
@inproceedings{guo_calibration_2017,
	title = {On {Calibration} of {Modern} {Neural} {Networks}},
	url = {https://proceedings.mlr.press/v70/guo17a.html},
	abstract = {Confidence calibration – the problem of predicting probability estimates representative of the true correctness likelihood – is important for classification models in many applications. We discover that modern neural networks, unlike those from a decade ago, are poorly calibrated. Through extensive experiments, we observe that depth, width, weight decay, and Batch Normalization are important factors influencing calibration. We evaluate the performance of various post-processing calibration methods on state-of-the-art architectures with image and document classification datasets. Our analysis and experiments not only offer insights into neural network learning, but also provide a simple and straightforward recipe for practical settings: on most datasets, temperature scaling – a single-parameter variant of Platt Scaling – is surprisingly effective at calibrating predictions.},
	language = {en},
	urldate = {2025-06-22},
	booktitle = {Proceedings of the 34th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Guo, Chuan and Pleiss, Geoff and Sun, Yu and Weinberger, Kilian Q.},
	month = jul,
	year = {2017},
	note = {ISSN: 2640-3498},
	pages = {1321--1330},
}

@article{duits_barretts_2015,
	title = {Barrett's oesophagus patients with low-grade dysplasia can be accurately risk-stratified after histological review by an expert pathology panel},
	volume = {64},
	copyright = {Published by the BMJ Publishing Group Limited. For permission to use (where not already granted under a licence) please go to http://group.bmj.com/group/rights-licensing/permissions},
	issn = {0017-5749, 1468-3288},
	url = {https://gut.bmj.com/content/64/5/700},
	doi = {10.1136/gutjnl-2014-307278},
	abstract = {Objective Reported malignant progression rates for low-grade dysplasia (LGD) in Barrett's oesophagus (BO) vary widely. Expert histological review of LGD is advised, but limited data are available on its clinical value. This retrospective cohort study aimed to determine the value of an expert pathology panel organised in the Dutch Barrett's Advisory Committee (BAC) by investigating the incidence rates of high-grade dysplasia (HGD) and oesophageal adenocarcinoma (OAC) after expert histological review of LGD.
Design We included all BO cases referred to the BAC for histological review of LGD diagnosed between 2000 and 2011. The diagnosis of the expert panel was related to the histological outcome during endoscopic follow-up. Primary endpoint was development of HGD or OAC.
Results 293 LGD patients (76\% men; mean 63 years±11.9) were included. Following histological review, 73\% was downstaged to non-dysplastic BO (NDBO) or indefinite for dysplasia (IND). In 27\% the initial LGD diagnosis was confirmed. Endoscopic follow-up was performed in 264 patients (90\%) with a median follow-up of 39 months (IQR 16–72). For confirmed LGD, the risk of HGD/OAC was 9.1\% per patient-year. Patients downstaged to NDBO or IND had a malignant progression risk of 0.6\% and 0.9\% per patient-year, respectively.
Conclusions Confirmed LGD in BO has a markedly increased risk of malignant progression. However, the vast majority of patients with community LGD will be downstaged after expert review and have a low progression risk. Therefore, all BO patients with LGD should undergo expert histological review of the diagnosis for adequate risk stratification.},
	language = {en},
	number = {5},
	urldate = {2022-03-28},
	journal = {Gut},
	author = {Duits, Lucas C. and Phoa, K. Nadine and Curvers, Wouter L. and Kate, Fiebo J. W. ten and Meijer, Gerrit A. and Seldenrijk, Cees A. and Offerhaus, G. Johan and Visser, Mike and Meijer, Sybren L. and Krishnadath, Kausilia K. and Tijssen, Jan G. P. and Mallant-Hent, Rosalie C. and Bergman, Jacques J. G. H. M.},
	month = may,
	year = {2015},
	pmid = {25034523},
	keywords = {Histopathology, Barrett's Carcinoma, Barrett's Oesophagus, Dysplasia, Oesophageal Cancer},
	pages = {700--706},
}


@article{zhang_learning_2023,
	title = {Learning from multiple annotators for medical image segmentation},
	volume = {138},
	issn = {0031-3203},
	url = {https://www.sciencedirect.com/science/article/pii/S0031320323001012},
	doi = {10.1016/j.patcog.2023.109400},
	abstract = {Supervised machine learning methods have been widely developed for segmentation tasks in recent years. However, the quality of labels has high impact on the predictive performance of these algorithms. This issue is particularly acute in the medical image domain, where both the cost of annotation and the inter-observer variability are high. Different human experts contribute estimates of the ”actual” segmentation labels in a typical label acquisition process, influenced by their personal biases and competency levels. The performance of automatic segmentation algorithms is limited when these noisy labels are used as the expert consensus label. In this work, we use two coupled CNNs to jointly learn, from purely noisy observations alone, the reliability of individual annotators and the expert consensus label distributions. The separation of the two is achieved by maximally describing the annotator’s “unreliable behavior” (we call it “maximally unreliable”) while achieving high fidelity with the noisy training data. We first create a toy segmentation dataset using MNIST and investigate the properties of the proposed algorithm. We then use three public medical imaging segmentation datasets to demonstrate our method’s efficacy, including both simulated (where necessary) and real-world annotations: 1) ISBI2015 (multiple-sclerosis lesions); 2) BraTS (brain tumors); 3) LIDC-IDRI (lung abnormalities). Finally, we create a real-world multiple sclerosis lesion dataset (QSMSC at UCL: Queen Square Multiple Sclerosis Center at UCL, UK) with manual segmentations from 4 different annotators (3 radiologists with different level skills and 1 expert to generate the expert consensus label). In all datasets, our method consistently outperforms competing methods and relevant baselines, especially when the number of annotations is small and the amount of disagreement is large. The studies also reveal that the system is capable of capturing the complicated spatial characteristics of annotators’ mistakes.},
	urldate = {2025-12-10},
	journal = {Pattern Recognition},
	author = {Zhang, Le and Tanno, Ryutaro and Xu, Moucheng and Huang, Yawen and Bronik, Kevin and Jin, Chen and Jacob, Joseph and Zheng, Yefeng and Shao, Ling and Ciccarelli, Olga and Barkhof, Frederik and Alexander, Daniel C.},
	month = jun,
	year = {2023},
	keywords = {Segmentation, Label fusion, Multi-Annotator},
	pages = {109400},
}


@article{karimi_deep_2020,
	title = {Deep learning with noisy labels: {Exploring} techniques and remedies in medical image analysis},
	volume = {65},
	issn = {1361-8415},
	shorttitle = {Deep learning with noisy labels},
	url = {https://www.sciencedirect.com/science/article/pii/S1361841520301237},
	doi = {10.1016/j.media.2020.101759},
	abstract = {Supervised training of deep learning models requires large labeled datasets. There is a growing interest in obtaining such datasets for medical image analysis applications. However, the impact of label noise has not received sufficient attention. Recent studies have shown that label noise can significantly impact the performance of deep learning models in many machine learning and computer vision applications. This is especially concerning for medical applications, where datasets are typically small, labeling requires domain expertise and suffers from high inter- and intra-observer variability, and erroneous predictions may influence decisions that directly impact human health. In this paper, we first review the state-of-the-art in handling label noise in deep learning. Then, we review studies that have dealt with label noise in deep learning for medical image analysis. Our review shows that recent progress on handling label noise in deep learning has gone largely unnoticed by the medical image analysis community. To help achieve a better understanding of the extent of the problem and its potential remedies, we conducted experiments with three medical imaging datasets with different types of label noise, where we investigated several existing strategies and developed new methods to combat the negative effect of label noise. Based on the results of these experiments and our review of the literature, we have made recommendations on methods that can be used to alleviate the effects of different types of label noise on deep models trained for medical image analysis. We hope that this article helps the medical image analysis researchers and developers in choosing and devising new techniques that effectively handle label noise in deep learning.},
	urldate = {2025-12-10},
	journal = {Medical Image Analysis},
	author = {Karimi, Davood and Dou, Haoran and Warfield, Simon K. and Gholipour, Ali},
	month = oct,
	year = {2020},
	keywords = {Deep learning, Machine learning, Big data, Label noise, Medical image annotation},
	pages = {101759},
}

@misc{teuwen_dlup_2024,
  author       = {Teuwen, J. and Romor, L. and Pai, A. and Schirris, Y. and Marcus, E.},
  title        = {{DLUP: Deep Learning Utilities for Pathology}},
  year         = {2024},
  howpublished = {\url{https://github.com/NKI-AI/dlup}},
  note         = {Accessed: August 2024}
}

@article{van_der_wel_adherence_2019,
	title = {Adherence to pre-set benchmark quality criteria to qualify as expert assessor of dysplasia in {Barrett}'s esophagus biopsies - towards digital review of {Barrett}'s esophagus},
	volume = {7},
	issn = {2050-6406},
	doi = {10.1177/2050640619853441},
	abstract = {BACKGROUND: Dysplasia assessment of Barrett's esophagus biopsies is associated with low observer agreement; guidelines advise expert review. We have developed a web-based review panel for dysplastic Barrett's esophagus biopsies.
OBJECTIVE: The purpose of this study was to test if 10 gastrointestinal pathologists working at Dutch Barrett's esophagus expert centres met pre-set benchmark scores for quality criteria.
METHODS: Ten gastrointestinal pathologists twice assessed 60 digitalized Barrett's esophagus cases, enriched for dysplasia; then randomised (7520 assessments). We tested predefined benchmark quality criteria: (a) percentage of 'indefinite for dysplasia' diagnoses, benchmark score ≤14\% for all cases, ≤16\% for dysplastic subset, (b) intra-observer agreement; benchmark score ≥0.66/≥0.39, (c) percentage agreement with 'gold standard diagnosis'; benchmark score ≥82\%/≥73\%, (d) proportion of cases with high-grade dysplasia underdiagnosed as non-dysplastic Barrett's esophagus; benchmark score ≤1/78 (≤1.28\%) assessments for dysplastic subset.
RESULTS: Gastrointestinal pathologists had seven years' Barrett's esophagus-experience, handling seven Barrett's esophagus-cases weekly. Three met stringent benchmark scores; all cases and dysplastic subset, three met extended benchmark scores. Four pathologists lacked one quality criterion to meet benchmark scores.
CONCLUSION: Predefined benchmark scores for expert assessment of Barrett's esophagus dysplasia biopsies are stringent and met by some gastrointestinal pathologists. The majority of assessors however, only showed limited deviation from benchmark scores. We expect further training with group discussions will lead to adherence of all participating gastrointestinal pathologists to quality criteria, and therefore eligible to join the review panel.},
	language = {eng},
	number = {7},
	journal = {United European Gastroenterology Journal},
	author = {van der Wel, M. J. and Klaver, E. and Duits, L. C. and Pouw, R. E. and Seldenrijk, C. A. and Offerhaus, Gja and Visser, M. and Ten Kate, Fjw and Biermann, K. and Brosens, Laa and Doukas, M. and Huysentruyt, C. and Karrenbeld, A. and Kats-Ugurlu, G. and van der Laan, J. S. and van Lijnschoten, G. and Moll, Fcp and Ooms, Ahag and Tijssen, J. G. and Bergman, Jjghm and Meijer, S. L.},
	month = aug,
	year = {2019},
	pmid = {31428413},
	pmcid = {PMC6683647},
	keywords = {Humans, Observer Variation, Netherlands, Risk Factors, Barrett Esophagus, Internet, Barrett’s esophagus, benchmark quality criteria, Benchmarking, Biopsy, Cell Transformation, Neoplastic, consensus gold standard diagnosis, digital microscopy, Esophagus, Guideline Adherence, Microscopy, observer agreement, Pathologists, review panel, whole slide imaging},
	pages = {889--896},
}


@misc{noauthor_guo_calibration_2017_2021,
	title = {guo\_calibration\_2017 {\textbar} {TransferLab} — {appliedAI} {Institute}},
	url = {https://transferlab.ai/refs/guo_calibration_2017/},
	abstract = {Reference abstract: Conﬁdence calibration – the problem of predicting probability estimates representative of the true correctness likelihood – is important for classiﬁcation models in many applications. We discover that modern neural networks, unlike those from a decade ago, are poorly calibrated. …},
	language = {en-gb},
	urldate = {2025-06-22},
	month = apr,
	year = {2021},
}

@misc{jiang_trust_2018,
	title = {To {Trust} {Or} {Not} {To} {Trust} {A} {Classifier}},
	url = {http://arxiv.org/abs/1805.11783},
	doi = {10.48550/arXiv.1805.11783},
	abstract = {Knowing when a classifier's prediction can be trusted is useful in many applications and critical for safely using AI. While the bulk of the effort in machine learning research has been towards improving classifier performance, understanding when a classifier's predictions should and should not be trusted has received far less attention. The standard approach is to use the classifier's discriminant or confidence score; however, we show there exists an alternative that is more effective in many situations. We propose a new score, called the trust score, which measures the agreement between the classifier and a modified nearest-neighbor classifier on the testing example. We show empirically that high (low) trust scores produce surprisingly high precision at identifying correctly (incorrectly) classified examples, consistently outperforming the classifier's confidence score as well as many other baselines. Further, under some mild distributional assumptions, we show that if the trust score for an example is high (low), the classifier will likely agree (disagree) with the Bayes-optimal classifier. Our guarantees consist of non-asymptotic rates of statistical consistency under various nonparametric settings and build on recent developments in topological data analysis.},
	urldate = {2025-06-22},
	publisher = {arXiv},
	author = {Jiang, Heinrich and Kim, Been and Guan, Melody Y. and Gupta, Maya},
	month = oct,
	year = {2018},
	note = {arXiv:1805.11783 [stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@misc{poceviciute_generalisation_2022,
	title = {Generalisation effects of predictive uncertainty estimation in deep learning for digital pathology},
	url = {http://arxiv.org/abs/2112.09693},
	doi = {10.48550/arXiv.2112.09693},
	abstract = {Deep learning (DL) has shown great potential in digital pathology applications. The robustness of a diagnostic DL-based solution is essential for safe clinical deployment. In this work we evaluate if adding uncertainty estimates for DL predictions in digital pathology could result in increased value for the clinical applications, by boosting the general predictive performance or by detecting mispredictions. We compare the effectiveness of model-integrated methods (MC dropout and Deep ensembles) with a model-agnostic approach (Test time augmentation, TTA). Moreover, four uncertainty metrics are compared. Our experiments focus on two domain shift scenarios: a shift to a different medical center and to an underrepresented subtype of cancer. Our results show that uncertainty estimates increase reliability by reducing a model's sensitivity to classification threshold selection as well as by detecting between 70{\textbackslash}\% and 90{\textbackslash}\% of the mispredictions done by the model. Overall, the deep ensembles method achieved the best performance closely followed by TTA.},
	urldate = {2025-06-21},
	publisher = {arXiv},
	author = {Pocevičiūtė, Milda and Eilertsen, Gabriel and Jarkman, Sofia and Lundström, Claes},
	month = may,
	year = {2022},
	note = {arXiv:2112.09693 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Electrical Engineering and Systems Science - Image and Video Processing},
}

@misc{noauthor_master_nodate,
	title = {Master {Thesis} - {Jan} {Grove}},
	url = {https://www.overleaf.com/project/67d0919a1bd2d8ef93fc835b},
	abstract = {An online LaTeX editor that’s easy to use. No installation, real-time collaboration, version control, hundreds of LaTeX templates, and more.},
	language = {en},
	urldate = {2025-06-21},
}

@misc{gal_dropout_2016,
	title = {Dropout as a {Bayesian} {Approximation}: {Representing} {Model} {Uncertainty} in {Deep} {Learning}},
	shorttitle = {Dropout as a {Bayesian} {Approximation}},
	url = {http://arxiv.org/abs/1506.02142},
	doi = {10.48550/arXiv.1506.02142},
	abstract = {Deep learning tools have gained tremendous attention in applied machine learning. However such tools for regression and classification do not capture model uncertainty. In comparison, Bayesian models offer a mathematically grounded framework to reason about model uncertainty, but usually come with a prohibitive computational cost. In this paper we develop a new theoretical framework casting dropout training in deep neural networks (NNs) as approximate Bayesian inference in deep Gaussian processes. A direct result of this theory gives us tools to model uncertainty with dropout NNs -- extracting information from existing models that has been thrown away so far. This mitigates the problem of representing uncertainty in deep learning without sacrificing either computational complexity or test accuracy. We perform an extensive study of the properties of dropout's uncertainty. Various network architectures and non-linearities are assessed on tasks of regression and classification, using MNIST as an example. We show a considerable improvement in predictive log-likelihood and RMSE compared to existing state-of-the-art methods, and finish by using dropout's uncertainty in deep reinforcement learning.},
	urldate = {2025-06-21},
	publisher = {arXiv},
	author = {Gal, Yarin and Ghahramani, Zoubin},
	month = oct,
	year = {2016},
	note = {arXiv:1506.02142 [stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@misc{noauthor_gal_dropout_2016_2023,
	title = {gal\_dropout\_2016 {\textbar} {TransferLab} — {appliedAI} {Institute}},
	url = {https://transferlab.ai/refs/gal_dropout_2016/},
	abstract = {Reference abstract: Deep learning tools have gained tremendous attention in applied machine learning. However such tools for regression and classification do not capture model uncertainty. In comparison, Bayesian models offer a mathematically grounded framework to reason about model uncertainty, but …},
	language = {en-gb},
	urldate = {2025-06-21},
	month = feb,
	year = {2023},
}

@misc{lakshminarayanan_simple_2017,
	title = {Simple and {Scalable} {Predictive} {Uncertainty} {Estimation} using {Deep} {Ensembles}},
	url = {http://arxiv.org/abs/1612.01474},
	doi = {10.48550/arXiv.1612.01474},
	abstract = {Deep neural networks (NNs) are powerful black box predictors that have recently achieved impressive performance on a wide spectrum of tasks. Quantifying predictive uncertainty in NNs is a challenging and yet unsolved problem. Bayesian NNs, which learn a distribution over weights, are currently the state-of-the-art for estimating predictive uncertainty; however these require significant modifications to the training procedure and are computationally expensive compared to standard (non-Bayesian) NNs. We propose an alternative to Bayesian NNs that is simple to implement, readily parallelizable, requires very little hyperparameter tuning, and yields high quality predictive uncertainty estimates. Through a series of experiments on classification and regression benchmarks, we demonstrate that our method produces well-calibrated uncertainty estimates which are as good or better than approximate Bayesian NNs. To assess robustness to dataset shift, we evaluate the predictive uncertainty on test examples from known and unknown distributions, and show that our method is able to express higher uncertainty on out-of-distribution examples. We demonstrate the scalability of our method by evaluating predictive uncertainty estimates on ImageNet.},
	urldate = {2025-06-21},
	publisher = {arXiv},
	author = {Lakshminarayanan, Balaji and Pritzel, Alexander and Blundell, Charles},
	month = nov,
	year = {2017},
	note = {arXiv:1612.01474 [stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@article{dolezal_uncertainty-informed_2022,
	title = {Uncertainty-informed deep learning models enable high-confidence predictions for digital histopathology},
	volume = {13},
	copyright = {2022 The Author(s)},
	issn = {2041-1723},
	url = {https://www.nature.com/articles/s41467-022-34025-x},
	doi = {10.1038/s41467-022-34025-x},
	abstract = {A model’s ability to express its own predictive uncertainty is an essential attribute for maintaining clinical user confidence as computational biomarkers are deployed into real-world medical settings. In the domain of cancer digital histopathology, we describe a clinically-oriented approach to uncertainty quantification for whole-slide images, estimating uncertainty using dropout and calculating thresholds on training data to establish cutoffs for low- and high-confidence predictions. We train models to identify lung adenocarcinoma vs. squamous cell carcinoma and show that high-confidence predictions outperform predictions without uncertainty, in both cross-validation and testing on two large external datasets spanning multiple institutions. Our testing strategy closely approximates real-world application, with predictions generated on unsupervised, unannotated slides using predetermined thresholds. Furthermore, we show that uncertainty thresholding remains reliable in the setting of domain shift, with accurate high-confidence predictions of adenocarcinoma vs. squamous cell carcinoma for out-of-distribution, non-lung cancer cohorts.},
	language = {en},
	number = {1},
	urldate = {2025-06-21},
	journal = {Nature Communications},
	author = {Dolezal, James M. and Srisuwananukorn, Andrew and Karpeyev, Dmitry and Ramesh, Siddhi and Kochanny, Sara and Cody, Brittany and Mansfield, Aaron S. and Rakshit, Sagar and Bansal, Radhika and Bois, Melanie C. and Bungum, Aaron O. and Schulte, Jefree J. and Vokes, Everett E. and Garassino, Marina Chiara and Husain, Aliya N. and Pearson, Alexander T.},
	month = nov,
	year = {2022},
	note = {Publisher: Nature Publishing Group},
	keywords = {Cancer screening, Computational science, Lung cancer, Machine learning},
	pages = {6572},
}

@book{krippendorff_content_2018,
	title = {Content {Analysis}: {An} {Introduction} to {Its} {Methodology}},
	isbn = {978-1-5063-9567-8},
	shorttitle = {Content {Analysis}},
	abstract = {What matters in people’s social lives? What motivates and inspires our society? How do we enact what we know? Since the first edition published in 1980, Content Analysis has helped shape and define the field. In the highly anticipated Fourth Edition, award-winning scholar and author Klaus Krippendorff introduces you to the most current method of analyzing the textual fabric of contemporary society. Students and scholars will learn to treat data not as physical events but as communications that are created and disseminated to be seen, read, interpreted, enacted, and reflected upon according to the meanings they have for their recipients. Interpreting communications as texts in the contexts of their social uses distinguishes content analysis from other empirical methods of inquiry. Organized into three parts, Content Analysis first examines the conceptual aspects of content analysis, then discusses components such as unitizing and sampling, and concludes by showing readers how to trace the analytical paths and apply evaluative techniques. The Fourth Edition has been completely revised to offer you the most current techniques and research on content analysis, including new information on reliability and social media. You will also gain practical advice and experience for teaching academic and commercial researchers how to conduct content analysis.},
	language = {en},
	publisher = {SAGE Publications},
	author = {Krippendorff, Klaus},
	month = may,
	year = {2018},
	note = {Google-Books-ID: nE1aDwAAQBAJ},
	keywords = {Language Arts \& Disciplines / Communication Studies, Reference / Research},
}

@book{krippendorff_content_nodate,
	address = {Thousand Oaks, Calif.},
	edition = {2. ed., [ Nachdr.]},
	title = {Content analysis: an introduction to its methodology},
	isbn = {978-0-7619-1544-7 978-0-7619-1545-4},
	shorttitle = {Content analysis},
	language = {en},
	publisher = {Sage Publ},
	author = {Krippendorff, Klaus},
}

@misc{krippendorff_computing_2011,
	title = {Computing {Krippendorff}'s {Alpha}-{Reliability}},
	url = {https://scispace.com/papers/computing-krippendorff-s-alpha-reliability-341bzpy7ww},
	abstract = {Krippendorff’s alpha () is a reliability coefficient developed to measure the agreement among observers, coders, judges, raters, annotators or measuring instruments drawing distinctions among typically unstructured phenomena or assign computable values to them.  emerged in content analysis but is widely applicable wherever two or more methods of generating data are applied to the same set of objects, predefined units of analysis or items and the question is how much the resulting data can be trusted to represent something worthy of analysis.},
	language = {en},
	urldate = {2025-06-18},
	author = {Krippendorff, Klaus},
	month = jan,
	year = {2011},
}

@article{krippendorff_computing_nodate,
	title = {Computing {Krippendorff}’s {Alpha}-{Reliability}},
	language = {en},
	author = {Krippendorff, Klaus},
}

@misc{noauthor_master_nodate-1,
	title = {Master {Thesis} - {Jan} {Grove}},
	url = {https://www.overleaf.com/project/67d0919a1bd2d8ef93fc835b},
	abstract = {An online LaTeX editor that’s easy to use. No installation, real-time collaboration, version control, hundreds of LaTeX templates, and more.},
	language = {en},
	urldate = {2025-06-16},
}

@article{krippendorff_estimating_1970,
	title = {Estimating the reliability, systematic error and random error of interval data},
	volume = {30},
	issn = {1552-3888},
	doi = {10.1177/001316447003000105},
	abstract = {Formulae are developed permitting estimation of data reliability across Os, individual reliability, and unit reliability. An illustration is given. (PsycINFO Database Record (c) 2016 APA, all rights reserved)},
	number = {1},
	journal = {Educational and Psychological Measurement},
	author = {Krippendorff, Klaus},
	year = {1970},
	note = {Place: US
Publisher: Sage Publications},
	keywords = {Errors, Estimation, Statistical Analysis, Statistical Reliability},
	pages = {61--70},
}

@article{krippendorff_reliability_1978,
	title = {Reliability of {Binary} {Attribute} {Data}},
	volume = {34},
	issn = {0006-341X},
	url = {https://www.jstor.org/stable/2529602},
	number = {1},
	urldate = {2025-06-16},
	journal = {Biometrics},
	author = {Krippendorff, Klaus and Fleiss, Joseph L.},
	year = {1978},
	note = {Publisher: International Biometric Society},
	pages = {142--144},
}

@misc{noauthor_estimating_nodate,
	title = {Estimating the {Reliability}, {Systematic} {Error} and {Random} {Error} of {Interval} {Data} - {Klaus} {Krippendorff}, 1970},
	url = {https://journals.sagepub.com/doi/abs/10.1177/001316447003000105?casa_token=F5_RRnjJgZcAAAAA:J4E1wyJckNb4az0-rTTfZIA-Ntp3b7pNcDygeJLg3jYC_SZ_crRgfTJjSZFCvGT7B6GcsvlJeUhNVg&casa_token=Rh8GO4iiwYkAAAAA:jR9UIyzbACey7ACCA5jB093lQzS-0tucPOTSJImYQVejIHTIsvywfjOlLUOliqnUPceVQyg__U8Uxw},
	urldate = {2025-06-16},
}

@misc{zimmermann_virchow2_2024,
	title = {Virchow2: {Scaling} {Self}-{Supervised} {Mixed} {Magnification} {Models} in {Pathology}},
	shorttitle = {Virchow2},
	url = {http://arxiv.org/abs/2408.00738},
	doi = {10.48550/arXiv.2408.00738},
	abstract = {Foundation models are rapidly being developed for computational pathology applications. However, it remains an open question which factors are most important for downstream performance with data scale and diversity, model size, and training algorithm all playing a role. In this work, we propose algorithmic modifications, tailored for pathology, and we present the result of scaling both data and model size, surpassing previous studies in both dimensions. We introduce three new models: Virchow2, a 632 million parameter vision transformer, Virchow2G, a 1.9 billion parameter vision transformer, and Virchow2G Mini, a 22 million parameter distillation of Virchow2G, each trained with 3.1 million histopathology whole slide images, with diverse tissues, originating institutions, and stains. We achieve state of the art performance on 12 tile-level tasks, as compared to the top performing competing models. Our results suggest that data diversity and domain-specific methods can outperform models that only scale in the number of parameters, but, on average, performance benefits from the combination of domain-specific methods, data scale, and model scale.},
	urldate = {2025-06-13},
	publisher = {arXiv},
	author = {Zimmermann, Eric and Vorontsov, Eugene and Viret, Julian and Casson, Adam and Zelechowski, Michal and Shaikovski, George and Tenenholtz, Neil and Hall, James and Klimstra, David and Yousfi, Razik and Fuchs, Thomas and Fusi, Nicolo and Liu, Siqi and Severson, Kristen},
	month = nov,
	year = {2024},
	note = {arXiv:2408.00738 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}

@inproceedings{jensen_improving_2019,
	address = {Cham},
	title = {Improving {Uncertainty} {Estimation} in {Convolutional} {Neural} {Networks} {Using} {Inter}-rater {Agreement}},
	isbn = {978-3-030-32251-9},
	doi = {10.1007/978-3-030-32251-9_59},
	abstract = {Modern neural networks are pushing the boundaries of medical image classification. For some tasks in dermatology, state of the art models are able to beat human experts in terms of accuracy and type I/II errors. However, in the domain of medical applications, models should also be evaluated on how well they capture uncertainty in samples and labels. This aspect is key to building trust in computer-assisted systems, otherwise largely considered to be black boxes by their users. A common practice in supervised learning is to collect multiple evaluations per sample, which is particularly useful when inter-rater agreement is expected to be low. At the same time, model training traditionally uses label fusion, such as majority voting, to produce a single label for each sample. In this paper, we experimentally show that models trained to predict skin conditions become overconfident when this approach is used; i.e. the probability estimates of the model exceeds the true correctness likelihood. Additionally, we show that a better calibrated model is obtained when training with a label sampling scheme that takes advantage of inter-rater variability during training. The calibration improvements come at no cost in terms of model accuracy. Our approach is combined and contrasted with other recent techniques in uncertainty estimation. All experiments are evaluated on a proprietary dataset consisting of 31017 images of skin, where up to 12 experts have diagnosed each image.},
	language = {en},
	booktitle = {Medical {Image} {Computing} and {Computer} {Assisted} {Intervention} – {MICCAI} 2019},
	publisher = {Springer International Publishing},
	author = {Jensen, Martin Holm and Jørgensen, Dan Richter and Jalaboi, Raluca and Hansen, Mads Eiler and Olsen, Martin Aastrup},
	editor = {Shen, Dinggang and Liu, Tianming and Peters, Terry M. and Staib, Lawrence H. and Essert, Caroline and Zhou, Sean and Yap, Pew-Thian and Khan, Ali},
	year = {2019},
	keywords = {Dermatology, Model calibration, Supervised deep learning},
	pages = {540--548},
}

@misc{vorontsov_virchow_2024,
	title = {Virchow: {A} {Million}-{Slide} {Digital} {Pathology} {Foundation} {Model}},
	shorttitle = {Virchow},
	url = {http://arxiv.org/abs/2309.07778},
	doi = {10.48550/arXiv.2309.07778},
	abstract = {The use of artificial intelligence to enable precision medicine and decision support systems through the analysis of pathology images has the potential to revolutionize the diagnosis and treatment of cancer. Such applications will depend on models' abilities to capture the diverse patterns observed in pathology images. To address this challenge, we present Virchow, a foundation model for computational pathology. Using self-supervised learning empowered by the DINOv2 algorithm, Virchow is a vision transformer model with 632 million parameters trained on 1.5 million hematoxylin and eosin stained whole slide images from diverse tissue and specimen types, which is orders of magnitude more data than previous works. The Virchow model enables the development of a pan-cancer detection system with 0.949 overall specimen-level AUC across 17 different cancer types, while also achieving 0.937 AUC on 7 rare cancer types. The Virchow model sets the state-of-the-art on the internal and external image tile level benchmarks and slide level biomarker prediction tasks. The gains in performance highlight the importance of training on massive pathology image datasets, suggesting scaling up the data and network architecture can improve the accuracy for many high-impact computational pathology applications where limited amounts of training data are available.},
	urldate = {2025-03-08},
	publisher = {arXiv},
	author = {Vorontsov, Eugene and Bozkurt, Alican and Casson, Adam and Shaikovski, George and Zelechowski, Michal and Liu, Siqi and Severson, Kristen and Zimmermann, Eric and Hall, James and Tenenholtz, Neil and Fusi, Nicolo and Mathieu, Philippe and Eck, Alexander van and Lee, Donghun and Viret, Julian and Robert, Eric and Wang, Yi Kan and Kunz, Jeremy D. and Lee, Matthew C. H. and Bernhard, Jan and Godrich, Ran A. and Oakley, Gerard and Millar, Ewan and Hanna, Matthew and Retamero, Juan and Moye, William A. and Yousfi, Razik and Kanan, Christopher and Klimstra, David and Rothrock, Brandon and Fuchs, Thomas J.},
	month = jan,
	year = {2024},
	note = {arXiv:2309.07778 [eess]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Electrical Engineering and Systems Science - Image and Video Processing, Quantitative Biology - Tissues and Organs},
}

@misc{lu_towards_2023,
	title = {Towards a {Visual}-{Language} {Foundation} {Model} for {Computational} {Pathology}},
	url = {http://arxiv.org/abs/2307.12914},
	doi = {10.48550/arXiv.2307.12914},
	abstract = {The accelerated adoption of digital pathology and advances in deep learning have enabled the development of powerful models for various pathology tasks across a diverse array of diseases and patient cohorts. However, model training is often difficult due to label scarcity in the medical domain and the model's usage is limited by the specific task and disease for which it is trained. Additionally, most models in histopathology leverage only image data, a stark contrast to how humans teach each other and reason about histopathologic entities. We introduce CONtrastive learning from Captions for Histopathology (CONCH), a visual-language foundation model developed using diverse sources of histopathology images, biomedical text, and notably over 1.17 million image-caption pairs via task-agnostic pretraining. Evaluated on a suite of 13 diverse benchmarks, CONCH can be transferred to a wide range of downstream tasks involving either or both histopathology images and text, achieving state-of-the-art performance on histology image classification, segmentation, captioning, text-to-image and image-to-text retrieval. CONCH represents a substantial leap over concurrent visual-language pretrained systems for histopathology, with the potential to directly facilitate a wide array of machine learning-based workflows requiring minimal or no further supervised fine-tuning.},
	urldate = {2025-03-08},
	publisher = {arXiv},
	author = {Lu, Ming Y. and Chen, Bowen and Williamson, Drew F. K. and Chen, Richard J. and Liang, Ivy and Ding, Tong and Jaume, Guillaume and Odintsov, Igor and Zhang, Andrew and Le, Long Phi and Gerber, Georg and Parwani, Anil V. and Mahmood, Faisal},
	month = jul,
	year = {2023},
	note = {arXiv:2307.12914 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition},
}

@misc{kofler_approaching_2023,
	title = {Approaching {Peak} {Ground} {Truth}},
	url = {http://arxiv.org/abs/2301.00243},
	doi = {10.48550/arXiv.2301.00243},
	abstract = {Machine learning models are typically evaluated by computing similarity with reference annotations and trained by maximizing similarity with such. Especially in the biomedical domain, annotations are subjective and suffer from low inter- and intra-rater reliability. Since annotations only reflect one interpretation of the real world, this can lead to sub-optimal predictions even though the model achieves high similarity scores. Here, the theoretical concept of PGT is introduced. PGT marks the point beyond which an increase in similarity with the {\textbackslash}emph\{reference annotation\} stops translating to better RWMP. Additionally, a quantitative technique to approximate PGT by computing inter- and intra-rater reliability is proposed. Finally, four categories of PGT-aware strategies to evaluate and improve model performance are reviewed.},
	urldate = {2025-03-08},
	publisher = {arXiv},
	author = {Kofler, Florian and Wahle, Johannes and Ezhov, Ivan and Wagner, Sophia and Al-Maskari, Rami and Gryska, Emilia and Todorov, Mihail and Bukas, Christina and Meissen, Felix and Peng, Tingying and Ertürk, Ali and Rueckert, Daniel and Heckemann, Rolf and Kirschke, Jan and Zimmer, Claus and Wiestler, Benedikt and Menze, Bjoern and Piraud, Marie},
	month = mar,
	year = {2023},
	note = {arXiv:2301.00243 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
}

@article{kofler_are_2023,
	title = {Are we using appropriate segmentation metrics? {Identifying} correlates of human expert perception for {CNN} training beyond rolling the {DICE} coefficient},
	volume = {2},
	issn = {2766-905X},
	shorttitle = {Are we using appropriate segmentation metrics?},
	url = {https://www.melba-journal.org/papers/2023:002.html},
	doi = {10.59275/j.melba.2023-dg1f},
	abstract = {F Kofler, I Ezhov, F Isensee, F Balsiger, C Berger, M Koerner, B Demiray, J Rackerseder, J Paetzold, H Li, S Shit, R McKinley, M Piraud, S Bakas, C Zimmer, N Navab, J Kirschke, B Wiestler, B Menze},
	language = {english},
	number = {May 2023 issue},
	urldate = {2025-03-08},
	journal = {Machine Learning for Biomedical Imaging},
	author = {Kofler, Florian and Ezhov, Ivan and Isensee, Fabian and Balsiger, Fabian and Berger, Christoph and Koerner, Maximilian and Demiray, Beatrice and Rackerseder, Julia and Paetzold, Johannes and Li, Hongwei and Shit, Suprosanna and McKinley, Richard and Piraud, Marie and Bakas, Spyridon and Zimmer, Claus and Navab, Nassir and Kirschke, Jan and Wiestler, Benedikt and Menze, Bjoern},
	month = may,
	year = {2023},
	pages = {27--71},
}

@article{fanshawe_assessing_2008,
	title = {Assessing {Agreement} between {Multiple} {Raters} with {Missing} {Rating} {Information}, {Applied} to {Breast} {Cancer} {Tumour} {Grading}},
	volume = {3},
	issn = {1932-6203},
	url = {https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0002925},
	doi = {10.1371/journal.pone.0002925},
	abstract = {Background We consider the problem of assessing inter-rater agreement when there are missing data and a large number of raters. Previous studies have shown only ‘moderate’ agreement between pathologists in grading breast cancer tumour specimens. We analyse a large but incomplete data-set consisting of 24177 grades, on a discrete 1–3 scale, provided by 732 pathologists for 52 samples. Methodology/Principal Findings We review existing methods for analysing inter-rater agreement for multiple raters and demonstrate two further methods. Firstly, we examine a simple non-chance-corrected agreement score based on the observed proportion of agreements with the consensus for each sample, which makes no allowance for missing data. Secondly, treating grades as lying on a continuous scale representing tumour severity, we use a Bayesian latent trait method to model cumulative probabilities of assigning grade values as functions of the severity and clarity of the tumour and of rater-specific parameters representing boundaries between grades 1–2 and 2–3. We simulate from the fitted model to estimate, for each rater, the probability of agreement with the majority. Both methods suggest that there are differences between raters in terms of rating behaviour, most often caused by consistent over- or under-estimation of the grade boundaries, and also considerable variability in the distribution of grades assigned to many individual samples. The Bayesian model addresses the tendency of the agreement score to be biased upwards for raters who, by chance, see a relatively ‘easy’ set of samples. Conclusions/Significance Latent trait models can be adapted to provide novel information about the nature of inter-rater agreement when the number of raters is large and there are missing data. In this large study there is substantial variability between pathologists and uncertainty in the identity of the ‘true’ grade of many of the breast cancer tumours, a fact often ignored in clinical studies.},
	language = {en},
	number = {8},
	urldate = {2025-02-20},
	journal = {PLOS ONE},
	author = {Fanshawe, Thomas R. and Lynch, Andrew G. and Ellis, Ian O. and Green, Andrew R. and Hanka, Rudolf},
	month = aug,
	year = {2008},
	note = {Publisher: Public Library of Science},
	keywords = {Breast cancer, Breast tumors, Cancers and neoplasms, Histology, Normal distribution, Pathologists, Probability distribution, Simulation and modeling},
	pages = {e2925},
}

@article{tang_interpretable_2019,
	title = {Interpretable classification of {Alzheimer}’s disease pathologies with a convolutional neural network pipeline},
	volume = {10},
	copyright = {2019 The Author(s)},
	issn = {2041-1723},
	url = {https://www.nature.com/articles/s41467-019-10212-1},
	doi = {10.1038/s41467-019-10212-1},
	abstract = {Neuropathologists assess vast brain areas to identify diverse and subtly-differentiated morphologies. Standard semi-quantitative scoring approaches, however, are coarse-grained and lack precise neuroanatomic localization. We report a proof-of-concept deep learning pipeline that identifies specific neuropathologies—amyloid plaques and cerebral amyloid angiopathy—in immunohistochemically-stained archival slides. Using automated segmentation of stained objects and a cloud-based interface, we annotate {\textgreater} 70,000 plaque candidates from 43 whole slide images (WSIs) to train and evaluate convolutional neural networks. Networks achieve strong plaque classification on a 10-WSI hold-out set (0.993 and 0.743 areas under the receiver operating characteristic and precision recall curve, respectively). Prediction confidence maps visualize morphology distributions at high resolution. Resulting network-derived amyloid beta (Aβ)-burden scores correlate well with established semi-quantitative scores on a 30-WSI blinded hold-out. Finally, saliency mapping demonstrates that networks learn patterns agreeing with accepted pathologic features. This scalable means to augment a neuropathologist’s ability suggests a route to neuropathologic deep phenotyping.},
	language = {en},
	number = {1},
	urldate = {2025-02-20},
	journal = {Nature Communications},
	author = {Tang, Ziqi and Chuang, Kangway V. and DeCarli, Charles and Jin, Lee-Way and Beckett, Laurel and Keiser, Michael J. and Dugger, Brittany N.},
	month = may,
	year = {2019},
	note = {Publisher: Nature Publishing Group},
	keywords = {Alzheimer's disease, Image processing, Machine learning, Pathology},
	pages = {2173},
}

@article{bujang_guidelines_2022,
	title = {Guidelines of the minimum sample size requirements for {Kappa} agreement test},
	volume = {14},
	copyright = {https://creativecommons.org/licenses/by/4.0},
	issn = {2282-0930},
	url = {https://riviste.unimi.it/index.php/ebph/article/view/17614},
	doi = {10.2427/12267},
	abstract = {Background and aims: 
    To estimate sample size for Cohen’s kappa agreement test can be challenging especially when it is expected that the true marginal rating frequencies are not the same. This study aims to present the tables that display a minimum sample size determination for an agreement test when certain assumptions are hold. 
Method: 
    We adopted the sample size formula provided by Flack and colleagues (1988) to calculate the minimum sample sizes required using PASS software. The power is pre-specified to be at least 80\% and the alpha to be less than 0.05. The effect sizes were derived from several pre-specified estimates such as the pattern of the true marginal rating frequencies and the difference between the two kappa coefficients in the hypothesis testing. 
Results: 
    When the true marginal rating frequencies are the same, the minimum sample size determination can range from 2 to 698 depending on the actual value of the effect size. When the true marginal rating frequencies are not the same, then the majority of the minimum sample size required for this condition is more than double than that required sample size when the true marginal rating frequencies are the same. 
Conclusion: 
  Concerning that the sample size formula could produce a very extreme small sample size, therefore, the determination of K1 and K2 should be based on reasonable estimates. We recommend for all sample size determinations for Cohen’s kappa agreement test, the true marginal rating frequencies can be assumed the same. Otherwise, it will be necessary to multiple the estimated minimum sample size by two to accommodate if the true marginal rating frequencies is not the same.},
	number = {2},
	urldate = {2025-02-19},
	journal = {Epidemiology, Biostatistics, and Public Health},
	author = {Bujang, Mohamad Adam and Baharum, Nurakmal},
	month = mar,
	year = {2022},
}

@article{noauthor_pdf_2024,
	title = {({PDF}) {Guidelines} of the minimum sample size requirements for {Cohen}’s {Kappa}},
	url = {https://www.researchgate.net/publication/320148141_Guidelines_of_the_minimum_sample_size_requirements_for_Cohen's_Kappa},
	doi = {10.2427/12267},
	abstract = {PDF {\textbar} Background: To estimate sample size for Cohen’s kappa agreement test can be challenging especially when dealing with various effect sizes. This... {\textbar} Find, read and cite all the research you need on ResearchGate},
	language = {en},
	urldate = {2025-02-19},
	journal = {ResearchGate},
	month = oct,
	year = {2024},
}

@article{mchugh_interrater_2012,
	title = {Interrater reliability: the kappa statistic},
	volume = {22},
	issn = {1330-0962},
	shorttitle = {Interrater reliability},
	url = {https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3900052/},
	abstract = {The kappa statistic is frequently used to test interrater reliability. The importance of rater reliability lies in the fact that it represents the extent to which the data collected in the study are correct representations of the variables measured. Measurement of the extent to which data collectors (raters) assign the same score to the same variable is called interrater reliability. While there have been a variety of methods to measure interrater reliability, traditionally it was measured as percent agreement, calculated as the number of agreement scores divided by the total number of scores. In 1960, Jacob Cohen critiqued use of percent agreement due to its inability to account for chance agreement. He introduced the Cohen’s kappa, developed to account for the possibility that raters actually guess on at least some variables due to uncertainty. Like most correlation statistics, the kappa can range from −1 to +1. While the kappa is one of the most commonly used statistics to test interrater reliability, it has limitations. Judgments about what level of kappa should be acceptable for health research are questioned. Cohen’s suggested interpretation may be too lenient for health related studies because it implies that a score as low as 0.41 might be acceptable. Kappa and percent agreement are compared, and levels for both kappa and percent agreement that should be demanded in healthcare studies are suggested.},
	number = {3},
	urldate = {2025-02-19},
	journal = {Biochemia Medica},
	author = {McHugh, Mary L.},
	month = oct,
	year = {2012},
	pmid = {23092060},
	pmcid = {PMC3900052},
	pages = {276--282},
}

@article{cohen_coefficient_1960,
	title = {A {Coefficient} of {Agreement} for {Nominal} {Scales}},
	volume = {20},
	issn = {0013-1644},
	url = {https://doi.org/10.1177/001316446002000104},
	doi = {10.1177/001316446002000104},
	language = {en},
	number = {1},
	urldate = {2025-02-19},
	journal = {Educational and Psychological Measurement},
	author = {Cohen, Jacob},
	month = apr,
	year = {1960},
	note = {Publisher: SAGE Publications Inc},
	pages = {37--46},
}

@article{noauthor_pdf_2024-1,
	title = {({PDF}) {A} {Machine} {Learning} {Framework} for {Assessing} {Experts}’ {Decision} {Quality}},
	url = {https://www.researchgate.net/publication/384947334_A_Machine_Learning_Framework_for_Assessing_Experts'_Decision_Quality},
	doi = {10.1287/mnsc.2021.03357},
	abstract = {PDF {\textbar} Expert workers make non-trivial decisions with significant implications. Experts’ decision accuracy is, thus, a fundamental aspect of their... {\textbar} Find, read and cite all the research you need on ResearchGate},
	language = {en},
	urldate = {2025-02-18},
	journal = {ResearchGate},
	month = nov,
	year = {2024},
}

@misc{song_learning_2022,
	title = {Learning from {Noisy} {Labels} with {Deep} {Neural} {Networks}: {A} {Survey}},
	shorttitle = {Learning from {Noisy} {Labels} with {Deep} {Neural} {Networks}},
	url = {http://arxiv.org/abs/2007.08199},
	doi = {10.48550/arXiv.2007.08199},
	abstract = {Deep learning has achieved remarkable success in numerous domains with help from large amounts of big data. However, the quality of data labels is a concern because of the lack of high-quality labels in many real-world scenarios. As noisy labels severely degrade the generalization performance of deep neural networks, learning from noisy labels (robust training) is becoming an important task in modern deep learning applications. In this survey, we first describe the problem of learning with label noise from a supervised learning perspective. Next, we provide a comprehensive review of 62 state-of-the-art robust training methods, all of which are categorized into five groups according to their methodological difference, followed by a systematic comparison of six properties used to evaluate their superiority. Subsequently, we perform an in-depth analysis of noise rate estimation and summarize the typically used evaluation methodology, including public noisy datasets and evaluation metrics. Finally, we present several promising research directions that can serve as a guideline for future studies. All the contents will be available at https://github.com/songhwanjun/Awesome-Noisy-Labels.},
	urldate = {2025-02-18},
	publisher = {arXiv},
	author = {Song, Hwanjun and Kim, Minseok and Park, Dongmin and Shin, Yooju and Lee, Jae-Gil},
	month = mar,
	year = {2022},
	note = {arXiv:2007.08199 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning},
}

@misc{noauthor_maximum_nodate,
	title = {Maximum {Likelihood} {Estimation} of {Observer} {Error}-{Rates} {Using} the {EM} {Algorithm} on {JSTOR}},
	url = {https://www.jstor.org/stable/2346806?seq=9},
	urldate = {2025-02-17},
}

@inproceedings{welinder_multidimensional_2010,
	title = {The {Multidimensional} {Wisdom} of {Crowds}},
	url = {https://www.semanticscholar.org/paper/The-Multidimensional-Wisdom-of-Crowds-Welinder-Branson/c2e250b4b49a9aa04b68dfd40dc69b022b1f8b3d},
	abstract = {Distributing labeling tasks among hundreds or thousands of annotators is an increasingly important method for annotating large datasets. We present a method for estimating the underlying value (e.g. the class) of each image from (noisy) annotations provided by multiple annotators. Our method is based on a model of the image formation and annotation process. Each image has different characteristics that are represented in an abstract Euclidean space. Each annotator is modeled as a multidimensional entity with variables representing competence, expertise and bias. This allows the model to discover and represent groups of annotators that have different sets of skills and knowledge, as well as groups of images that differ qualitatively. We find that our model predicts ground truth labels on both synthetic and real data more accurately than state of the art methods. Experiments also show that our model, starting from a set of binary labels, may discover rich information, such as different "schools of thought" amongst the annotators, and can group together images belonging to separate categories.},
	urldate = {2025-02-17},
	author = {Welinder, P. and Branson, Steve and Belongie, Serge J. and Perona, P.},
	month = dec,
	year = {2010},
}

@inproceedings{noauthor_pdf_nodate,
	title = {({PDF}) {The} {Multidimensional} {Wisdom} of {Crowds}.},
	url = {https://www.researchgate.net/publication/221619287_The_Multidimensional_Wisdom_of_Crowds},
	abstract = {PDF {\textbar} Distributing labeling tasks among hundreds or thousands of annotators is an in- creasingly important method for annotating large datasets. We... {\textbar} Find, read and cite all the research you need on ResearchGate},
	language = {en},
	urldate = {2025-02-17},
	booktitle = {{ResearchGate}},
}

@inproceedings{mnih_learning_2010,
	address = {Berlin, Heidelberg},
	title = {Learning to {Detect} {Roads} in {High}-{Resolution} {Aerial} {Images}},
	isbn = {978-3-642-15567-3},
	doi = {10.1007/978-3-642-15567-3_16},
	abstract = {Reliably extracting information from aerial imagery is a difficult problem with many practical applications. One specific case of this problem is the task of automatically detecting roads. This task is a difficult vision problem because of occlusions, shadows, and a wide variety of non-road objects. Despite 30 years of work on automatic road detection, no automatic or semi-automatic road detection system is currently on the market and no published method has been shown to work reliably on large datasets of urban imagery. We propose detecting roads using a neural network with millions of trainable weights which looks at a much larger context than was used in previous attempts at learning the task. The network is trained on massive amounts of data using a consumer GPU. We demonstrate that predictive performance can be substantially improved by initializing the feature detectors using recently developed unsupervised learning methods as well as by taking advantage of the local spatial coherence of the output labels. We show that our method works reliably on two challenging urban datasets that are an order of magnitude larger than what was used to evaluate previous approaches.},
	language = {en},
	booktitle = {Computer {Vision} – {ECCV} 2010},
	publisher = {Springer},
	author = {Mnih, Volodymyr and Hinton, Geoffrey E.},
	editor = {Daniilidis, Kostas and Maragos, Petros and Paragios, Nikos},
	year = {2010},
	keywords = {Aerial Image, Aerial Imagery, Hide Unit, Road Network, Stochastic Gradient Descent},
	pages = {210--223},
}

@inproceedings{noauthor_pdf_nodate-1,
	title = {({PDF}) {The} {Multidimensional} {Wisdom} of {Crowds}.},
	url = {https://www.researchgate.net/publication/221619287_The_Multidimensional_Wisdom_of_Crowds},
	abstract = {PDF {\textbar} Distributing labeling tasks among hundreds or thousands of annotators is an in- creasingly important method for annotating large datasets. We... {\textbar} Find, read and cite all the research you need on ResearchGate},
	language = {en},
	urldate = {2025-02-17},
	booktitle = {{ResearchGate}},
}

@article{krause_grader_2018,
	title = {Grader {Variability} and the {Importance} of {Reference} {Standards} for {Evaluating} {Machine} {Learning} {Models} for {Diabetic} {Retinopathy}},
	volume = {125},
	issn = {1549-4713},
	doi = {10.1016/j.ophtha.2018.01.034},
	abstract = {PURPOSE: Use adjudication to quantify errors in diabetic retinopathy (DR) grading based on individual graders and majority decision, and to train an improved automated algorithm for DR grading.
DESIGN: Retrospective analysis.
PARTICIPANTS: Retinal fundus images from DR screening programs.
METHODS: Images were each graded by the algorithm, U.S. board-certified ophthalmologists, and retinal specialists. The adjudicated consensus of the retinal specialists served as the reference standard.
MAIN OUTCOME MEASURES: For agreement between different graders as well as between the graders and the algorithm, we measured the (quadratic-weighted) kappa score. To compare the performance of different forms of manual grading and the algorithm for various DR severity cutoffs (e.g., mild or worse DR, moderate or worse DR), we measured area under the curve (AUC), sensitivity, and specificity.
RESULTS: Of the 193 discrepancies between adjudication by retinal specialists and majority decision of ophthalmologists, the most common were missing microaneurysm (MAs) (36\%), artifacts (20\%), and misclassified hemorrhages (16\%). Relative to the reference standard, the kappa for individual retinal specialists, ophthalmologists, and algorithm ranged from 0.82 to 0.91, 0.80 to 0.84, and 0.84, respectively. For moderate or worse DR, the majority decision of ophthalmologists had a sensitivity of 0.838 and specificity of 0.981. The algorithm had a sensitivity of 0.971, specificity of 0.923, and AUC of 0.986. For mild or worse DR, the algorithm had a sensitivity of 0.970, specificity of 0.917, and AUC of 0.986. By using a small number of adjudicated consensus grades as a tuning dataset and higher-resolution images as input, the algorithm improved in AUC from 0.934 to 0.986 for moderate or worse DR.
CONCLUSIONS: Adjudication reduces the errors in DR grading. A small set of adjudicated DR grades allows substantial improvements in algorithm performance. The resulting algorithm's performance was on par with that of individual U.S. Board-Certified ophthalmologists and retinal specialists.},
	language = {eng},
	number = {8},
	journal = {Ophthalmology},
	author = {Krause, Jonathan and Gulshan, Varun and Rahimy, Ehsan and Karth, Peter and Widner, Kasumi and Corrado, Greg S. and Peng, Lily and Webster, Dale R.},
	month = aug,
	year = {2018},
	pmid = {29548646},
	keywords = {Algorithms, Clinical Competence, Diabetic Retinopathy, Female, Humans, Machine Learning, Male, Mass Screening, Middle Aged, Ophthalmologists, ROC Curve, Reference Standards, Retrospective Studies},
	pages = {1264--1272},
}

@article{carbonneau_multiple_2018,
	title = {Multiple instance learning: {A} survey of problem characteristics and applications},
	volume = {77},
	issn = {0031-3203},
	shorttitle = {Multiple instance learning},
	url = {https://www.sciencedirect.com/science/article/pii/S0031320317304065},
	doi = {10.1016/j.patcog.2017.10.009},
	abstract = {Multiple instance learning (MIL) is a form of weakly supervised learning where training instances are arranged in sets, called bags, and a label is provided for the entire bag. This formulation is gaining interest because it naturally fits various problems and allows to leverage weakly labeled data. Consequently, it has been used in diverse application fields such as computer vision and document classification. However, learning from bags raises important challenges that are unique to MIL. This paper provides a comprehensive survey of the characteristics which define and differentiate the types of MIL problems. Until now, these problem characteristics have not been formally identified and described. As a result, the variations in performance of MIL algorithms from one data set to another are difficult to explain. In this paper, MIL problem characteristics are grouped into four broad categories: the composition of the bags, the types of data distribution, the ambiguity of instance labels, and the task to be performed. Methods specialized to address each category are reviewed. Then, the extent to which these characteristics manifest themselves in key MIL application areas are described. Finally, experiments are conducted to compare the performance of 16 state-of-the-art MIL methods on selected problem characteristics. This paper provides insight on how the problem characteristics affect MIL algorithms, recommendations for future benchmarking and promising avenues for research. Code is available on-line at https://github.com/macarbonneau/MILSurvey.},
	urldate = {2025-02-13},
	journal = {Pattern Recognition},
	author = {Carbonneau, Marc-André and Cheplygina, Veronika and Granger, Eric and Gagnon, Ghyslain},
	month = may,
	year = {2018},
	keywords = {Classification, Computer aided diagnosis, Computer vision, Document classification, Drug activity prediction, Multi-instance learning, Multiple instance learning, Weakly supervised learning},
	pages = {329--353},
}

@article{tavolara_contrastive_2022,
	title = {Contrastive {Multiple} {Instance} {Learning}: {An} {Unsupervised} {Framework} for {Learning} {Slide}-{Level} {Representations} of {Whole} {Slide} {Histopathology} {Images} without {Labels}},
	volume = {14},
	copyright = {http://creativecommons.org/licenses/by/3.0/},
	issn = {2072-6694},
	shorttitle = {Contrastive {Multiple} {Instance} {Learning}},
	url = {https://www.mdpi.com/2072-6694/14/23/5778},
	doi = {10.3390/cancers14235778},
	abstract = {Recent methods in computational pathology have trended towards semi- and weakly-supervised methods requiring only slide-level labels. Yet, even slide-level labels may be absent or irrelevant to the application of interest, such as in clinical trials. Hence, we present a fully unsupervised method to learn meaningful, compact representations of WSIs. Our method initially trains a tile-wise encoder using SimCLR, from which subsets of tile-wise embeddings are extracted and fused via an attention-based multiple-instance learning framework to yield slide-level representations. The resulting set of intra-slide-level and inter-slide-level embeddings are attracted and repelled via contrastive loss, respectively. This resulted in slide-level representations with self-supervision. We applied our method to two tasks— (1) non-small cell lung cancer subtyping (NSCLC) as a classification prototype and (2) breast cancer proliferation scoring (TUPAC16) as a regression prototype—and achieved an AUC of 0.8641 ± 0.0115 and correlation (R2) of 0.5740 ± 0.0970, respectively. Ablation experiments demonstrate that the resulting unsupervised slide-level feature space can be fine-tuned with small datasets for both tasks. Overall, our method approaches computational pathology in a novel manner, where meaningful features can be learned from whole-slide images without the need for annotations of slide-level labels. The proposed method stands to benefit computational pathology, as it theoretically enables researchers to benefit from completely unlabeled whole-slide images.},
	language = {en},
	number = {23},
	urldate = {2025-02-13},
	journal = {Cancers},
	author = {Tavolara, Thomas E. and Gurcan, Metin N. and Niazi, M. Khalid Khan},
	month = jan,
	year = {2022},
	note = {Number: 23
Publisher: Multidisciplinary Digital Publishing Institute},
	keywords = {contrastive, deep learning, histopathology, multiple instance learning, self-supervised},
	pages = {5778},
}

@article{carbonneau_multiple_2018-1,
	title = {Multiple instance learning: {A} survey of problem characteristics and applications},
	volume = {77},
	issn = {0031-3203},
	shorttitle = {Multiple instance learning},
	url = {https://www.sciencedirect.com/science/article/pii/S0031320317304065},
	doi = {10.1016/j.patcog.2017.10.009},
	abstract = {Multiple instance learning (MIL) is a form of weakly supervised learning where training instances are arranged in sets, called bags, and a label is provided for the entire bag. This formulation is gaining interest because it naturally fits various problems and allows to leverage weakly labeled data. Consequently, it has been used in diverse application fields such as computer vision and document classification. However, learning from bags raises important challenges that are unique to MIL. This paper provides a comprehensive survey of the characteristics which define and differentiate the types of MIL problems. Until now, these problem characteristics have not been formally identified and described. As a result, the variations in performance of MIL algorithms from one data set to another are difficult to explain. In this paper, MIL problem characteristics are grouped into four broad categories: the composition of the bags, the types of data distribution, the ambiguity of instance labels, and the task to be performed. Methods specialized to address each category are reviewed. Then, the extent to which these characteristics manifest themselves in key MIL application areas are described. Finally, experiments are conducted to compare the performance of 16 state-of-the-art MIL methods on selected problem characteristics. This paper provides insight on how the problem characteristics affect MIL algorithms, recommendations for future benchmarking and promising avenues for research. Code is available on-line at https://github.com/macarbonneau/MILSurvey.},
	urldate = {2025-02-13},
	journal = {Pattern Recognition},
	author = {Carbonneau, Marc-André and Cheplygina, Veronika and Granger, Eric and Gagnon, Ghyslain},
	month = may,
	year = {2018},
	keywords = {Classification, Computer aided diagnosis, Computer vision, Document classification, Drug activity prediction, Multi-instance learning, Multiple instance learning, Weakly supervised learning},
	pages = {329--353},
}

@article{nagpal_development_2019,
	title = {Development and validation of a deep learning algorithm for improving {Gleason} scoring of prostate cancer},
	volume = {2},
	copyright = {2019 The Author(s)},
	issn = {2398-6352},
	url = {https://www.nature.com/articles/s41746-019-0112-2},
	doi = {10.1038/s41746-019-0112-2},
	abstract = {For prostate cancer patients, the Gleason score is one of the most important prognostic factors, potentially determining treatment independent of the stage. However, Gleason scoring is based on subjective microscopic examination of tumor morphology and suffers from poor reproducibility. Here we present a deep learning system (DLS) for Gleason scoring whole-slide images of prostatectomies. Our system was developed using 112 million pathologist-annotated image patches from 1226 slides, and evaluated on an independent validation dataset of 331 slides. Compared to a reference standard provided by genitourinary pathology experts, the mean accuracy among 29 general pathologists was 0.61 on the validation set. The DLS achieved a significantly higher diagnostic accuracy of 0.70 (p = 0.002) and trended towards better patient risk stratification in correlations to clinical follow-up data. Our approach could improve the accuracy of Gleason scoring and subsequent therapy decisions, particularly where specialist expertise is unavailable. The DLS also goes beyond the current Gleason system to more finely characterize and quantitate tumor morphology, providing opportunities for refinement of the Gleason system itself.},
	language = {en},
	number = {1},
	urldate = {2025-02-13},
	journal = {npj Digital Medicine},
	author = {Nagpal, Kunal and Foote, Davis and Liu, Yun and Chen, Po-Hsuan Cameron and Wulczyn, Ellery and Tan, Fraser and Olson, Niels and Smith, Jenny L. and Mohtashamian, Arash and Wren, James H. and Corrado, Greg S. and MacDonald, Robert and Peng, Lily H. and Amin, Mahul B. and Evans, Andrew J. and Sangoi, Ankur R. and Mermel, Craig H. and Hipp, Jason D. and Stumpe, Martin C.},
	month = jun,
	year = {2019},
	note = {Publisher: Nature Publishing Group},
	keywords = {Prostate cancer},
	pages = {1--10},
}

@article{hou_patch-based_2016,
	title = {Patch-based {Convolutional} {Neural} {Network} for {Whole} {Slide} {Tissue} {Image} {Classification}},
	volume = {2016},
	issn = {1063-6919},
	doi = {10.1109/CVPR.2016.266},
	abstract = {Convolutional Neural Networks (CNN) are state-of-the-art models for many image classification tasks. However, to recognize cancer subtypes automatically, training a CNN on gigapixel resolution Whole Slide Tissue Images (WSI) is currently computationally impossible. The differentiation of cancer subtypes is based on cellular-level visual features observed on image patch scale. Therefore, we argue that in this situation, training a patch-level classifier on image patches will perform better than or similar to an image-level classifier. The challenge becomes how to intelligently combine patch-level classification results and model the fact that not all patches will be discriminative. We propose to train a decision fusion model to aggregate patch-level predictions given by patch-level CNNs, which to the best of our knowledge has not been shown before. Furthermore, we formulate a novel Expectation-Maximization (EM) based method that automatically locates discriminative patches robustly by utilizing the spatial relationships of patches. We apply our method to the classification of glioma and non-small-cell lung carcinoma cases into subtypes. The classification accuracy of our method is similar to the inter-observer agreement between pathologists. Although it is impossible to train CNNs on WSIs, we experimentally demonstrate using a comparable non-cancer dataset of smaller images that a patch-based CNN can outperform an image-based CNN.},
	language = {eng},
	journal = {Proceedings. IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	author = {Hou, Le and Samaras, Dimitris and Kurc, Tahsin M. and Gao, Yi and Davis, James E. and Saltz, Joel H.},
	year = {2016},
	pmid = {27795661},
	pmcid = {PMC5085270},
	pages = {2424--2433},
}

@article{campanella_clinical-grade_2019,
	title = {Clinical-grade computational pathology using weakly supervised deep learning on whole slide images},
	volume = {25},
	copyright = {2019 The Author(s), under exclusive licence to Springer Nature America, Inc.},
	issn = {1546-170X},
	url = {https://www.nature.com/articles/s41591-019-0508-1},
	doi = {10.1038/s41591-019-0508-1},
	abstract = {The development of decision support systems for pathology and their deployment in clinical practice have been hindered by the need for large manually annotated datasets. To overcome this problem, we present a multiple instance learning-based deep learning system that uses only the reported diagnoses as labels for training, thereby avoiding expensive and time-consuming pixel-wise manual annotations. We evaluated this framework at scale on a dataset of 44,732 whole slide images from 15,187 patients without any form of data curation. Tests on prostate cancer, basal cell carcinoma and breast cancer metastases to axillary lymph nodes resulted in areas under the curve above 0.98 for all cancer types. Its clinical application would allow pathologists to exclude 65–75\% of slides while retaining 100\% sensitivity. Our results show that this system has the ability to train accurate classification models at unprecedented scale, laying the foundation for the deployment of computational decision support systems in clinical practice.},
	language = {en},
	number = {8},
	urldate = {2025-02-13},
	journal = {Nature Medicine},
	author = {Campanella, Gabriele and Hanna, Matthew G. and Geneslaw, Luke and Miraflor, Allen and Werneck Krauss Silva, Vitor and Busam, Klaus J. and Brogi, Edi and Reuter, Victor E. and Klimstra, David S. and Fuchs, Thomas J.},
	month = aug,
	year = {2019},
	note = {Publisher: Nature Publishing Group},
	keywords = {Computer modelling, Computer science, High-throughput screening, Pathology},
	pages = {1301--1309},
}

@article{chen_microscope_2019,
	title = {Microscope 2.0: {An} {Augmented} {Reality} {Microscope} with {Real}-time {Artificial} {Intelligence} {Integration}},
	volume = {25},
	issn = {1078-8956, 1546-170X},
	shorttitle = {Microscope 2.0},
	url = {http://arxiv.org/abs/1812.00825},
	doi = {10.1038/s41591-019-0539-7},
	abstract = {The brightfield microscope is instrumental in the visual examination of both biological and physical samples at sub-millimeter scales. One key clinical application has been in cancer histopathology, where the microscopic assessment of the tissue samples is used for the diagnosis and staging of cancer and thus guides clinical therapy. However, the interpretation of these samples is inherently subjective, resulting in significant diagnostic variability. Moreover, in many regions of the world, access to pathologists is severely limited due to lack of trained personnel. In this regard, Artificial Intelligence (AI) based tools promise to improve the access and quality of healthcare. However, despite significant advances in AI research, integration of these tools into real-world cancer diagnosis workflows remains challenging because of the costs of image digitization and difficulties in deploying AI solutions. Here we propose a cost-effective solution to the integration of AI: the Augmented Reality Microscope (ARM). The ARM overlays AI-based information onto the current view of the sample through the optical pathway in real-time, enabling seamless integration of AI into the regular microscopy workflow. We demonstrate the utility of ARM in the detection of lymph node metastases in breast cancer and the identification of prostate cancer with a latency that supports real-time workflows. We anticipate that ARM will remove barriers towards the use of AI in microscopic analysis and thus improve the accuracy and efficiency of cancer diagnosis. This approach is applicable to other microscopy tasks and AI algorithms in the life sciences and beyond.},
	number = {9},
	urldate = {2025-02-13},
	journal = {Nature Medicine},
	author = {Chen, Po-Hsuan Cameron and Gadepalli, Krishna and MacDonald, Robert and Liu, Yun and Nagpal, Kunal and Kohlberger, Timo and Dean, Jeffrey and Corrado, Greg S. and Hipp, Jason D. and Stumpe, Martin C.},
	month = sep,
	year = {2019},
	note = {arXiv:1812.00825 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	pages = {1453--1457},
}

@article{quellec_multiple-instance_2017,
	title = {Multiple-{Instance} {Learning} for {Medical} {Image} and {Video} {Analysis}},
	volume = {10},
	issn = {1941-1189},
	url = {https://ieeexplore.ieee.org/abstract/document/7812612},
	doi = {10.1109/RBME.2017.2651164},
	abstract = {Multiple-instance learning (MIL) is a recent machine-learning paradigm that is particularly well suited to medical image and video analysis (MIVA) tasks. Based solely on class labels assigned globally to images or videos, MIL algorithms learn to detect relevant patterns locally in images or videos. These patterns are then used for classification at a global level. Because supervision relies on global labels, manual segmentations are not needed to train MIL algorithms, unlike traditional single-instance learning (SIL) algorithms. Consequently, these solutions are attracting increasing interest from the MIVA community: since the term was coined by Dietterich et al. in 1997, 73 research papers about MIL have been published in the MIVA literature. This paper reviews the existing strategies for modeling MIVA tasks as MIL problems, recommends general-purpose MIL algorithms for each type of MIVA tasks, and discusses MIVA-specific MIL algorithms. Various experiments performed in medical image and video datasets are compiled in order to back up these discussions. This meta-analysis shows that, besides being more convenient than SIL solutions, MIL algorithms are also more accurate in many cases. In other words, MIL is the ideal solution for many MIVA tasks. Recent trends are discussed, and future directions are proposed for this emerging paradigm.},
	urldate = {2025-02-13},
	journal = {IEEE Reviews in Biomedical Engineering},
	author = {Quellec, Gwenolé and Cazuguel, Guy and Cochener, Béatrice and Lamard, Mathieu},
	year = {2017},
	note = {Conference Name: IEEE Reviews in Biomedical Engineering},
	keywords = {Algorithm design and analysis, Biomedical image processing, Cancer, Feature extraction, Image segmentation, Lesions, Medical image analysis, medical video analysis, multiple-instance learning (MIL)},
	pages = {213--234},
}

@article{raykar_supervised_nodate,
	title = {Supervised {Learning} from {Multiple} {Experts}: {Whom} to trust when everyone lies a bit},
	abstract = {We describe a probabilistic approach for supervised learning when we have multiple experts/annotators providing (possibly noisy) labels but no absolute gold standard. The proposed algorithm evaluates the diﬀerent experts and also gives an estimate of the actual hidden labels. Experimental results indicate that the proposed method is superior to the commonly used majority voting baseline.},
	language = {en},
	author = {Raykar, Vikas C and Yu, Shipeng and Zhao, Linda H and Jerebko, Anna and Florin, Charles and Valadez, Gerardo Hermosillo and Bogoni, Luca and Moy, Linda and Moy, Linda},
}

@article{raykar_supervised_nodate-1,
	title = {Supervised {Learning} from {Multiple} {Experts}: {Whom} to trust when everyone lies a bit},
	abstract = {We describe a probabilistic approach for supervised learning when we have multiple experts/annotators providing (possibly noisy) labels but no absolute gold standard. The proposed algorithm evaluates the diﬀerent experts and also gives an estimate of the actual hidden labels. Experimental results indicate that the proposed method is superior to the commonly used majority voting baseline.},
	language = {en},
	author = {Raykar, Vikas C and Yu, Shipeng and Zhao, Linda H and Jerebko, Anna and Florin, Charles and Valadez, Gerardo Hermosillo and Bogoni, Luca and Moy, Linda and Moy, Linda},
}

@article{dritsa_how_2024,
	title = {How {Design} {Researchers} {Make} {Sense} of {Data} {Visualizations} in {Data}-{Driven} {Design}: {An} {Uncertainty}-{Aware} {Sensemaking} {Model}},
	volume = {31},
	issn = {1073-0516, 1557-7325},
	shorttitle = {How {Design} {Researchers} {Make} {Sense} of {Data} {Visualizations} in {Data}-{Driven} {Design}},
	url = {https://dl.acm.org/doi/10.1145/3685268},
	doi = {10.1145/3685268},
	abstract = {While data is the cornerstone of modern design strategies, design researchers frequently struggle when performing data work. This creates a need to design tools that enable design researchers to actively engage with data. However, this presupposes understanding how design researchers create meaning from data representations, as the way of visualizing the data, along with other factors, can significantly impact the extracted insights, increasing uncertainty about the quality of the outcome. As a response to this problem, we explore how design researchers make sense of data in a case study: making sense of paired subjective and objective sleep and stress data visualizations. By synthesizing our findings from two user studies, we construct a sensemaking model which highlights how uncertainty related to data qualities, visualization parameters, and the viewer's background affects the insight-generation process. Our findings have implications for the future development of tools and techniques for visual data sensemaking for designers.},
	language = {en},
	number = {6},
	urldate = {2025-02-08},
	journal = {ACM Transactions on Computer-Human Interaction},
	author = {Dritsa, Dimitra and Houben, Steven},
	month = dec,
	year = {2024},
	pages = {1--53},
}

@article{graber_incidence_2013,
	title = {The incidence of diagnostic error in medicine},
	volume = {22},
	copyright = {Published by the BMJ Publishing Group Limited. For permission to use (where not already granted under a licence) please go to http://group.bmj.com/group/rights-licensing/permissions. this is an open access article distributed in accordance with the creative commons attribution non commercial (cc by-nc 3.0) license, which permits others to distribute, remix, adapt, build upon this work non-commercially, and license their derivative works on different terms, provided the original work is properly cited and the use is non-commercial. see: http://creativecommons.org/licenses/by-nc/3.0/},
	issn = {2044-5415, 2044-5423},
	url = {https://qualitysafety.bmj.com/content/22/Suppl_2/ii21},
	doi = {10.1136/bmjqs-2012-001615},
	abstract = {A wide variety of research studies suggest that breakdowns in the diagnostic process result in a staggering toll of harm and patient deaths. These include autopsy studies, case reviews, surveys of patient and physicians, voluntary reporting systems, using standardised patients, second reviews, diagnostic testing audits and closed claims reviews. Although these different approaches provide important information and unique insights regarding diagnostic errors, each has limitations and none is well suited to establishing the incidence of diagnostic error in actual practice, or the aggregate rate of error and harm. We argue that being able to measure the incidence of diagnostic error is essential to enable research studies on diagnostic error, and to initiate quality improvement projects aimed at reducing the risk of error and harm. Three approaches appear most promising in this regard: (1) using ‘trigger tools’ to identify from electronic health records cases at high risk for diagnostic error; (2) using standardised patients (secret shoppers) to study the rate of error in practice; (3) encouraging both patients and physicians to voluntarily report errors they encounter, and facilitating this process.},
	language = {en},
	number = {Suppl 2},
	urldate = {2025-02-05},
	journal = {BMJ Quality \& Safety},
	author = {Graber, Mark L.},
	month = oct,
	year = {2013},
	pmid = {23771902},
	note = {Publisher: BMJ Publishing Group Ltd
Section: Narrative review},
	keywords = {Decision making, Diagnostic errors, Medical error, measurement/epidemiology, Patient safety},
	pages = {ii21--ii27},
}

@misc{guan_who_2018,
	title = {Who {Said} {What}: {Modeling} {Individual} {Labelers} {Improves} {Classification}},
	shorttitle = {Who {Said} {What}},
	url = {http://arxiv.org/abs/1703.08774},
	doi = {10.48550/arXiv.1703.08774},
	abstract = {Data are often labeled by many different experts with each expert only labeling a small fraction of the data and each data point being labeled by several experts. This reduces the workload on individual experts and also gives a better estimate of the unobserved ground truth. When experts disagree, the standard approaches are to treat the majority opinion as the correct label or to model the correct label as a distribution. These approaches, however, do not make any use of potentially valuable information about which expert produced which label. To make use of this extra information, we propose modeling the experts individually and then learning averaging weights for combining them, possibly in sample-specific ways. This allows us to give more weight to more reliable experts and take advantage of the unique strengths of individual experts at classifying certain types of data. Here we show that our approach leads to improvements in computer-aided diagnosis of diabetic retinopathy. We also show that our method performs better than competing algorithms by Welinder and Perona (2010), and by Mnih and Hinton (2012). Our work offers an innovative approach for dealing with the myriad real-world settings that use expert opinions to define labels for training.},
	urldate = {2025-02-03},
	publisher = {arXiv},
	author = {Guan, Melody Y. and Gulshan, Varun and Dai, Andrew M. and Hinton, Geoffrey E.},
	month = jan,
	year = {2018},
	note = {arXiv:1703.08774 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
}

@inproceedings{mnih_learning_2012,
	title = {Learning to {Label} {Aerial} {Images} from {Noisy} {Data}},
	url = {https://www.semanticscholar.org/paper/Learning-to-Label-Aerial-Images-from-Noisy-Data-Mnih-Hinton/cdfdaccd946dc2fe4863aed048b12b5d2282f285},
	abstract = {When training a system to label images, the amount of labeled training data tends to be a limiting factor. We consider the task of learning to label aerial images from existing maps. These provide abundant labels, but the labels are often incomplete and sometimes poorly registered. We propose two robust loss functions for dealing with these kinds of label noise and use the loss functions to train a deep neural network on two challenging aerial image datasets. The robust loss functions lead to big improvements in performance and our best system substantially outperforms the best published results on the task we consider.},
	urldate = {2025-02-04},
	author = {Mnih, Volodymyr and Hinton, Geoffrey E.},
	month = jun,
	year = {2012},
}

@inproceedings{welinder_online_2010,
	title = {Online crowdsourcing: {Rating} annotators and obtaining cost-effective labels},
	shorttitle = {Online crowdsourcing},
	url = {https://ieeexplore.ieee.org/document/5543189},
	doi = {10.1109/CVPRW.2010.5543189},
	abstract = {Labeling large datasets has become faster, cheaper, and easier with the advent of crowdsourcing services like Amazon Mechanical Turk. How can one trust the labels obtained from such services? We propose a model of the labeling process which includes label uncertainty, as well a multi-dimensional measure of the annotators' ability. From the model we derive an online algorithm that estimates the most likely value of the labels and the annotator abilities. It finds and prioritizes experts when requesting labels, and actively excludes unreliable annotators. Based on labels already obtained, it dynamically chooses which images will be labeled next, and how many labels to request in order to achieve a desired level of confidence. Our algorithm is general and can handle binary, multi-valued, and continuous annotations (e.g. bounding boxes). Experiments on a dataset containing more than 50,000 labels show that our algorithm reduces the number of labels required, and thus the total cost of labeling, by a large factor while keeping error rates low on a variety of datasets.},
	urldate = {2025-02-04},
	booktitle = {2010 {IEEE} {Computer} {Society} {Conference} on {Computer} {Vision} and {Pattern} {Recognition} - {Workshops}},
	author = {Welinder, Peter and Perona, Pietro},
	month = jun,
	year = {2010},
	note = {ISSN: 2160-7516},
	keywords = {Adaptation model, Computer vision, Costs, Error analysis, Labeling, Noise figure, Outsourcing},
	pages = {25--32},
}

@article{elmore_variability_1994,
	title = {Variability in radiologists' interpretations of mammograms},
	volume = {331},
	issn = {0028-4793},
	doi = {10.1056/NEJM199412013312206},
	abstract = {BACKGROUND: Despite the proved value of mammography in screening for breast cancer, its efficacy depends on radiologists' interpretations. The variability in such interpretations is not well understood.
METHODS: Using a technique of stratified random sampling, we selected 150 mammograms obtained in 1987: 27 from women with histopathologically confirmed breast cancer and 123 from women with no evidence of breast cancer after three years of follow-up examinations. Ten radiologists, who were unaware of the diagnoses and research hypothesis, each interpreted the 150 mammograms. Disagreement was analyzed within pairs of the 10 radiologists, as well as for the group of 150 women as a whole.
RESULTS: The diagnostic consistency between pairs of radiologists was moderate, with a median weighted percentage of agreement of 78 percent (weighted kappa, 0.47). The frequency of the radiologists' recommendations for an immediate workup ranged from 74 to 96 percent for mammograms from the women with cancer and from 11 to 65 percent for films from the women without cancer. A substantial disagreement in management recommendations--in which one radiologist recommended routine follow-up and another recommended a biopsy for the same patient--occurred in 3 percent of the pairwise comparisons but in 25 percent of the comparisons for the group of women as a whole. When two or more radiologists recommended a biopsy for the same patient, a disagreement in the stated location (right or left breast) occurred in 2 percent of the pairwise comparisons among the radiologists but in 9 percent of comparisons for the group of women as a whole. Because some disagreement was likely, given that 10 radiologists read each film, the pairwise comparison is a more conservative estimate of disagreement.
CONCLUSIONS: Although mammography is of value in screening women for breast cancer, radiologists can differ, sometimes substantially, in their interpretations of mammograms and in their recommendations for management. Efforts to improve accuracy and reduce variability in interpretation may increase the effectiveness of mammography in detecting early breast cancers.},
	language = {eng},
	number = {22},
	journal = {The New England Journal of Medicine},
	author = {Elmore, J. G. and Wells, C. K. and Lee, C. H. and Howard, D. H. and Feinstein, A. R.},
	month = dec,
	year = {1994},
	pmid = {7969300},
	keywords = {Breast Diseases, Breast Neoplasms, Clinical Competence, Diagnosis, Differential, Female, Humans, Mammography, Observer Variation, Patient Selection, Radiology, Sensitivity and Specificity},
	pages = {1493--1499},
}

@book{vanderWel_myrthle_2019,
	title = {What makes an expert {Barrett}’s pathologist?: {Concordance} and pathologist expertise within a digital review panel},
	isbn = {978-94-6375-637-2},
	shorttitle = {What makes an expert {Barrett}’s pathologist?},
	url = {https://dare.uva.nl/search?identifier=dbd5a0c3-bb77-4663-b483-b9944a37f085},
	language = {en},
	urldate = {2025-02-03},
	author = {van der Wel, M. J.},
	year = {2019},
}

@misc{tanno_learning_2019,
	title = {Learning {From} {Noisy} {Labels} {By} {Regularized} {Estimation} {Of} {Annotator} {Confusion}},
	url = {http://arxiv.org/abs/1902.03680},
	doi = {10.48550/arXiv.1902.03680},
	abstract = {The predictive performance of supervised learning algorithms depends on the quality of labels. In a typical label collection process, multiple annotators provide subjective noisy estimates of the “truth” under the inﬂuence of their varying skill-levels and biases. Blindly treating these noisy labels as the ground truth limits the accuracy of learning algorithms in the presence of strong disagreement. This problem is critical for applications in domains such as medical imaging where both the annotation cost and inter-observer variability are high. In this work, we present a method for simultaneously learning the individual annotator model and the underlying true label distribution, using only noisy observations. Each annotator is modeled by a confusion matrix that is jointly estimated along with the classiﬁer predictions. We propose to add a regularization term to the loss function that encourages convergence to the true annotator confusion matrix. We provide a theoretical argument as to how the regularization is essential to our approach both for the case of single annotator and multiple annotators. Despite the simplicity of the idea, experiments on image classiﬁcation tasks with both simulated and real labels show that our method either outperforms or performs on par with the state-of-the-art methods and is capable of estimating the skills of annotators even with a single label available per image.},
	language = {en},
	urldate = {2025-02-03},
	publisher = {arXiv},
	author = {Tanno, Ryutaro and Saeedi, Ardavan and Sankaranarayanan, Swami and Alexander, Daniel C. and Silberman, Nathan},
	month = jun,
	year = {2019},
	note = {arXiv:1902.03680 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning},
}

@article{wang_transformer-based_2022,
	title = {Transformer-based unsupervised contrastive learning for histopathological image classification},
	volume = {81},
	issn = {1361-8415},
	url = {https://www.sciencedirect.com/science/article/pii/S1361841522002043},
	doi = {10.1016/j.media.2022.102559},
	abstract = {A large-scale and well-annotated dataset is a key factor for the success of deep learning in medical image analysis. However, assembling such large annotations is very challenging, especially for histopathological images with unique characteristics (e.g., gigapixel image size, multiple cancer types, and wide staining variations). To alleviate this issue, self-supervised learning (SSL) could be a promising solution that relies only on unlabeled data to generate informative representations and generalizes well to various downstream tasks even with limited annotations. In this work, we propose a novel SSL strategy called semantically-relevant contrastive learning (SRCL), which compares relevance between instances to mine more positive pairs. Compared to the two views from an instance in traditional contrastive learning, our SRCL aligns multiple positive instances with similar visual concepts, which increases the diversity of positives and then results in more informative representations. We employ a hybrid model (CTransPath) as the backbone, which is designed by integrating a convolutional neural network (CNN) and a multi-scale Swin Transformer architecture. The CTransPath is pretrained on massively unlabeled histopathological images that could serve as a collaborative local–global feature extractor to learn universal feature representations more suitable for tasks in the histopathology image domain. The effectiveness of our SRCL-pretrained CTransPath is investigated on five types of downstream tasks (patch retrieval, patch classification, weakly-supervised whole-slide image classification, mitosis detection, and colorectal adenocarcinoma gland segmentation), covering nine public datasets. The results show that our SRCL-based visual representations not only achieve state-of-the-art performance in each dataset, but are also more robust and transferable than other SSL methods and ImageNet pretraining (both supervised and self-supervised methods). Our code and pretrained model are available at https://github.com/Xiyue-Wang/TransPath.},
	urldate = {2025-02-03},
	journal = {Medical Image Analysis},
	author = {Wang, Xiyue and Yang, Sen and Zhang, Jun and Wang, Minghui and Zhang, Jing and Yang, Wei and Huang, Junzhou and Han, Xiao},
	month = oct,
	year = {2022},
	keywords = {Feature extraction, Histopathology, Self-supervised learning, Transformer},
	pages = {102559},
}

@misc{ilse_attention-based_2018,
	title = {Attention-based {Deep} {Multiple} {Instance} {Learning}},
	url = {http://arxiv.org/abs/1802.04712},
	doi = {10.48550/arXiv.1802.04712},
	abstract = {Multiple instance learning (MIL) is a variation of supervised learning where a single class label is assigned to a bag of instances. In this paper, we state the MIL problem as learning the Bernoulli distribution of the bag label where the bag label probability is fully parameterized by neural networks. Furthermore, we propose a neural network-based permutation-invariant aggregation operator that corresponds to the attention mechanism. Notably, an application of the proposed attention-based operator provides insight into the contribution of each instance to the bag label. We show empirically that our approach achieves comparable performance to the best MIL methods on benchmark MIL datasets and it outperforms other methods on a MNIST-based MIL dataset and two real-life histopathology datasets without sacrificing interpretability.},
	urldate = {2025-02-03},
	publisher = {arXiv},
	author = {Ilse, Maximilian and Tomczak, Jakub M. and Welling, Max},
	month = jun,
	year = {2018},
	note = {arXiv:1802.04712 [cs]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@article{wong_deep_2022,
	title = {Deep learning from multiple experts improves identification of amyloid neuropathologies},
	volume = {10},
	issn = {2051-5960},
	url = {https://actaneurocomms.biomedcentral.com/articles/10.1186/s40478-022-01365-0},
	doi = {10.1186/s40478-022-01365-0},
	abstract = {Pathologists can label pathologies differently, making it challenging to yield consistent assessments in the absence of one ground truth. To address this problem, we present a deep learning (DL) approach that draws on a cohort of experts, weighs each contribution, and is robust to noisy labels. We collected 100,495 annotations on 20,099 candidate amyloid beta neuropathologies (cerebral amyloid angiopathy (CAA), and cored and diffuse plaques) from three institutions, independently annotated by five experts. DL methods trained on a consensus-of-two strategy yielded 12.6–26\% improvements by area under the precision recall curve (AUPRC) when compared to those that learned individualized annotations. This strategy surpassed individual-expert models, even when unfairly assessed on benchmarks favoring them. Moreover, ensembling over individual models was robust to hidden random annotators. In blind prospective tests of 52,555 subsequent expert-annotated images, the models labeled pathologies like their human counterparts (consensus model AUPRC = 0.74 cored; 0.69 CAA). This study demonstrates a means to combine multiple ground truths into a common-ground DL model that yields consistent diagnoses informed by multiple and potentially variable expert opinions.},
	language = {en},
	number = {1},
	urldate = {2025-02-03},
	journal = {Acta Neuropathologica Communications},
	author = {Wong, Daniel R. and Tang, Ziqi and Mew, Nicholas C. and Das, Sakshi and Athey, Justin and McAleese, Kirsty E. and Kofler, Julia K. and Flanagan, Margaret E. and Borys, Ewa and White, Charles L. and Butte, Atul J. and Dugger, Brittany N. and Keiser, Michael J.},
	month = dec,
	year = {2022},
	pages = {66},
}

@misc{lu_data_2020,
	title = {Data {Efficient} and {Weakly} {Supervised} {Computational} {Pathology} on {Whole} {Slide} {Images}},
	url = {http://arxiv.org/abs/2004.09666},
	doi = {10.48550/arXiv.2004.09666},
	abstract = {The rapidly emerging field of computational pathology has the potential to enable objective diagnosis, therapeutic response prediction and identification of new morphological features of clinical relevance. However, deep learning-based computational pathology approaches either require manual annotation of gigapixel whole slide images (WSIs) in fully-supervised settings or thousands of WSIs with slide-level labels in a weakly-supervised setting. Moreover, whole slide level computational pathology methods also suffer from domain adaptation and interpretability issues. These challenges have prevented the broad adaptation of computational pathology for clinical and research purposes. Here we present CLAM - Clustering-constrained attention multiple instance learning, an easy-to-use, high-throughput, and interpretable WSI-level processing and learning method that only requires slide-level labels while being data efficient, adaptable and capable of handling multi-class subtyping problems. CLAM is a deep-learning-based weakly-supervised method that uses attention-based learning to automatically identify sub-regions of high diagnostic value in order to accurately classify the whole slide, while also utilizing instance-level clustering over the representative regions identified to constrain and refine the feature space. In three separate analyses, we demonstrate the data efficiency and adaptability of CLAM and its superior performance over standard weakly-supervised classification. We demonstrate that CLAM models are interpretable and can be used to identify well-known and new morphological features. We further show that models trained using CLAM are adaptable to independent test cohorts, cell phone microscopy images, and biopsies. CLAM is a general-purpose and adaptable method that can be used for a variety of different computational pathology tasks in both clinical and research settings.},
	urldate = {2025-02-03},
	publisher = {arXiv},
	author = {Lu, Ming Y. and Williamson, Drew F. K. and Chen, Tiffany Y. and Chen, Richard J. and Barbieri, Matteo and Mahmood, Faisal},
	month = may,
	year = {2020},
	note = {arXiv:2004.09666 [eess]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Electrical Engineering and Systems Science - Image and Video Processing, Quantitative Biology - Tissues and Organs},
}

@inproceedings{guan2018said,
  title={Who said what: Modeling individual labelers improves classification},
  author={Guan, Melody and Gulshan, Varun and Dai, Andrew and Hinton, Geoffrey},
  booktitle={Proceedings of the AAAI conference on artificial intelligence},
  volume={32},
  number={1},
  year={2018}
}
