@inproceedings{
hendrycks2017a,
title={A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks},
author={Dan Hendrycks and Kevin Gimpel},
booktitle={International Conference on Learning Representations},
year={2017},
url={https://openreview.net/forum?id=Hkg4TI9xl}
}

@inproceedings{
lee2018training,
title={Training Confidence-calibrated Classifiers for Detecting Out-of-Distribution Samples},
author={Kimin Lee and Honglak Lee and Kibok Lee and Jinwoo Shin},
booktitle={International Conference on Learning Representations},
year={2018},
url={https://openreview.net/forum?id=ryiAv2xAZ},
}



@inproceedings{lee_simple_2018,
	title = {A {Simple} {Unified} {Framework} for {Detecting} {Out}-of-{Distribution} {Samples} and {Adversarial} {Attacks}},
	volume = {31},
	url = {https://papers.nips.cc/paper_files/paper/2018/hash/abdeb6f575ac5c6676b747bca8d09cc2-Abstract.html},
	abstract = {Detecting test samples drawn sufficiently far away from the training distribution statistically or adversarially is a fundamental requirement for deploying a good classifier in many real-world machine learning applications. However, deep neural networks with the softmax classifier are known to produce highly overconfident posterior distributions even for such abnormal samples. In this paper, we propose a simple yet effective method for detecting any abnormal samples, which is applicable to any pre-trained softmax neural classifier. We obtain the class conditional Gaussian distributions with respect to (low- and upper-level) features of the deep models under Gaussian discriminant analysis, which result in a confidence score based on the Mahalanobis distance. While most prior methods have been evaluated for detecting either out-of-distribution or adversarial samples, but not both, the proposed method achieves the state-of-the-art performances for both cases in our experiments. Moreover, we found that our proposed method is more robust in harsh cases, e.g., when the training dataset has noisy labels or small number of samples. Finally, we show that the proposed method enjoys broader usage by applying it to class-incremental learning: whenever out-of-distribution samples are detected, our classification rule can incorporate new classes well without further training deep models.},
	urldate = {2023-10-11},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Lee, Kimin and Lee, Kibok and Lee, Honglak and Shin, Jinwoo},
	year = {2018},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\J8PN6KD6\\Lee et al. - 2018 - A Simple Unified Framework for Detecting Out-of-Di.pdf:application/pdf},
}

@inproceedings{widmann_calibration_2019,
	title = {Calibration tests in multi-class classification: {A} unifying framework},
	volume = {32},
	shorttitle = {Calibration tests in multi-class classification},
	url = {https://proceedings.neurips.cc/paper/2019/hash/1c336b8080f82bcc2cd2499b4c57261d-Abstract.html},
	abstract = {In safety-critical applications a probabilistic model is usually required to be calibrated, i.e., to capture the uncertainty of its predictions accurately. In multi-class classification, calibration of the most confident predictions only is often not sufficient. We propose and study calibration measures for multi-class classification that generalize existing measures such as the expected calibration error, the maximum calibration error, and the maximum mean calibration error. We propose and evaluate empirically different consistent and unbiased estimators for a specific class of measures based on matrix-valued kernels. Importantly, these estimators can be interpreted as test statistics associated with well-defined bounds and approximations of the p-value under the null hypothesis that the model is calibrated, significantly improving the interpretability of calibration measures, which otherwise lack any meaningful unit or scale.},
	urldate = {2023-10-11},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Widmann, David and Lindsten, Fredrik and Zachariah, Dave},
	year = {2019},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\GUWDEWZD\\Widmann et al. - 2019 - Calibration tests in multi-class classification A.pdf:application/pdf},
}

@inproceedings{guo_calibration_2017,
	title = {On {Calibration} of {Modern} {Neural} {Networks}},
	url = {https://proceedings.mlr.press/v70/guo17a.html},
	abstract = {Confidence calibration – the problem of predicting probability estimates representative of the true correctness likelihood – is important for classification models in many applications. We discover that modern neural networks, unlike those from a decade ago, are poorly calibrated. Through extensive experiments, we observe that depth, width, weight decay, and Batch Normalization are important factors influencing calibration. We evaluate the performance of various post-processing calibration methods on state-of-the-art architectures with image and document classification datasets. Our analysis and experiments not only offer insights into neural network learning, but also provide a simple and straightforward recipe for practical settings: on most datasets, temperature scaling – a single-parameter variant of Platt Scaling – is surprisingly effective at calibrating predictions.},
	language = {en},
	urldate = {2023-10-11},
	booktitle = {Proceedings of the 34th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Guo, Chuan and Pleiss, Geoff and Sun, Yu and Weinberger, Kilian Q.},
	month = jul,
	year = {2017},
	note = {ISSN: 2640-3498},
	pages = {1321--1330},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\4CJ7N9I3\\Guo et al. - 2017 - On Calibration of Modern Neural Networks.pdf:application/pdf;Supplementary PDF:C\:\\Users\\yunru\\Zotero\\storage\\W335DJQX\\Guo et al. - 2017 - On Calibration of Modern Neural Networks.pdf:application/pdf},
}

@article{silva_filho_classifier_2023,
	title = {Classifier calibration: a survey on how to assess and improve predicted class probabilities},
	volume = {112},
	issn = {1573-0565},
	shorttitle = {Classifier calibration},
	url = {https://doi.org/10.1007/s10994-023-06336-7},
	doi = {10.1007/s10994-023-06336-7},
	abstract = {This paper provides both an introduction to and a detailed overview of the principles and practice of classifier calibration. A well-calibrated classifier correctly quantifies the level of uncertainty or confidence associated with its instance-wise predictions. This is essential for critical applications, optimal decision making, cost-sensitive classification, and for some types of context change. Calibration research has a rich history which predates the birth of machine learning as an academic field by decades. However, a recent increase in the interest on calibration has led to new methods and the extension from binary to the multiclass setting. The space of options and issues to consider is large, and navigating it requires the right set of concepts and tools. We provide both introductory material and up-to-date technical details of the main concepts and methods, including proper scoring rules and other evaluation metrics, visualisation approaches, a comprehensive account of post-hoc calibration methods for binary and multiclass classification, and several advanced topics.},
	language = {en},
	number = {9},
	urldate = {2023-11-23},
	journal = {Mach Learn},
	author = {Silva Filho, Telmo and Song, Hao and Perello-Nieto, Miquel and Santos-Rodriguez, Raul and Kull, Meelis and Flach, Peter},
	month = sep,
	year = {2023},
	keywords = {Classification, Uncertainty, Calibration, Confidence, Evaluation, Multiclass},
	pages = {3211--3260},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\KT9RX87P\\Silva Filho et al. - 2023 - Classifier calibration a survey on how to assess .pdf:application/pdf},
}

@misc{karpusha_calibrated_2020,
	title = {Calibrated neighborhood aware confidence measure for deep metric learning},
	url = {http://arxiv.org/abs/2006.04935},
	abstract = {Deep metric learning has gained promising improvement in recent years following the success of deep learning. It has been successfully applied to problems in few-shot learning, image retrieval, and open-set classifications. However, measuring the confidence of a deep metric learning model and identifying unreliable predictions is still an open challenge. This paper focuses on defining a calibrated and interpretable confidence metric that closely reflects its classification accuracy. While performing similarity comparison directly in the latent space using the learned distance metric, our approach approximates the distribution of data points for each class using a Gaussian kernel smoothing function. The post-processing calibration algorithm with proposed confidence metric on the held-out validation dataset improves generalization and robustness of state-of-the-art deep metric learning models while provides an interpretable estimation of the confidence. Extensive tests on four popular benchmark datasets (Caltech-UCSD Birds, Stanford Online Product, Stanford Car-196, and In-shop Clothes Retrieval) show consistent improvements even at the presence of distribution shifts in test data related to additional noise or adversarial examples.},
	urldate = {2023-11-24},
	publisher = {arXiv},
	author = {Karpusha, Maryna and Yun, Sunghee and Fehervari, Istvan},
	month = jun,
	year = {2020},
	note = {arXiv:2006.04935 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\RW7D6RBW\\2006.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\TZBIVHN3\\Karpusha et al. - 2020 - Calibrated neighborhood aware confidence measure f.pdf:application/pdf},
}

@misc{sun_out--distribution_2022,
	title = {Out-of-{Distribution} {Detection} with {Deep} {Nearest} {Neighbors}},
	url = {http://arxiv.org/abs/2204.06507},
	abstract = {Out-of-distribution (OOD) detection is a critical task for deploying machine learning models in the open world. Distance-based methods have demonstrated promise, where testing samples are detected as OOD if they are relatively far away from in-distribution (ID) data. However, prior methods impose a strong distributional assumption of the underlying feature space, which may not always hold. In this paper, we explore the efficacy of non-parametric nearest-neighbor distance for OOD detection, which has been largely overlooked in the literature. Unlike prior works, our method does not impose any distributional assumption, hence providing stronger flexibility and generality. We demonstrate the effectiveness of nearest-neighbor-based OOD detection on several benchmarks and establish superior performance. Under the same model trained on ImageNet-1k, our method substantially reduces the false positive rate (FPR@TPR95) by 24.77\% compared to a strong baseline SSD+, which uses a parametric approach Mahalanobis distance in detection. Code is available: https://github.com/deeplearning-wisc/knn-ood.},
	urldate = {2023-11-24},
	publisher = {arXiv},
	author = {Sun, Yiyou and Ming, Yifei and Zhu, Xiaojin and Li, Yixuan},
	month = dec,
	year = {2022},
	note = {arXiv:2204.06507 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\2RUN3M46\\2204.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\FZVP747V\\Sun et al. - 2022 - Out-of-Distribution Detection with Deep Nearest Ne.pdf:application/pdf},
}

@misc{ding_revisiting_2020,
	title = {Revisiting the {Evaluation} of {Uncertainty} {Estimation} and {Its} {Application} to {Explore} {Model} {Complexity}-{Uncertainty} {Trade}-{Off}},
	url = {http://arxiv.org/abs/1903.02050},
	abstract = {Accurately estimating uncertainties in neural network predictions is of great importance in building trusted DNNs-based models, and there is an increasing interest in providing accurate uncertainty estimation on many tasks, such as security cameras and autonomous driving vehicles. In this paper, we focus on the two main use cases of uncertainty estimation, i.e. selective prediction and confidence calibration. We first reveal potential issues of commonly used quality metrics for uncertainty estimation in both use cases, and propose our new metrics to mitigate them. We then apply these new metrics to explore the trade-off between model complexity and uncertainty estimation quality, a critically missing work in the literature. Our empirical experiment results validate the superiority of the proposed metrics, and some interesting trends about the complexity-uncertainty trade-off are observed.},
	urldate = {2023-11-24},
	publisher = {arXiv},
	author = {Ding, Yukun and Liu, Jinglan and Xiong, Jinjun and Shi, Yiyu},
	month = jul,
	year = {2020},
	note = {arXiv:1903.02050 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\YT23IW2C\\1903.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\LF9WXQDI\\Ding et al. - 2020 - Revisiting the Evaluation of Uncertainty Estimatio.pdf:application/pdf},
}

@misc{wei_mitigating_2022,
	title = {Mitigating {Neural} {Network} {Overconfidence} with {Logit} {Normalization}},
	url = {http://arxiv.org/abs/2205.09310},
	abstract = {Detecting out-of-distribution inputs is critical for safe deployment of machine learning models in the real world. However, neural networks are known to suffer from the overconfidence issue, where they produce abnormally high confidence for both in- and out-of-distribution inputs. In this work, we show that this issue can be mitigated through Logit Normalization (LogitNorm) -- a simple fix to the cross-entropy loss -- by enforcing a constant vector norm on the logits in training. Our method is motivated by the analysis that the norm of the logit keeps increasing during training, leading to overconfident output. Our key idea behind LogitNorm is thus to decouple the influence of output's norm during network optimization. Trained with LogitNorm, neural networks produce highly distinguishable confidence scores between in- and out-of-distribution data. Extensive experiments demonstrate the superiority of LogitNorm, reducing the average FPR95 by up to 42.30\% on common benchmarks.},
	urldate = {2023-11-24},
	publisher = {arXiv},
	author = {Wei, Hongxin and Xie, Renchunzi and Cheng, Hao and Feng, Lei and An, Bo and Li, Yixuan},
	month = jun,
	year = {2022},
	note = {arXiv:2205.09310 [cs]},
	keywords = {Computer Science - Machine Learning, I.2.6},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\8BEKCT8B\\2205.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\YDCRS679\\Wei et al. - 2022 - Mitigating Neural Network Overconfidence with Logi.pdf:application/pdf},
}

@misc{noauthor_unified_nodate,
	title = {{UNIFIED} {UNCERTAINTY} {CALIBRATION} {ICLR2024}},
	shorttitle = {{UNIFIED} {UNCERTAINTY} {CALIBRATION} {ICLR2024}},
	url = {https://openreview.net/pdf?id=56jIlazr6a},
	urldate = {2023-11-24},
	file = {pdf.pdf:C\:\\Users\\yunru\\Zotero\\storage\\NNW3HVZC\\pdf.pdf:application/pdf},
}

@inproceedings{vaicenavicius_evaluating_2019,
	title = {Evaluating model calibration in classification},
	url = {https://proceedings.mlr.press/v89/vaicenavicius19a.html},
	abstract = {Probabilistic classifiers output a probability distribution on target classes rather than just a class prediction. Besides providing a clear separation of prediction and decision making, the main advantage of probabilistic models is their ability to represent uncertainty about predictions. In safety-critical applications, it is pivotal for a model to possess an adequate sense of uncertainty, which for probabilistic classifiers translates into outputting probability distributions that are consistent with the empirical frequencies observed from realized outcomes. A classifier with such a property is called calibrated. In this work, we develop a general theoretical calibration evaluation framework grounded in probability theory, and point out subtleties present in model calibration evaluation that lead to refined interpretations of existing evaluation techniques. Lastly, we propose new ways to quantify and visualize miscalibration in probabilistic classification, including novel multidimensional reliability diagrams.},
	language = {en},
	urldate = {2023-11-24},
	booktitle = {Proceedings of the {Twenty}-{Second} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Vaicenavicius, Juozas and Widmann, David and Andersson, Carl and Lindsten, Fredrik and Roll, Jacob and Schön, Thomas},
	month = apr,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {3459--3467},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\MCE44PTM\\Vaicenavicius et al. - 2019 - Evaluating model calibration in classification.pdf:application/pdf},
}

@inproceedings{kumar_verified_2019,
	title = {Verified {Uncertainty} {Calibration}},
	volume = {32},
	url = {https://papers.nips.cc/paper_files/paper/2019/hash/f8c0c968632845cd133308b1a494967f-Abstract.html},
	urldate = {2023-11-30},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Kumar, Ananya and Liang, Percy S and Ma, Tengyu},
	year = {2019},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\G7IH7EUR\\Kumar et al. - 2019 - Verified Uncertainty Calibration.pdf:application/pdf},
}

@inproceedings{pleiss_fairness_2017,
	title = {On {Fairness} and {Calibration}},
	volume = {30},
	url = {https://proceedings.neurips.cc/paper/2017/hash/b8b9c74ac526fffbeb2d39ab038d1cd7-Abstract.html},
	abstract = {The machine learning community has become increasingly concerned with the potential for bias and discrimination in predictive models. This has motivated a growing line of work on what it means for a classification procedure to be "fair." In this paper, we investigate the tension between minimizing error disparity across different population groups while maintaining calibrated probability estimates. We show that calibration is compatible only with a single error constraint (i.e. equal false-negatives rates across groups), and show that any algorithm that satisfies this relaxation is no better than randomizing a percentage of predictions for an existing classifier. These unsettling findings, which extend and generalize existing results, are empirically confirmed on several datasets.},
	urldate = {2023-12-11},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Pleiss, Geoff and Raghavan, Manish and Wu, Felix and Kleinberg, Jon and Weinberger, Kilian Q},
	year = {2017},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\36FPXFXT\\Pleiss et al. - 2017 - On Fairness and Calibration.pdf:application/pdf},
}

@inproceedings{joo_being_2020,
	title = {Being {Bayesian} about {Categorical} {Probability}},
	url = {https://proceedings.mlr.press/v119/joo20a.html},
	abstract = {Neural networks utilize the softmax as a building block in classification tasks, which contains an overconfidence problem and lacks an uncertainty representation ability. As a Bayesian alternative to the softmax, we consider a random variable of a categorical probability over class labels. In this framework, the prior distribution explicitly models the presumed noise inherent in the observed label, which provides consistent gains in generalization performance in multiple challenging tasks. The proposed method inherits advantages of Bayesian approaches that achieve better uncertainty estimation and model calibration. Our method can be implemented as a plug-and-play loss function with negligible computational overhead compared to the softmax with the cross-entropy loss function.},
	language = {en},
	urldate = {2023-12-15},
	booktitle = {Proceedings of the 37th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Joo, Taejong and Chung, Uijung and Seo, Min-Gwan},
	month = nov,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {4950--4961},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\ZYM5V8CG\\Joo et al. - 2020 - Being Bayesian about Categorical Probability.pdf:application/pdf},
}

@misc{muller_when_2020,
	title = {When {Does} {Label} {Smoothing} {Help}?},
	url = {http://arxiv.org/abs/1906.02629},
	urldate = {2023-12-18},
	publisher = {arXiv},
	author = {Müller, Rafael and Kornblith, Simon and Hinton, Geoffrey},
	month = jun,
	year = {2020},
	note = {arXiv:1906.02629 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\H485JDEW\\1906.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\8MPECD9H\\Müller et al. - 2020 - When Does Label Smoothing Help.pdf:application/pdf},
}

@misc{matsubara_tce_2023,
	title = {{TCE}: {A} {Test}-{Based} {Approach} to {Measuring} {Calibration} {Error}},
	shorttitle = {{TCE}},
	url = {http://arxiv.org/abs/2306.14343},
	abstract = {This paper proposes a new metric to measure the calibration error of probabilistic binary classifiers, called test-based calibration error (TCE). TCE incorporates a novel loss function based on a statistical test to examine the extent to which model predictions differ from probabilities estimated from data. It offers (i) a clear interpretation, (ii) a consistent scale that is unaffected by class imbalance, and (iii) an enhanced visual representation with repect to the standard reliability diagram. In addition, we introduce an optimality criterion for the binning procedure of calibration error metrics based on a minimal estimation error of the empirical probabilities. We provide a novel computational algorithm for optimal bins under bin-size constraints. We demonstrate properties of TCE through a range of experiments, including multiple real-world imbalanced datasets and ImageNet 1000.},
	urldate = {2023-12-18},
	publisher = {arXiv},
	author = {Matsubara, Takuo and Tax, Niek and Mudd, Richard and Guy, Ido},
	month = jun,
	year = {2023},
	note = {arXiv:2306.14343 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\QVWAWA6Y\\2306.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\DAJ8G72M\\Matsubara et al. - 2023 - TCE A Test-Based Approach to Measuring Calibratio.pdf:application/pdf},
}

@inproceedings{li_regularization_2020,
	title = {Regularization via {Structural} {Label} {Smoothing}},
	url = {https://proceedings.mlr.press/v108/li20e.html},
	abstract = {Regularization is an effective way to promote the generalization performance of machine learning models. In this paper, we focus on label smoothing, a form of output distribution regularization that prevents overfitting of a neural network by softening the ground-truth labels in the training data in an attempt to penalize overconfident outputs. Existing approaches typically use cross-validation to impose this smoothing, which is uniform across all training data. In this paper, we show that such label smoothing imposes a quantifiable bias in the Bayes error rate of the training data, with regions of the feature space with high overlap and low marginal likelihood having a lower bias and regions of low overlap and high marginal likelihood having a higher bias. These theoretical results motivate a simple objective function for data-dependent smoothing to mitigate the potential negative consequences of the operation while maintaining its desirable properties as a regularizer. We call this approach Structural Label Smoothing (SLS). We implement SLS and empirically validate on synthetic, Higgs, SVHN, CIFAR-10, and CIFAR-100 datasets. The results confirm our theoretical insights and demonstrate the effectiveness of the proposed method in comparison to traditional label smoothing.},
	language = {en},
	urldate = {2023-12-19},
	booktitle = {Proceedings of the {Twenty} {Third} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Li, Weizhi and Dasarathy, Gautam and Berisha, Visar},
	month = jun,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {1453--1463},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\IVWXZFBB\\Li et al. - 2020 - Regularization via Structural Label Smoothing.pdf:application/pdf},
}

@article{lienen_label_2021,
	title = {From {Label} {Smoothing} to {Label} {Relaxation}},
	volume = {35},
	copyright = {Copyright (c) 2021 Association for the Advancement of Artificial Intelligence},
	issn = {2374-3468},
	url = {https://ojs.aaai.org/index.php/AAAI/article/view/17041},
	doi = {10.1609/aaai.v35i10.17041},
	abstract = {Regularization of (deep) learning models can be realized at the model, loss, or data level. As a technique somewhere in-between loss and data, label smoothing turns deterministic class labels into probability distributions, for example by uniformly distributing a certain part of the probability mass over all classes. A predictive model is then trained on these distributions as targets, using cross-entropy as loss function. While this method has shown improved performance compared to non-smoothed cross-entropy, we argue that the use of a smoothed though still precise probability distribution as a target can be questioned from a theoretical perspective. As an alternative, we propose a generalized technique called label relaxation, in which the target is a set of probabilities represented in terms of an upper probability distribution. This leads to a genuine relaxation of the target instead of a distortion, thereby reducing the risk of incorporating an undesirable bias in the learning process. Methodically, label relaxation leads to the minimization of a novel type of loss function, for which we propose a suitable closed-form expression for model optimization. The effectiveness of the approach is demonstrated in an empirical study on image data.},
	language = {en},
	number = {10},
	urldate = {2023-12-20},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
	author = {Lienen, Julian and Hüllermeier, Eyke},
	month = may,
	year = {2021},
	note = {Number: 10},
	keywords = {(Deep) Neural Network Learning Theory},
	pages = {8583--8591},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\YG6C8ZD2\\Lienen and Hüllermeier - 2021 - From Label Smoothing to Label Relaxation.pdf:application/pdf},
}

@misc{hinton_distilling_2015,
	title = {Distilling the {Knowledge} in a {Neural} {Network}},
	url = {http://arxiv.org/abs/1503.02531},
	abstract = {A very simple way to improve the performance of almost any machine learning algorithm is to train many different models on the same data and then to average their predictions. Unfortunately, making predictions using a whole ensemble of models is cumbersome and may be too computationally expensive to allow deployment to a large number of users, especially if the individual models are large neural nets. Caruana and his collaborators have shown that it is possible to compress the knowledge in an ensemble into a single model which is much easier to deploy and we develop this approach further using a different compression technique. We achieve some surprising results on MNIST and we show that we can significantly improve the acoustic model of a heavily used commercial system by distilling the knowledge in an ensemble of models into a single model. We also introduce a new type of ensemble composed of one or more full models and many specialist models which learn to distinguish fine-grained classes that the full models confuse. Unlike a mixture of experts, these specialist models can be trained rapidly and in parallel.},
	urldate = {2023-12-21},
	publisher = {arXiv},
	author = {Hinton, Geoffrey and Vinyals, Oriol and Dean, Jeff},
	month = mar,
	year = {2015},
	note = {arXiv:1503.02531 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Neural and Evolutionary Computing},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\E45PJQZU\\1503.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\KXXD9RHI\\Hinton et al. - 2015 - Distilling the Knowledge in a Neural Network.pdf:application/pdf},
}

@inproceedings{zhang_self-distillation_2020,
	title = {Self-{Distillation} as {Instance}-{Specific} {Label} {Smoothing}},
	volume = {33},
	url = {https://proceedings.neurips.cc/paper/2020/hash/1731592aca5fb4d789c4119c65c10b4b-Abstract.html},
	abstract = {It has been recently demonstrated that multi-generational self-distillation can improve generalization. Despite this intriguing observation, reasons for the enhancement remain poorly understood. In this paper, we first demonstrate experimentally that the improved performance of multi-generational self-distillation is in part associated with the increasing diversity in teacher predictions. With this in mind, we offer a new interpretation for teacher-student training as amortized MAP estimation, such that teacher predictions enable instance-specific regularization. Our framework allows us to theoretically relate self-distillation to label smoothing, a commonly used technique that regularizes predictive uncertainty, and suggests the importance of predictive diversity in addition to predictive uncertainty. We present experimental results using multiple datasets and neural network architectures that, overall, demonstrate the utility of predictive diversity. Finally, we propose a novel instance-specific label smoothing technique that promotes predictive diversity without the need for a separately trained teacher model. We provide an empirical evaluation of the proposed method, which, we find, often outperforms classical label smoothing.},
	urldate = {2023-12-22},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Zhang, Zhilu and Sabuncu, Mert},
	year = {2020},
	pages = {2184--2195},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\2DYRBGYK\\Zhang and Sabuncu - 2020 - Self-Distillation as Instance-Specific Label Smoot.pdf:application/pdf},
}

@inproceedings{kull_beyond_2019,
	title = {Beyond temperature scaling: {Obtaining} well-calibrated multi-class probabilities with {Dirichlet} calibration},
	volume = {32},
	shorttitle = {Beyond temperature scaling},
	urldate = {2024-01-03},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems} (NeurIPS)},
	author = {Kull, Meelis and Perello Nieto, Miquel and Kängsepp, Markus and Silva Filho, Telmo and Song, Hao and Flach, Peter},
	year = {2019},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\VD8SYI95\\Kull et al. - 2019 - Beyond temperature scaling Obtaining well-calibra.pdf:application/pdf},
}

@inproceedings{zhang_when_2022,
	title = {When and {How} {Mixup} {Improves} {Calibration}},
	url = {https://proceedings.mlr.press/v162/zhang22f.html},
	abstract = {In many machine learning applications, it is important for the model to provide confidence scores that accurately capture its prediction uncertainty. Although modern learning methods have achieved great success in predictive accuracy, generating calibrated confidence scores remains a major challenge. Mixup, a popular yet simple data augmentation technique based on taking convex combinations of pairs of training examples, has been empirically found to significantly improve confidence calibration across diverse applications. However, when and how Mixup helps calibration is still a mystery. In this paper, we theoretically prove that Mixup improves calibration in high-dimensional settings by investigating natural statistical models. Interestingly, the calibration benefit of Mixup increases as the model capacity increases. We support our theories with experiments on common architectures and datasets. In addition, we study how Mixup improves calibration in semi-supervised learning. While incorporating unlabeled data can sometimes make the model less calibrated, adding Mixup training mitigates this issue and provably improves calibration. Our analysis provides new insights and a framework to understand Mixup and calibration.},
	language = {en},
	urldate = {2024-01-05},
	booktitle = {Proceedings of the 39th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Zhang, Linjun and Deng, Zhun and Kawaguchi, Kenji and Zou, James},
	month = jun,
	year = {2022},
	note = {ISSN: 2640-3498},
	pages = {26135--26160},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\6243E6EK\\Zhang et al. - 2022 - When and How Mixup Improves Calibration.pdf:application/pdf},
}

@inproceedings{kumar_trainable_2018,
	title = {Trainable {Calibration} {Measures} for {Neural} {Networks} from {Kernel} {Mean} {Embeddings}},
	url = {https://proceedings.mlr.press/v80/kumar18a.html},
	abstract = {Modern neural networks have recently been found to be poorly calibrated, primarily in the direction of over-confidence. Methods like entropy penalty and temperature smoothing improve calibration by clamping confidence, but in doing so compromise the many legitimately confident predictions. We propose a more principled fix that minimizes an explicit calibration error during training. We present MMCE, a RKHS kernel based measure of calibration that is efficiently trainable alongside the negative likelihood loss without careful hyper-parameter tuning. Theoretically too, MMCE is a sound measure of calibration that is minimized at perfect calibration, and whose finite sample estimates are consistent and enjoy fast convergence rates. Extensive experiments on several network architectures demonstrate that MMCE is a fast, stable, and accurate method to minimize calibration error while maximally preserving the number of high confidence predictions.},
	language = {en},
	urldate = {2024-01-08},
	booktitle = {Proceedings of the 35th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Kumar, Aviral and Sarawagi, Sunita and Jain, Ujjwal},
	month = jul,
	year = {2018},
	note = {ISSN: 2640-3498},
	pages = {2805--2814},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\N2U9U479\\Kumar et al. - 2018 - Trainable Calibration Measures for Neural Networks.pdf:application/pdf},
}

@article{boken_appropriateness_2021,
	title = {On the appropriateness of {Platt} scaling in classifier calibration},
	volume = {95},
	issn = {0306-4379},
	url = {https://www.sciencedirect.com/science/article/pii/S0306437920301083},
	doi = {10.1016/j.is.2020.101641},
	abstract = {Many applications using data mining and machine learning techniques require posterior probability estimates besides often highly accurate predictions. Classifier calibration is a separate branch of machine learning that aims at transforming classifier predictions into posterior class probabilities and thus are useful additional extensions in the respective applications. Among the existing state-of-the-art classifier calibration techniques, Platt scaling (sometimes also called sigmoid or logistic calibration) actually is the only parametric one while almost all of its competing methods do not rely on parametric assumptions. Platt scaling is controversially discussed in the classifier calibration literature, despite good empirical results reported in many domains there are many authors criticizing it. Interestingly, none of these criticisms properly deal with the underlying parametric assumptions. Instead, even incorrect statements exist. Thus, the first contribution of this work is to review such criticism and to present a proof of the true parametric assumptions. In fact, these are more general and valid for different probability distributions, which is an immediate consequence. Next, the relationship between Platt scaling and a different, relatively new classifier calibration technique called beta calibration is analyzed and it is shown that these two are actually equivalent: Their only difference lies in the characteristics of the classifier whose predictions are calibrated. Thus, the proven validity of Platt scaling additionally translates directly into a proven optimality of beta calibration. Furthermore, evaluating classifier calibration techniques is a highly non-trivial problem as the true posteriors cannot be used as a reference. Hence, the existing evaluation metrics are reviewed as well because there exist relatively popular evaluation criteria that should not be used at all. Finally, the theoretical findings are supported by a simulation study.},
	urldate = {2024-01-08},
	journal = {Information Systems},
	author = {Böken, Björn},
	month = jan,
	year = {2021},
	keywords = {Classifier calibration, Logistic regression, Platt scaling, Posterior probability, Proper scoring rules},
	pages = {101641},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\6WBQYLL5\\Böken - 2021 - On the appropriateness of Platt scaling in classifier calibration.pdf:application/pdf;ScienceDirect Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\KJ8T5RKN\\S0306437920301083.html:text/html},
}

@inproceedings{wenger_non-parametric_2020,
	title = {Non-{Parametric} {Calibration} for {Classification}},
	url = {https://proceedings.mlr.press/v108/wenger20a.html},
	abstract = {Many applications of classification methods not only require high accuracy but also reliable estimation of predictive uncertainty. However, while many current classification frameworks, in particular deep neural networks, achieve high accuracy, they tend to incorrectly estimate uncertainty. In this paper, we propose a method that adjusts the confidence estimates of a general classifier such that they approach the probability of classifying correctly. In contrast to existing approaches, our calibration method employs a non-parametric representation using a latent Gaussian process, and is specifically designed for multi-class classification. It can be applied to any classifier that outputs confidence estimates and is not limited to neural networks. We also provide a theoretical analysis regarding the over- and underconfidence of a classifier and its relationship to calibration, as well as an empirical outlook for calibrated active learning. In experiments we show the universally strong performance of our method across different classifiers and benchmark data sets, in particular for state-of-the art neural network architectures.},
	language = {en},
	urldate = {2024-01-09},
	booktitle = {Proceedings of the {Twenty} {Third} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Wenger, Jonathan and Kjellström, Hedvig and Triebel), Rudolph},
	month = jun,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {178--190},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\FTK69CYX\\Wenger et al. - 2020 - Non-Parametric Calibration for Classification.pdf:application/pdf},
}

@misc{pearce_understanding_2021,
	title = {Understanding {Softmax} {Confidence} and {Uncertainty}},
	url = {http://arxiv.org/abs/2106.04972},
	abstract = {It is often remarked that neural networks fail to increase their uncertainty when predicting on data far from the training distribution. Yet naively using softmax confidence as a proxy for uncertainty achieves modest success in tasks exclusively testing for this, e.g., out-of-distribution (OOD) detection. This paper investigates this contradiction, identifying two implicit biases that do encourage softmax confidence to correlate with epistemic uncertainty: 1) Approximately optimal decision boundary structure, and 2) Filtering effects of deep networks. It describes why low-dimensional intuitions about softmax confidence are misleading. Diagnostic experiments quantify reasons softmax confidence can fail, finding that extrapolations are less to blame than overlap between training and OOD data in final-layer representations. Pre-trained/fine-tuned networks reduce this overlap.},
	urldate = {2024-01-16},
	publisher = {arXiv},
	author = {Pearce, Tim and Brintrup, Alexandra and Zhu, Jun},
	month = jun,
	year = {2021},
	note = {arXiv:2106.04972 [cs, stat]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\STBVMYVY\\2106.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\SKST2BT4\\Pearce et al. - 2021 - Understanding Softmax Confidence and Uncertainty.pdf:application/pdf},
}

@article{noauthor_hiding_2023,
	title = {Hiding in a {Plain} {Sight}: {Out}-of-{Distribution} {Detection} from {Logit} {Space} {Embeddings}},
	shorttitle = {Hiding in a {Plain} {Sight}},
	url = {https://openreview.net/forum?id=2nGufrROnC},
	abstract = {Although deep learning (DL) models have revolutionized the field of machine learning (ML), these classification models cannot easily distinguish the in-distribution (ID) versus the out-of-distribution (OOD) data at the test phase. This paper analyzes the landscape of ID and OOD data embeddings and demonstrates that OOD data is always embedded toward the center in the logit space. Furthermore, IDs data are embedded far from the center towards the positive regions of the logit space, thus ensuring minimal overlap between ID and OOD embeddings. Based on these observations, we propose to make the classification model sensitive to the OOD data by incorporating the configuration of the logit space into the predictive response. Hence, we estimate the distribution of the ID logits by utilizing a density estimator over the training data logits. Our proposed approach is data and architecture-agnostic and could be easily incorporated with a trained model without exposure to OOD data. We ran experiments on the popular image datasets and obtained state-of-the-art performance and an improvement of up to 10\${\textbackslash}\%\$ on AUCROC on the Google genome dataset.},
	language = {en},
	urldate = {2024-01-16},
	journal = {Transactions on Machine Learning Research},
	month = sep,
	year = {2023},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\5Y4SUC2K\\2023 - Hiding in a Plain Sight Out-of-Distribution Detec.pdf:application/pdf},
}

@inproceedings{wan_rethinking_2018,
	address = {Salt Lake City, UT},
	title = {Rethinking {Feature} {Distribution} for {Loss} {Functions} in {Image} {Classification}},
	isbn = {978-1-5386-6420-9},
	url = {https://ieeexplore.ieee.org/document/8579048/},
	doi = {10.1109/CVPR.2018.00950},
	abstract = {We propose a large-margin Gaussian Mixture (L-GM) loss for deep neural networks in classiﬁcation tasks. Different from the softmax cross-entropy loss, our proposal is established on the assumption that the deep features of the training set follow a Gaussian Mixture distribution. By involving a classiﬁcation margin and a likelihood regularization, the L-GM loss facilitates both a high classiﬁcation performance and an accurate modeling of the training feature distribution. As such, the L-GM loss is superior to the softmax loss and its major variants in the sense that besides classiﬁcation, it can be readily used to distinguish abnormal inputs, such as the adversarial examples, based on their features’ likelihood to the training feature distribution. Extensive experiments on various recognition benchmarks like MNIST, CIFAR, ImageNet and LFW, as well as on adversarial examples demonstrate the effectiveness of our proposal.},
	language = {en},
	urldate = {2024-01-16},
	booktitle = {2018 {IEEE}/{CVF} {Conference} on {Computer} {Vision} and {Pattern} {Recognition}},
	publisher = {IEEE},
	author = {Wan, Weitao and Zhong, Yuanyi and Li, Tianpeng and Chen, Jiansheng},
	month = jun,
	year = {2018},
	pages = {9117--9126},
	file = {Wan et al. - 2018 - Rethinking Feature Distribution for Loss Functions.pdf:C\:\\Users\\yunru\\Zotero\\storage\\V3IVM4GJ\\Wan et al. - 2018 - Rethinking Feature Distribution for Loss Functions.pdf:application/pdf},
}

@inproceedings{marx_calibration_2023,
	title = {Calibration by {Distribution} {Matching}: {Trainable} {Kernel} {Calibration} {Metrics}},
	shorttitle = {Calibration by {Distribution} {Matching}},
	url = {https://openreview.net/forum?id=IhxD94i5ra},
	abstract = {Calibration ensures that probabilistic forecasts meaningfully capture uncertainty by requiring that predicted probabilities align with empirical frequencies. However, many existing calibration methods are specialized for post-hoc recalibration, which can worsen the sharpness of forecasts. Drawing on the insight that calibration can be viewed as a distribution matching task, we introduce kernel-based calibration metrics that unify and generalize popular forms of calibration for both classification and regression. These metrics admit differentiable sample estimates, making it easy to incorporate a calibration objective into empirical risk minimization. Furthermore, we provide intuitive mechanisms to tailor calibration metrics to a decision task, and enforce accurate loss estimation and no regret decisions. Our empirical evaluation demonstrates that employing these metrics as regularizers enhances calibration, sharpness, and decision-making across a range of regression and classification tasks, outperforming methods relying solely on post-hoc recalibration.},
	language = {en},
	urldate = {2024-01-18},
	author = {Marx, Charles Thomas and Zalouk, Sofian and Ermon, Stefano},
	month = nov,
	year = {2023},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\YTLEN8CT\\Marx et al. - 2023 - Calibration by Distribution Matching Trainable Ke.pdf:application/pdf},
}

@misc{popordanoska_consistent_2022,
	title = {A {Consistent} and {Differentiable} {Lp} {Canonical} {Calibration} {Error} {Estimator}},
	url = {http://arxiv.org/abs/2210.07810},
	abstract = {Calibrated probabilistic classiﬁers are models whose predicted probabilities can directly be interpreted as uncertainty estimates. It has been shown recently that deep neural networks are poorly calibrated and tend to output overconﬁdent predictions. As a remedy, we propose a low-bias, trainable calibration error estimator based on Dirichlet kernel density estimates, which asymptotically converges to the true Lp calibration error. This novel estimator enables us to tackle the strongest notion of multiclass calibration, called canonical (or distribution) calibration, while other common calibration methods are tractable only for top-label and marginal calibration. The computational complexity of our estimator is O(n2), the convergence rate is O(n−1/2), and it is unbiased up to O(n−2), achieved by a geometric series debiasing scheme. In practice, this means that the estimator can be applied to small subsets of data, enabling efﬁcient estimation and mini-batch updates. The proposed method has a natural choice of kernel, and can be used to generate consistent estimates of other quantities based on conditional expectation, such as the sharpness of a probabilistic classiﬁer. Empirical results validate the correctness of our estimator, and demonstrate its utility in canonical calibration error estimation and calibration error regularized risk minimization.},
	language = {en},
	urldate = {2024-01-19},
	publisher = {arXiv},
	author = {Popordanoska, Teodora and Sayer, Raphael and Blaschko, Matthew B.},
	month = oct,
	year = {2022},
	note = {arXiv:2210.07810 [cs, stat]},
	keywords = {Statistics - Machine Learning, Computer Science - Computer Vision and Pattern Recognition},
	file = {Popordanoska et al. - 2022 - A Consistent and Differentiable Lp Canonical Calib.pdf:C\:\\Users\\yunru\\Zotero\\storage\\XK5VYN9B\\Popordanoska et al. - 2022 - A Consistent and Differentiable Lp Canonical Calib.pdf:application/pdf},
}

@article{ciosek_conservative_2020,
	title = {{CONSERVATIVE} {UNCERTAINTY} {ESTIMATION} {BY} {FITTING} {PRIOR} {NETWORKS}},
	abstract = {Obtaining high-quality uncertainty estimates is essential for many applications of deep neural networks. In this paper, we theoretically justify a scheme for estimating uncertainties, based on sampling from a prior distribution. Crucially, the uncertainty estimates are shown to be conservative in the sense that they never underestimate a posterior uncertainty obtained by a hypothetical Bayesian algorithm. We also show concentration, implying that the uncertainty estimates converge to zero as we get more data. Uncertainty estimates obtained from random priors can be adapted to any deep network architecture and trained using standard supervised learning pipelines. We provide experimental evaluation of random priors on calibration and out-of-distribution detection on typical computer vision tasks, demonstrating that they outperform deep ensembles in practice.},
	language = {en},
	author = {Ciosek, Kamil and Fortuin, Vincent and Tomioka, Ryota and Hofmann, Katja and Turner, Richard},
	year = {2020},
	file = {Ciosek et al. - 2020 - CONSERVATIVE UNCERTAINTY ESTIMATION BY FITTING PRI.pdf:C\:\\Users\\yunru\\Zotero\\storage\\P7VD7GP5\\Ciosek et al. - 2020 - CONSERVATIVE UNCERTAINTY ESTIMATION BY FITTING PRI.pdf:application/pdf},
}

@inproceedings{lakshminarayanan_simple_2017,
	title = {Simple and {Scalable} {Predictive} {Uncertainty} {Estimation} using {Deep} {Ensembles}},
	volume = {30},
	url = {https://proceedings.neurips.cc/paper_files/paper/2017/hash/9ef2ed4b7fd2c810847ffa5fa85bce38-Abstract.html},
	abstract = {Deep neural networks (NNs) are powerful black box predictors that have recently achieved impressive performance on a wide spectrum of tasks. Quantifying predictive uncertainty in NNs is a challenging and yet unsolved problem.  Bayesian NNs, which learn a distribution over weights, are currently the state-of-the-art for estimating predictive uncertainty; however these require significant modifications to the training procedure and are computationally expensive compared to standard (non-Bayesian)  NNs.  We propose an alternative to Bayesian NNs that is simple to implement, readily parallelizable, requires very little hyperparameter tuning, and yields high quality predictive uncertainty estimates. Through a series of experiments on classification and regression benchmarks, we demonstrate that our method produces well-calibrated uncertainty estimates which are as good or better than approximate Bayesian NNs. To assess robustness to dataset shift, we evaluate the predictive uncertainty on test examples from known and unknown distributions, and show that our method is able to express higher uncertainty on out-of-distribution examples. We demonstrate the scalability of our method by evaluating predictive uncertainty estimates on ImageNet.},
	urldate = {2024-02-01},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Lakshminarayanan, Balaji and Pritzel, Alexander and Blundell, Charles},
	year = {2017},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\CVEMTSRP\\Lakshminarayanan et al. - 2017 - Simple and Scalable Predictive Uncertainty Estimat.pdf:application/pdf},
}

@inproceedings{zhao_calibrating_2021,
	title = {Calibrating {Predictions} to {Decisions}: {A} {Novel} {Approach} to {Multi}-{Class} {Calibration}},
	volume = {34},
	shorttitle = {Calibrating {Predictions} to {Decisions}},
	url = {https://proceedings.neurips.cc/paper/2021/hash/bbc92a647199b832ec90d7cf57074e9e-Abstract.html},
	abstract = {When facing uncertainty, decision-makers want predictions they can trust. A machine learning provider can convey confidence to decision-makers by guaranteeing their predictions are distribution calibrated--- amongst the inputs that receive a predicted vector of class probabilities q, the actual distribution over classes is given by q. For multi-class prediction problems, however, directly optimizing predictions under distribution calibration tends to be infeasible, requiring sample complexity that grows exponentially in the number of classes C. In this work, we introduce a new notion---decision calibration---that requires the predicted distribution and true distribution over classes to be indistinguishable'' to downstream decision-makers. This perspective gives a new characterization of distribution calibration: a predictor is distribution calibrated if and only if it is decision calibrated with respect to all decision-makers. Our main result shows that under a mild restriction, unlike distribution calibration, decision calibration is actually feasible. We design a recalibration algorithm that provably achieves decision calibration efficiently, provided that the decision-makers have a bounded number of actions (e.g., polynomial in C). We validate our recalibration algorithm empirically: compared to existing methods, decision calibration improves decision-making on skin lesion and ImageNet classification with modern neural network predictors.},
	urldate = {2024-03-20},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Zhao, Shengjia and Kim, Michael and Sahoo, Roshni and Ma, Tengyu and Ermon, Stefano},
	year = {2021},
	pages = {22313--22324},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\BGFTKQWZ\\Zhao et al. - 2021 - Calibrating Predictions to Decisions A Novel Appr.pdf:application/pdf},
}

@misc{kuhn_semantic_2023,
	title = {Semantic {Uncertainty}: {Linguistic} {Invariances} for {Uncertainty} {Estimation} in {Natural} {Language} {Generation}},
	shorttitle = {Semantic {Uncertainty}},
	url = {http://arxiv.org/abs/2302.09664},
	abstract = {We introduce a method to measure uncertainty in large language models. For tasks like question answering, it is essential to know when we can trust the natural language outputs of foundation models. We show that measuring uncertainty in natural language is challenging because of "semantic equivalence" -- different sentences can mean the same thing. To overcome these challenges we introduce semantic entropy -- an entropy which incorporates linguistic invariances created by shared meanings. Our method is unsupervised, uses only a single model, and requires no modifications to off-the-shelf language models. In comprehensive ablation studies we show that the semantic entropy is more predictive of model accuracy on question answering data sets than comparable baselines.},
	urldate = {2024-03-21},
	publisher = {arXiv},
	author = {Kuhn, Lorenz and Gal, Yarin and Farquhar, Sebastian},
	month = apr,
	year = {2023},
	note = {arXiv:2302.09664 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Computer Science - Computation and Language},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\TF9JRP4C\\2302.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\ZY6MBEXA\\Kuhn et al. - 2023 - Semantic Uncertainty Linguistic Invariances for U.pdf:application/pdf},
}

@misc{gupta_calibration_2021,
	title = {Calibration of {Neural} {Networks} using {Splines}},
	url = {http://arxiv.org/abs/2006.12800},
	abstract = {Calibrating neural networks is of utmost importance when employing them in safety-critical applications where the downstream decision making depends on the predicted probabilities. Measuring calibration error amounts to comparing two empirical distributions. In this work, we introduce a binning-free calibration measure inspired by the classical Kolmogorov-Smirnov (KS) statistical test in which the main idea is to compare the respective cumulative probability distributions. From this, by approximating the empirical cumulative distribution using a differentiable function via splines, we obtain a recalibration function, which maps the network outputs to actual (calibrated) class assignment probabilities. The spine-fitting is performed using a held-out calibration set and the obtained recalibration function is evaluated on an unseen test set. We tested our method against existing calibration approaches on various image classification datasets and our spline-based recalibration approach consistently outperforms existing methods on KS error as well as other commonly used calibration measures. Our Code is available at https://github.com/kartikgupta-at-anu/spline-calibration.},
	urldate = {2024-04-02},
	publisher = {arXiv},
	author = {Gupta, Kartik and Rahimi, Amir and Ajanthan, Thalaiyasingam and Mensink, Thomas and Sminchisescu, Cristian and Hartley, Richard},
	month = dec,
	year = {2021},
	note = {arXiv:2006.12800 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\BPBHPXY7\\2006.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\J6VWEB5B\\Gupta et al. - 2021 - Calibration of Neural Networks using Splines.pdf:application/pdf},
}

@article{vaz_quantication_nodate,
	title = {Quantiﬁcation {Under} {Prior} {Probability} {Shift}: the {Ratio} {Estimator} and its {Extensions}},
	abstract = {The quantiﬁcation problem consists of determining the prevalence of a given label in a target population. However, one often has access to the labels in a sample from the training population but not in the target population. A common assumption in this situation is that of prior probability shift, that is, once the labels are known, the distribution of the features is the same in the training and target populations. In this paper, we derive a new lower bound for the risk of the quantiﬁcation problem under the prior shift assumption. Complementing this lower bound, we present a new approximately minimax class of estimators, ratio estimators, which generalize several previous proposals in the literature. Using a weaker version of the prior shift assumption, which can be tested, we show that ratio estimators can be used to build conﬁdence intervals for the quantiﬁcation problem. We also extend the ratio estimator so that it can: (i) incorporate labels from the target population, when they are available and (ii) estimate how the prevalence of positive labels varies according to a function of certain covariates.},
	language = {en},
	author = {Vaz, Afonso Fernandes and Izbicki, Rafael and Stern, Rafael Bassi},
	file = {Vaz et al. - Quantiﬁcation Under Prior Probability Shift the R.pdf:C\:\\Users\\yunru\\Zotero\\storage\\JIHJ4GGY\\Vaz et al. - Quantiﬁcation Under Prior Probability Shift the R.pdf:application/pdf},
}

@misc{noauthor_maximum_nodate,
	title = {Maximum likelihood estimation {\textbar} {Theory}, assumptions, properties},
	url = {https://www.statlect.com/fundamentals-of-statistics/maximum-likelihood},
	urldate = {2024-04-04},
	file = {Maximum likelihood estimation | Theory, assumptions, properties:C\:\\Users\\yunru\\Zotero\\storage\\8JVAQTKX\\maximum-likelihood.html:text/html},
}

@misc{blasiok_unifying_2023,
	title = {A {Unifying} {Theory} of {Distance} from {Calibration}},
	url = {http://arxiv.org/abs/2211.16886},
	abstract = {We study the fundamental question of how to define and measure the distance from calibration for probabilistic predictors. While the notion of perfect calibration is well-understood, there is no consensus on how to quantify the distance from perfect calibration. Numerous calibration measures have been proposed in the literature, but it is unclear how they compare to each other, and many popular measures such as Expected Calibration Error (ECE) fail to satisfy basic properties like continuity. We present a rigorous framework for analyzing calibration measures, inspired by the literature on property testing. We propose a ground-truth notion of distance from calibration: the \${\textbackslash}ell\_1\$ distance to the nearest perfectly calibrated predictor. We define a consistent calibration measure as one that is polynomially related to this distance. Applying our framework, we identify three calibration measures that are consistent and can be estimated efficiently: smooth calibration, interval calibration, and Laplace kernel calibration. The former two give quadratic approximations to the ground truth distance, which we show is information-theoretically optimal in a natural model for measuring calibration which we term the prediction-only access model. Our work thus establishes fundamental lower and upper bounds on measuring the distance to calibration, and also provides theoretical justification for preferring certain metrics (like Laplace kernel calibration) in practice.},
	urldate = {2024-04-08},
	publisher = {arXiv},
	author = {Błasiok, Jarosław and Gopalan, Parikshit and Hu, Lunjia and Nakkiran, Preetum},
	month = mar,
	year = {2023},
	note = {arXiv:2211.16886 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\2XH23SZZ\\2211.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\6CKW5KNC\\Błasiok et al. - 2023 - A Unifying Theory of Distance from Calibration.pdf:application/pdf},
}

@misc{blasiok_smooth_2023,
	title = {Smooth {ECE}: {Principled} {Reliability} {Diagrams} via {Kernel} {Smoothing}},
	shorttitle = {Smooth {ECE}},
	url = {http://arxiv.org/abs/2309.12236},
	abstract = {Calibration measures and reliability diagrams are two fundamental tools for measuring and interpreting the calibration of probabilistic predictors. Calibration measures quantify the degree of miscalibration, and reliability diagrams visualize the structure of this miscalibration. However, the most common constructions of reliability diagrams and calibration measures -- binning and ECE -- both suffer from well-known flaws (e.g. discontinuity). We show that a simple modification fixes both constructions: first smooth the observations using an RBF kernel, then compute the Expected Calibration Error (ECE) of this smoothed function. We prove that with a careful choice of bandwidth, this method yields a calibration measure that is well-behaved in the sense of (B\{{\textbackslash}l\}asiok, Gopalan, Hu, and Nakkiran 2023a) -- a consistent calibration measure. We call this measure the SmoothECE. Moreover, the reliability diagram obtained from this smoothed function visually encodes the SmoothECE, just as binned reliability diagrams encode the BinnedECE. We also provide a Python package with simple, hyperparameter-free methods for measuring and plotting calibration: `pip install relplot{\textbackslash}`.},
	urldate = {2024-04-08},
	publisher = {arXiv},
	author = {Błasiok, Jarosław and Nakkiran, Preetum},
	month = sep,
	year = {2023},
	note = {arXiv:2309.12236 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\FFC4DK64\\2309.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\CH9Y2946\\Błasiok and Nakkiran - 2023 - Smooth ECE Principled Reliability Diagrams via Ke.pdf:application/pdf},
}

@article{xie_calibrating_2024,
	title = {{CALIBRATING} {LANGUAGE} {MODELS} {WITH} {ADAPTIVE} {TEMPERATURE} {SCALING}},
	abstract = {The effectiveness of large language models (LLMs) is not only measured by their ability to generate accurate outputs but also by their calibration—how well their confidence scores reflect the probability of their outputs being correct. While unsupervised pre-training has been shown to yield LLMs with well-calibrated conditional probabilities, recent studies have shown that after fine-tuning with reinforcement learning from human feedback (RLHF), the calibration of these models degrades significantly. In this work, we introduce Adaptive Temperature Scaling (ATS), a post-hoc calibration method that predicts a temperature scaling parameter for each token prediction. The predicted temperature values adapt based on token-level features and are fit over a standard supervised fine-tuning (SFT) dataset. The adaptive nature of ATS addresses the varying degrees of calibration shift that can occur after RLHF fine-tuning. ATS improves calibration by over 1050\% across three downstream natural language evaluation benchmarks compared to prior calibration methods and does not impede performance improvements from RLHF.},
	language = {en},
	author = {Xie, Johnathan and Chen, Annie S and Lee, Yoonho and Mitchell, Eric and Finn, Chelsea},
	year = {2024},
	file = {Xie et al. - 2024 - CALIBRATING LANGUAGE MODELS WITH ADAPTIVE TEMPERAT.pdf:C\:\\Users\\yunru\\Zotero\\storage\\8DG4GWCA\\Xie et al. - 2024 - CALIBRATING LANGUAGE MODELS WITH ADAPTIVE TEMPERAT.pdf:application/pdf},
}

@article{papamakarios_normalizing_nodate,
	title = {Normalizing {Flows} for {Probabilistic} {Modeling} and {Inference}},
	abstract = {Normalizing ﬂows provide a general mechanism for deﬁning expressive probability distributions, only requiring the speciﬁcation of a (usually simple) base distribution and a series of bijective transformations. There has been much recent work on normalizing ﬂows, ranging from improving their expressive power to expanding their application. We believe the ﬁeld has now matured and is in need of a uniﬁed perspective. In this review, we attempt to provide such a perspective by describing ﬂows through the lens of probabilistic modeling and inference. We place special emphasis on the fundamental principles of ﬂow design, and discuss foundational topics such as expressive power and computational trade-oﬀs. We also broaden the conceptual framing of ﬂows by relating them to more general probability transformations. Lastly, we summarize the use of ﬂows for tasks such as generative modeling, approximate inference, and supervised learning.},
	language = {en},
	author = {Papamakarios, George and Nalisnick, Eric and Rezende, Danilo Jimenez and Mohamed, Shakir and Lakshminarayanan, Balaji},
	file = {Papamakarios et al. - Normalizing Flows for Probabilistic Modeling and I.pdf:C\:\\Users\\yunru\\Zotero\\storage\\LF8JCYCJ\\Papamakarios et al. - Normalizing Flows for Probabilistic Modeling and I.pdf:application/pdf},
}

@misc{rahimi_post-hoc_2022,
	title = {Post-hoc {Calibration} of {Neural} {Networks} by g-{Layers}},
	url = {http://arxiv.org/abs/2006.12807},
	abstract = {Calibration of neural networks is a critical aspect to consider when incorporating machine learning models in real-world decision-making systems where the conﬁdence of decisions are equally important as the decisions themselves. In recent years, there is a surge of research on neural network calibration and the majority of the works can be categorized into post-hoc calibration methods, deﬁned as methods that learn an additional function to calibrate an already trained base network. In this work, we intend to understand the post-hoc calibration methods from a theoretical point of view. Especially, it is known that minimizing Negative Log-Likelihood (NLL) will lead to a calibrated network on the training set if the global optimum is attained (Bishop 1994). Nevertheless, it is not clear learning an additional function in a post-hoc manner would lead to calibration in the theoretical sense. To this end, we prove that even though the base network (f ) does not lead to the global optimum of NLL, by adding additional g-layers and minimizing NLL by optimizing the parameters of g one can obtain a calibrated network g ◦ f . This not only provides a less stringent condition to obtain a calibrated network but also provides a theoretical justiﬁcation of post-hoc calibration methods. Our experiments on various image classiﬁcation benchmarks conﬁrm the theory.},
	language = {en},
	urldate = {2024-06-04},
	publisher = {arXiv},
	author = {Rahimi, Amir and Mensink, Thomas and Gupta, Kartik and Ajanthan, Thalaiyasingam and Sminchisescu, Cristian and Hartley, Richard},
	month = feb,
	year = {2022},
	note = {arXiv:2006.12807 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Computer Vision and Pattern Recognition},
	file = {Rahimi et al. - 2022 - Post-hoc Calibration of Neural Networks by g-Layer.pdf:C\:\\Users\\yunru\\Zotero\\storage\\63INNW48\\Rahimi et al. - 2022 - Post-hoc Calibration of Neural Networks by g-Layer.pdf:application/pdf},
}

@inproceedings{wang_rethinking_2021,
	title = {Rethinking {Calibration} of {Deep} {Neural} {Networks}: {Do} {Not} {Be} {Afraid} of {Overconfidence}},
	volume = {34},
	shorttitle = {Rethinking {Calibration} of {Deep} {Neural} {Networks}},
	url = {https://proceedings.neurips.cc/paper/2021/hash/61f3a6dbc9120ea78ef75544826c814e-Abstract.html},
	abstract = {Capturing accurate uncertainty quantification of the prediction from deep neural networks is important in many real-world decision-making applications. A reliable predictor is expected to be accurate when it is confident about its predictions and indicate high uncertainty when it is likely to be inaccurate. However, modern neural networks have been found to be poorly calibrated, primarily in the direction of overconfidence. In recent years, there is a surge of research on model calibration by leveraging implicit or explicit regularization techniques during training, which obtain well calibration by avoiding overconfident outputs. In our study, we empirically found that despite the predictions obtained from these regularized models are better calibrated, they suffer from not being as calibratable, namely, it is harder to further calibrate their predictions with post-hoc calibration methods like temperature scaling and histogram binning. We conduct a series of empirical studies showing that overconfidence may not hurt final calibration performance if post-hoc calibration is allowed, rather, the penalty of confident outputs will compress the room of potential improvements in post-hoc calibration phase. Our experimental findings point out a new direction to improve calibration of DNNs by considering main training and post-hoc calibration as a unified framework.},
	urldate = {2024-06-04},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Wang, Deng-Bao and Feng, Lei and Zhang, Min-Ling},
	year = {2021},
	pages = {11809--11820},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\786FKB6M\\Wang et al. - 2021 - Rethinking Calibration of Deep Neural Networks Do.pdf:application/pdf},
}

@inproceedings{karandikar_soft_2021,
	title = {Soft {Calibration} {Objectives} for {Neural} {Networks}},
	volume = {34},
	url = {https://proceedings.neurips.cc/paper_files/paper/2021/hash/f8905bd3df64ace64a68e154ba72f24c-Abstract.html},
	abstract = {Optimal decision making requires that classifiers produce uncertainty estimates consistent with their empirical accuracy. However, deep neural networks are often under- or over-confident in their predictions. Consequently, methods have been developed to improve the calibration of their predictive uncertainty both during training and post-hoc. In this work, we propose differentiable losses to improve calibration based on a soft (continuous) version of the binning operation underlying popular calibration-error estimators. When incorporated into training, these soft calibration losses achieve state-of-the-art single-model ECE across multiple datasets with less than 1\% decrease in accuracy. For instance, we observe an 82\% reduction in ECE (70\% relative to the post-hoc rescaled ECE) in exchange for a 0.7\% relative decrease in accuracy relative to the cross entropy baseline on CIFAR-100.When incorporated post-training, the soft-binning-based calibration error objective improves upon temperature scaling, a popular recalibration method.  Overall, experiments across losses and datasets demonstrate that using calibration-sensitive procedures yield better uncertainty estimates under dataset shift than the standard practice of using a cross entropy loss and post-hoc recalibration methods.},
	urldate = {2024-06-04},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Karandikar, Archit and Cain, Nicholas and Tran, Dustin and Lakshminarayanan, Balaji and Shlens, Jonathon and Mozer, Michael C and Roelofs, Becca},
	year = {2021},
	pages = {29768--29779},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\KDG4KL3I\\Karandikar et al. - 2021 - Soft Calibration Objectives for Neural Networks.pdf:application/pdf},
}

@article{xiong_proximity-informed_2023,
	title = {Proximity-{Informed} {Calibration} for {Deep} {Neural} {Networks}},
	volume = {36},
	url = {https://proceedings.neurips.cc/paper_files/paper/2023/hash/d826f5aadb26db488b8686097ceea2d1-Abstract-Conference.html},
	language = {en},
	urldate = {2024-06-06},
	journal = {Advances in Neural Information Processing Systems},
	author = {Xiong, Miao and Deng, Ailin and Koh, Pang Wei W. and Wu, Jiaying and Li, Shen and Xu, Jianqing and Hooi, Bryan},
	month = dec,
	year = {2023},
	pages = {68511--68538},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\GUIFXF3R\\Xiong et al. - 2023 - Proximity-Informed Calibration for Deep Neural Net.pdf:application/pdf},
}

@inproceedings{hebert-johnson_multicalibration_2018,
	title = {Multicalibration: {Calibration} for the ({Computationally}-{Identifiable}) {Masses}},
	shorttitle = {Multicalibration},
	url = {https://proceedings.mlr.press/v80/hebert-johnson18a.html},
	abstract = {We develop and study multicalibration as a new measure of fairness in machine learning that aims to mitigate inadvertent or malicious discrimination that is introduced at training time (even from ground truth data). Multicalibration guarantees meaningful (calibrated) predictions for every subpopulation that can be identified within a specified class of computations. The specified class can be quite rich; in particular, it can contain many overlapping subgroups of a protected group. We demonstrate that in many settings this strong notion of protection from discrimination is provably attainable and aligned with the goal of obtaining accurate predictions. Along the way, we present algorithms for learning a multicalibrated predictor, study the computational complexity of this task, and illustrate tight connections to the agnostic learning model.},
	language = {en},
	urldate = {2024-06-09},
	booktitle = {Proceedings of the 35th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Hebert-Johnson, Ursula and Kim, Michael and Reingold, Omer and Rothblum, Guy},
	month = jul,
	year = {2018},
	note = {ISSN: 2640-3498},
	pages = {1939--1948},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\NCWCLQIF\\Hebert-Johnson et al. - 2018 - Multicalibration Calibration for the (Computation.pdf:application/pdf;Supplementary PDF:C\:\\Users\\yunru\\Zotero\\storage\\UNLLFJGV\\Hebert-Johnson et al. - 2018 - Multicalibration Calibration for the (Computation.pdf:application/pdf},
}

@inproceedings{sahoo_reliable_2021,
	title = {Reliable {Decisions} with {Threshold} {Calibration}},
	volume = {34},
	url = {https://proceedings.neurips.cc/paper_files/paper/2021/hash/0e65972dce68dad4d52d063967f0a705-Abstract.html},
	abstract = {Decision makers rely on probabilistic forecasts to predict the loss of different decision rules before deployment. When the forecasted probabilities match the true frequencies, predicted losses will be accurate. Although perfect forecasts are typically impossible, probabilities can be calibrated to match the true frequencies on average. However, we find that this {\textbackslash}textit\{average\} notion of calibration, which is typically used in practice, does not necessarily guarantee accurate decision loss prediction. Specifically in the regression setting, the loss of threshold decisions, which are decisions based on whether the forecasted outcome falls above or below a cutoff, might not be predicted accurately. We propose a stronger notion of calibration called threshold calibration, which is exactly the condition required to ensure that decision loss is predicted accurately for threshold decisions. We provide an efficient algorithm which takes an uncalibrated forecaster as input and provably outputs a threshold-calibrated forecaster. Our procedure allows downstream decision makers to confidently estimate the loss of any threshold decision under any threshold loss function. Empirically, threshold calibration improves decision loss prediction without compromising on the quality of the decisions in two real-world settings: hospital scheduling decisions and resource allocation decisions.},
	urldate = {2024-06-10},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Sahoo, Roshni and Zhao, Shengjia and Chen, Alyssa and Ermon, Stefano},
	year = {2021},
	pages = {1831--1844},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\5FST5US9\\Sahoo et al. - 2021 - Reliable Decisions with Threshold Calibration.pdf:application/pdf},
}

@misc{zhao_calibrate_2021,
	title = {Calibrate {Before} {Use}: {Improving} {Few}-{Shot} {Performance} of {Language} {Models}},
	shorttitle = {Calibrate {Before} {Use}},
	url = {http://arxiv.org/abs/2102.09690},
	abstract = {GPT-3 can perform numerous tasks when provided a natural language prompt that contains a few training examples. We show that this type of few-shot learning can be unstable: the choice of prompt format, training examples, and even the order of the training examples can cause accuracy to vary from near chance to near state-of-the-art. We demonstrate that this instability arises from the bias of language models towards predicting certain answers, e.g., those that are placed near the end of the prompt or are common in the pretraining data. To mitigate this, we ﬁrst estimate the model’s bias towards each answer by asking for its prediction when given the training prompt and a content-free test input such as “N/A”. We then ﬁt calibration parameters that cause the prediction for this input to be uniform across answers. On a diverse set of tasks, this contextual calibration procedure substantially improves GPT-3 and GPT-2’s average accuracy (up to 30.0\% absolute) and reduces variance across different choices of the prompt.},
	language = {en},
	urldate = {2024-06-11},
	publisher = {arXiv},
	author = {Zhao, Tony Z. and Wallace, Eric and Feng, Shi and Klein, Dan and Singh, Sameer},
	month = jun,
	year = {2021},
	note = {arXiv:2102.09690 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Computation and Language},
	file = {Zhao et al. - 2021 - Calibrate Before Use Improving Few-Shot Performan.pdf:C\:\\Users\\yunru\\Zotero\\storage\\AF65NGF9\\Zhao et al. - 2021 - Calibrate Before Use Improving Few-Shot Performan.pdf:application/pdf},
}

@misc{fujisawa_pac-bayes_2024,
	title = {{PAC}-{Bayes} {Analysis} for {Recalibration} in {Classification}},
	url = {http://arxiv.org/abs/2406.06227},
	abstract = {Nonparametric estimation with binning is widely employed in the calibration error evaluation and the recalibration of machine learning models. Recently, theoretical analyses of the bias induced by this estimation approach have been actively pursued; however, the understanding of the generalization of the calibration error to unknown data remains limited. In addition, although many recalibration algorithms have been proposed, their generalization performance lacks theoretical guarantees. To address this problem, we conduct a generalization analysis of the calibration error under the probably approximately correct (PAC) Bayes framework. This approach enables us to derive a first optimizable upper bound for the generalization error in the calibration context. We then propose a generalization-aware recalibration algorithm based on our generalization theory. Numerical experiments show that our algorithm improves the Gaussian-process-based recalibration performance on various benchmark datasets and models.},
	language = {en},
	urldate = {2024-06-14},
	publisher = {arXiv},
	author = {Fujisawa, Masahiro and Futami, Futoshi},
	month = jun,
	year = {2024},
	note = {arXiv:2406.06227 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {Fujisawa and Futami - 2024 - PAC-Bayes Analysis for Recalibration in Classifica.pdf:C\:\\Users\\yunru\\Zotero\\storage\\XNM9CG69\\Fujisawa and Futami - 2024 - PAC-Bayes Analysis for Recalibration in Classifica.pdf:application/pdf},
}

@book{rahimi_post-hoc_2020,
	title = {Post-hoc {Calibration} of {Neural} {Networks}},
	abstract = {Calibration of neural networks is a critical aspect to consider when incorporating machine learning models in real-world decision-making systems where the confidence of decisions are equally important as the decisions themselves. In recent years, there is a surge of research on neural network calibration and the majority of the works can be categorized into post-hoc calibration methods, defined as methods that learn an additional function to calibrate an already trained base network. In this work, we intend to understand the post-hoc calibration methods from a theoretical point of view. Especially, it is known that minimizing Negative Log-Likelihood (NLL) will lead to a calibrated network on the training set if the global optimum is attained (Bishop, 1994). Nevertheless, it is not clear learning an additional function in a post-hoc manner would lead to calibration in the theoretical sense. To this end, we prove that even though the base network (\$f\$) does not lead to the global optimum of NLL, by adding additional layers (\$g\$) and minimizing NLL by optimizing the parameters of \$g\$ one can obtain a calibrated network \$g {\textbackslash}circ f\$. This not only provides a less stringent condition to obtain a calibrated network but also provides a theoretical justification of post-hoc calibration methods. Our experiments on various image classification benchmarks confirm the theory.},
	author = {Rahimi, Amir and Gupta, Kartik and Ajanthan, Thalaiyasingam and Mensink, Thomas and Sminchisescu, Cristian and Hartley, Richard},
	month = jun,
	year = {2020},
}

@article{wang_calibration_nodate,
	title = {Calibration {Bottleneck}: {Over}-compressed {Representations} are {Less} {Calibratable}},
	abstract = {Although deep neural networks have achieved remarkable success, they often exhibit a significant deficiency in reliable uncertainty calibration. This paper focus on model calibratability, which assesses how amenable a model is to be well recalibrated post-hoc. We find that the widely used weight decay regularizer detrimentally affects model calibratability, subsequently leading to a decline in final calibration performance after post-hoc calibration. To identify the underlying causes leading to poor calibratability, we delve into the calibratability of intermediate features across the hidden layers. We observe a U-shaped trend in the calibratability of intermediate features from the bottom to the top layers, which indicates that over-compression of the top representation layers significantly hinders model calibratability. Based on the observations, this paper introduces a weak classifier hypothesis, i.e., given a weak classification head that has not been over-trained, the representation module can be better learned to produce more calibratable features. Consequently, we propose a progressively layer-peeled training (PLP) method to exploit this hypothesis, thereby enhancing model calibratability. Our comparative experiments show the effectiveness of our method, which improves model calibration and also yields competitive predictive performance.},
	language = {en},
	author = {Wang, Deng-Bao and Zhang, Min-Ling},
	file = {Wang and Zhang - Calibration Bottleneck Over-compressed Representa.pdf:C\:\\Users\\yunru\\Zotero\\storage\\IJ8BXEWB\\Wang and Zhang - Calibration Bottleneck Over-compressed Representa.pdf:application/pdf},
}

@article{minderer_revisiting_nodate,
	title = {Revisiting the {Calibration} of {Modern} {Neural} {Networks}},
	abstract = {Accurate estimation of predictive uncertainty (model calibration) is essential for the safe application of neural networks. Many instances of miscalibration in modern neural networks have been reported, suggesting a trend that newer, more accurate models produce poorly calibrated predictions. Here, we revisit this question for recent state-of-the-art image classiﬁcation models. We systematically relate model calibration and accuracy, and ﬁnd that the most recent models, notably those not using convolutions, are among the best calibrated. Trends observed in prior model generations, such as decay of calibration with distribution shift or model size, are less pronounced in recent architectures. We also show that model size and amount of pretraining do not fully explain these differences, suggesting that architecture is a major determinant of calibration properties.},
	language = {en},
	author = {Minderer, Matthias and Djolonga, Josip and Romijnders, Rob and Hubis, Frances and Zhai, Xiaohua and Houlsby, Neil and Tran, Dustin and Lucic, Mario},
	file = {Minderer et al. - Revisiting the Calibration of Modern Neural Networ.pdf:C\:\\Users\\yunru\\Zotero\\storage\\MRZJTJ8R\\Minderer et al. - Revisiting the Calibration of Modern Neural Networ.pdf:application/pdf},
}

@misc{nakkiran_deep_2019,
	title = {Deep {Double} {Descent}: {Where} {Bigger} {Models} and {More} {Data} {Hurt}},
	shorttitle = {Deep {Double} {Descent}},
	url = {http://arxiv.org/abs/1912.02292},
	abstract = {We show that a variety of modern deep learning tasks exhibit a “double-descent” phenomenon where, as we increase model size, performance ﬁrst gets worse and then gets better. Moreover, we show that double descent occurs not just as a function of model size, but also as a function of the number of training epochs. We unify the above phenomena by deﬁning a new complexity measure we call the effective model complexity and conjecture a generalized double descent with respect to this measure. Furthermore, our notion of model complexity allows us to identify certain regimes where increasing (even quadrupling) the number of train samples actually hurts test performance.},
	language = {en},
	urldate = {2024-06-20},
	publisher = {arXiv},
	author = {Nakkiran, Preetum and Kaplun, Gal and Bansal, Yamini and Yang, Tristan and Barak, Boaz and Sutskever, Ilya},
	month = dec,
	year = {2019},
	note = {arXiv:1912.02292 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Neural and Evolutionary Computing},
	file = {Nakkiran et al. - 2019 - Deep Double Descent Where Bigger Models and More .pdf:C\:\\Users\\yunru\\Zotero\\storage\\Q6IZ5IUM\\Nakkiran et al. - 2019 - Deep Double Descent Where Bigger Models and More .pdf:application/pdf},
}

@misc{blasiok_when_2023,
	title = {When {Does} {Optimizing} a {Proper} {Loss} {Yield} {Calibration}?},
	url = {http://arxiv.org/abs/2305.18764},
	doi = {10.48550/arXiv.2305.18764},
	abstract = {Optimizing proper loss functions is popularly believed to yield predictors with good calibration properties; the intuition being that for such losses, the global optimum is to predict the ground-truth probabilities, which is indeed calibrated. However, typical machine learning models are trained to approximately minimize loss over restricted families of predictors, that are unlikely to contain the ground truth. Under what circumstances does optimizing proper loss over a restricted family yield calibrated models? What precise calibration guarantees does it give? In this work, we provide a rigorous answer to these questions. We replace the global optimality with a local optimality condition stipulating that the (proper) loss of the predictor cannot be reduced much by post-processing its predictions with a certain family of Lipschitz functions. We show that any predictor with this local optimality satisfies smooth calibration as defined in Kakade-Foster (2008), B\{{\textbackslash}l\}asiok et al. (2023). Local optimality is plausibly satisfied by well-trained DNNs, which suggests an explanation for why they are calibrated from proper loss minimization alone. Finally, we show that the connection between local optimality and calibration error goes both ways: nearly calibrated predictors are also nearly locally optimal.},
	urldate = {2024-06-20},
	publisher = {arXiv},
	author = {Błasiok, Jarosław and Gopalan, Parikshit and Hu, Lunjia and Nakkiran, Preetum},
	month = dec,
	year = {2023},
	note = {arXiv:2305.18764 [cs, math, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Mathematics - Statistics Theory},
	file = {arXiv Fulltext PDF:C\:\\Users\\yunru\\Zotero\\storage\\2MQDETN7\\Błasiok et al. - 2023 - When Does Optimizing a Proper Loss Yield Calibrati.pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\CBNJHBJT\\2305.html:text/html},
}

@misc{hebert-johnson_calibration_2018,
	title = {Calibration for the ({Computationally}-{Identifiable}) {Masses}},
	url = {http://arxiv.org/abs/1711.08513},
	doi = {10.48550/arXiv.1711.08513},
	abstract = {As algorithms increasingly inform and influence decisions made about individuals, it becomes increasingly important to address concerns that these algorithms might be discriminatory. The output of an algorithm can be discriminatory for many reasons, most notably: (1) the data used to train the algorithm might be biased (in various ways) to favor certain populations over others; (2) the analysis of this training data might inadvertently or maliciously introduce biases that are not borne out in the data. This work focuses on the latter concern. We develop and study multicalbration -- a new measure of algorithmic fairness that aims to mitigate concerns about discrimination that is introduced in the process of learning a predictor from data. Multicalibration guarantees accurate (calibrated) predictions for every subpopulation that can be identified within a specified class of computations. We think of the class as being quite rich; in particular, it can contain many overlapping subgroups of a protected group. We show that in many settings this strong notion of protection from discrimination is both attainable and aligned with the goal of obtaining accurate predictions. Along the way, we present new algorithms for learning a multicalibrated predictor, study the computational complexity of this task, and draw new connections to computational learning models such as agnostic learning.},
	urldate = {2024-06-27},
	publisher = {arXiv},
	author = {Hébert-Johnson, Úrsula and Kim, Michael P. and Reingold, Omer and Rothblum, Guy N.},
	month = mar,
	year = {2018},
	note = {arXiv:1711.08513 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Data Structures and Algorithms},
	file = {arXiv Fulltext PDF:C\:\\Users\\yunru\\Zotero\\storage\\CQCKZVNW\\Hébert-Johnson et al. - 2018 - Calibration for the (Computationally-Identifiable).pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\NYYUJKY7\\1711.html:text/html},
}

@misc{geifman_bias-reduced_2019,
	title = {Bias-{Reduced} {Uncertainty} {Estimation} for {Deep} {Neural} {Classifiers}},
	url = {http://arxiv.org/abs/1805.08206},
	abstract = {We consider the problem of uncertainty estimation in the context of (non-Bayesian) deep neural classiﬁcation. In this context, all known methods are based on extracting uncertainty signals from a trained network optimized to solve the classiﬁcation problem at hand. We demonstrate that such techniques tend to introduce biased estimates for instances whose predictions are supposed to be highly conﬁdent. We argue that this deﬁciency is an artifact of the dynamics of training with SGD-like optimizers, and it has some properties similar to overﬁtting. Based on this observation, we develop an uncertainty estimation algorithm that selectively estimates the uncertainty of highly conﬁdent points, using earlier snapshots of the trained model, before their estimates are jittered (and way before they are ready for actual classiﬁcation). We present extensive experiments indicating that the proposed algorithm provides uncertainty estimates that are consistently better than all known methods.},
	language = {en},
	urldate = {2024-06-28},
	publisher = {arXiv},
	author = {Geifman, Yonatan and Uziel, Guy and El-Yaniv, Ran},
	month = apr,
	year = {2019},
	note = {arXiv:1805.08206 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {1805.pdf:C\:\\Users\\yunru\\Zotero\\storage\\PF8GU42T\\1805.pdf:application/pdf},
}

@article{yang_beyond_nodate,
	title = {Beyond {Probability} {Partitions}: {Calibrating} {Neural} {Networks} with {Semantic} {Aware} {Grouping}},
	abstract = {Research has shown that deep networks tend to be overly optimistic about their predictions, leading to an underestimation of prediction errors. Due to the limited nature of data, existing studies have proposed various methods based on model prediction probabilities to bin the data and evaluate calibration error. We propose a more generalized definition of calibration error called Partitioned Calibration Error (PCE), revealing that the key difference among these calibration error metrics lies in how the data space is partitioned. We put forth an intuitive proposition that an accurate model should be calibrated across any partition, suggesting that the input space partitioning can extend beyond just the partitioning of prediction probabilities, and include partitions directly related to the input. Through semanticrelated partitioning functions, we demonstrate that the relationship between model accuracy and calibration lies in the granularity of the partitioning function. This highlights the importance of partitioning criteria for training a calibrated and accurate model. To validate the aforementioned analysis, we propose a method that involves jointly learning a semantic aware grouping function based on deep model features and logits to partition the data space into subsets. Subsequently, a separate calibration function is learned for each subset. Experimental results demonstrate that our approach achieves significant performance improvements across multiple datasets and network architectures, thus highlighting the importance of the partitioning function for calibration.},
	language = {en},
	author = {Yang, Jia-Qi and Zhan, De-Chuan},
	file = {Yang and Zhan - Beyond Probability Partitions Calibrating Neural .pdf:C\:\\Users\\yunru\\Zotero\\storage\\VAHFIV3Q\\Yang and Zhan - Beyond Probability Partitions Calibrating Neural .pdf:application/pdf},
}

@inproceedings{ece_flaw,
author = {Chidambaram, Muthu and Lee, Holden and McSwiggen, Colin and Rezchikov, Semon},
title = {How flawed is ECE? an analysis via logit smoothing},
year = {2024},
publisher = {JMLR.org},
abstract = {Informally, a model is calibrated if its predictions are correct with a probability that matches the confidence of the prediction. By far the most common method in the literature for measuring calibration is the expected calibration error (ECE). Recent work, however, has pointed out drawbacks of ECE, such as the fact that it is discontinuous in the space of predictors. In this work, we ask: how fundamental are these issues, and what are their impacts on existing results? Towards this end, we completely characterize the discontinuities of ECE with respect to general probability measures on Polish spaces. We then use the nature of these discontinuities to motivate a novel continuous, easily estimated miscalibration metric, which we term Logit-Smoothed ECE (LS-ECE). By comparing the ECE and LS-ECE of pre-trained image classification models, we show in initial experiments that binned ECE closely tracks LS-ECE, indicating that the theoretical pathologies of ECE may be avoidable in practice.},
booktitle = {Proceedings of the 41st International Conference on Machine Learning},
articleno = {334},
numpages = {19},
location = {Vienna, Austria},
series = {ICML'24}
}



@article{ma_characterizing_2018,
	title = {{CHARACTERIZING} {ADVERSARIAL} {SUBSPACES} {USING} {LOCAL} {INTRINSIC} {DIMENSIONALITY}},
	abstract = {Deep Neural Networks (DNNs) have recently been shown to be vulnerable against adversarial examples, which are carefully crafted instances that can mislead DNNs to make errors during prediction. To better understand such attacks, a characterization is needed of the properties of regions (the so-called ‘adversarial subspaces’) in which adversarial examples lie. We tackle this challenge by characterizing the dimensional properties of adversarial regions, via the use of Local Intrinsic Dimensionality (LID). LID assesses the space-ﬁlling capability of the region surrounding a reference example, based on the distance distribution of the example to its neighbors. We ﬁrst provide explanations about how adversarial perturbation can affect the LID characteristic of adversarial regions, and then show empirically that LID characteristics can facilitate the distinction of adversarial examples generated using state-of-the-art attacks. As a proof-of-concept, we show that a potential application of LID is to distinguish adversarial examples, and the preliminary results show that it can outperform several state-of-the-art detection measures by large margins for ﬁve attack strategies considered in this paper across three benchmark datasets . Our analysis of the LID characteristic for adversarial regions not only motivates new directions of effective adversarial defense, but also opens up more challenges for developing new attacks to better understand the vulnerabilities of DNNs.},
	language = {en},
	author = {Ma, Xingjun and Li, Bo and Wang, Yisen and Erfani, Sarah M and Wijewickrema, Sudanthi and Schoenebeck, Grant and Song, Dawn and Houle, Michael E and Bailey, James},
	year = {2018},
	file = {Ma et al. - 2018 - CHARACTERIZING ADVERSARIAL SUBSPACES USING LOCAL I.pdf:C\:\\Users\\yunru\\Zotero\\storage\\TJLHCENJ\\Ma et al. - 2018 - CHARACTERIZING ADVERSARIAL SUBSPACES USING LOCAL I.pdf:application/pdf},
}

@inproceedings{zhang_mix-n-match_2020,
	title = {Mix-n-{Match} : {Ensemble} and {Compositional} {Methods} for {Uncertainty} {Calibration} in {Deep} {Learning}},
	shorttitle = {Mix-n-{Match}},
	url = {https://proceedings.mlr.press/v119/zhang20k.html},
	abstract = {This paper studies the problem of post-hoc calibration of machine learning classifiers. We introduce the following desiderata for uncertainty calibration: (a) accuracy-preserving, (b) data-efficient, and (c) high expressive power. We show that none of the existing methods satisfy all three requirements, and demonstrate how Mix-n-Match calibration strategies (i.e., ensemble and composition) can help achieve remarkably better data-efficiency and expressive power while provably maintaining the classification accuracy of the original classifier. Mix-n-Match strategies are generic in the sense that they can be used to improve the performance of any off-the-shelf calibrator. We also reveal potential issues in standard evaluation practices. Popular approaches (e.g., histogram-based expected calibration error (ECE)) may provide misleading results especially in small-data regime. Therefore, we propose an alternative data-efficient kernel density-based estimator for a reliable evaluation of the calibration performance and prove its asymptotically unbiasedness and consistency. Our approaches outperform state-of-the-art solutions on both the calibration as well as the evaluation tasks in most of the experimental settings. Our codes are available at https://github.com/zhang64- llnl/Mix-n-Match-Calibration.},
	language = {en},
	urldate = {2024-07-30},
	booktitle = {Proceedings of the 37th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Zhang, Jize and Kailkhura, Bhavya and Han, T. Yong-Jin},
	month = nov,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {11117--11128},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\5CZM9U37\\Zhang et al. - 2020 - Mix-n-Match  Ensemble and Compositional Methods f.pdf:application/pdf;Supplementary PDF:C\:\\Users\\yunru\\Zotero\\storage\\W38RK339\\Zhang et al. - 2020 - Mix-n-Match  Ensemble and Compositional Methods f.pdf:application/pdf},
}

@inproceedings{tomani_parameterized_2022,
	address = {Cham},
	title = {Parameterized {Temperature} {Scaling} for {Boosting} the {Expressive} {Power} in {Post}-{Hoc} {Uncertainty} {Calibration}},
	abstract = {We address the problem of uncertainty calibration and introduce a novel calibration method, Parametrized Temperature Scaling (PTS). Standard deep neural networks typically yield uncalibrated predictions, which can be transformed into calibrated confidence scores using post-hoc calibration methods. In this contribution, we demonstrate that the performance of accuracy-preserving state-of-the-art post-hoc calibrators is limited by their intrinsic expressive power. We generalize temperature scaling by computing prediction-specific temperatures, parameterized by a neural network. We show with extensive experiments that our novel accuracy-preserving approach consistently outperforms existing algorithms across a large number of model architectures, datasets and metrics (Source code available at: https://github.com/tochris/pts-uncertainty).},
	language = {en},
	booktitle = {European Conference on Computer Vision – {ECCV} 2022},
	author = {Tomani, Christian and Cremers, Daniel and Buettner, Florian},
	year = {2022},
	pages = {555--569}
}

@inproceedings{tomani_beyond_2023,
	title = {Beyond {In}-{Domain} {Scenarios}: {Robust} {Density}-{Aware} {Calibration}},
	shorttitle = {Beyond {In}-{Domain} {Scenarios}},
	url = {https://proceedings.mlr.press/v202/tomani23a.html},
	abstract = {Calibrating deep learning models to yield uncertainty-aware predictions is crucial as deep neural networks get increasingly deployed in safety-critical applications. While existing post-hoc calibration methods achieve impressive results on in-domain test datasets, they are limited by their inability to yield reliable uncertainty estimates in domain-shift and out-of-domain (OOD) scenarios. We aim to bridge this gap by proposing DAC, an accuracy-preserving as well as Density-Aware Calibration method based on k-nearest-neighbors (KNN). In contrast to existing post-hoc methods, we utilize hidden layers of classifiers as a source for uncertainty-related information and study their importance. We show that DAC is a generic method that can readily be combined with state-of-the-art post-hoc methods. DAC boosts the robustness of calibration performance in domain-shift and OOD, while maintaining excellent in-domain predictive uncertainty estimates. We demonstrate that DAC leads to consistently better calibration across a large number of model architectures, datasets, and metrics. Additionally, we show that DAC improves calibration substantially on recent large-scale neural networks pre-trained on vast amounts of data.},
	language = {en},
	urldate = {2024-08-29},
	booktitle = {Proceedings of the 40th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Tomani, Christian and Waseda, Futa Kai and Shen, Yuesong and Cremers, Daniel},
	month = jul,
	year = {2023},
	note = {ISSN: 2640-3498},
	pages = {34344--34368},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\QJ645XE2\\Tomani et al. - 2023 - Beyond In-Domain Scenarios Robust Density-Aware C.pdf:application/pdf},
}

@inproceedings{roelofs_mitigating_2022,
	title = {Mitigating {Bias} in {Calibration} {Error} {Estimation}},
	url = {https://proceedings.mlr.press/v151/roelofs22a.html},
	abstract = {For an AI system to be reliable, the confidence it expresses in its decisions must match its accuracy. To assess the degree of match, examples are typically binned by confidence and the per-bin mean confidence and accuracy are compared. Most research in calibration focuses on techniques to reduce this empirical measure of calibration error, ECE\_bin. We instead focus on assessing statistical bias in this empirical measure, and we identify better estimators. We propose a framework through which we can compute the bias of a particular estimator for an evaluation data set of a given size. The framework involves synthesizing model outputs that have the same statistics as common neural architectures on popular data sets. We find that binning-based estimators with bins of equal mass (number of instances) have lower bias than estimators with bins of equal width. Our results indicate two reliable calibration-error estimators: the debiased estimator (Brocker, 2012; Ferro and Fricker, 2012) and a method we propose, ECE\_sweep, which uses equal-mass bins and chooses the number of bins to be as large as possible while preserving monotonicity in the calibration function. With these estimators, we observe improvements in the effectiveness of recalibration methods and in the detection of model miscalibration.},
	language = {en},
	urldate = {2024-09-03},
	booktitle = {Proceedings of {The} 25th {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Roelofs, Rebecca and Cain, Nicholas and Shlens, Jonathon and Mozer, Michael C.},
	month = may,
	year = {2022},
	note = {ISSN: 2640-3498},
	pages = {4036--4054},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\MPTXSACY\\Roelofs et al. - 2022 - Mitigating Bias in Calibration Error Estimation.pdf:application/pdf},
}

@misc{nixon_measuring_2020,
	title = {Measuring {Calibration} in {Deep} {Learning}},
	url = {http://arxiv.org/abs/1904.01685},
	abstract = {Overconﬁdence and underconﬁdence in machine learning classiﬁers is measured by calibration: the degree to which the probabilities predicted for each class match the accuracy of the classiﬁer on that prediction.},
	language = {en},
	urldate = {2024-09-04},
	publisher = {arXiv},
	author = {Nixon, Jeremy and Dusenberry, Mike and Jerfel, Ghassen and Nguyen, Timothy and Liu, Jeremiah and Zhang, Linchuan and Tran, Dustin},
	month = aug,
	year = {2020},
	note = {arXiv:1904.01685 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\FLC9XHJX\\Nixon et al. - 2020 - Measuring Calibration in Deep Learning.pdf:application/pdf},
}

@article{olah_zoom_2020,
	title = {Zoom {In}: {An} {Introduction} to {Circuits}},
	volume = {5},
	issn = {2476-0757},
	shorttitle = {Zoom {In}},
	url = {https://distill.pub/2020/circuits/zoom-in},
	doi = {10.23915/distill.00024.001},
	number = {3},
	urldate = {2024-09-05},
	journal = {Distill},
	author = {Olah, Chris and Cammarata, Nick and Schubert, Ludwig and Goh, Gabriel and Petrov, Michael and Carter, Shan},
	month = mar,
	year = {2020},
	pages = {10.23915/distill.00024.001},
}

@misc{ji_bin-wise_2019,
	title = {Bin-wise {Temperature} {Scaling} ({BTS}): {Improvement} in {Confidence} {Calibration} {Performance} through {Simple} {Scaling} {Techniques}},
	shorttitle = {Bin-wise {Temperature} {Scaling} ({BTS})},
	url = {http://arxiv.org/abs/1908.11528},
	abstract = {The prediction reliability of neural networks is important in many applications. Speciﬁcally, in safety-critical domains, such as cancer prediction or autonomous driving, a reliable conﬁdence of models prediction is critical for the interpretation of the results. Modern deep neural networks have achieved a signiﬁcant improvement in performance for many different image classiﬁcation tasks. However, these networks tend to be poorly calibrated in terms of output conﬁdence. Temperature scaling is an efﬁcient postprocessing-based calibration scheme and obtains well calibrated results. In this study, we leverage the concept of temperature scaling to build a sophisticated bin-wise scaling. Furthermore, we adopt augmentation of validation samples for elaborated scaling. The proposed methods consistently improve calibration performance with various datasets and deep convolutional neural network models.},
	language = {en},
	urldate = {2024-09-16},
	publisher = {arXiv},
	author = {Ji, Byeongmoon and Jung, Hyemin and Yoon, Jihyeun and Kim, Kyungyul and Shin, Younghak},
	month = sep,
	year = {2019},
	note = {arXiv:1908.11528 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\E3BAV538\\Ji et al. - 2019 - Bin-wise Temperature Scaling (BTS) Improvement in Confidence Calibration Performance through Simple.pdf:application/pdf},
}

@misc{rahimi_post-hoc_2022-1,
	title = {Post-hoc {Calibration} of {Neural} {Networks} by g-{Layers}},
	url = {http://arxiv.org/abs/2006.12807},
	abstract = {Calibration of neural networks is a critical aspect to consider when incorporating machine learning models in real-world decision-making systems where the conﬁdence of decisions are equally important as the decisions themselves. In recent years, there is a surge of research on neural network calibration and the majority of the works can be categorized into post-hoc calibration methods, deﬁned as methods that learn an additional function to calibrate an already trained base network. In this work, we intend to understand the post-hoc calibration methods from a theoretical point of view. Especially, it is known that minimizing Negative Log-Likelihood (NLL) will lead to a calibrated network on the training set if the global optimum is attained (Bishop 1994). Nevertheless, it is not clear learning an additional function in a post-hoc manner would lead to calibration in the theoretical sense. To this end, we prove that even though the base network (f ) does not lead to the global optimum of NLL, by adding additional g-layers and minimizing NLL by optimizing the parameters of g one can obtain a calibrated network g ◦ f . This not only provides a less stringent condition to obtain a calibrated network but also provides a theoretical justiﬁcation of post-hoc calibration methods. Our experiments on various image classiﬁcation benchmarks conﬁrm the theory.},
	language = {en},
	urldate = {2024-10-01},
	publisher = {arXiv},
	author = {Rahimi, Amir and Mensink, Thomas and Gupta, Kartik and Ajanthan, Thalaiyasingam and Sminchisescu, Cristian and Hartley, Richard},
	month = feb,
	year = {2022},
	note = {arXiv:2006.12807 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Computer Vision and Pattern Recognition},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\T2FXAWGQ\\Rahimi et al. - 2022 - Post-hoc Calibration of Neural Networks by g-Layers.pdf:application/pdf},
}

@misc{shen_thermometer_2024,
	title = {Thermometer: {Towards} {Universal} {Calibration} for {Large} {Language} {Models}},
	shorttitle = {Thermometer},
	url = {http://arxiv.org/abs/2403.08819},
	abstract = {We consider the issue of calibration in large language models (LLM). Recent studies have found that common interventions such as instruction tuning often result in poorly calibrated LLMs. Although calibration is well-explored in traditional applications, calibrating LLMs is uniquely challenging. These challenges stem as much from the severe computational requirements of LLMs as from their versatility, which allows them to be applied to diverse tasks. Addressing these challenges, we propose THERMOMETER, a calibration approach tailored to LLMs. THERMOMETER learns an auxiliary model, given data from multiple tasks, for calibrating a LLM. It is computationally efficient, preserves the accuracy of the LLM, and produces better-calibrated responses for new tasks. Extensive empirical evaluations across various benchmarks demonstrate the effectiveness of the proposed method1.},
	language = {en},
	urldate = {2024-10-03},
	publisher = {arXiv},
	author = {Shen, Maohao and Das, Subhro and Greenewald, Kristjan and Sattigeri, Prasanna and Wornell, Gregory and Ghosh, Soumya},
	month = jun,
	year = {2024},
	note = {arXiv:2403.08819 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Computation and Language},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\MAMMPRIA\\Shen et al. - 2024 - Thermometer Towards Universal Calibration for Large Language Models.pdf:application/pdf},
}

@misc{shen_post-hoc_2022,
	title = {Post-hoc {Uncertainty} {Learning} using a {Dirichlet} {Meta}-{Model}},
	url = {http://arxiv.org/abs/2212.07359},
	abstract = {It is known that neural networks have the problem of being over-conﬁdent when directly using the output label distribution to generate uncertainty measures. Existing methods mainly resolve this issue by retraining the entire model to impose the uncertainty quantiﬁcation capability so that the learned model can achieve desired performance in accuracy and uncertainty prediction simultaneously. However, training the model from scratch is computationally expensive and may not be feasible in many situations. In this work, we consider a more practical post-hoc uncertainty learning setting, where a well-trained base model is given, and we focus on the uncertainty quantiﬁcation task at the second stage of training. We propose a novel Bayesian meta-model to augment pretrained models with better uncertainty quantiﬁcation abilities, which is effective and computationally efﬁcient. Our proposed method requires no additional training data and is ﬂexible enough to quantify different uncertainties and easily adapt to different application settings, including out-of-domain data detection, misclassiﬁcation detection, and trustworthy transfer learning. We demonstrate our proposed meta-model approach’s ﬂexibility and superior empirical performance on these applications over multiple representative image classiﬁcation benchmarks.},
	language = {en},
	urldate = {2024-10-03},
	publisher = {arXiv},
	author = {Shen, Maohao and Bu, Yuheng and Sattigeri, Prasanna and Ghosh, Soumya and Das, Subhro and Wornell, Gregory},
	month = dec,
	year = {2022},
	note = {arXiv:2212.07359 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\68N33QX6\\Shen et al. - 2022 - Post-hoc Uncertainty Learning using a Dirichlet Meta-Model.pdf:application/pdf},
}

@inproceedings{noauthor_reassessing_2024,
	title = {Reassessing {How} to {Compare} and {Improve} the {Calibration} of {Machine} {Learning} {Models}},
	url = {https://openreview.net/forum?id=X0epAjg0hd},
	abstract = {A machine learning model is calibrated if its predicted probability for an outcome matches the observed frequency for that outcome conditional on the model prediction. This property has become increasingly important as the impact of machine learning models has continued to spread to various domains. As a result, there are now a dizzying number of recent papers on measuring and improving the calibration of (specifically deep learning) models. In this work, we reassess the reporting of calibration metrics in the recent literature. We show that there exist trivial recalibration approaches that can appear seemingly state-of-the-art unless calibration and prediction metrics (i.e. test accuracy) are accompanied by additional generalization metrics such as negative log-likelihood. We then use a calibration-based decomposition of Bregman divergences to develop a new extension to reliability diagrams that jointly visualizes calibration and generalization error, and show how our visualization can be used to detect trade-offs between calibration and generalization. Along the way, we prove novel results regarding the relationship between full calibration error and confidence calibration error for Bregman divergences. We also establish the consistency of the kernel regression estimator for calibration error used in our visualization approach, which generalizes existing consistency results in the literature.},
	language = {en},
	urldate = {2024-10-22},
	month = oct,
	year = {2024},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\ZPA973BP\\2024 - Reassessing How to Compare and Improve the Calibration of Machine Learning Models.pdf:application/pdf},
}

@misc{chidambaram_limitations_2024,
	title = {On the {Limitations} of {Temperature} {Scaling} for {Distributions} with {Overlaps}},
	url = {http://arxiv.org/abs/2306.00740},
	abstract = {Despite the impressive generalization capabilities of deep neural networks, they have been repeatedly shown to be overconfident when they are wrong. Fixing this issue is known as model calibration, and has consequently received much attention in the form of modified training schemes and post-training calibration procedures such as temperature scaling. While temperature scaling is frequently used because of its simplicity, it is often outperformed by modified training schemes. In this work, we identify a specific bottleneck for the performance of temperature scaling. We show that for empirical risk minimizers for a general set of distributions in which the supports of classes have overlaps, the performance of temperature scaling degrades with the amount of overlap between classes, and asymptotically becomes no better than random when there are a large number of classes. On the other hand, we prove that optimizing a modified form of the empirical risk induced by the Mixup data augmentation technique can in fact lead to reasonably good calibration performance, showing that training-time calibration may be necessary in some situations. We also verify that our theoretical results reflect practice by showing that Mixup significantly outperforms empirical risk minimization (with respect to multiple calibration metrics) on image classification benchmarks with class overlaps introduced in the form of label noise.},
	urldate = {2024-10-22},
	publisher = {arXiv},
	author = {Chidambaram, Muthu and Ge, Rong},
	month = feb,
	year = {2024},
	note = {arXiv:2306.00740},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {Preprint PDF:C\:\\Users\\yunru\\Zotero\\storage\\E8G32HWK\\Chidambaram and Ge - 2024 - On the Limitations of Temperature Scaling for Distributions with Overlaps.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\5WNUTY8F\\2306.html:text/html},
}

@misc{runje_constrained_2023,
	title = {Constrained {Monotonic} {Neural} {Networks}},
	url = {http://arxiv.org/abs/2205.11775},
	abstract = {Wider adoption of neural networks in many critical domains such as finance and healthcare is being hindered by the need to explain their predictions and to impose additional constraints on them. Monotonicity constraint is one of the most requested properties in real-world scenarios and is the focus of this paper. One of the oldest ways to construct a monotonic fully connected neural network is to constrain signs on its weights. Unfortunately, this construction does not work with popular non-saturated activation functions as it can only approximate convex functions. We show this shortcoming can be fixed by constructing two additional activation functions from a typical unsaturated monotonic activation function and employing each of them on the part of neurons. Our experiments show this approach of building monotonic neural networks has better accuracy when compared to other state-of-the-art methods, while being the simplest one in the sense of having the least number of parameters, and not requiring any modifications to the learning procedure or post-learning steps. Finally, we prove it can approximate any continuous monotone function on a compact subset of \${\textbackslash}mathbb\{R\}{\textasciicircum}n\$.},
	urldate = {2024-10-23},
	publisher = {arXiv},
	author = {Runje, Davor and Shankaranarayana, Sharath M.},
	month = may,
	year = {2023},
	note = {arXiv:2205.11775},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
	file = {Preprint PDF:C\:\\Users\\yunru\\Zotero\\storage\\29LYZPTA\\Runje and Shankaranarayana - 2023 - Constrained Monotonic Neural Networks.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\FE2WQKXE\\2205.html:text/html},
}

@article{nolte_expressive_2023,
	title = {{EXPRESSIVE} {MONOTONIC} {NEURAL} {NETWORKS}},
	abstract = {The monotonic dependence of the outputs of a neural network on some of its inputs is a crucial inductive bias in many scenarios where domain knowledge dictates such behavior. This is especially important for interpretability and fairness considerations. In a broader context, scenarios in which monotonicity is important can be found in finance, medicine, physics, and other disciplines. It is thus desirable to build neural network architectures that implement this inductive bias provably. In this work, we propose a weight-constrained architecture with a single residual connection to achieve exact monotonic dependence in any subset of the inputs. The weight constraint scheme directly controls the Lipschitz constant of the neural network and thus provides the additional benefit of robustness. Compared to currently existing techniques used for monotonicity, our method is simpler in implementation and in theory foundations, has negligible computational overhead, is guaranteed to produce monotonic dependence, and is highly expressive. We show how the algorithm is used to train powerful, robust, and interpretable discriminators that achieve competitive performance compared to current state-of-the-art methods across various benchmarks, from social applications to the classification of the decays of subatomic particles produced at the CERN Large Hadron Collider.},
	language = {en},
	author = {Nolte, Niklas and Kitouni, Ouail and Williams, Mike},
	year = {2023},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\QSMB7Y9T\\Nolte et al. - 2023 - EXPRESSIVE MONOTONIC NEURAL NETWORKS.pdf:application/pdf},
}

@inproceedings{bai_dont_2021,
	title = {Don’t {Just} {Blame} {Over}-parametrization for {Over}-confidence: {Theoretical} {Analysis} of {Calibration} in {Binary} {Classification}},
	shorttitle = {Don’t {Just} {Blame} {Over}-parametrization for {Over}-confidence},
	url = {https://proceedings.mlr.press/v139/bai21c.html},
	abstract = {Modern machine learning models with high accuracy are often miscalibrated—the predicted top probability does not reflect the actual accuracy, and tends to be {\textbackslash}emph\{over-confident\}. It is commonly believed that such over-confidence is mainly due to {\textbackslash}emph\{over-parametrization\}, in particular when the model is large enough to memorize the training data and maximize the confidence. In this paper, we show theoretically that over-parametrization is not the only reason for over-confidence. We prove that {\textbackslash}emph\{logistic regression is inherently over-confident\}, in the realizable, under-parametrized setting where the data is generated from the logistic model, and the sample size is much larger than the number of parameters. Further, this over-confidence happens for general well-specified binary classification problems as long as the activation is symmetric and concave on the positive part. Perhaps surprisingly, we also show that over-confidence is not always the case—there exists another activation function (and a suitable loss function) under which the learned classifier is {\textbackslash}emph\{under-confident\} at some probability values. Overall, our theory provides a precise characterization of calibration in realizable binary classification, which we verify on simulations and real data experiments.},
	language = {en},
	urldate = {2024-10-25},
	booktitle = {Proceedings of the 38th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Bai, Yu and Mei, Song and Wang, Huan and Xiong, Caiming},
	month = jul,
	year = {2021},
	note = {ISSN: 2640-3498},
	pages = {566--576},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\AXQ3UD3T\\Bai et al. - 2021 - Don’t Just Blame Over-parametrization for Over-confidence Theoretical Analysis of Calibration in Bi.pdf:application/pdf;Supplementary PDF:C\:\\Users\\yunru\\Zotero\\storage\\TD2QQHUE\\Bai et al. - 2021 - Don’t Just Blame Over-parametrization for Over-confidence Theoretical Analysis of Calibration in Bi.pdf:application/pdf},
}

@inproceedings{rahimi_intra_2020,
author = {Rahimi, Amir and Shaban, Amirreza and Cheng, Ching-An and Hartley, Richard and Boots, Byron},
title = {Intra order-preserving functions for calibration of multi-class neural networks},
year = {2020},
isbn = {9781713829546},
publisher = {Curran Associates Inc.},
address = {Red Hook, NY, USA},
abstract = {Predicting calibrated confidence scores for multi-class deep networks is important for avoiding rare but costly mistakes. A common approach is to learn a post-hoc calibration function that transforms the output of the original network into calibrated confidence scores while maintaining the network's accuracy. However, previous post-hoc calibration techniques work only with simple calibration functions, potentially lacking sufficient representation to calibrate the complex function landscape of deep networks. In this work, we aim to learn general post-hoc calibration functions that can preserve the top-k predictions of any deep network. We call this family of functions intra order-preserving functions. We propose a new neural network architecture that represents a class of intra order-preserving functions by combining common neural network components. Additionally, we introduce order-invariant and diagonal sub-families, which can act as regulariza-tion for better generalization when the training data size is small. We show the effectiveness of the proposed method across a wide range of datasets and classifiers. Our method outperforms state-of-the-art post-hoc calibration methods, namely temperature scaling and Dirichlet calibration, in several evaluation metrics for the task.},
booktitle = {Proceedings of the 34th International Conference on Neural Information Processing Systems},
articleno = {1129},
numpages = {12},
location = {Vancouver, BC, Canada},
series = {NIPS '20}
}

@misc{wehenkel_unconstrained_2021,
	title = {Unconstrained {Monotonic} {Neural} {Networks}},
	url = {http://arxiv.org/abs/1908.05164},
	abstract = {Monotonic neural networks have recently been proposed as a way to define invertible transformations. These transformations can be combined into powerful autoregressive flows that have been shown to be universal approximators of continuous probability distributions. Architectures that ensure monotonicity typically enforce constraints on weights and activation functions, which enables invertibility but leads to a cap on the expressiveness of the resulting transformations. In this work, we propose the Unconstrained Monotonic Neural Network (UMNN) architecture based on the insight that a function is monotonic as long as its derivative is strictly positive. In particular, this latter condition can be enforced with a free-form neural network whose only constraint is the positiveness of its output. We evaluate our new invertible building block within a new autoregressive flow (UMNN-MAF) and demonstrate its effectiveness on density estimation experiments. We also illustrate the ability of UMNNs to improve variational inference.},
	urldate = {2024-10-28},
	publisher = {arXiv},
	author = {Wehenkel, Antoine and Louppe, Gilles},
	month = mar,
	year = {2021},
	note = {arXiv:1908.05164},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Neural and Evolutionary Computing},
	file = {Preprint PDF:C\:\\Users\\yunru\\Zotero\\storage\\LNP986XB\\Wehenkel and Louppe - 2021 - Unconstrained Monotonic Neural Networks.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\LWWEDIXB\\1908.html:text/html},
}

@inproceedings{jung_scaling_2023,
	title = {Scaling of {Class}-wise {Training} {Losses} for {Post}-hoc {Calibration}},
	url = {https://proceedings.mlr.press/v202/jung23a.html},
	abstract = {The class-wise training losses often diverge as a result of the various levels of intra-class and inter-class appearance variation, and we find that the diverging class-wise training losses cause the uncalibrated prediction with its reliability. To resolve the issue, we propose a new calibration method to synchronize the class-wise training losses. We design a new training loss to alleviate the variance of class-wise training losses by using multiple class-wise scaling factors. Since our framework can compensate the training losses of overfitted classes with those of under-fitted classes, the integrated training loss is preserved, preventing the performance drop even after the model calibration. Furthermore, our method can be easily employed in the post-hoc calibration methods, allowing us to use the pre-trained model as an initial model and reduce the additional computation for model calibration. We validate the proposed framework by employing it in the various post-hoc calibration methods, which generally improves calibration performance while preserving accuracy, and discover through the investigation that our approach performs well with unbalanced datasets and untuned hyperparameters.},
	language = {en},
	urldate = {2024-10-29},
	booktitle = {Proceedings of the 40th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Jung, Seungjin and Seo, Seungmo and Jeong, Yonghyun and Choi, Jongwon},
	month = jul,
	year = {2023},
	note = {ISSN: 2640-3498},
	pages = {15421--15434},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\UUYQSSCP\\Jung et al. - 2023 - Scaling of Class-wise Training Losses for Post-hoc Calibration.pdf:application/pdf},
}

@article{xiong_can_2024,
	title = {{CAN} {LLMS} {EXPRESS} {THEIR} {UNCERTAINTY}? {AN} {EMPIRICAL} {EVALUATION} {OF} {CONFIDENCE} {ELICI}- {TATION} {IN} {LLMS}},
	abstract = {Empowering large language models (LLMs) to accurately express confidence in their answers is essential for reliable and trustworthy decision-making. Previous confidence elicitation methods, which primarily rely on white-box access to internal model information or model fine-tuning, have become less suitable for LLMs, especially closed-source commercial APIs. This leads to a growing need to explore the untapped area of black-box approaches for LLM uncertainty estimation. To better break down the problem, we define a systematic framework with three components: prompting strategies for eliciting verbalized confidence, sampling methods for generating multiple responses, and aggregation techniques for computing consistency. We then benchmark these methods on two key tasks—confidence calibration and failure prediction—across five types of datasets (e.g., commonsense and arithmetic reasoning) and five widely-used LLMs including GPT-4 and LLaMA 2 Chat. Our analysis uncovers several key insights: 1) LLMs, when verbalizing their confidence, tend to be overconfident, potentially imitating human patterns of expressing confidence. 2) As model capability scales up, both calibration and failure prediction performance improve, yet still far from ideal performance. 3) Employing our proposed strategies, such as human-inspired prompts, consistency among multiple responses, and better aggregation strategies can help mitigate this overconfidence from various perspectives. 4) Comparisons with white-box methods indicate that while white-box methods perform better, the gap is narrow, e.g., 0.522 to 0.605 in AUROC. Despite these advancements, none of these techniques consistently outperform others, and all investigated methods struggle in challenging tasks, such as those requiring professional knowledge, indicating significant scope for improvement. We believe this study can serve as a strong baseline and provide insights for eliciting confidence in black-box LLMs. The code is publicly available at https://github.com/MiaoXiong2320/llm-uncertainty.},
	language = {en},
	author = {Xiong, Miao and Hu, Zhiyuan and Lu, Xinyang and Li, Yifei and Fu, Jie and He, Junxian and Hooi, Bryan},
	year = {2024},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\Z5HVR6S4\\Xiong et al. - 2024 - CAN LLMS EXPRESS THEIR UNCERTAINTY AN EMPIRICAL EVALUATION OF CONFIDENCE ELICI- TATION IN LLMS.pdf:application/pdf},
}

@misc{runje_constrained_2023-1,
	title = {Constrained {Monotonic} {Neural} {Networks}},
	url = {http://arxiv.org/abs/2205.11775},
	abstract = {Wider adoption of neural networks in many critical domains such as finance and healthcare is being hindered by the need to explain their predictions and to impose additional constraints on them. Monotonicity constraint is one of the most requested properties in real-world scenarios and is the focus of this paper. One of the oldest ways to construct a monotonic fully connected neural network is to constrain signs on its weights. Unfortunately, this construction does not work with popular non-saturated activation functions as it can only approximate convex functions. We show this shortcoming can be fixed by constructing two additional activation functions from a typical unsaturated monotonic activation function and employing each of them on the part of neurons. Our experiments show this approach of building monotonic neural networks has better accuracy when compared to other state-of-the-art methods, while being the simplest one in the sense of having the least number of parameters, and not requiring any modifications to the learning procedure or post-learning steps. Finally, we prove it can approximate any continuous monotone function on a compact subset of \${\textbackslash}mathbb\{R\}{\textasciicircum}n\$.},
	urldate = {2024-11-04},
	publisher = {arXiv},
	author = {Runje, Davor and Shankaranarayana, Sharath M.},
	month = may,
	year = {2023},
	note = {arXiv:2205.11775},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
	file = {Preprint PDF:C\:\\Users\\yunru\\Zotero\\storage\\NPLU2DBH\\Runje and Shankaranarayana - 2023 - Constrained Monotonic Neural Networks.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\ZMZK5TVC\\2205.html:text/html},
}

@inproceedings{luo_local_2022,
	title = {Local calibration: metrics and recalibration},
	shorttitle = {Local calibration},
	url = {https://proceedings.mlr.press/v180/luo22a.html},
	abstract = {Probabilistic classifiers output confidence scores along with their predictions, and these confidence scores should be calibrated, i.e., they should reflect the reliability of the prediction. Confidence scores that minimize standard metrics such as the expected calibration error (ECE) accurately measure the reliability {\textbackslash}textit\{on average\} across the entire population. However, it is in general impossible to measure the reliability of an {\textbackslash}textit\{individual\} prediction. In this work, we propose the local calibration error (LCE) to span the gap between average and individual reliability. For each individual prediction, the LCE measures the average reliability of a set of similar predictions, where similarity is quantified by a kernel function on a pretrained feature space and by a binning scheme over predicted model confidences. We show theoretically that the LCE can be estimated sample-efficiently from data, and empirically find that it reveals miscalibration modes that are more fine-grained than the ECE can detect. Our key result is a novel \{{\textbackslash}textbf\{lo\}cal {\textbackslash}textbf\{re\}calibration\} method {\textbackslash}method\{\}, to improve confidence scores for individual predictions and decrease the LCE. Experimentally, we show that our recalibration method produces more accurate confidence scores, which improves downstream fairness and decision making on classification tasks with both image and tabular data.},
	language = {en},
	urldate = {2024-11-28},
	booktitle = {Proceedings of the {Thirty}-{Eighth} {Conference} on {Uncertainty} in {Artificial} {Intelligence}},
	publisher = {PMLR},
	author = {Luo, Rachel and Bhatnagar, Aadyot and Bai, Yu and Zhao, Shengjia and Wang, Huan and Xiong, Caiming and Savarese, Silvio and Ermon, Stefano and Schmerling, Edward and Pavone, Marco},
	month = aug,
	year = {2022},
	note = {ISSN: 2640-3498},
	pages = {1286--1295},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\4R23NKWX\\Luo et al. - 2022 - Local calibration metrics and recalibration.pdf:application/pdf;Supplementary PDF:C\:\\Users\\yunru\\Zotero\\storage\\IKUNJU85\\Luo et al. - 2022 - Local calibration metrics and recalibration.pdf:application/pdf},
}

@article{coz_confidence_nodate,
	title = {Confidence {Calibration} of {Classifiers} with {Many} {Classes}},
	abstract = {For classification models based on neural networks, the maximum predicted class probability is often used as a confidence score. This score rarely predicts well the probability of making a correct prediction and requires a post-processing calibration step. However, many confidence calibration methods fail for problems with many classes. To address this issue, we transform the problem of calibrating a multiclass classifier into calibrating a single surrogate binary classifier. This approach allows for more efficient use of standard calibration methods. We evaluate our approach on numerous neural networks used for image or text classification and show that it significantly enhances existing calibration methods. Our code can be accessed at the following link: https://github.com/allglc/tva-calibration.},
	language = {en},
	author = {Coz, Adrien Le and Herbin, Stéphane and Adjed, Faouzi},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\U76FY9HM\\Coz et al. - Confidence Calibration of Classifiers with Many Classes.pdf:application/pdf},
}

@article{zhang_pid_nodate,
	title = {A {PID} {Controller} {Approach} for {Adaptive} {Probability}-dependent {Gradient} {Decay} in {Model} {Calibration}},
	abstract = {Modern deep learning models often exhibit overconfident predictions, inadequately capturing uncertainty. During model optimization, the expected calibration error tends to overfit earlier than classification accuracy, indicating distinct optimization objectives for classification error and calibration error. To ensure consistent optimization of both model accuracy and model calibration, we propose a novel method incorporating a probability-dependent gradient decay coefficient into loss function. This coefficient exhibits a strong correlation with the overall confidence level. To maintain model calibration during optimization, we utilize a proportionalintegral-derivative (PID) controller to dynamically adjust this gradient decay rate, where the adjustment relies on the proposed relative calibration error feedback in each epoch, thereby preventing the model from exhibiting over-confidence or under-confidence. Within the PID control system framework, the proposed relative calibration error serves as the control system output, providing an indication of the overall confidence level, while the gradient decay rate functions as the controlled variable. Moreover, recognizing the impact of gradient amplitude of adaptive decay rates, we implement an adaptive learning rate mechanism for gradient compensation to prevent inadequate learning of over-small or over-large gradient. Empirical experiments validate the efficacy of our PID-based adaptive gradient decay rate approach, ensuring consistent optimization of model calibration and model accuracy. The code of implementation is available in https://github.com/UHIF/PID\_AGD.},
	language = {en},
	author = {Zhang, Siyuan and Xie, Linbo},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\WIIQ38PM\\Zhang and Xie - A PID Controller Approach for Adaptive Probability-dependent Gradient Decay in Model Calibration.pdf:application/pdf},
}

@article{hansen_when_nodate,
	title = {When is {Multicalibration} {Post}-{Processing} {Necessary}?},
	abstract = {Calibration is a well-studied property of predictors which guarantees meaningful uncertainty estimates. Multicalibration is a related notion — originating in algorithmic fairness — which requires predictors to be simultaneously calibrated over a potentially complex and overlapping collection of protected subpopulations (such as groups defined by ethnicity, race, or income). We conduct the first comprehensive study evaluating the usefulness of multicalibration post-processing across a broad set of tabular, image, and language datasets for models spanning from simple decision trees to 90 million parameter fine-tuned LLMs. Our findings can be summarized as follows: (1) models which are calibrated out of the box tend to be relatively multicalibrated without any additional post-processing; (2) multicalibration post-processing can help inherently uncalibrated models and also large vision and language models; and (3) traditional calibration measures may sometimes provide multicalibration implicitly. More generally, we distill many independent observations which may be useful for practical and effective applications of multicalibration post-processing in real-world contexts. We also release a python package implementing multicalibration algorithms, available via ‘pip install multicalibration’.},
	language = {en},
	author = {Hansen, Dutch and Devic, Siddartha and Nakkiran, Preetum and Sharan, Vatsal},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\ZQ2ZRQXU\\Hansen et al. - When is Multicalibration Post-Processing Necessary.pdf:application/pdf},
}

@misc{ma_meta-cal_2021,
	title = {Meta-{Cal}: {Well}-controlled {Post}-hoc {Calibration} by {Ranking}},
	shorttitle = {Meta-{Cal}},
	url = {http://arxiv.org/abs/2105.04290},
	doi = {10.48550/arXiv.2105.04290},
	abstract = {In many applications, it is desirable that a classifier not only makes accurate predictions, but also outputs calibrated posterior probabilities. However, many existing classifiers, especially deep neural network classifiers, tend to be uncalibrated. Post-hoc calibration is a technique to recalibrate a model by learning a calibration map. Existing approaches mostly focus on constructing calibration maps with low calibration errors, however, this quality is inadequate for a calibrator being useful. In this paper, we introduce two constraints that are worth consideration in designing a calibration map for post-hoc calibration. Then we present Meta-Cal, which is built from a base calibrator and a ranking model. Under some mild assumptions, two high-probability bounds are given with respect to these constraints. Empirical results on CIFAR-10, CIFAR-100 and ImageNet and a range of popular network architectures show our proposed method significantly outperforms the current state of the art for post-hoc multi-class classification calibration.},
	urldate = {2025-01-07},
	publisher = {arXiv},
	author = {Ma, Xingchen and Blaschko, Matthew B.},
	month = jun,
	year = {2021},
	note = {arXiv:2105.04290 [stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {Preprint PDF:C\:\\Users\\yunru\\Zotero\\storage\\WY8MLG83\\Ma and Blaschko - 2021 - Meta-Cal Well-controlled Post-hoc Calibration by Ranking.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\LYVRDA3K\\2105.html:text/html},
}

@misc{prieto_grokking_2025,
	title = {Grokking at the {Edge} of {Numerical} {Stability}},
	url = {http://arxiv.org/abs/2501.04697},
	doi = {10.48550/arXiv.2501.04697},
	abstract = {Grokking, the sudden generalization that occurs after prolonged overfitting, is a surprising phenomenon challenging our understanding of deep learning. Although significant progress has been made in understanding grokking, the reasons behind the delayed generalization and its dependence on regularization remain unclear. In this work, we argue that without regularization, grokking tasks push models to the edge of numerical stability, introducing floating point errors in the Softmax function, which we refer to as Softmax Collapse (SC). We demonstrate that SC prevents grokking and that mitigating SC enables grokking without regularization. Investigating the root cause of SC, we find that beyond the point of overfitting, the gradients strongly align with what we call the na{\textbackslash}"ive loss minimization (NLM) direction. This component of the gradient does not alter the model's predictions but decreases the loss by scaling the logits, typically by scaling the weights along their current direction. We show that this scaling of the logits explains the delay in generalization characteristic of grokking and eventually leads to SC, halting further learning. To validate our hypotheses, we introduce two key contributions that address the challenges in grokking tasks: StableMax, a new activation function that prevents SC and enables grokking without regularization, and \${\textbackslash}perp\$Grad, a training algorithm that promotes quick generalization in grokking tasks by preventing NLM altogether. These contributions provide new insights into grokking, elucidating its delayed generalization, reliance on regularization, and the effectiveness of existing grokking-inducing methods. Code for this paper is available at https://github.com/LucasPrietoAl/grokking-at-the-edge-of-numerical-stability.},
	urldate = {2025-01-19},
	publisher = {arXiv},
	author = {Prieto, Lucas and Barsbey, Melih and Mediano, Pedro A. M. and Birdal, Tolga},
	month = jan,
	year = {2025},
	note = {arXiv:2501.04697 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {Preprint PDF:C\:\\Users\\yunru\\Zotero\\storage\\DKBHPSSI\\Prieto et al. - 2025 - Grokking at the Edge of Numerical Stability.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\LCRNAFR7\\2501.html:text/html},
}




@inproceedings{maletzke_accurately_2021,
	title = {Accurately {Quantifying} under {Score} {Variability}},
	doi = {10.1109/ICDM51629.2021.00149},
	abstract = {The quantification objective is to predict the class distribution of a data sample. Therefore, this task intrinsically involves a drift in the class distribution that causes a mismatch between the training and test sets. However, existing quantification approaches assume that the feature distribution is stationary. We analyse for the first time how score-based quantifiers are affected by concept drifts and propose a novel drift-resilient quantifier for binary classes. Our proposal does not model the different types of concept drifts. Instead, we model the changes that such changes cause in the classification scores. This observation simplifies our analysis since distribution changes can only increase, decrease or maintain the overlap of the positive and negative classes in a rank induced by the scores. Our paper has two main contributions. The first one is MoSS, a model for synthetic scores. We use this model to show that state-of-the-art quantifiers underperform in the occurrence of any concept drift that changes the score distribution. Our second contribution is a quantifier, DySyn, that uses MoSS to estimate the class distribution. We show that DySyn statistically outperforms state-of-the-art quantifiers in a comprehensive comparison with real-world and benchmark datasets in the presence of concept drifts.},
	booktitle = {2021 {IEEE} {International} {Conference} on {Data} {Mining} ({ICDM})},
	author = {Maletzke, André and Reis, Denis Dos and Hassan, Waqar and Batista, Gustavo},
	month = dec,
	year = {2021},
	note = {ISSN: 2374-8486},
	keywords = {Data mining, Proposals, Benchmark testing, concept drift, Conferences, Counting, machine learning, Reliability, Training, Training data},
	pages = {1228--1233},
	file = {ICDM21_extended.pdf:C\:\\Users\\yunru\\Zotero\\storage\\ED99XE62\\ICDM21_extended.pdf:application/pdf;IEEE Xplore Abstract Record:C\:\\Users\\yunru\\Zotero\\storage\\S3W6J3U8\\9679104.html:text/html;IEEE Xplore Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\NQRFUQ9H\\Maletzke et al. - 2021 - Accurately Quantifying under Score Variability.pdf:application/pdf},
}

@article{maletzke_dys_2019,
	title = {{DyS}: {A} {Framework} for {Mixture} {Models} in {Quantification}},
	volume = {33},
	shorttitle = {{DyS}},
	doi = {10.1609/aaai.v33i01.33014552},
	abstract = {Quantification is an expanding research topic in Machine Learning literature. While in classification we are interested in obtaining the class of individual observations, in quantification we want to estimate the total number of instances that belong to each class. This subtle difference allows the development of several algorithms that incur smaller and more consistent errors than counting the classes issued by a classifier. Among such new quantification methods, one particular family stands out due to its accuracy, simplicity, and ability to operate with imbalanced training samples: Mixture Models (MM). Despite these desirable traits, MM, as a class of algorithms, lacks a more in-depth understanding concerning the influence of internal parameters on its performance. In this paper, we generalize MM with a base framework called DyS: Distribution y-Similarity. With this framework, we perform a thorough evaluation of the most critical design decisions of MM models. For instance, we assess 15 dissimilarity functions to compare histograms with varying numbers of bins from 2 to 110 and, for the first time, make a connection between quantification accuracy and test sample size, with experiments covering 24 public benchmark datasets. We conclude that, when tuned, Topsøe is the histogram distance function that consistently leads to smaller quantification errors and, therefore, is recommended to general use, notwithstanding Hellinger Distance’s popularity. To rid MM models of the dependency on a choice for the number of histogram bins, we introduce two dissimilarity functions that can operate directly on observations. We show that SORD, one of such measures, presents performance that is slightly inferior to the tuned Topsøe, while not requiring the sensible parameterization of the number of bins.},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
	author = {Maletzke, André and Reis, Denis and Cherman, Everton and Batista, Gustavo},
	month = jul,
	year = {2019},
	pages = {4552--4560},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\K6FZYDCH\\Maletzke et al. - 2019 - DyS A Framework for Mixture Models in Quantificat.pdf:application/pdf},
}

@inproceedings{abushaqra_piets_2021,
	title = {{PIETS}: {Parallelised} {Irregularity} {Encoders} for {Forecasting} with {Heterogeneous} {Time}-{Series}},
	shorttitle = {{PIETS}},
	doi = {10.1109/ICDM51629.2021.00109},
	abstract = {Heterogeneity and irregularity of multi-source data sets present a significant challenge to time-series analysis. In the literature, the fusion of multi-source time-series has been achieved either by using ensemble learning models which ignore temporal patterns and correlation within features or by defining a fixed-size window to select specific parts of the data sets. On the other hand, many studies have shown major improvement to handle the irregularity of time-series, yet none of these studies has been applied to multi-source data. In this work, we design a novel architecture, PIETS, to model heterogeneous time-series. PIETS has the following characteristics: (1) irregularity encoders for multi-source samples that can leverage all available information and accelerate the convergence of the model; (2) parallelised neural networks to enable flexibility and avoid information overwhelming; and (3) attention mechanism that highlights different information and gives high importance to the most related data. Through extensive experiments on real-world data sets related to COVID-19, we show that the proposed architecture is able to effectively model heterogeneous temporal data and outperforms other state-of-the-art approaches in the prediction task.},
	booktitle = {2021 {IEEE} {International} {Conference} on {Data} {Mining} ({ICDM})},
	author = {Abushaqra, Futoon M. and Xue, Hao and Ren, Yongli and Salim, Flora D.},
	month = dec,
	year = {2021},
	note = {ISSN: 2374-8486},
	keywords = {Predictive models, Training, Analytical models, Correlation, COVID-19, Data models, heterogeneous data sources, irregular time-series, sequences modeling, temporal data analysis., Time series analysis},
	pages = {976--981},
	file = {IEEE Xplore Abstract Record:C\:\\Users\\yunru\\Zotero\\storage\\QQ6TTNBE\\references.html:text/html;IEEE Xplore Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\AYMIFS2J\\Abushaqra et al. - 2021 - PIETS Parallelised Irregularity Encoders for Fore.pdf:application/pdf},
}

@inproceedings{qi_framework_2021,
	address = {Cham},
	series = {Lecture {Notes} in {Computer} {Science}},
	title = {A {Framework} for {Deep} {Quantification} {Learning}},
	isbn = {978-3-030-67658-2},
	doi = {10.1007/978-3-030-67658-2_14},
	abstract = {A quantification learning task estimates class ratios or class distribution given a test set. Quantification learning is useful for a variety of application domains such as commerce, public health, and politics. For instance, it is desirable to automatically estimate the proportion of customer satisfaction in different aspects from product reviews to improve customer relationships. We formulate the quantification learning problem as a maximum likelihood problem and propose the first end-to-end Deep Quantification Network (DQN) framework. DQN jointly learns quantification feature representations and directly predicts the class distribution. Compared to classification-based quantification methods, DQN avoids three separate steps: classification of individual instances, calculation of the predicted class ratios, and class ratio adjustment to account for classification errors. We evaluated DQN on four public datasets, ranging from movie and product reviews to multi-class news. We compared DQN against six existing quantification methods and conducted a sensitivity analysis of DQN performance. Compared to the best existing method in our study, (1) DQN reduces Mean Absolute Error (MAE) by about 35\%. (2) DQN uses around 40\% less training samples to achieve a comparable MAE.},
	language = {en},
	booktitle = {Machine {Learning} and {Knowledge} {Discovery} in {Databases}},
	publisher = {Springer International Publishing},
	author = {Qi, Lei and Khaleel, Mohammed and Tavanapong, Wallapak and Sukul, Adisak and Peterson, David},
	editor = {Hutter, Frank and Kersting, Kristian and Lijffijt, Jefrey and Valera, Isabel},
	year = {2021},
	keywords = {Deep learning, Class distribution estimate, Quantification learning},
	pages = {232--248},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\EN74FFPQ\\Qi et al. - 2021 - A Framework for Deep Quantification Learning.pdf:application/pdf;González et al. - 2017 - A Review on Quantification Learning.pdf:C\:\\Users\\yunru\\Zotero\\storage\\RD5YUQKM\\González et al. - 2017 - A Review on Quantification Learning.pdf:application/pdf},
}

@article{gonzalez-castro_class_2013,
	title = {Class distribution estimation based on the {Hellinger} distance},
	volume = {218},
	issn = {0020-0255},
	url = {https://www.sciencedirect.com/science/article/pii/S0020025512004069},
	doi = {10.1016/j.ins.2012.05.028},
	abstract = {Class distribution estimation (quantification) plays an important role in many practical classification problems. Firstly, it is important in order to adapt the classifier to the operational conditions when they differ from those assumed in learning. Additionally, there are some real domains where the quantification task is itself valuable due to the high variability of the class prior probabilities. Our novel quantification approach for two-class problems is based on distributional divergence measures. The mismatch between the test data distribution and validation distributions generated in a fully controlled way is measured by the Hellinger distance in order to estimate the prior probability that minimizes this divergence. Experimental results on several binary classification problems show the benefits of this approach when compared to such approaches as counting the predicted class labels and other methods based on the classifier confusion matrix or on posterior probability estimations. We also illustrate these techniques as well as their robustness against the base classifier performance (a neural network) with a boar semen quality control setting. Empirical results show that the quantification can be conducted with a mean absolute error lower than 0.008, which seems very promising in this field.},
	language = {en},
	urldate = {2022-11-17},
	journal = {Information Sciences},
	author = {González-Castro, Víctor and Alaiz-Rodríguez, Rocío and Alegre, Enrique},
	month = jan,
	year = {2013},
	keywords = {Class distribution shift, Class prior probability estimation, Hellinger distance, Quantification},
	pages = {146--164},
	file = {ScienceDirect Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\ZHZVS2AN\\González-Castro et al. - 2013 - Class distribution estimation based on the Helling.pdf:application/pdf;ScienceDirect Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\AGDNPFWZ\\S0020025512004069.html:text/html},
}

@article{gonzalez_review_2017,
	title = {A {Review} on {Quantification} {Learning}},
	volume = {50},
	issn = {0360-0300},
	url = {https://doi.org/10.1145/3117807},
	doi = {10.1145/3117807},
	abstract = {The task of quantification consists in providing an aggregate estimation (e.g., the class distribution in a classification problem) for unseen test sets, applying a model that is trained using a training set with a different data distribution. Several real-world applications demand this kind of method that does not require predictions for individual examples and just focuses on obtaining accurate estimates at an aggregate level. During the past few years, several quantification methods have been proposed from different perspectives and with different goals. This article presents a unified review of the main approaches with the aim of serving as an introductory tutorial for newcomers in the field.},
	number = {5},
	urldate = {2023-02-20},
	journal = {ACM Comput. Surv.},
	author = {González, Pablo and Castaño, Alberto and Chawla, Nitesh V. and Coz, Juan José Del},
	month = sep,
	year = {2017},
	keywords = {Class distribution estimation, prevalence estimation, quantification},
	pages = {74:1--74:40},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\DGHXA44L\\González et al. - 2017 - A Review on Quantification Learning.pdf:application/pdf},
}

@inproceedings{esuli_recurrent_2018,
	address = {New York, NY, USA},
	series = {{CIKM} '18},
	title = {A {Recurrent} {Neural} {Network} for {Sentiment} {Quantification}},
	isbn = {978-1-4503-6014-2},
	url = {https://doi.org/10.1145/3269206.3269287},
	doi = {10.1145/3269206.3269287},
	abstract = {Quantification is a supervised learning task that consists in predicting, given a set of classes C and a set D of unlabelled items, the prevalence (or relative frequency) p\_c(D) of each class c{\textbackslash}in{\textbackslash}mathcalC in D. Quantification can in principle be solved by classifying all the unlabelled items and counting how many of them have been attributed to each class. However, this "classify and count" approach has been shown to yield suboptimal quantification accuracy; this has established quantification as a task of its own, and given rise to a number of methods specifically devised for it. We propose a recurrent neural network architecture for quantification (that we call QuaNet) that observes the classification predictions to learn higher-order "quantification embeddings", which are then refined by incorporating quantification predictions of simple classify-and-count-like methods. We test QuaNet on sentiment quantification on text, showing that it substantially outperforms several state-of-the-art baselines.},
	urldate = {2023-02-21},
	booktitle = {Proceedings of the 27th {ACM} {International} {Conference} on {Information} and {Knowledge} {Management}},
	publisher = {Association for Computing Machinery},
	author = {Esuli, Andrea and Moreo Fernández, Alejandro and Sebastiani, Fabrizio},
	month = oct,
	year = {2018},
	keywords = {deep learning, quantification, neural networks, opinion mining, sentiment analysis},
	pages = {1775--1778},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\PV4BEBDC\\Esuli et al. - 2018 - A Recurrent Neural Network for Sentiment Quantific.pdf:application/pdf},
}

@inproceedings{tachet_des_combes_domain_2020,
	title = {Domain {Adaptation} with {Conditional} {Distribution} {Matching} and {Generalized} {Label} {Shift}},
	volume = {33},
	url = {https://proceedings.neurips.cc/paper/2020/hash/dfbfa7ddcfffeb581f50edcf9a0204bb-Abstract.html},
	urldate = {2023-03-05},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Tachet des Combes, Remi and Zhao, Han and Wang, Yu-Xiang and Gordon, Geoffrey J},
	year = {2020},
	pages = {19276--19289},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\DMRXWTTZ\\Tachet des Combes et al. - 2020 - Domain Adaptation with Conditional Distribution Ma.pdf:application/pdf},
}

@article{saerens_adjusting_2002,
	title = {Adjusting the {Outputs} of a {Classifier} to {New} a {Priori} {Probabilities}: {A} {Simple} {Procedure}},
	volume = {14},
	issn = {0899-7667},
	shorttitle = {Adjusting the {Outputs} of a {Classifier} to {New} a {Priori} {Probabilities}},
	url = {https://doi.org/10.1162/089976602753284446},
	doi = {10.1162/089976602753284446},
	abstract = {It sometimes happens (for instance in case control studies) that a classifier is trained on a data set that does not reflect the true a priori probabilities of the target classes on real-world data. This may have a negative effect on the classification accuracy obtained on the real-world data set, especially when the classifier's decisions are based on the a posteriori probabilities of class membership. Indeed, in this case, the trained classifier provides estimates of the a posteriori probabilities that are not valid for this real-world data set (they rely on the a priori probabilities of the training set). Applying the classifier as is (without correcting its outputs with respect to these new conditions) on this new data set may thus be suboptimal. In this note, we present a simple iterative procedure for adjusting the outputs of the trained classifier with respect to these new a priori probabilities without having to refit the model, even when these probabilities are not known in advance. As a by-product, estimates of the new a priori probabilities are also obtained. This iterative algorithm is a straightforward instance of the expectation-maximization (EM) algorithm and is shown to maximize the likelihood of the new data. Thereafter, we discuss a statistical test that can be applied to decide if the a priori class probabilities have changed from the training set to the real-world data. The procedure is illustrated on different classification problems involving a multilayer neural network, and comparisons with a standard procedure for a priori probability estimation are provided. Our original method, based on the EM algorithm, is shown to be superior to the standard one for a priori probability estimation. Experimental results also indicate that the classifier with adjusted outputs always performs better than the original one in terms of classification accuracy, when the a priori probability conditions differ from the training set to the real-world data. The gain in classification accuracy can be significant.},
	number = {1},
	urldate = {2023-03-05},
	journal = {Neural Computation},
	author = {Saerens, Marco and Latinne, Patrice and Decaestecker, Christine},
	month = jan,
	year = {2002},
	pages = {21--41},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\A6Q5XVMU\\Saerens et al. - 2002 - Adjusting the Outputs of a Classifier to New a Pri.pdf:application/pdf},
}

@inproceedings{maletzke_quantification_2017,
	title = {Quantification in {Data} {Streams}: {Initial} {Results}},
	shorttitle = {Quantification in {Data} {Streams}},
	doi = {10.1109/BRACIS.2017.74},
	abstract = {In the last decades, learning from data streams has attracted the attention of researchers and practitioners due to its large number of applications. These applications have motivated the research community to propose a significant number of methods that can be used to solve problems in diverse tasks, more prominently in classification, prediction, and clustering. However, a relevant task known as quantification has remained largely unexplored. The quantification goal is to provide an estimate of the class prevalence in an unlabeled set. In this paper, we discuss the relevance and challenges of quantification for data streams and compare how it differs from the batch setting, in which quantification has attracted more attention from the research community. We propose an algorithm to estimate the class distribution in a data stream and frame our algorithm in the active learning framework. In addition, we define two other approaches as baseline and topline strategies for this problem. The experimental results demonstrate that our algorithm has significantly higher quantification accuracy than the baseline and almost as large as the topline while requiring a fraction of the true labels requested by the latter approach.},
	booktitle = {2017 {Brazilian} {Conference} on {Intelligent} {Systems} ({BRACIS})},
	author = {Maletzke, André G. and dos Reis, Denis M. and Batista, Gustavo E.A.P.A.},
	month = oct,
	year = {2017},
	keywords = {Data mining, Estimation, concept drift, Training, quantification, data stream, Insects, Mathematical model, Surveillance, verification latency},
	pages = {43--48},
	file = {IEEE Xplore Abstract Record:C\:\\Users\\yunru\\Zotero\\storage\\RCGSZVUN\\8247027.html:text/html;IEEE Xplore Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\PBJUT6H9\\Maletzke et al. - 2017 - Quantification in Data Streams Initial Results.pdf:application/pdf},
}

@inproceedings{dos_reis_fast_2016,
	address = {New York, NY, USA},
	series = {{KDD} '16},
	title = {Fast {Unsupervised} {Online} {Drift} {Detection} {Using} {Incremental} {Kolmogorov}-{Smirnov} {Test}},
	isbn = {978-1-4503-4232-2},
	url = {https://doi.org/10.1145/2939672.2939836},
	doi = {10.1145/2939672.2939836},
	abstract = {Data stream research has grown rapidly over the last decade. Two major features distinguish data stream from batch learning: stream data are generated on the fly, possibly in a fast and variable rate; and the underlying data distribution can be non-stationary, leading to a phenomenon known as concept drift. Therefore, most of the research on data stream classification focuses on proposing efficient models that can adapt to concept drifts and maintain a stable performance over time. However, specifically for the classification task, the majority of such methods rely on the instantaneous availability of true labels for all already classified instances. This is a strong assumption that is rarely fulfilled in practical applications. Hence there is a clear need for efficient methods that can detect concept drifts in an unsupervised way. One possibility is the well-known Kolmogorov-Smirnov test, a statistical hypothesis test that checks whether two samples differ. This work has two main contributions. The first one is the Incremental Kolmogorov-Smirnov algorithm that allows performing the Kolmogorov-Smirnov hypothesis test instantly using two samples that change over time, where the change is an insertion and/or removal of an observation. Our algorithm employs a randomized tree and is able to perform the insertion and removal operations in O(log N) with high probability and calculate the Kolmogorov-Smirnov test in O(1), where N is the number of sample observations. This is a significant speed-up compared to the O(N log N) cost of the non-incremental implementation. The second contribution is the use of the Incremental Kolmogorov-Smirnov test to detect concept drifts without true labels. Classification algorithms adapted to use the test rely on a limited portion of those labels just to update the classification model after a concept drift is detected.},
	urldate = {2023-03-14},
	booktitle = {Proceedings of the 22nd {ACM} {SIGKDD} {International} {Conference} on {Knowledge} {Discovery} and {Data} {Mining}},
	publisher = {Association for Computing Machinery},
	author = {dos Reis, Denis Moreira and Flach, Peter and Matwin, Stan and Batista, Gustavo},
	month = aug,
	year = {2016},
	keywords = {concept drift, data stream, cartesian tree, kolmogorov-smirnov, lazy propagation, treap},
	pages = {1545--1554},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\JGMC6K38\\dos Reis et al. - 2016 - Fast Unsupervised Online Drift Detection Using Inc.pdf:application/pdf},
}

@inproceedings{souza_data_2015,
	title = {Data {Stream} {Classification} {Guided} by {Clustering} on {Nonstationary} {Environments} and {Extreme} {Verification} {Latency}},
	isbn = {978-1-61197-401-0},
	url = {https://epubs.siam.org/doi/10.1137/1.9781611974010.98},
	doi = {10.1137/1.9781611974010.98},
	language = {en},
	urldate = {2023-03-15},
	booktitle = {Proceedings of the 2015 {SIAM} {International} {Conference} on {Data} {Mining}},
	publisher = {Society for Industrial and Applied Mathematics},
	author = {Souza, Vinícius M. A. and Silva, Diego F. and Gama, João and Batista, Gustavo E. A. P. A.},
	month = jun,
	year = {2015},
	pages = {873--881},
	file = {Submitted Version:C\:\\Users\\yunru\\Zotero\\storage\\NFI4INFL\\Souza et al. - 2015 - Data Stream Classification Guided by Clustering on.pdf:application/pdf},
}

@incollection{bifet_learning_2007,
	series = {Proceedings},
	title = {Learning from {Time}-{Changing} {Data} with {Adaptive} {Windowing}},
	isbn = {978-0-89871-630-6},
	url = {https://epubs.siam.org/doi/abs/10.1137/1.9781611972771.42},
	abstract = {We present a new approach for dealing with distribution change and concept drift when learning from data sequences that may vary with time. We use sliding windows whose size, instead of being fixed a priori, is recomputed online according to the rate of change observed from the data in the window itself. This delivers the user or programmer from having to guess a time-scale for change. Contrary to many related works, we provide rigorous guarantees of performance, as bounds on the rates of false positives and false negatives.
Using ideas from data stream algorithmics, we develop a time- and memory-efficient version of this algorithm, called ADWIN2. We show how to combine ADWIN2 with the Naïve Bayes (NB) predictor, in two ways: one, using it to monitor the error rate of the current model and declare when revision is necessary and, two, putting it inside the NB predictor to maintain up-to-date estimations of conditional probabilities in the data. We test our approach using synthetic and real data streams and compare them to both fixed-size and variable-size window strategies with good results.
MSC codes
Data Streams
Time-Changing Data
Concept and Distribution Drift
Naïve Bayes},
	urldate = {2023-04-03},
	booktitle = {Proceedings of the 2007 {SIAM} {International} {Conference} on {Data} {Mining} ({SDM})},
	publisher = {Society for Industrial and Applied Mathematics},
	author = {Bifet, Albert and Gavaldà, Ricard},
	month = apr,
	year = {2007},
	doi = {10.1137/1.9781611972771.42},
	pages = {443--448},
	file = {adwin06.pdf:C\:\\Users\\yunru\\Zotero\\storage\\A4WLBDCZ\\adwin06.pdf:application/pdf;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\QKPE8DFH\\Bifet and Gavaldà - 2007 - Learning from Time-Changing Data with Adaptive Win.pdf:application/pdf},
}

@article{gonzalez_why_2017,
	title = {Why is quantification an interesting learning problem?},
	volume = {6},
	issn = {2192-6360},
	url = {https://doi.org/10.1007/s13748-016-0103-3},
	doi = {10.1007/s13748-016-0103-3},
	abstract = {There are real applications that do not demand to classify or to make predictions about individual objects, but to estimate some magnitude about a group of them. For instance, one of these cases happens in sentiment analysis and opinion mining. Some applications require to classify opinions as positives or negatives, but there are also others, even more useful sometimes, that just need an estimation of which is the proportion of each class during a concrete period of time. “How many tweets about our new product were positive yesterday?” Practitioners should apply quantification algorithms to tackle this kind of problems, instead of just using off-the-shelf classification methods, because classifiers are suboptimal in the context of quantification tasks. Unfortunately, quantification learning is still relatively an under explored area in machine learning. The goal of this paper is to show that quantification learning is an interesting open problem. To support its benefits, we shall show an application to analyze Twitter comments in which even the most simple quantification methods outperform classification approaches.},
	language = {en},
	number = {1},
	urldate = {2023-07-06},
	journal = {Prog Artif Intell},
	author = {González, Pablo and Díez, Jorge and Chawla, Nitesh and del Coz, Juan José},
	month = mar,
	year = {2017},
	keywords = {Quantification, Opinion mining, Population shift, Prevalence estimation, Sentiment analysis},
	pages = {53--58},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\SYRINKBN\\González et al. - 2017 - Why is quantification an interesting learning prob.pdf:application/pdf},
}

@article{hopkins_method_2010,
	title = {A {Method} of {Automated} {Nonparametric} {Content} {Analysis} for {Social} {Science}},
	volume = {54},
	copyright = {©2010, Midwest Political Science Association},
	issn = {1540-5907},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/j.1540-5907.2009.00428.x},
	doi = {10.1111/j.1540-5907.2009.00428.x},
	abstract = {The increasing availability of digitized text presents enormous opportunities for social scientists. Yet hand coding many blogs, speeches, government records, newspapers, or other sources of unstructured text is infeasible. Although computer scientists have methods for automated content analysis, most are optimized to classify individual documents, whereas social scientists instead want generalizations about the population of documents, such as the proportion in a given category. Unfortunately, even a method with a high percent of individual documents correctly classified can be hugely biased when estimating category proportions. By directly optimizing for this social science goal, we develop a method that gives approximately unbiased estimates of category proportions even when the optimal classifier performs poorly. We illustrate with diverse data sets, including the daily expressed opinions of thousands of people about the U.S. presidency. We also make available software that implements our methods and large corpora of text for further analysis.},
	language = {en},
	number = {1},
	urldate = {2023-07-07},
	journal = {American Journal of Political Science},
	author = {Hopkins, Daniel J. and King, Gary},
	year = {2010},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/j.1540-5907.2009.00428.x},
	pages = {229--247},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\NICIRIL3\\Hopkins and King - 2010 - A Method of Automated Nonparametric Content Analys.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\RFIPDR8N\\j.1540-5907.2009.00428.html:text/html},
}

@misc{yusuf_differentiable_2020,
	title = {Differentiable {Histogram} with {Hard}-{Binning}},
	url = {http://arxiv.org/abs/2012.06311},
	doi = {10.48550/arXiv.2012.06311},
	abstract = {The simplicity and expressiveness of a histogram render it a useful feature in different contexts including deep learning. Although the process of computing a histogram is non-differentiable, researchers have proposed differentiable approximations, which have some limitations. A differentiable histogram that directly approximates the hard-binning operation in conventional histograms is proposed. It combines the strength of existing differentiable histograms and overcomes their individual challenges. In comparison to a histogram computed using Numpy, the proposed histogram has an absolute approximation error of 0.000158.},
	urldate = {2023-07-10},
	publisher = {arXiv},
	author = {Yusuf, Ibrahim and Igwegbe, George and Azeez, Oluwafemi},
	month = nov,
	year = {2020},
	note = {arXiv:2012.06311 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:C\:\\Users\\yunru\\Zotero\\storage\\5X6LW5E8\\Yusuf et al. - 2020 - Differentiable Histogram with Hard-Binning.pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\9N7EFVDU\\2012.html:text/html},
}

@article{sadeghi_histnet_2022,
	title = {{HistNet}: {Histogram}-based convolutional neural network with {Chi}-squared deep metric learning for facial expression recognition},
	volume = {608},
	issn = {0020-0255},
	shorttitle = {{HistNet}},
	url = {https://www.sciencedirect.com/science/article/pii/S0020025522006867},
	doi = {10.1016/j.ins.2022.06.092},
	abstract = {Facial expression recognition is a challenging problem in machine learning. There is much research has been conducted in this field; however, the accuracy of facial expression recognition, especially in uncontrolled conditions, needs much improvement. In this paper, a deep histogram metric learning in a Convolutional Neural Network (CNN) is presented for facial expression recognition. The proposed CNN utilizes a histogram calculation layer to provide statistical description of feature maps at the output of the convolutional layers. To train the proposed CNN in histogram space, a learnable matrix (equivalent to the fully connected layer) is introduced in chi-squared distance equation. Then, the modified equation is used in the loss function. The recognition rates of the proposed CNN for seven-class facial expression recognition on four well-known databases including CK+, MMI, SFEW, and RAF-DB are 98.47\%, 83.41\%, 61.01\%, and 89.28\%, respectively. The results show superiority of the proposed CNN compared to the state-of-the-art methods.},
	language = {en},
	urldate = {2023-07-10},
	journal = {Information Sciences},
	author = {Sadeghi, Hamid and Raie, Abolghasem-A.},
	month = aug,
	year = {2022},
	keywords = {Chi-squared distance, Convolutional neural network, Deep Learning, Facial expression recognition, Histogram network, Metric learning},
	pages = {472--488},
	file = {ScienceDirect Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\7X65YMRX\\Sadeghi and Raie - 2022 - HistNet Histogram-based convolutional neural netw.pdf:application/pdf;ScienceDirect Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\N9LMHJWI\\S0020025522006867.html:text/html},
}

@article{castano_equivalence_2023,
	title = {An {Equivalence} {Analysis} of {Binary} {Quantification} {Methods}},
	volume = {37},
	copyright = {Copyright (c) 2023 Association for the Advancement of Artificial Intelligence},
	issn = {2374-3468},
	url = {https://ojs.aaai.org/index.php/AAAI/article/view/25849},
	doi = {10.1609/aaai.v37i6.25849},
	abstract = {Quantification (or prevalence estimation) algorithms aim at predicting the class distribution of unseen sets (or bags) of examples. These methods are useful for two main tasks: 1) quantification applications, for instance when we need to track the proportions of several groups of interest over time, and 2) domain adaptation problems, in which we usually need to adapt a previously trained classifier to a different --albeit related-- target distribution according to the estimated prevalences. This paper analyzes several binary quantification algorithms showing that not only do they share a common framework but are, in fact, equivalent. Inspired by this study, we propose a new method that extends one of the approaches analyzed. After an empirical evaluation of all these methods using synthetic and benchmark datasets, the paper concludes recommending three of them due to their precision, efficiency, and diversity.},
	language = {en},
	number = {6},
	urldate = {2023-07-10},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
	author = {Castaño, Alberto and Alonso, Jaime and González, Pablo and Coz, Juan José del},
	month = jun,
	year = {2023},
	note = {Number: 6},
	keywords = {ML: Multi-Instance/Multi-View Learning},
	pages = {6944--6952},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\66QHBZI7\\Castaño et al. - 2023 - An Equivalence Analysis of Binary Quantification M.pdf:application/pdf},
}

@misc{kendall_multi-task_2018,
	title = {Multi-{Task} {Learning} {Using} {Uncertainty} to {Weigh} {Losses} for {Scene} {Geometry} and {Semantics}},
	url = {http://arxiv.org/abs/1705.07115},
	abstract = {Numerous deep learning applications benefit from multi-task learning with multiple regression and classification objectives. In this paper we make the observation that the performance of such systems is strongly dependent on the relative weighting between each task's loss. Tuning these weights by hand is a difficult and expensive process, making multi-task learning prohibitive in practice. We propose a principled approach to multi-task deep learning which weighs multiple loss functions by considering the homoscedastic uncertainty of each task. This allows us to simultaneously learn various quantities with different units or scales in both classification and regression settings. We demonstrate our model learning per-pixel depth regression, semantic and instance segmentation from a monocular input image. Perhaps surprisingly, we show our model can learn multi-task weightings and outperform separate models trained individually on each task.},
	urldate = {2023-07-10},
	publisher = {arXiv},
	author = {Kendall, Alex and Gal, Yarin and Cipolla, Roberto},
	month = apr,
	year = {2018},
	note = {arXiv:1705.07115 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\NSEZUFN9\\1705.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\7VXMN3A3\\Kendall et al. - 2018 - Multi-Task Learning Using Uncertainty to Weigh Los.pdf:application/pdf},
}

@inproceedings{forman_pragmatic_2006,
	address = {New York, NY, USA},
	series = {{KDD} '06},
	title = {Pragmatic text mining: minimizing human effort to quantify many issues in call logs},
	isbn = {978-1-59593-339-3},
	shorttitle = {Pragmatic text mining},
	url = {https://dl.acm.org/doi/10.1145/1150402.1150520},
	doi = {10.1145/1150402.1150520},
	abstract = {We discuss our experiences in analyzing customer-support issues from the unstructured free-text fields of technical-support call logs. The identification of frequent issues and their accurate quantification is essential in order to track aggregate costs broken down by issue type, to appropriately target engineering resources, and to provide the best diagnosis, support and documentation for most common issues. We present a new set of techniques for doing this efficiently on an industrial scale, without requiring manual coding of calls in the call center. Our approach involves (1) a new text clustering method to identify common and emerging issues; (2) a method to rapidly train large numbers of categorizers in a practical, interactive manner; and (3) a method to accurately quantify categories, even in the face of inaccurate classifications and training sets that necessarily cannot match the class distribution of each new month's data. We present our methodology and a tool we developed and deployed that uses these methods for tracking ongoing support issues and discovering emerging issues at HP.},
	urldate = {2023-07-20},
	booktitle = {Proceedings of the 12th {ACM} {SIGKDD} international conference on {Knowledge} discovery and data mining},
	publisher = {Association for Computing Machinery},
	author = {Forman, George and Kirshenbaum, Evan and Suermondt, Jaap},
	month = aug,
	year = {2006},
	keywords = {quantification, applications, log processing, supervised machine learning, text classification, text mining},
	pages = {852--861},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\KT8LHHIE\\Forman et al. - 2006 - Pragmatic text mining minimizing human effort to .pdf:application/pdf},
}

@inproceedings{forman_quantifying_2006,
	address = {New York, NY, USA},
	series = {{KDD} '06},
	title = {Quantifying trends accurately despite classifier error and class imbalance},
	isbn = {978-1-59593-339-3},
	url = {https://dl.acm.org/doi/10.1145/1150402.1150423},
	doi = {10.1145/1150402.1150423},
	abstract = {This paper promotes a new task for supervised machine learning research: quantification - the pursuit of learning methods for accurately estimating the class distribution of a test set, with no concern for predictions on individual cases. A variant for cost quantification addresses the need to total up costs according to categories predicted by imperfect classifiers. These tasks cover a large and important family of applications that measure trends over time.The paper establishes a research methodology, and uses it to evaluate several proposed methods that involve selecting the classification threshold in a way that would spoil the accuracy of individual classifications. In empirical tests, Median Sweep methods show outstanding ability to estimate the class distribution, despite wide disparity in testing and training conditions. The paper addresses shifting class priors and costs, but not concept drift in general.},
	urldate = {2024-01-31},
	booktitle = {Proceedings of the 12th {ACM} {SIGKDD} international conference on {Knowledge} discovery and data mining},
	publisher = {Association for Computing Machinery},
	author = {Forman, George},
	month = aug,
	year = {2006},
	keywords = {quantification, classification, text mining, cost quantification},
	pages = {157--166},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\3VKWRSB7\\Forman - 2006 - Quantifying trends accurately despite classifier e.pdf:application/pdf},
}

@misc{firat_unified_2016,
	title = {Unified {Framework} for {Quantification}},
	url = {http://arxiv.org/abs/1606.00868},
	abstract = {Quantification is the machine learning task of estimating test-data class proportions that are not necessarily similar to those in training. Apart from its intrinsic value as an aggregate statistic, quantification output can also be used to optimize classifier probabilities, thereby increasing classification accuracy. We unify major quantification approaches under a constrained multi-variate regression framework, and use mathematical programming to estimate class proportions for different loss functions. With this modeling approach, we extend existing binary-only quantification approaches to multi-class settings as well. We empirically verify our unified framework by experimenting with several multi-class datasets including the Stanford Sentiment Treebank and CIFAR-10.},
	urldate = {2024-03-25},
	publisher = {arXiv},
	author = {Firat, Aykut},
	month = jun,
	year = {2016},
	note = {arXiv:1606.00868 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\FH64F7WN\\1606.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\KKMEYIVQ\\Firat - 2016 - Unified Framework for Quantification.pdf:application/pdf},
}

@misc{azizzadenesheli_regularized_2019,
	title = {Regularized {Learning} for {Domain} {Adaptation} under {Label} {Shifts}},
	url = {http://arxiv.org/abs/1903.09734},
	abstract = {We propose Regularized Learning under Label shifts (RLLS), a principled and a practical domain-adaptation algorithm to correct for shifts in the label distribution between a source and a target domain. We first estimate importance weights using labeled source data and unlabeled target data, and then train a classifier on the weighted source samples. We derive a generalization bound for the classifier on the target domain which is independent of the (ambient) data dimensions, and instead only depends on the complexity of the function class. To the best of our knowledge, this is the first generalization bound for the label-shift problem where the labels in the target domain are not available. Based on this bound, we propose a regularized estimator for the small-sample regime which accounts for the uncertainty in the estimated weights. Experiments on the CIFAR-10 and MNIST datasets show that RLLS improves classification accuracy, especially in the low sample and large-shift regimes, compared to previous methods.},
	urldate = {2024-03-30},
	publisher = {arXiv},
	author = {Azizzadenesheli, Kamyar and Liu, Anqi and Yang, Fanny and Anandkumar, Animashree},
	month = mar,
	year = {2019},
	note = {arXiv:1903.09734 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\3P9NJ3QM\\1903.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\LL8DSNW6\\Azizzadenesheli et al. - 2019 - Regularized Learning for Domain Adaptation under L.pdf:application/pdf},
}

@misc{lipton_detecting_2018,
	title = {Detecting and {Correcting} for {Label} {Shift} with {Black} {Box} {Predictors}},
	url = {http://arxiv.org/abs/1802.03916},
	abstract = {Faced with distribution shift between training and test set, we wish to detect and quantify the shift, and to correct our classifiers without test set labels. Motivated by medical diagnosis, where diseases (targets) cause symptoms (observations), we focus on label shift, where the label marginal \$p(y)\$ changes but the conditional \$p(x{\textbar} y)\$ does not. We propose Black Box Shift Estimation (BBSE) to estimate the test distribution \$p(y)\$. BBSE exploits arbitrary black box predictors to reduce dimensionality prior to shift correction. While better predictors give tighter estimates, BBSE works even when predictors are biased, inaccurate, or uncalibrated, so long as their confusion matrices are invertible. We prove BBSE's consistency, bound its error, and introduce a statistical test that uses BBSE to detect shift. We also leverage BBSE to correct classifiers. Experiments demonstrate accurate estimates and improved prediction, even on high-dimensional datasets of natural images.},
	urldate = {2024-03-30},
	publisher = {arXiv},
	author = {Lipton, Zachary C. and Wang, Yu-Xiang and Smola, Alex},
	month = jul,
	year = {2018},
	note = {arXiv:1802.03916 [cs, stat]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Neural and Evolutionary Computing},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\6TCFKIJ9\\1802.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\AXWG42MD\\Lipton et al. - 2018 - Detecting and Correcting for Label Shift with Blac.pdf:application/pdf},
}

@misc{monte_answer_2023,
	title = {Answer to "{What} is the difference between {Covariate} {Shift}, {Label} {Shift}, {Concept} {Shift}, {Concept} {Drift}, and {Prior} {Probability} {Shift}?"},
	shorttitle = {Answer to "{What} is the difference between {Covariate} {Shift}, {Label} {Shift}, {Concept} {Shift}, {Concept} {Drift}, and {Prior} {Probability} {Shift}?},
	url = {https://datascience.stackexchange.com/a/122466},
	urldate = {2024-03-30},
	journal = {Data Science Stack Exchange},
	author = {monte},
	month = jun,
	year = {2023},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\Q3Y9VYVB\\what-is-the-difference-between-covariate-shift-label-shift-concept-shift-conc.html:text/html},
}

@misc{huyen_data_2022,
	title = {Data {Distribution} {Shifts} and {Monitoring}},
	url = {https://huyenchip.com/2022/02/07/data-distribution-shifts-and-monitoring.html},
	abstract = {Note: This note is a work-in-progress, created for the course CS 329S: Machine Learning Systems Design (Stanford, 2022).},
	language = {en},
	urldate = {2024-03-30},
	journal = {Chip Huyen},
	author = {Huyen, Chip},
	month = feb,
	year = {2022},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\24NWPI2E\\data-distribution-shifts-and-monitoring.html:text/html},
}

@inproceedings{alexandari_maximum_2020,
  title={Maximum likelihood with bias-corrected calibration is hard-to-beat at label shift adaptation},
  author={Alexandari, Amr and Kundaje, Anshul and Shrikumar, Avanti},
  booktitle={International Conference on Machine Learning},
  pages={222--232},
  year={2020},
  organization={PMLR}
}




@article{scholkopf_causal_nodate,
	title = {On {Causal} and {Anticausal} {Learning}},
	abstract = {We consider the problem of function estimation in the case where an underlying causal model can be inferred. This has implications for popular scenarios such as covariate shift, concept drift, transfer learning and semi-supervised learning. We argue that causal knowledge may facilitate some approaches for a given problem, and rule out others. In particular, we formulate a hypothesis for when semi-supervised learning can help, and corroborate it with empirical results.},
	language = {en},
	author = {Schölkopf, Bernhard and Janzing, Dominik and Peters, Jonas and Sgouritsa, Eleni and Zhang, Kun and Mooij, Joris},
	file = {Schölkopf et al. - On Causal and Anticausal Learning.pdf:C\:\\Users\\yunru\\Zotero\\storage\\ERBE98W3\\Schölkopf et al. - On Causal and Anticausal Learning.pdf:application/pdf},
}

@inproceedings{garg_unified_2020,
	title = {A {Unified} {View} of {Label} {Shift} {Estimation}},
	volume = {33},
	url = {https://proceedings.neurips.cc/paper/2020/hash/219e052492f4008818b8adb6366c7ed6-Abstract.html},
	urldate = {2024-03-30},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Garg, Saurabh and Wu, Yifan and Balakrishnan, Sivaraman and Lipton, Zachary},
	year = {2020},
	pages = {3290--3300},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\ERVQH9PN\\Garg et al. - 2020 - A Unified View of Label Shift Estimation.pdf:application/pdf},
}

@misc{noauthor_mle_nodate,
	title = {{MLE} vs {MAP}: the connection between {Maximum} {Likelihood} and {Maximum} {A} {Posteriori} {Estimation} - {Agustinus} {Kristiadi}},
	shorttitle = {{MLE} vs {MAP}},
	url = {https://agustinus.kristia.de/techblog/2017/01/01/mle-vs-map/},
	abstract = {In this post, we will see what is the difference between Maximum Likelihood Estimation (MLE) and Maximum A Posteriori (MAP).},
	language = {en},
	urldate = {2024-03-30},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\6KKRY3YT\\mle-vs-map.html:text/html},
}

@misc{suhre_answer_2021,
	title = {Answer to "{Relation} between {MAP}, {EM}, and {MLE}"},
	url = {https://stats.stackexchange.com/a/524802/401528},
	urldate = {2024-03-30},
	journal = {Cross Validated},
	author = {Suhre, Yannik},
	month = may,
	year = {2021},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\PN8SWIJV\\relation-between-map-em-and-mle.html:text/html},
}

@article{vaz_quantification_2019,
	title = {Quantification {Under} {Prior} {Probability} {Shift}: the {Ratio} {Estimator} and its {Extensions}},
	volume = {20},
	issn = {1533-7928},
	shorttitle = {Quantification {Under} {Prior} {Probability} {Shift}},
	url = {http://jmlr.org/papers/v20/18-456.html},
	abstract = {The quantification problem consists of determining the prevalence of a given label in a target population. However, one often has access to the labels in a sample from the training population but not in the target population. A common assumption in this situation is that of prior probability shift, that is, once the labels are known, the distribution of the features is the same in the training and target populations. In this paper, we derive a new lower bound for the risk of the quantification problem under the prior shift assumption. Complementing this lower bound, we present a new approximately minimax class of estimators, ratio estimators, which generalize several previous proposals in the literature. Using a weaker version of the prior shift assumption, which can be tested, we show that ratio estimators can be used to build confidence intervals for the quantification problem. We also extend the ratio estimator so that it can: (i) incorporate labels from the target population, when they are available and (ii) estimate how the prevalence of positive labels varies according to a function of certain covariates.},
	number = {79},
	urldate = {2024-04-03},
	journal = {Journal of Machine Learning Research},
	author = {Vaz, Afonso Fernandes and Izbicki, Rafael and Stern, Rafael Bassi},
	year = {2019},
	pages = {1--33},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\PXYS6Q27\\Vaz et al. - 2019 - Quantification Under Prior Probability Shift the .pdf:application/pdf},
}

@misc{garg_rlsbench_2023,
	title = {{RLSbench}: {Domain} {Adaptation} {Under} {Relaxed} {Label} {Shift}},
	shorttitle = {{RLSbench}},
	url = {http://arxiv.org/abs/2302.03020},
	abstract = {Despite the emergence of principled methods for domain adaptation under label shift, their sensitivity to shifts in class conditional distributions is precariously under explored. Meanwhile, popular deep domain adaptation heuristics tend to falter when faced with label proportions shifts. While several papers modify these heuristics in attempts to handle label proportions shifts, inconsistencies in evaluation standards, datasets, and baselines make it difficult to gauge the current best practices. In this paper, we introduce RLSbench, a large-scale benchmark for relaxed label shift, consisting of \${\textgreater}\$500 distribution shift pairs spanning vision, tabular, and language modalities, with varying label proportions. Unlike existing benchmarks, which primarily focus on shifts in class-conditional \$p(x{\textbar}y)\$, our benchmark also focuses on label marginal shifts. First, we assess 13 popular domain adaptation methods, demonstrating more widespread failures under label proportion shifts than were previously known. Next, we develop an effective two-step meta-algorithm that is compatible with most domain adaptation heuristics: (i) pseudo-balance the data at each epoch; and (ii) adjust the final classifier with target label distribution estimate. The meta-algorithm improves existing domain adaptation heuristics under large label proportion shifts, often by 2--10{\textbackslash}\% accuracy points, while conferring minimal effect (\${\textless}\$0.5{\textbackslash}\%) when label proportions do not shift. We hope that these findings and the availability of RLSbench will encourage researchers to rigorously evaluate proposed methods in relaxed label shift settings. Code is publicly available at https://github.com/acmi-lab/RLSbench.},
	urldate = {2024-04-09},
	publisher = {arXiv},
	author = {Garg, Saurabh and Erickson, Nick and Sharpnack, James and Smola, Alex and Balakrishnan, Sivaraman and Lipton, Zachary C.},
	month = jun,
	year = {2023},
	note = {arXiv:2302.03020 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\Q8DYNCEG\\2302.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\FPJ3B754\\Garg et al. - 2023 - RLSbench Domain Adaptation Under Relaxed Label Sh.pdf:application/pdf},
}

@misc{tasche_class_2022,
	title = {Class {Prior} {Estimation} under {Covariate} {Shift}: {No} {Problem}?},
	shorttitle = {Class {Prior} {Estimation} under {Covariate} {Shift}},
	url = {http://arxiv.org/abs/2206.02449},
	abstract = {We show that in the context of classification the property of source and target distributions to be related by covariate shift may be lost if the information content captured in the covariates is reduced, for instance by dropping components or mapping into a lower-dimensional or finite space. As a consequence, under covariate shift simple approaches to class prior estimation in the style of classify and count with or without adjustment are infeasible. We prove that transformations of the covariates that preserve the covariate shift property are necessarily sufficient in the statistical sense for the full set of covariates. A probing algorithm as alternative approach to class prior estimation under covariate shift is proposed.},
	urldate = {2024-04-23},
	publisher = {arXiv},
	author = {Tasche, Dirk},
	month = aug,
	year = {2022},
	note = {arXiv:2206.02449 [cs, math, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, 62B05, G.3, I.5.1, Mathematics - Statistics Theory},
	file = {arXiv.org Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\FNV2FRZF\\2206.html:text/html;Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\5V4U4B7T\\Tasche - 2022 - Class Prior Estimation under Covariate Shift No P.pdf:application/pdf},
}

@article{zhang_domain_nodate,
	title = {Domain {Adaptation} under {Target} and {Conditional} {Shift}},
	abstract = {Let X denote the feature and Y the target. We consider domain adaptation under three possible scenarios: (1) the marginal PY changes, while the conditional PX{\textbar}Y stays the same (target shift), (2) the marginal PY is ﬁxed, while the conditional PX{\textbar}Y changes with certain constraints (conditional shift), and (3) the marginal PY changes, and the conditional PX{\textbar}Y changes with constraints (generalized target shift). Using background knowledge, causal interpretations allow us to determine the correct situation for a problem at hand. We exploit importance reweighting or sample transformation to ﬁnd the learning machine that works well on test data, and propose to estimate the weights or transformations by reweighting or transforming training data to reproduce the covariate distribution on the test domain. Thanks to kernel embedding of conditional as well as marginal distributions, the proposed approaches avoid distribution estimation, and are applicable for high-dimensional problems. Numerical evaluations on synthetic and realworld data sets demonstrate the eﬀectiveness of the proposed framework.},
	language = {en},
	author = {Zhang, Kun and Scholkopf, Bernhard and Muandet, Krikamol and Wang, Zhikun},
	file = {Zhang et al. - Domain Adaptation under Target and Conditional Shi.pdf:C\:\\Users\\yunru\\Zotero\\storage\\P2IC5NA3\\Zhang et al. - Domain Adaptation under Target and Conditional Shi.pdf:application/pdf},
}

@misc{garg_unified_2020-1,
	title = {A {Unified} {View} of {Label} {Shift} {Estimation}},
	url = {http://arxiv.org/abs/2003.07554},
	abstract = {Under label shift, the label distribution p(y) might change but the class-conditional distributions p(x{\textbar}y) do not. There are two dominant approaches for estimating the label marginal. BBSE, a moment-matching approach based on confusion matrices, is provably consistent and provides interpretable error bounds. However, a maximum likelihood estimation approach, which we call MLLS, dominates empirically. In this paper, we present a uniﬁed view of the two methods and the ﬁrst theoretical characterization of MLLS. Our contributions include (i) consistency conditions for MLLS, which include calibration of the classiﬁer and a confusion matrix invertibility condition that BBSE also requires; (ii) a uniﬁed framework, casting BBSE as roughly equivalent to MLLS for a particular choice of calibration method; and (iii) a decomposition of MLLS’s ﬁnite-sample error into terms reﬂecting miscalibration and estimation error. Our analysis attributes BBSE’s statistical inefﬁciency to a loss of information due to coarse calibration. Experiments on synthetic data, MNIST, and CIFAR10 support our ﬁndings.},
	language = {en},
	urldate = {2024-05-03},
	publisher = {arXiv},
	author = {Garg, Saurabh and Wu, Yifan and Balakrishnan, Sivaraman and Lipton, Zachary C.},
	month = oct,
	year = {2020},
	note = {arXiv:2003.07554 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {Garg et al. - 2020 - A Unified View of Label Shift Estimation.pdf:C\:\\Users\\yunru\\Zotero\\storage\\87BB2847\\Garg et al. - 2020 - A Unified View of Label Shift Estimation.pdf:application/pdf},
}

@incollection{noauthor_frontmatter_1992,
	title = {Frontmatter},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.fmatter},
	abstract = {The prelims comprise: Half Title Title Copyright Dedication Contents Preface},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.fmatter},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.fmatter},
	pages = {i--xv},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\72L36CER\\1992 - Frontmatter.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\Z28WNUUD\\0471725293.html:text/html},
}

@incollection{noauthor_general_1992,
	title = {General {Introduction}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch1},
	abstract = {This chapter includes the following topics: Introduction Basic Notation Allocation Rules Decision-Theoretic Approach Unavailability of Group-Prior Probabilities Training Data Sample-Based Allocation Rules Parametric Allocation Rules Assessment of Model Fit Error Rates of Allocation Rules Posterior Probabilities of Group Membership Distances Between Groups},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch1},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch1},
	keywords = {discriminant analysis, optimal allocation, probability (statistics), statistical methodology, vectors},
	pages = {1--26},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\Y8AMRPQM\\0471725293.html:text/html},
}

@incollection{noauthor_likelihood-based_1992,
	title = {Likelihood-{Based} {Approaches} to {Discrimination}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch2},
	abstract = {This chapter includes the following topics: Maximum Likelihood Estimation of Group Parameters A Bayesian Approach Estimation of Group Proportions Estimating Disease Prevalence Misclassified Training Data Partially Classified Training Data Maximum Likelihood Estimation for Partial Classification Maximum Likelihood Estimation for Partial Nonrandom Classification Classification Likelihood Approach Absence of Classified Data Group-Conditional Mixture Densities},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch2},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch2},
	keywords = {discriminant analysis, distribution function, group-indicator vectors, likelihood estimation, matrices},
	pages = {27--51},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\E2YY7FQG\\0471725293.html:text/html},
}

@incollection{noauthor_discrimination_1992,
	title = {Discrimination via {Normal} {Models}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch3},
	abstract = {This chapter includes the following topics: Introduction Heteroscedastic Normal Model Homoscedastic Normal Model Some Other Normal Theory-Based Rules Predictive Discrimination Covariance-Adjusted Discrimination Discrimination with Repeated Measurements Partially Classified Data Linear Projections of Homoscedastic Feature Data Linear Projections of Heteroscedastic Feature Data},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch3},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch3},
	keywords = {discriminant analysis, distribution function, group-covariance matrices, likelihood ratio, normal models},
	pages = {52--100},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\85E9NR6G\\0471725293.html:text/html},
}

@incollection{noauthor_distributional_1992,
	title = {Distributional {Results} for {Discrimination} via {Normal} {Models}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch4},
	abstract = {This chapter includes the following topics: Introduction Distribution of Sample NLDF (W-Statistic) Moments of Conditional Error Rates of Sample NLDR Distributions of Conditional Error Rates of Sample NLDR Constrained Allocation with the Sample NLDR Distributional Results for Quadratic Discrimination},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch4},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch4},
	keywords = {discriminant analysis, normal models, asymptotic expansions, distribution (statistics), normal-based linear discriminant rule (NLDR)},
	pages = {101--128},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\HGVB4ZBA\\0471725293.html:text/html},
}

@incollection{noauthor_practical_1992,
	title = {Some {Practical} {Aspects} and {Variants} of {Normal} {Theory}-{Based} {Discriminant} {Rules}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch5},
	abstract = {This chapter includes the following topics: Introduction Regularization in Quadratic Discrimination Linear Versus Quadratic Normal-Based Discriminant Analysis Some Models for Variants of the Sample NQDR Regularized Discriminant Analysis (RDA) Robustness of Sample NLDR and NQDR Robust Estimation of Group Parameters},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch5},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch5},
	keywords = {discriminant analysis, group-covariance matrices, normal models, maximum likelihood estimates, robustness},
	pages = {129--167},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\2RSLBG72\\0471725293.html:text/html},
}

@incollection{noauthor_data_1992,
	title = {Data {Analytic} {Considerations} with {Normal} {Theory}-{Based} {Discriminant} {Analysis}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch6},
	abstract = {This chapter includes the following topics: Introduction Assessment of Normality and Homoscedasticity Data-Based Transformations of Feature Data Typicality of a Feature Vector Sample Canonical Variates Some Other Methods of Dimension Reduction to Reveal Group Structure Example: Detection of Hemophilia A Carriers Example: Statistical Diagnosis of Diabetes Example: Testing for Existence of Subspecies in Fisher's Iris Data},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch6},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch6},
	keywords = {discriminant analysis, likelihood ratio, normal models, normality, variance analysis},
	pages = {168--215},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\B7545FBY\\0471725293.html:text/html},
}

@incollection{noauthor_parametric_1992,
	title = {Parametric {Discrimination} via {Nonnormal} {Models}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch7},
	abstract = {This chapter includes the following topics: Introduction Discrete Feature Data Parametric Formulation for Discrete Feature Data Location Model for Mixed Features Error Rates of Location Model-Based Rules Adjustments to Sample NLDR for Mixed Feature Data Some Nonnormal Models for Continuous Feature Data Case Study of Renal Venous Renin in Hypertension Example: Discrimination Between Depositional Environments},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch7},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch7},
	keywords = {normal models, maximum likelihood estimates, discrete feature, error rates, parametric distribution},
	pages = {216--254},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\DX8IC5US\\0471725293.html:text/html},
}

@incollection{noauthor_logistic_1992,
	title = {Logistic {Discrimination}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch8},
	abstract = {This chapter includes the following topics: Introduction Maximum Likelihood Estimation of Logistic Regression Coefficients Bias Correction of MLE for g = 2 Groups Assessing the Fit and Performance of Logistic Model Logistic Versus Normal-Based Linear Discriminant Analysis Example: Differential Diagnosis of Some Liver Diseases},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch8},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch8},
	keywords = {discriminant analysis, distribution function, maximum likelihood estimates, discrete feature, logistic discrimination},
	pages = {255--282},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\DY8Q7TMK\\0471725293.html:text/html},
}

@incollection{noauthor_nonparametric_1992,
	title = {Nonparametric {Discrimination}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch9},
	abstract = {This chapter includes the following topics: Introduction Multinomial-Based Discrimination Nonparametric Estimation of Group-Conditional Densities Selection of Smoothing Parameters in Kernel Estimates of Group-Conditional Densities Alternatives to Fixed Kernel Density Estimates Comparative Performance of Kernel-Based Discriminant Rules Nearest Neighbor Rules Tree-Structured Allocation Rules Some Other Nonparametric Discriminant Procedures},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch9},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch9},
	keywords = {discriminant analysis, distribution function, kernel density estimation, multinomial distribution, nonparametric discrimination},
	pages = {283--336},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\CKW3HZNC\\0471725293.html:text/html},
}

@incollection{noauthor_estimation_1992,
	title = {Estimation of {Error} {Rates}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch10},
	abstract = {This chapter includes the following topics: Introduction Some Nonparametric Error-Rate Estimators The Bootstrap Variants of the Bootstrap Smoothing of the Apparent Error Rate Parametric Error-Rate Esimators Confidence Intervals Some Other Topics in Error-Rate Estimation},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch10},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch10},
	keywords = {probability (statistics), distribution function, asymptotic bias, bootstrap, error-rate estimators},
	pages = {337--377},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\DTI46VPV\\0471725293.html:text/html},
}

@incollection{noauthor_assessing_1992,
	title = {Assessing the {Reliability} of the {Estimated} {Posterior} {Probabilities} of {Group} {Membership}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch11},
	abstract = {This chapter includes the following topics: Introduction Distribution of Sample Posterior Probabilities Further Approaches to Interval Estimation of Posterior Probabilities of Group Membership},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch11},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch11},
	keywords = {covariance matrices, estimation theory, group membership, log odds, posterior probabilities},
	pages = {378--388},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\9ZMZITKH\\0471725293.html:text/html},
}

@incollection{noauthor_selection_1992,
	title = {Selection of {Feature} {Variables} in {Discriminant} {Analysis}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch12},
	abstract = {This chapter includes the following topics: Introduction Test for No Additional Information Some Selection Procedures Error-Rate-Based Procedures The F-Test and Error-Rate-Based Variable Selections Assessment of the Allocatory Capacity of the Selected Feature Variables},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch12},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch12},
	keywords = {discriminant analysis, normal models, error rates, asymptotic bias, feature variables},
	pages = {389--412},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\RPKPXVBL\\0471725293.html:text/html},
}

@incollection{noauthor_statistical_1992,
	title = {Statistical {Image} {Analysis}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.ch13},
	abstract = {This chapter includes the following topics: Introduction Markov Random Fields Noncontextual Methods of Segmentation Smoothing Methods Individual Contextual Allocation of Pixels ICM Algorithm Global Maximization of the Posterior Distribution of the Image Incomplete-Data Formulation of Image Segmentation Correlated Training Data},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.ch13},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.ch13},
	keywords = {discriminant analysis, image segmentation, Markov processes, pixels, statistical image analysis},
	pages = {413--446},
	file = {Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\YD28ITW5\\0471725293.html:text/html},
}

@incollection{noauthor_references_1992,
	title = {References},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.refs},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.refs},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.refs},
	pages = {447--506},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\7NVZ9DJJ\\1992 - References.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\HZ6HHEH3\\0471725293.html:text/html},
}

@incollection{noauthor_author_1992,
	title = {Author {Index}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.indauth},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.indauth},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.indauth},
	pages = {507--517},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\9DKURW4A\\1992 - Author Index.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\XCL34Y52\\0471725293.html:text/html},
}

@incollection{noauthor_subject_1992,
	title = {Subject {Index}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.indsub},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.indsub},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.indsub},
	pages = {519--526},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\IWYNSZMI\\1992 - Subject Index.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\KPZA3T4H\\0471725293.html:text/html},
}

@incollection{noauthor_wiley_1992,
	title = {Wiley {Series} in {Probability} and {Statistics}},
	isbn = {978-0-471-72529-9},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/0471725293.scard},
	language = {en},
	urldate = {2024-05-07},
	booktitle = {Discriminant {Analysis} and {Statistical} {Pattern} {Recognition}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1992},
	doi = {10.1002/0471725293.scard},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/0471725293.scard},
	pages = {527--534},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\TPWBKC8C\\1992 - Wiley Series in Probability and Statistics.pdf:application/pdf;Snapshot:C\:\\Users\\yunru\\Zotero\\storage\\8Y5YKGAF\\0471725293.html:text/html},
}

@article{wen_class_2024,
	title = {{CLASS} {PROBABILITY} {MATCHING} {WITH} {CALIBRATED} {NETWORKS} {FOR} {LABEL} {SHIFT} {ADAPTATION}},
	abstract = {We consider the domain adaptation problem in the context of label shift, where the label distributions between source and target domain differ, but the conditional distributions of features given the label are the same. To solve the label shift adaptation problem, we develop a novel matching framework named class probability matching (CPM). It is inspired by a new understanding of the source domain’s class probability, as well as a specific relationship between class probability ratios and feature probability ratios between the source and target domains. CPM is able to maintain the same theoretical guarantees as the existing feature probability matching framework, while significantly improving the computational efficiency due to directly matching the probabilities of the label variable. Within the CPM framework, we propose an algorithm named class probability matching with calibrated networks (CPMCN) for target domain classification. From the theoretical perspective, we establish a generalization bound of the CPMCN method in order to explain the benefits of introducing calibrated networks. From the experimental perspective, real data comparisons show that CPMCN outperforms existing matching-based and EM-based algorithms.},
	language = {en},
	author = {Wen, Hongwei and Betken, Annika and Hang, Hanyuan},
	year = {2024},
	file = {Wen et al. - 2024 - CLASS PROBABILITY MATCHING WITH CALIBRATED NETWORK.pdf:C\:\\Users\\yunru\\Zotero\\storage\\G2E83AFN\\Wen et al. - 2024 - CLASS PROBABILITY MATCHING WITH CALIBRATED NETWORK.pdf:application/pdf},
}

@inproceedings{baby_online_2023,
	title = {Online {Label} {Shift}: {Optimal} {Dynamic} {Regret} meets {Practical} {Algorithms}},
	shorttitle = {Online {Label} {Shift}},
	url = {https://openreview.net/forum?id=Ki6DqBXss4},
	abstract = {This paper focuses on supervised and unsupervised online label shift, where the class marginals \$Q(y)\$ varies but the class-conditionals \$Q(x{\textbar}y)\$ remain invariant. In the unsupervised setting, our goal is to adapt a learner, trained on some offline labeled data, to changing label distributions given unlabeled online data. In the supervised setting, we must both learn a classifier and adapt to the dynamically evolving class marginals given only labeled online data. We develop novel algorithms that reduce the adaptation problem to online regression and guarantee optimal dynamic regret without any prior knowledge of the extent of drift in the label distribution. Our solution is based on bootstrapping the estimates of *online regression oracles* that track the drifting proportions. Experiments across numerous simulated and real-world online label shift scenarios demonstrate the superior performance of our proposed approaches, often achieving 1-3\% improvement in accuracy while being sample and computationally efficient. Code is publicly available at https://github.com/Anon-djiwh/OnlineLabelShift},
	language = {en},
	urldate = {2024-07-30},
	author = {Baby, Dheeraj and Garg, Saurabh and Yen, Tzu-Ching and Balakrishnan, Sivaraman and Lipton, Zachary Chase and Wang, Yu-Xiang},
	month = nov,
	year = {2023},
	file = {Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\T7A6V7WW\\Baby et al. - 2023 - Online Label Shift Optimal Dynamic Regret meets P.pdf:application/pdf},
}

@inproceedings{denham_gain--lose-_2021,
	title = {Gain-{Some}-{Lose}-{Some}: {Reliable} {Quantification} {Under} {General} {Dataset} {Shift}},
	shorttitle = {Gain-{Some}-{Lose}-{Some}},
	url = {https://ieeexplore.ieee.org/document/9679152/?arnumber=9679152},
	doi = {10.1109/ICDM51629.2021.00121},
	abstract = {When applying supervised learning to estimate class distributions of unlabelled samples (so-called quantification), dataset shift is an expected yet challenging problem. Existing quantification methods make strong assumptions on the nature of dataset shift that often will not hold in practice. We propose a novel Gain-Some-Lose-Some (GSLS) model that accounts for more general conditions of dataset shift. We present a method for fitting the GSLS model without any labelled instances from the target sample, and experimentally demonstrate that GSLS can produce reliable quantification prediction intervals under broader conditions of shift than existing quantification methods.},
	urldate = {2024-08-26},
	booktitle = {2021 {IEEE} {International} {Conference} on {Data} {Mining} ({ICDM})},
	author = {Denham, Benjamin and Lai, Edmund M-K and Sinha, Roopak and Naeem, M. Asif},
	month = dec,
	year = {2021},
	note = {ISSN: 2374-8486},
	keywords = {Data mining, Predictive models, Conferences, Reliability, quantification, dataset shift, Fitting, prediction intervals, Supervised learning},
	pages = {1048--1053},
	file = {IEEE Xplore Abstract Record:C\:\\Users\\yunru\\Zotero\\storage\\YEQ9GU5R\\9679152.html:text/html;IEEE Xplore Full Text PDF:C\:\\Users\\yunru\\Zotero\\storage\\Q4YIIREE\\Denham et al. - 2021 - Gain-Some-Lose-Some Reliable Quantification Under.pdf:application/pdf},
}

@inproceedings{fan_label_2024,
	address = {Barcelona Spain},
	title = {Label {Shift} {Correction} via {Bidirectional} {Marginal} {Distribution} {Matching}},
	isbn = {979-8-4007-0490-1},
	url = {https://dl.acm.org/doi/10.1145/3637528.3671867},
	doi = {10.1145/3637528.3671867},
	language = {en},
	urldate = {2024-08-29},
	booktitle = {Proceedings of the 30th {ACM} {SIGKDD} {Conference} on {Knowledge} {Discovery} and {Data} {Mining}},
	publisher = {ACM},
	author = {Fan, Ruidong and Ouyang, Xiao and Tao, Hong and Hou, Chenping},
	month = aug,
	year = {2024},
	pages = {735--746},
}

@article{wu_online_nodate,
	title = {Online {Adaptation} to {Label} {Distribution} {Shift}},
	abstract = {Machine learning models often encounter distribution shifts when deployed in the real world. In this paper, we focus on adaptation to label distribution shift in the online setting, where the test-time label distribution is continually changing and the model must dynamically adapt to it without observing the true label. Leveraging a novel analysis, we show that the lack of true label does not hinder estimation of the expected test loss, which enables the reduction of online label shift adaptation to conventional online learning. Informed by this observation, we propose adaptation algorithms inspired by classical online learning techniques such as Follow The Leader (FTL) and Online Gradient Descent (OGD) and derive their regret bounds. We empirically verify our ﬁndings under both simulated and real world label distribution shifts and show that OGD is particularly effective and robust to a variety of challenging label shift scenarios.},
	language = {en},
	author = {Wu, Ruihan and Guo, Chuan and Su, Yi and Weinberger, Kilian Q},
	file = {PDF:C\:\\Users\\yunru\\Zotero\\storage\\FKLQU3DQ\\Wu et al. - Online Adaptation to Label Distribution Shift.pdf:application/pdf},
}
@article{van2019calibration,
  title={Calibration: the Achilles heel of predictive analytics},
  author={Van Calster, Ben and McLernon, David J and Van Smeden, Maarten and Wynants, Laure and Steyerberg, Ewout W and Topic Group ‘Evaluating diagnostic tests and prediction models’ of the STRATOS initiative Bossuyt Patrick Collins Gary S. Macaskill Petra McLernon David J. Moons Karel GM Steyerberg Ewout W. Van Calster Ben van Smeden Maarten Vickers Andrew J.},
  journal={BMC medicine},
  volume={17},
  number={1},
  pages={230},
  year={2019},
  publisher={Springer}
}


@article{bieshaar2018cooperative,
  title={Cooperative starting movement detection of cyclists using convolutional neural networks and a boosted stacking ensemble},
  author={Bieshaar, Maarten and Zernetsch, Stefan and Hubert, Andreas and Sick, Bernhard and Doll, Konrad},
  journal={IEEE Transactions on Intelligent Vehicles},
  volume={3},
  number={4},
  pages={534--544},
  year={2018},
  publisher={IEEE}
}


@article{tang2014effects,
  title={The effects of visualization and interactivity on calibration in financial decision-making},
  author={Tang, Fengchun and Hess, Traci J and Valacich, Joseph S and Sweeney, John T},
  journal={Behavioral Research in Accounting},
  volume={26},
  number={1},
  pages={25--58},
  year={2014},
  publisher={American Accounting Assocation}
}


@inproceedings{naeini2015obtaining,
  title={Obtaining well calibrated probabilities using bayesian binning},
  author={Naeini, Mahdi Pakdaman and Cooper, Gregory and Hauskrecht, Milos},
  booktitle={Proceedings of the AAAI conference on artificial intelligence},
  volume={29},
  number={1},
  year={2015}
}

@article{krizhevsky2009learning,
  title={Learning multiple layers of features from tiny images},
  author={Krizhevsky, Alex and Hinton, Geoffrey and others},
  year={2009},
  publisher={Toronto, ON, Canada}
}

@INPROCEEDINGS{5206848,
  author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Kai Li and Li Fei-Fei},
  booktitle={2009 IEEE Conference on Computer Vision and Pattern Recognition}, 
  title={ImageNet: A large-scale hierarchical image database}, 
  year={2009},
  volume={},
  number={},
  pages={248-255},
  keywords={Large-scale systems;Image databases;Explosions;Internet;Robustness;Information retrieval;Image retrieval;Multimedia databases;Ontologies;Spine},
  doi={10.1109/CVPR.2009.5206848}}


@inproceedings{HB,
author = {Zadrozny, Bianca and Elkan, Charles},
title = {Obtaining calibrated probability estimates from decision trees and naive Bayesian classifiers},
year = {2001},
isbn = {1558607781},
publisher = {Morgan Kaufmann Publishers Inc.},
address = {San Francisco, CA, USA},
booktitle = {Proceedings of the Eighteenth International Conference on Machine Learning},
pages = {609–616},
numpages = {8},
series = {ICML '01}
}

@inproceedings{Vit,
  author       = {Alexey Dosovitskiy and
                  Lucas Beyer and
                  Alexander Kolesnikov and
                  Dirk Weissenborn and
                  Xiaohua Zhai and
                  Thomas Unterthiner and
                  Mostafa Dehghani and
                  Matthias Minderer and
                  Georg Heigold and
                  Sylvain Gelly and
                  Jakob Uszkoreit and
                  Neil Houlsby},
  title        = {An Image is Worth 16x16 Words: Transformers for Image Recognition
                  at Scale},
  booktitle    = {9th International Conference on Learning Representations, {ICLR}},
  year         = {2021}
 
}

@ARTICLE{lenet,
  author={Lecun, Y. and Bottou, L. and Bengio, Y. and Haffner, P.},
  journal={Proceedings of the IEEE}, 
  title={Gradient-based learning applied to document recognition}, 
  year={1998},
  volume={86},
  number={11},
  pages={2278-2324},
  keywords={Neural networks;Pattern recognition;Machine learning;Optical character recognition software;Character recognition;Feature extraction;Multi-layer neural network;Optical computing;Hidden Markov models;Principal component analysis},
  doi={10.1109/5.726791}}

@INPROCEEDINGS {densenet,
author = { Huang, Gao and Liu, Zhuang and Van Der Maaten, Laurens and Weinberger, Kilian Q. },
booktitle = { 2017 IEEE Conference on Computer Vision and Pattern Recognition (CVPR) },
title = {{ Densely Connected Convolutional Networks }},
year = {2017},
volume = {},
ISSN = {1063-6919},
pages = {2261-2269},
abstract = { Recent work has shown that convolutional networks can be substantially deeper, more accurate, and efficient to train if they contain shorter connections between layers close to the input and those close to the output. In this paper, we embrace this observation and introduce the Dense Convolutional Network (DenseNet), which connects each layer to every other layer in a feed-forward fashion. Whereas traditional convolutional networks with L layers have L connections-one between each layer and its subsequent layer-our network has L(L+1)/2 direct connections. For each layer, the feature-maps of all preceding layers are used as inputs, and its own feature-maps are used as inputs into all subsequent layers. DenseNets have several compelling advantages: they alleviate the vanishing-gradient problem, strengthen feature propagation, encourage feature reuse, and substantially reduce the number of parameters. We evaluate our proposed architecture on four highly competitive object recognition benchmark tasks (CIFAR-10, CIFAR-100, SVHN, and ImageNet). DenseNets obtain significant improvements over the state-of-the-art on most of them, whilst requiring less memory and computation to achieve high performance. Code and pre-trained models are available at https://github.com/liuzhuang13/DenseNet. },
keywords = {Training;Convolution;Network architecture;Convolutional codes;Neural networks;Road transportation},
doi = {10.1109/CVPR.2017.243},
url = {https://doi.ieeecomputersociety.org/10.1109/CVPR.2017.243},
publisher = {IEEE Computer Society},
address = {Los Alamitos, CA, USA},
month =Jul}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}


@article{zagoruyko2016wide,
  title={Wide residual networks},
  author={Zagoruyko, Sergey},
  journal={arXiv preprint arXiv:1605.07146},
  year={2016}
}


@INPROCEEDINGS{inception,
  author={Szegedy, Christian and Vanhoucke, Vincent and Ioffe, Sergey and Shlens, Jon and Wojna, Zbigniew},
  booktitle={2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 
  title={Rethinking the Inception Architecture for Computer Vision}, 
  year={2016},
  volume={},
  number={},
  pages={2818-2826},
  keywords={Convolution;Computer architecture;Training;Computational efficiency;Computer vision;Benchmark testing;Computational modeling},
  doi={10.1109/CVPR.2016.308}}

@article{kraft1988software,
  title={A software package for sequential quadratic programming},
  author={Kraft, Dieter},
  journal={Forschungsbericht- Deutsche Forschungs- und Versuchsanstalt fur Luft- und Raumfahrt},
  year={1988}
}

@article{platt1999probabilistic,
  title={Probabilistic outputs for support vector machines and comparisons to regularized likelihood methods},
  author={Platt, John and others},
  journal={Advances in large margin classifiers},
  volume={10},
  number={3},
  pages={61--74},
  year={1999},
  publisher={Cambridge, MA}
}

@article{hinton2015distilling,
  title={Distilling the Knowledge in a Neural Network},
  author={Hinton, Geoffrey},
  journal={arXiv preprint arXiv:1503.02531},
  year={2015}
}


@inproceedings{zadrozny2002transforming,
  title={Transforming classifier scores into accurate multiclass probability estimates},
  author={Zadrozny, Bianca and Elkan, Charles},
  booktitle={Proceedings of the eighth ACM SIGKDD international conference on Knowledge discovery and data mining},
  pages={694--699},
  year={2002}
}


@article{mukhoti2020calibrating,
  title={Calibrating deep neural networks using focal loss},
  author={Mukhoti, Jishnu and Kulharia, Viveka and Sanyal, Amartya and Golodetz, Stuart and Torr, Philip and Dokania, Puneet},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={15288--15299},
  year={2020}
}


@inproceedings{dual_focal_loss,
author = {Tao, Linwei and Dong, Minjing and Xu, Chang},
title = {Dual focal loss for calibration},
year = {2023},
publisher = {JMLR.org},
abstract = {The use of deep neural networks in real-world applications require well-calibrated networks with confidence scores that accurately reflect the actual probability. However, it has been found that these networks often provide over-confident predictions, which leads to poor calibration. Recent efforts have sought to address this issue by focal loss to reduce over-confidence, but this approach can also lead to under-confident predictions. While different variants of focal loss have been explored, it is difficult to find a balance between over-confidence and under-confidence. In our work, we propose a new loss function by focusing on dual logits. Our method not only considers the ground truth logit, but also take into account the highest logit ranked after the ground truth logit. By maximizing the gap between these two logits, our proposed dual focal loss can achieve a better balance between overconfidence and under-confidence. We provide theoretical evidence to support our approach and demonstrate its effectiveness through evaluations on multiple models and datasets, where it achieves state-of-the-art performance. Code is available at https://github.com/Linwei94/DualFocalLoss},
booktitle = {Proceedings of the 40th International Conference on Machine Learning},
articleno = {1410},
numpages = {17},
location = {Honolulu, Hawaii, USA},
series = {ICML'23}
}


@inproceedings{wei2022mitigating,
  title={Mitigating neural network overconfidence with logit normalization},
  author={Wei, Hongxin and Xie, Renchunzi and Cheng, Hao and Feng, Lei and An, Bo and Li, Yixuan},
  booktitle={International conference on machine learning},
  pages={23631--23644},
  year={2022},
  organization={PMLR}
}

@book{scott2015multivariate,
  title={Multivariate density estimation: theory, practice, and visualization},
  author={Scott, David W},
  year={2015},
  publisher={John Wiley \& Sons}
}


@article{paszke2019pytorch,
  title={Pytorch: An imperative style, high-performance deep learning library},
  author={Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and others},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}


@inproceedings{zhang2025label,
  title={Label Shift Estimation With Incremental Prior Update},
  author={Zhang, Yunrui and Batista, Gustavo and Kanhere, Salii S},
  booktitle={Proceedings of the 2025 SIAM International Conference on Data Mining (SDM)},
  pages={134--142},
  year={2025},
  organization={SIAM}
}

@article{zhang2025revisit,
  title={Revisit Time Series Classification Benchmark: The Impact of Temporal Information for Classification},
  author={Zhang, Yunrui and Batista, Gustavo and Kanhere, Salil S},
  journal={arXiv preprint arXiv:2503.20264},
  year={2025}
}

@inproceedings{tan2025pathsovergraph,
  title={Paths-over-graph: Knowledge graph empowered large language model reasoning},
  author={Tan, Xingyu and Wang, Xiaoyang and Liu, Qing and Xu, Xiwei and Yuan, Xin and Zhang, Wenjie},
  booktitle={Proceedings of the ACM on Web Conference 2025},
  pages={3505--3522},
  year={2025}
}