@book{vovk2005algorithmic,
  title={Algorithmic learning in a random world},
  author={Vovk, Vladimir and Gammerman, Alexander and Shafer, Glenn},
  volume={29},
  year={2005},
  publisher={Springer}
}

@article{angelopoulos2022gentle,
journal = {Foundations and Trends® in Machine Learning},
title = {Conformal Prediction: A Gentle Introduction},
doi = {10.1561/2200000101},
issn = {1935-8237},
number = {4},
pages = {494-591},
author = {Anastasios N. Angelopoulos and Stephen Bates},
url = {http://dx.doi.org/10.1561/2200000101},
year = {2023},
volume = {16}
}


@misc{chzhen2021setvalued,
      title={Set-valued classification -- overview via a unified framework}, 
      author={Evgenii Chzhen and Christophe Denis and Mohamed Hebiri and Titouan Lorieul},
      year={2021},
      eprint={2102.12318},
      archivePrefix={arXiv},
      primaryClass={stat.ML}
}

@inproceedings{
kasa2023empirically,
title={Empirically Validating Conformal Prediction on Modern Vision Architectures Under Distribution Shift and Long-tailed Data},
author={Kevin Kasa and Graham W. Taylor},
booktitle={ICML 2023 Workshop on Structured Probabilistic Inference {\&} Generative Modeling},
year={2023},
url={https://openreview.net/forum?id=TMxpy0aluL}
}


@article{zhang2024discounted,
  title={Discounted Adaptive Online Prediction},
  author={Zhang, Zhiyu and Bombara, David and Yang, Heng},
  journal={International Conference on Machine Learning},
  year={2024}
}

@article{angelopoulos2023conformal,
  title={Conformal PID control for time series prediction},
  author={Angelopoulos, Anastasios and Candes, Emmanuel and Tibshirani, Ryan J},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2023}
}

@article{gibbs2021adaptive,
  title={Adaptive conformal inference under distribution shift},
  author={Gibbs, Isaac and Candes, Emmanuel},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={1660--1672},
  year={2021}
}

@article{roth2022uncertain,
  title={Uncertain: Modern topics in uncertainty estimation},
  author={Roth, Aaron},
journal={Unpublished Lecture Notes},
  year={2022},
url={https://www.cis.upenn.edu/~aaroth/uncertainty-notes.pdf}
}

@misc{gibbs2023conformal,
      title={Conformal Inference for Online Prediction with Arbitrary Distribution Shifts}, 
      author={Isaac Gibbs and Emmanuel Candès},
      year={2023},
      eprint={2208.08401},
      archivePrefix={arXiv},
      primaryClass={stat.ME}
}

@misc{angelopoulos2024online,
      title={Online conformal prediction with decaying step sizes}, 
      author={Anastasios N. Angelopoulos and Rina Foygel Barber and Stephen Bates},
      year={2024},
      eprint={2402.01139},
      archivePrefix={arXiv},
      primaryClass={stat.ML}
}

@article{sadinle_least_2019,
	title = {Least {Ambiguous} {Set}-{Valued} {Classifiers} with {Bounded} {Error} {Levels}},
	volume = {114},
	issn = {0162-1459, 1537-274X},
	url = {http://arxiv.org/abs/1609.00451},
	doi = {10.1080/01621459.2017.1395341},
	abstract = {In most classification tasks there are observations that are ambiguous and therefore difficult to correctly label. Set-valued classifiers output sets of plausible labels rather than a single label, thereby giving a more appropriate and informative treatment to the labeling of ambiguous instances. We introduce a framework for multiclass set-valued classification, where the classifiers guarantee user-defined levels of coverage or confidence (the probability that the true label is contained in the set) while minimizing the ambiguity (the expected size of the output). We first derive oracle classifiers assuming the true distribution to be known. We show that the oracle classifiers are obtained from level sets of the functions that define the conditional probability of each class. Then we develop estimators with good asymptotic and finite sample properties. The proposed estimators build on existing single-label classifiers. The optimal classifier can sometimes output the empty set, but we provide two solutions to fix this issue that are suitable for various practical needs.},
	number = {525},
	urldate = {2023-02-02},
	journal = {Journal of the American Statistical Association},
	author = {Sadinle, Mauricio and Lei, Jing and Wasserman, Larry},
	month = jan,
	year = {2019},
	note = {arXiv:1609.00451 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Theory, Statistics - Methodology},
	pages = {223--234},
	annote = {Comment: Final version to be published in the Journal of the American Statistical Association at https://www.tandfonline.com/doi/abs/10.1080/01621459.2017.1395341?journalCode=uasa20},
	file = {arXiv Fulltext PDF:C\:\\Users\\Kevin\\Zotero\\storage\\WUY59NXL\\Sadinle et al. - 2019 - Least Ambiguous Set-Valued Classifiers with Bounde.pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\Kevin\\Zotero\\storage\\5HPL9FDJ\\1609.html:text/html},
}


@misc{cplecture,
  author        = {Ryan Tibshirani},
  title         = {Advanced Topics in Statistical Learning: Conformal Prediction},
  year          = {2023},
  publisher={Faculdade de Engenharia da Universidade do Porto},
  howpublished = "\url{https://www.stat.berkeley.edu/~ryantibs/statlearn-s23/lectures/conformal.pdf}",

}



@article{lei2017distributionfree,
  title={Distribution-free predictive inference for regression},
  author={Lei, Jing and G’Sell, Max and Rinaldo, Alessandro and Tibshirani, Ryan J and Wasserman, Larry},
  journal={Journal of the American Statistical Association},
  volume={113},
  number={523},
  pages={1094--1111},
  year={2018},
  publisher={Taylor \& Francis}
}


@inproceedings{wang2021tent,
  title={Tent: Fully Test-Time Adaptation by Entropy Minimization},
  author={Wang, Dequan and Shelhamer, Evan and Liu, Shaoteng and Olshausen, Bruno and Darrell, Trevor},
  booktitle={International Conference on Learning Representations},
  year={2021}
}


@inproceedings{Recht2019DoIC,
  title={Do ImageNet Classifiers Generalize to ImageNet?},
  author={Benjamin Recht and Rebecca Roelofs and Ludwig Schmidt and Vaishaal Shankar},
  booktitle={International Conference on Machine Learning},
  year={2019}
}

@article{Hendrycks2018BenchmarkingNN,
  title={Benchmarking Neural Network Robustness to Common Corruptions and Perturbations},
  author={Dan Hendrycks and Thomas Dietterich},
  journal={Proceedings of the International Conference on Learning Representations},
  year={2019}
}

@article{hendrycks2021many,
  title={The Many Faces of Robustness: A Critical Analysis of Out-of-Distribution Generalization},
  author={Dan Hendrycks and Steven Basart and Norman Mu and Saurav Kadavath and Frank Wang and Evan Dorundo and Rahul Desai and Tyler Zhu and Samyak Parajuli and Mike Guo and Dawn Song and Jacob Steinhardt and Justin Gilmer},
  journal={ICCV},
  year={2021}
}

@article{hendrycks2021nae,
  title={Natural Adversarial Examples},
  author={Dan Hendrycks and Kevin Zhao and Steven Basart and Jacob Steinhardt and Dawn Song},
  journal={CVPR},
  year={2021}
}


@inproceedings{saocp,
author = {Bhatnagar, Aadyot and Wang, Huan and Xiong, Caiming and Bai, Yu},
title = {Improved online conformal prediction via strongly adaptive online learning},
year = {2023},
publisher = {JMLR.org},
abstract = {We study the problem of uncertainty quantification via prediction sets, in an online setting where the data distribution may vary arbitrarily over time. Recent work develops online conformal prediction techniques that leverage regret minimization algorithms from the online learning literature to learn prediction sets with approximately valid coverage and small regret. However, standard regret minimization could be insufficient for handling changing environments, where performance guarantees may be desired not only over the full time horizon but also in all (sub-)intervals of time. We develop new online conformal prediction methods that minimize the strongly adaptive regret, which measures the worst-case regret over all intervals of a fixed length. We prove that our methods achieve near-optimal strongly adaptive regret for all interval lengths simultaneously, and approximately valid coverage. Experiments show that our methods consistently obtain better coverage and smaller prediction sets than existing methods on real-world tasks, such as time series forecasting and image classification under distribution shift.},
booktitle = {Proceedings of the 40th International Conference on Machine Learning},
articleno = {100},
numpages = {27},
location = {<conf-loc>, <city>Honolulu</city>, <state>Hawaii</state>, <country>USA</country>, </conf-loc>},
series = {ICML'23}
}
@inproceedings{wilds2021,
  title = {{WILDS}: A Benchmark of in-the-Wild Distribution Shifts},
  author = {Pang Wei Koh and Shiori Sagawa and Henrik Marklund and Sang Michael Xie and Marvin Zhang and Akshay Balsubramani and Weihua Hu and Michihiro Yasunaga and Richard Lanas Phillips and Irena Gao and Tony Lee and Etienne David and Ian Stavness and Wei Guo and Berton A. Earnshaw and Imran S. Haque and Sara Beery and Jure Leskovec and Anshul Kundaje and Emma Pierson and Sergey Levine and Chelsea Finn and Percy Liang},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2021}
}

@InProceedings{niu2022efficient,
  title={Efficient Test-Time Model Adaptation without Forgetting},
  author={Niu, Shuaicheng and Wu, Jiaxiang and Zhang, Yifan and Chen, Yaofo and Zheng, Shijian and Zhao, Peilin and Tan, Mingkui},
  booktitle = {The Internetional Conference on Machine Learning},
  year = {2022}
}

@article{Huang2016DenselyCC,
  title={Densely Connected Convolutional Networks},
  author={Gao Huang and Zhuang Liu and Kilian Q. Weinberger},
  journal={2017 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
  year={2016},
  pages={2261-2269},
  url={https://api.semanticscholar.org/CorpusID:9433631}
}

@article{brown2020language,
  title={Language models are few-shot learners},
  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={1877--1901},
  year={2020}
}

@inproceedings{shao2023reasonnet,
  title={Reasonnet: End-to-end driving with temporal and global reasoning},
  author={Shao, Hao and Wang, Letian and Chen, Ruobing and Waslander, Steven L and Li, Hongsheng and Liu, Yu},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={13723--13733},
  year={2023}
}

@article{Jumper2021HighlyAP,
  title={Highly accurate protein structure prediction with AlphaFold},
  author={John M. Jumper and Richard Evans and Alexander Pritzel and Tim Green and Michael Figurnov and Olaf Ronneberger and Kathryn Tunyasuvunakool and Russ Bates and Augustin Z{\'i}dek and Anna Potapenko and Alex Bridgland and Clemens Meyer and Simon A A Kohl and Andy Ballard and Andrew Cowie and Bernardino Romera-Paredes and Stanislav Nikolov and Rishub Jain and Jonas Adler and Trevor Back and Stig Petersen and David A. Reiman and Ellen Clancy and Michal Zielinski and Martin Steinegger and Michalina Pacholska and Tamas Berghammer and Sebastian Bodenstein and David Silver and Oriol Vinyals and Andrew W. Senior and Koray Kavukcuoglu and Pushmeet Kohli and Demis Hassabis},
  journal={Nature},
  year={2021},
  volume={596},
  pages={583 - 589},
}

@article{Silver2016MasteringTG,
  title={Mastering the game of Go with deep neural networks and tree search},
  author={David Silver and Aja Huang and Chris J. Maddison and Arthur Guez and L. Sifre and George van den Driessche and Julian Schrittwieser and Ioannis Antonoglou and Vedavyas Panneershelvam and Marc Lanctot and Sander Dieleman and Dominik Grewe and John Nham and Nal Kalchbrenner and Ilya Sutskever and Timothy P. Lillicrap and Madeleine Leach and Koray Kavukcuoglu and Thore Graepel and Demis Hassabis},
  journal={Nature},
  year={2016},
  volume={529},
  pages={484-489},
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@article{hu2023Toward,
  author    = {Yafei Hu and Quanting Xie and Vidhi Jain and Jonathan Francis and Jay Patrikar 
                and Nikhil Keetha and Seungchan Kim and Yaqi Xie and Tianyi Zhang and Shibo Zhao 
                and Yu-Quan Chong and Chen Wang and Katia Sycara and Matthew Johnson-Roberson 
                and Dhruv Batra and Xiaolong Wang and Sebastian Scherer and Zsolt Kira and 
                Fei Xia and Yonatan Bisk},
  title     = {Toward General-Purpose Robots via Foundation Models: A Survey and Meta-Analysis},
  booktitle = {arXiv preprint: arXiv:2312.08782 },
  year      = {2023},
}


@inproceedings{lu2022fair,
  title={Fair conformal predictors for applications in medical imaging},
  author={Lu, Charles and Lemay, Andr{\'e}anne and Chang, Ken and H{\"o}bel, Katharina and Kalpathy-Cramer, Jayashree},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={36},
  number={11},
  pages={12008--12016},
  year={2022}
}

@article{Bergquist_2022,
   title={Uncertainty in lung cancer stage for survival estimation via set‐valued classification},
   volume={41},
   ISSN={1097-0258},
   url={http://dx.doi.org/10.1002/sim.9448},
   DOI={10.1002/sim.9448},
   number={19},
   journal={Statistics in Medicine},
   publisher={Wiley},
   author={Bergquist, Savannah and Brooks, Gabriel A. and Landrum, Mary Beth and Keating, Nancy L. and Rose, Sherri},
   year={2022},
   month=jun, pages={3772–3788} }

@article{cpdrug,
   title={Predicting With Confidence: Using Conformal Prediction in Drug Discovery},
   journal={Journal of pharmaceutical sciences},
   publisher={Wiley},
   author={Jonathan Alvarsson and Staffan Arvidsson McShane and Ulf Norinder and Ola Spjuth},
   year={2021},
   month=jun, pages={42-49} 
}

@inproceedings{knowno2023,
    title={Robots That Ask For Help: Uncertainty Alignment for Large Language Model Planners},
    author={Ren, Allen Z. and Dixit, Anushri and Bodrova, Alexandra and Singh, Sumeet and Tu, Stephen and Brown, Noah and Xu, Peng and Takayama, Leila and Xia, Fei and Varley, Jake and Xu, Zhenjia and Sadigh, Dorsa and Zeng, Andy and Majumdar, Anirudha},
    booktitle={Proceedings of the Conference on Robot Learning (CoRL)},
    year={2023}
}

@article{lindemann2023safe,
  title={Safe planning in dynamic environments using conformal prediction},
  author={Lindemann, Lars and Cleaveland, Matthew and Shim, Gihyun and Pappas, George J},
  journal={IEEE Robotics and Automation Letters},
  year={2023},
  publisher={IEEE}
}

@inproceedings{bhatnagar2023improved,
  title={Improved online conformal prediction via strongly adaptive online learning},
  author={Bhatnagar, Aadyot and Wang, Huan and Xiong, Caiming and Bai, Yu},
  booktitle={International Conference on Machine Learning},
  pages={2337--2363},
  year={2023},
  organization={PMLR}
}

@article{barber2023conformal,
  title={Conformal prediction beyond exchangeability},
  author={Barber, Rina Foygel and Candes, Emmanuel J and Ramdas, Aaditya and Tibshirani, Ryan J},
  journal={The Annals of Statistics},
  volume={51},
  number={2},
  pages={816--845},
  year={2023},
  publisher={Institute of Mathematical Statistics}
}

@article{tibshirani2019conformal,
  title={Conformal prediction under covariate shift},
  author={Tibshirani, Ryan J and Foygel Barber, Rina and Candes, Emmanuel and Ramdas, Aaditya},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@inproceedings{zaffran2022adaptive,
  title={Adaptive conformal predictions for time series},
  author={Zaffran, Margaux and F{\'e}ron, Olivier and Goude, Yannig and Josse, Julie and Dieuleveut, Aymeric},
  booktitle={International Conference on Machine Learning},
  pages={25834--25866},
  year={2022},
  organization={PMLR}
}

@article{feldman2022achieving,
  title={Achieving risk control in online learning settings},
  author={Feldman, Shai and Ringel, Liran and Bates, Stephen and Romano, Yaniv},
  journal={arXiv preprint arXiv:2205.09095},
  year={2022}
}

@inproceedings{Zinkevichonline,
  author       = {Martin Zinkevich},
  title        = {Online Convex Programming and Generalized Infinitesimal Gradient Ascent},
  booktitle    = {Machine Learning, Proceedings of the Twentieth International Conference
                  {(ICML} 2003), August 21-24, 2003, Washington, DC, {USA}},
  pages        = {928--936},
  publisher    = {{AAAI} Press},
  year         = {2003},
  url          = {http://www.aaai.org/Library/ICML/2003/icml03-120.php},
  timestamp    = {Thu, 13 Dec 2012 12:01:46 +0100},
  biburl       = {https://dblp.org/rec/conf/icml/Zinkevich03.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}


@article{nado2020evaluating,
  title={Evaluating prediction-time batch normalization for robustness under covariate shift},
  author={Nado, Zachary and Padhy, Shreyas and Sculley, D and D'Amour, Alexander and Lakshminarayanan, Balaji and Snoek, Jasper},
  journal={arXiv preprint arXiv:2006.10963},
  year={2020}
}

@article{schneider2020improving,
  title={Improving robustness against common corruptions by covariate shift adaptation},
  author={Schneider, Steffen and Rusak, Evgenia and Eck, Luisa and Bringmann, Oliver and Brendel, Wieland and Bethge, Matthias},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={11539--11551},
  year={2020}
}

@article{khurana2021sita,
  title={Sita: Single image test-time adaptation},
  author={Khurana, Ansh and Paul, Sujoy and Rai, Piyush and Biswas, Soma and Aggarwal, Gaurav},
  journal={arXiv preprint arXiv:2112.02355},
  year={2021}
}

@article{zhang2022memo,
  title={Memo: Test time robustness via adaptation and augmentation},
  author={Zhang, Marvin and Levine, Sergey and Finn, Chelsea},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={38629--38642},
  year={2022}
}

@inproceedings{song2023ecotta,
  title={Ecotta: Memory-efficient continual test-time adaptation via self-distilled regularization},
  author={Song, Junha and Lee, Jungsoo and Kweon, In So and Choi, Sungha},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={11920--11929},
  year={2023}
}


@inproceedings{PapadopoulosCpKNN,
author = {Papadopoulos, Harris and Gammerman, Alex and Vovk, Volodya},
title = {Normalized nonconformity measures for regression Conformal Prediction},
year = {2008},
isbn = {9780889867109},
publisher = {ACTA Press},
address = {USA},
abstract = {In this paper we apply Conformal Prediction (CP) to the k-Nearest Neighbours Regression (k-NNR) algorithm and propose a way of extending the typical nonconformity measure used for regression so far. Unlike traditional regression methods which produce point predictions, Conformal Predictors output predictive regions that satisfy a given confidence level. When the regular regression nonconformity measure is used the resulting predictive regions have more or less the same width for all examples in the test set. However, it would be more natural for the size of the regions to vary according to how difficult to predict each example is. We define two new nonconformity measures, which produce predictive regions of variable width depending on the expected accuracy of the algorithm on each example. As a consequence, the resulting predictive regions are in most cases much tighter than those produced by the simple regression measure.},
booktitle = {Proceedings of the 26th IASTED International Conference on Artificial Intelligence and Applications},
pages = {64–69},
numpages = {6},
keywords = {regression, nonconformity measure, machine learning, Nearest Neighbours, Conformal Prediction},
location = {Innsbruck, Austria},
series = {AIA '08}
}

@InProceedings{izbicki2019flexible,
  title = 	 {Flexible distribution-free conditional predictive bands using density estimators},
  author =       {Izbicki, Rafael and Shimizu, Gilson and Stern, Rafael},
  booktitle = 	 {Proceedings of the Twenty Third International Conference on Artificial Intelligence and Statistics},
  pages = 	 {3068--3077},
  year = 	 {2020},
  editor = 	 {Chiappa, Silvia and Calandra, Roberto},
  volume = 	 {108},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {26--28 Aug},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v108/izbicki20a/izbicki20a.pdf},
  url = 	 {https://proceedings.mlr.press/v108/izbicki20a.html},
  abstract = 	 {Conformal methods create prediction bands that control average coverage assuming solely i.i.d. data. Besides average coverage, one might also desire to control conditional coverage, that is, coverage for every new testing point. However, without strong assumptions, conditional coverage is unachievable. Given this limitation, the literature has focused on methods with asymptotical conditional coverage. In order to obtain this property, these methods require strong conditions on the dependence between the target variable and the features. We introduce two conformal methods based on conditional density estimators that do not depend on this type of assumption to obtain asymptotic conditional coverage: Dist-split and CD-split. While Dist-split asymptotically obtains optimal intervals, which are easier to interpret than general regions, CD-split obtains optimal size regions, which are smaller than intervals. CD-split also obtains local coverage by creating prediction bands locally on a partition of the features space. This partition is data-driven and scales to high-dimensional settings. In a wide variety of simulated scenarios, our methods have a better control of conditional coverage and have smaller length than previously proposed methods.}
}


@article{romano2019conformalized,
  title={Conformalized quantile regression},
  author={Romano, Yaniv and Patterson, Evan and Candes, Emmanuel},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@INPROCEEDINGS{Johanssonbagged,
  author={Johansson, Ulf and Sönströd, Cecilia and Linusson, Henrik},
  booktitle={2015 International Joint Conference on Neural Networks (IJCNN)}, 
  title={Efficient conformal regressors using bagged neural nets}, 
  year={2015},
  volume={},
  number={},
  pages={1-8},
  keywords={Space heating;Artificial neural networks;Calibration;Prediction algorithms;Iterative closest point algorithm;Conformal prediction;Regression;Bagging;Neural networks},
  doi={10.1109/IJCNN.2015.7280763}}


@inproceedings{seedat2023improving,
  title={Improving adaptive conformal prediction using self-supervised learning},
  author={Seedat, Nabeel and Jeffares, Alan and Imrie, Fergus and van der Schaar, Mihaela},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={10160--10177},
  year={2023},
  organization={PMLR}
}


@InProceedings{rossellini24aUncertCQR,
  title = 	 { Integrating Uncertainty Awareness into Conformalized Quantile Regression },
  author =       {Rossellini, Raphael and Foygel Barber, Rina and Willett, Rebecca},
  booktitle = 	 {Proceedings of The 27th International Conference on Artificial Intelligence and Statistics},
  pages = 	 {1540--1548},
  year = 	 {2024},
  editor = 	 {Dasgupta, Sanjoy and Mandt, Stephan and Li, Yingzhen},
  volume = 	 {238},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {02--04 May},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v238/rossellini24a/rossellini24a.pdf},
  url = 	 {https://proceedings.mlr.press/v238/rossellini24a.html},
  abstract = 	 { Conformalized Quantile Regression (CQR) is a recently proposed method for constructing prediction intervals for a response $Y$ given covariates $X$, without making distributional assumptions. However, existing constructions of CQR can be ineffective for problems where the quantile regressors perform better in certain parts of the feature space than others. The reason is that the prediction intervals of CQR do not distinguish between two forms of uncertainty: first, the variability of the conditional distribution of $Y$ given $X$ (i.e., aleatoric uncertainty), and second, our uncertainty in estimating this conditional distribution (i.e., epistemic uncertainty). This can lead to intervals that are overly narrow in regions where epistemic uncertainty is high. To address this, we propose a new variant of the CQR methodology, Uncertainty-Aware CQR (UACQR), that explicitly separates these two sources of uncertainty to adjust quantile regressors differentially across the feature space. Compared to CQR, our methods enjoy the same distribution-free theoretical coverage guarantees, while demonstrating in our experiments stronger conditional coverage properties in simulated settings and real-world data sets alike. }
}

@article{press2024rdumb,
  title={RDumb: A simple approach that questions our progress in continual test-time adaptation},
  author={Press, Ori and Schneider, Steffen and K{\"u}mmerer, Matthias and Bethge, Matthias},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2023}
}

@inproceedings{yang23cvpr-object,
  title={Object pose estimation with statistical guarantees: Conformal keypoint detection and geometric uncertainty propagation},
  author={Yang, Heng and Pavone, Marco},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={8947--8958},
  year={2023}
}

@article{gao24arxiv-closure,
  title={CLOSURE: Fast Quantification of Pose Uncertainty Sets},
  author={Gao, Yihuai and Tang, Yukai and Qi, Han and Yang, Heng},
  journal={arXiv preprint arXiv:2403.09990},
  year={2024}
}


@INPROCEEDINGS{lekeufack2023conformal,
  author={Lekeufack, Jordan and Angelopoulos, Anastasios N. and Bajcsy, Andrea and Jordan, Michael I. and Malik, Jitendra},
  booktitle={2024 IEEE International Conference on Robotics and Automation (ICRA)}, 
  title={Conformal Decision Theory: Safe Autonomous Decisions from Imperfect Predictions}, 
  year={2024},
  volume={},
  number={},
  pages={11668-11675},
  keywords={Robot motion;Pedestrians;Machine learning algorithms;Decision theory;Switches;Prediction algorithms;Throughput},
  doi={10.1109/ICRA57147.2024.10610041}}


@article{kumar2023conformal,
  title={Conformal prediction with large language models for multi-choice question answering},
  author={Kumar, Bhawesh and Lu, Charlie and Gupta, Gauri and Palepu, Anil and Bellamy, David and Raskar, Ramesh and Beam, Andrew},
  journal={arXiv preprint arXiv:2305.18404},
  year={2023}
}

@inproceedings{
quach2024conformal,
title={Conformal Language Modeling},
author={Victor Quach and Adam Fisch and Tal Schuster and Adam Yala and Jae Ho Sohn and Tommi S. Jaakkola and Regina Barzilay},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024},
url={https://openreview.net/forum?id=pzUhfQ74c5}
}

@article{bastani2022practical,
  title={Practical adversarial multivalid conformal prediction},
  author={Bastani, Osbert and Gupta, Varun and Jung, Christopher and Noarov, Georgy and Ramalingam, Ramya and Roth, Aaron},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={29362--29373},
  year={2022}
}

@inproceedings{guo2017calibration,
  title={On calibration of modern neural networks},
  author={Guo, Chuan and Pleiss, Geoff and Sun, Yu and Weinberger, Kilian Q},
  booktitle={International conference on machine learning},
  pages={1321--1330},
  year={2017},
  organization={PMLR}
}

@article{liang2023holistic,
  title={Holistic Evaluation of Language Models},
  author={Liang, Percy and Bommasani, Rishi and Lee, Tony and Tsipras, Dimitris and Soylu, Dilara and Yasunaga, Michihiro and Zhang, Yian and Narayanan, Deepak and Wu, Yuhuai and Kumar, Ananya and others},
  journal={Transactions on Machine Learning Research},
  year={2023}
}

@article{grandvalet2004semi,
  title={Semi-supervised learning by entropy minimization},
  author={Grandvalet, Yves and Bengio, Yoshua},
  journal={Advances in Neural Information Processing Systems},
  volume={17},
  year={2004}
}

@article{minderer2021revisiting,
  title={Revisiting the calibration of modern neural networks},
  author={Minderer, Matthias and Djolonga, Josip and Romijnders, Rob and Hubis, Frances and Zhai, Xiaohua and Houlsby, Neil and Tran, Dustin and Lucic, Mario},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={15682--15694},
  year={2021}
}

@inproceedings{deng2009imagenet,
  title={Imagenet: A large-scale hierarchical image database},
  author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Li, Kai and Fei-Fei, Li},
  booktitle={2009 IEEE conference on computer vision and pattern recognition},
  pages={248--255},
  year={2009},
  organization={Ieee}
}

@article{beery2020iwildcam,
    title={The iWildCam 2020 Competition Dataset},
    author={Beery, Sara and Cole, Elijah and Gjoka, Arvi},
    journal={arXiv preprint arXiv:2004.10340},
    year={2020}
}

@inproceedings{taylor2019rxrx1,
  author = {Taylor, J. and Earnshaw, B. and Mabey, B. and Victors, M. and  Yosinski, J.},
  title = {RxRx1: An Image Set for Cellular Morphological Variation Across Many Experimental Batches.},
  year = {2019},
  booktitle = {International Conference on Learning Representations (ICLR)},
  booksubtitle = {AI for Social Good Workshop},
}

@inproceedings{christie2018functional,
  title={Functional Map of the World},
  author={Christie, Gordon and Fendley, Neil and Wilson, James and Mukherjee, Ryan},
  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},  
  year={2018}
}

@inproceedings{
angelopoulos2021uncertainty,
title={Uncertainty Sets for Image Classifiers using Conformal Prediction},
author={Anastasios Nikolas Angelopoulos and Stephen Bates and Michael Jordan and Jitendra Malik},
booktitle={International Conference on Learning Representations},
year={2021},
url={https://openreview.net/forum?id=eNdiU_DbM9}
}

@inproceedings{press2024entropy,
  title={The Entropy Enigma: Success and Failure of Entropy Minimization},
  author={Press, Ori and Shwartz-Ziv, Ravid and LeCun, Yann and Bethge, Matthias},
  booktitle={International Conference on Machine Learning},
  year={2024},
  organization={PMLR}
}

@article{cauchois2024robust,
  title={Robust validation: Confident predictions even when distributions shift},
  author={Cauchois, Maxime and Gupta, Suyash and Ali, Alnur and Duchi, John C},
  journal={Journal of the American Statistical Association},
  pages={1--66},
  year={2024},
  publisher={Taylor \& Francis}
}


@inproceedings{mohri2024language,
author = {Mohri, Christopher and Hashimoto, Tatsunori},
title = {Language models with conformal factuality guarantees},
year = {2024},
publisher = {JMLR.org},
abstract = {Guaranteeing the correctness and factuality of language model (LM) outputs is a major open problem. In this work, we propose conformal factuality, a framework that can ensure high probability correctness guarantees for LMs by connecting language modeling and conformal prediction. We observe that the correctness of an LM output is equivalent to an uncertainty quantification problem, where the uncertainty sets are defined as the entailment set of an LM's output. Using this connection, we show that conformal prediction in language models corresponds to a back-off algorithm that provides high probability correctness guarantees by progressively making LM outputs less specific (and expanding the associated uncertainty sets). This approach applies to any blackbox LM and requires very few human-annotated samples. Evaluations of our approach on closed book QA (FActScore, NaturalQuestions) and reasoning tasks (MATH) show that our approach can provide 80-90\% correctness guarantees while retaining the majority of the LM's original output.},
booktitle = {Proceedings of the 41st International Conference on Machine Learning},
articleno = {1468},
numpages = {19},
location = {Vienna, Austria},
series = {ICML'24}
}

@inproceedings{gui2024conformal,
 author = {Gui, Yu and Jin, Ying and Ren, Zhimei},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Globerson and L. Mackey and D. Belgrave and A. Fan and U. Paquet and J. Tomczak and C. Zhang},
 pages = {73884--73919},
 publisher = {Curran Associates, Inc.},
 title = {Conformal Alignment: Knowing When to Trust Foundation Models with Guarantees},
 volume = {37},
 year = {2024}
}


@article{cherian2024large,
  title={Large language model validity via enhanced conformal prediction methods},
  author={Cherian, John and Gibbs, Isaac and Candes, Emmanuel},
  journal={Advances in Neural Information Processing Systems},
  volume={37},
  pages={114812--114842},
  year={2024}
}

@InProceedings{ai2024not,
  title = 	 {Not all distributional shifts are equal: Fine-grained robust conformal inference},
  author =       {Ai, Jiahao and Ren, Zhimei},
  booktitle = 	 {Proceedings of the 41st International Conference on Machine Learning},
  pages = 	 {641--665},
  year = 	 {2024},
  editor = 	 {Salakhutdinov, Ruslan and Kolter, Zico and Heller, Katherine and Weller, Adrian and Oliver, Nuria and Scarlett, Jonathan and Berkenkamp, Felix},
  volume = 	 {235},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {21--27 Jul},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v235/main/assets/ai24a/ai24a.pdf},
  url = 	 {https://proceedings.mlr.press/v235/ai24a.html},
  abstract = 	 {We introduce a fine-grained framework for uncertainty quantification of predictive models under distributional shifts. This framework distinguishes the shift in covariate distributions from that in the conditional relationship between the outcome ($Y$) and the covariates ($X$). We propose to reweight the training samples to adjust for an identifiable shift in covariate distribution while protecting against the worst-case conditional distribution shift bounded in an $f$-divergence ball. Based on ideas from conformal inference and distributionally robust learning, we present an algorithm that outputs (approximately) valid and efficient prediction intervals in the presence of distributional shifts. As a use case, we apply the framework to sensitivity analysis of individual treatment effects with hidden confounding. The proposed methods are evaluated in simulations and four real data applications, demonstrating superior robustness and efficiency compared with existing benchmarks.}
}

@inproceedings{ge2024optimal,
 author = {Ge, Jiawei and Mukherjee, Debarghya and Fan, Jianqing},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Globerson and L. Mackey and D. Belgrave and A. Fan and U. Paquet and J. Tomczak and C. Zhang},
 pages = {73605--73637},
 publisher = {Curran Associates, Inc.},
 title = {Optimal Aggregation of Prediction Intervals under Unsupervised Domain Shift},
 volume = {37},
 year = {2024}
}


@inproceedings{hendrycks2017baseline,
  title={A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks},
  author={Hendrycks, Dan and Gimpel, Kevin},
  booktitle={International Conference on Learning Representations},
  year={2017}
}

@inproceedings{kang2024deep,
  title={Deep Neural Networks Tend To Extrapolate Predictably},
  author={Kang, Katie and Setlur, Amrith and Tomlin, Claire and Levine, Sergey},
  booktitle={International Conference on Learning Representations},
year={2024}
}


@article{denis2017confidence,
  title={Confidence sets with expected sizes for multiclass classification},
  author={Denis, Christophe and Hebiri, Mohamed},
  journal={Journal of Machine Learning Research},
  volume={18},
  number={102},
  pages={1--28},
  year={2017}
}

@article{romano2020classification,
  title={Classification with valid and adaptive coverage},
  author={Romano, Yaniv and Sesia, Matteo and Candes, Emmanuel},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={3581--3591},
  year={2020}
}

@article{gibbs2024conformal,
  title={Conformal inference for online prediction with arbitrary distribution shifts},
  author={Gibbs, Isaac and Cand{\`e}s, Emmanuel J},
  journal={Journal of Machine Learning Research},
  volume={25},
  number={162},
  pages={1--36},
  year={2024}
}