@inproceedings{ijcai2021-knowledge,
  title     = {{Neighborhood Intervention Consistency: Measuring Confidence for Knowledge Graph Link Prediction}},
  author    = {K. Wang and Y. Liu and Q. Sheng},
  booktitle = {Proceedings of the Thirtieth International Joint Conference on
               Artificial Intelligence, {IJCAI}},
  pages     = {2090--2096},
  year      = {2021},
  month     = {8},
}

@article{Code,
  author       = {L. Leman and G. Chouraqui},
  title        = {Geometric Calibrator source code},
  howpublished = {Github},
  note ={\url{https://github.com/NoSleepDeveloper/Geometric-Calibrator}},
  year         = {2022}
}


@inproceedings{NEURIPS2019_8ca01ea9,
 author = {Kull, M. and Perello Nieto, M. and K\"{a}ngsepp, M. and Silva Filho, T. and Song, H. and Flach, P.},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 title = {{Beyond Temperature Scaling: Obtaining Well-calibrated Multi-class Probabilities with Dirichlet Calibration}},
  volume = {32},
 year = {2019}
}

@InProceedings{pmlr-v70-guo17a,
  title = 	 {{On Calibration of Modern Neural Networks}},
  author =       {C. Guo and G. Pleiss and Y. Sun and K. Q. Weinberger},
  booktitle = 	 {Proceedings of the 34th International Conference on Machine Learning},
  pages = 	 {1321--1330},
  year = 	 {2017},
  volume = 	 {70},
  series = 	 {Proceedings of Machine Learning Research},
  pdf = 	 {http://proceedings.mlr.press/v70/guo17a/guo17a.pdf},
  abstract = 	 {Confidence calibration – the problem of predicting probability estimates representative of the true correctness likelihood – is important for classification models in many applications. We discover that modern neural networks, unlike those from a decade ago, are poorly calibrated. Through extensive experiments, we observe that depth, width, weight decay, and Batch Normalization are important factors influencing calibration. We evaluate the performance of various post-processing calibration methods on state-of-the-art architectures with image and document classification datasets. Our analysis and experiments not only offer insights into neural network learning, but also provide a simple and straightforward recipe for practical settings: on most datasets, temperature scaling – a single-parameter variant of Platt Scaling – is surprisingly effective at calibrating predictions.},
}

@inproceedings{ThulasidasanCBB19,
  title = {{On Mixup Training: Improved Calibration and Predictive Uncertainty for Deep Neural Networks}},
  author = {S. Thulasidasan and G. Chennupati and J. A. Bilmes and T. Bhattacharya and S. Michalak},
  year = {2019},
  researchr = {https://researchr.org/publication/ThulasidasanCBB19},
  cites = {0},
  citedby = {0},
  pages = {13888-13899},
  booktitle = {Advances in Neural Information Processing Systems 32: Annual Conference on Neural Information Processing Systems, NeurIPS},
}


@inproceedings{gupta2021calibration,
  author    = {Chirag Gupta and
               Aaditya Ramdas},
  editor    = {Marina Meila and
               Tong Zhang},
  title     = {Distribution-Free Calibration Guarantees for Histogram Binning without
               Sample Splitting},
  booktitle = {Proceedings of the 38th International Conference on Machine Learning,
               {ICML} 2021, 18-24 July 2021, Virtual Event},
  series    = {Proceedings of Machine Learning Research},
  volume    = {139},
  pages     = {3942--3952},
  publisher = {{PMLR}},
  year      = {2021},
  url       = {http://proceedings.mlr.press/v139/gupta21b.html},
  timestamp = {Wed, 25 Aug 2021 17:11:17 +0200},
  biburl    = {https://dblp.org/rec/conf/icml/GuptaR21.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{
Ashukha2020Pitfalls,
title={{Pitfalls of In-Domain Uncertainty Estimation and Ensembling in Deep Learning}},
author={A. Ashukha and A. Lyzhov and D. Molchanov and D. Vetrov},
booktitle={International Conference on Learning Representations},
year={2020},
}

@inproceedings{NEURIPS2019_f1748d6b,
 author = {M\"{u}ller, R. and Kornblith, S. and Hinton, G. E},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {4696-4705},
 title = {When does label smoothing help?},
  volume = {32},
 year = {2019}
}


@inproceedings{NEURIPS2019_a2b15837,
 author = {Hendrycks, D. and Mazeika, M. and Kadavath, S. and Song, D.},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 title = {{Using Self-Supervised Learning Can Improve Model Robustness and Uncertainty}},
 volume = {32},
 year = {2019}
}

@inproceedings{NIPS2017_84ddfb34,
 author = {Gal, Y. and Hron, J. and Kendall, A.},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 title = {{Concrete Dropout}},
 volume = {30},
 year = {2017}
}

@inproceedings{SnapshotEnsemble,
  author    = {G. Huang and
               Y. Li and
               G. Pleiss and
               Z. Liu and
               J. E. Hopcroft and
               K. Q. Weinberger},
  title     = {Snapshot Ensembles: Train 1, Get {M} for Free},
  booktitle = {5th International Conference on Learning Representations, {ICLR}},
  year      = {2017},
  timestamp = {Thu, 25 Jul 2019 14:26:05 +0200},
  biburl    = {https://dblp.org/rec/conf/iclr/HuangLP0HW17.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{SWAG,
 author = {Maddox, W. J. and Izmailov, P. and Garipov, T. and Vetrov, D. P. and Wilson, A. G.},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {12},
 title = {{A Simple Baseline for Bayesian Uncertainty in Deep Learning}},
 volume = {32},
 year = {2019}
}


@inproceedings{FastEnsembling,
  author    = {T. Garipov and
               P. Izmailov and
               D. Podoprikhin and
               D. P. Vetrov and
               A. G. Wilson},
  title     = {{Loss Surfaces, Mode Connectivity, and Fast Ensembling of DNNs}},
  booktitle = {Advances in Neural Information Processing Systems 31: Annual Conference
               on Neural Information Processing Systems 2018, NeurIPS 2018},
  pages     = {8803--8812},
  year      = {2018},
  timestamp = {Thu, 21 Jan 2021 15:15:21 +0100},
  biburl    = {https://dblp.org/rec/conf/nips/GaripovIPVW18.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{DeepEnsembles,
author = {Lakshminarayanan, B. and Pritzel, A. and Blundell, C.},
title = {Simple and Scalable Predictive Uncertainty Estimation Using Deep Ensembles},
year = {2017},
abstract = {Deep neural networks (NNs) are powerful black box predictors that have recently achieved impressive performance on a wide spectrum of tasks. Quantifying predictive uncertainty in NNs is a challenging and yet unsolved problem. Bayesian NNs, which learn a distribution over weights, are currently the state-of-the-art for estimating predictive uncertainty; however these require significant modifications to the training procedure and are computationally expensive compared to standard (non-Bayesian) NNs. We propose an alternative to Bayesian NNs that is simple to implement, readily parallelizable, requires very little hyperparameter tuning, and yields high quality predictive uncertainty estimates. Through a series of experiments on classification and regression benchmarks, we demonstrate that our method produces well-calibrated uncertainty estimates which are as good or better than approximate Bayesian NNs. To assess robustness to dataset shift, we evaluate the predictive uncertainty on test examples from known and unknown distributions, and show that our method is able to express higher uncertainty on out-of-distribution examples. We demonstrate the scalability of our method by evaluating predictive uncertainty estimates on ImageNet.},
booktitle = {Proceedings of the 31st International Conference on Neural Information Processing Systems},
pages = {6405–6416},
numpages = {12},
series = {NIPS'17}
}


@InProceedings{pmlr-v48-gal16,
  title = 	 {{Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning}},
  author = 	 {Gal, Y. and Ghahramani, Z.},
  booktitle = 	 {Proceedings of The 33rd International Conference on Machine Learning},
  pages = 	 {1050--1059},
  year = 	 {2016},
  volume = 	 {48},
  series = 	 {Proceedings of Machine Learning Research},
  pdf = 	 {http://proceedings.mlr.press/v48/gal16.pdf},
  abstract = 	 {Deep learning tools have gained tremendous attention in applied machine learning. However such tools for regression and classification do not capture model uncertainty. In comparison, Bayesian models offer a mathematically grounded framework to reason about model uncertainty, but usually come with a prohibitive computational cost. In this paper we develop a new theoretical framework casting dropout training in deep neural networks (NNs) as approximate Bayesian inference in deep Gaussian processes. A direct result of this theory gives us tools to model uncertainty with dropout NNs â€“ extracting information from existing models that has been thrown away so far. This mitigates the problem of representing uncertainty in deep learning without sacrificing either computational complexity or test accuracy. We perform an extensive study of the properties of dropoutâ€™s uncertainty. Various network architectures and non-linearities are assessed on tasks of regression and classification, using MNIST as an example. We show a considerable improvement in predictive log-likelihood and RMSE compared to existing state-of-the-art methods, and finish by using dropoutâ€™s uncertainty in deep reinforcement learning.}
}





@InProceedings{pmlr-v97-hendrycks19a,
  title = 	 {{Using Pre-Training Can Improve Model Robustness and Uncertainty}},
  author =       {Hendrycks, D. and Lee, K. and Mazeika, M.},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {2712--2721},
  year = 	 {2019},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  pdf = 	 {http://proceedings.mlr.press/v97/hendrycks19a/hendrycks19a.pdf},
  abstract = 	 {He et al. (2018) have called into question the utility of pre-training by showing that training from scratch can often yield similar performance to pre-training. We show that although pre-training may not improve performance on traditional classification metrics, it improves model robustness and uncertainty estimates. Through extensive experiments on label corruption, class imbalance, adversarial examples, out-of-distribution detection, and confidence calibration, we demonstrate large gains from pre-training and complementary effects with task-specific methods. We show approximately a 10\% absolute improvement over the previous state-of-the-art in adversarial robustness. In some cases, using pre-training without task-specific methods also surpasses the state-of-the-art, highlighting the need for pre-training when evaluating future methods on robustness and uncertainty tasks.}
}



@InProceedings{pmlr-v161-ma21a,
  title = 	 {{Improving Uncertainty Calibration of Deep Neural Networks via Truth Discovery and Geometric Optimization}},
  author =       {Ma, C. and Huang, Z. and Xian, J. and Gao, M. and Xu, J.},
  booktitle = 	 {Proceedings of the Thirty-Seventh Conference on Uncertainty in Artificial Intelligence},
  pages = 	 {75--85},
  year = 	 {2021},
  volume = 	 {161},
  series = 	 {Proceedings of Machine Learning Research},
  pdf = 	 {https://proceedings.mlr.press/v161/ma21a/ma21a.pdf},
  abstract = 	 {Deep Neural Networks (DNNs), despite their tremendous success in recent years, could still cast doubts on their predictions due to the intrinsic uncertainty associated with their learning process. Ensemble techniques and post-hoc calibrations are two types of approaches that have individually shown promise in improving the uncertainty calibration of DNNs. However, the synergistic effect of the two types of methods has not been well explored. In this paper, we propose a truth discovery framework to integrate ensemble-based and post-hoc calibration methods. Using the geometric variance of the ensemble candidates as a good indicator for sample uncertainty, we design an accuracy-preserving truth estimator with provably no accuracy drop. Furthermore, we show that post-hoc calibration can also be enhanced by truth discovery-regularized optimization. On large-scale datasets including CIFAR and ImageNet, our method shows consistent improvement against state-of-the-art calibration approaches on both histogram-based and kernel density-based evaluation metrics. Our code is available at https://github.com/horsepurve/truly-uncertain.}
}




@inproceedings{Niculescu2005Predicting,
author = {A. Niculescu-Mizil and R. Caruana},
title = {{Predicting Good Probabilities with Supervised Learning}},
year = {2005},
booktitle = {Proceedings of the 22nd International Conference on Machine Learning},
pages = {625–632},
numpages = {8},
location = {Bonn, Germany},
series = {ICML}
}
@inproceedings{CNNCalibration,
author = {C. Guo and G. Pleiss and Y. Sun and  K.Q. Weinberger},
title = {{On Calibration of Modern Neural Networks}},
year = {2017},
booktitle = {Proceedings of the 34th International Conference on Machine Learning},
pages = {1321–1330},
numpages = {10},
location = {Sydney, NSW, Australia},
series = {ICML}
}


@article{Ding2020,
  author    = {Z.Ding and
               X.Han and
               P.Liu and
               M.Niethammer},
  title     = {Local Temperature Scaling for Probability Calibration},
  journal   = {CoRR},
  volume    = {abs/2008.05105},
  year      = {2020},
  biburl    = {https://dblp.org/rec/journals/corr/abs-2008-05105.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{wenger2019,
  author    = {J.Wenger and
               H.Kjellstrm and
               R.Triebel},
  title     = {Non-Parametric Calibration for Classification},
  journal   = {CoRR},
  volume    = {abs/1906.04933},
  year      = {2019},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1906-04933.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{Survey1,
	author = {A. Niculescu-Mizil and R. Caruana},
	title = {{An Empirical Comparison of Supervised Learning Algorithms}},
	booktitle = {Proceedings of the 23rd International Conference on Machine Learning},
	series = {ICML},
	year = {2006},
	location = {Pittsburgh, Pennsylvania, USA},
	pages = {161--168},
	numpages = {8},
	acmid = {1143865}
} 

@Article{Sun2007,
	author={Y. Sun and S. Todorovic and J. Li},
	title={{Increasing the Robustness of Boosting Algorithms within the Linear-programming Framework}},
	journal="The Journal of VLSI Signal Processing Systems for Signal, Image, and Video Technology",
	year="2007",
	volume="48",
	number="1",
	pages="5--20",
	abstract="AdaBoost has been successfully used in many signal classification systems. However, it has been observed that on highly noisy data AdaBoost easily leads to overfitting, which seriously constrains its applicability. In this paper, we address this problem by proposing a new regularized boosting algorithm LPnorm2-AdaBoost (LPNA). This algorithm arises from a close connection between AdaBoost and linear programming. In the algorithm, skewness of the data distribution is controlled during the training to prevent outliers from spoiling decision boundaries. To this end, a smooth convex penalty function (l                        2 norm) is introduced in the objective function of a minimax problem. A stabilized column generation technique is used to transform the optimization problem into a simple linear programming problem. The effectiveness of the proposed algorithm is demonstrated through experiments on many diverse datasets.",
}


@inproceedings{GBTheory,
	author = {L. Mason and  J. Baxter and P. Bartlett and M. Frean},
	title = {{Boosting Algorithms As Gradient Descent}},
	booktitle = {Proceedings of the 12th International Conference on Neural Information Processing Systems},
	series = {NIPS},
	year = {1999},
	location = {Denver, CO},
	pages = {512--518},
	numpages = {7},
	acmid = {3009730},
} 



@article{MNIST,
  author = {Y. LeCun and C. Cortes},
  biburl = {https://www.bibsonomy.org/bibtex/2935bad99fa1f65e03c25b315aa3c1032/mhwombat},
  groups = {public},
  howpublished = {http://yann.lecun.com/exdb/mnist/},
  interhash = {21b9d0558bd66279df9452562df6e6f3},
  intrahash = {935bad99fa1f65e03c25b315aa3c1032},
  keywords = {MSc _checked character_recognition mnist network neural},
  lastchecked = {2016-01-14 14:24:11},
  timestamp = {2016-07-12T19:25:30.000+0200},
  title = {{MNIST} handwritten digit database},
  note = {\url{http://yann.lecun.com/exdb/mnist/}},
  username = {mhwombat},
  year = 2010
}


@article{CIFAR10,
title= {CIFAR-10 (Canadian Institute for Advanced Research)},
author= {A.Krizhevsky and V.Nair and G.Hinton},
year= {2009},
url= {http://www.cs.toronto.edu/~kriz/cifar.html},
abstract= {The CIFAR-10 dataset consists of 60000 32x32 colour images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images. 
The dataset is divided into five training batches and one test batch, each with 10000 images. The test batch contains exactly 1000 randomly-selected images from each class. The training batches contain the remaining images in random order, but some training batches may contain more images from one class than another. Between them, the training batches contain exactly 5000 images from each class. },
keywords= {Dataset},
terms= {}
}

@article{fashion_MNIST,
  author    = {H. Xiao and K. Rasul and R. Vollgraf},
  title     = {{Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms}},
  journal   = {CoRR},
  volume    = {abs/1708.07747},
  year      = {2017},
  eprinttype = {arXiv},
  eprint    = {1708.07747},
  timestamp = {Mon, 13 Aug 2018 16:47:27 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1708-07747.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@Misc{numpy,
  author =    {E. Jones and T. Oliphant and P. Peterson and others},
  title =     {{SciPy}: Open source scientific tools for {Python}},
  year =      {2001--2018},
  url = "http://www.scipy.org/",
}


@Misc{mnist-back,
  author =    {H. Larochelle and  D. Erhan and A. Courville},
  title =     {MnistVariations},
  year =      {2017},
  url = "http://www.iro.umontreal.ca/~lisa/twiki/bin/view.cgi/Public/MnistVariations",
}




@INPROCEEDINGS{Survey2, 
	author={C. Leistner and A. Saffari and P. M. Roth and H. Bischof}, 
	booktitle={12th International Conference on Computer Vision Workshops, ICCV}, 
	title={{On Robustness of On-line Boosting - a Competitive Study}}, 
	year={2009}, 
	volume={}, 
	number={}, 
	pages={1362-1369}, 
	keywords={computer vision;object detection;unsupervised learning;computer vision applications;visual object tracking;class-label noise;machine learning problems;online boosting algorithm;online gradient-boost;Boosting;Application software;Computer vision;Noise robustness;Machine learning;Object detection;Detectors;Computer graphics;Plugs;Machine learning algorithms}, 
	ISSN={}
	}
@inproceedings{Chapelle:2010:YLR:3045754.3045756,
	author = {O. Chapelle and Y. Chang},
	title = {Yahoo! Learning to Rank Challenge Overview},
	booktitle = {Proceedings of the 2010 International Conference on Yahoo! Learning to Rank Challenge - Volume 14},
	series = {YLRC'10},
	year = {2010},
	location = {Haifa, Israel},
	pages = {1--24},
	numpages = {24},
	acmid = {3045756},
	publisher = {JMLR.org},
} 
@article{transport,
	author = {Y. Zhang and A. Haghani},
    journal = {Transportation Research Part C: Emerging Technologies},
    volume = {58},
    pages = {308-324},
    year = {2015},
	title = {{A Gradient Boosting Method to Improve Travel Time Prediction}},
	booktitle = {Transportation Research Part C Emerging Technologies}
}


@INPROCEEDINGS{990517, 
	author={P. Viola and M. Jones}, 
	booktitle={Proceedings of the 2001 IEEE Computer Society Conference on Computer Vision and Pattern Recognition. CVPR}, 
	title={{Rapid object detection using a boosted cascade of simple features}}, 
	year={2001}, 
	volume={1}, 
	number={}, 
	pages={I-511-I-518 vol.1}, 
	keywords={feature extraction;image classification;image representation;learning (artificial intelligence);object detection;AdaBoost;background regions;boosted simple feature cascade;classifiers;face detection;image processing;image representation;integral image;machine learning;object specific focus-of-attention mechanism;rapid object detection;real-time applications;statistical guarantees;visual object detection;Detectors;Face detection;Filters;Focusing;Image representation;Machine learning;Object detection;Pixel;Robustness;Skin}, 
	ISSN={1063-6919}, 
	month={},}
@article{DBLP:journals/corr/YangYLL15,
	author    = {B. Yang and J. Yan and Z. Lei and S.Z. Li},
	title     = {{Convolutional Channel Features For Pedestrian, Face and Edge Detection}},
	journal   = {CoRR},
	volume    = {abs/1504.07339},
	year      = {2015},
	archivePrefix = {arXiv},
	eprint    = {1504.07339},
	timestamp = {Wed, 07 Jun 2017 14:41:54 +0200},
	biburl    = {https://dblp.org/rec/bib/journals/corr/YangYLL15},
	bibsource = {dblp computer science bibliography, https://dblp.org}
}
@article{FREUND1997119,
	title = {{A Decision-Theoretic Generalization of On-Line Learning and an Application to Boosting}},
	journal = "Journal of Computer and System Sciences",
	volume = "55",
	number = "1",
	pages = "119 - 139",
	year = "1997",
	issn = "0022-0000",
	author = "Y. Freund and R.E. Schapire"
}


@InProceedings{marta,
author="X. Huang and M. Kwiatkowska and S. Wang and M. Wu",
title={{Safety Verification of Deep Neural Networks}},
booktitle="Computer Aided Verification",
year="2017",
pages="3--29",
}


@inproceedings{z3,
 author = {L. De Moura and N. Bj{\o}rner},
 title = {Z3: An Efficient SMT Solver},
 booktitle = {Proceedings of the Theory and Practice of Software, 14th International Conference on Tools and Algorithms for the Construction and Analysis of Systems},
 series = {TACAS'08/ETAPS'08},
 year = {2008},
 isbn = {3-540-78799-2, 978-3-540-78799-0},
 location = {Budapest, Hungary},
 pages = {337--340},
 numpages = {4},
 acmid = {1792766},
 publisher = {Springer-Verlag},
 address = {Berlin, Heidelberg},
}


@INPROCEEDINGS{Brat,
author={G. Brat and A. Jonsson},
booktitle={Proceedings. 2005 IEEE International Joint Conference on Neural Networks, 2005.},
title={Challenges in verification and validation of autonomous systems for space exploration},
year={2005},
volume={5},
number={},
pages={2909-2914 vol. 5},
keywords={aerospace computing;space research;autonomous system;space exploration;space mission;Communication system control;Costs;Humans;Mars;Moon;Planets;Space exploration;Space missions;Space technology;Space vehicles},
ISSN={2161-4393},
month={July},}



@inproceedings{KatzBDJK17,
  author    = {G. Katz and C.W. Barrett and D.L. Dill and K. Julian and M.J. Kochenderfer},
  title     = {{Reluplex: An Efficient {SMT} Solver for Verifying Deep Neural Networks}},
  booktitle = {Computer Aided Verification - 29th International Conference, {CAV}},
  pages     = {97--117},
  year      = {2017},
  timestamp = {Fri, 14 Jul 2017 12:55:54 +0200},
  biburl    = {https://dblp.org/rec/bib/conf/cav/KatzBDJK17},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@article{GTSRB,
   author = {S. Houben and J. Stallkamp and J. Salmen and M. Schlipsing and C. Igel},
    journal={The 2013 International Joint Conference on Neural Networks (IJCNN)},
    year={2013},
    pages={1-8},
    title = {{Detection of Traffic Signs in Real-World Images: The {G}erman {T}raffic {S}ign {D}etection {B}enchmark}},
    number = {1288},
}
@article{Houben2013DetectionOT,
  title={Detection of traffic signs in real-world images: The German traffic sign detection benchmark},
  author={Sebastian Houben and Johannes Stallkamp and Jan Salmen and Marc Schlipsing and C. Igel},

}



@article{SignLanguage,
   author={Techperson},
   note ={\url{https://www.kaggle.com/datamunge/sign-language-mnist/} },
   title = {{Sign Language MNIST}},
   year = {2017},
}
@article{KMNIST,
  author    = {T. Clanuwat and M. Bober{-}Irizar and A. Kitamoto and A. Lamb and K. Yamamoto and D. Ha},
  title     = {{Deep Learning for Classical Japanese Literature}},
  journal   = {CoRR},
  volume    = {abs/1812.01718},
  year      = {2018},
  url       = {http://arxiv.org/abs/1812.01718},
  eprinttype = {arXiv},
  eprint    = {1812.01718},
  timestamp = {Thu, 14 Oct 2021 09:15:14 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1812-01718.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}




@inproceedings{Robustness-Moosavi, 
	author={S. Moosavi-Dezfooli and A. Fawzi and O. Fawzi and P. Frossard}, 
	booktitle={IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 
	title={{Universal Adversarial Perturbations}}, 
	year={2017}, 
	pages={86-94}, 
	keywords={image classification;neural nets;probability;universal adversarial perturbations;deep neural network classifier;image-agnostic;very small perturbation vector;natural images;universal perturbations;deep neural networks;Neural networks;Visualization;Optimization;Training;Computer architecture;Correlation;Robustness}, 
	ISSN={1063-6919}, 
}

@inproceedings{mooly,
  author    = {N. Narodytska and S.P. Kasiviswanathan and L. Ryzhyk and M. Sagiv and T. Walsh},
  title     = {{Verifying Properties of Binarized Deep Neural Networks}},
  booktitle = {32nd AAAI Conference on Artificial Intelligence, AAAI 2018},
  year      = {2018},
  pages = {6615--6624},
  timestamp = {Thu, 03 May 2018 17:03:19 +0200},
  biburl    = {https://dblp.org/rec/bib/conf/aaai/NarodytskaKRSW18},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@MISC{space_bugs,
	title = {{Houston, we have a bug: 9 famous software glitches in space}},
	author ={P. Johnson},
	howpublished = {https://www.arnnet.com.au/slideshow/581015/pictures-houston-we-bug-9-famous-software-glitches-space/},
	year = {2018}
}


@InProceedings{nuXmv,
author="R. Cavada and A. Cimatti and M. Dorigatti and A. Griggio and A. Mariotti and A. Micheli and S. Mover and M. Roveri and S. Tonetta",
title="The nuXmv Symbolic Model Checker",
booktitle="Computer Aided Verification",
year="2014",
publisher="Springer International Publishing",
address="Cham",
pages="334--342",
abstract="This paper describes the nuXmv symbolic model checker for finite- and infinite-state synchronous transition systems. nuXmv is the evolution of the nuXmv open source model checker. It builds on and extends nuXmv along two main directions. For finite-state systems it complements the basic verification techniques of nuXmv with state-of-the-art verification algorithms. For infinite-state systems, it extends the nuXmv language with new data types, namely Integers and Reals, and it provides advanced SMT-based model checking techniques.",
isbn="978-3-319-08867-9"
}


@book{hastie01statisticallearning,
  added-at = {2008-05-16T16:17:42.000+0200},
  address = {New York, NY, USA},
  author = {T. Hastie, R. Tibshirani and J. Friedman},
  publisher = {Springer New York Inc.},
  series = {Springer Series in Statistics},
  title = {The Elements of Statistical Learning},
  pages="364--367",
  year = 2001
}



@ARTICLE{Biggio,
author={B. Biggio and G. Fumera and F. Roli},
journal={IEEE Transactions on Knowledge and Data Engineering},
title={{Security Evaluation of Pattern Classifiers under Attack}},
year={2014},
volume={26},
number={4},
pages={984-996},
keywords={biometrics (access control);pattern classification;security of data;adversarial environments;biometric authentication;classical design methods;design phase;network intrusion detection;pattern classification systems;pattern classification theory;pattern classifiers;security evaluation;spam filtering;Algorithm design and analysis;Analytical models;Data models;Performance evaluation;Security;Testing;Training;Pattern classification;adversarial classification;performance evaluation;robustness evaluation;security evaluation},
}


@inproceedings{Zhou,
 author = {Y. Zhou and M. Kantarcioglu and B. Thuraisingham and B. Xi},
 title = {{Adversarial Support Vector Machine Learning}},
 booktitle = {Proceedings of the 18th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining},
 series = {KDD},
 year = {2012},
 location = {Beijing, China},
 pages = {1059--1067},
 numpages = {9},
 acmid = {2339697},
 keywords = {adversarial learning, attack models, robust SVM},
} 


@inproceedings{Ehlers17,
  author    = {R. Ehlers},
  booktitle = {Automated Technology for Verification and Analysis},
  title     = {{Formal Verification of Piece-Wise Linear Feed-Forward Neural Networks}},
  year={2017},
  pages = {269-286},
  editor = {D. D'Souza and K. N. Kumar},
  volume = {10482},
  series = {Lecture Notes in Computer Science},
  publisher = {Springer},
}




@Inbook{Biggio2014,
author="B. Biggio and I. Corona and B. Nelson and B.I.P. Rubinstein and D. Maiorca and G. Fumera and G. Giacinto and F. Roli",
title={{Security Evaluation of Support Vector Machines in Adversarial Environments}},
bookTitle="Support Vector Machines Applications",
year="2014",
pages="105--153",
abstract="Support vector machines (SVMs) are among the most popular classification techniques adopted in security applications like malware detection, intrusion detection, and spam filtering. However, if SVMs are to be incorporated in real-world security systems, they must be able to cope with attack patterns that can either mislead the learning algorithm (poisoning), evade detection (evasion) or gain information about their internal parameters (privacy breaches). The main contributions of this chapter are twofold. First, we introduce a formal general framework for the empirical evaluation of the security of machine-learning systems. Second, according to our framework, we demonstrate the feasibility of evasion, poisoning and privacy attacks against SVMs in real-world security problems. For each attack technique, we evaluate its impact and discuss whether (and how) it can be countered through an adversary-aware design of SVMs. Our experiments are easily reproducible thanks to open-source code that we have made available, together with all the employed datasets, on a public repository.",
}


@article{biggio1,
author = {B. Biggio and G. Fumera and F. Roli},
title = {{Pattern recognition systems under attack: Design issues and research challenges}},
journal = {International Journal of Pattern Recognition and Artificial Intelligence},
volume = {28},
number = {07},
pages = {1460002},
year = {2014},
eprint = {https://www.worldscientific.com/doi/pdf/10.1142/S0218001414600027}
}




@inproceedings{Gehr2018AISA,
author = {T. Gehr and M. Mirman and D. Drachsler-Cohen and P. Tsankov and S. Chaudhuri and M.T. Vechev},
year = {2018},
month = {05},
pages = {3-18},
title = {AI2: Safety and Robustness Certification of Neural Networks with Abstract Interpretation},
doi = {10.1109/SP.2018.00058}
}




@article{accidents,
    author = {F.M. Favaro and N. Nader and S.O. Eurich and M. Tripp and N. Varadaraju},
    journal = {PLOS ONE},
    publisher = {Public Library of Science},
    title = {{Examining accident reports involving autonomous vehicles in California}},
    year = {2017},
    month = {09},
    volume = {12},
    pages = {1-20},
    abstract = {Autonomous Vehicle technology is quickly expanding its market and has found in Silicon Valley, California, a strong foothold for preliminary testing on public roads. In an effort to promote safety and transparency to consumers, the California Department of Motor Vehicles has mandated that reports of accidents involving autonomous vehicles be drafted and made available to the public. The present work shows an in-depth analysis of the accident reports filed by different manufacturers that are testing autonomous vehicles in California (testing data from September 2014 to March 2017). The data provides important information on autonomous vehicles accidents’ dynamics, related to the most frequent types of collisions and impacts, accident frequencies, and other contributing factors. The study also explores important implications related to future testing and validation of semi-autonomous vehicles, tracing the investigation back to current literature as well as to the current regulatory panorama.},
    number = {9},
}


@incollection{BSST09,
	author    = {C.W. Barrett and R. Sebastiani and S.A. Seshia and C. Tinelli},
	title     = {{Satisfiability Modulo Theories}},
	booktitle = {Handbook of Satisfiability},
	pages     = {825--885},
	year      = {2009},
	publisher = {{IOS} Press}
}


@article{RFTheory,
	author = {L. Breiman},
	date = {2001/10/01},
	date-added = {2022-01-07 07:54:59 +0200},
	date-modified = {2022-01-07 07:54:59 +0200},
	id = {Breiman2001},
	isbn = {1573-0565},
	journal = {Machine Learning},
	number = {1},
	pages = {5--32},
	title = {{Random Forests}},
	volume = {45},
	year = {2001},
	bdsk-url-1 = {https://doi.org/10.1023/A:1010933404324}}
	
	
@article{CNN,
title = {{Recent Advances in Convolutional Neural Networks}},
journal = {Pattern Recognition},
volume = {77},
pages = {354-377},
year = {2018},
author = {J. Gu and Z. Wang and J. Kuen and L. Ma and A. Shahroudy and B. Shuai and T. Liu and X. Wang and G. Wang and J. Cai and T. Chen}
}





@article{CNN-Fashion-Config,
author = {J. Brownlee},
year = {2020},
title = {Deep Learning CNN for Fashion-MNIST Clothing Classification},
howpublished = {https://machinelearningmastery.com/how-to-develop-a-cnn-from-scratch-for-fashion-mnist-clothing-classification/},
}

@article{CNN-MNIST-Config,
author = {J. Brownlee},
year = {2020},
title = {How to Develop a CNN for MNIST Handwritten Digit Classification},
howpublished = {https://machinelearningmastery.com/how-to-develop-a-convolutional-neural-network-from-scratch-for-mnist-handwritten-digit-classification/},
}

@article{CNN-KMNIST-Config,
author = {G. Preda},
year = {2018},
title = {Classifying Cursive hiragana(崩し字) KMNIST using CNN},
howpublished = {https://www.kaggle.com/gpreda/classifying-cursive-hiragana-kmnist-using-cnn/notebook},
}

@article{CNN-GTSRB-Config,
author = {S. Sharma},
year = {2020},
title = {GTSRB - CNN (98\% Test Accuracy)},
howpublished = {https://www.kaggle.com/shivank856/gtsrb-cnn-98-test-accuracy},
}

%calibration
@conference {naeini2015obtaining,
	title = {{Obtaining Well Calibrated Probabilities Using Bayesian Binning}},
	booktitle = {AAAI},
	year = {2015},
	pages = {2901{\textendash}2907},
	author = {M.P. Naeini and G.F. Cooper and M. Hauskrecht}
}

@inproceedings{Xing2020distance,
title={{Distance-Based Learning from Errors for Confidence Calibration}},
author={C. Xing and S. Arik and Z. Zhang and T. Pfister},
booktitle={International Conference on Learning Representations (ICLR)},
year={2020},
}




@InProceedings{pmlr-v119-zhang20k,
  title = 	 {{Mix-n-Match : Ensemble and Compositional Methods for Uncertainty Calibration in Deep Learning}},
  author =       {Zhang, J. and Kailkhura, B. and Han, T. YJ},
  booktitle = 	 {Proceedings of the 37th International Conference on Machine Learning},
  pages = 	 {11117--11128},
  year = 	 {2020},
  volume = 	 {119},
  series = 	 {Proceedings of Machine Learning Research},
  pdf = 	 {http://proceedings.mlr.press/v119/zhang20k/zhang20k.pdf},
  abstract = 	 {This paper studies the problem of post-hoc calibration of machine learning classifiers. We introduce the following desiderata for uncertainty calibration: (a) accuracy-preserving, (b) data-efficient, and (c) high expressive power. We show that none of the existing methods satisfy all three requirements, and demonstrate how Mix-n-Match calibration strategies (i.e., ensemble and composition) can help achieve remarkably better data-efficiency and expressive power while provably maintaining the classification accuracy of the original classifier. Mix-n-Match strategies are generic in the sense that they can be used to improve the performance of any off-the-shelf calibrator. We also reveal potential issues in standard evaluation practices. Popular approaches (e.g., histogram-based expected calibration error (ECE)) may provide misleading results especially in small-data regime. Therefore, we propose an alternative data-efficient kernel density-based estimator for a reliable evaluation of the calibration performance and prove its asymptotically unbiasedness and consistency. Our approaches outperform state-of-the-art solutions on both the calibration as well as the evaluation tasks in most of the experimental settings. Our codes are available at https://github.com/zhang64- llnl/Mix-n-Match-Calibration.}
}





@article{Krishnan2020Improving,
 author = {Krishnan, R. and Tickoo, O.},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {18237--18248},
 title = {{Improving Model Calibration with Accuracy versus Uncertainty Optimization}},
 volume = {33},
 year = {2020}
}


@article{Seo2018Confidence,
  author    = {S. Seo and P.H. Seo and B. Han},
  title     = {{Confidence Calibration in Deep Neural Networks through Stochastic Inferences}},
  journal   = {CoRR},
  volume    = {abs/1809.10877},
  year      = {2018},
  eprinttype = {arXiv},
  eprint    = {1809.10877},
  timestamp = {Fri, 05 Oct 2018 11:34:52 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1809-10877.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{Ana2019Verified,
  author = {A. Kumar and P. Liang and Te. Ma},
  booktitle = {{Advances in Neural Information Processing Systems (NeurIPS)}},
  title = {{Verified Uncertainty Calibration}},
  year = {2019},
  volume = {32},
}

@inproceedings{gupta2021distribution,
  title={{Distribution-free Calibration Guarantees for Histogram Binning without Sample Splitting}},
  author={Gupta, C. and Ramdas, A.},
  booktitle={International Conference on Machine Learning},
  pages={3942--3952},
  year={2021},
  organization={PMLR}
}
@article{Zadrozny2002Transforming,
author = {Zadrozny, B. and Elkan, C.},
year = {2002},
month = {08},
pages = {694–699},
title = {{Transforming Classifier Scores into Accurate Multiclass Probability Estimates}},
journal = {Proceedings of the ACM SIGKDD International Conference on Knowledge Discovery and Data Mining},
}


@article{scikit-learn,
 title={{Scikit-learn: Machine Learning in {P}ython}},
 author={Pedregosa , F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
 journal={Journal of Machine Learning Research},
 volume={12},
 pages={2825--2830},
 year={2011}
}

@inproceedings{DBLP:conf/aaai/EinzigerGSS19,
  author    = {G. Einziger and
               M. Goldstein and
               Y. Sa'ar and
               I. Segall},
  title     = {{Verifying Robustness of Gradient Boosted Models}},
  booktitle = {The Thirty-Third Conference on Artificial Intelligence, 
               },
  pages     = {2446--2453},
  publisher = {{AAAI} },
  year      = {2019},
  timestamp = {Tue, 02 Feb 2021 08:00:09 +0100},
  biburl    = {https://dblp.org/rec/conf/aaai/EinzigerGSS19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@Misc{Github,
  author =    {L. Leman and G. Chouraqui and L. Cohen and G. Einzinger},
  title =     {Geometric Uncertainty Github},
  year =      {2022},
  url = "https://github.com/NoSleepDeveloper/Geometric-Calibrator",
}


%new from review
@article{Dalitz09Reject,
author = {Dalitz, Christoph},
year = {2009},
month = {01},
pages = {16-38},
title = {Reject Options and Confidence Measures for kNN Classifiers},
volume = {8},
journal = {Schriftenreihe des Fachbereichs Elektrotechnik und Informatik der Hochschule Niederrhein}
}