@article{dnns,
title = {Deep learning in neural networks: An overview},
journal = {Neural Networks},
volume = {61},
pages = {85-117},
year = {2015},
issn = {0893-6080},
doi = {https://doi.org/10.1016/j.neunet.2014.09.003},
author = {Jürgen Schmidhuber},
keywords = {primary}}



@inproceedings{webb2018statistical,
  title={A Statistical Approach to Assessing Neural Network Robustness},
  author={Webb, Stefan and Rainforth, Tom and Teh, Yee Whye and Kumar, M Pawan},
  booktitle={International Conference on Learning Representations},
  year={2019}
}

@article{nais_morio,
title = {Extreme quantile estimation with nonparametric adaptive importance sampling},
journal = {Simulation Modelling Practice and Theory},
volume = {27},
pages = {76-89},
year = {2012},
issn = {1569-190X},
doi = {https://doi.org/10.1016/j.simpat.2012.05.008},
url = {https://www.sciencedirect.com/science/article/pii/S1569190X1200072X},
author = {Jérôme Morio},
keywords = {Statistics, Rare event, Quantile estimation, Importance sampling},
abstract = {In this article, we propose a nonparametric adaptive importance sampling (NAIS) algorithm to estimate rare event quantile. Indeed, Importance Sampling (IS) is a well-known adapted random simulation technique. It consists in generating random weighted samples from an auxiliary distribution rather than the distribution of interest. The optimization of this auxiliary distribution is often very difficult in practice. First, we review how to define the optimal auxiliary density of IS for quantile estimation in the general case and then propose a nonparametric method based on Gaussian kernel density estimator to approach the optimal auxiliary density that does not assume an initial PDF guess. This method is then finally applied to theoretical cases to demonstrate the efficiency of the proposed NAIS algorithm and on the quantile estimation of the temperature of a forest fire detection simulator.}
}
@misc{beskos2014convergence,
      title={On the Convergence of Adaptive Sequential Monte Carlo Methods}, 
      author={Alexandros Beskos and Ajay Jasra and Nikolas Kantas and Alexandre Thiery},
      year={2014},
      eprint={1306.6462},
      archivePrefix={arXiv},
      primaryClass={stat.CO}
}


@INPROCEEDINGS{acaxu,
  author={Owen, Michael P. and Panken, Adam and Moss, Robert and Alvarez, Luis and Leeper, Charles},
  booktitle={2019 IEEE/AIAA 38th Digital Avionics Systems Conference (DASC)}, 
  title={{ACAS Xu}: Integrated Collision Avoidance and Detect and Avoid Capability for {UAS}}, 
  year={2019},
  volume={},
  number={},
  pages={1-10},
  doi={10.1109/DASC43569.2019.9081758}}
  
@article{dnnacasxu,
author = {Julian, Kyle and Kochenderfer, Mykel and Owen, Michael},
year = {2018},
month = {11},
pages = {1-11},
title = {Deep Neural Network Compression for Aircraft Collision Avoidance Systems},
volume = {42},
journal = {Journal of Guidance, Control, and Dynamics},
doi = {10.2514/1.G003724}
}



@InProceedings{fastlin,
  title = 	 {Towards Fast Computation of Certified Robustness for {R}e{LU} Networks},
  author =       {Weng, Lily and Zhang, Huan and Chen, Hongge and Song, Zhao and Hsieh, Cho-Jui and Daniel, Luca and Boning, Duane and Dhillon, Inderjit},
  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
  pages = 	 {5276--5285},
  year = 	 {2018},
  editor = 	 {Dy, Jennifer and Krause, Andreas},
  volume = 	 {80},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {10--15 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v80/weng18a/weng18a.pdf},
  url = 	 {http://proceedings.mlr.press/v80/weng18a.html},
  abstract = 	 {Verifying the robustness property of a general Rectified Linear Unit (ReLU) network is an NP-complete problem. Although finding the exact minimum adversarial distortion is hard, giving a certified lower bound of the minimum distortion is possible. Current available methods of computing such a bound are either time-consuming or deliver low quality bounds that are too loose to be useful. In this paper, we exploit the special structure of ReLU networks and provide two computationally efficient algorithms (Fast-Lin, Fast-Lip) that are able to certify non-trivial lower bounds of minimum adversarial distortions. Experiments show that (1) our methods deliver bounds close to (the gap is 2-3X) exact minimum distortions found by Reluplex in small networks while our algorithms are more than 10,000 times faster; (2) our methods deliver similar quality of bounds (the gap is within 35% and usually around 10%; sometimes our bounds are even better) for larger networks compared to the methods based on solving linear programming problems but our algorithms are 33-14,000 times faster; (3) our method is capable of solving large MNIST and CIFAR networks up to 7 layers with more than 10,000 neurons within tens of seconds on a single CPU core. In addition, we show that there is no polynomial time algorithm that can approximately find the minimum $\ell_1$ adversarial distortion of a ReLU network with a $0.99\ln n$ approximation ratio unless NP=P, where $n$ is the number of neurons in the network.}
}

@inproceedings{cnncert,
  author = "Akhilan Boopathy AND Tsui-Wei Weng AND Pin-Yu Chen AND Sijia Liu AND Luca Daniel",
  title = "{CNN-Cert}: An Efficient Framework for Certifying Robustness of Convolutional
Neural Networks",
  booktitle = "AAAI",
  year = "2019",
  month = "Jan"
}

@inproceedings{crown,
  author = "Huan Zhang AND Tsui-Wei Weng AND Pin-Yu Chen AND Cho-Jui Hsieh AND Luca Daniel",
  title = "Efficient Neural Network Robustness Certification with General Activation Functions",
  booktitle = "Advances in Neural Information Processing Systems (NuerIPS)",
  year = "2018",
  month = "dec"
}

@inproceedings{convex_relaxation,
 author = {Salman, Hadi and Yang, Greg and Zhang, Huan and Hsieh, Cho-Jui and Zhang, Pengchuan},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {A Convex Relaxation Barrier to Tight Robustness Verification of Neural Networks},
 url = {https://proceedings.neurips.cc/paper/2019/file/246a3c5544feb054f3ea718f61adfa16-Paper.pdf},
 volume = {32},
 year = {2019}
}


@article{probabilistic_robustness,
  title={Robustness of Neural Networks: A Probabilistic and Practical Approach},
  author={Ravi Mangal and A. Nori and A. Orso},
  journal={2019 IEEE/ACM 41st International Conference on Software Engineering: New Ideas and Emerging Results (ICSE-NIER)},
  year={2019},
  pages={93-96}, 
  url = {https://arxiv.org/pdf/1902.05983.pdf}
}

@InProceedings{consequences,
  title = 	 {Adversarial Examples Are a Natural Consequence of Test Error in Noise},
  author =       {Gilmer, Justin and Ford, Nicolas and Carlini, Nicholas and Cubuk, Ekin},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {2280--2289},
  year = 	 {2019},
  editor = 	 {Kamalika Chaudhuri and Ruslan Salakhutdinov},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--15 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/gilmer19a/gilmer19a.pdf},
  url = 	 {
http://proceedings.mlr.press/v97/gilmer19a.html
},
  abstract = 	 {Over the last few years, the phenomenon of <em>adversarial examples</em> — maliciously constructed inputs that fool trained machine learning models — has captured the attention of the research community, especially when restricted to small modifications of a correctly handled input. Less surprisingly, image classifiers also lack human-level performance on randomly corrupted images, such as images with additive Gaussian noise. In this paper we provide both empirical and theoretical evidence that these are two manifestations of the same underlying phenomenon, and therefore the adversarial robustness and corruption robustness research programs are closely related. This suggests that improving adversarial robustness should go hand in hand with improving performance in the presence of more general and realistic image corruptions. This yields a computationally tractable evaluation metric for defenses to consider: test error in noisy image distributions.}
}

@article{abstract,
author = {Singh, Gagandeep and Gehr, Timon and P\"{u}schel, Markus and Vechev, Martin},
title = {An Abstract Domain for Certifying Neural Networks},
year = {2019},
issue_date = {January 2019},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {3},
number = {POPL},
url = {https://doi.org/10.1145/3290354},
doi = {10.1145/3290354},
abstract = {We present a novel method for scalable and precise certification of deep neural networks. The key technical insight behind our approach is a new abstract domain which combines floating point polyhedra with intervals and is equipped with abstract transformers specifically tailored to the setting of neural networks. Concretely, we introduce new transformers for affine transforms, the rectified linear unit (ReLU), sigmoid, tanh, and maxpool functions.  We implemented our method in a system called DeepPoly and evaluated it extensively on a range of datasets, neural architectures (including defended networks), and specifications. Our experimental results indicate that DeepPoly is more precise than prior work while scaling to large networks.  We also show how to combine DeepPoly with a form of abstraction refinement based on trace partitioning. This enables us to prove, for the first time, the robustness of the network when the input image is subjected to complex perturbations such as rotations that employ linear interpolation.},
journal = {Proc. ACM Program. Lang.},
month = jan,
articleno = {41},
numpages = {30},
keywords = {Adversarial attacks, Abstract Interpretation, Deep Learning}
}

      



@article{Jovanovic2021CertifiedDW,
  title={Certified Defenses: Why Tighter Relaxations May Hurt Training?},
  author={Nikola Jovanovi'c and Mislav Balunovic and Maximilian Baader and Martin T. Vechev},
  journal={ArXiv},
  year={2021},
  volume={abs/2102.06700}

}
@article{Biggio_2013,
   title={Evasion Attacks against Machine Learning at Test Time},
   ISBN={9783642387098},
   ISSN={1611-3349},
   url={http://dx.doi.org/10.1007/978-3-642-40994-3_25},
   DOI={10.1007/978-3-642-40994-3_25},
   journal={Lecture Notes in Computer Science},
   publisher={Springer Berlin Heidelberg},
   author={Biggio, Battista and Corona, Igino and Maiorca, Davide and Nelson, Blaine and Šrndić, Nedim and Laskov, Pavel and Giacinto, Giorgio and Roli, Fabio},
   year={2013},
   pages={387–402}
}


@InProceedings{scalable_verification,
      title={Scalable Quantitative Verification For Deep Neural Networks}, 
      author={Teodora Baluta and Zheng Leong Chua and Kuldeep S. Meel and Prateek Saxena},
      year={2021},
      booktitle = {Proc. of Int. Conf. on Software Engineering},
      eprint={2002.06864},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@CONFERENCE{ImageNet_VSS09,
        AUTHOR = {Deng, J. and Li, K. and Do, M. and Su, H. and Fei-Fei, L.},        
        TITLE = {{Construction and Analysis of a Large Scale Image Ontology}},
        ORGANIZATION = {Vision Sciences Society},
        YEAR = {2009},
        BIBSOURCE = "http://www.image-net.org/papers/ImageNet_VSS2009.bib"}



@InProceedings{reluplex, 
author="Katz, Guy
and Barrett, Clark
and Dill, David L.
and Julian, Kyle
and Kochenderfer, Mykel J.",
editor="Majumdar, Rupak
and Kun{\v{c}}ak, Viktor",
title="Reluplex: An Efficient {SMT} Solver for Verifying Deep Neural Networks",
booktitle="Computer Aided Verification",
year="2017",
publisher="Springer International Publishing",
address="Cham",
pages="97--117",
abstract="Deep neural networks have emerged as a widely used and effective means for tackling complex, real-world problems. However, a major obstacle in applying them to safety-critical systems is the great difficulty in providing formal guarantees about their behavior. We present a novel, scalable, and efficient technique for verifying properties of deep neural networks (or providing counter-examples). The technique is based on the simplex method, extended to handle the non-convex Rectified Linear Unit (ReLU) activation function, which is a crucial ingredient in many modern neural networks. The verification procedure tackles neural networks as a whole, without making any simplifying assumptions. We evaluated our technique on a prototype deep neural network implementation of the next-generation airborne collision avoidance system for unmanned aircraft (ACAS Xu). Results show that our technique can successfully prove properties of networks that are an order of magnitude larger than the largest networks verified using existing methods.",
isbn="978-3-319-63387-9", 
url="https://arxiv.org/abs/1312.6199"
}

@inproceedings{mnist,
 author = {LeCun, Yann and Boser, Bernhard and Denker, John and Henderson, Donnie and Howard, R. and Hubbard, Wayne and Jackel, Lawrence},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {D. Touretzky},
 pages = {},
 publisher = {Morgan-Kaufmann},
 title = {Handwritten Digit Recognition with a Back-Propagation Network},
 url = {https://proceedings.neurips.cc/paper/1989/file/53c3bce66e43be4f209556518c2fcb54-Paper.pdf},
 volume = {2},
 year = {1990}
}

@article{guyader,
author = {Guyader, Arnaud and Hengartner, Nicolas and Matzner-Løber, E.},
year = {2011},
month = {10},
pages = {171-196},
title = {Simulation and Estimation of Extreme Quantiles and Extreme Probabilities},
volume = {64},
journal = {Applied Mathematics \& Optimization},
doi = {10.1007/s00245-011-9135-z}
}

@article{smc,
url = {http://dx.doi.org/10.1561/2200000074},
year = {2019},
volume = {12},
journal = {Foundations and Trends® in Machine Learning},
title = {Elements of Sequential Monte Carlo},
doi = {10.1561/2200000074},
issn = {1935-8237},
number = {3},
pages = {307-392},
author = {Christian A. Naesseth and Fredrik Lindsten and Thomas B. Schön}
}

@InProceedings{proven, 
    title = {{PROVEN}: Verifying Robustness of Neural Networks with a Probabilistic Approach}, 
    author = {Weng, Lily and Chen, Pin-Yu and Nguyen, Lam and Squillante, Mark and Boopathy, Akhilan and Oseledets, Ivan and Daniel, Luca}, 
    booktitle = {Proceedings of the 36th International Conference on Machine Learning}, pages = {6727--6736}, year = {2019}, 
    volume = {97}, series = {Proceedings of Machine Learning Research}, month = {09--15 Jun}, publisher = {PMLR}, pdf = {http://proceedings.mlr.press/v97/weng19a/weng19a.pdf}, url = { http://proceedings.mlr.press/v97/weng19a.html } }
    
@InProceedings{uniform_robustness,
  title = 	 {Robustness of classifiers to uniform $\ell_p$ and Gaussian noise},
  author = 	 {Jean-Yves Franceschi and Alhussein Fawzi and Omar Fawzi},
  booktitle = 	 {Proceedings of the Twenty-First International Conference on Artificial Intelligence and Statistics},
  pages = 	 {1280--1288},
  year = 	 {2018},
  editor = 	 {Amos Storkey and Fernando Perez-Cruz},
  volume = 	 {84},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--11 Apr},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v84/franceschi18a/franceschi18a.pdf},
  url = 	 {
http://proceedings.mlr.press/v84/franceschi18a.html
},
  abstract = 	 {We study the robustness of classifiers to various kinds of random noise models. In particular, we consider noise drawn uniformly from the $\ell_p$ ball for $p ∈[1, ∞]$ and Gaussian noise with an arbitrary covariance matrix. We characterize this robustness to random noise in terms of the distance to the decision boundary of the classifier. This analysis applies to linear classifiers as well as classifiers with locally approximately flat decision boundaries, a condition which is satisfied by state-of-the-art deep neural networks. The predicted robustness is verified experimentally.}
}

@book{melchers2018structural,
  title={Structural reliability analysis and prediction},
  author={Melchers, Robert E and Beck, Andr{\'e} T},
  year={2018},
  publisher={John wiley \& sons}
}


@inproceedings{NEURIPS2018_f2f44698,
	Author = {Singh, Gagandeep and Gehr, Timon and Mirman, Matthew and P\"{u}schel, Markus and Vechev, Martin},
	Booktitle = {Advances in Neural Information Processing Systems},
	Editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
	Publisher = {Curran Associates, Inc.},
	Title = {Fast and Effective Robustness Certification},
	Url = {https://proceedings.neurips.cc/paper/2018/file/f2f446980d8e971ef3da97af089481c3-Paper.pdf},
	Volume = {31},
	Year = {2018},
	Bdsk-Url-1 = {https://proceedings.neurips.cc/paper/2018/file/f2f446980d8e971ef3da97af089481c3-Paper.pdf}
}

@inproceedings{hairer2011yet,
  title={Yet another look at Harris’ ergodic theorem for Markov chains},
  author={Hairer, Martin and Mattingly, Jonathan C},
  booktitle={Seminar on Stochastic Analysis, Random Fields and Applications VI},
  pages={109--117},
  year={2011},
  organization={Springer}
}



@article{hastings,
 ISSN = {00063444},
 URL = {http://www.jstor.org/stable/2334940},
 abstract = {A generalization of the sampling method introduced by Metropolis et al. (1953) is presented along with an exposition of the relevant theory, techniques of application and methods and difficulties of assessing the error in Monte Carlo estimates. Examples of the methods, including the generation of random orthogonal matrices and potential applications of the methods to numerical problems arising in statistics, are discussed.},
 author = {W. K. Hastings},
 journal = {Biometrika},
 number = {1},
 pages = {97--109},
 publisher = {[Oxford University Press, Biometrika Trust]},
 title = {Monte Carlo Sampling Methods Using Markov Chains and Their Applications},
 urldate = {2022-09-21},
 volume = {57},
 year = {1970}
}

@inproceedings{adv_ex,
title	= {Explaining and Harnessing Adversarial Examples},
author	= {Ian Goodfellow and Jonathon Shlens and Christian Szegedy},
year	= {2015},
URL	= {http://arxiv.org/abs/1412.6572},
booktitle	= {International Conference on Learning Representations}
}


@misc{franceschi2018robustness,
      title={Robustness of classifiers to uniform $\ell\_p$ and Gaussian noise}, 
      author={Jean-Yves Franceschi and Alhussein Fawzi and Omar Fawzi},
      year={2018},
      eprint={1802.07971},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}
@inproceedings{
imagenetC,
title={Benchmarking Neural Network Robustness to Common Corruptions and Perturbations},
author={Dan Hendrycks and Thomas Dietterich},
booktitle={International Conference on Learning Representations},
year={2019},
url={https://openreview.net/forum?id=HJz6tiCqYm},
}
@misc{beskos2014convergence,
      title={On the Convergence of Adaptive Sequential Monte Carlo Methods}, 
      author={Alexandros Beskos and Ajay Jasra and Nikolas Kantas and Alexandre Thiery},
      year={2014},
      eprint={1306.6462},
      archivePrefix={arXiv},
      primaryClass={stat.CO}
}

@article{resnet,
  author    = {Kaiming He and
               Xiangyu Zhang and
               Shaoqing Ren and
               Jian Sun},
  title     = {Deep Residual Learning for Image Recognition},
  journal   = {CoRR},
  volume    = {abs/1512.03385},
  year      = {2015},
  url       = {http://arxiv.org/abs/1512.03385},
  eprinttype = {arXiv},
  eprint    = {1512.03385},
  timestamp = {Wed, 17 Apr 2019 17:23:45 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/HeZRS15.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@ARTICLE{auto,  author={Koopman, Philip and Wagner, Michael},  journal={IEEE Intelligent Transportation Systems Magazine},   title={Autonomous Vehicle Safety: An Interdisciplinary Challenge},   year={2017},  volume={9},  number={1},  pages={90-96},  doi={10.1109/MITS.2016.2583491}}

@InProceedings{auto_car,
  title = 	 { Deep Probabilistic Accelerated Evaluation: A Robust Certifiable Rare-Event Simulation Methodology for Black-Box Safety-Critical Systems },
  author =       {Arief, Mansur and Huang, Zhiyuan and Koushik Senthil Kumar, Guru and Bai, Yuanlu and He, Shengyi and Ding, Wenhao and Lam, Henry and Zhao, Ding},
  booktitle = 	 {Proceedings of The 24th International Conference on Artificial Intelligence and Statistics},
  pages = 	 {595--603},
  year = 	 {2021},
  editor = 	 {Banerjee, Arindam and Fukumizu, Kenji},
  volume = 	 {130},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {13--15 Apr},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v130/arief21a/arief21a.pdf},
  url = 	 {https://proceedings.mlr.press/v130/arief21a.html},
  abstract = 	 { Evaluating the reliability of intelligent physical systems against rare safety-critical events poses a huge testing burden for real-world applications. Simulation provides a useful platform to evaluate the extremal risks of these systems before their deployments. Importance Sampling (IS), while proven to be powerful for rare-event simulation, faces challenges in handling these learning-based systems due to their black-box nature that fundamentally undermines its efficiency guarantee, which can lead to under-estimation without diagnostically detected. We propose a framework called Deep Probabilistic Accelerated Evaluation (Deep-PrAE) to design statistically guaranteed IS, by converting black-box samplers that are versatile but could lack guarantees, into one with what we call a relaxed efficiency certificate that allows accurate estimation of bounds on the safety-critical event probability. We present the theory of Deep-PrAE that combines the dominating point concept with rare-event set learning via deep neural network classifiers, and demonstrate its effectiveness in numerical examples including the safety-testing of an intelligent driving algorithm. }
}


@misc{lamperski2020projected,
      title={Projected Stochastic Gradient Langevin Algorithms for Constrained Sampling and Non-Convex Learning}, 
      author={Andrew Lamperski},
      year={2020},
      eprint={2012.12137},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{ford2019adversarial,
      title={Adversarial Examples Are a Natural Consequence of Test Error in Noise}, 
      author={Nic Ford and Justin Gilmer and Nicolas Carlini and Dogus Cubuk},
      year={2019},
      eprint={1901.10513},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}







@misc{beskos2014convergence,
      title={On the Convergence of Adaptive Sequential Monte Carlo Methods}, 
      author={Alexandros Beskos and Ajay Jasra and Nikolas Kantas and Alexandre Thiery},
      year={2014},
      eprint={1306.6462},
      archivePrefix={arXiv},
      primaryClass={stat.CO}
}


@INPROCEEDINGS{acaxu,
  author={Owen, Michael P. and Panken, Adam and Moss, Robert and Alvarez, Luis and Leeper, Charles},
  booktitle={2019 IEEE/AIAA 38th Digital Avionics Systems Conference (DASC)}, 
  title={{ACAS Xu}: Integrated Collision Avoidance and Detect and Avoid Capability for {UAS}}, 
  year={2019},
  volume={},
  number={},
  pages={1-10},
  doi={10.1109/DASC43569.2019.9081758}}
@ARTICLE{ais_review,
  author={Bugallo, Monica F. and Elvira, Victor and Martino, Luca and Luengo, David and Miguez, Joaquin and Djuric, Petar M.},
  journal={IEEE Signal Processing Magazine}, 
  title={Adaptive Importance Sampling: The past, the present, and the future}, 
  year={2017},
  volume={34},
  number={4},
  pages={60-79},
  keywords={Sampling methods;Artificial intelligence;Bayes methods;Signal processing algorithms;Probability distribution;Monte Carlo methods;Signal processing},
  doi={10.1109/MSP.2017.2699226}}

@article{dnnacasxu,
author = {Julian, Kyle and Kochenderfer, Mykel and Owen, Michael},
year = {2018},
month = {11},
pages = {1-11},
title = {Deep Neural Network Compression for Aircraft Collision Avoidance Systems},
volume = {42},
journal = {Journal of Guidance, Control, and Dynamics},
doi = {10.2514/1.G003724}
}

@article{FT,
author = {Fearnhead, Paul and Taylor, Benjamin},
year = {2010},
month = {05},
pages = {},
title = {An Adaptive Sequential Monte Carlo Sampler},
volume = {8},
journal = {Bayesian Analysis},
doi = {10.1214/13-BA814}
}
@article{nataf,
  TITLE = {{{\'E}tude graphique de d{\'e}termination de distributions de probabilit{\'e}s planes dont les marges sont donn{\'e}es}},
  AUTHOR = {Nataf, Andr{\'e}},
  URL = {https://hal.science/hal-04095227},
  JOURNAL = {{Annales de l'ISUP}},
  PUBLISHER = {{Publications de l'Institut de Statistique de l'Universit{\'e} de Paris}},
  VOLUME = {XI},
  NUMBER = {3},
  PAGES = {[257]-260},
  YEAR = {1962},
  PDF = {https://hal.science/hal-04095227/file/Pages%20de%20%281962%29-8_Vol%20XI-fascicule%203_article%205.pdf},
  HAL_ID = {hal-04095227},
  HAL_VERSION = {v1},
}

@Article{ESS,
  author={Víctor Elvira and Luca Martino and Christian P. Robert},
  title={{Rethinking the Effective Sample Size}},
  journal={International Statistical Review},
  year=2022,
  volume={90},
  number={3},
  pages={525-550},
  month={December},
  keywords={},
  doi={10.1111/insr.12500},
  abstract={The effective sample size (ESS) is widely used in sample‐based simulation methods for assessing the quality of a Monte Carlo approximation of a given distribution and of related integrals. In this paper, we revisit the approximation of the ESS in the specific context of importance sampling. The derivation of this approximation, that we will denote as ESS^, is partially available in a 1992 foundational technical report of Augustine Kong. This approximation has been widely used in the last 25 years due to its simplicity as a practical rule of thumb in a wide variety of importance sampling methods. However, we show that the multiple assumptions and approximations in the derivation of ESS^ make it difficult to be considered even as a reasonable approximation of the ESS. We extend the discussion of the ESS^ in the multiple importance sampling setting, we display numerical examples and we discuss several avenues for developing alternative metrics. This paper does not cover the use of ESS for Markov chain Monte Carlo algorithms.},
  url={https://ideas.repec.org/a/bla/istatr/v90y2022i3p525-550.html}
}





@inproceedings{cnncert,
  author = "Akhilan Boopathy AND Tsui-Wei Weng AND Pin-Yu Chen AND Sijia Liu AND Luca Daniel",
  title = "{CNN-Cert}: An Efficient Framework for Certifying Robustness of Convolutional
Neural Networks",
  booktitle = "AAAI",
  year = "2019",
  month = "Jan"
}

@inproceedings{crown,
  author = "Huan Zhang AND Tsui-Wei Weng AND Pin-Yu Chen AND Cho-Jui Hsieh AND Luca Daniel",
  title = "Efficient Neural Network Robustness Certification with General Activation Functions",
  booktitle = "Advances in Neural Information Processing Systems (NuerIPS)",
  year = "2018",
  month = "dec"
}

@inproceedings{convex_relaxation,
 author = {Salman, Hadi and Yang, Greg and Zhang, Huan and Hsieh, Cho-Jui and Zhang, Pengchuan},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {A Convex Relaxation Barrier to Tight Robustness Verification of Neural Networks},
 url = {https://proceedings.neurips.cc/paper/2019/file/246a3c5544feb054f3ea718f61adfa16-Paper.pdf},
 volume = {32},
 year = {2019}
}


@article{probabilistic_robustness,
  title={Robustness of Neural Networks: A Probabilistic and Practical Approach},
  author={Ravi Mangal and A. Nori and A. Orso},
  journal={2019 IEEE/ACM 41st International Conference on Software Engineering: New Ideas and Emerging Results (ICSE-NIER)},
  year={2019},
  pages={93-96}, 
  url = {https://arxiv.org/pdf/1902.05983.pdf}
}

@InProceedings{consequences,
  title = 	 {Adversarial Examples Are a Natural Consequence of Test Error in Noise},
  author =       {Gilmer, Justin and Ford, Nicolas and Carlini, Nicholas and Cubuk, Ekin},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {2280--2289},
  year = 	 {2019},
  editor = 	 {Kamalika Chaudhuri and Ruslan Salakhutdinov},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--15 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/gilmer19a/gilmer19a.pdf},
  url = 	 {
http://proceedings.mlr.press/v97/gilmer19a.html
},
  abstract = 	 {Over the last few years, the phenomenon of <em>adversarial examples</em> — maliciously constructed inputs that fool trained machine learning models — has captured the attention of the research community, especially when restricted to small modifications of a correctly handled input. Less surprisingly, image classifiers also lack human-level performance on randomly corrupted images, such as images with additive Gaussian noise. In this paper we provide both empirical and theoretical evidence that these are two manifestations of the same underlying phenomenon, and therefore the adversarial robustness and corruption robustness research programs are closely related. This suggests that improving adversarial robustness should go hand in hand with improving performance in the presence of more general and realistic image corruptions. This yields a computationally tractable evaluation metric for defenses to consider: test error in noisy image distributions.}
}

@article{abstract,
author = {Singh, Gagandeep and Gehr, Timon and P\"{u}schel, Markus and Vechev, Martin},
title = {An Abstract Domain for Certifying Neural Networks},
year = {2019},
issue_date = {January 2019},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {3},
number = {POPL},
url = {https://doi.org/10.1145/3290354},
doi = {10.1145/3290354},
abstract = {We present a novel method for scalable and precise certification of deep neural networks. The key technical insight behind our approach is a new abstract domain which combines floating point polyhedra with intervals and is equipped with abstract transformers specifically tailored to the setting of neural networks. Concretely, we introduce new transformers for affine transforms, the rectified linear unit (ReLU), sigmoid, tanh, and maxpool functions.  We implemented our method in a system called DeepPoly and evaluated it extensively on a range of datasets, neural architectures (including defended networks), and specifications. Our experimental results indicate that DeepPoly is more precise than prior work while scaling to large networks.  We also show how to combine DeepPoly with a form of abstraction refinement based on trace partitioning. This enables us to prove, for the first time, the robustness of the network when the input image is subjected to complex perturbations such as rotations that employ linear interpolation.},
journal = {Proc. ACM Program. Lang.},
month = jan,
articleno = {41},
numpages = {30},
keywords = {Adversarial attacks, Abstract Interpretation, Deep Learning}
}

      



@article{Jovanovic2021CertifiedDW,
  title={Certified Defenses: Why Tighter Relaxations May Hurt Training?},
  author={Nikola Jovanovi'c and Mislav Balunovic and Maximilian Baader and Martin T. Vechev},
  journal={ArXiv},
  year={2021},
  volume={abs/2102.06700}

}
@article{Biggio_2013,
   title={Evasion Attacks against Machine Learning at Test Time},
   ISBN={9783642387098},
   ISSN={1611-3349},
   url={http://dx.doi.org/10.1007/978-3-642-40994-3_25},
   DOI={10.1007/978-3-642-40994-3_25},
   journal={Lecture Notes in Computer Science},
   publisher={Springer Berlin Heidelberg},
   author={Biggio, Battista and Corona, Igino and Maiorca, Davide and Nelson, Blaine and Šrndić, Nedim and Laskov, Pavel and Giacinto, Giorgio and Roli, Fabio},
   year={2013},
   pages={387–402}
}


@InProceedings{scalable_verification,
      title={Scalable Quantitative Verification For Deep Neural Networks}, 
      author={Teodora Baluta and Zheng Leong Chua and Kuldeep S. Meel and Prateek Saxena},
      year={2021},
      booktitle = {Proc. of Int. Conf. on Software Engineering},
      eprint={2002.06864},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@CONFERENCE{ImageNet_VSS09,
        AUTHOR = {Deng, J. and Li, K. and Do, M. and Su, H. and Fei-Fei, L.},        
        TITLE = {{Construction and Analysis of a Large Scale Image Ontology}},
        ORGANIZATION = {Vision Sciences Society},
        YEAR = {2009},
        BIBSOURCE = "http://www.image-net.org/papers/ImageNet_VSS2009.bib"}

@inbook{Neal2011,
	doi = {10.1201/b10905},
	url = {https://doi.org/10.1201/b10905},
	year = 2011,
	month = {may},
	publisher = {Chapman and Hall/{CRC}},
	title = {Handbook of Markov Chain Monte Carlo},
	author = {Radford M. Neal},
	chapter = {MCMC using Hamiltonian dynamics}
}%editor = {Steve Brooks and Andrew Gelman and Galin Jones and Xiao-Li Meng},


@InProceedings{reluplex, 
author="Katz, Guy
and Barrett, Clark
and Dill, David L.
and Julian, Kyle
and Kochenderfer, Mykel J.",
editor="Majumdar, Rupak
and Kun{\v{c}}ak, Viktor",
title="Reluplex: An Efficient {SMT} Solver for Verifying Deep Neural Networks",
booktitle="Computer Aided Verification",
year="2017",
publisher="Springer International Publishing",
address="Cham",
pages="97--117",
abstract="Deep neural networks have emerged as a widely used and effective means for tackling complex, real-world problems. However, a major obstacle in applying them to safety-critical systems is the great difficulty in providing formal guarantees about their behavior. We present a novel, scalable, and efficient technique for verifying properties of deep neural networks (or providing counter-examples). The technique is based on the simplex method, extended to handle the non-convex Rectified Linear Unit (ReLU) activation function, which is a crucial ingredient in many modern neural networks. The verification procedure tackles neural networks as a whole, without making any simplifying assumptions. We evaluated our technique on a prototype deep neural network implementation of the next-generation airborne collision avoidance system for unmanned aircraft (ACAS Xu). Results show that our technique can successfully prove properties of networks that are an order of magnitude larger than the largest networks verified using existing methods.",
isbn="978-3-319-63387-9", 
url="https://arxiv.org/abs/1312.6199"
}

@inproceedings{mnist,
 author = {LeCun, Yann and Boser, Bernhard and Denker, John and Henderson, Donnie and Howard, R. and Hubbard, Wayne and Jackel, Lawrence},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {D. Touretzky},
 pages = {},
 publisher = {Morgan-Kaufmann},
 title = {Handwritten Digit Recognition with a Back-Propagation Network},
 url = {https://proceedings.neurips.cc/paper/1989/file/53c3bce66e43be4f209556518c2fcb54-Paper.pdf},
 volume = {2},
 year = {1990}
}

@article{guyader,
author = {Guyader, Arnaud and Hengartner, Nicolas and Matzner-Løber, E.},
year = {2011},
month = {10},
pages = {171-196},
title = {Simulation and Estimation of Extreme Quantiles and Extreme Probabilities},
volume = {64},
journal = {Applied Mathematics \& Optimization},
doi = {10.1007/s00245-011-9135-z}
}

@ARTICLE{optimal_scale,
    author = {Gareth O. Roberts and Jeffrey S. Rosenthal},
    title = {Optimal scaling of discrete approximations to Langevin diffusions},
    journal = {J. R. Statist. Soc. B},
    year = {1997},
    volume = {60},
    pages = {255--268}
}
@article{smc,
url = {http://dx.doi.org/10.1561/2200000074},
year = {2019},
volume = {12},
journal = {Foundations and Trends® in Machine Learning},
title = {Elements of Sequential Monte Carlo},
doi = {10.1561/2200000074},
issn = {1935-8237},
number = {3},
pages = {307-392},
author = {Christian A. Naesseth and Fredrik Lindsten and Thomas B. Schön}
}

@InProceedings{proven, 
    title = {{PROVEN}: Verifying Robustness of Neural Networks with a Probabilistic Approach}, 
    author = {Weng, Lily and Chen, Pin-Yu and Nguyen, Lam and Squillante, Mark and Boopathy, Akhilan and Oseledets, Ivan and Daniel, Luca}, 
    booktitle = {Proceedings of the 36th International Conference on Machine Learning}, pages = {6727--6736}, year = {2019}, 
    volume = {97}, series = {Proceedings of Machine Learning Research}, month = {09--15 Jun}, publisher = {PMLR}, pdf = {http://proceedings.mlr.press/v97/weng19a/weng19a.pdf}, url = { http://proceedings.mlr.press/v97/weng19a.html } }
    
@InProceedings{uniform_robustness,
  title = 	 {Robustness of classifiers to uniform $\ell_p$ and Gaussian noise},
  author = 	 {Jean-Yves Franceschi and Alhussein Fawzi and Omar Fawzi},
  booktitle = 	 {Proceedings of the Twenty-First International Conference on Artificial Intelligence and Statistics},
  pages = 	 {1280--1288},
  year = 	 {2018},
  editor = 	 {Amos Storkey and Fernando Perez-Cruz},
  volume = 	 {84},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--11 Apr},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v84/franceschi18a/franceschi18a.pdf},
  url = 	 {
http://proceedings.mlr.press/v84/franceschi18a.html
},
  abstract = 	 {We study the robustness of classifiers to various kinds of random noise models. In particular, we consider noise drawn uniformly from the $\ell_p$ ball for $p ∈[1, ∞]$ and Gaussian noise with an arbitrary covariance matrix. We characterize this robustness to random noise in terms of the distance to the decision boundary of the classifier. This analysis applies to linear classifiers as well as classifiers with locally approximately flat decision boundaries, a condition which is satisfied by state-of-the-art deep neural networks. The predicted robustness is verified experimentally.}
}

@book{melchers2018structural,
  title={Structural reliability analysis and prediction},
  author={Melchers, Robert E and Beck, Andr{\'e} T},
  year={2018},
  publisher={John wiley \& sons}
}


@inproceedings{NEURIPS2018_f2f44698,
	Author = {Singh, Gagandeep and Gehr, Timon and Mirman, Matthew and P\"{u}schel, Markus and Vechev, Martin},
	Booktitle = {Advances in Neural Information Processing Systems},
	Editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
	Publisher = {Curran Associates, Inc.},
	Title = {Fast and Effective Robustness Certification},
	Url = {https://proceedings.neurips.cc/paper/2018/file/f2f446980d8e971ef3da97af089481c3-Paper.pdf},
	Volume = {31},
	Year = {2018},
	Bdsk-Url-1 = {https://proceedings.neurips.cc/paper/2018/file/f2f446980d8e971ef3da97af089481c3-Paper.pdf}
}

@inproceedings{hairer2011yet,
  title={Yet another look at Harris’ ergodic theorem for Markov chains},
  author={Hairer, Martin and Mattingly, Jonathan C},
  booktitle={Seminar on Stochastic Analysis, Random Fields and Applications VI},
  pages={109--117},
  year={2011},
  organization={Springer}
}


@article{Kahn1951,
  added-at = {2020-01-29T18:29:49.000+0100},
  author = {Kahn, Herman and Harris, Theodore E},
  biburl = {https://www.bibsonomy.org/bibtex/22f9cb11c7aaa4af53ede2235b936238b/salotz},
  interhash = {628f57b0f02692c5c87991899fbcb706},
  intrahash = {2f9cb11c7aaa4af53ede2235b936238b},
  journal = {National Bureau of Standards applied mathematics series},
  keywords = {enhanced-sampling history path-sampling splitting-methods},
  pages = {27--30},
  timestamp = {2020-01-29T18:29:49.000+0100},
  title = {Estimation of particle transmission by random sampling},
  volume = 12,
  year = 1951
}




@inproceedings{ishikawa2021efficient,
  title={Efficient debiased evidence estimation by multilevel Monte Carlo sampling},
  author={Ishikawa, Kei and Goda, Takashi},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={34--43},
  year={2021},
  organization={PMLR}
}

@inproceedings{podkopaev2021distribution,
  title={Distribution-free uncertainty quantification for classification under label shift},
  author={Podkopaev, Aleksandr and Ramdas, Aaditya},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={844--853},
  year={2021},
  organization={PMLR}
}

@inproceedings{wang2021statistically,
  title={Statistically robust neural network classification},
  author={Wang, Benjie and Webb, Stefan and Rainforth, Tom},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={1735--1745},
  year={2021},
  organization={PMLR}
}


@InProceedings{pmlr-v161-ulmer21a,
  title = 	 {Know your limits: Uncertainty estimation with ReLU classifiers fails at reliable OOD detection},
  author =       {Ulmer, Dennis and Cin\`a, Giovanni},
  booktitle = 	 {Proceedings of the Thirty-Seventh Conference on Uncertainty in Artificial Intelligence},
  pages = 	 {1766--1776},
  year = 	 {2021},
  editor = 	 {de Campos, Cassio and Maathuis, Marloes H.},
  volume = 	 {161},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {27--30 Jul},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v161/ulmer21a/ulmer21a.pdf},
  url = 	 {https://proceedings.mlr.press/v161/ulmer21a.html},
  abstract = 	 {A crucial requirement for reliable deployment of deep learning models for safety-critical applications is the ability to identify out-of-distribution (OOD) data points, samples which differ from the training data and on which a model might underperform. Previous work has attempted to tackle this problem using uncertainty estimation techniques. However, there is empirical evidence that a large family of these techniques do not detect OOD reliably in classification tasks. This paper gives a theoretical explanation for said experimental findings and illustrates it on synthetic data. We prove that such techniques are not able to reliably identify OOD samples in a classification setting, since their level of confidence is generalized to unseen areas of the feature space. This result stems from the interplay between the representation of ReLU networks as piece-wise affine transformations, the saturating nature of activation functions like softmax, and the most widely-used uncertainty metrics.}
}

@article{buchholz2021adaptive,
  title={Adaptive tuning of hamiltonian monte carlo within sequential monte carlo},
  author={Buchholz, Alexander and Chopin, Nicolas and Jacob, Pierre E},
  journal={Bayesian Analysis},
  volume={16},
  number={3},
  pages={745--771},
  year={2021},
  publisher={International Society for Bayesian Analysis}
}

@ARTICLE{Foi2008,
  author={Foi, Alessandro and Trimeche, Mejdi and Katkovnik, Vladimir and Egiazarian, Karen},
  journal={IEEE Transactions on Image Processing}, 
  title={Practical Poissonian-Gaussian Noise Modeling and Fitting for Single-Image Raw-Data}, 
  year={2008},
  volume={17},
  number={10},
  pages={1737-1754},
  doi={10.1109/TIP.2008.2001399}}

@inproceedings{efficient,
  TITLE = {{Efficient Statistical Assessment of Neural Network Corruption Robustness}},
  AUTHOR = {Tit, Karim and Furon, Teddy and Rousset, Mathias},
  URL = {https://hal.archives-ouvertes.fr/hal-03407011},
  BOOKTITLE = {{NeurIPS 2021 - 35th Conference on Neural Information Processing Systems}},
  ADDRESS = {Sydney (virtual), Australia},
  SERIES = {Advances in Neural Information Processing Systems proceedings},
  VOLUME = {34},
  YEAR = {2021},
  MONTH = Dec,
  PDF = {https://hal.archives-ouvertes.fr/hal-03407011v2/file/Robustness_Estimation_with_Last_Particle__NeurIPS_pre_print_-6.pdf},
  HAL_ID = {hal-03407011},
  HAL_VERSION = {v2},
}

@article{mae_vs_rmse,
 ISSN = {0936577X, 16161572},
 URL = {http://www.jstor.org/stable/24869236},
 abstract = {ABSTRACT:The relative abilities of 2, dimensioned statistics—the root-mean-square error (RMSE) and the mean absolute error (MAE)—to describe average model-performance error are examined. The RMSE is of special interest because it is widely reported in the climatic and environmental literature; nevertheless, it is an inappropriate and misinterpreted measure of average error. RMSE is inappropriate because it is a function of 3 characteristics of a set of errors, rather than of one (the average error). RMSE varies with the variability within the distribution of error magnitudes and with the square root of the number of errors (n1/2), as well as with the average-error magnitude (MAE). Our findings indicate that MAE is a more natural measure of average error, and (unlike RMSE) is unambiguous. Dimensioned evaluations and inter-comparisons of average model-performance error, therefore, should be based on MAE.},
 author = {Cort J. Willmott and Kenji Matsuura},
 journal = {Climate Research},
 number = {1},
 pages = {79--82},
 publisher = {Inter-Research Science Center},
 title = {Advantages of the mean absolute error (MAE) over the root mean square error (RMSE) in assessing average model performance},
 urldate = {2022-05-11},
 volume = {30},
 year = {2005}
}
@INPROCEEDINGS{mobilenet,  author={Sandler, Mark and Howard, Andrew and Zhu, Menglong and Zhmoginov, Andrey and Chen, Liang-Chieh},  booktitle={2018 IEEE/CVF Conference on Computer Vision and Pattern Recognition},   title={MobileNetV2: Inverted Residuals and Linear Bottlenecks},   year={2018},  volume={},  number={},  pages={4510-4520},  doi={10.1109/CVPR.2018.00474}}

@INPROCEEDINGS{imagenet,

  author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Kai Li and Li Fei-Fei},

  booktitle={2009 IEEE Conference on Computer Vision and Pattern Recognition}, 

  title={ImageNet: A large-scale hierarchical image database}, 

  year={2009},

  volume={},

  number={},

  pages={248-255},

  doi={10.1109/CVPR.2009.5206848}}



