\begin{thebibliography}{45}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
  \providecommand{\doi}[1]{doi: #1}\else
  \providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi

\bibitem[Carlini and Wagner(2017)]{cw}
Nicholas Carlini and David Wagner.
\newblock Towards evaluating the robustness of neural networks.
\newblock In \emph{Proc. {IEEE S\&P}}, pages 39--57, 2017.

\bibitem[Chen and Hsieh(2020)]{sdarts}
Xiangning Chen and Cho{-}Jui Hsieh.
\newblock Stabilizing differentiable architecture search via perturbation-based
  regularization.
\newblock In \emph{Proc. ICML}, pages 1554--1565, 2020.

\bibitem[Chen et~al.(2019)Chen, Xie, Wu, and Tian]{p-darts}
Xin Chen, Lingxi Xie, Jun Wu, and Qi~Tian.
\newblock Progressive differentiable architecture search: {Bridging} the depth
  gap between search and evaluation.
\newblock In \emph{Proc. ICCV}, pages 1294--1303, 2019.

\bibitem[Chrabaszcz et~al.(2017)Chrabaszcz, Loshchilov, and
  Hutter]{imagenet-16-120}
Patryk Chrabaszcz, Ilya Loshchilov, and Frank Hutter.
\newblock A downsampled variant of {ImageNet} as an alternative to the {CIFAR}
  datasets.
\newblock {arXiv:1707.08819}, 2017.

\bibitem[Chu et~al.(2019)Chu, Zhang, Xu, and Li]{fairnas}
Xiangxiang Chu, Bo~Zhang, Ruijun Xu, and Jixiang Li.
\newblock {FairNAS}: Rethinking evaluation fairness of weight sharing neural
  architecture search.
\newblock {arXiv:1907.01845}, 2019.

\bibitem[Chu et~al.(2020)Chu, Wang, Zhang, Lu, Wei, and Yan]{darts-}
Xiangxiang Chu, Xiaoxing Wang, Bo~Zhang, Shun Lu, Xiaolin Wei, and Junchi Yan.
\newblock {DARTS-: Robustly} stepping out of performance collapse without
  indicators.
\newblock {arXiv:2009.01027}, 2020.

\bibitem[Cortes et~al.(2017)Cortes, Gonzalvo, Kuznetsov, Mohri, and
  Yang]{adanet}
Corinna Cortes, Xavier Gonzalvo, Vitaly Kuznetsov, Mehryar Mohri, and Scott
  Yang.
\newblock Adanet: {A}daptive structural learning of artificial neural networks.
\newblock In \emph{Proc. ICML}, pages 874--883, 2017.

\bibitem[Croce and Hein(2020)]{autoattack}
Francesco Croce and Matthias Hein.
\newblock Reliable evaluation of adversarial robustness with an ensemble of
  diverse parameter-free attacks.
\newblock In \emph{Proc. {ICML}}, pages 2206--2216, 2020.

\bibitem[Deng et~al.(2009)Deng, Dong, Socher, Li, Li, and Fei-Fei]{imagenet}
J.~Deng, W.~Dong, R.~Socher, L.-J. Li, K.~Li, and L.~Fei-Fei.
\newblock {ImageNet: A} large-scale hierarchical image database.
\newblock In \emph{Proc. CVPR}, 2009.

\bibitem[Devries and Taylor(2017)]{cutout}
Terrance Devries and Graham~W. Taylor.
\newblock Improved regularization of convolutional neural networks with cutout.
\newblock {arXiv:1708.04552}, 2017.

\bibitem[Dietterich(2000)]{eml}
Thomas~G. Dietterich.
\newblock Ensemble methods in machine learning.
\newblock In \emph{Multiple Classifier Systems}, pages 1--15, 2000.

\bibitem[Dong and Yang(2019{\natexlab{a}})]{gdas}
Xuanyi Dong and Yi~Yang.
\newblock Searching for a robust neural architecture in four {GPU} hours.
\newblock In \emph{Proc. CVPR}, pages 1761--1770, 2019{\natexlab{a}}.

\bibitem[Dong and Yang(2019{\natexlab{b}})]{setn}
Xuanyi Dong and Yi~Yang.
\newblock One-shot neural architecture search via self-evaluated template
  network.
\newblock In \emph{Proc. {ICCV}}, pages 3680--3689, 2019{\natexlab{b}}.

\bibitem[Dong and Yang(2020)]{nasbench201}
Xuanyi Dong and Yi~Yang.
\newblock {NAS-Bench-201}: Extending the scope of reproducible neural
  architecture search.
\newblock In \emph{Proc. ICLR}, 2020.

\bibitem[Fort et~al.(2019)Fort, Hu, and Lakshminarayanan]{disagreement}
Stanislav Fort, Huiyi Hu, and Balaji Lakshminarayanan.
\newblock Deep ensembles: {A} loss landscape perspective.
\newblock {arXiv:1912.02757}, 2019.

\bibitem[Gal and Ghahramani(2016)]{mc-dropout}
Yarin Gal and Zoubin Ghahramani.
\newblock Dropout as a {Bayesian} approximation: Representing model uncertainty
  in deep learning.
\newblock In \emph{Proc. ICML}, pages 1050--1059, 2016.

\bibitem[Goodfellow et~al.(2015)Goodfellow, Shlens, and Szegedy]{fgsm}
Ian~J. Goodfellow, Jonathon Shlens, and Christian Szegedy.
\newblock Explaining and harnessing adversarial examples.
\newblock In \emph{Proc. ICLR}, 2015.

\bibitem[Guo et~al.(2020)Guo, Yang, Xu, Liu, and Lin]{robnet}
Minghao Guo, Yuzhe Yang, Rui Xu, Ziwei Liu, and Dahua Lin.
\newblock When {NAS} meets robustness: In search of robust architectures
  against adversarial attacks.
\newblock In \emph{Proc. {CVPR}}, pages 628--637, 2020.

\bibitem[He et~al.(2016)He, Zhang, Ren, and Sun]{resnet}
Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun.
\newblock Deep residual learning for image recognition.
\newblock In \emph{Proc. CVPR}, pages 770--778, 2016.

\bibitem[Hong et~al.(2020)Hong, Li, Zhang, Tang, Wang, Li, and Yu]{dropnas}
Weijun Hong, Guilin Li, Weinan Zhang, Ruiming Tang, Yunhe Wang, Zhenguo Li, and
  Yong Yu.
\newblock {DropNAS}: Grouped operation dropout for differentiable architecture
  search.
\newblock In \emph{Proc. IJCAI}, pages 2326--2332, 2020.

\bibitem[Huang et~al.(2017)Huang, Li, Pleiss, Liu, Hopcroft, and
  Weinberger]{snapshot}
Gao Huang, Yixuan Li, Geoff Pleiss, Zhuang Liu, John~E. Hopcroft, and Kilian~Q.
  Weinberger.
\newblock Snapshot ensembles: {T}rain 1, get {M} for free.
\newblock In \emph{Proc. ICLR}, 2017.

\bibitem[Jang et~al.(2017)Jang, Gu, and Poole]{gumbel-softmax}
Eric Jang, Shixiang Gu, and Ben Poole.
\newblock Categorical reparameterization with {Gumbel-Softmax}.
\newblock In \emph{Proc. ICLR}, 2017.

\bibitem[Kingma and Ba(2015)]{adam}
Diederik~P. Kingma and Jimmy Ba.
\newblock Adam: {A} method for stochastic optimization.
\newblock In \emph{Proc. ICLR}, 2015.

\bibitem[Kingma and Welling(2014)]{vae}
Diederik~P. Kingma and Max Welling.
\newblock Auto-encoding variational {Bayes}.
\newblock In \emph{Proc. ICLR}, 2014.

\bibitem[Krizhevsky(2009)]{cifar}
Alex Krizhevsky.
\newblock Learning multiple layers of features from tiny images.
\newblock Master's thesis, Department of Computer Science, University of
  Toronto, 2009.

\bibitem[Lakshminarayanan et~al.(2017)Lakshminarayanan, Pritzel, and
  Blundell]{deepens}
Balaji Lakshminarayanan, Alexander Pritzel, and Charles Blundell.
\newblock Simple and scalable predictive uncertainty estimation using deep
  ensembles.
\newblock In \emph{Proc. NIPS}, pages 6402--6413, 2017.

\bibitem[Li and Talwalkar(2019)]{rsps}
Liam Li and Ameet Talwalkar.
\newblock Random search and reproducibility for neural architecture search.
\newblock In \emph{Proc. {UAI}}, pages 367--377, 2019.

\bibitem[Liu et~al.(2018)Liu, Zoph, Neumann, Shlens, Hua, Li, Fei{-}Fei,
  Yuille, Huang, and Murphy]{pnas}
Chenxi Liu, Barret Zoph, Maxim Neumann, Jonathon Shlens, Wei Hua, Li{-}Jia Li,
  Li~Fei{-}Fei, Alan~L. Yuille, Jonathan Huang, and Kevin Murphy.
\newblock Progressive neural architecture search.
\newblock In \emph{Proc. ECCV}, pages 19--35, 2018.

\bibitem[Liu et~al.(2019)Liu, Simonyan, and Yang]{darts}
Hanxiao Liu, Karen Simonyan, and Yiming Yang.
\newblock {DARTS: Differentiable} architecture search.
\newblock In \emph{Proc. ICLR}, 2019.

\bibitem[Liu and Wang(2016)]{svgd}
Qiang Liu and Dilin Wang.
\newblock Stein variational gradient descent: {A} general purpose {Bayesian}
  inference algorithm.
\newblock In \emph{Proc. NIPS}, pages 2370--2378, 2016.

\bibitem[Maddison et~al.(2017)Maddison, Mnih, and Teh]{concrete}
Chris~J. Maddison, Andriy Mnih, and Yee~Whye Teh.
\newblock The {Concrete} distribution: {A} continuous relaxation of discrete
  random variables.
\newblock In \emph{Proc. ICLR}, 2017.

\bibitem[Madry et~al.(2018)Madry, Makelov, Schmidt, Tsipras, and Vladu]{pgd}
Aleksander Madry, Aleksandar Makelov, Ludwig Schmidt, Dimitris Tsipras, and
  Adrian Vladu.
\newblock Towards deep learning models resistant to adversarial attacks.
\newblock In \emph{Proc. ICLR}, 2018.

\bibitem[Parikh and Boyd(2014)]{proximal}
Neal Parikh and Stephen~P. Boyd.
\newblock Proximal algorithms.
\newblock \emph{Foundations and Trends\textregistered\ in Optimization},
  1\penalty0 (3):\penalty0 127--239, 2014.

\bibitem[Pham et~al.(2018)Pham, Guan, Zoph, Le, and Dean]{enas}
Hieu Pham, Melody~Y. Guan, Barret Zoph, Quoc~V. Le, and Jeff Dean.
\newblock Efficient neural architecture search via parameter sharing.
\newblock In \emph{Proc. ICML}, pages 4092--4101, 2018.

\bibitem[Real et~al.(2019)Real, Aggarwal, Huang, and Le]{amoebanet}
Esteban Real, Alok Aggarwal, Yanping Huang, and Quoc~V. Le.
\newblock Regularized evolution for image classifier architecture search.
\newblock In \emph{Proc. AAAI}, pages 4780--4789, 2019.

\bibitem[Snoek et~al.(2012)Snoek, Larochelle, and Adams]{bo}
Jasper Snoek, Hugo Larochelle, and Ryan~P. Adams.
\newblock Practical {Bayesian} optimization of machine learning algorithms.
\newblock In \emph{Proc. NIPS}, pages 2960--2968, 2012.

\bibitem[Strauss et~al.(2017)Strauss, Hanselmann, Junginger, and
  Ulmer]{ensemble-for-defense}
Thilo Strauss, Markus Hanselmann, Andrej Junginger, and Holger Ulmer.
\newblock Ensemble methods as a defense to adversarial perturbations against
  deep neural networks.
\newblock {arXiv:1709.03423}, 2017.

\bibitem[van~der Maaten and Hinton(2008)]{t-sne}
Laurens van~der Maaten and Geoffrey Hinton.
\newblock Visualizing data using t-{SNE}.
\newblock \emph{JMLR}, 9\penalty0 (86):\penalty0 2579--2605, 2008.

\bibitem[Xie et~al.(2019)Xie, Zheng, Liu, and Lin]{snas}
Sirui Xie, Hehui Zheng, Chunxiao Liu, and Liang Lin.
\newblock {SNAS:} {Stochastic} neural architecture search.
\newblock In \emph{Proc. ICLR}, 2019.

\bibitem[Xu et~al.(2020)Xu, Xie, Zhang, Chen, Qi, Tian, and Xiong]{pc-darts}
Yuhui Xu, Lingxi Xie, Xiaopeng Zhang, Xin Chen, Guo{-}Jun Qi, Qi~Tian, and
  Hongkai Xiong.
\newblock {PC-DARTS:} {P}artial channel connections for memory-efficient
  architecture search.
\newblock In \emph{Proc. ICLR}, 2020.

\bibitem[Zaidi et~al.(2021)Zaidi, Zela, Elsken, Holmes, Hutter, and Teh]{nes}
Sheheryar Zaidi, Arber Zela, Thomas Elsken, Chris~C. Holmes, Frank Hutter, and
  Yee~Whye Teh.
\newblock Neural ensemble search for uncertainty estimation and dataset shift.
\newblock In \emph{Proc. NeurIPS}, 2021.

\bibitem[Zhou(2012)]{zhou-ensemble}
Zhi-Hua Zhou.
\newblock \emph{Ensemble Methods: {F}oundations and Algorithms}.
\newblock CRC Press, 2012.

\bibitem[Zhuo et~al.(2018)Zhuo, Liu, Shi, Zhu, Chen, and Zhang]{mp-svgd}
Jingwei Zhuo, Chang Liu, Jiaxin Shi, Jun Zhu, Ning Chen, and Bo~Zhang.
\newblock Message passing stein variational gradient descent.
\newblock In \emph{Proc. ICML}, pages 6013--6022, 2018.

\bibitem[Zoph and Le(2017)]{nas}
Barret Zoph and Quoc~V. Le.
\newblock Neural architecture search with reinforcement learning.
\newblock In \emph{Proc. ICLR}, 2017.

\bibitem[Zoph et~al.(2018)Zoph, Vasudevan, Shlens, and Le]{nasnet}
Barret Zoph, Vijay Vasudevan, Jonathon Shlens, and Quoc~V. Le.
\newblock Learning transferable architectures for scalable image recognition.
\newblock In \emph{Proc. CVPR}, pages 8697--8710, 2018.

\end{thebibliography}
