\relax 
\providecommand\hyper@newdestlabel[2]{}
\providecommand\HyperFirstAtBeginDocument{\AtBeginDocument}
\HyperFirstAtBeginDocument{\ifx\hyper@anchor\@undefined
\global\let\oldcontentsline\contentsline
\gdef\contentsline#1#2#3#4{\oldcontentsline{#1}{#2}{#3}}
\global\let\oldnewlabel\newlabel
\gdef\newlabel#1#2{\newlabelxx{#1}#2}
\gdef\newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}}
\AtEndDocument{\ifx\hyper@anchor\@undefined
\let\contentsline\oldcontentsline
\let\newlabel\oldnewlabel
\fi}
\fi}
\global\let\hyper@last\relax 
\gdef\HyperFirstAtBeginDocument#1{#1}
\providecommand\HyField@AuxAddToFields[1]{}
\providecommand\HyField@AuxAddToCoFields[2]{}
\providecommand\babel@aux[2]{}
\@nameuse{bbl@beforestart}
\bibstyle{plainnat}
\citation{nas}
\citation{enas,darts,p-darts}
\citation{adanet,mc-dropout,deepens}
\citation{nes}
\citation{nes}
\citation{zhou-ensemble}
\citation{zhou-ensemble}
\providecommand \oddpage@label [2]{}
\babel@aux{american}{}
\@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{1}{section.1}\protected@file@percent }
\citation{amoebanet,nas,nasnet}
\citation{enas}
\citation{p-darts,sdarts,darts-,darts,snas}
\citation{eml}
\citation{mc-dropout}
\citation{deepens}
\citation{snapshot}
\citation{nes}
\citation{nes}
\citation{svgd}
\citation{svgd}
\citation{svgd}
\citation{svgd}
\citation{nes}
\@writefile{toc}{\contentsline {section}{\numberline {2}Related Works \& Background}{2}{section.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {2.1}Neural Architecture Search}{2}{subsection.2.1}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {2.2}Neural Network Ensembles}{2}{subsection.2.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {2.3}Stein Variational Gradient Descent}{2}{subsection.2.3}\protected@file@percent }
\newlabel{sec:bg-svgd}{{2.3}{2}{Stein Variational Gradient Descent}{subsection.2.3}{}}
\newlabel{eq:min-kl}{{1}{2}{Stein Variational Gradient Descent}{equation.2.1}{}}
\newlabel{eq:svgd-iter}{{2}{2}{Stein Variational Gradient Descent}{equation.2.2}{}}
\newlabel{eq:svgd-approx}{{5}{2}{Stein Variational Gradient Descent}{equation.2.5}{}}
\citation{darts}
\citation{nes}
\citation{darts,enas}
\citation{fairnas}
\citation{zhou-ensemble}
\@writefile{toc}{\contentsline {section}{\numberline {3}Neural Ensemble Search via Bayesian Sampling}{3}{section.3}\protected@file@percent }
\newlabel{eq:nes}{{6}{3}{Neural Ensemble Search via Bayesian Sampling}{equation.3.6}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces An illustration of the model training of~supernet. The supernet here consists of three candidate architectures with $r_i$ indicating the selection of one architecture and ${\bm  {\theta }}_i^{t}$ denoting its model parameters at step $t$. In every training step, only one architecture is uniformly sampled to update its parameters and all other architectures will be ignored.\relax }}{3}{figure.caption.1}\protected@file@percent }
\providecommand*\caption@xref[2]{\@setref\relax\@undefined{#1}}
\newlabel{fig:train}{{1}{3}{An illustration of the model training of~supernet. The supernet here consists of three candidate architectures with $r_i$ indicating the selection of one architecture and $\vtheta _i^{t}$ denoting its model parameters at step $t$. In every training step, only one architecture is uniformly sampled to update its parameters and all other architectures will be ignored.\relax }{figure.caption.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.1}Model Training of Supernet}{3}{subsection.3.1}\protected@file@percent }
\newlabel{sec:oneshot}{{3.1}{3}{Model Training of Supernet}{subsection.3.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.2}Distribution of Architectures}{3}{subsection.3.2}\protected@file@percent }
\newlabel{sec:posterior}{{3.2}{3}{Distribution of Architectures}{subsection.3.2}{}}
\citation{vae}
\citation{vae}
\citation{snas}
\citation{zhou-ensemble}
\citation{gdas,snas}
\newlabel{eq:bayes}{{7}{4}{Distribution of Architectures}{equation.3.7}{}}
\newlabel{eq:approx}{{8}{4}{Distribution of Architectures}{equation.3.8}{}}
\newlabel{eq:elbo}{{9}{4}{Distribution of Architectures}{equation.3.9}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.3}Bayesian Sampling}{4}{subsection.3.3}\protected@file@percent }
\newlabel{sec:sampling}{{3.3}{4}{Bayesian Sampling}{subsection.3.3}{}}
\@writefile{loa}{\contentsline {algorithm}{\numberline {1}{\ignorespaces NES via Bayesian Sampling (NESBS)\relax }}{4}{figure.caption.2}\protected@file@percent }
\newlabel{alg:nes}{{1}{4}{NES via Bayesian Sampling (NESBS)\relax }{figure.caption.2}{}}
\@writefile{loa}{\contentsline {algorithm}{\numberline {2}{\ignorespaces MC Sampling\relax }}{4}{figure.caption.2}\protected@file@percent }
\newlabel{alg:mc}{{2}{4}{MC Sampling\relax }{figure.caption.2}{}}
\@writefile{loa}{\contentsline {algorithm}{\numberline {3}{\ignorespaces SVGD-RD\relax }}{4}{figure.caption.2}\protected@file@percent }
\newlabel{alg:svgd-rd}{{3}{4}{SVGD-RD\relax }{figure.caption.2}{}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {3.3.1}Monte Carlo Sampling (MC Sampling)}{4}{subsubsection.3.3.1}\protected@file@percent }
\newlabel{sec:mc}{{3.3.1}{4}{Monte Carlo Sampling (MC Sampling)}{subsubsection.3.3.1}{}}
\citation{mp-svgd}
\citation{proximal}
\citation{nasbench201}
\citation{nasbench201}
\citation{resnet}
\citation{enas}
\citation{darts}
\citation{gdas}
\citation{setn}
\citation{rsps}
\citation{deepens}
\citation{nes}
\citation{nasbench201}
\citation{zhou-ensemble}
\citation{cifar}
\citation{imagenet-16-120}
\@writefile{lof}{\contentsline {figure}{\numberline {2}{\ignorespaces Impact of $\delta $ in SVGD-RD. We use contours and dots to denote the density of target distribution and sampled particles, respectively. The target distribution is chosen to be $p({\bm  {x}}){=}(1/Z)\left [{\mathcal  {N}}({\bm  {x}}| {\bm  {u}}_1, \Sigma _1) + {\mathcal  {N}}({\bm  {x}}| {\bm  {u}}_2, \Sigma _2)\right ]$ where ${\bm  {u}}_1{=}(-1, 0)$, ${\bm  {u}}_2{=}(0, 1)$, $\Sigma _1{=}\Sigma _2{=}\diag  ((0.25, 0.5))$, and $Z$ denotes the normalization constant. Those sampled particles are obtained from Algorithm \ref  {alg:svgd-rd} using $L{=}1000$, $n{=}15$, $\epsilon _l{=}0.1$, and a \emph  {radial basis function} (RBF) kernel. Notably, SVGD-RD tends to sample particles with more diverse probability densities as $\delta $ is increased, hence indicating a controllable (via $\delta $) diversity in our SVGD-RD algorithm. Meanwhile, SVGD-RD can consistently sample particles with high probability densities under varying $\delta $.\relax }}{5}{figure.caption.3}\protected@file@percent }
\newlabel{fig:diversity}{{2}{5}{Impact of $\delta $ in SVGD-RD. We use contours and dots to denote the density of target distribution and sampled particles, respectively. The target distribution is chosen to be $p(\vx ){=}(1/Z)\left [\gN (\vx | \vu _1, \Sigma _1) + \gN (\vx | \vu _2, \Sigma _2)\right ]$ where $\vu _1{=}(-1, 0)$, $\vu _2{=}(0, 1)$, $\Sigma _1{=}\Sigma _2{=}\diag ((0.25, 0.5))$, and $Z$ denotes the normalization constant. Those sampled particles are obtained from Algorithm \ref {alg:svgd-rd} using $L{=}1000$, $n{=}15$, $\epsilon _l{=}0.1$, and a \emph {radial basis function} (RBF) kernel. Notably, SVGD-RD tends to sample particles with more diverse probability densities as $\delta $ is increased, hence indicating a controllable (via $\delta $) diversity in our SVGD-RD algorithm. Meanwhile, SVGD-RD can consistently sample particles with high probability densities under varying $\delta $.\relax }{figure.caption.3}{}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {3.3.2}SVGD with Regularized Diversity (SVGD-RD)}{5}{subsubsection.3.3.2}\protected@file@percent }
\newlabel{sec:svgd-rd}{{3.3.2}{5}{SVGD with Regularized Diversity (SVGD-RD)}{subsubsection.3.3.2}{}}
\newlabel{eq:min-kl-max-div}{{10}{5}{SVGD with Regularized Diversity (SVGD-RD)}{equation.3.10}{}}
\@writefile{loe}{\contentsline {proposition}{\ifthmt@listswap Proposition~1\else \numberline {1}Proposition\fi }{5}{proposition.1}\protected@file@percent }
\newlabel{prop:svgdrd-update}{{1}{5}{}{proposition.1}{}}
\@writefile{loe}{\contentsline {proposition}{\ifthmt@listswap Proposition~2\else \numberline {2}Proposition\fi }{5}{proposition.2}\protected@file@percent }
\newlabel{prop:exploitation}{{2}{5}{}{proposition.2}{}}
\@writefile{toc}{\contentsline {section}{\numberline {4}Experiments}{5}{section.4}\protected@file@percent }
\newlabel{sec:exps}{{4}{5}{Experiments}{section.4}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.1}Search in NAS-Bench-201}{5}{subsection.4.1}\protected@file@percent }
\newlabel{sec:nasbench}{{4.1}{5}{Search in NAS-Bench-201}{subsection.4.1}{}}
\citation{darts}
\citation{imagenet}
\citation{nasnet}
\citation{amoebanet}
\citation{pnas}
\citation{enas}
\citation{darts}
\citation{gdas}
\citation{p-darts}
\citation{darts-}
\citation{sdarts}
\citation{nes}
\citation{mc-dropout}
\citation{ensemble-for-defense}
\citation{fgsm}
\citation{pgd}
\citation{cw}
\citation{autoattack}
\citation{robnet}
\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces Comparison of architectures selected by different NAS and ensemble (search) algorithms in NAS-Bench-201 with ensemble size $n=3$. Test errors are reported with the mean and standard error of three independent trials and our search costs are evaluated on a single Nvidia $1080$Ti GPU. Results marked by $\dagger $ are reported by \citet  {nasbench201}.\relax }}{6}{table.caption.4}\protected@file@percent }
\newlabel{tab:nasbench201}{{1}{6}{Comparison of architectures selected by different NAS and ensemble (search) algorithms in NAS-Bench-201 with ensemble size $n=3$. Test errors are reported with the mean and standard error of three independent trials and our search costs are evaluated on a single Nvidia $1080$Ti GPU. Results marked by $\dagger $ are reported by \citet {nasbench201}.\relax }{table.caption.5}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.2}Search in The DARTS Search Space}{6}{subsection.4.2}\protected@file@percent }
\newlabel{sec:darts}{{4.2}{6}{Search in The DARTS Search Space}{subsection.4.2}{}}
\@writefile{toc}{\contentsline {paragraph}{Ensemble for classification.}{6}{paragraph*.10}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Ensemble for adversarial defense.}{6}{paragraph*.11}\protected@file@percent }
\newlabel{sec:exp-adversarial}{{4.2}{6}{Ensemble for adversarial defense}{paragraph*.11}{}}
\citation{disagreement}
\citation{t-sne}
\@writefile{lot}{\contentsline {table}{\numberline {2}{\ignorespaces Comparison of different image classifiers on CIFAR-10/100. Results of MC DropPath are from a drop~probability of $0.01$ and our search costs are evaluated on Nvidia $1080$Ti.\relax }}{7}{table.caption.6}\protected@file@percent }
\newlabel{tab:accuracy-cifar}{{2}{7}{Comparison of different image classifiers on CIFAR-10/100. Results of MC DropPath are from a drop~probability of $0.01$ and our search costs are evaluated on Nvidia $1080$Ti.\relax }{table.caption.7}{}}
\@writefile{lot}{\contentsline {table}{\numberline {3}{\ignorespaces Comparison of image classifiers on ImageNet. The ensemble size is set to $n=3$ for NES-RS and NESBS.\relax }}{7}{table.caption.8}\protected@file@percent }
\newlabel{tab:accuracy-imagenet}{{3}{7}{Comparison of image classifiers on ImageNet. The ensemble size is set to $n=3$ for NES-RS and NESBS.\relax }{table.caption.9}{}}
\citation{zhou-ensemble}
\@writefile{lot}{\contentsline {table}{\numberline {4}{\ignorespaces Comparison of adversarial defense among different ensemble (search) algorithms on CIFAR-10/100 under white-box adversarial attacks. The \emph  {Attack} and \emph  {Defense} columns denote the test \emph  {accuracy} under the attack using a single model randomly sampled from an ensemble and the defense using the whole ensemble, respectively. Each result reports the mean and standard deviation of test accuracies for $3$ rounds of the attack-defense process with an ensemble size of $n=3$.\relax }}{8}{table.caption.12}\protected@file@percent }
\newlabel{tab:adversarial}{{4}{8}{Comparison of adversarial defense among different ensemble (search) algorithms on CIFAR-10/100 under white-box adversarial attacks. The \emph {Attack} and \emph {Defense} columns denote the test \emph {accuracy} under the attack using a single model randomly sampled from an ensemble and the defense using the whole ensemble, respectively. Each result reports the mean and standard deviation of test accuracies for $3$ rounds of the attack-defense process with an ensemble size of $n=3$.\relax }{table.caption.12}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.3}Single-model Performances and Diverse Model Predictions}{8}{subsection.4.3}\protected@file@percent }
\newlabel{sec:trade-off}{{4.3}{8}{Single-model Performances and Diverse Model Predictions}{subsection.4.3}{}}
\@writefile{lot}{\contentsline {table}{\numberline {5}{\ignorespaces Quantitative comparison of the single-model performances (measured by ATE (\%), smaller is better) and the diversity of model predictions (measured by PPD (\%), larger is better) achieved by different ensemble (search) algorithms with an ensemble size of $3$ on CIFAR-10/100.\relax }}{8}{table.caption.13}\protected@file@percent }
\newlabel{tab:perf_vs_diver}{{5}{8}{Quantitative comparison of the single-model performances (measured by ATE (\%), smaller is better) and the diversity of model predictions (measured by PPD (\%), larger is better) achieved by different ensemble (search) algorithms with an ensemble size of $3$ on CIFAR-10/100.\relax }{table.caption.13}{}}
\@writefile{toc}{\contentsline {section}{\numberline {5}Conclusion}{8}{section.5}\protected@file@percent }
\citation{pc-darts}
\citation{concrete}
\citation{gumbel-softmax}
\citation{adam}
\citation{bo}
\citation{cutout}
\citation{dropnas}
\bibdata{shu_406}
\bibcite{cw}{{1}{2017}{{Carlini and Wagner}}{{}}}
\bibcite{sdarts}{{2}{2020}{{Chen and Hsieh}}{{}}}
\bibcite{p-darts}{{3}{2019}{{Chen et~al.}}{{Chen, Xie, Wu, and Tian}}}
\bibcite{imagenet-16-120}{{4}{2017}{{Chrabaszcz et~al.}}{{Chrabaszcz, Loshchilov, and Hutter}}}
\bibcite{fairnas}{{5}{2019}{{Chu et~al.}}{{Chu, Zhang, Xu, and Li}}}
\bibcite{darts-}{{6}{2020}{{Chu et~al.}}{{Chu, Wang, Zhang, Lu, Wei, and Yan}}}
\bibcite{adanet}{{7}{2017}{{Cortes et~al.}}{{Cortes, Gonzalvo, Kuznetsov, Mohri, and Yang}}}
\bibcite{autoattack}{{8}{2020}{{Croce and Hein}}{{}}}
\bibcite{imagenet}{{9}{2009}{{Deng et~al.}}{{Deng, Dong, Socher, Li, Li, and Fei-Fei}}}
\bibcite{cutout}{{10}{2017}{{Devries and Taylor}}{{}}}
\bibcite{eml}{{11}{2000}{{Dietterich}}{{}}}
\bibcite{gdas}{{12}{2019{a}}{{Dong and Yang}}{{}}}
\bibcite{setn}{{13}{2019{b}}{{Dong and Yang}}{{}}}
\bibcite{nasbench201}{{14}{2020}{{Dong and Yang}}{{}}}
\bibcite{disagreement}{{15}{2019}{{Fort et~al.}}{{Fort, Hu, and Lakshminarayanan}}}
\bibcite{mc-dropout}{{16}{2016}{{Gal and Ghahramani}}{{}}}
\bibcite{fgsm}{{17}{2015}{{Goodfellow et~al.}}{{Goodfellow, Shlens, and Szegedy}}}
\bibcite{robnet}{{18}{2020}{{Guo et~al.}}{{Guo, Yang, Xu, Liu, and Lin}}}
\bibcite{resnet}{{19}{2016}{{He et~al.}}{{He, Zhang, Ren, and Sun}}}
\bibcite{dropnas}{{20}{2020}{{Hong et~al.}}{{Hong, Li, Zhang, Tang, Wang, Li, and Yu}}}
\bibcite{snapshot}{{21}{2017}{{Huang et~al.}}{{Huang, Li, Pleiss, Liu, Hopcroft, and Weinberger}}}
\bibcite{gumbel-softmax}{{22}{2017}{{Jang et~al.}}{{Jang, Gu, and Poole}}}
\bibcite{adam}{{23}{2015}{{Kingma and Ba}}{{}}}
\@writefile{lof}{\contentsline {figure}{\numberline {3}{\ignorespaces Qualitative comparison of (a) the single-model performances and (b) the diverse model predictions achieved by different ensemble (search) algorithms with an ensemble size of $n=3$ on CIFAR-10. Each architecture in (b) is independently evaluated for ten times to visualize their model predictions, which follows from DeepEns.\relax }}{9}{figure.caption.14}\protected@file@percent }
\newlabel{fig:perf_vs_diver}{{3}{9}{Qualitative comparison of (a) the single-model performances and (b) the diverse model predictions achieved by different ensemble (search) algorithms with an ensemble size of $n=3$ on CIFAR-10. Each architecture in (b) is independently evaluated for ten times to visualize their model predictions, which follows from DeepEns.\relax }{figure.caption.14}{}}
\bibcite{vae}{{24}{2014}{{Kingma and Welling}}{{}}}
\bibcite{cifar}{{25}{2009}{{Krizhevsky}}{{}}}
\bibcite{deepens}{{26}{2017}{{Lakshminarayanan et~al.}}{{Lakshminarayanan, Pritzel, and Blundell}}}
\bibcite{rsps}{{27}{2019}{{Li and Talwalkar}}{{}}}
\bibcite{pnas}{{28}{2018}{{Liu et~al.}}{{Liu, Zoph, Neumann, Shlens, Hua, Li, Fei{-}Fei, Yuille, Huang, and Murphy}}}
\bibcite{darts}{{29}{2019}{{Liu et~al.}}{{Liu, Simonyan, and Yang}}}
\bibcite{svgd}{{30}{2016}{{Liu and Wang}}{{}}}
\bibcite{concrete}{{31}{2017}{{Maddison et~al.}}{{Maddison, Mnih, and Teh}}}
\bibcite{pgd}{{32}{2018}{{Madry et~al.}}{{Madry, Makelov, Schmidt, Tsipras, and Vladu}}}
\bibcite{proximal}{{33}{2014}{{Parikh and Boyd}}{{}}}
\bibcite{enas}{{34}{2018}{{Pham et~al.}}{{Pham, Guan, Zoph, Le, and Dean}}}
\bibcite{amoebanet}{{35}{2019}{{Real et~al.}}{{Real, Aggarwal, Huang, and Le}}}
\bibcite{bo}{{36}{2012}{{Snoek et~al.}}{{Snoek, Larochelle, and Adams}}}
\bibcite{ensemble-for-defense}{{37}{2017}{{Strauss et~al.}}{{Strauss, Hanselmann, Junginger, and Ulmer}}}
\bibcite{t-sne}{{38}{2008}{{van~der Maaten and Hinton}}{{}}}
\bibcite{snas}{{39}{2019}{{Xie et~al.}}{{Xie, Zheng, Liu, and Lin}}}
\bibcite{pc-darts}{{40}{2020}{{Xu et~al.}}{{Xu, Xie, Zhang, Chen, Qi, Tian, and Xiong}}}
\bibcite{nes}{{41}{2021}{{Zaidi et~al.}}{{Zaidi, Zela, Elsken, Holmes, Hutter, and Teh}}}
\bibcite{zhou-ensemble}{{42}{2012}{{Zhou}}{{}}}
\bibcite{mp-svgd}{{43}{2018}{{Zhuo et~al.}}{{Zhuo, Liu, Shi, Zhu, Chen, and Zhang}}}
\bibcite{nas}{{44}{2017}{{Zoph and Le}}{{}}}
\bibcite{nasnet}{{45}{2018}{{Zoph et~al.}}{{Zoph, Vasudevan, Shlens, and Le}}}
\newlabel{LastPage}{{}{10}{}{page.10}{}}
\xdef\lastpage@lastpage{10}
\xdef\lastpage@lastpageHy{10}
\gdef \@abspage@last{10}
