@inproceedings{zou2021benign,
  title={Benign overfitting of constant-stepsize sgd for linear regression},
  author={Zou, Difan and Wu, Jingfeng and Braverman, Vladimir and Gu, Quanquan and Kakade, Sham},
  booktitle={Conference on Learning Theory},
  pages={4633--4635},
  year={2021},
  organization={PMLR}
}


@inproceedings{dan2020sharp,
  title={Sharp statistical guaratees for adversarially robust gaussian classification},
  author={Dan, Chen and Wei, Yuting and Ravikumar, Pradeep},
  booktitle={International Conference on Machine Learning},
  pages={2345--2355},
  year={2020},
  organization={PMLR}
}

@article{taheri2020asymptotic,
  title={Asymptotic behavior of adversarial training in binary classification},
  author={Taheri, Hossein and Pedarsani, Ramtin and Thrampoulidis, Christos},
  journal={arXiv preprint arXiv:2010.13275},
  year={2020}
}

@article{shamir2022implicit,
  title={The Implicit Bias of Benign Overfitting},
  author={Shamir, Ohad},
  journal={arXiv preprint arXiv:2201.11489},
  year={2022}
}



@article{cao2022benign,
  title={Benign Overfitting in Two-layer Convolutional Neural Networks},
  author={Cao, Yuan and Chen, Zixiang and Belkin, Mikhail and Gu, Quanquan},
  journal={arXiv preprint arXiv:2202.06526},
  year={2022}
}
@inproceedings{frei2022benign,
  title={Benign overfitting without linearity: Neural network classifiers trained by gradient descent for noisy linear data},
  author={Frei, Spencer and Chatterji, Niladri S and Bartlett, Peter},
  booktitle={Conference on Learning Theory},
  pages={2668--2703},
  year={2022},
  organization={PMLR}
}

@article{javanmard2020precise,
  title={Precise statistical analysis of classification accuracies for adversarial training},
  author={Javanmard, Adel and Soltanolkotabi, Mahdi},
  journal={arXiv preprint arXiv:2010.11213},
  year={2020}
}




@article{chatterji2022foolish,
  title={Foolish crowds support benign overfitting},
  author={Chatterji, Niladri S and Long, Philip M},
  journal={Journal of Machine Learning Research},
  volume={23},
  number={125},
  pages={1--12},
  year={2022}
}


@article{wald2022malign,
  title={Malign Overfitting: Interpolation Can Provably Preclude Invariance},
  author={Wald, Yoav and Yona, Gal and Shalit, Uri and Carmon, Yair},
  journal={arXiv preprint arXiv:2211.15724},
  year={2022}
}






@incollection{Bengio+chapter2007,
author = {Bengio, Yoshua and LeCun, Yann},
booktitle = {Large Scale Kernel Machines},
publisher = {MIT Press},
title = {Scaling Learning Algorithms Towards {AI}},
year = {2007}
}



@article{lecun2010mnist,
  title={MNIST handwritten digit database},
  author={LeCun, Yann and Cortes, Corinna and Burges, CJ},
  year={2010}
}

 

@article{zagoruyko2016wide,
  title={Wide residual networks},
  author={Zagoruyko, Sergey and Komodakis, Nikos},
  journal={arXiv preprint arXiv:1605.07146},
  year={2016}
}

@inproceedings{
brendel2018decisionbased,
title={Decision-Based Adversarial Attacks: Reliable Attacks Against Black-Box Machine Learning Models},
author={Wieland Brendel and Jonas Rauber and Matthias Bethge},
booktitle={ICLR},
year={2018}
}

@article{chen2019hopskipjumpattack,
  title={Hopskipjumpattack: A query-efficient decision-based attack},
  author={Chen, Jianbo and Jordan, Michael I and Wainwright, Martin J},
  journal={arXiv preprint arXiv:1904.02144},
  volume={3},
  year={2019}
}

@inproceedings{
cheng2018queryefficient,
title={Query-Efficient Hard-label Black-box Attack: An Optimization-based Approach},
author={Minhao Cheng and Thong Le and Pin-Yu Chen and Huan Zhang and JinFeng Yi and Cho-Jui Hsieh},
booktitle={ICLR},
year={2019}
}

@inproceedings{
cheng2020signopt,
title={Sign-OPT: A Query-Efficient Hard-label Adversarial Attack},
author={Minhao Cheng and Simranjit Singh and Patrick H. Chen and Pin-Yu Chen and Sijia Liu and Cho-Jui Hsieh},
booktitle={ICLR},
year={2020}
}

@article{ma2018characterizing,
  title={Characterizing adversarial subspaces using local intrinsic dimensionality},
  author={Ma, Xingjun and Li, Bo and Wang, Yisen and Erfani, Sarah M and Wijewickrema, Sudanthi and Schoenebeck, Grant and Song, Dawn and Houle, Michael E and Bailey, James},
  journal={ICLR},
  year={2018}
}

@article{samangouei2018defense,
  title={Defense-gan: Protecting classifiers against adversarial attacks using generative models},
  author={Samangouei, Pouya and Kabkab, Maya and Chellappa, Rama},
  journal={ICLR},
  year={2018}
}

@article{song2017pixeldefend,
  title={Pixeldefend: Leveraging generative models to understand and defend against adversarial examples},
  author={Song, Yang and Kim, Taesup and Nowozin, Sebastian and Ermon, Stefano and Kushman, Nate},
  journal={ICLR},
  year={2018}
}

@article{xie2017mitigating,
  title={Mitigating adversarial effects through randomization},
  author={Xie, Cihang and Wang, Jianyu and Zhang, Zhishuai and Ren, Zhou and Yuille, Alan},
  journal={ICLR},
  year={2018}
}

@article{guo2017countering,
  title={Countering adversarial images using input transformations},
  author={Guo, Chuan and Rana, Mayank and Cisse, Moustapha and Van Der Maaten, Laurens},
  journal={ICLR},
  year={2018}
}

@inproceedings{wang2019convergence,
  title={On the Convergence and Robustness of Adversarial Training},
  author={Wang, Yisen and Ma, Xingjun and Bailey, James and Yi, Jinfeng and Zhou, Bowen and Gu, Quanquan},
  booktitle={ICML},
  pages={6586--6595},
  year={2019}
}

@article{hinton2012deep,
  title={Deep neural networks for acoustic modeling in speech recognition},
  author={Hinton, Geoffrey and Deng, Li and Yu, Dong and Dahl, George and Mohamed, Abdel-rahman and Jaitly, Navdeep and Senior, Andrew and Vanhoucke, Vincent and Nguyen, Patrick and Kingsbury, Brian and others},
  journal={IEEE Signal processing magazine},
  volume={29},
  year={2012}
}

@article{sutskever2012imagenet,
  title={Imagenet classification with deep convolutional neural networks},
  author={Sutskever, Ilya and Hinton, Geoffrey E and Krizhevsky, A},
  journal={NeurIPS},
  pages={1097--1105},
  year={2012}
}


@article{kannan2018adversarial,
  title={Adversarial logit pairing},
  author={Kannan, Harini and Kurakin, Alexey and Goodfellow, Ian},
  journal={arXiv preprint arXiv:1803.06373},
  year={2018}
}

@inproceedings{zhang2019theoretically,
  title={Theoretically Principled Trade-off between Robustness and Accuracy},
  author={Zhang, Hongyang and Yu, Yaodong and Jiao, Jiantao and Xing, Eric and El Ghaoui, Laurent and Jordan, Michael},
  booktitle={ICML},
  pages={7472--7482},
  year={2019}
}

@inproceedings{uesato2018adversarial,
  title={Adversarial Risk and the Dangers of Evaluating Against Weak Attacks},
  author={Uesato, Jonathan and O’Donoghue, Brendan and Kohli, Pushmeet and Oord, Aaron},
  booktitle={ICML},
  pages={5025--5034},
  year={2018}
}

@article{andriushchenko2019square,
  title={Square Attack: a query-efficient black-box adversarial attack via random search},
  author={Andriushchenko, Maksym and Croce, Francesco and Flammarion, Nicolas and Hein, Matthias},
  journal={arXiv preprint arXiv:1912.00049},
  year={2019}
}

@inproceedings{
AlDujaili2020Sign,
title={Sign Bits Are All You Need for Black-Box Attacks},
author={Abdullah Al-Dujaili and Una-May O'Reilly},
booktitle={ICLR},
year={2020}
}

@article{balasubramanian2018zeroth,
  title={Zeroth-order (Non)-Convex Stochastic Optimization via Conditional Gradient and Gradient Updates},
  author={Balasubramanian, Krishnakumar and Ghadimi, Saeed},
  journal={arXiv preprint arXiv:1809.06474},
  year={2018}
}

@inproceedings{he2016identity,
  title={Identity mappings in deep residual networks},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={ECCV},
  pages={630--645},
  year={2016},
  organization={Springer}
}

@article{li2018query,
  title={Query-Efficient Black-Box Attack by Active Learning},
  author={Li, Pengcheng and Yi, Jinfeng and Zhang, Lijun},
  journal={arXiv preprint arXiv:1809.04913},
  year={2018}
}

@article{xu2017can,
  title={Can you fool AI with adversarial examples on a visual Turing test?},
  author={Xu, Xiaojun and Chen, Xinyun and Liu, Chang and Rohrbach, Anna and Darell, Trevor and Song, Dawn},
  journal={arXiv preprint arXiv:1709.08693},
  year={2017}
}

@article{su2018robustness,
  title={Is Robustness the Cost of Accuracy?--A Comprehensive Study on the Robustness of 18 Deep Image Classification Models},
  author={Su, Dong and Zhang, Huan and Chen, Hongge and Yi, Jinfeng and Chen, Pin-Yu and Gao, Yupeng},
  journal={ECCV},
  year= {2018},
}

@article{chen2017show,
  title={Show-and-Fool: Crafting Adversarial Examples for Neural Image Captioning},
  author={Chen, Hongge and Zhang, Huan and Chen, Pin-Yu and Yi, Jinfeng and Hsieh, Cho-Jui},
  journal={arXiv preprint arXiv:1712.02051},
  year={2017}
}

@article{DBLP:journals/corr/abs-1805-11770,
  author    = {Chun{-}Chen Tu and
               Pai{-}Shun Ting and
               Pin{-}Yu Chen and
               Sijia Liu and
               Huan Zhang and
               Jinfeng Yi and
               Cho{-}Jui Hsieh and
               Shin{-}Ming Cheng},
  title     = {AutoZOOM: Autoencoder-based Zeroth Order Optimization Method for Attacking Black-box Neural Networks},
  journal   = {CoRR},
  volume    = {abs/1805.11770},
  year      = {2018},
  archivePrefix = {arXiv},
}

@article{bhagoji2017exploring,
  title={Exploring the Space of Black-box Attacks on Deep Neural Networks},
  author={Bhagoji, Arjun Nitin and He, Warren and Li, Bo and Song, Dawn},
  journal={arXiv:1712.09491},
  year={2017}
}

@article{liu2016delving,
  title={Delving into transferable adversarial examples and black-box attacks},
  author={Liu, Yanpei and Chen, Xinyun and Liu, Chang and Song, Dawn},
  journal={ICDM},
  year={2018}
}

@article{cheng2018query,
  title={Query-Efficient Hard-label Black-box Attack: An Optimization-based Approach},
  author={Cheng, Minhao and Le, Thong and Chen, Pin-Yu and Yi, Jinfeng and Zhang, Huan and Hsieh, Cho-Jui},
  journal={arXiv:1807.04457},
  year={2018}
}

@article{Hinton06,
author = {Hinton, Geoffrey E. and Osindero, Simon and Teh, Yee Whye},
journal = {Neural Computation},
pages = {1527--1554},
title = {A Fast Learning Algorithm for Deep Belief Nets},
volume = {18},
year = {2006}
}

@book{goodfellow2016deep,
title={Deep learning},
author={Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron and Bengio, Yoshua},
volume={1},
year={2016},
publisher={MIT Press}
}


@article{gao2018information,
  title={On the information-adaptive variants of the ADMM: an iteration complexity perspective},
  author={Gao, Xiang and Jiang, Bo and Zhang, Shuzhong},
  journal={Journal of Scientific Computing},
  volume={76},
  number={1},
  pages={327--363},
  year={2018},
  publisher={Springer}
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={CVPR},
  pages={770--778},
  year={2016}
}

@article{krizhevsky2009learning,
  title={Learning multiple layers of features from tiny images},
  author={Krizhevsky, Alex and Hinton, Geoffrey and others},
  year={2009},
  publisher={Citeseer}
}


@inproceedings{athalye2018obfuscated,
  title={Obfuscated Gradients Give a False Sense of Security: Circumventing Defenses to Adversarial Examples},
  author={Athalye, Anish and Carlini, Nicholas and Wagner, David},
  booktitle = {ICML},
  year={2018}
}

@inproceedings{papernot2016distillation,
  title={Distillation as a defense to adversarial perturbations against deep neural networks},
  author={Papernot, Nicolas and McDaniel, Patrick and Wu, Xi and Jha, Somesh and Swami, Ananthram},
  booktitle={SP},
  pages={582--597},
  year={2016},
  organization={IEEE}
}

@book{boyd2004convex,
  title={Convex optimization},
  author={Boyd, Stephen and Boyd, Stephen P and Vandenberghe, Lieven},
  year={2004},
  publisher={Cambridge university press}
}

@inproceedings{salman2019provably,
  title={Provably robust deep learning via adversarially trained smoothed classifiers},
  author={Salman, Hadi and Li, Jerry and Razenshteyn, Ilya and Zhang, Pengchuan and Zhang, Huan and Bubeck, Sebastien and Yang, Greg},
  booktitle={NeurIPS},
  pages={11292--11303},
  year={2019}
}

@inproceedings{cohen2019certified,
  title={Certified Adversarial Robustness via Randomized Smoothing},
  author={Cohen, Jeremy and Rosenfeld, Elan and Kolter, Zico},
  booktitle={ICML},
  pages={1310--1320},
  year={2019}
}

@article{micikevicius2017mixed,
  title={Mixed precision training},
  author={Micikevicius, Paulius and Narang, Sharan and Alben, Jonah and Diamos, Gregory and Elsen, Erich and Garcia, David and Ginsburg, Boris and Houston, Michael and Kuchaiev, Oleksii and Venkatesh, Ganesh and others},
  journal={arXiv preprint arXiv:1710.03740},
  year={2017}
}

@inproceedings{smith2017cyclical,
  title={Cyclical learning rates for training neural networks},
  author={Smith, Leslie N},
  booktitle={WACV},
  pages={464--472},
  year={2017},
  organization={IEEE}
}

@inproceedings{croce2020minimally,
  author    = {F. Croce and M. Hein},
  title     = {Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack},
  booktitle = {ICML},
  year      = {2020}
}

@inproceedings{qin2019adversarial,
  title={Adversarial robustness through local linearization},
  author={Qin, Chongli and Martens, James and Gowal, Sven and Krishnan, Dilip and Dvijotham, Krishnamurthy and Fawzi, Alhussein and De, Soham and Stanforth, Robert and Kohli, Pushmeet},
  booktitle={NeurIPS},
  pages={13847--13856},
  year={2019}
}

@inproceedings{alayrac2019labels,
  title={Are Labels Required for Improving Adversarial Robustness?},
  author={Alayrac, Jean-Baptiste and Uesato, Jonathan and Huang, Po-Sen and Fawzi, Alhussein and Stanforth, Robert and Kohli, Pushmeet},
  booktitle={NeurIPS},
  pages={12214--12223},
  year={2019}
}

@inproceedings{carmon2019unlabeled,
  title={Unlabeled data improves adversarial robustness},
  author={Carmon, Yair and Raghunathan, Aditi and Schmidt, Ludwig and Duchi, John C and Liang, Percy S},
  booktitle={NeurIPS},
  pages={11192--11203},
  year={2019}
}

@inproceedings{hendrycks2019using,
  title={Using Pre-Training Can Improve Model Robustness and Uncertainty},
  author={Hendrycks, Dan and Lee, Kimin and Mazeika, Mantas},
  booktitle={ICML},
  pages={2712--2721},
  year={2019}
}

@inproceedings{shafahi2019adversarial,
  title={Adversarial training for free!},
  author={Shafahi, Ali and Najibi, Mahyar and Ghiasi, Mohammad Amin and Xu, Zheng and Dickerson, John and Studer, Christoph and Davis, Larry S and Taylor, Gavin and Goldstein, Tom},
  booktitle={NeurIPS},
  pages={3358--3369},
  year={2019}
}

@inproceedings{chen2018frank,
  title={A Frank-Wolfe framework for efficient and effective adversarial attacks},
  author={Chen, Jinghui and Zhou, Dongruo and Yi, Jinfeng and Gu, Quanquan},
  booktitle={AAAI},
  year={2020}
}

@inproceedings{krizhevsky2012imagenet,
  title={Imagenet classification with deep convolutional neural networks},
  author={Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
  booktitle={Neurips},
  pages={1097--1105},
  year={2012}
}

@inproceedings{girshick2015fast,
  title={Fast r-cnn},
  author={Girshick, Ross},
  booktitle={ICCV},
  pages={1440--1448},
  year={2015}
}

@inproceedings{ren2015faster,
  title={Faster r-cnn: Towards real-time object detection with region proposal networks},
  author={Ren, Shaoqing and He, Kaiming and Girshick, Ross and Sun, Jian},
  booktitle={Neurips},
  pages={91--99},
  year={2015}
}

@article{mohamed2012acoustic,
  title={Acoustic modeling using deep belief networks},
  author={Mohamed, Abdel-rahman and Dahl, George E and Hinton, Geoffrey and others},
  journal={IEEE Trans. Audio, Speech \& Language Processing},
  volume={20},
  number={1},
  pages={14--22},
  year={2012}
}

@inproceedings{bahdanau2016end,
  title={End-to-end attention-based large vocabulary speech recognition},
  author={Bahdanau, Dzmitry and Chorowski, Jan and Serdyuk, Dmitriy and Brakel, Philemon and Bengio, Yoshua},
  booktitle={ICASSP},
  pages={4945--4949},
  year={2016},
  organization={IEEE}
}

@article{szegedy2013intriguing,
  title={Intriguing properties of neural networks},
  author={Szegedy, Christian and Zaremba, Wojciech and Sutskever, Ilya and Bruna, Joan and Erhan, Dumitru and Goodfellow, Ian and Fergus, Rob},
  journal={arXiv preprint arXiv:1312.6199},
  year={2013}
}

@article{kurakin2016adversarial,
  title={Adversarial examples in the physical world},
  author={Kurakin, Alexey and Goodfellow, Ian and Bengio, Samy},
  journal={arXiv preprint arXiv:1607.02533},
  year={2016}
}

@inproceedings{carlini2016hidden,
  title={Hidden Voice Commands.},
  author={Carlini, Nicholas and Mishra, Pratyush and Vaidya, Tavish and Zhang, Yuankai and Sherr, Micah and Shields, Clay and Wagner, David and Zhou, Wenchao},
  booktitle={USENIX Security Symposium},
  pages={513--530},
  year={2016}
}


@article{ilyas2018prior,
  title={Prior convictions: Black-box adversarial attacks with bandits and priors},
  author={Ilyas, Andrew and Engstrom, Logan and Madry, Aleksander},
  journal={International Conference on Learning Representations},
  year={2019}
}

@article{dhillon2018stochastic,
  title={Stochastic activation pruning for robust adversarial defense},
  author={Dhillon, Guneet S and Azizzadenesheli, Kamyar and Lipton, Zachary C and Bernstein, Jeremy and Kossaifi, Jean and Khanna, Aran and Anandkumar, Anima},
  journal={ICLR},
  year={2018}
}

@inproceedings{meng2017magnet,
  title={Magnet: a two-pronged defense against adversarial examples},
  author={Meng, Dongyu and Chen, Hao},
  booktitle={CCS},
  pages={135--147},
  year={2017}
}

@inproceedings{liao2018defense,
  title={Defense against adversarial attacks using high-level representation guided denoiser},
  author={Liao, Fangzhou and Liang, Ming and Dong, Yinpeng and Pang, Tianyu and Hu, Xiaolin and Zhu, Jun},
  booktitle={CVPR},
  pages={1778--1787},
  year={2018}
}

@article{xu2017feature,
  title={Feature squeezing: Detecting adversarial examples in deep neural networks},
  author={Xu, Weilin and Evans, David and Qi, Yanjun},
  journal={arXiv preprint arXiv:1704.01155},
  year={2017}
}

@inproceedings{moon2019parsimonious,
  title={Parsimonious Black-Box Adversarial Attacks via Efficient Combinatorial Optimization},
  author={Moon, Seungyong and An, Gaon and Song, Hyun Oh},
  booktitle={ICML},
  pages={4636--4645},
  year={2019}
}


@inproceedings{li2019nattack,
  title={NATTACK: Learning the Distributions of Adversarial Examples for an Improved Black-Box Attack on Deep Neural Networks},
  author={Li, Yandong and Li, Lijun and Wang, Liqiang and Zhang, Tong and Gong, Boqing},
  booktitle={ICML},
  pages={3866--3876},
  year={2019}
}

@article{goodfellow6572explaining,
  title={Explaining and harnessing adversarial examples},
  author={Goodfellow, Ian J and Shlens, Jonathon and Szegedy, Christian},
  journal={ICLR},
  year={2015}
}

@inproceedings{croce2020reliable,
    title = {Reliable evaluation of adversarial robustness with an ensemble of diverse parameter-free attacks},
    author = {Francesco Croce and Matthias Hein},
    booktitle = {ICML},
    year = {2020}
}

@article{andriushchenko2020understanding,
  title={Understanding and Improving Fast Adversarial Training},
  author={Andriushchenko, Maksym and Flammarion, Nicolas},
  journal={Advances in Neural Information Processing Systems},
  year={2020}
}

@article{tashiro2020diversity,
  title={Diversity can be Transferred: Output Diversification for White-and Black-box Attacks},
  author={Tashiro, Yusuke and Song, Yang and Ermon, Stefano},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}

@inproceedings{chen2020rays,
  title={RayS: A Ray Searching Method for Hard-label Adversarial Attack},
  author={Chen, Jinghui and Gu, Quanquan},
  booktitle={SIGKDD},
  year={2020}
}

@article{rice2020overfitting,
  title={Overfitting in adversarially robust deep learning},
  author={Rice, Leslie and Wong, Eric and Kolter, J Zico},
  journal={ICML},
  year={2020}
}

@article{li2020towards,
  title={Towards Understanding Fast Adversarial Training},
  author={Li, Bai and Wang, Shiqi and Jana, Suman and Carin, Lawrence},
  journal={arXiv preprint arXiv:2006.03089},
  year={2020}
}

@inproceedings{
Wong2020Fast,
title={Fast is better than free: Revisiting adversarial training},
author={Eric Wong and Leslie Rice and J. Zico Kolter},
booktitle={ICLR},
year={2020},
}

@article{duchi2012randomized,
  title={Randomized smoothing for stochastic optimization},
  author={Duchi, John C and Bartlett, Peter L and Wainwright, Martin J},
  journal={SIAM Journal on Optimization},
  volume={22},
  number={2},
  pages={674--701},
  year={2012},
  publisher={SIAM}
}

@inproceedings{pattanaik2018robust,
  title={Robust deep reinforcement learning with adversarial attacks},
  author={Pattanaik, Anay and Tang, Zhenyi and Liu, Shuijing and Bommannan, Gautham and Chowdhary, Girish},
  booktitle={Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={2040--2042},
  year={2018},
  organization={IFAAMAS}
}


@article{baluja2017adversarial,
  title={Adversarial transformation networks: Learning to generate adversarial examples},
  author={Baluja, Shumeet and Fischer, Ian},
  journal={arXiv preprint arXiv:1703.09387},
  year={2017}
}

@article{xiao2018generating,
  title={Generating adversarial examples with adversarial networks},
  author={Xiao, Chaowei and Li, Bo and Zhu, Jun-Yan and He, Warren and Liu, Mingyan and Song, Dawn},
  journal={arXiv preprint arXiv:1801.02610},
  year={2018}
}

@article{su2017one,
  title={One pixel attack for fooling deep neural networks},
  author={Su, Jiawei and Vargas, Danilo Vasconcellos and Kouichi, Sakurai},
  journal={arXiv preprint arXiv:1710.08864},
  year={2017}
}

@inproceedings{papernot2016limitations,
  title={The limitations of deep learning in adversarial settings},
  author={Papernot, Nicolas and McDaniel, Patrick and Jha, Somesh and Fredrikson, Matt and Celik, Z Berkay and Swami, Ananthram},
  booktitle={EuroS\&P},
  pages={372--387},
  year={2016},
  organization={IEEE}
}

@inproceedings{moosavi2016deepfool,
  title={Deepfool: a simple and accurate method to fool deep neural networks},
  author={Moosavi-Dezfooli, Seyed-Mohsen and Fawzi, Alhussein and Frossard, Pascal},
  booktitle={CVPR},
  pages={2574--2582},
  year={2016}
}

@article{moosavi2017universal,
  title={Universal adversarial perturbations},
  author={Moosavi-Dezfooli, Seyed-Mohsen and Fawzi, Alhussein and Fawzi, Omar and Frossard, Pascal},
  journal={arXiv preprint},
  year={2017}
}

@misc{
zhang2020adversarial,
title={Adversarial Interpolation Training: A Simple Approach for Improving Model Robustness},
author={Haichao Zhang and Wei Xu},
year={2020},
url={https://openreview.net/forum?id=Syejj0NYvr}
}

@inproceedings{zhang2019defense,
  title={Defense against adversarial attacks using feature scattering-based adversarial training},
  author={Zhang, Haichao and Wang, Jianyu},
  booktitle={Neurips},
  pages={1829--1839},
  year={2019}
}

@misc{kim2020sensible,
title={Sensible adversarial learning},
author={Jungeum Kim and Xiao Wang},
year={2020},
url={https://openreview.net/forum?id=rJlf_RVKwr}
}


@article{papernot2016transferability,
  title={Transferability in machine learning: from phenomena to black-box attacks using adversarial samples},
  author={Papernot, Nicolas and McDaniel, Patrick and Goodfellow, Ian},
  journal={arXiv preprint arXiv:1605.07277},
  year={2016}
}

@inproceedings{carlini2017towards,
  title={Towards evaluating the robustness of neural networks},
  author={Carlini, Nicholas and Wagner, David},
  booktitle={SP},
  pages={39--57},
  year={2017},
  organization={IEEE}
}

@article{chen2017ead,
  title={EAD: elastic-net attacks to deep neural networks via adversarial examples},
  author={Chen, Pin-Yu and Sharma, Yash and Zhang, Huan and Yi, Jinfeng and Hsieh, Cho-Jui},
  journal={arXiv preprint arXiv:1709.04114},
  year={2017}
}

@inproceedings{chen2017zoo,
  title={Zoo: Zeroth order optimization based black-box attacks to deep neural networks without training substitute models},
  author={Chen, Pin-Yu and Zhang, Huan and Sharma, Yash and Yi, Jinfeng and Hsieh, Cho-Jui},
  booktitle={AISec},
  pages={15--26},
  year={2017},
  organization={ACM}
}

@inproceedings{ilyas2018black,
  title={Black-box Adversarial Attacks with Limited Queries and Information},
  author={Ilyas, Andrew and Engstrom, Logan and Athalye, Anish and Lin, Jessy and Athalye, Anish and Engstrom, Logan and Ilyas, Andrew and Kwok, Kevin},
  booktitle={ICML},
  year={2018}
}


@article{kingma2014adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik P and Ba, Jimmy},
  journal={ICLR},
  year={2015}
}

@article{sinha2018certifying,
  title={Certifying some distributional robustness with principled adversarial training},
  author={Sinha, Aman and Namkoong, Hongseok and Duchi, John},
  journal={ICLR},
  year={2018}
}

@article{madry2017towards,
  title={Towards deep learning models resistant to adversarial attacks},
  author={Madry, Aleksander and Makelov, Aleksandar and Schmidt, Ludwig and Tsipras, Dimitris and Vladu, Adrian},
  journal={ICML},
  year={2018}
}

@article{wierstra2014natural,
  title={Natural evolution strategies},
  author={Wierstra, Daan and Schaul, Tom and Glasmachers, Tobias and Sun, Yi and Peters, Jan and Schmidhuber, J{\"u}rgen},
  journal={The Journal of Machine Learning Research},
  volume={15},
  number={1},
  pages={949--980},
  year={2014},
  publisher={JMLR. org}
}

@inproceedings{szegedy2016rethinking,
  title={Rethinking the inception architecture for computer vision},
  author={Szegedy, Christian and Vanhoucke, Vincent and Ioffe, Sergey and Shlens, Jon and Wojna, Zbigniew},
  booktitle={CVPR},
  pages={2818--2826},
  year={2016}
}

@article{salimans2017evolution,
  title={Evolution strategies as a scalable alternative to reinforcement learning},
  author={Salimans, Tim and Ho, Jonathan and Chen, Xi and Sidor, Szymon and Sutskever, Ilya},
  journal={arXiv preprint arXiv:1703.03864},
  year={2017}
}


@article{lacoste2016convergence,
  title={Convergence rate of Frank-Wolfe for non-convex objectives},
  author={Lacoste-Julien, Simon},
  journal={arXiv preprint arXiv:1607.00345},
  year={2016}
}

@inproceedings{reddi2016stochastic,
  title={Stochastic Frank-Wolfe methods for nonconvex optimization},
  author={Reddi, Sashank J and Sra, Suvrit and P{\'o}czos, Barnab{\'a}s and Smola, Alex},
  booktitle={Allerton},
  pages={1244--1251},
  year={2016},
  organization={IEEE}
}

@article{shamir2017optimal,
  title={An optimal algorithm for bandit and zero-order convex optimization with two-point feedback},
  author={Shamir, Ohad},
  journal={Journal of Machine Learning Research},
  volume={18},
  number={52},
  pages={1--11},
  year={2017},
  publisher={Microtome Publishing}
}

@inproceedings{flaxman2005online,
  title={Online convex optimization in the bandit setting: gradient descent without a gradient},
  author={Flaxman, Abraham D and Kalai, Adam Tauman and McMahan, H Brendan},
  booktitle={SODA},
  pages={385--394},
  year={2005},
  organization={Society for Industrial and Applied Mathematics}
}

@article{santurkar2018does,
  title={How Does Batch Normalization Help Optimization?(No, It Is Not About Internal Covariate Shift)},
  author={Santurkar, Shibani and Tsipras, Dimitris and Ilyas, Andrew and Madry, Aleksander},
  journal={arXiv preprint arXiv:1805.11604},
  year={2018}
}

@inproceedings{deng2009imagenet,
  title={Imagenet: A large-scale hierarchical image database},
  author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Li, Kai and Fei-Fei, Li},
  booktitle={CVPR},
  pages={248--255},
  year={2009},
  organization={Ieee}
}


 
@article{staibdistributionally,
  title={Distributionally Robust Deep Learning as a Generalization of Adversarial Training},
  author={Staib, Matthew and Jegelka, Stefanie},
  journal={Machine Learning and Computer Security Workshop},
  year={2017}
}


@article{yu2017generalized,
  title={Generalized conditional gradient for sparse estimation},
  author={Yu, Yaoliang and Zhang, Xinhua and Schuurmans, Dale},
  journal={The Journal of Machine Learning Research},
  volume={18},
  number={1},
  pages={5279--5324},
  year={2017},
  publisher={JMLR. org}
}


@article{hu2017generating,
  title={Generating adversarial malware examples for black-box attacks based on GAN},
  author={Hu, Weiwei and Tan, Ying},
  journal={arXiv preprint arXiv:1702.05983},
  year={2017}
}



@inproceedings{papernot2017practical,
  title={Practical black-box attacks against machine learning},
  author={Papernot, Nicolas and McDaniel, Patrick and Goodfellow, Ian and Jha, Somesh and Celik, Z Berkay and Swami, Ananthram},
  booktitle={AsiaCCS},
  pages={506--519},
  year={2017},
  organization={ACM}
}

@article{frank1956algorithm,
  title={An algorithm for quadratic programming},
  author={Frank, Marguerite and Wolfe, Philip},
  journal={Naval research logistics quarterly},
  volume={3},
  number={1-2},
  pages={95--110},
  year={1956},
  publisher={Wiley Online Library}
}


@inproceedings{jaggi2013revisiting,
  title={Revisiting Frank-Wolfe: Projection-Free Sparse Convex Optimization.},
  author={Jaggi, Martin},
  booktitle={ICML},
  pages={427--435},
  year={2013}
}

@article{chatterji2020finite,
  title={Finite-sample analysis of interpolating linear classifiers in the overparameterized regime},
  author={Chatterji, Niladri S and Long, Philip M},
  journal={arXiv preprint arXiv:2004.12019},
  year={2020}
}


@inproceedings{
Li2020Implicit,
title={Implicit Bias of Gradient Descent based Adversarial Training on Separable Data},
author={Yan Li and Ethan X.Fang and Huan Xu and Tuo Zhao},
booktitle={International Conference on Learning Representations},
year={2020},
}

@book{lindvall2002lectures,
  title={Lectures on the coupling method},
  author={Lindvall, Torgny},
  year={2002},
  publisher={Courier Corporation}
}

@article{vershynin2010introduction,
  title={Introduction to the non-asymptotic analysis of random matrices},
  author={Vershynin, Roman},
  journal={arXiv preprint arXiv:1011.3027},
  year={2010}
}

@article{soudry2018implicit,
  title={The implicit bias of gradient descent on separable data},
  author={Soudry, Daniel and Hoffer, Elad and Nacson, Mor Shpigel and Gunasekar, Suriya and Srebro, Nathan},
  journal={The Journal of Machine Learning Research},
  volume={19},
  number={1},
  pages={2822--2878},
  year={2018},
  publisher={JMLR. org}
}

@inproceedings{nacson2019stochastic,
  title={Stochastic gradient descent on separable data: Exact convergence with a fixed learning rate},
  author={Nacson, Mor Shpigel and Srebro, Nathan and Soudry, Daniel},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={3051--3059},
  year={2019},
  organization={PMLR}
}

@inproceedings{ji2019implicit,
  title={The implicit bias of gradient descent on nonseparable data},
  author={Ji, Ziwei and Telgarsky, Matus},
  booktitle={Conference on Learning Theory},
  pages={1772--1798},
  year={2019},
  organization={PMLR}
}

@inproceedings{gunasekar2018characterizing,
  title={Characterizing implicit bias in terms of optimization geometry},
  author={Gunasekar, Suriya and Lee, Jason and Soudry, Daniel and Srebro, Nathan},
  booktitle={International Conference on Machine Learning},
  pages={1832--1841},
  year={2018},
  organization={PMLR}
}

@inproceedings{gunasekar2018implicit,
  title={Implicit regularization in matrix factorization},
  author={Gunasekar, Suriya and Woodworth, Blake and Bhojanapalli, Srinadh and Neyshabur, Behnam and Srebro, Nathan},
  booktitle={2018 Information Theory and Applications Workshop (ITA)},
  pages={1--10},
  year={2018},
  organization={IEEE}
}

@article{arora2019implicit,
  title={Implicit regularization in deep matrix factorization},
  author={Arora, Sanjeev and Cohen, Nadav and Hu, Wei and Luo, Yuping},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@inproceedings{
Lyu2020Gradient,
title={Gradient Descent Maximizes the Margin of Homogeneous Neural Networks},
author={Kaifeng Lyu and Jian Li},
booktitle={International Conference on Learning Representations},
year={2020},
}

@article{neyshabur2017implicit,
  title={Implicit regularization in deep learning},
  author={Neyshabur, Behnam},
  journal={arXiv preprint arXiv:1709.01953},
  year={2017}
}

@inproceedings{
sanyal2021how,
title={How Benign is Benign Overfitting ?},
author={Amartya Sanyal and Puneet K. Dokania and Varun Kanade and Philip Torr},
booktitle={International Conference on Learning Representations},
year={2021},
}

@inproceedings{glorot2010understanding,
  title={Understanding the difficulty of training deep feedforward neural networks},
  author={Glorot, Xavier and Bengio, Yoshua},
  booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
  pages={249--256},
  year={2010},
  organization={JMLR Workshop and Conference Proceedings}
}

@article{bartlett2020benign,
  title={Benign overfitting in linear regression},
  author={Bartlett, Peter L and Long, Philip M and Lugosi, G{\'a}bor and Tsigler, Alexander},
  journal={Proceedings of the National Academy of Sciences},
  volume={117},
  number={48},
  pages={30063--30070},
  year={2020},
  publisher={National Acad Sciences}
}

@article{gao2019convergence,
  title={Convergence of adversarial training in overparametrized neural networks},
  author={Gao, Ruiqi and Cai, Tianle and Li, Haochuan and Hsieh, Cho-Jui and Wang, Liwei and Lee, Jason D},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  pages={13029--13040},
  year={2019}
}

@article{zhang2020over,
  title={Over-parameterized adversarial training: An analysis overcoming the curse of dimensionality},
  author={Zhang, Yi and Plevrakis, Orestis and Du, Simon S and Li, Xingguo and Song, Zhao and Arora, Sanjeev},
  journal={Advances in Neural Information Processing Systems},
  year={2020}
}

@inproceedings{wang2020improving,
  title={Improving adversarial robustness requires revisiting misclassified examples},
  author={Wang, Yisen and Zou, Difan and Yi, Jinfeng and Bailey, James and Ma, Xingjun and Gu, Quanquan},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@article{charles2019convergence,
  title={Convergence and margin of adversarial training on separable data},
  author={Charles, Zachary and Rajput, Shashank and Wright, Stephen and Papailiopoulos, Dimitris},
  journal={arXiv preprint arXiv:1905.09209},
  year={2019}
}

@inproceedings{chen2020more,
  title={More data can expand the generalization gap between adversarially robust and standard models},
  author={Chen, Lin and Min, Yifei and Zhang, Mingrui and Karbasi, Amin},
  booktitle={International Conference on Machine Learning},
  pages={1670--1680},
  year={2020},
  organization={PMLR}
}

@article{schmidt2018adversarially,
  title={Adversarially robust generalization requires more data},
  author={Schmidt, Ludwig and Santurkar, Shibani and Tsipras, Dimitris and Talwar, Kunal and M{\k{a}}dry, Aleksander},
  journal={Advances in Neural Information Processing Systems},
  year={2018}
}

@inproceedings{montasser2020efficiently,
  title={Efficiently learning adversarially robust halfspaces with noise},
  author={Montasser, Omar and Goel, Surbhi and Diakonikolas, Ilias and Srebro, Nathan},
  booktitle={International Conference on Machine Learning},
  pages={7010--7021},
  year={2020},
  organization={PMLR}
}

@article{diakonikolas2020complexity,
  title={The complexity of adversarially robust proper learning of halfspaces with agnostic noise},
  author={Diakonikolas, Ilias and Kane, Daniel M and Manurangsi, Pasin},
  journal={Advances in Neural Information Processing Systems},
  year={2020}
}

@article{zou2021provable,
  title={Provable Robustness of Adversarial Training for Learning Halfspaces with Noise},
  author={Zou, Difan and Frei, Spencer and Gu, Quanquan},
  journal={International Conference on Machine Learning},
  year={2021}
}

@article{wu2020wider,
  title={Do Wider Neural Networks Really Help Adversarial Robustness?},
  author={Wu, Boxi and Chen, Jinghui and Cai, Deng and He, Xiaofei and Gu, Quanquan},
  journal={arXiv preprint arXiv:2010.01279},
  year={2020}
}


@article{raghunathan2020understanding,
  title={Understanding and mitigating the tradeoff between robustness and accuracy},
  author={Raghunathan, Aditi and Xie, Sang Michael and Yang, Fanny and Duchi, John and Liang, Percy},
  journal={arXiv preprint arXiv:2002.10716},
  year={2020}
}

@article{gowal2020uncovering,
  title={Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples},
  author={Gowal, Sven and Qin, Chongli and Uesato, Jonathan and Mann, Timothy and Kohli, Pushmeet},
  journal={arXiv preprint arXiv:2010.03593},
  year={2020}
}

@article{wu2020adversarial,
  title={Adversarial Weight Perturbation Helps Robust Generalization},
  author={Wu, Dongxian and Xia, Shu-Tao and Wang, Yisen},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}

@article{yang2020closer,
  title={A closer look at accuracy vs. robustness},
  author={Yang, Yao-Yuan and Rashtchian, Cyrus and Zhang, Hongyang and Salakhutdinov, Russ R and Chaudhuri, Kamalika},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}


@inproceedings{zhang2020understanding,
  title={Understanding the intrinsic robustness of image distributions using conditional generative models},
  author={Zhang, Xiao and Chen, Jinghui and Gu, Quanquan and Evans, David},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3883--3893},
  year={2020},
  organization={PMLR}
}

@inproceedings{fawzi2018adversarial,
  title={Adversarial vulnerability for any classifier},
  author={Fawzi, Alhussein and Fawzi, Hamza and Fawzi, Omar},
  booktitle={Proceedings of the 32nd International Conference on Neural Information Processing Systems},
  pages={1186--1195},
  year={2018}
}

@article{dobriban2020provable,
  title={Provable tradeoffs in adversarially robust classification},
  author={Dobriban, Edgar and Hassani, Hamed and Hong, David and Robey, Alexander},
  journal={arXiv preprint arXiv:2006.05161},
  year={2020}
}


@inproceedings{
tsipras2018robustness,
title={Robustness May Be at Odds with Accuracy},
author={Dimitris Tsipras and Shibani Santurkar and Logan Engstrom and Alexander Turner and Aleksander Madry},
booktitle={International Conference on Learning Representations},
year={2019},
}

@article{allen2020feature,
  title={Feature purification: How adversarial training performs robust deep learning},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi},
  journal={arXiv preprint arXiv:2005.10190},
  year={2020}
}

@inproceedings{yin2019rademacher,
  title={Rademacher complexity for adversarially robust generalization},
  author={Yin, Dong and Kannan, Ramchandran and Bartlett, Peter},
  booktitle={International Conference on Machine Learning},
  pages={7085--7094},
  year={2019},
  organization={PMLR}
}

@article{cullina2018pac,
  title={PAC-learning in the presence of evasion adversaries},
  author={Cullina, Daniel and Bhagoji, Arjun Nitin and Mittal, Prateek},
  journal={Proceedings of the 32nd International Conference on Neural Information Processing Systems},
  year={2018}
}

@inproceedings{
ji2018gradient,
title={Gradient descent aligns the layers of deep linear networks},
author={Ziwei Ji and Matus Telgarsky},
booktitle={International Conference on Learning Representations},
year={2019},
}

 
 
 
 
@article{tsigler2020benign,
  title={Benign overfitting in ridge regression},
  author={Tsigler, Alexander and Bartlett, Peter L},
  journal={arXiv preprint arXiv:2009.14286},
  year={2020}
}



@article{cao2021risk,
  title={Risk Bounds for Over-parameterized Maximum Margin Classification on Sub-Gaussian Mixtures},
  author={Cao, Yuan and Gu, Quanquan and Belkin, Mikhail},
  journal={arXiv preprint arXiv:2104.13628},
  year={2021}
}


@inproceedings{wang2021benign,
  title={Benign overfitting in binary classification of gaussian mixtures},
  author={Wang, Ke and Thrampoulidis, Christos},
  booktitle={ICASSP 2021-2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
  pages={4030--4034},
  year={2021},
  organization={IEEE}
}




@article{belkin2019reconciling,
  title={Reconciling modern machine-learning practice and the classical bias--variance trade-off},
  author={Belkin, Mikhail and Hsu, Daniel and Ma, Siyuan and Mandal, Soumik},
  journal={Proceedings of the National Academy of Sciences},
  volume={116},
  number={32},
  pages={15849--15854},
  year={2019},
  publisher={National Acad Sciences}
}


@article{belkin2019two,
  title={Two models of double descent for weak features},
  author={Belkin, Mikhail and Hsu, Daniel and Xu, Ji},
  journal={arXiv preprint arXiv:1903.07571},
  year={2019}
}



@article{hastie2019surprises,
  title={Surprises in high-dimensional ridgeless least squares interpolation},
  author={Hastie, Trevor and Montanari, Andrea and Rosset, Saharon and Tibshirani, Ryan J},
  journal={arXiv preprint arXiv:1903.08560},
  year={2019}
}


@article{mei2019generalization,
  title={The generalization error of random features regression: Precise asymptotics and double descent curve},
  author={Mei, Song and Montanari, Andrea},
  journal={arXiv preprint arXiv:1908.05355},
  year={2019}
}





@article{wu2020optimal,
  title={On the Optimal Weighted $\ell_2$ Regularization in Overparameterized Linear Regression},
  author={Wu, Denny and Xu, Ji},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}



@inproceedings{belkin2018understand,
  title={To understand deep learning we need to understand kernel learning},
  author={Belkin, Mikhail and Ma, Siyuan and Mandal, Soumik},
  booktitle={International Conference on Machine Learning},
  pages={541--549},
  year={2018},
  organization={PMLR}
}

@article{dong2021exploring,
  title={Exploring Memorization in Adversarial Training},
  author={Dong, Yinpeng and Xu, Ke and Yang, Xiao and Pang, Tianyu and Deng, Zhijie and Su, Hang and Zhu, Jun},
  journal={arXiv preprint arXiv:2106.01606},
  year={2021}
}

@article{liu2021impact,
  title={On the Impact of Hard Adversarial Instances on Overfitting in Adversarial Training},
  author={Liu, Chen and Huang, Zhichao and Salzmann, Mathieu and Zhang, Tong and S{\"u}sstrunk, Sabine},
  journal={arXiv preprint arXiv:2112.07324},
  year={2021}
}



@inproceedings{zhang2016understanding,
  title={Understanding deep learning requires rethinking generalization},
  author={Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
  booktitle={International Conference on Learning Representations},
  year={2017}
}

 