@article{magnani2009convex,
  title={Convex piecewise-linear fitting},
  author={Magnani, Alessandro and Boyd, Stephen P},
  journal={Optimization and Engineering},
  volume={10},
  pages={1--17},
  year={2009},
  publisher={Springer}
}

@article{ovchinnikov2002max,
  title={Max-Min Representation of Piecewise Linear Functions},
  author={Ovchinnikov, Sergei},
  journal={Contributions to Algebra and Geometry},
  volume={43},
  number={1},
  pages={297--302},
  year={2002}
}

@inproceedings{humayun2023splinecam,
  title={SplineCam: Exact Visualization and Characterization of Deep Network Geometry and Decision Boundaries},
  author={Humayun, Ahmed Imtiaz and Balestriero, Randall and Balakrishnan, Guha and Baraniuk, Richard G},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={3789--3798},
  year={2023}
}

@article{maclagan2009introduction,
  title={Introduction to tropical geometry},
  author={Maclagan, Diane and Sturmfels, Bernd},
  journal={Graduate Studies in Mathematics},
  volume={161},
  year={2009}
}

@inproceedings{zhang2018tropical,
  title={Tropical geometry of deep neural networks},
  author={Zhang, Liwen and Naitzat, Gregory and Lim, Lek-Heng},
  booktitle={International Conference on Machine Learning},
  pages={5824--5832},
  year={2018},
  organization={PMLR}
}

@inproceedings{trimmel2020tropex,
  title={TropEx: An Algorithm for Extracting Linear Terms in Deep Neural Networks},
  author={Trimmel, Martin and Petzka, Henning and Sminchisescu, Cristian},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@article{alfarra2020decision,
  title={On the decision boundaries of neural networks: A tropical geometry perspective},
  author={Alfarra, Motasem and Bibi, Adel and Hammoud, Hasan and Gaafar, Mohamed and Ghanem, Bernard},
  journal={arXiv preprint arXiv:2002.08838},
  year={2020}
}

@article{maragos2021tropical,
  title={Tropical geometry and machine learning},
  author={Maragos, Petros and Charisopoulos, Vasileios and Theodosis, Emmanouil},
  journal={Proceedings of the IEEE},
  volume={109},
  number={5},
  pages={728--755},
  year={2021},
  publisher={IEEE}
}

@article{montufar2021sharp,
  title={Sharp bounds for the number of regions of maxout networks and vertices of Minkowski sums},
  author={Mont{\'u}far, Guido and Ren, Yue and Zhang, Leon},
  journal={arXiv preprint arXiv:2104.08135},
  year={2021}
}

@article{charisopoulos2018tropical,
  title={A tropical approach to neural networks with piecewise linear activations},
  author={Charisopoulos, Vasileios and Maragos, Petros},
  journal={arXiv preprint arXiv:1805.08749},
  year={2018}
}

@article{hertrich2021towards,
  title={Towards Lower Bounds on the Depth of ReLU Neural Networks},
  author={Hertrich, Christoph and Basu, Amitabh and Di Summa, Marco and Skutella, Martin},
  journal={arXiv preprint arXiv:2105.14835},
  year={2021}
}


@inproceedings{langley00,
 author    = {P. Langley},
 title     = {Crafting Papers on Machine Learning},
 year      = {2000},
 pages     = {1207--1216},
 editor    = {Pat Langley},
 booktitle     = {Proceedings of the 17th International Conference
              on Machine Learning (ICML 2000)},
 address   = {Stanford, CA},
 publisher = {Morgan Kaufmann}
}

@TechReport{mitchell80,
  author = 	 "T. M. Mitchell",
  title = 	 "The Need for Biases in Learning Generalizations",
  institution =  "Computer Science Department, Rutgers University",
  year = 	 "1980",
  address =	 "New Brunswick, MA",
}

@phdthesis{kearns89,
  author = {M. J. Kearns},
  title =  {Computational Complexity of Machine Learning},
  school = {Department of Computer Science, Harvard University},
  year =   {1989}
}

@Book{MachineLearningI,
  editor = 	 "R. S. Michalski and J. G. Carbonell and T.
		  M. Mitchell",
  title = 	 "Machine Learning: An Artificial Intelligence
		  Approach, Vol. I",
  publisher = 	 "Tioga",
  year = 	 "1983",
  address =	 "Palo Alto, CA"
}

@Book{DudaHart2nd,
  author =       "R. O. Duda and P. E. Hart and D. G. Stork",
  title =        "Pattern Classification",
  publisher =    "John Wiley and Sons",
  edition =      "2nd",
  year =         "2000"
}

@misc{anonymous,
  title= {Suppressed for Anonymity},
  author= {Author, N. N.},
  year= {2021}
}

@InCollection{Newell81,
  author =       "A. Newell and P. S. Rosenbloom",
  title =        "Mechanisms of Skill Acquisition and the Law of
                  Practice", 
  booktitle =    "Cognitive Skills and Their Acquisition",
  pages =        "1--51",
  publisher =    "Lawrence Erlbaum Associates, Inc.",
  year =         "1981",
  editor =       "J. R. Anderson",
  chapter =      "1",
  address =      "Hillsdale, NJ"
}


@Article{Samuel59,
  author = 	 "A. L. Samuel",
  title = 	 "Some Studies in Machine Learning Using the Game of
		  Checkers",
  journal =	 "IBM Journal of Research and Development",
  year =	 "1959",
  volume =	 "3",
  number =	 "3",
  pages =	 "211--229"
}


@incollection{Bengio+chapter2007,
author = {Bengio, Yoshua and LeCun, Yann},
booktitle = {Large Scale Kernel Machines},
publisher = {MIT Press},
title = {Scaling Learning Algorithms Towards {AI}},
year = {2007}
}

@article{Hinton06,
author = {Hinton, Geoffrey E. and Osindero, Simon and Teh, Yee Whye},
journal = {Neural Computation},
pages = {1527--1554},
title = {A Fast Learning Algorithm for Deep Belief Nets},
volume = {18},
year = {2006}
}

@book{goodfellow2016deep,
title={Deep learning},
author={Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron and Bengio, Yoshua},
volume={1},
year={2016},
publisher={MIT Press}
}

@article{zhang2004statistical,
  title={Statistical behavior and consistency of classification methods based on convex risk minimization},
  author={Zhang, Tong},
  journal={The Annals of Statistics},
  pages={56--85},
  year={2004},
  publisher={JSTOR}
}

@article{steinwart2007fast,
  title={Fast rates for support vector machines using {G}aussian kernels},
  author={Steinwart, Ingo and Scovel, Clint and others},
  journal={The Annals of Statistics},
  volume={35},
  number={2},
  pages={575--607},
  year={2007},
  publisher={Institute of Mathematical Statistics}
}

@article{bauer2019deep,
  title={On deep learning as a remedy for the curse of dimensionality in nonparametric regression},
  author={Bauer, Benedikt and Kohler, Michael and others},
  journal={The Annals of Statistics},
  volume={47},
  number={4},
  pages={2261--2285},
  year={2019},
  publisher={Institute of Mathematical Statistics}
}

@article{bietti2019inductive,
  title={On the inductive bias of neural tangent kernels},
  author={Bietti, Alberto and Mairal, Julien},
  journal={arXiv preprint arXiv:1905.12173},
  year={2019}
}

@book{demengel2012functional,
  title={Functional Spaces for the Theory of Elliptic Partial Differential Equations},
  author={Demengel, Fran{\c{c}}oise and Demengel, Gilbert and Ern{\'e}, Reinie},
  year={2012},
  publisher={Springer}
}

@inproceedings{jacot2018neural,
  title={Neural tangent kernel: Convergence and generalization in neural networks},
  author={Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8571--8580},
  year={2018}
}

@inproceedings{hu2021regularization,
  title={Regularization Matters: A Nonparametric Perspective on Overparametrized Neural Network},
  author={Hu, Tianyang and Wang, Wenjia and Lin, Cong and Cheng, Guang},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={829--837},
  year={2021},
  organization={PMLR}
}

@article{arora2019fine,
  title={Fine-grained analysis of optimization and generalization for overparameterized two-layer neural networks},
  author={Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and Wang, Ruosong},
  journal={arXiv preprint arXiv:1901.08584},
  year={2019}
}

@inproceedings{li2018learning,
  title={Learning overparameterized neural networks via stochastic gradient descent on structured data},
  author={Li, Yuanzhi and Liang, Yingyu},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8157--8166},
  year={2018}
}

@article{du2018gradient,
  title={Gradient descent provably optimizes over-parameterized neural networks},
  author={Du, Simon S and Zhai, Xiyu and Poczos, Barnabas and Singh, Aarti},
  journal={arXiv preprint arXiv:1810.02054},
  year={2018}
}

@inproceedings{hu2020simple,
  title={Simple and effective regularization methods for training on noisily labeled data with generalization guarantee},
  author={Hu, Wei and Li, Zhiyuan and Yu, Dingli},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@article{BrezisMironescu19,
	author = {Ha{\"\i}m Brezis and Petru Mironescu},
	journal = {Journal of Functional Analysis},
	number = {8},
	pages = {2839-2864},
	title = {Where {Sobolev} interacts with {Gagliardo--Nirenberg}},
	volume = {277},
	year = {2019}}

	
@article{papyan2020prevalence,
  title={Prevalence of neural collapse during the terminal phase of deep learning training},
  author={Papyan, Vardan and Han, XY and Donoho, David L},
  journal={Proceedings of the National Academy of Sciences},
  volume={117},
  number={40},
  pages={24652--24663},
  year={2020},
  publisher={National Acad Sciences}
}

@inproceedings{demirkaya2020exploring,
  title={Exploring the role of loss functions in multiclass classification},
  author={Demirkaya, Ahmet and Chen, Jiasi and Oymak, Samet},
  booktitle={2020 54th Annual Conference on Information Sciences and Systems (CISS)},
  pages={1--5},
  year={2020},
  organization={IEEE}
}

@article{hui2020evaluation,
  title={Evaluation of neural architectures trained with square loss vs cross-entropy in classification tasks},
  author={Hui, Like and Belkin, Mikhail},
  journal={arXiv preprint arXiv:2006.07322},
  year={2020}
}

@article{pang2019rethinking,
  title={Rethinking softmax cross-entropy loss for adversarial robustness},
  author={Pang, Tianyu and Xu, Kun and Dong, Yinpeng and Du, Chao and Chen, Ning and Zhu, Jun},
  journal={arXiv preprint arXiv:1905.10626},
  year={2019}
}

@article{bai2021don,
  title={Don't Just Blame Over-parametrization for Over-confidence: Theoretical Analysis of Calibration in Binary Classification},
  author={Bai, Yu and Mei, Song and Wang, Huan and Xiong, Caiming},
  journal={arXiv preprint arXiv:2102.07856},
  year={2021}
}

@article{khosla2020supervised,
  title={Supervised contrastive learning},
  author={Khosla, Prannay and Teterwak, Piotr and Wang, Chen and Sarna, Aaron and Tian, Yonglong and Isola, Phillip and Maschinot, Aaron and Liu, Ce and Krishnan, Dilip},
  journal={arXiv preprint arXiv:2004.11362},
  year={2020}
}

@article{yu2020learning,
  title={Learning diverse and discriminative representations via the principle of maximal coding rate reduction},
  author={Yu, Yaodong and Chan, Kwan Ho Ryan and You, Chong and Song, Chaobing and Ma, Yi},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}

@inproceedings{feng2020can,
  title={Can cross entropy loss be robust to label noise?},
  author={Feng, Lei and Shu, Senlin and Lin, Zhuoyi and Lv, Fengmao and Li, Li and An, Bo},
  booktitle={IJCAI},
  pages={2206--2212},
  year={2020}
}

@inproceedings{vaicenavicius2019evaluating,
  title={Evaluating model calibration in classification},
  author={Vaicenavicius, Juozas and Widmann, David and Andersson, Carl and Lindsten, Fredrik and Roll, Jacob and Sch{\"o}n, Thomas},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={3459--3467},
  year={2019},
  organization={PMLR}
}

@inproceedings{guo2017calibration,
  title={On calibration of modern neural networks},
  author={Guo, Chuan and Pleiss, Geoff and Sun, Yu and Weinberger, Kilian Q},
  booktitle={International Conference on Machine Learning},
  pages={1321--1330},
  year={2017},
  organization={PMLR}
}

@article{muthukumar2020classification,
  title={Classification vs regression in overparameterized regimes: Does the loss function matter?},
  author={Muthukumar, Vidya and Narang, Adhyyan and Subramanian, Vignesh and Belkin, Mikhail and Hsu, Daniel and Sahai, Anant},
  journal={arXiv preprint arXiv:2005.08054},
  year={2020}
}

@article{zhang2017mixup,
  title={mixup: Beyond empirical risk minimization},
  author={Zhang, Hongyi and Cisse, Moustapha and Dauphin, Yann N and Lopez-Paz, David},
  journal={arXiv preprint arXiv:1710.09412},
  year={2017}
}

@article{hinton2015distilling,
  title={Distilling the knowledge in a neural network},
  author={Hinton, Geoffrey and Vinyals, Oriol and Dean, Jeff},
  journal={arXiv preprint arXiv:1503.02531},
  year={2015}
}

@article{menon2020distillation,
  title={A statistical perspective on distillation},
  author={Menon, Aditya Krishna and Rawat, Ankit Singh and Reddi, Sashank J and Kim, Seungyeon and Kumar, Sanjiv},
  journal={arXiv preprint arXiv:2005.10419},
  year={2020}
}

@inproceedings{menon2021statistical,
  title={A statistical perspective on distillation},
  author={Menon, Aditya K and Rawat, Ankit Singh and Reddi, Sashank and Kim, Seungyeon and Kumar, Sanjiv},
  booktitle={International Conference on Machine Learning},
  pages={7632--7642},
  year={2021},
  organization={PMLR}
}

@article{arora2019theoretical,
  title={A theoretical analysis of contrastive unsupervised representation learning},
  author={Arora, Sanjeev and Khandeparkar, Hrishikesh and Khodak, Mikhail and Plevrakis, Orestis and Saunshi, Nikunj},
  journal={arXiv preprint arXiv:1902.09229},
  year={2019}
}

@article{bartlett2006convexity,
  title={Convexity, classification, and risk bounds},
  author={Bartlett, Peter L and Jordan, Michael I and McAuliffe, Jon D},
  journal={Journal of the American Statistical Association},
  volume={101},
  number={473},
  pages={138--156},
  year={2006},
  publisher={Taylor \& Francis}
}

@article{mroueh2012multiclass,
  title={Multiclass learning with simplex coding},
  author={Mroueh, Youssef and Poggio, Tomaso and Rosasco, Lorenzo and Slotine, Jean-Jacques},
  journal={arXiv preprint arXiv:1209.1360},
  year={2012}
}

@inproceedings{an2007face,
  title={Face recognition using kernel ridge regression},
  author={An, Senjian and Liu, Wanquan and Venkatesh, Svetha},
  booktitle={2007 IEEE Conference on Computer Vision and Pattern Recognition},
  pages={1--7},
  year={2007},
  organization={IEEE}
}

@inproceedings{yang2008stagewise,
  title={A stagewise least square loss function for classification},
  author={Yang, Shuang-Hong and Hu, Bao-Gang},
  booktitle={Proceedings of the 2008 SIAM International Conference on Data Mining},
  pages={120--131},
  year={2008},
  organization={SIAM}
}

@book{edmunds2008function,
  title={Function Spaces, Entropy Numbers, Differential Operators},
  author={Edmunds, David Eric and Triebel, Hans},
  volume={120},
  year={2008},
  publisher={Cambridge University Press}
}

@book{bennett1988interpolation,
  title={Interpolation of Operators},
  author={Bennett, Colin and Sharpley, Robert C},
  year={1988},
  publisher={Academic press}
}

@article{geifman2020similarity,
  title={On the similarity between the {L}aplace and neural tangent kernels},
  author={Geifman, Amnon and Yadav, Abhay and Kasten, Yoni and Galun, Meirav and Jacobs, David and Basri, Ronen},
  journal={arXiv preprint arXiv:2007.01580},
  year={2020}
}

@article{chen2020deep,
  title={Deep neural tangent kernel and {L}aplace kernel have the same {RKHS}},
  author={Chen, Lin and Xu, Sheng},
  journal={arXiv preprint arXiv:2009.10683},
  year={2020}
}

@book{wendland2004scattered,
  title={Scattered Data Approximation},
  author={Wendland, Holger},
  year={2004},
  publisher={Cambridge University Press}
}

@article{elsayed2018large,
  title={Large margin deep networks for classification},
  author={Elsayed, Gamaleldin F and Krishnan, Dilip and Mobahi, Hossein and Regan, Kevin and Bengio, Samy},
  journal={arXiv preprint arXiv:1803.05598},
  year={2018}
}

@article{ding2018mma,
  title={Mma training: Direct input space margin maximization through adversarial training},
  author={Ding, Gavin Weiguang and Sharma, Yash and Lui, Kry Yik Chau and Huang, Ruitong},
  journal={arXiv preprint arXiv:1812.02637},
  year={2018}
}

@inproceedings{he2019parametric,
  title={Parametric noise injection: Trainable randomness to improve deep neural network robustness against adversarial attack},
  author={He, Zhezhi and Rakin, Adnan Siraj and Fan, Deliang},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={588--597},
  year={2019}
}

@article{zagoruyko2016wide,
  title={Wide residual networks},
  author={Zagoruyko, Sergey and Komodakis, Nikos},
  journal={arXiv preprint arXiv:1605.07146},
  year={2016}
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
  pages={770--778},
  year={2016}
}

@article{audibert2007fast,
  title={Fast learning rates for plug-in classifiers},
  author={Audibert, Jean-Yves and Tsybakov, Alexandre B},
  journal={The Annals of statistics},
  volume={35},
  number={2},
  pages={608--633},
  year={2007},
  publisher={Institute of Mathematical Statistics}
}

@book{hastie2009elements,
  title={The elements of statistical learning: data mining, inference, and prediction},
  author={Hastie, Trevor and Tibshirani, Robert and Friedman, Jerome},
  year={2009},
  publisher={Springer Science \& Business Media}
}

@article{kohler2007rate,
  title={On the rate of convergence of local averaging plug-in classification rules under a margin condition},
  author={Kohler, Michael and Krzyzak, Adam},
  journal={IEEE Transactions on Information Theory},
  volume={53},
  number={5},
  pages={1735--1742},
  year={2007},
  publisher={IEEE}
}

@article{madry2017towards,
  title={Towards deep learning models resistant to adversarial attacks},
  author={Madry, Aleksander and Makelov, Aleksandar and Schmidt, Ludwig and Tsipras, Dimitris and Vladu, Adrian},
  journal={arXiv preprint arXiv:1706.06083},
  year={2017}
}

@inproceedings{croce2020reliable,
  title={Reliable evaluation of adversarial robustness with an ensemble of diverse parameter-free attacks},
  author={Croce, Francesco and Hein, Matthias},
  booktitle={International Conference on Machine Learning},
  pages={2206--2216},
  year={2020},
  organization={PMLR}
}

@article{kim2018fast,
  title={Fast convergence rates of deep neural networks for classification},
  author={Kim, Yongdai and Ohn, Ilsang and Kim, Dongha},
  journal={arXiv preprint arXiv:1812.03599},
  year={2018}
}

@article{mammen1999smooth,
  title={Smooth discrimination analysis},
  author={Mammen, Enno and Tsybakov, Alexandre B},
  journal={The Annals of Statistics},
  volume={27},
  number={6},
  pages={1808--1829},
  year={1999},
  publisher={Institute of Mathematical Statistics}
}

@article{ji2019polylogarithmic,
  title={Polylogarithmic width suffices for gradient descent to achieve arbitrarily small test error with shallow {ReLU} networks},
  author={Ji, Ziwei and Telgarsky, Matus},
  journal={arXiv preprint arXiv:1909.12292},
  year={2019}
}

@article{lyu2019gradient,
  title={Gradient descent maximizes the margin of homogeneous neural networks},
  author={Lyu, Kaifeng and Li, Jian},
  journal={arXiv preprint arXiv:1906.05890},
  year={2019}
}

@inproceedings{cao2020generalization,
  title={Generalization error bounds of gradient descent for learning over-parameterized deep relu networks},
  author={Cao, Yuan and Gu, Quanquan},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={3349--3356},
  year={2020}
}

@article{han2021neural,
  title={Neural Collapse Under MSE Loss: Proximity to and Dynamics on the Central Path},
  author={Han, XY and Papyan, Vardan and Donoho, David L},
  journal={arXiv preprint arXiv:2106.02073},
  year={2021}
}

@article{kornblith2020demystifying,
  title={Demystifying Loss Functions for Classification},
  author={Kornblith, Simon and Lee, Honglak and Chen, Ting and Norouzi, Mohammad},
  year={2020}
}

@techreport{poggio2019generalization,
  title={Generalization in deep network classifiers trained with the square loss},
  author={Poggio, Tomaso and Liao, Qianli},
  year={2019},
  institution={CBMM Memo No}
}

@inproceedings{graf2021dissecting,
  title={Dissecting supervised constrastive learning},
  author={Graf, Florian and Hofer, Christoph and Niethammer, Marc and Kwitt, Roland},
  booktitle={International Conference on Machine Learning},
  pages={3821--3830},
  year={2021},
  organization={PMLR}
}

@article{fang2021exploring,
  title={Exploring deep neural networks via layer-peeled model: Minority collapse in imbalanced training},
  author={Fang, Cong and He, Hangfeng and Long, Qi and Su, Weijie J},
  journal={Proceedings of the National Academy of Sciences},
  volume={118},
  number={43},
  year={2021},
  publisher={National Acad Sciences}
}

@article{Aronszajn50,
	author = {N. Aronszajn},
	journal = {Trans. Amer. Math. Soc.},
	number = {3},
	pages = {337--404},
	title = {Theory of reproducing kernels},
	volume = {68},
	year = {1950}}
	
@book{BerlinetThomas-Agnan04,
    author = {Berlinet, Alain  and Thomas-Agnan, Christine},
    title = {Reproducing Kernel Hilbert Spaces in Probability and Statistics},
    year = {2004},
    publisher = {Springer}}
    
@article{mixon2020neural,
  title={Neural collapse with unconstrained features},
  author={Mixon, Dustin G and Parshall, Hans and Pi, Jianzong},
  journal={arXiv preprint arXiv:2011.11619},
  year={2020}
}

@article{nitanda2020optimal,
  title={Optimal rates for averaged stochastic gradient descent under neural tangent kernel regime},
  author={Nitanda, Atsushi and Suzuki, Taiji},
  journal={arXiv preprint arXiv:2006.12297},
  year={2020}
}

@article{nitanda2019gradient,
  title={Gradient descent can learn less over-parameterized two-layer neural networks on classification problems},
  author={Nitanda, Atsushi and Chinot, Geoffrey and Suzuki, Taiji},
  journal={arXiv preprint arXiv:1905.09870},
  year={2019}
}


@article{cao2019generalization,
  title={Generalization bounds of stochastic gradient descent for wide and deep neural networks},
  author={Cao, Yuan and Gu, Quanquan},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  pages={10836--10846},
  year={2019}
}

@article{schmidt2020nonparametric,
  title={Nonparametric regression using deep neural networks with ReLU activation function},
  author={Schmidt-Hieber, Johannes},
  journal={The Annals of Statistics},
  volume={48},
  number={4},
  pages={1875--1897},
  year={2020},
  publisher={Institute of Mathematical Statistics}
}

@article{farrell2021deep,
  title={Deep neural networks for estimation and inference},
  author={Farrell, Max H and Liang, Tengyuan and Misra, Sanjog},
  journal={Econometrica},
  volume={89},
  number={1},
  pages={181--213},
  year={2021},
  publisher={Wiley Online Library}
}

@article{kohler2020statistical,
  title={Statistical theory for image classification using deep convolutional neural networks with cross-entropy loss},
  author={Kohler, Michael and Langer, Sophie},
  journal={arXiv preprint arXiv:2011.13602},
  year={2020}
}

@article{muhammad2021mixacm,
  title={MixACM: Mixup-Based Robustness Transfer via Distillation of Activated Channel Maps},
  author={Muhammad, Awais and Zhou, Fengwei and Xie, Chuanlong and Li, Jiawei and Bae, Sung-Ho and Li, Zhenguo},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{srivastava2014dropout,
  title={Dropout: a simple way to prevent neural networks from overfitting},
  author={Srivastava, Nitish and Hinton, Geoffrey and Krizhevsky, Alex and Sutskever, Ilya and Salakhutdinov, Ruslan},
  journal={The journal of machine learning research},
  volume={15},
  number={1},
  pages={1929--1958},
  year={2014},
  publisher={JMLR. org}
}

@article{liang2018well,
  title={On How Well Generative Adversarial Networks Learn Densities: Nonparametric and Parametric Results},
  author={Liang, Tengyuan},
  journal={arXiv preprint arXiv:1811.03179},
  year={2018}
}

@incollection{NIPS2019_8921,
title = {Dynamics of stochastic gradient descent for two-layer neural networks in the teacher-student setup},
author = {Goldt, Sebastian and Advani, Madhu and Saxe, Andrew M and Krzakala, Florent and Zdeborov\'{a}, Lenka},
booktitle = {Advances in Neural Information Processing Systems 32},
editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d' Alch\'{e}-Buc and E. Fox and R. Garnett},
pages = {6979--6989},
year = {2019},
publisher = {Curran Associates, Inc.}
}


@incollection{NIPS2014_5484,
title = {Do Deep Nets Really Need to be Deep?},
author = {Ba, Jimmy and Caruana, Rich},
booktitle = {Advances in Neural Information Processing Systems 27},
editor = {Z. Ghahramani and M. Welling and C. Cortes and N. D. Lawrence and K. Q. Weinberger},
pages = {2654--2662},
year = {2014},
publisher = {Curran Associates, Inc.}
}


@incollection{NIPS2018_7584,
title = {The committee machine: Computational to statistical gaps in learning a two-layers neural network},
author = {Aubin, Benjamin and Maillard, Antoine and barbier, jean and Krzakala, Florent and Macris, Nicolas and Zdeborov\'{a}, Lenka},
booktitle = {Advances in Neural Information Processing Systems 31},
editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
pages = {3223--3234},
year = {2018},
publisher = {Curran Associates, Inc.}
}


@incollection{NIPS2019_8921,
title = {Dynamics of stochastic gradient descent for two-layer neural networks in the teacher-student setup},
author = {Goldt, Sebastian and Advani, Madhu and Saxe, Andrew M and Krzakala, Florent and Zdeborov\'{a}, Lenka},
booktitle = {Advances in Neural Information Processing Systems 32},
editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
pages = {6979--6989},
year = {2019},
publisher = {Curran Associates, Inc.}
}


@book{engelb01,
  title={Statistical Mechanics of Learning},
  author={Engel, A and Broeck, C.V.},
  year={2002},
  publisher={Cambridge University Press}
}


@article{DBLP:journals/sac/MaceC98,
  author    = {C. W. H. Mace and
               A. C. C. Coolen},
  title     = {Statistical mechanical analysis of the dynamics of learning in perceptrons},
  journal   = {Statistics and Computing},
  volume    = {8},
  number    = {1},
  pages     = {55--88},
  year      = {1998},
  url       = {https://doi.org/10.1023/A:1008896910704},
  doi       = {10.1023/A:1008896910704},
  timestamp = {Sun, 28 May 2017 13:20:58 +0200},
  biburl    = {https://dblp.org/rec/bib/journals/sac/MaceC98},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@article{bai19,
  title={Rate Optimal Variational Bayesian Inference for
Sparse DNN},
  author={Bai, Jincheng and Song, Qifan and Cheng, Guang},
  journal={arXiv preprint arXiv:1910.04355},
  year={2019}
}



@MISC{Saad96dynamicsof,
    author = {David Saad and Sara A. Solla},
    title = {Dynamics of On-Line Gradient Descent Learning for Multilayer Neural Networks},
    year = {1996}
}

@article{lyu2019gradient,
  title={Gradient Descent Maximizes the Margin of Homogeneous Neural Networks},
  author={Lyu, Kaifeng and Li, Jian},
  journal={arXiv preprint arXiv:1906.05890},
  year={2019}
}

@article{collobert2006large,
  title={Large scale transductive SVMs},
  author={Collobert, Ronan and Sinz, Fabian and Weston, Jason and Bottou, L{\'e}on},
  journal={Journal of Machine Learning Research},
  volume={7},
  number={Aug},
  pages={1687--1712},
  year={2006}
}

@article{shen2003psi,
  title={On $\psi$-learning},
  author={Shen, Xiaotong and Tseng, George C and Zhang, Xuegong and Wong, Wing Hung},
  journal={Journal of the American Statistical Association},
  volume={98},
  number={463},
  pages={724--734},
  year={2003},
  publisher={Taylor \& Francis}
}

@article{lin2002support,
  title={Support vector machines and the Bayes rule in classification},
  author={Lin, Yi},
  journal={Data Mining and Knowledge Discovery},
  volume={6},
  number={3},
  pages={259--275},
  year={2002},
  publisher={Springer}
}

@article{soudry2018implicit,
  title={The implicit bias of gradient descent on separable data},
  author={Soudry, Daniel and Hoffer, Elad and Nacson, Mor Shpigel and Gunasekar, Suriya and Srebro, Nathan},
  journal={The Journal of Machine Learning Research},
  volume={19},
  number={1},
  pages={2822--2878},
  year={2018},
  publisher={JMLR. org}
}

@article{hinz2019framework,
  title={A framework for the construction of upper bounds on the number of affine linear regions of ReLU feed-forward neural networks},
  author={Hinz, Peter and van de Geer, Sara},
  journal={IEEE Transactions on Information Theory},
  year={2019},
  publisher={IEEE}
}

@article{guo2019robust,
  title={Robust student network learning},
  author={Guo, Tianyu and Xu, Chang and He, Shiyi and Shi, Boxin and Xu, Chao and Tao, Dacheng},
  journal={IEEE transactions on neural networks and learning systems},
  year={2019},
  publisher={IEEE}
}

@article{tian2018theoretical,
  title={A theoretical framework for deep locally connected relu network},
  author={Tian, Yuandong},
  journal={arXiv preprint arXiv:1809.10829},
  year={2018}
}
@article{tian2019over,
  title={Over-parameterization as a Catalyst for Better Generalization of Deep ReLU network},
  author={Tian, Yuandong},
  journal={arXiv preprint arXiv:1909.13458},
  year={2019}
}

@article{ILSVRC15,
Author = {Olga Russakovsky and Jia Deng and Hao Su and Jonathan Krause and Sanjeev Satheesh and Sean Ma and Zhiheng Huang and Andrej Karpathy and Aditya Khosla and Michael Bernstein and Alexander C. Berg and Li Fei-Fei},
Title = {{ImageNet Large Scale Visual Recognition Challenge}},
Year = {2015},
journal   = {International Journal of Computer Vision (IJCV)},
doi = {10.1007/s11263-015-0816-y},
volume={115},
number={3},
pages={211-252}
}

@inproceedings{gunasekar2018implicit,
  title={Implicit bias of gradient descent on linear convolutional networks},
  author={Gunasekar, Suriya and Lee, Jason D and Soudry, Daniel and Srebro, Nati},
  booktitle={Advances in Neural Information Processing Systems},
  pages={9461--9471},
  year={2018}
}

@inproceedings{montufar2014number,
  title={On the number of linear regions of deep neural networks},
  author={Montufar, Guido F and Pascanu, Razvan and Cho, Kyunghyun and Bengio, Yoshua},
  booktitle={Advances in neural information processing systems},
  pages={2924--2932},
  year={2014}
}

@inproceedings{krizhevsky2012imagenet,
  title={Imagenet classification with deep convolutional neural networks},
  author={Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
  booktitle={Advances in neural information processing systems},
  pages={1097--1105},
  year={2012}
}

@inproceedings{raghu2017expressive,
  title={On the expressive power of deep neural networks},
  author={Raghu, Maithra and Poole, Ben and Kleinberg, Jon and Ganguli, Surya and Dickstein, Jascha Sohl},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={2847--2854},
  year={2017},
  organization={JMLR. org}
}

@article{serra2017bounding,
  title={Bounding and counting linear regions of deep neural networks},
  author={Serra, Thiago and Tjandraatmadja, Christian and Ramalingam, Srikumar},
  journal={arXiv preprint arXiv:1711.02114},
  year={2017}
}

@article{nguyen2017iris,
  title={Iris recognition with off-the-shelf CNN features: A deep learning perspective},
  author={Nguyen, Kien and Fookes, Clinton and Ross, Arun and Sridharan, Sridha},
  journal={IEEE Access},
  volume={6},
  pages={18848--18855},
  year={2017},
  publisher={IEEE}
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}
@inproceedings{ngiam2011multimodal,
  title={Multimodal deep learning},
  author={Ngiam, Jiquan and Khosla, Aditya and Kim, Mingyu and Nam, Juhan and Lee, Honglak and Ng, Andrew Y},
  booktitle={Proceedings of the 28th international conference on machine learning (ICML-11)},
  pages={689--696},
  year={2011}
}

@inproceedings{imagenet_cvpr09,
        AUTHOR = {Deng, J. and Dong, W. and Socher, R. and Li, L.-J. and Li, K. and Fei-Fei, L.},
        TITLE = {{ImageNet: A Large-Scale Hierarchical Image Database}},
        BOOKTITLE = {CVPR09},
        YEAR = {2009},
        BIBSOURCE = "http://www.image-net.org/papers/imagenet_cvpr09.bib"}

@article{simonyan2014very,
  title={Very deep convolutional networks for large-scale image recognition},
  author={Simonyan, Karen and Zisserman, Andrew},
  journal={arXiv preprint arXiv:1409.1556},
  year={2014}
}

@inproceedings{zeiler2014visualizing,
  title={Visualizing and understanding convolutional networks},
  author={Zeiler, Matthew D and Fergus, Rob},
  booktitle={European conference on computer vision},
  pages={818--833},
  year={2014},
  organization={Springer}
}


@book{korostelev2012minimax,
  title={Minimax theory of image reconstruction},
  author={Korostelev, Aleksandr Petrovich and Tsybakov, Alexandre B},
  volume={82},
  year={2012},
  publisher={Springer Science \& Business Media}
}

@article{schmidt2017nonparametric,
  title={Nonparametric regression using deep neural networks with ReLU activation function},
   author={Schmidt-Hieber, Johannes\vspace{0mm}},
  journal={The Annals of Statistics},
  year={2019},
  note={Forthcoming}
}

@inproceedings{bartlett2017spectrally,
  title={Spectrally-normalized margin bounds for neural networks},
  author={Bartlett, Peter L and Foster, Dylan J and Telgarsky, Matus J},
  booktitle={Advances in Neural Information Processing Systems},
  pages={6240--6249},
  year={2017}
}


@article{ioffe2015batch,
  title={Batch normalization: Accelerating deep network training by reducing internal covariate shift},
  author={Ioffe, Sergey and Szegedy, Christian},
  journal={arXiv preprint arXiv:1502.03167},
  year={2015}
}

@inproceedings{hoffer2017train,
  title={Train longer, generalize better: closing the generalization gap in large batch training of neural networks},
  author={Hoffer, Elad and Hubara, Itay and Soudry, Daniel},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1731--1741},
  year={2017}
}

@article{ledent2019improved,
  title={Improved Generalisation Bounds for Deep Learning Through $ L\^{}$\backslash$infty $ Covering Numbers},
  author={Ledent, Antoine and Lei, Yunwen and Kloft, Marius},
  journal={arXiv preprint arXiv:1905.12430},
  year={2019}
}

@article{hastie2019surprises,
  title={Surprises in high-dimensional ridgeless least squares interpolation},
  author={Hastie, Trevor and Montanari, Andrea and Rosset, Saharon and Tibshirani, Ryan J},
  journal={arXiv preprint arXiv:1903.08560},
  year={2019}
}

@article{petersen2018optimal,
  title={Optimal approximation of piecewise smooth functions using deep ReLU neural networks},
  author={Petersen, Philipp and Voigtlaender, Felix},
  journal={Neural Networks},
  volume={108},
  pages={296--330},
  year={2018},
  publisher={Elsevier}
}

@article{yarotsky2017error,
  title={Error bounds for approximations with deep ReLU networks},
  author={Yarotsky, Dmitry},
  journal={Neural Networks},
  volume={94},
  pages={103--114},
  year={2017},
  publisher={Elsevier}
}

@article{bauer2019deep,
  title={On deep learning as a remedy for the curse of dimensionality in nonparametric regression},
  author={Bauer, Benedikt and Kohler, Michael and others},
  journal={The Annals of Statistics},
  volume={47},
  number={4},
  pages={2261--2285},
  year={2019},
  publisher={Institute of Mathematical Statistics}
}

@article{martin2018implicit,
  title={Implicit self-regularization in deep neural networks: Evidence from random matrix theory and implications for learning},
  author={Martin, Charles H and Mahoney, Michael W},
  journal={arXiv preprint arXiv:1810.01075},
  year={2018}
}

@article{kim2018fast,
  title={Fast convergence rates of deep neural networks for classification},
  author={Kim, Yongdai and Ohn, Ilsang and Kim, Dongha},
  journal={arXiv preprint arXiv:1812.03599},
  year={2018}
}

@article{imaizumi2018deep,
  title={Deep neural networks learn non-smooth functions effectively},
  author={Imaizumi, Masaaki and Fukumizu, Kenji},
  journal={arXiv preprint arXiv:1802.04474},
  year={2018}
}

@article{suzuki2018adaptivity,
  title={Adaptivity of deep ReLU network for learning in Besov and mixed smooth Besov spaces: optimal rate and curse of dimensionality},
  author={Suzuki, Taiji},
  journal={arXiv preprint arXiv:1810.08033},
  year={2018}
}

@inproceedings{goodfellow2014generative,
  title={Generative adversarial nets},
  author={Goodfellow, Ian and Pouget-Abadie, Jean and Mirza, Mehdi and Xu, Bing and Warde-Farley, David and Ozair, Sherjil and Courville, Aaron and Bengio, Yoshua},
  booktitle={Advances in neural information processing systems},
  pages={2672--2680},
  year={2014}
}

@article{arjovsky2017wasserstein,
  title={Wasserstein gan},
  author={Arjovsky, Martin and Chintala, Soumith and Bottou, L{\'e}on},
  journal={arXiv preprint arXiv:1701.07875},
  year={2017}
}

@article{mroueh2017sobolev,
  title={Sobolev gan},
  author={Mroueh, Youssef and Li, Chun-Liang and Sercu, Tom and Raj, Anant and Cheng, Yu},
  journal={arXiv preprint arXiv:1711.04894},
  year={2017}
}

@article{advani2017high,
  title={High-dimensional dynamics of generalization error in neural networks},
  author={Advani, Madhu S and Saxe, Andrew M},
  journal={arXiv preprint arXiv:1710.03667},
  year={2017}
}

@article{ghorbani2019linearized,
  title={Linearized two-layers neural networks in high dimension},
  author={Ghorbani, Behrooz and Mei, Song and Misiakiewicz, Theodor and Montanari, Andrea},
  journal={arXiv preprint arXiv:1904.12191},
  year={2019}
}

@inproceedings{li2015generative,
  title={Generative moment matching networks},
  author={Li, Yujia and Swersky, Kevin and Zemel, Rich},
  booktitle={International Conference on Machine Learning},
  pages={1718--1727},
  year={2015}
}

@article{steinwart2007fast,
  title={Fast rates for support vector machines using Gaussian kernels},
  author={Steinwart, Ingo and Scovel, Clint and others},
  journal={The Annals of Statistics},
  volume={35},
  number={2},
  pages={575--607},
  year={2007},
  publisher={Institute of Mathematical Statistics}
}

@article{lecun2015deep,
  title={Deep learning},
  author={LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
  journal={nature},
  volume={521},
  number={7553},
  pages={436},
  year={2015},
  publisher={Nature Publishing Group}
}

@article{tsybakov2004optimal,
  title={Optimal aggregation of classifiers in statistical learning},
  author={Tsybakov, Alexander B and others},
  journal={The Annals of Statistics},
  volume={32},
  number={1},
  pages={135--166},
  year={2004},
  publisher={Institute of Mathematical Statistics}
}

@article{tsybakov2005square,
  title={Square root penalty: adaptation to the margin in classification and in edge estimation},
  author={Tsybakov, Alexandre B and van de Geer, Sara A and others},
  journal={The Annals of Statistics},
  volume={33},
  number={3},
  pages={1203--1224},
  year={2005},
  publisher={Institute of Mathematical Statistics}
}


@article{cybenko1989approximations,
  title={Approximations by superpositions of a sigmoidal function},
  author={Cybenko, George},
  journal={Mathematics of Control, Signals and Systems},
  volume={2},
  pages={183--192},
  year={1989}
}

@article{stone1982optimal,
  title={Optimal global rates of convergence for nonparametric regression},
  author={Stone, Charles J},
  journal={The annals of statistics},
  pages={1040--1053},
  year={1982},
  publisher={JSTOR}
}

@article{arora2019fine,
  title={Fine-grained analysis of optimization and generalization for overparameterized two-layer neural networks},
  author={Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and Wang, Ruosong},
  journal={arXiv preprint arXiv:1901.08584},
  year={2019}
}

@article{liu2019optimal,
  title={Optimal Nonparametric Inference via Deep Neural Network},
  author={Liu, Ruiqi and Boukai, Ben and Shang, Zuofeng},
  journal={arXiv preprint arXiv:1902.01687},
  year={2019}
}

@article{arora2016understanding,
  title={Understanding deep neural networks with rectified linear units},
  author={Arora, Raman and Basu, Amitabh and Mianjy, Poorya and Mukherjee, Anirbit},
  journal={arXiv preprint arXiv:1611.01491},
  year={2016}
}

@article{hanin2017universal,
  title={Universal function approximation by deep neural nets with bounded width and relu activations},
  author={Hanin, Boris},
  journal={arXiv preprint arXiv:1708.02691},
  year={2017}
}

@inproceedings{kawaguchi2016deep,
  title={Deep learning without poor local minima},
  author={Kawaguchi, Kenji},
  booktitle={Advances in neural information processing systems},
  pages={586--594},
  year={2016}
}

@article{du2018gradient,
  title={Gradient descent provably optimizes over-parameterized neural networks},
  author={Du, Simon S and Zhai, Xiyu and Poczos, Barnabas and Singh, Aarti},
  journal={arXiv preprint arXiv:1810.02054},
  year={2018}
}

@inproceedings{neyshabur2017exploring,
  title={Exploring generalization in deep learning},
  author={Neyshabur, Behnam and Bhojanapalli, Srinadh and McAllester, David and Srebro, Nati},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5947--5956},
  year={2017}
}

@inproceedings{neyshabur2015norm,
  title={Norm-based capacity control in neural networks},
  author={Neyshabur, Behnam and Tomioka, Ryota and Srebro, Nathan},
  booktitle={Conference on Learning Theory},
  pages={1376--1401},
  year={2015}
}

@article{zhou2017landscape,
  title={The landscape of deep learning algorithms},
  author={Zhou, Pan and Feng, Jiashi},
  journal={arXiv preprint arXiv:1705.07038},
  year={2017}
}

@misc{fu1802local,
  title={Local geometry of one-hidden-layer neural networks for logistic regression},
  author={Fu, Haoyu and Chi, Yuejie and Liang, Yingbin},
  year={1802}
}

@article{liang2018understanding,
  title={Understanding the loss surface of neural networks for binary classification},
  author={Liang, Shiyu and Sun, Ruoyu and Li, Yixuan and Srikant, Rayadurgam},
  journal={arXiv preprint arXiv:1803.00909},
  year={2018}
}

@article{allen2018convergence,
  title={A convergence theory for deep learning via over-parameterization},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  journal={arXiv preprint arXiv:1811.03962},
  year={2018}
}

@article{allen2018learning,
  title={Learning and generalization in overparameterized neural networks, going beyond two layers},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Liang, Yingyu},
  journal={arXiv preprint arXiv:1811.04918},
  year={2018}
}

@inproceedings{liang2018adding,
  title={Adding one neuron can eliminate all bad local minima},
  author={Liang, Shiyu and Sun, Ruoyu and Lee, Jason D and Srikant, Rayadurgam},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4350--4360},
  year={2018}
}

@article{kawaguchi2019elimination,
  title={Elimination of all bad local minima in deep learning},
  author={Kawaguchi, Kenji and Kaelbling, Leslie Pack},
  journal={arXiv preprint arXiv:1901.00279},
  year={2019}
}

@inproceedings{jacot2018neural,
  title={Neural tangent kernel: Convergence and generalization in neural networks},
  author={Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
  booktitle={Advances in neural information processing systems},
  pages={8571--8580},
  year={2018}
}

@article{arora2019exact,
  title={On exact computation with an infinitely wide neural net},
  author={Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and Salakhutdinov, Ruslan and Wang, Ruosong},
  journal={arXiv preprint arXiv:1904.11955},
  year={2019}
}

@article{chizat2018note,
  title={A note on lazy training in supervised differentiable programming},
  author={Chizat, Lenaic and Bach, Francis},
  journal={arXiv preprint arXiv:1812.07956},
  year={2018}
}

@article{yang2019scaling,
  title={Scaling limits of wide neural networks with weight sharing: Gaussian process behavior, gradient independence, and neural tangent kernel derivation},
  author={Yang, Greg},
  journal={arXiv preprint arXiv:1902.04760},
  year={2019}
}

@article{vapnik1999overview,
  title={An overview of statistical learning theory},
  author={Vapnik, Vladimir N},
  journal={IEEE transactions on neural networks},
  volume={10},
  number={5},
  pages={988--999},
  year={1999},
  publisher={Citeseer}
}

@article{neyshabur2017pac,
  title={A pac-bayesian approach to spectrally-normalized margin bounds for neural networks},
  author={Neyshabur, Behnam and Bhojanapalli, Srinadh and Srebro, Nathan},
  journal={arXiv preprint arXiv:1707.09564},
  year={2017}
}

@incollection{mcallester2003simplified,
  title={Simplified PAC-Bayesian margin bounds},
  author={McAllester, David},
  booktitle={Learning theory and Kernel machines},
  pages={203--215},
  year={2003},
  publisher={Springer}
}

@article{zhang2016understanding,
  title={Understanding deep learning requires rethinking generalization},
  author={Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
  journal={arXiv preprint arXiv:1611.03530},
  year={2016}
}

@article{arora2019exact,
  title={On exact computation with an infinitely wide neural net},
  author={Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and Salakhutdinov, Ruslan and Wang, Ruosong},
  journal={arXiv preprint arXiv:1904.11955},
  year={2019}
}

@inproceedings{deng2009imagenet,
  title={Imagenet: A large-scale hierarchical image database},
  author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Li, Kai and Fei-Fei, Li},
  booktitle={2009 IEEE conference on computer vision and pattern recognition},
  pages={248--255},
  year={2009},
  organization={Ieee}
}

@article{cortes1995support,
  title={Support-vector networks},
  author={Cortes, Corinna and Vapnik, Vladimir},
  journal={Machine learning},
  volume={20},
  number={3},
  pages={273--297},
  year={1995},
  publisher={Springer}
}

@article{bousquet2019fast,
  title={Fast classification rates without standard margin assumptions},
  author={Bousquet, Olivier and Zhivotovskiy, Nikita},
  journal={arXiv preprint arXiv:1910.12756},
  year={2019}
}

@article{massart2006risk,
  title={Risk bounds for statistical learning},
  author={Massart, Pascal and N{\'e}d{\'e}lec, {\'E}lodie and others},
  journal={The Annals of Statistics},
  volume={34},
  number={5},
  pages={2326--2366},
  year={2006},
  publisher={Institute of Mathematical Statistics}
}

@article{chow1970optimum,
  title={On optimum recognition error and reject tradeoff},
  author={Chow, C},
  journal={IEEE Transactions on information theory},
  volume={16},
  number={1},
  pages={41--46},
  year={1970},
  publisher={IEEE}
}

@article{srebro2010optimistic,
  title={Optimistic rates for learning with a smooth loss},
  author={Srebro, Nathan and Sridharan, Karthik and Tewari, Ambuj},
  journal={arXiv preprint arXiv:1009.3896},
  year={2010}
}

@article{yang1999minimax,
  title={Minimax nonparametric classification. I. Rates of convergence},
  author={Yang, Yuhong},
  journal={IEEE Transactions on Information Theory},
  volume={45},
  number={7},
  pages={2271--2284},
  year={1999},
  publisher={IEEE}
}

@article{kohler2019rate,
  title={On the rate of convergence of fully connected very deep neural network regression estimates},
  author={Kohler, Michael and Langer, Sophie},
  journal={arXiv preprint arXiv:1908.11133},
  year={2019}
}

@article{farrell2018deep,
  title={Deep neural networks for estimation and inference: Application to causal effects and other semiparametric estimands},
  author={Farrell, Max H and Liang, Tengyuan and Misra, Sanjog},
  journal={arXiv preprint arXiv:1809.09953},
  year={2018}
}

@article{nakada2019adaptive,
  title={Adaptive approximation and estimation of deep neural network to intrinsic dimensionality},
  author={Nakada, Ryumei and Imaizumi, Masaaki},
  journal={arXiv preprint arXiv:1907.02177},
  year={2019}
}

@article{oono2019approximation,
  title={Approximation and non-parametric estimation of ResNet-type convolutional neural networks},
  author={Oono, Kenta and Suzuki, Taiji},
  journal={arXiv preprint arXiv:1903.10047},
  year={2019}
}

@inproceedings{chen2019efficient,
  title={Efficient approximation of deep relu networks for functions on low dimensional manifolds},
  author={Chen, Minshuo and Jiang, Haoming and Liao, Wenjing and Zhao, Tuo},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8172--8182},
  year={2019}
}

@inproceedings{lu2017expressive,
  title={The expressive power of neural networks: A view from the width},
  author={Lu, Zhou and Pu, Hongming and Wang, Feicheng and Hu, Zhiqiang and Wang, Liwei},
  booktitle={Advances in neural information processing systems},
  pages={6231--6239},
  year={2017}
}

@article{zhao2006model,
  title={On model selection consistency of Lasso},
  author={Zhao, Peng and Yu, Bin},
  journal={Journal of Machine learning research},
  volume={7},
  number={Nov},
  pages={2541--2563},
  year={2006}
}

@article{yarotsky2019phase,
  title={The phase diagram of approximation rates for deep neural networks},
  author={Yarotsky, Dmitry and Zhevnerchuk, Anton},
  journal={arXiv preprint arXiv:1906.09477},
  year={2019}
}

@book{sara2000,
  title={Empirical Processes in M-estimation},
  author={Van De Geer, Sara},
  year={2000},
  publisher={Cambridge University Press}
}

@inproceedings{cao2019tight,
  title={Tight Sample Complexity of Learning One-hidden-layer Convolutional Neural Networks},
  author={Cao, Yuan and Gu, Quanquan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={10611--10621},
  year={2019}
}

@article{zhang2018learning,
  title={Learning one-hidden-layer relu networks via gradient descent},
  author={Zhang, Xiao and Yu, Yaodong and Wang, Lingxiao and Gu, Quanquan},
  journal={Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics (AISTATS)},
  year={2019}
}

@article{telgarsky2015representation,
  title={Representation benefits of deep feedforward networks},
  author={Telgarsky, Matus},
  journal={arXiv preprint arXiv:1509.08101},
  year={2015}
}

@article{bartlett2019nearly,
  title={Nearly-tight VC-dimension and Pseudodimension Bounds for Piecewise Linear Neural Networks.},
  author={Bartlett, Peter L and Harvey, Nick and Liaw, Christopher and Mehrabian, Abbas},
  journal={Journal of Machine Learning Research},
  volume={20},
  number={63},
  pages={1--17},
  year={2019}
}

@article{vapnik1971uniform,
  title={On the Uniform Convergence of Relative Frequencies of Events to Their Probabilities},
  author={Vapnik, VN and Chervonenkis, A Ya},
  journal={Theory of Probability \& Its Applications},
  volume={16},
  number={2},
  pages={264--280},
  year={1971},
  publisher={SIAM}
}

@article{vaart1997weak,
  title={Weak convergence and empirical processes with applications to statistics},
  author={Vaart, AW van der and Wellner, Jon A},
  journal={Journal of the Royal Statistical Society-Series A Statistics in Society},
  volume={160},
  number={3},
  pages={596--608},
  year={1997},
  publisher={London: Royal Statistical Society, 1988-}
}

@book{anthony2009neural,
  title={Neural network learning: Theoretical foundations},
  author={Anthony, Martin and Bartlett, Peter L},
  year={2009},
  publisher={cambridge university press}
}

@article{kohler2020statistical,
  title={Statistical theory for image classification using deep convolutional neural networks with cross-entropy loss},
  author={Kohler, Michael and Langer, Sophie},
  journal={arXiv preprint arXiv:2011.13602},
  year={2020}
}


@article{hu2020sharp,
  title={Sharp Rate of Convergence for Deep Neural Network Classifiers under the Teacher-Student Setting},
  author={Hu, Tianyang and Shang, Zuofeng and Cheng, Guang},
  journal={arXiv preprint arXiv:2001.06892},
  year={2020}
}

@inproceedings{croce2019provable,
  title={Provable robustness of relu networks via maximization of linear regions},
  author={Croce, Francesco and Andriushchenko, Maksym and Hein, Matthias},
  booktitle={the 22nd International Conference on Artificial Intelligence and Statistics},
  pages={2057--2066},
  year={2019},
  organization={PMLR}
}

@article{tsipras2018robustness,
  title={Robustness may be at odds with accuracy},
  author={Tsipras, Dimitris and Santurkar, Shibani and Engstrom, Logan and Turner, Alexander and Madry, Aleksander},
  journal={arXiv preprint arXiv:1805.12152},
  year={2018}
}

@article{bousquet2002stability,
  title={Stability and generalization},
  author={Bousquet, Olivier and Elisseeff, Andr{\'e}},
  journal={The Journal of Machine Learning Research},
  volume={2},
  pages={499--526},
  year={2002},
  publisher={JMLR. org}
}

@book{vapnik1999nature,
  title={The nature of statistical learning theory},
  author={Vapnik, Vladimir},
  year={1999},
  publisher={Springer science \& business media}
}

@book{james2013introduction,
  title={An introduction to statistical learning},
  author={James, Gareth and Witten, Daniela and Hastie, Trevor and Tibshirani, Robert},
  volume={112},
  year={2013},
  publisher={Springer}
}

@article{goodfellow2014explaining,
  title={Explaining and harnessing adversarial examples},
  author={Goodfellow, Ian J and Shlens, Jonathon and Szegedy, Christian},
  journal={arXiv preprint arXiv:1412.6572},
  year={2014}
}

@incollection{prechelt1998early,
  title={Early stopping-but when?},
  author={Prechelt, Lutz},
  booktitle={Neural Networks: Tricks of the trade},
  pages={55--69},
  year={1998},
  publisher={Springer}
}

@inproceedings{woodworth2019kernel,
  title={Kernel and deep regimes in overparametrized models},
  author={Woodworth, Blake and Genesekar, Suriya and Lee, Jason and Soudry, Daniel and Srebro, Nathan},
  booktitle={Conference on Learning Theory (COLT)},
  year={2019}
}

@inproceedings{chizat2020implicit,
  title={Implicit bias of gradient descent for wide two-layer neural networks trained with the logistic loss},
  author={Chizat, Lenaic and Bach, Francis},
  booktitle={Conference on Learning Theory},
  pages={1305--1338},
  year={2020},
  organization={PMLR}
}

@article{razin2020implicit,
  title={Implicit regularization in deep learning may not be explainable by norms},
  author={Razin, Noam and Cohen, Nadav},
  journal={arXiv preprint arXiv:2005.06398},
  year={2020}
}

@article{shafahi2019adversarially,
  title={Adversarially robust transfer learning},
  author={Shafahi, Ali and Saadatpanah, Parsa and Zhu, Chen and Ghiasi, Amin and Studer, Christoph and Jacobs, David and Goldstein, Tom},
  journal={arXiv preprint arXiv:1905.08232},
  year={2019}
}

@inproceedings{goldblum2020adversarially,
  title={Adversarially robust distillation},
  author={Goldblum, Micah and Fowl, Liam and Feizi, Soheil and Goldstein, Tom},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={3996--4003},
  year={2020}
}

@inproceedings{cohen2019certified,
  title={Certified adversarial robustness via randomized smoothing},
  author={Cohen, Jeremy and Rosenfeld, Elan and Kolter, Zico},
  booktitle={International Conference on Machine Learning},
  pages={1310--1320},
  year={2019},
  organization={PMLR}
}

@inproceedings{katz2017reluplex,
  title={Reluplex: An efficient SMT solver for verifying deep neural networks},
  author={Katz, Guy and Barrett, Clark and Dill, David L and Julian, Kyle and Kochenderfer, Mykel J},
  booktitle={International Conference on Computer Aided Verification},
  pages={97--117},
  year={2017},
  organization={Springer}
}

@article{tjeng2017evaluating,
  title={Evaluating robustness of neural networks with mixed integer programming},
  author={Tjeng, Vincent and Xiao, Kai and Tedrake, Russ},
  journal={arXiv preprint arXiv:1711.07356},
  year={2017}
}

@article{hein2017formal,
  title={Formal guarantees on the robustness of a classifier against adversarial manipulation},
  author={Hein, Matthias and Andriushchenko, Maksym},
  journal={arXiv preprint arXiv:1705.08475},
  year={2017}
}

@article{raghunathan2018certified,
  title={Certified defenses against adversarial examples},
  author={Raghunathan, Aditi and Steinhardt, Jacob and Liang, Percy},
  journal={arXiv preprint arXiv:1801.09344},
  year={2018}
}

@inproceedings{weng2018towards,
  title={Towards fast computation of certified robustness for relu networks},
  author={Weng, Lily and Zhang, Huan and Chen, Hongge and Song, Zhao and Hsieh, Cho-Jui and Daniel, Luca and Boning, Duane and Dhillon, Inderjit},
  booktitle={International Conference on Machine Learning},
  pages={5276--5285},
  year={2018},
  organization={PMLR}
}

@inproceedings{lyu2015unified,
  title={A unified gradient regularization family for adversarial examples},
  author={Lyu, Chunchuan and Huang, Kaizhu and Liang, Hai-Ning},
  booktitle={2015 IEEE international conference on data mining},
  pages={301--309},
  year={2015},
  organization={IEEE}
}

@article{roth2019adversarial,
  title={Adversarial training is a form of data-dependent operator norm regularization},
  author={Roth, Kevin and Kilcher, Yannic and Hofmann, Thomas},
  journal={arXiv preprint arXiv:1906.01527},
  year={2019}
}

@article{chan2019jacobian,
  title={Jacobian adversarially regularized networks for robustness},
  author={Chan, Alvin and Tay, Yi and Ong, Yew Soon and Fu, Jie},
  journal={arXiv preprint arXiv:1912.10185},
  year={2019}
}

@article{hu2021understanding,
  title={Understanding Square Loss in Training Overparametrized Neural Network Classifiers},
  author={Hu, Tianyang and Wang, Jun and Wang, Wenjia and Li, Zhenguo},
  journal={arXiv preprint arXiv:2112.03657},
  year={2021}
}

@article{xie2020smooth,
  title={Smooth adversarial training},
  author={Xie, Cihang and Tan, Mingxing and Gong, Boqing and Yuille, Alan and Le, Quoc V},
  journal={arXiv preprint arXiv:2006.14536},
  year={2020}
}

@inproceedings{bao2020calibrated,
  title={Calibrated surrogate losses for adversarially robust classification},
  author={Bao, Han and Scott, Clay and Sugiyama, Masashi},
  booktitle={Conference on Learning Theory},
  pages={408--451},
  year={2020},
  organization={PMLR}
}

@article{atzmon2019controlling,
  title={Controlling neural level sets},
  author={Atzmon, Matan and Haim, Niv and Yariv, Lior and Israelov, Ofer and Maron, Haggai and Lipman, Yaron},
  journal={arXiv preprint arXiv:1905.11911},
  year={2019}
}

@inproceedings{cui2021learnable,
  title={Learnable boundary guided adversarial training},
  author={Cui, Jiequan and Liu, Shu and Wang, Liwei and Jia, Jiaya},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={15721--15730},
  year={2021}
}

@inproceedings{bubeck2021law,
  title={A law of robustness for two-layers neural networks},
  author={Bubeck, S{\'e}bastien and Li, Yuanzhi and Nagaraj, Dheeraj M},
  booktitle={Conference on Learning Theory},
  pages={804--820},
  year={2021},
  organization={PMLR}
}

@article{hu2022minimax,
  title={Minimax Optimal Deep Neural Network Classifiers Under Smooth Decision Boundary},
  author={Hu, Tianyang and Liu, Ruiqi and Shang, Zuofeng and Cheng, Guang},
  journal={arXiv preprint arXiv:2207.01602},
  year={2022}
}

@inproceedings{hanin2019complexity,
  title={Complexity of linear regions in deep networks},
  author={Hanin, Boris and Rolnick, David},
  booktitle={International Conference on Machine Learning},
  pages={2596--2604},
  year={2019},
  organization={PMLR}
}

@article{dherin2022neural,
  title={Why neural networks find simple solutions: the many regularizers of geometric complexity},
  author={Dherin, Benoit and Munn, Michael and Rosca, Mihaela and Barrett, David GT},
  journal={arXiv preprint arXiv:2209.13083},
  year={2022}
}

@inproceedings{chen2019topological,
  title={A topological regularizer for classifiers via persistent homology},
  author={Chen, Chao and Ni, Xiuyan and Bai, Qinxun and Wang, Yusu},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={2573--2582},
  year={2019},
  organization={PMLR}
}

@article{lei2022understanding,
  title={Understanding deep learning via decision boundary},
  author={Lei, Shiye and He, Fengxiang and Yuan, Yancheng and Tao, Dacheng},
  journal={arXiv preprint arXiv:2206.01515},
  year={2022}
}

@article{yang2020boundary,
  title={Boundary thickness and robustness in learning models},
  author={Yang, Yaoqing and Khanna, Rajiv and Yu, Yaodong and Gholami, Amir and Keutzer, Kurt and Gonzalez, Joseph E and Ramchandran, Kannan and Mahoney, Michael W},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={6223--6234},
  year={2020}
}

@article{krizhevsky2009learning,
author = {Krizhevsky, Alex},
year = {2009},
title = {Learning Multiple Layers of Features from Tiny Images},
journal = {University of Toronto}
}

@article{bubeck2023universal,
  title={A universal law of robustness via isoperimetry},
  author={Bubeck, S{\'e}bastien and Sellke, Mark},
  journal={Journal of the ACM},
  volume={70},
  number={2},
  pages={1--18},
  year={2023},
  publisher={ACM New York, NY}
}

@inproceedings{bubeck2021law,
  title={A law of robustness for two-layers neural networks},
  author={Bubeck, S{\'e}bastien and Li, Yuanzhi and Nagaraj, Dheeraj M},
  booktitle={Conference on Learning Theory},
  pages={804--820},
  year={2021},
  organization={PMLR}
}

@article{ding2023random,
  title={Random Smoothing Regularization in Kernel Gradient Descent Learning},
  author={Ding, Liang and Hu, Tianyang and Jiang, Jiahang and Li, Donghao and Wang, Wenjia and Yao, Yuan},
  journal={arXiv preprint arXiv:2305.03531},
  year={2023}
}

@article{zhu2020bounding,
    title={Bounding the number of linear regions in local area for neural networks with relu activations},
    author={Zhu, Rui and Lin, Bo and Tang, Haixu},
    journal={arXiv preprint arXiv:2007.06803},
    year={2020}
}
