@inproceedings{CL-LZ-MB:20,
 author = {Liu, Chaoyue and Zhu, Libin and Belkin, Misha},
 booktitle = {Advances in Neural Information Processing Systems},
 title = {On the linearity of large non-linear models: when and why the tangent kernel is constant},
 year = {2020}
}

@article{CL-LZ-MB:21,
  title =	 {Loss landscapes and optimization in over-parameterized non-linear systems and neural networks},
  author =	 {C. Liu and L. Zhu and M. Belkin},
  journal =	 {Applied and Computational Harmonic Analysis},
  volume = {59},
    pages = {85-116},
    year = {2022}
}


@inproceedings{
AB-PCV-LZ-MB:22,
title={Restricted Strong Convexity of Deep Learning Models with Smooth Activations},
author={Arindam Banerjee and Pedro Cisneros-Velarde and Libin Zhu and Misha Belkin},
booktitle={The Eleventh International Conference on Learning Representations },
year={2023}
}

@article{oymak2020hermite,
  title =	 {Towards moderate overparameterization: global convergence guarantees for
training shallow neural networks},
  author =	 {S. Oymak and M. Soltanolkotabi},
  journal =	 {IEEE Journal on Selected
Areas in Information Theory},
  year =	 {2020}
}

@inproceedings{ng2020hermite1,
 author = {Q. Nguyen and M. Mondelli},
 title = {Global convergence of deep networks with one wide layer followed by pyramidal
topology},
 booktitle = {Neural Information Processing Systems (NeurIPS)},
 year = {2020}
}


@inproceedings{ng2021hermite2,
 author = {Quynh Nguyen and Marco Mondelli and Guido Montufar},
 booktitle = {International Conference on Machine Learning (ICML)},
 title = {Tight Bounds on the Smallest Eigenvalue of the Neural Tangent Kernel for
Deep ReLU Networks},
 year = {2021}
}




@inproceedings{AM-SO-ZS-DPW:22,
 author = {Alexander Munteanu and Simon Omlor and Zhao Song and David P. Woodruff},
 booktitle = {International Conference on Machine Learning (ICML)},
 title = {Bounding the Width of Neural Networks via Coupled Initialization
– A Worst Case Analysis –},
 year = {2022}
}


@inproceedings{salman2019adverse,
 author = {Hadi Salman and Greg Yang and Jerry Li and Pengchuan Zhang and Huan Zhang and Ilya Razenshteyn and Sébastien Bubeck},
 booktitle = {Neural Information Processing Systems (NeurIPS)},
 title = {Provably Robust Deep Learning via Adversarially Trained Smoothed Classifiers},
 year = {2019}
}

@inproceedings{cohen2019adverse,
 author = {Jeremy Cohen and Elan Rosenfeld and Zico Kolter},
 booktitle = {International Conference on Machine Learning (ICML)},
 title = {Certified adversarial robustness via randomized smoothing},
 year = {2020}
}

@inproceedings{he2015init,
 author = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
 booktitle = {International Conference on Computer Vision (ICCV)},
 title = {Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification},
 year = {2015}
}

@inproceedings{ng2021opt,
 author = {Quynh Nguyen},
 booktitle = {International Conference on Machine Learning (ICML)},
 title = {On the Proof of Global Convergence of Gradient Descent for Deep ReLU Networks with Linear Widths},
 year = {2021}
}


@inproceedings{bubeck2021robust,
 author = {Sebastien Bubeck and Mark Sellke},
 booktitle = {Neural Information Processing Systems (NeurIPS)},
 title = {A Universal Law of Robustness via Isoperimetry},
 year = {2021}
}

@inproceedings{an2021gradnorm,
  title={Can We Use Gradient Norm as a Measure of Generalization Error for Model Selection in Practice?},
  author={Haozhe An and Haoyi Xiong and Xuhong Li and Xingjian Li and Dejing Dou and Zhanxing Zhu},
  booktitle={Preprint},
  year={2021}
}

@inproceedings{negrea2019information,
  title={Information-Theoretic Generalization Bounds for SGLD via Data-Dependent Estimates.},
  author={Negrea, Jeffrey and Haghifam, Mahdi and Dziugaite, Gintare Karolina and Khisti, Ashish and Roy, Daniel M},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}

@inproceedings{baner22sgld,
  title={Stability Based Generalization Bounds for Exponential Family Langevin Dynamics},
  author={A. Banerjee and T. Chen and X. Li and Y. Zhou},
  booktitle={International Conference on Machine Learning (ICML)},
  year={2022}
}

@inproceedings{bcfs14,
  title={Estimation with Norm Regularization},
  author={A. Banerjee and S. Chen and F. Fazayeli and V. Sivakumar},
  booktitle={Advances in Neural Information Processing Systems (NIPS)},
  year={2014}
}


@inproceedings{chba15,
  title={Structured Estimation with Atomic Norms: General Bounds and Applications},
  author={S. Chen and A. Banerjee},
  booktitle={Advances in Neural Information Processing Systems (NIPS)},
  year={2015}
}


@article{negahban2012mest,
	year = {2012},
	volume = {27},
	number = {4},
	author = {Sahand Negahban and Pradeep Ravikumar and Martin Wainwright and Bin Yu},
	title = {A Unified Framework for High-Dimensional Analysis of M-Estimators with Decomposable Regularizers},
	journal = {Statistical Science}
}

@article{negahban2012matrix,
  title={Restricted strong convexity and weighted matrix completion: Optimal bounds with noise},
  author={Negahban, Sahand and Wainwright, Martin J},
  journal={The Journal of Machine Learning Research},
  volume={13},
  number={1},
  pages={1665--1697},
  year={2012}
}


@InProceedings{ZAZ-YL-ZS:19,
  title = 	 {A Convergence Theory for Deep Learning via Over-Parameterization},
  author =       {Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {242--252},
  year = 	 {2019},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  publisher =    {PMLR}
}

@inproceedings{karimi2016linear,
  title={Linear convergence of gradient and proximal-gradient methods under the polyak-{L}ojasiewicz condition},
  author={Karimi, Hamed and Nutini, Julie and Schmidt, Mark},
  booktitle={Joint European Conference on Machine Learning and Knowledge Discovery in Databases},
  pages={795--811},
  year={2016},
  organization={Springer}
}


@InProceedings{FD-KV-MS-FH:18,
  title = 	 {Essentially No Barriers in Neural Network Energy Landscape},
  author =       {Draxler, Felix and Veschgini, Kambis and Salmhofer, Manfred and Hamprecht, Fred},
  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
  pages = 	 {1309--1318},
  year = 	 {2018},
  volume = 	 {80},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {10--15 Jul},
  publisher =    {PMLR}
}

@inproceedings{TG-DP-DV-AW:18,
 author = {Garipov, Timur and Izmailov, Pavel and Podoprikhin, Dmitrii and Vetrov, Dmitry P and Wilson, Andrew G},
 booktitle = {Advances in Neural Information Processing Systems},
 publisher = {Curran Associates, Inc.},
 title = {Loss Surfaces, Mode Connectivity, and Fast Ensembling of DNNs},
 volume = {31},
 year = {2018}
}

@inproceedings{AG-NSK-CX-RS:18,
  author    = {Gotmare, Akhilesh and Keskar, Nitish Shirish and Xiong, Caiming and Socher, Richard},
  title     = {Using Mode Connectivity for Loss Landscape Analysis},
  year      = {2018},
  booktitle = {Workshop on Modern Trends in Nonconvex Optimization for Machine Learning, part of the 35th International Conference on Machine Learning},
}


@InProceedings{BS-FG-AJ-FS-CH-WG-JB:21,
  title = 	 {Geometry of the Loss Landscape in Overparameterized Neural Networks: Symmetries and Invariances},
  author =       {Simsek, Berfin and Ged, Fran{\c{c}}ois and Jacot, Arthur and Spadaro, Francesco and Hongler, Clement and Gerstner, Wulfram and Brea, Johanni},
  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
  pages = 	 {9722--9732},
  year = 	 {2021},
  volume = 	 {139},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {18--24 Jul},
  publisher =    {PMLR}
} 


@inproceedings{QN-PB-MM:21,
 author = {Nguyen, Quynh and Br\'{e}chet, Pierre and Mondelli, Marco},
 booktitle = {Advances in Neural Information Processing Systems},
 publisher = {Curran Associates, Inc.},
 title = {When are Solutions Connected in Deep Networks?},
 year = {2021}
}

@inproceedings{LS-UE-VUG-YD-LB:18,
  author    = {Sagun, Levent and Evci, Utku and G\"{u}ney, V. U\v{g}ur and Dauphin, Yann and Bottou, Le\'{o}n},
  title     = {Empirical Analysis of the Hessian of Over-Parametrized Neural Networks},
  year      = {2018},
  booktitle = {Workshop track at the Sixth International Conference on Learning Representations},
}

@book{MJW:19, 
place={Cambridge}, series={Cambridge Series in Statistical and Probabilistic Mathematics}, title={High-Dimensional Statistics: A Non-Asymptotic Viewpoint}, DOI={10.1017/9781108627771}, publisher={Cambridge University Press}, author={Wainwright, Martin J.}, year={2019}, collection={Cambridge Series in Statistical and Probabilistic Mathematics}}

@inproceedings{HL-ZX-GT-CS-TG:18,
author = {Li, Hao and Xu, Zheng and Taylor, Gavin and Studer, Christoph and Goldstein, Tom},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Visualizing the Loss Landscape of Neural Nets},
 volume = {31},
 year = {2018}
}

@article{EN-AR-CU:21,
  title =	 {Increasing Depth Leads to U-Shaped Test Risk in Over-parameterized Convolutional Networks},
  author =	 {Nichami, Eshaan and Radhakrishnan, Adityanarayanan and Uhler, Caroline},
  journal =	 {arXiv},
  year =	 2021
}

@inproceedings{LW-CM-WE:18,
 author = {Wu, Lei and Ma, Chao and E, Weinan},
 booktitle = {Advances in Neural Information Processing Systems},
 title = {How SGD Selects the Global Minima in Over-parameterized Learning: A Dynamical Stability Perspective},
 volume = {31},
 year = {2018}
}

@InProceedings{SD-JL-HL-LW-XZ:19,
  title = 	 {Gradient Descent Finds Global Minima of Deep Neural Networks},
  author =       {Du, Simon and Lee, Jason and Li, Haochuan and Wang, Liwei and Zhai, Xiyu},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {1675--1685},
  year = 	 {2019},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--15 Jun},
  publisher =    {PMLR}
}

@article{DZ-YC-DZ-QG:20,
  title = {Gradient descent optimizes over-parameterized deep ReLU networks},
  author =	 {Zou, Difan and Cao, Yuan and Zhou, Dongruo and Gu, Quanquan},
  journal =	 {Machine Learning},
  volume = {109},
  pages = {467--492},
  year = {2020}
}

@inproceedings{YLA-YL:18,
 author = {Li, Yuanzhi and Liang, Yingyu},
 booktitle = {Advances in Neural Information Processing Systems},
 publisher = {Curran Associates, Inc.},
 title = {Learning Overparameterized Neural Networks via Stochastic Gradient Descent on Structured Data},
 volume = {31},
 year = {2018}
}



@article{AA-SN-MJW:12,
author = {Alekh Agarwal and Sahand Negahban and Martin J. Wainwright},
title = {Fast global convergence of gradient methods for high-dimensional statistical recovery},
volume = {40},
journal = {The Annals of Statistics},
number = {5},
publisher = {Institute of Mathematical Statistics},
pages = {2452 -- 2482},
year = {2012}
}

@InProceedings{DR-MR:21,
  title = 	 { Learning with Gradient Descent and Weakly Convex Losses},
  author =       {Richards, Dominic and Rabbat, Mike},
  booktitle = 	 {Proceedings of The 24th International Conference on Artificial Intelligence and Statistics},
  pages = 	 {1990--1998},
  year = 	 {2021},
  volume = 	 {130},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {13--15 Apr},
  publisher =    {PMLR}
}

@inproceedings{DR-IK:21,
 author = {Richards, Dominic and Kuzborskij, Ilja},
 booktitle = {Advances in Neural Information Processing Systems},
 publisher = {Curran Associates, Inc.},
 title = {Stability & Generalisation of Gradient Descent for Shallow Neural Networks without the Neural Tangent Kernel},
 year = {2021}
}


@InProceedings{MH-BR-YS:16,
  title = 	 {Train faster, generalize better: Stability of stochastic gradient descent},
  author = 	 {Hardt, Moritz and Recht, Ben and Singer, Yoram},
  booktitle = 	 {Proceedings of The 33rd International Conference on Machine Learning},
  pages = 	 {1225--1234},
  year = 	 {2016},
  volume = 	 {48},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {20--22 Jun},
  publisher =    {PMLR}
}


@InProceedings{IK-CL:18,
  title = 	 {Data-Dependent Stability of Stochastic Gradient Descent},
  author = {Kuzborskij, Ilja and Lampert, Christoph},
  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
  pages = 	 {2815--2824},
  year = 	 {2018},
  volume = 	 {80},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {10--15 Jul},
  publisher =    {PMLR}
}

@inproceedings{AJ-FG-CH:18,
 author = {Jacot, Arthur and Gabriel, Franck and Hongler, Clement},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Neural Tangent Kernel: Convergence and Generalization in Neural Networks},
 volume = {31},
 year = {2018}
}

@inproceedings{JL-LX-SS-YB-RN-JSD-JP:19,
 author = {Lee, Jaehoon and Xiao, Lechao and Schoenholz, Samuel and Bahri, Yasaman and Novak, Roman and Sohl-Dickstein, Jascha and Pennington, Jeffrey},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Wide Neural Networks of Any Depth Evolve as Linear Models Under Gradient Descent},
 volume = {32},
 year = {2019}
}


@InProceedings{SS-RS:21,
  title = 	 {The Dynamics of Gradient Descent for Overparametrized Neural Networks},
  author =       {Satpathi, Siddhartha and Srikant, R},
  booktitle = 	 {Proceedings of the 3rd Conference on Learning for Dynamics and Control},
  pages = 	 {373--384},
  year = 	 {2021},
  volume = 	 {144},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {07 -- 08 June},
  publisher =    {PMLR},
}

@inproceedings{RK-XW-HL-YZ-ZL-WH-RG-SA:19,
 author = {Kuditipudi, Rohith and Wang, Xiang and Lee, Holden and Zhang, Yi and Li, Zhiyuan and Hu, Wei and Ge, Rong and Arora, Sanjeev},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Explaining Landscape Connectivity of Low-cost Solutions for Multilayer Nets},
 volume = {32},
 year = {2019}
}

@ARTICLE{CF-HD-TZ:21,
  author={Fang, Cong and Dong, Hanze and Zhang, Tong},
  journal={Proceedings of the IEEE}, 
  title={Mathematical Models of Overparameterized Neural Networks}, 
  year={2021},
  volume={109},
  number={5},
  pages={683-703}
  }
  
@article{JG-MG-PEP-MM-TG:21,
  title =	 {Stochastic Training is Not Necessary for Generalization},
  author =	 {Geiping, Jonas and  Goldblum, Micah and Pope, Phillip E. and Moeller, Michael and  Goldstein, Tom},
  journal =	 {arXiv},
  year =	 2021
} 
 
@inproceedings{DZ-QG:19,
 author = {Zou, Difan and Gu, Quanquan},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {An Improved Analysis of Training Over-parameterized Deep Neural Networks},
 volume = {32},
 year = {2019}
}

@article{JF-CM-YZ:21,
author = {Jianqing Fan and Cong Ma and Yiqiao Zhong},
title = {A Selective Overview of Deep Learning},
volume = {36},
journal = {Statistical Science},
number = {2},
publisher = {Institute of Mathematical Statistics},
pages = {264 -- 290},
year = {2021}
}


@article{PB-AM-AR:21, title={Deep learning: a statistical viewpoint}, volume={30},
journal={Acta Numerica}, publisher={Cambridge University Press}, author={Bartlett, Peter L. and Montanari, Andrea and Rakhlin, Alexander}, year={2021}, pages={87–201}
}

@inproceedings{SA-SSD-WH-ZL-RS-RW:19,
 author = {Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and Salakhutdinov, Russ R and Wang, Ruosong},
 booktitle = {Advances in Neural Information Processing Systems},
 publisher = {Curran Associates, Inc.},
 title = {On Exact Computation with an Infinitely Wide Neural Net},
 volume = {32},
 year = {2019}
}

@inproceedings{RV:12, place={Cambridge}, title={Introduction to the non-asymptotic analysis of random matrices},
booktitle={Compressed Sensing: Theory and Applications}, publisher={Cambridge University Press}, author={Vershynin, Roman}, year={2012}, pages={210–268}
}

@article{LC-EO-FB:19,
  title={On lazy training in differentiable programming},
  author={Chizat, Lenaic and Oyallon, Edouard and Bach, Francis},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}



@article{MG-SS-AJ-MW:20,
  title={Disentangling feature and lazy training in deep neural networks},
  author={Geiger, Mario and Spigler, Stefano and Jacot, Arthur and Wyart, Matthieu},
  journal={Journal of Statistical Mechanics: Theory and Experiment},
  volume={2020},
  number={11},
  pages={113301},
  year={2020},
  publisher={IOP Publishing}
}

@article{GY-EJH:20,
  title={Feature learning in infinite-width neural networks},
  author={Yang, Greg and Hu, Edward J},
  journal={arXiv preprint arXiv:2011.14522},
  year={2020}
}


@article{SF-GKD-MP-SK-DR-SG:20,
  title={Deep learning versus kernel learning: an empirical study of loss landscape geometry and the time evolution of the neural tangent kernel},
  author={Fort, Stanislav and Dziugaite, Gintare Karolina and Paul, Mansheej and Kharaghani, Sepideh and Roy, Daniel M and Ganguli, Surya},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={5850--5861},
  year={2020}
}


@inproceedings{JH-HY:20,
  title={Dynamics of deep neural networks and neural tangent hierarchy},
  author={Huang, Jiaoyang and Yau, Horng-Tzer},
  booktitle={International conference on machine learning},
  pages={4542--4551},
  year={2020},
  organization={PMLR}
}


@inproceedings{ZJ-MT:19,
  title={Polylogarithmic width suffices for gradient descent to achieve arbitrarily small test error with shallow ReLU networks},
  author={Ji, Ziwei and Telgarsky, Matus},
  booktitle={International Conference on Learning Representations},
  year={2019}
}



@inproceedings{SA-SD-WH-ZL-RW:19,
  title={Fine-grained analysis of optimization and generalization for overparameterized two-layer neural networks},
  author={Arora, Sanjeev and Du, Simon and Hu, Wei and Li, Zhiyuan and Wang, Ruosong},
  booktitle={International Conference on Machine Learning},
  pages={322--332},
  year={2019},
  organization={PMLR}
}



@article{AM-YZ:20,
  title={The interpolation phase transition in neural networks: Memorization and generalization under lazy training},
  author={Montanari, Andrea and Zhong, Yiqiao},
  journal={arXiv preprint arXiv:2007.12826},
  year={2020}
}


@article{JZ-SS-JP-BA-LX-RN-JSD:20,
  title={Finite versus infinite neural networks: an empirical study},
  author={Lee, Jaehoon and Schoenholz, Samuel and Pennington, Jeffrey and Adlam, Ben and Xiao, Lechao and Novak, Roman and Sohl-Dickstein, Jascha},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={15156--15172},
  year={2020}
}

@ARTICLE{xsede,
author = {J. Towns and T. Cockerill and M. Dahan and I. Foster and K. Gaither and A. Grimshaw and V. Hazlewood and S. Lathrop and D. Lifka and G. D. Peterson and R. Roskies and J. R. Scott and N. Wilkins-Diehr},
journal = {Computing in Science \& Engineering},
title = {XSEDE: Accelerating Scientific Discovery},
year = {2014},
volume = {16},
number = {5},
pages = {62-74},
keywords={Knowledge discovery;Scientific computing;Digital systems;Materials engineering;Supercomputers},
doi = {10.1109/MCSE.2014.80},
url = {doi.ieeecomputersociety.org/10.1109/MCSE.2014.80},
ISSN = {1521-9615}
}



@inproceedings{
bombari2022memorization,
title={Memorization and Optimization in Deep Neural Networks with Minimum Over-parameterization},
author={Simone Bombari and Mohammad Hossein Amani and Marco Mondelli},
booktitle={Advances in Neural Information Processing Systems},
year={2022}
}