% Encoding: UTF-8

@Article{Samuel59,
  author  = {A. L. Samuel},
  title   = {Some Studies in Machine Learning Using the Game of Checkers},
  journal = {IBM Journal of Research and Development},
  year    = {1959},
  volume  = {3},
  number  = {3},
  pages   = {211--229},
}

@article{hochreiter1997flat,
  title={Flat minima},
  author={Hochreiter, Sepp and Schmidhuber, J{\"u}rgen},
  journal={Neural computation},
  year={1997},
  publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…}
}

@article{
    keskar2017on,
    title={On Large-Batch Training for Deep Learning: Generalization Gap and Sharp Minima},
    author={Nitish Shirish Keskar and Dheevatsa Mudigere and Jorge Nocedal and Mikhail Smelyanskiy and Ping Tak Peter Tang},
    journal={ICLR},
    year={2017}
}

@article{foret2020sharpness,
  title={Sharpness-aware minimization for efficiently improving generalization},
  author={Foret, Pierre and Kleiner, Ariel and Mobahi, Hossein and Neyshabur, Behnam},
  journal={ICLR},
  year={2021}
}

@article{izmailov2018averaging,
  title={Averaging weights leads to wider optima and better generalization},
  author={Izmailov, P and Wilson, AG and Podoprikhin, D and Vetrov, D and Garipov, T},
  journal={UAI},
  year={2018}
}

@article{orvieto2022anticorrelated,
  title={Anticorrelated Noise Injection for Improved Generalization},
  author={Orvieto, Antonio and Kersting, Hans and Proske, Frank and Bach, Francis and Lucchi, Aurelien},
  journal={ICML},
  year={2022}
}

@article{
    chaudhari2017entropysgd,
    title={Entropy-{SGD}: Biasing Gradient Descent Into Wide Valleys},
    author={Pratik Chaudhari and Anna Choromanska and Stefano Soatto and Yann LeCun and Carlo Baldassi and Christian Borgs and Jennifer Chayes and Levent Sagun and Riccardo Zecchina},
    journal={ICLR},
    year={2017}
}

@article{zhao2022penalizing,
  title={Penalizing gradient norm for efficiently improving generalization in deep learning},
  author={Zhao, Yang and Zhang, Hao and Hu, Xiuyuan},
  journal={ICML},
  year={2022},
}

@article{dauphin2024neglected,
  title={Neglected Hessian component explains mysteries in Sharpness regularization},
  author={Dauphin, Yann N and Agarwala, Atish and Mobahi, Hossein},
  journal={arXiv},
  year={2024}
}

@article{andriushchenko2022towards,
  title={Towards understanding sharpness-aware minimization},
  author={Andriushchenko, Maksym and Flammarion, Nicolas},
  journal={ICML},
  year={2022}
}

@article{kaddour2022flat,
  title={When do flat minima optimizers work?},
  author={Kaddour, Jean and Liu, Linqing and Silva, Ricardo and Kusner, Matt J},
  journal={NeurIPS},
  year={2022}
}

@article{wensharpness,
  title={How Sharpness-Aware Minimization Minimizes Sharpness?},
  author={Wen, Kaiyue and Ma, Tengyu and Li, Zhiyuan},
  journal={ICLR},
  year={2023}
}

@article{kim2023stability,
  title={Stability Analysis of Sharpness-Aware Minimization},
  author={Kim, Hoki and Park, Jinseong and Choi, Yujin and Lee, Jaewook},
  journal={arXiv preprint arXiv:2301.06308},
  year={2023}
}

@article{leesnip,
  title={Snip: Single-shot network pruning based on connection sensitivity},
  author={Lee, Namhoon and Ajanthan, Thalaiyasingam and Torr, Philip},
  journal={ICLR},
  year={2019}
}

@article{jacot2018neural,
  title={Neural tangent kernel: Convergence and generalization in neural networks},
  author={Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
  journal={NeurIPS},
  year={2018}
}

@article{telgarsky2016benefits,
  title={Benefits of depth in neural networks},
  author={Telgarsky, Matus},
  journal={COLT},
  year={2016},
  organization={PMLR}
}

@article{chenvision,
  title={When Vision Transformers Outperform ResNets without Pre-training or Strong Data Augmentations},
  author={Chen, Xiangning and Hsieh, Cho-Jui and Gong, Boqing},
  journal={ICLR},
  year={2022}
}

@article{bahri2022sharpness,
  title={Sharpness-Aware Minimization Improves Language Model Generalization},
  author={Bahri, Dara and Mobahi, Hossein and Tay, Yi},
  journal={ACL},
  year={2022}
}

@article{kleinberg2018alternative,
  title={An alternative view: When does SGD escape local minima?},
  author={Kleinberg, Bobby and Li, Yuanzhi and Yuan, Yang},
  journal={ICML},
  year={2018},
  organization={PMLR}
}

@article{smithbayesian,
  title={A Bayesian Perspective on Generalization and Stochastic Gradient Descent},
  author={Smith, Samuel L and Le, Quoc V},
  journal={ICLR},
  year={2018}
}

@article{belkin2019reconciling,
  title={Reconciling modern machine-learning practice and the classical bias--variance trade-off},
  author={Belkin, Mikhail and Hsu, Daniel and Ma, Siyuan and Mandal, Soumik},
  journal={PNAS},
  year={2019},
  publisher={National Acad Sciences}
}

@article{bartlett2020benign,
  title={Benign overfitting in linear regression},
  author={Bartlett, Peter L and Long, Philip M and Lugosi, G{\'a}bor and Tsigler, Alexander},
  journal={PNAS},
  year={2020},
  publisher={National Acad Sciences}
}

@article{liu2022loss,
  title={Loss landscapes and optimization in over-parameterized non-linear systems and neural networks},
  author={Liu, Chaoyue and Zhu, Libin and Belkin, Mikhail},
  journal={ACHA},
  year={2022}
}

@article{simsek2021geometry,
  title={Geometry of the loss landscape in overparameterized neural networks: Symmetries and invariances},
  author={Simsek, Berfin and Ged, Fran{\c{c}}ois and Jacot, Arthur and Spadaro, Francesco and Hongler, Cl{\'e}ment and Gerstner, Wulfram and Brea, Johanni},
  journal={ICML},
  year={2021},
  organization={PMLR}
}

@article{juneja2022linear,
  title={Linear Connectivity Reveals Generalization Strategies},
  author={Juneja, Jeevesh and Bansal, Rachit and Cho, Kyunghyun and Sedoc, Jo{\~a}o and Saphra, Naomi},
  journal={Workshop on Spurious Correlations, Invariance and Stability, ICML},
  year={2022}
}

@article{garipov2018loss,
  title={Loss surfaces, mode connectivity, and fast ensembling of dnns},
  author={Garipov, Timur and Izmailov, Pavel and Podoprikhin, Dmitrii and Vetrov, Dmitry P and Wilson, Andrew G},
  journal={NeurIPS},
  year={2018}
}

@article{lecun1989optimal,
  title={Optimal brain damage},
  author={LeCun, Yann and Denker, John and Solla, Sara},
  journal={NeurIPS},
  year={1989}
}

@article{hoefler2021sparsity,
  title={Sparsity in deep learning: Pruning and growth for efficient inference and training in neural networks},
  author={Hoefler, Torsten and Alistarh, Dan and Ben-Nun, Tal and Dryden, Nikoli and Peste, Alexandra},
  journal={JMLR},
  year={2021},
  publisher={JMLRORG}
}

@article{krizhevsky2009learning,
  title={Learning multiple layers of features from tiny images},
  author={Krizhevsky, Alex and Hinton, Geoffrey and others},
  year={2009},
  publisher={Toronto, ON, Canada}
}

@article{lecun2010mnist,
  title={MNIST handwritten digit database},
  author={LeCun, Yann and Cortes, Corinna and Burges, CJ},
  journal={ATT Labs},
  year={2010}
}

@article{leeunderstanding,
  title={Understanding the effects of data parallelism and sparsity on neural network training},
  author={Lee, Namhoon and Ajanthan, Thalaiyasingam and Torr, Philip and Jaggi, Martin},
  journal={ICLR},
  year={2021}
}

@article{he2022sparse,
  title={Sparse Double Descent: Where Network Pruning Aggravates Overfitting},
  author={He, Zheng and Xie, Zeke and Zhu, Quanzhi and Qin, Zengchang},
  journal={ICML},
  year={2022},
  organization={PMLR}
}

@article{lubanagradient,
  title={A Gradient Flow Framework For Analyzing Network Pruning},
  author={Lubana, Ekdeep Singh and Dick, Robert},
  journal={ICLR},
  year={2021}
}

@article{wangpicking,
  title={Picking Winning Tickets Before Training by Preserving Gradient Flow},
  author={Wang, Chaoqi and Zhang, Guodong and Grosse, Roger},
  journal={ICLR},
  year={2020}
}

@article{tanaka2020pruning,
  title={Pruning neural networks without any data by iteratively conserving synaptic flow},
  author={Tanaka, Hidenori and Kunin, Daniel and Yamins, Daniel L and Ganguli, Surya},
  journal={NeurIPS},
  year={2020}
}

@article{franklelottery,
  title={The Lottery Ticket Hypothesis: Finding Sparse, Trainable Neural Networks},
  author={Frankle, Jonathan and Carbin, Michael},
  journal={ICLR},
  year={2019}
}

@article{li2018visualizing,
  title={Visualizing the loss landscape of neural nets},
  author={Li, Hao and Xu, Zheng and Taylor, Gavin and Studer, Christoph and Goldstein, Tom},
  journal={NeurIPS},
  year={2018}
}

@article{lee2019signal,
  title={A signal propagation perspective for pruning neural networks at initialization},
  author={Lee, Namhoon and Ajanthan, Thalaiyasingam and Gould, Stephen and Torr, Philip HS},
  journal={ICLR},
  year={2020}
}

@article{kalchbrenner2018efficient,
  title={Efficient neural audio synthesis},
  author={Kalchbrenner, Nal and Elsen, Erich and Simonyan, Karen and Noury, Seb and Casagrande, Norman and Lockhart, Edward and Stimberg, Florian and Oord, Aaron and Dieleman, Sander and Kavukcuoglu, Koray},
  journal={ICML},
  year={2018},
}

@article{zhang2022opt,
  title={Opt: Open pre-trained transformer language models},
  author={Zhang, Susan and Roller, Stephen and Goyal, Naman and Artetxe, Mikel and Chen, Moya and Chen, Shuohui and Dewan, Christopher and Diab, Mona and Li, Xian and Lin, Xi Victoria and others},
  journal={arXiv preprint arXiv:2205.01068},
  year={2022}
}

@article{muthukumar2020harmless,
  title={Harmless interpolation of noisy data in regression},
  author={Muthukumar, Vidya and Vodrahalli, Kailas and Subramanian, Vignesh and Sahai, Anant},
  journal={IEEE JSAIT},
  year={2020},
  publisher={IEEE}
}

@article{bommasani2021opportunities,
  title={On the opportunities and risks of foundation models},
  author={Bommasani, Rishi and Hudson, Drew A and Adeli, Ehsan and Altman, Russ and Arora, Simran and von Arx, Sydney and Bernstein, Michael S and Bohg, Jeannette and Bosselut, Antoine and Brunskill, Emma and others},
  journal={arXiv preprint arXiv:2108.07258},
  year={2021}
}

@article{wu2022alignment,
  title={The alignment property of SGD noise and how it helps select flat minima: A stability analysis},
  author={Wu, Lei and Wang, Mingze and Su, Weijie},
  journal={NeurIPS},
  year={2022}
}

@article{ma2018power,
  title={The power of interpolation: Understanding the effectiveness of SGD in modern over-parametrized learning},
  author={Ma, Siyuan and Bassily, Raef and Belkin, Mikhail},
  journal={ICML},
  year={2018},
}

@article{bassily2018exponential,
  title={On exponential convergence of sgd in non-convex over-parametrized learning},
  author={Bassily, Raef and Belkin, Mikhail and Ma, Siyuan},
  journal={arXiv preprint arXiv:1811.02564},
  year={2018}
}

@article{wu2018sgd,
  title={How sgd selects the global minima in over-parameterized learning: A dynamical stability perspective},
  author={Wu, Lei and Ma, Chao and others},
  journal={NeurIPS},
  year={2018}
}

@article{agarwala2023sam,
  title={SAM operates far from home: eigenvalue regularization as a dynamical phenomenon},
  author={Agarwala, Atish and Dauphin, Yann},
  journal={ICML},
  year={2023},
}

@article{si2023practical,
  title={Practical Sharpness-Aware Minimization Cannot Converge All the Way to Optima},
  author={Si, Dongkuk and Yun, Chulhee},
  journal={NeurIPS},
  year={2023}
}

@article{zhuang2021surrogate,
  title={Surrogate Gap Minimization Improves Sharpness-Aware Training},
  author={Zhuang, Juntang and Gong, Boqing and Yuan, Liangzhe and Cui, Yin and Adam, Hartwig and Dvornek, Nicha C and s Duncan, James and Liu, Ting and others},
  journal={ICLR},
  year={2022}
}

@article{mi2022make,
  title={Make sharpness-aware minimization stronger: A sparsified perturbation approach},
  author={Mi, Peng and Shen, Li and Ren, Tianhe and Zhou, Yiyi and Sun, Xiaoshuai and Ji, Rongrong and Tao, Dacheng},
  journal={NeurIPS},
  year={2022}
}

@article{jiang2022adaptive,
  title={An Adaptive Policy to Employ Sharpness-Aware Minimization},
  author={Jiang, Weisen and Yang, Hansi and Zhang, Yu and Kwok, James},
  journal={ICLR},
  year={2022}
}

@article{sun2023adasam,
  title={Adasam: Boosting sharpness-aware minimization with adaptive learning rate and momentum for training deep neural networks},
  author={Sun, Hao and Shen, Li and Zhong, Qihuang and Ding, Liang and Chen, Shixiang and Sun, Jingwei and Li, Jing and Sun, Guangzhong and Tao, Dacheng},
  journal={Neural Networks},
  year={2024}
}

@article{belkin2021fit,
  title={Fit without fear: remarkable mathematical phenomena of deep learning through the prism of interpolation},
  author={Belkin, Mikhail},
  journal={Acta Numerica},
  year={2021},
  publisher={Cambridge University Press}
}

@article{polyak1964gradient,
  title={Gradient methods for solving equations and inequalities},
  author={Polyak, Boris},
  journal={USSR Comput. Math. \& Math. Phys.},
  year={1964},
}

@article{karimi2016linear,
  title={Linear convergence of gradient and proximal-gradient methods under the polyak-${\l}$ojasiewicz condition},
  author={Karimi, Hamed and Nutini, Julie and Schmidt, Mark},
  journal={ECML-PKDD},
  year={2016},
}

@article{dai2023crucial,
  title={The Crucial Role of Normalization in Sharpness-Aware Minimization},
  author={Dai, Yan and Ahn, Kwangjun and Sra, Suvrit},
  journal={NeurIPS},
  year={2023}
}

@article{compagnoni2023sde,
  title={An sde for modeling sam: Theory and insights},
  author={Compagnoni, Enea Monzio and Biggio, Luca and Orvieto, Antonio and Proske, Frank Norbert and Kersting, Hans and Lucchi, Aurelien},
  journal={ICML},
  year={2023},
}

@article{jiang2019fantastic,
  title={Fantastic generalization measures and where to find them},
  author={Jiang, Yiding and Neyshabur, Behnam and Mobahi, Hossein and Krishnan, Dilip and Bengio, Samy},
  journal={ICLR},
  year={2020},
}

@article{ma2021linear,
  title={On linear stability of sgd and input-smoothness of neural networks},
  author={Ma, Chao and Ying, Lexing},
  journal={NeurIPS},
  year={2021}
}

@article{laurent2018deep,
  title={Deep linear networks with arbitrary loss: All local minima are global},
  author={Laurent, Thomas and Brecht, James},
  journal={ICML},
  year={2018},
}

@article{zhang2017understanding,
  title={Understanding deep learning requires rethinking generalization},
  author={Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
  journal={ICLR},
  year={2017},
}

@article{cohen2020gradient,
  title={Gradient Descent on Neural Networks Typically Occurs at the Edge of Stability},
  author={Cohen, Jeremy and Kaur, Simran and Li, Yuanzhi and Kolter, J Zico and Talwalkar, Ameet},
  journal={ICLR},
  year={2021}
}

@article{woodworth2020kernel,
  title={Kernel and rich regimes in overparametrized models},
  author={Woodworth, Blake and Gunasekar, Suriya and Lee, Jason D and Moroshko, Edward and Savarese, Pedro and Golan, Itay and Soudry, Daniel and Srebro, Nathan},
  journal={COLT},
  year={2020},
}

@article{bottou2018optimization,
  title={Optimization methods for large-scale machine learning},
  author={Bottou, L{\'e}on and Curtis, Frank E and Nocedal, Jorge},
  journal={SIAM review},
  year={2018},
}
@article{ghadimi2013stochastic,
  title={Stochastic first-and zeroth-order methods for nonconvex stochastic programming},
  author={Ghadimi, Saeed and Lan, Guanghui},
  journal={SIAM Journal on Optimization},
  year={2013},
}

@article{loizou2021stochastic,
  title={Stochastic polyak step-size for sgd: An adaptive learning rate for fast convergence},
  author={Loizou, Nicolas and Vaswani, Sharan and Laradji, Issam Hadj and Lacoste-Julien, Simon},
  journal={AISTATS},
  year={2021},
}

@article{lojasiewicz1963topological,
  title={A topological property of real analytic subsets},
  author={Lojasiewicz, Stanislaw},
  journal={Coll. du CNRS, Les {\'e}quations aux d{\'e}riv{\'e}es partielles},
  year={1963}
}

@article{dziugaite2017computing,
  title={Computing nonvacuous generalization bounds for deep (stochastic) neural networks with many more parameters than training data},
  author={Dziugaite, Gintare Karolina and Roy, Daniel M},
  journal={UAI},
  year={2017}
}

@article{neyshabur2017exploring,
  title={Exploring generalization in deep learning},
  author={Neyshabur, Behnam and Bhojanapalli, Srinadh and McAllester, David and Srebro, Nati},
  journal={NeurIPS},
  year={2017}
}

@article{wei2019data,
  title={Data-dependent sample complexity of deep neural networks via lipschitz augmentation},
  author={Wei, Colin and Ma, Tengyu},
  journal={NeurIPS},
  year={2019}
}

@article{kwon2021asam,
  title={Asam: Adaptive sharpness-aware minimization for scale-invariant learning of deep neural networks},
  author={Kwon, Jungmin and Kim, Jeongseop and Park, Hyunseo and Choi, In Kwon},
  journal={ICML},
  year={2021},
}

@article{du2021efficient,
  title={Efficient Sharpness-aware Minimization for Improved Training of Neural Networks},
  author={Du, Jiawei and Yan, Hanshu and Feng, Jiashi and Zhou, Joey Tianyi and Zhen, Liangli and Goh, Rick Siow Mong and Tan, Vincent},
  journal={ICLR},
  year={2022}
}

% all local minima are global

@article{gupta2015deep,
  title={Deep learning with limited numerical precision},
  author={Gupta, Suyog and Agrawal, Ankur and Gopalakrishnan, Kailash and Narayanan, Pritish},
  journal={ICML},
  year={2015},
}

@article{kaplan2020scaling,
  title={Scaling laws for neural language models},
  author={Kaplan, Jared and McCandlish, Sam and Henighan, Tom and Brown, Tom B and Chess, Benjamin and Child, Rewon and Gray, Scott and Radford, Alec and Wu, Jeffrey and Amodei, Dario},
  journal={arXiv preprint arXiv:2001.08361},
  year={2020}
}

@article{dehghani2023scaling,
  title={Scaling vision transformers to 22 billion parameters},
  author={Dehghani, Mostafa and Djolonga, Josip and Mustafa, Basil and Padlewski, Piotr and Heek, Jonathan and Gilmer, Justin and Steiner, Andreas Peter and Caron, Mathilde and Geirhos, Robert and Alabdulmohsin, Ibrahim and others},
  journal={ICML},
  year={2023},
}

@article{sagun2017empirical,
  title={Empirical analysis of the hessian of over-parametrized neural networks},
  author={Sagun, Levent and Evci, Utku and Guney, V Ugur and Dauphin, Yann and Bottou, Leon},
  journal={ICLR Workshop Track},
  year={2018}
}

@article{neyshabur2018towards,
  title={Towards understanding the role of over-parametrization in generalization of neural networks},
  author={Neyshabur, Behnam and Li, Zhiyuan and Bhojanapalli, Srinadh and LeCun, Yann and Srebro, Nathan},
  journal={ICLR},
  year={2019}
}

@article{du2018power,
  title={On the power of over-parametrization in neural networks with quadratic activation},
  author={Du, Simon and Lee, Jason},
  journal={ICML},
  year={2018},
}

@article{kawaguchi2016deep,
  title={Deep learning without poor local minima},
  author={Kawaguchi, Kenji},
  journal={NeurIPS},
  year={2016}
}

@article{lee2021vision,
  title={Vision transformer for small-size datasets},
  author={Lee, Seung Hoon and Lee, Seunghyun and Song, Byung Cheol},
  journal={arXiv preprint arXiv:2112.13492},
  year={2021}
}

@article{chen2022transmix,
  title={Transmix: Attend to mix for vision transformers},
  author={Chen, Jie-Neng and Sun, Shuyang and He, Ju and Torr, Philip HS and Yuille, Alan and Bai, Song},
  journal={CVPR},
  year={2022}
}

@article{liu2023ten,
  title={Ten lessons we have learned in the new" sparseland": A short handbook for sparse neural network researchers},
  author={Liu, Shiwei and Wang, Zhangyang},
  journal={arXiv preprint arXiv:2302.02596},
  year={2023}
}

@article{zhou2021learning,
  title={Learning n: m fine-grained structured sparse neural networks from scratch},
  author={Zhou, Aojun and Ma, Yukun and Zhu, Junnan and Liu, Jianbo and Zhang, Zhijie and Yuan, Kun and Sun, Wenxiu and Li, Hongsheng},
  journal={ICLR},
  year={2021}
}

@article{lyu2019gradient,
  title={Gradient descent maximizes the margin of homogeneous neural networks},
  author={Lyu, Kaifeng and Li, Jian},
  journal={ICLR},
  year={2020}
}

@article{xie2020diffusion,
  title={A diffusion theory for deep learning dynamics: Stochastic gradient descent exponentially favors flat minima},
  author={Xie, Zeke and Sato, Issei and Sugiyama, Masashi},
  journal={ICLR},
  year={2021}
}

@article{scao2022bloom,
  title={Bloom: A 176b-parameter open-access multilingual language model},
  author={Scao, Teven Le and Fan, Angela and Akiki, Christopher and Pavlick, Ellie and Ili{\'c}, Suzana and Hesslow, Daniel and Castagn{\'e}, Roman and Luccioni, Alexandra Sasha and Yvon, Fran{\c{c}}ois and Gall{\'e}, Matthias and others},
  journal={arXiv preprint arXiv:2211.05100},
  year={2022}
}

@article{gholami2022survey,
  title={A survey of quantization methods for efficient neural network inference},
  author={Gholami, Amir and Kim, Sehoon and Dong, Zhen and Yao, Zhewei and Mahoney, Michael W and Keutzer, Kurt},
  journal={Low-Power Computer Vision},
  year={2022},
}

@article{packwood2011moments,
  title={Moments of sums of independent and identically distributed random variables},
  author={Packwood, Daniel M},
  journal={arXiv preprint arXiv:1105.6283},
  year={2011}
}

@article{vardi2023implicit,
  title={On the implicit bias in deep-learning algorithms},
  author={Vardi, Gal},
  journal={Communications of the ACM},
  year={2023},
}

@article{neyshabur2017implicit,
  title={Implicit regularization in deep learning},
  author={Neyshabur, Behnam},
  journal={arXiv preprint arXiv:1709.01953},
  year={2017}
}

@article{allen2019convergence,
  title={A convergence theory for deep learning via over-parameterization},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  journal={ICML},
  year={2019},
}

@article{dinh2017sharp,
  title={Sharp minima can generalize for deep nets},
  author={Dinh, Laurent and Pascanu, Razvan and Bengio, Samy and Bengio, Yoshua},
  journal={ICML},
  year={2017},
}

@article{andriushchenko2022modern,
  title={A Modern Look at the Relationship between Sharpness and Generalization},
  author={Andriushchenko, Maksym and Croce, Francesco and M{\"u}ller, Maximilian and Hein, Matthias and Flammarion, Nicolas},
  journal={ICML},
  year={2023},
}

@article{arora2022understanding,
  title={Understanding gradient descent on the edge of stability in deep learning},
  author={Arora, Sanjeev and Li, Zhiyuan and Panigrahi, Abhishek},
  journal={ICML},
  year={2022},
}

@article{qu2022generalized,
  title={Generalized federated learning via sharpness aware minimization},
  author={Qu, Zhe and Li, Xingyu and Duan, Rui and Liu, Yao and Tang, Bo and Lu, Zhuo},
  journal={ICML},
  year={2022},
}

@article{chizat2019lazy,
  title={On lazy training in differentiable programming},
  author={Chizat, Lenaic and Oyallon, Edouard and Bach, Francis},
  journal={NeurIPS},
  year={2019}
}

@article{wang2022generalization,
  title={Generalization error bounds for deep neural networks trained by sgd},
  author={Wang, Mingze and Ma, Chao},
  journal={arXiv preprint arXiv:2206.03299},
  year={2022}
}

@article{behdin2023msam,
  title={mSAM: Micro-Batch-Averaged Sharpness-Aware Minimization},
  author={Behdin, Kayhan and Song, Qingquan and Gupta, Aman and Acharya, Ayan and Durfee, David and Ocejo, Borja and Keerthi, Sathiya and Mazumder, Rahul},
  journal={arXiv preprint arXiv:2302.09693},
  year={2023}
}

@article{angluin1988learning,
  title={Learning from noisy examples},
  author={Angluin, Dana and Laird, Philip},
  journal={Machine learning},
  year={1988},
}
@article{natarajan2013learning,
  title={Learning with noisy labels},
  author={Natarajan, Nagarajan and Dhillon, Inderjit S and Ravikumar, Pradeep K and Tewari, Ambuj},
  journal={NeurIPS},
  year={2013}
}

@article{allen2019learning,
  title={Learning and generalization in overparameterized neural networks, going beyond two layers},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Liang, Yingyu},
  journal={NeurIPS},
  year={2019}
}

@article{hu2020open,
  title={Open graph benchmark: Datasets for machine learning on graphs},
  author={Hu, Weihua and Fey, Matthias and Zitnik, Marinka and Dong, Yuxiao and Ren, Hongyu and Liu, Bowen and Catasta, Michele and Leskovec, Jure},
  journal={NeurIPS},
  year={2020}
}

@article{kipf2016semi,
  title={Semi-supervised classification with graph convolutional networks},
  author={Kipf, Thomas N and Welling, Max},
  journal={ICLR},
  year={2017}
}

@article{hochreiter1997long,
  title={Long short-term memory},
  author={Hochreiter, Sepp and Schmidhuber, J{\"u}rgen},
  journal={Neural computation},
  year={1997},
}

@article{socher2013recursive,
  title={Recursive deep models for semantic compositionality over a sentiment treebank},
  author={Socher, Richard and Perelygin, Alex and Wu, Jean and Chuang, Jason and Manning, Christopher D and Ng, Andrew Y and Potts, Christopher},
  journal={EMNLP},
  year={2013}
}

@article{dosovitskiy2020image,
  title={An image is worth 16x16 words: Transformers for image recognition at scale},
  author={Dosovitskiy, Alexey and Beyer, Lucas and Kolesnikov, Alexander and Weissenborn, Dirk and Zhai, Xiaohua and Unterthiner, Thomas and Dehghani, Mostafa and Minderer, Matthias and Heigold, Georg and Gelly, Sylvain and others},
  journal={ICLR},
  year={2021}
}

@article{lecun1998gradient,
  title={Gradient-based learning applied to document recognition},
  author={LeCun, Yann and Bottou, L{\'e}on and Bengio, Yoshua and Haffner, Patrick},
  journal={Proceedings of the IEEE},
  year={1998},
}

@article{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  journal={CVPR},
  year={2016}
}

@article{deng2009imagenet,
  title={Imagenet: A large-scale hierarchical image database},
  author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Li, Kai and Fei-Fei, Li},
  journal={CVPR},
  year={2009},
}

@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={NeurIPS},
  year={2017}
}

@article{nivre2016universal,
  title={Universal dependencies v1: A multilingual treebank collection},
  author={Nivre, Joakim and De Marneffe, Marie-Catherine and Ginter, Filip and Goldberg, Yoav and Hajic, Jan and Manning, Christopher D and McDonald, Ryan and Petrov, Slav and Pyysalo, Sampo and Silveira, Natalia and others},
  journal={LREC},
  year={2016}
}

@article{schulman2017proximal,
  title={Proximal policy optimization algorithms},
  author={Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
  journal={arXiv preprint arXiv:1707.06347},
  year={2017}
}

@article{konda1999actor,
  title={Actor-critic algorithms},
  author={Konda, Vijay and Tsitsiklis, John},
  journal={NeurIPS},
  year={1999}
}

@article{mnih2013playing,
  title={Playing atari with deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},
  journal={Workshop on Deep Learning, NeurIPS},
  year={2013}
}

@article{huang2022cleanrl,
  author  = {Shengyi Huang and Rousslan Fernand Julien Dossa and Chang Ye and Jeff Braga and Dipam Chakraborty and Kinal Mehta and João G.M. Araújo},
  title   = {CleanRL: High-quality Single-file Implementations of Deep Reinforcement Learning Algorithms},
  journal = {JMLR},
  year    = {2022},
}

@software{flax2020github,
  author = {Jonathan Heek and Anselm Levskaya and Avital Oliver and Marvin Ritter and Bertrand Rondepierre and Andreas Steiner and Marc van {Z}ee},
  title = {{F}lax: A neural network library and ecosystem for {JAX}},
  year = {2023},
}

@article{liu2019accelerating,
  title={Accelerating SGD with momentum for over-parameterized learning},
  author={Liu, Chaoyue and Belkin, Mikhail},
  journal={International Conference on Learning Representations},
  year={2019}
}

@article{vaswani2019fast,
  title={Fast and faster convergence of sgd for over-parameterized models and an accelerated perceptron},
  author={Vaswani, Sharan and Bach, Francis and Schmidt, Mark},
  journal={AISTATS},
  year={2019},
}

@article{meng2020fast,
  title={Fast and furious convergence: Stochastic second order methods under interpolation},
  author={Meng, Si Yi and Vaswani, Sharan and Laradji, Issam Hadj and Schmidt, Mark and Lacoste-Julien, Simon},
  journal={AISTATS},
  year={2020},
}

@article{vaswani2019painless,
  title={Painless stochastic gradient: Interpolation, line-search, and convergence rates},
  author={Vaswani, Sharan and Mishkin, Aaron and Laradji, Issam and Schmidt, Mark and Gidel, Gauthier and Lacoste-Julien, Simon},
  journal={NeurIPS},
  year={2019}
}

@article{varre2021last,
  title={Last iterate convergence of SGD for Least-Squares in the Interpolation regime.},
  author={Varre, Aditya Vardhan and Pillaud-Vivien, Loucas and Flammarion, Nicolas},
  journal={NeurIPS},
  year={2021}
}

@article{advani2020highdi,
  title={High-dimensional dynamics of generalization error in neural networks},
  author={Advani, Madhu S. and Saxe, Andrew M. and Sompolinsky, Haim},
  journal={Neural Networks},
  year={2020}
}

@article{zhang2019which,
  title={Which Algorithmic Choices Matter at Which Batch Sizes? Insights From a Noisy Quadratic Model},
  author={Zhang, Guodong and Li, Lala and Nado Zachar and Martens, James and Sachdeva, Sushant and Dahl, George and Shallue, Chris and Grosse, Roger B.},
  journal={NeurIPS},
  year={2019}
}

@article{arora2019fine,
  title={Fine-grained analysis of optimization and generalization for overparameterized two-layer neural networks},
  author={Arora, Sanjeev and Du, Simon and Hu, Wei and Li, Zhiyuan and Wang, Ruosong},
  journal={ICML},
  year={2019},
}

@article{frantar2023scaling,
  title={Scaling laws for sparsely-connected foundation models},
  author={Frantar, Elias and Riquelme, Carlos and Houlsby, Neil and Alistarh, Dan and Evci, Utku},
  journal={ICLR},
  year={2024}
}

@article{schuhmann2022laion,
  title={Laion-5b: An open large-scale dataset for training next generation image-text models},
  author={Schuhmann, Christoph and Beaumont, Romain and Vencu, Richard and Gordon, Cade and Wightman, Ross and Cherti, Mehdi and Coombes, Theo and Katta, Aarush and Mullis, Clayton and Wortsman, Mitchell and others},
  journal={NeurIPS},
  year={2022}
}

@article{yu2020hyper,
  title={Hyper-parameter optimization: A review of algorithms and applications},
  author={Yu, Tong and Zhu, Hong},
  journal={arXiv preprint arXiv:2003.05689},
  year={2020}
}

@article{brutzkus2019larger,
  title={Why do larger models generalize better? A theoretical perspective via the XOR problem},
  author={Brutzkus, Alon and Globerson, Amir},
  journal={ICML},
  year={2019},
}

@article{radford2021learning,
  title={Learning transferable visual models from natural language supervision},
  author={Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others},
  journal={ICML},
  year={2021},
}

@article{baek2024sam,
  title={Why is SAM Robust to Label Noise?},
  author={Baek, Christina and Kolter, Zico and Raghunathan, Aditi},
  journal={ICLR},
  year={2024}
}

@article{yu2021differentially,
  title={Differentially private fine-tuning of language models},
  author={Yu, Da and Naik, Saurabh and Backurs, Arturs and Gopi, Sivakanth and Inan, Huseyin A and Kamath, Gautam and Kulkarni, Janardhan and Lee, Yin Tat and Manoel, Andre and Wutschitz, Lukas and others},
  journal={ICLR},
  year={2022}
}

@article{ouyang2022training,
  title={Training language models to follow instructions with human feedback},
  author={Ouyang, Long and Wu, Jeffrey and Jiang, Xu and Almeida, Diogo and Wainwright, Carroll and Mishkin, Pamela and Zhang, Chong and Agarwal, Sandhini and Slama, Katarina and Ray, Alex and others},
  journal={NeurIPS},
  year={2022}
}

@article{du2019gradient,
  title={Gradient descent finds global minima of deep neural networks},
  author={Du, Simon and Lee, Jason and Li, Haochuan and Wang, Liwei and Zhai, Xiyu},
  journal={ICML},
  year={2019},
}

@software{jax2018github,
  author = {James Bradbury and Roy Frostig and Peter Hawkins and Matthew James Johnson and Chris Leary and Dougal Maclaurin and George Necula and Adam Paszke and Jake Vander{P}las and Skye Wanderman-{M}ilne and Qiao Zhang},
  title = {{JAX}: composable transformations of {P}ython+{N}um{P}y programs},
  year = {2018},
}

@article{simonyan2014very,
  title={Very deep convolutional networks for large-scale image recognition},
  author={Simonyan, Karen and Zisserman, Andrew},
  journal={ICLR},
  year={2015}
}

@article{alayrac2022flamingo,
  title={Flamingo: a visual language model for few-shot learning},
  author={Alayrac, Jean-Baptiste and Donahue, Jeff and Luc, Pauline and Miech, Antoine and Barr, Iain and Hasson, Yana and Lenc, Karel and Mensch, Arthur and Millican, Katherine and Reynolds, Malcolm and others},
  journal={NeurIPS},
  year={2022}
}

@article{radford2023robust,
  title={Robust speech recognition via large-scale weak supervision},
  author={Radford, Alec and Kim, Jong Wook and Xu, Tao and Brockman, Greg and McLeavey, Christine and Sutskever, Ilya},
  journal={ICML},
  year={2023},
}

@article{gunasekar2018characterizing,
  title={Characterizing implicit bias in terms of optimization geometry},
  author={Gunasekar, Suriya and Lee, Jason and Soudry, Daniel and Srebro, Nathan},
  journal={ICML},
  year={2018},
}

@article{song2022learning,
  title={Learning from noisy labels with deep neural networks: A survey},
  author={Song, Hwanjun and Kim, Minseok and Park, Dongmin and Shin, Yooju and Lee, Jae-Gil},
  journal={TNNLS},
  year={2022},
}

@article{mishra2021accelerating,
  title={Accelerating sparse deep neural networks},
  author={Mishra, Asit and Latorre, Jorge Albericio and Pool, Jeff and Stosic, Darko and Stosic, Dusan and Venkatesh, Ganesh and Yu, Chong and Micikevicius, Paulius},
  journal={arXiv preprint arXiv:2104.08378},
  year={2021}
}

@article{jiang2024mixtral,
  title={Mixtral of experts},
  author={Jiang, Albert Q and Sablayrolles, Alexandre and Roux, Antoine and Mensch, Arthur and Savary, Blanche and Bamford, Chris and Chaplot, Devendra Singh and Casas, Diego de las and Hanna, Emma Bou and Bressand, Florian and others},
  journal={arXiv preprint arXiv:2401.04088},
  year={2024}
}

@article{andriushchenko2024sharpness,
  title={Sharpness-aware minimization leads to low-rank features},
  author={Andriushchenko, Maksym and Bahri, Dara and Mobahi, Hossein and Flammarion, Nicolas},
  journal={NeurIPS},
  year={2023}
}

@article{ying2019overview,
  title={An overview of overfitting and its solutions},
  author={Ying, Xue},
  journal={Journal of physics},
  year={2019},
}

@article{gower2020variance,
  title={Variance-reduced methods for machine learning},
  author={Gower, Robert M and Schmidt, Mark and Bach, Francis and Richt{\'a}rik, Peter},
  journal={IEEE},
  year={2020},
}


@article{nutini2022let,
  title={Let's make block coordinate descent converge faster: faster greedy rules, message-passing, active-set complexity, and superlinear convergence},
  author={Nutini, Julie and Laradji, Issam and Schmidt, Mark},
  journal={JMLR},
  year={2022}
}


@article{ma2023recent,
  title={Recent Advances in the Generalization Theory of Neural Networks *},
  author={Ma, Tengyu and Damian, Alex},
  journal={ICML tutorial},
  year={2023},
}

@article{wilson2017marginal,
  title={The marginal value of adaptive gradient methods in machine learning},
  author={Wilson, Ashia C and Roelofs, Rebecca and Stern, Mitchell and Srebro, Nati and Recht, Benjamin},
  journal={NeurIPS},
  year={2017},
}

@article{andriushchenko2023sgd,
  title={SGD with large step sizes learns sparse features},
  author={Andriushchenko, Maksym and Varre, Aditya Vardhan and Pillaud-Vivien, Loucas and Flammarion, Nicolas},
  journal={ICML},
  year={2023},
}

@article{ji2020gradient,
  title={Gradient descent follows the regularization path for general losses},
  author={Ji, Ziwei and Dud{\'\i}k, Miroslav and Schapire, Robert E and Telgarsky, Matus},
  journal={CoLT},
  year={2020},
}

@article{nakkiran2020deep,
  title={Deep Double Descent: Where Bigger Models and More Data Hurt},
  author={Nakkiran, Preetum and Kaplun, Gal and Bansal, Yamini and Yang, Tristan and Barak, Boaz and Sutskever, Ilya},
  journal={ICLR},
  year={2020},
}

@article{belkin2018understand,
  title={To understand deep learning we need to understand kernel learning},
  author={Belkin, Mikhail and Ma, Siyuan and Mandal, Soumik},
  journal={ICML},
  year={2018},
}

@article{neyshabur2019role,
  title={The role of over-parametrization in generalization of neural networks},
  author={Neyshabur, Behnam and Li, Zhiyuan and Bhojanapalli, Srinadh and LeCun, Yann},
  journal={ICLR},
  year={2019}
}

@article{sagawa2020investigation,
  title={An investigation of why overparameterization exacerbates spurious correlations},
  author={Sagawa, Shiori and Raghunathan, Aditi and Koh, Pang Wei and Liang, Percy},
  journal={ICML},
  year={2020},
}

@article{nakkiran2020optimal,
  title={Optimal regularization can mitigate double descent},
  author={Nakkiran, Preetum and Venkat, Prayaag and Kakade, Sham and Ma, Tengyu},
  journal={ICLR},
  year={2021}
}


@article{agiollo2024approximating,
  title={Approximating Memorization Using Loss Surface Geometry for Dataset Pruning and Summarization},
  author={Agiollo, Andrea and Kim, Young In and Khanna, Rajiv},
  journal={SIGKDD},
  year={2024}
}

@article{huang2023robust,
  title={Robust generalization against photon-limited corruptions via worst-case sharpness minimization},
  author={Huang, Zhuo and Zhu, Miaoxi and Xia, Xiaobo and Shen, Li and Yu, Jun and Gong, Chen and Han, Bo and Du, Bo and Liu, Tongliang},
  journal={CVPR},
  year={2023}
}

@article{zou2024towards,
  title={Towards robust out-of-distribution generalization bounds via sharpness},
  author={Zou, Yingtian and Kawaguchi, Kenji and Liu, Yingnan and Liu, Jiashuo and Lee, Mong-Li and Hsu, Wynne},
  journal={ICLR},
  year={2024}
}

@article{kim2023fantastic,
  title={Fantastic robustness measures: The secrets of robust generalization},
  author={Kim, Hoki and Park, Jinseong and Choi, Yujin and Lee, Jaewook},
  journal={NeurIPS},
  year={2023}
}

