@article{sun2019optimization,
  title={Optimization for deep learning: theory and algorithms},
  author={Sun, Ruoyu},
  journal={arXiv preprint arXiv:1912.08957},
  year={2019}
}
@incollection{lecun2002efficient,
  title={Efficient backprop},
  author={LeCun, Yann and Bottou, L{\'e}on and Orr, Genevieve B and M{\"u}ller, Klaus-Robert},
  booktitle={Neural networks: Tricks of the trade},
  pages={9--50},
  year={2002},
  publisher={Springer}
}
@article{lee2019first,
  title={First-order methods almost always avoid strict saddle points},
  author={Lee, Jason D and Panageas, Ioannis and Piliouras, Georgios and Simchowitz, Max and Jordan, Michael I and Recht, Benjamin},
  journal={Mathematical programming},
  volume={176},
  pages={311--337},
  year={2019},
  publisher={Springer}
}
@book{nesterov2003introductory,
  title={Introductory lectures on convex optimization: A basic course},
  author={Nesterov, Yurii},
  volume={87},
  year={2003},
  publisher={Springer Science \& Business Media}
}
@article{gunasekar2017implicit,
  title={Implicit regularization in matrix factorization},
  author={Gunasekar, Suriya and Woodworth, Blake E and Bhojanapalli, Srinadh and Neyshabur, Behnam and Srebro, Nati},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}
@inproceedings{li2018algorithmic,
  title={Algorithmic regularization in over-parameterized matrix sensing and neural networks with quadratic activations},
  author={Li, Yuanzhi and Ma, Tengyu and Zhang, Hongyang},
  booktitle={Conference On Learning Theory},
  pages={2--47},
  year={2018},
  organization={PMLR}
}
@article{li2020towards,
  title={Towards resolving the implicit bias of gradient descent for matrix factorization: Greedy low-rank learning},
  author={Li, Zhiyuan and Luo, Yuping and Lyu, Kaifeng},
  journal={arXiv preprint arXiv:2012.09839},
  year={2020}
}
@article{gissin2019implicit,
  title={The implicit bias of depth: How incremental learning drives generalization},
  author={Gissin, Daniel and Shalev-Shwartz, Shai and Daniely, Amit},
  journal={arXiv preprint arXiv:1909.12051},
  year={2019}
}
@inproceedings{glorot2010understanding,
  title={Understanding the difficulty of training deep feedforward neural networks},
  author={Glorot, Xavier and Bengio, Yoshua},
  booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
  pages={249--256},
  year={2010},
  organization={JMLR Workshop and Conference Proceedings}
}
@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}
@article{ye2021global,
  title={Global convergence of gradient descent for asymmetric low-rank matrix factorization},
  author={Ye, Tian and Du, Simon S},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={1429--1439},
  year={2021}
}
@article{soltanolkotabi2023implicit,
  title={Implicit balancing and regularization: Generalization and convergence guarantees for overparameterized asymmetric matrix sensing},
  author={Soltanolkotabi, Mahdi and St{\"o}ger, Dominik and Xie, Changzhi},
  journal={arXiv preprint arXiv:2303.14244},
  year={2023}
}
@article{ma2022behind,
  title={Behind the scenes of gradient descent: A trajectory analysis via basis function decomposition},
  author={Ma, Jianhao and Guo, Lingjun and Fattahi, Salar},
  journal={arXiv preprint arXiv:2210.00346},
  year={2022}
}
@inproceedings{jin2023understanding,
  title={Understanding incremental learning of gradient descent: A fine-grained analysis of matrix sensing},
  author={Jin, Jikai and Li, Zhiyuan and Lyu, Kaifeng and Du, Simon Shaolei and Lee, Jason D},
  booktitle={International Conference on Machine Learning},
  pages={15200--15238},
  year={2023},
  organization={PMLR}
}
@inproceedings{lee2016gradient,
  title={Gradient descent only converges to minimizers},
  author={Lee, Jason D and Simchowitz, Max and Jordan, Michael I and Recht, Benjamin},
  booktitle={Conference on Learning Theory},
  pages={1246--1257},
  year={2016},
  organization={PMLR}
}
@article{stoger2021small,
  title={Small random initialization is akin to spectral learning: Optimization and generalization guarantees for overparameterized low-rank matrix reconstruction},
  author={St{\"o}ger, Dominik and Soltanolkotabi, Mahdi},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={23831--23843},
  year={2021}
}
@article{sun2016complete,
  title={Complete dictionary recovery over the sphere I: Overview and the geometric picture},
  author={Sun, Ju and Qu, Qing and Wright, John},
  journal={IEEE Transactions on Information Theory},
  volume={63},
  number={2},
  pages={853--884},
  year={2016},
  publisher={IEEE}
}
@article{chen2019gradient,
  title={Gradient descent with random initialization: Fast global convergence for nonconvex phase retrieval},
  author={Chen, Yuxin and Chi, Yuejie and Fan, Jianqing and Ma, Cong},
  journal={Mathematical Programming},
  volume={176},
  pages={5--37},
  year={2019},
  publisher={Springer}
}
@article{zhu2021global,
  title={The global optimization geometry of low-rank matrix optimization},
  author={Zhu, Zhihui and Li, Qiuwei and Tang, Gongguo and Wakin, Michael B},
  journal={IEEE Transactions on Information Theory},
  volume={67},
  number={2},
  pages={1308--1331},
  year={2021},
  publisher={IEEE}
}
@article{sun2018geometric,
  title={A geometric analysis of phase retrieval},
  author={Sun, Ju and Qu, Qing and Wright, John},
  journal={Foundations of Computational Mathematics},
  volume={18},
  pages={1131--1198},
  year={2018},
  publisher={Springer}
}
@inproceedings{ma2018implicit,
  title={Implicit regularization in nonconvex statistical estimation: Gradient descent converges linearly for phase retrieval and matrix completion},
  author={Ma, Cong and Wang, Kaizheng and Chi, Yuejie and Chen, Yuxin},
  booktitle={International Conference on Machine Learning},
  pages={3345--3354},
  year={2018},
  organization={PMLR}
}
@inproceedings{zhang2016understanding,
  title={Understanding deep learning requires rethinking generalization},
  author={Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
  booktitle={International Conference on Learning Representations},
  year={2016}
}
@article{ji2018gradient,
  title={Gradient descent aligns the layers of deep linear networks},
  author={Ji, Ziwei and Telgarsky, Matus},
  journal={arXiv preprint arXiv:1810.02032},
  year={2018}
}
@article{chi2019nonconvex,
  title={Nonconvex optimization meets low-rank matrix factorization: An overview},
  author={Chi, Yuejie and Lu, Yue M and Chen, Yuxin},
  journal={IEEE Transactions on Signal Processing},
  volume={67},
  number={20},
  pages={5239--5269},
  year={2019},
  publisher={IEEE}
}
@article{vardi2023implicit,
  title={On the implicit bias in deep-learning algorithms},
  author={Vardi, Gal},
  journal={Communications of the ACM},
  volume={66},
  number={6},
  pages={86--93},
  year={2023},
  publisher={ACM New York, NY, USA}
}
@article{soudry2018implicit,
  title={The implicit bias of gradient descent on separable data},
  author={Soudry, Daniel and Hoffer, Elad and Nacson, Mor Shpigel and Gunasekar, Suriya and Srebro, Nathan},
  journal={The Journal of Machine Learning Research},
  volume={19},
  number={1},
  pages={2822--2878},
  year={2018},
  publisher={JMLR. org}
}
@ARTICLE{8357489,
  author={Zhu, Zhihui and Li, Qiuwei and Tang, Gongguo and Wakin, Michael B.},
  journal={IEEE Transactions on Signal Processing}, 
  title={Global Optimality in Low-Rank Matrix Optimization}, 
  year={2018},
  volume={66},
  number={13},
  pages={3614-3628},
  keywords={Linear programming;Optimization;Sensors;Geometry;Matrix decomposition;Signal processing algorithms;Minimization;Low-rank matrix optimization;matrix sensing;nonconvex optimization;optimization geometry;strict saddle}}
@inproceedings{simon2023stepwise,
  title={On the stepwise nature of self-supervised learning},
  author={Simon, James B and Knutins, Maksis and Ziyin, Liu and Geisz, Daniel and Fetterman, Abraham J and Albrecht, Joshua},
  booktitle={International Conference on Machine Learning},
  pages={31852--31876},
  year={2023},
  organization={PMLR}
}
@article{sun2020optimization,
  title={Optimization for deep learning: An overview},
  author={Sun, Ruo-Yu},
  journal={Journal of the Operations Research Society of China},
  volume={8},
  number={2},
  pages={249--294},
  year={2020},
  publisher={Springer}
}
@article{candes2012exact,
  title={Exact matrix completion via convex optimization},
  author={Candes, Emmanuel and Recht, Benjamin},
  journal={Communications of the ACM},
  volume={55},
  number={6},
  pages={111--119},
  year={2012},
  publisher={ACM New York, NY, USA}
}
@article{recht2010guaranteed,
  title={Guaranteed minimum-rank solutions of linear matrix equations via nuclear norm minimization},
  author={Recht, Benjamin and Fazel, Maryam and Parrilo, Pablo A},
  journal={SIAM Review},
  volume={52},
  number={3},
  pages={471--501},
  year={2010},
  publisher={SIAM}
}

@article{chou2024gradient,
  title={Gradient descent for deep matrix factorization: Dynamics and implicit bias towards low rank},
  author={Chou, Hung-Hsu and Gieshoff, Carsten and Maly, Johannes and Rauhut, Holger},
  journal={Applied and Computational Harmonic Analysis},
  volume={68},
  pages={101595},
  year={2024},
  publisher={Elsevier}
}
@article{zhu2020global,
  title={The global optimization geometry of shallow linear neural networks},
  author={Zhu, Zhihui and Soudry, Daniel and Eldar, Yonina C and Wakin, Michael B},
  journal={Journal of Mathematical Imaging and Vision},
  volume={62},
  pages={279--292},
  year={2020},
  publisher={Springer}
}
@article{baldi1989neural,
  title={Neural networks and principal component analysis: Learning from examples without local minima},
  author={Baldi, Pierre and Hornik, Kurt},
  journal={Neural Networks},
  volume={2},
  number={1},
  pages={53--58},
  year={1989},
  publisher={Elsevier}
}
@article{jiang2023algorithmic,
  title={Algorithmic regularization in model-free overparametrized asymmetric matrix factorization},
  author={Jiang, Liwei and Chen, Yudong and Ding, Lijun},
  journal={SIAM Journal on Mathematics of Data Science},
  volume={5},
  number={3},
  pages={723--744},
  year={2023},
  publisher={SIAM}
}
@article{shechtman2015phase,
  title={Phase retrieval with application to optical imaging: a contemporary overview},
  author={Shechtman, Yoav and Eldar, Yonina C and Cohen, Oren and Chapman, Henry Nicholas and Miao, Jianwei and Segev, Mordechai},
  journal={IEEE Signal Processing Magazine},
  volume={32},
  number={3},
  pages={87--109},
  year={2015},
  publisher={IEEE}
}
@article{arora2019implicit,
  title={Implicit regularization in deep matrix factorization},
  author={Arora, Sanjeev and Cohen, Nadav and Hu, Wei and Luo, Yuping},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}
@book{wainwright2019high,
  title={High-dimensional statistics: A non-asymptotic viewpoint},
  author={Wainwright, Martin J},
  volume={48},
  year={2019},
  publisher={Cambridge university press}
}
@inproceedings{jakubovitz2019generalization,
  title={Generalization error in deep learning},
  author={Jakubovitz, Daniel and Giryes, Raja and Rodrigues, Miguel RD},
  booktitle={Compressed Sensing and Its Applications: Third International MATHEON Conference 2017},
  pages={153--193},
  year={2019},
  organization={Springer}
}