

@inproceedings{xu2018global,
  title={Global convergence of langevin dynamics based algorithms for nonconvex optimization},
  author={Xu, Pan and Chen, Jinghui and Zou, Difan and Gu, Quanquan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3122--3133},
  year={2018}
}




@article{otto2000generalization,
  title={Generalization of an inequality by Talagrand and links with the logarithmic Sobolev inequality},
  author={Otto, Felix and Villani, C{\'e}dric},
  journal={Journal of Functional Analysis},
  volume={173},
  number={2},
  pages={361--400},
  year={2000},
  publisher={Elsevier}
}

@article{fang2019over,
  title={Over Parameterized Two-level Neural Networks Can Learn Near Optimal Feature Representations},
  author={Fang, Cong and Dong, Hanze and Zhang, Tong},
  journal={arXiv preprint arXiv:1910.11508},
  year={2019}
}




@Article{zou2019gradient,
author="Zou, Difan
and Cao, Yuan
and Zhou, Dongruo
and Gu, Quanquan",
title="Gradient descent optimizes over-parameterized deep {ReLU} networks",
journal="Machine Learning",
year="2019",
month="Oct",
day="23"
}


@article{e2019comparative,
  title={A Comparative Analysis of the Optimization and Generalization Property of Two-layer Neural Network and Random Feature Models Under Gradient Descent Dynamics},
  author={E, Weinan and Ma, Chao and Wu, Lei and others},
  journal={arXiv preprint arXiv:1904.04326},
  year={2019}
}

@inproceedings{cao2019generalization,
  title={Generalization Error Bounds of Gradient Descent for Learning Over-parameterized Deep ReLU Networks},
  author={Cao, Yuan and Gu, Quanquan},
  booktitle={the Thirty-Fourth AAAI Conference on Artificial Intelligence},
  year={2020}
}


@inproceedings{brutzkus2017globally,
  title={Globally optimal gradient descent for a convnet with gaussian inputs},
  author={Brutzkus, Alon and Globerson, Amir},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={605--614},
  year={2017},
  organization={JMLR. org}
}
@inproceedings{oymak2018overparameterized,
  title={Overparameterized Nonlinear Learning: Gradient Descent Takes the Shortest Path?},
  author={Oymak, Samet and Soltanolkotabi, Mahdi},
  booktitle={International Conference on Machine Learning},
  pages={4951--4960},
  year={2019}
}
@article{zhang2019training,
  title={Training Over-parameterized Deep {ResNet} Is almost as Easy as Training a Two-layer Network},
  author={Zhang, Huishuai and Yu, Da and Chen, Wei and Liu, Tie-Yan},
  journal={arXiv preprint arXiv:1903.07120},
  year={2019}
}
@article{wu2019global,
  title={Global Convergence of Adaptive Gradient Methods for An Over-parameterized Neural Network},
  author={Wu, Xiaoxia and Du, Simon S and Ward, Rachel},
  journal={arXiv preprint arXiv:1902.07111},
  year={2019}
}


@inproceedings{cotter2011better,
  title={Better mini-batch algorithms via accelerated gradient methods},
  author={Cotter, Andrew and Shamir, Ohad and Srebro, Nati and Sridharan, Karthik},
  booktitle={Advances in neural information processing systems},
  pages={1647--1655},
  year={2011}
}
@article{polyak1963gradient,
  title={Gradient methods for minimizing functionals},
  author={Polyak, Boris Teodorovich},
  journal={Zhurnal Vychislitel'noi Matematiki i Matematicheskoi Fiziki},
  volume={3},
  number={4},
  pages={643--653},
  year={1963},
  publisher={Russian Academy of Sciences, Branch of Mathematical Sciences}
}
@article{shalev2013stochastic,
  title={Stochastic dual coordinate ascent methods for regularized loss minimization},
  author={Shalev-Shwartz, Shai and Zhang, Tong},
  journal={Journal of Machine Learning Research},
  volume={14},
  number={Feb},
  pages={567--599},
  year={2013}
}






@article{slepian1962one,
  title={The one-sided barrier problem for Gaussian noise},
  author={Slepian, David},
  journal={Bell Labs Technical Journal},
  volume={41},
  number={2},
  pages={463--501},
  year={1962},
  publisher={Wiley Online Library}
}




######################## Teacher Network



######################### Deep Linear Network
@article{saxe2013exact,
  title={Exact solutions to the nonlinear dynamics of learning in deep linear neural networks},
  author={Saxe, Andrew M and McClelland, James L and Ganguli, Surya},
  journal={arXiv preprint arXiv:1312.6120},
  year={2013}
}


@inproceedings{bartlett2018gradient,
  title={Gradient descent with identity initialization efficiently learns positive definite linear transformations},
  author={Bartlett, Peter and Helmbold, Dave and Long, Phil},
  booktitle={International Conference on Machine Learning},
  pages={520--529},
  year={2018}
}





#####################Optimization Landscape


######################Generalization






@article{hsu2012tail,
  title={A tail inequality for quadratic forms of subgaussian random vectors},
  author={Hsu, Daniel and Kakade, Sham and Zhang, Tong and others},
  journal={Electronic Communications in Probability},
  volume={17},
  year={2012},
  publisher={The Institute of Mathematical Statistics and the Bernoulli Society}
}


@article{dally2015high,
  title={High-performance hardware for machine learning},
  author={Dally, William},
  journal={NIPS Tutorial},
  year={2015}
}

@inproceedings{sutskever2014sequence,
  title={Sequence to sequence learning with neural networks},
  author={Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V},
  booktitle={Advances in neural information processing systems},
  pages={3104--3112},
  year={2014}
}

@inproceedings{szegedy2015going,
  title={Going deeper with convolutions},
  author={Szegedy, Christian and Liu, Wei and Jia, Yangqing and Sermanet, Pierre and Reed, Scott and Anguelov, Dragomir and Erhan, Dumitru and Vanhoucke, Vincent and Rabinovich, Andrew},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={1--9},
  year={2015}
}

@inproceedings{devlin2014fast,
  title={Fast and Robust Neural Network Joint Models for Statistical Machine Translation.},
  author={Devlin, Jacob and Zbib, Rabih and Huang, Zhongqiang and Lamar, Thomas and Schwartz, Richard M and Makhoul, John},
  booktitle={ACL (1)},
  pages={1370--1380},
  year={2014}
}
@article{bahdanau2014neural,
  title={Neural machine translation by jointly learning to align and translate},
  author={Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua},
  journal={arXiv preprint arXiv:1409.0473},
  year={2014}
}


@article{hornik1991approximation,
  title={Approximation capabilities of multilayer feedforward networks},
  author={Hornik, Kurt},
  journal={Neural networks},
  volume={4},
  number={2},
  pages={251--257},
  year={1991},
  publisher={Elsevier}
}

@inproceedings{daniely2016toward,
  title={Toward deeper understanding of neural networks: The power of initialization and a dual view on expressivity},
  author={Daniely, Amit and Frostig, Roy and Singer, Yoram},
  booktitle={Advances In Neural Information Processing Systems},
  pages={2253--2261},
  year={2016}
}

@inproceedings{cohen2016expressive,
  title={On the expressive power of deep learning: A tensor analysis},
  author={Cohen, Nadav and Sharir, Or and Shashua, Amnon},
  booktitle={Conference on Learning Theory},
  pages={698--728},
  year={2016}
}
@inproceedings{cohen2016convolutional,
  title={Convolutional rectifier networks as generalized tensor decompositions},
  author={Cohen, Nadav and Shashua, Amnon},
  booktitle={International Conference on Machine Learning},
  pages={955--963},
  year={2016}
}

@article{raghu2016expressive,
  title={On the expressive power of deep neural networks},
  author={Raghu, Maithra and Poole, Ben and Kleinberg, Jon and Ganguli, Surya and Sohl-Dickstein, Jascha},
  journal={arXiv preprint arXiv:1606.05336},
  year={2016}
}
@inproceedings{poole2016exponential,
  title={Exponential expressivity in deep neural networks through transient chaos},
  author={Poole, Ben and Lahiri, Subhaneil and Raghu, Maithreyi and Sohl-Dickstein, Jascha and Ganguli, Surya},
  booktitle={Advances In Neural Information Processing Systems},
  pages={3360--3368},
  year={2016}
}
@inproceedings{montufar2014number,
  title={On the number of linear regions of deep neural networks},
  author={Montufar, Guido F and Pascanu, Razvan and Cho, Kyunghyun and Bengio, Yoshua},
  booktitle={Advances in neural information processing systems},
  pages={2924--2932},
  year={2014}
}

@article{vsima2006training,
  title={Training a single sigmoidal neuron is hard},
  author={{\v{S}}{\'\i}ma, Ji{\v{r}}{\'\i}},
  journal={Training},
  volume={14},
  number={11},
  year={2006},
  publisher={MIT Press}
}
@inproceedings{livni2014computational,
  title={On the computational efficiency of training neural networks},
  author={Livni, Roi and Shalev-Shwartz, Shai and Shamir, Ohad},
  booktitle={Advances in Neural Information Processing Systems},
  pages={855--863},
  year={2014}
}
@article{shamir2016distribution,
  title={Distribution-specific hardness of learning neural networks},
  author={Shamir, Ohad},
  journal={arXiv preprint arXiv:1609.01037},
  year={2016}
}
@inproceedings{shalev2017failures,
  title={Failures of gradient-based deep learning},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Shammah, Shaked},
  booktitle={International Conference on Machine Learning},
  pages={3067--3075},
  year={2017}
}
@article{shalev2017weight,
  title={Weight Sharing is Crucial to Succesful Optimization},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Shammah, Shaked},
  journal={arXiv preprint arXiv:1706.00687},
  year={2017}
}
@inproceedings{blum1989training,
  title={Training a 3-node neural network is NP-complete},
  author={Blum, Avrim and Rivest, Ronald L},
  booktitle={Advances in neural information processing systems},
  pages={494--501},
  year={1989}
}
@article{goel2016reliably,
  title={Reliably learning the {ReLU} in polynomial time},
  author={Goel, Surbhi and Kanade, Varun and Klivans, Adam and Thaler, Justin},
  journal={arXiv preprint arXiv:1611.10258},
  year={2016}
}
@article{zhang2015learning,
  title={Learning halfspaces and neural networks with random initialization},
  author={Zhang, Yuchen and Lee, Jason D and Wainwright, Martin J and Jordan, Michael I},
  journal={arXiv preprint arXiv:1511.07948},
  year={2015}
}
@article{sedghi2014provable,
  title={Provable methods for training neural networks with sparse connectivity},
  author={Sedghi, Hanie and Anandkumar, Anima},
  journal={arXiv preprint arXiv:1412.2693},
  year={2014}
}
@article{janzamin2015beating,
  title={Beating the perils of non-convexity: Guaranteed training of neural networks using tensor methods},
  author={Janzamin, Majid and Sedghi, Hanie and Anandkumar, Anima},
  journal={arXiv preprint arXiv:1506.08473},
  year={2015}
}
@inproceedings{ge2015escaping,
  title={Escaping from saddle points—online stochastic gradient for tensor decomposition},
  author={Ge, Rong and Huang, Furong and Jin, Chi and Yuan, Yang},
  booktitle={Conference on Learning Theory},
  pages={797--842},
  year={2015}
}
@inproceedings{jin2016provable,
  title={Provable efficient online matrix completion via non-convex stochastic gradient descent},
  author={Jin, Chi and Kakade, Sham M and Netrapalli, Praneeth},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4520--4528},
  year={2016}
}
@article{jin2017escape,
  title={How to Escape Saddle Points Efficiently},
  author={Jin, Chi and Ge, Rong and Netrapalli, Praneeth and Kakade, Sham M and Jordan, Michael I},
  journal={arXiv preprint arXiv:1703.00887},
  year={2017}
}

@article{lu2017depth,
  title={Depth Creates No Bad Local Minima},
  author={Lu, Haihao and Kawaguchi, Kenji},
  journal={arXiv preprint arXiv:1702.08580},
  year={2017}
}

@inproceedings{freeman2016topology,
  title={Topology and Geometry of Half-Rectified Network Optimization},
  author={Freeman, C Daniel and Bruna, Joan},
  booktitle={International Conference on Learning Representations},
  year={2017}
}
@article{taghvaei2017regularization,
  title={How regularization affects the critical points in linear networks},
  author={Taghvaei, Amirhossein and Kim, Jin W and Mehta, Prashant G},
  journal={arXiv preprint arXiv:1709.09625},
  year={2017}
}

@article{feizi2017porcupine,
  title={Porcupine Neural Networks:(Almost) All Local Optima are Global},
  author={Feizi, Soheil and Javadi, Hamid and Zhang, Jesse and Tse, David},
  journal={arXiv preprint arXiv:1710.02196},
  year={2017}
}



@article{carmon2017convex,
  title={" Convex Until Proven Guilty": Dimension-Free Acceleration of Gradient Descent on Non-Convex Functions},
  author={Carmon, Yair and Hinder, Oliver and Duchi, John C and Sidford, Aaron},
  journal={arXiv preprint arXiv:1705.02766},
  year={2017}
}


@inproceedings{choromanska2015loss,
  title={The loss surfaces of multilayer networks},
  author={Choromanska, Anna and Henaff, Mikael and Mathieu, Michael and Arous, G{\'e}rard Ben and LeCun, Yann},
  booktitle={Artificial Intelligence and Statistics},
  pages={192--204},
  year={2015}
}



@article{arora2016provable,
  title={Provable learning of Noisy-or Networks},
  author={Arora, Sanjeev and Ge, Rong and Ma, Tengyu and Risteski, Andrej},
  journal={arXiv preprint arXiv:1612.08795},
  year={2016}
}

@inproceedings{zhang2017learnability,
  title={On the Learnability of Fully-Connected Neural Networks},
  author={Zhang, Yuchen and Lee, Jason and Wainwright, Martin and Jordan, Michael},
  booktitle={Artificial Intelligence and Statistics},
  pages={83--91},
  year={2017}
}
@inproceedings{jain2015fast,
  title={Fast exact matrix completion with finite samples},
  author={Jain, Prateek and Netrapalli, Praneeth},
  booktitle={Conference on Learning Theory},
  pages={1007--1034},
  year={2015}
}
@inproceedings{pan2016expressiveness,
  title={Expressiveness of rectifier networks},
  author={Pan, Xingyuan and Srikumar, Vivek},
  booktitle={International Conference on Machine Learning},
  pages={2427--2435},
  year={2016}
}

@article{barron1993universal,
  title={Universal approximation bounds for superpositions of a sigmoidal function},
  author={Barron, Andrew R},
  journal={IEEE Transactions on Information theory},
  volume={39},
  number={3},
  pages={930--945},
  year={1993},
  publisher={IEEE}
}

@article{mei2016landscape,
  title={The landscape of empirical risk for non-convex losses},
  author={Mei, Song and Bai, Yu and Montanari, Andrea},
  journal={arXiv preprint arXiv:1607.06534},
  year={2016}
}

@inproceedings{auer1996exponentially,
  title={Exponentially many local minima for single neurons},
  author={Auer, Peter and Herbster, Mark and Warmuth, Manfred K},
  booktitle={Advances in neural information processing systems},
  pages={316--322},
  year={1996}
}
@article{ge2017learning,
  title={Learning One-hidden-layer Neural Networks with Landscape Design},
  author={Ge, Rong and Lee, Jason D and Ma, Tengyu},
  journal={arXiv preprint arXiv:1711.00501},
  year={2017}
}
@inproceedings{kalai2009isotron,
  title={The Isotron Algorithm: High-Dimensional Isotonic Regression.},
  author={Kalai, Adam Tauman and Sastry, Ravi},
  booktitle={COLT},
  year={2009},
  organization={Citeseer}
}

@article{fu2018local,
  title={Local Geometry of One-Hidden-Layer Neural Networks for Logistic Regression},
  author={Fu, Haoyu and Chi, Yuejie and Liang, Yingbin},
  journal={arXiv preprint arXiv:1802.06463},
  year={2018}
}

@inproceedings{kakade2011efficient,
  title={Efficient learning of generalized linear and single index models with isotonic regression},
  author={Kakade, Sham M and Kanade, Varun and Shamir, Ohad and Kalai, Adam},
  booktitle={Advances in Neural Information Processing Systems},
  pages={927--935},
  year={2011}
}

@inproceedings{jain2013low,
  title={Low-rank matrix completion using alternating minimization},
  author={Jain, Prateek and Netrapalli, Praneeth and Sanghavi, Sujay},
  booktitle={Proceedings of the forty-fifth annual ACM symposium on Theory of computing},
  pages={665--674},
  year={2013},
  organization={ACM}
}

@inproceedings{sutskever2013importance,
  title={On the importance of initialization and momentum in deep learning},
  author={Sutskever, Ilya and Martens, James and Dahl, George and Hinton, Geoffrey},
  booktitle={International conference on machine learning},
  pages={1139--1147},
  year={2013}
}


@inproceedings{livni2014computational,
  title={On the computational efficiency of training neural networks},
  author={Livni, Roi and Shalev-Shwartz, Shai and Shamir, Ohad},
  booktitle={Advances in Neural Information Processing Systems},
  pages={855--863},
  year={2014}
}


@article{hornik1991approximation,
  title={Approximation capabilities of multilayer feedforward networks},
  author={Hornik, Kurt},
  journal={Neural networks},
  volume={4},
  number={2},
  pages={251--257},
  year={1991},
  publisher={Elsevier}
}


@article{barron1993universal,
  title={Universal approximation bounds for superpositions of a sigmoidal function},
  author={Barron, Andrew R},
  journal={IEEE Transactions on Information theory},
  volume={39},
  number={3},
  pages={930--945},
  year={1993},
  publisher={IEEE}
}




@article{neyshabur2014search,
  title={In search of the real inductive bias: On the role of implicit regularization in deep learning},
  author={Neyshabur, Behnam and Tomioka, Ryota and Srebro, Nathan},
  journal={arXiv preprint arXiv:1412.6614},
  year={2014}
}




@inproceedings{sutskever2013importance,
  title={On the importance of initialization and momentum in deep learning},
  author={Sutskever, Ilya and Martens, James and Dahl, George and Hinton, Geoffrey},
  booktitle={International conference on machine learning},
  pages={1139--1147},
  year={2013}
}

@inproceedings{blum1989training,
  title={Training a 3-node neural network is NP-complete},
  author={Blum, Avrim and Rivest, Ronald L},
  booktitle={Advances in neural information processing systems},
  pages={494--501},
  year={1989}
}

#########gaussian##############





@article{zhong2017learning,
  title={Learning Non-overlapping Convolutional Neural Networks with Multiple Kernels},
  author={Zhong, Kai and Song, Zhao and Dhillon, Inderjit S},
  journal={arXiv preprint arXiv:1711.03440},
  year={2017}
}


#############independent assumption 


@inproceedings{choromanska2015loss,
  title={The loss surfaces of multilayer networks},
  author={Choromanska, Anna and Henaff, Mikael and Mathieu, Michael and Arous, G{\'e}rard Ben and LeCun, Yann},
  booktitle={Artificial Intelligence and Statistics},
  pages={192--204},
  year={2015}
}





@inproceedings{bartlett2018gradient,
  title={Gradient descent with identity initialization efficiently learns positive definite linear transformations},
  author={Bartlett, Peter and Helmbold, Dave and Long, Phil},
  booktitle={International Conference on Machine Learning},
  pages={520--529},
  year={2018}
}

@inproceedings{langford2002not,
  title={(Not) bounding the true error},
  author={Langford, John and Caruana, Rich},
  booktitle={Advances in Neural Information Processing Systems},
  pages={809--816},
  year={2002}
}


###############generalization#################


@inproceedings{neyshabur2017exploring,
  title={Exploring generalization in deep learning},
  author={Neyshabur, Behnam and Bhojanapalli, Srinadh and McAllester, David and Srebro, Nati},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5947--5956},
  year={2017}
}

@inproceedings{bartlett2017spectrally,
  title={Spectrally-normalized margin bounds for neural networks},
  author={Bartlett, Peter L and Foster, Dylan J and Telgarsky, Matus J},
  booktitle={Advances in Neural Information Processing Systems},
  pages={6240--6249},
  year={2017}
}

@book{anthony2009neural,
  title={Neural network learning: Theoretical foundations},
  author={Anthony, Martin and Bartlett, Peter L},
  year={2009},
  publisher={cambridge university press}
}

@book{vapnik2013nature,
  title={The nature of statistical learning theory},
  author={Vapnik, Vladimir},
  year={2013},
  publisher={Springer science \& business media}
}


@article{arora2016understanding,
  title={Understanding deep neural networks with rectified linear units},
  author={Arora, Raman and Basu, Amitabh and Mianjy, Poorya and Mukherjee, Anirbit},
  journal={arXiv preprint arXiv:1611.01491},
  year={2016}
}


###########more citation#################




@inproceedings{safran2017spurious,
  title={Spurious Local Minima are Common in Two-Layer ReLU Neural Networks},
  author={Safran, Itay and Shamir, Ohad},
  booktitle={International Conference on Machine Learning},
  pages={4430--4438},
  year={2018}
}




@article{vaswani2018fast,
  title={Fast and faster convergence of sgd for over-parameterized models and an accelerated perceptron},
  author={Vaswani, Sharan and Bach, Francis and Schmidt, Mark},
  journal={arXiv preprint arXiv:1810.07288},
  year={2018}
}

@inproceedings{cotter2011better,
  title={Better mini-batch algorithms via accelerated gradient methods},
  author={Cotter, Andrew and Shamir, Ohad and Srebro, Nati and Sridharan, Karthik},
  booktitle={Advances in neural information processing systems},
  pages={1647--1655},
  year={2011}
}
@article{shalev2013stochastic,
  title={Stochastic dual coordinate ascent methods for regularized loss minimization},
  author={Shalev-Shwartz, Shai and Zhang, Tong},
  journal={Journal of Machine Learning Research},
  volume={14},
  number={Feb},
  pages={567--599},
  year={2013}
}




@inproceedings{neyshabur2017exploring,
  title={Exploring generalization in deep learning},
  author={Neyshabur, Behnam and Bhojanapalli, Srinadh and McAllester, David and Srebro, Nati},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5947--5956},
  year={2017}
}





@article{bartlett2002rademacher,
  title={Rademacher and {Gaussian} complexities: Risk bounds and structural results},
  author={Bartlett, Peter L and Mendelson, Shahar},
  journal={Journal of Machine Learning Research},
  volume={3},
  number={Nov},
  pages={463--482},
  year={2002}
}



@book{shalev2014understanding,
  title={Understanding machine learning: From theory to algorithms},
  author={Shalev-Shwartz, Shai and Ben-David, Shai},
  year={2014},
  publisher={Cambridge university press}
}



@inproceedings{ji2019polylogarithmic,
  title={Polylogarithmic width suffices for gradient descent to achieve arbitrarily small test error with shallow ReLU networks},
  author={Ji, Ziwei and Telgarsky, Matus},
  booktitle={International Conference on Learning Representations},
  year={2020}
}





@inproceedings{mei2019mean,
  title={Mean-field theory of two-layers neural networks: dimension-free bounds and kernel limit},
  author={Mei, Song and Misiakiewicz, Theodor and Montanari, Andrea},
  booktitle={Conference on Learning Theory},
  year={2019}
}




@inproceedings{allen2019can,
  title={What Can {ResNet} Learn Efficiently, Going Beyond Kernels?},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}



@article{nitanda2019refined,
  title={Refined Generalization Analysis of Gradient Descent for Over-parameterized Two-layer Neural Networks with Smooth Activations on Classification Problems},
  author={Nitanda, Atsushi and Suzuki, Taiji},
  journal={arXiv preprint arXiv:1905.09870},
  year={2019}
}


@article{woodworth2019kernel,
  title={Kernel and Deep Regimes in Overparametrized Models},
  author={Woodworth, Blake and Gunasekar, Suriya and Lee, Jason and Soudry, Daniel and Srebro, Nathan},
  journal={arXiv preprint arXiv:1906.05827},
  year={2019}
}






@article{venturi2018neural,
  title={Neural networks with finite intrinsic dimension have no spurious valleys},
  author={Venturi, Luca and Bandeira, Afonso and Bruna, Joan},
  journal={arXiv preprint arXiv:1802.06384},
  volume={15},
  year={2018}
}






@inproceedings{ji2019implicit,
  title={The implicit bias of gradient descent on nonseparable data},
  author={Ji, Ziwei and Telgarsky, Matus},
  booktitle={Conference on Learning Theory},
  pages={1772--1798},
  year={2019}
}





@article{bai2019beyond,
  title={Beyond Linearization: On Quadratic and Higher-Order Approximation of Wide Neural Networks},
  author={Bai, Yu and Lee, Jason D},
  journal={arXiv preprint arXiv:1910.01619},
  year={2019}
}




@article{belkin2019reconciling,
  title={Reconciling modern machine-learning practice and the classical bias--variance trade-off},
  author={Belkin, Mikhail and Hsu, Daniel and Ma, Siyuan and Mandal, Soumik},
  journal={Proceedings of the National Academy of Sciences},
  volume={116},
  number={32},
  pages={15849--15854},
  year={2019},
  publisher={National Acad Sciences}
}


@article{belkin2019two,
  title={Two models of double descent for weak features},
  author={Belkin, Mikhail and Hsu, Daniel and Xu, Ji},
  journal={arXiv preprint arXiv:1903.07571},
  year={2019}
}



@article{hastie2019surprises,
  title={Surprises in high-dimensional ridgeless least squares interpolation},
  author={Hastie, Trevor and Montanari, Andrea and Rosset, Saharon and Tibshirani, Ryan J},
  journal={arXiv preprint arXiv:1903.08560},
  year={2019}
}


@article{mei2019generalization,
  title={The generalization error of random features regression: Precise asymptotics and double descent curve},
  author={Mei, Song and Montanari, Andrea},
  journal={arXiv preprint arXiv:1908.05355},
  year={2019}
}


















@inproceedings{cotter2011better,
  title={Better mini-batch algorithms via accelerated gradient methods},
  author={Cotter, Andrew and Shamir, Ohad and Srebro, Nati and Sridharan, Karthik},
  booktitle={Advances in neural information processing systems},
  pages={1647--1655},
  year={2011}
}
@article{shalev2013stochastic,
  title={Stochastic dual coordinate ascent methods for regularized loss minimization},
  author={Shalev-Shwartz, Shai and Zhang, Tong},
  journal={Journal of Machine Learning Research},
  volume={14},
  number={Feb},
  pages={567--599},
  year={2013}
}


@article{soudry2017implicit,
  title={The implicit bias of gradient descent on separable data},
  author={Soudry, Daniel and Hoffer, Elad and Nacson, Mor Shpigel and Gunasekar, Suriya and Srebro, Nathan},
  journal={The Journal of Machine Learning Research},
  volume={19},
  number={1},
  pages={2822--2878},
  year={2018},
  publisher={JMLR. org}
}




@article{slepian1962one,
  title={The one-sided barrier problem for {Gaussian} noise},
  author={Slepian, David},
  journal={Bell Labs Technical Journal},
  volume={41},
  number={2},
  pages={463--501},
  year={1962},
  publisher={Wiley Online Library}
}




######################## Teacher Network



######################### Deep Linear Network
@article{saxe2013exact,
  title={Exact solutions to the nonlinear dynamics of learning in deep linear neural networks},
  author={Saxe, Andrew M and McClelland, James L and Ganguli, Surya},
  journal={arXiv preprint arXiv:1312.6120},
  year={2013}
}


@inproceedings{bartlett2018gradient,
  title={Gradient descent with identity initialization efficiently learns positive definite linear transformations},
  author={Bartlett, Peter and Helmbold, Dave and Long, Phil},
  booktitle={International Conference on Machine Learning},
  pages={520--529},
  year={2018}
}




#####################Optimization Landscape


######################Generalization







@article{hsu2012tail,
  title={A tail inequality for quadratic forms of subgaussian random vectors},
  author={Hsu, Daniel and Kakade, Sham and Zhang, Tong and others},
  journal={Electronic Communications in Probability},
  volume={17},
  year={2012},
  publisher={The Institute of Mathematical Statistics and the Bernoulli Society}
}


@article{dally2015high,
  title={High-performance hardware for machine learning},
  author={Dally, William},
  journal={NIPS Tutorial},
  year={2015}
}
@article{hinton2012deep,
  title={Deep neural networks for acoustic modeling in speech recognition: The shared views of four research groups},
  author={Hinton, Geoffrey and Deng, Li and Yu, Dong and Dahl, George E and Mohamed, Abdel-rahman and Jaitly, Navdeep and Senior, Andrew and Vanhoucke, Vincent and Nguyen, Patrick and Sainath, Tara N and others},
  journal={IEEE Signal Processing Magazine},
  volume={29},
  number={6},
  pages={82--97},
  year={2012},
  publisher={IEEE}
}
@inproceedings{sutskever2014sequence,
  title={Sequence to sequence learning with neural networks},
  author={Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V},
  booktitle={Advances in neural information processing systems},
  pages={3104--3112},
  year={2014}
}

@inproceedings{szegedy2015going,
  title={Going deeper with convolutions},
  author={Szegedy, Christian and Liu, Wei and Jia, Yangqing and Sermanet, Pierre and Reed, Scott and Anguelov, Dragomir and Erhan, Dumitru and Vanhoucke, Vincent and Rabinovich, Andrew},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={1--9},
  year={2015}
}

@inproceedings{devlin2014fast,
  title={Fast and Robust Neural Network Joint Models for Statistical Machine Translation.},
  author={Devlin, Jacob and Zbib, Rabih and Huang, Zhongqiang and Lamar, Thomas and Schwartz, Richard M and Makhoul, John},
  booktitle={ACL (1)},
  pages={1370--1380},
  year={2014}
}
@article{bahdanau2014neural,
  title={Neural machine translation by jointly learning to align and translate},
  author={Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua},
  journal={arXiv preprint arXiv:1409.0473},
  year={2014}
}


@article{hornik1991approximation,
  title={Approximation capabilities of multilayer feedforward networks},
  author={Hornik, Kurt},
  journal={Neural networks},
  volume={4},
  number={2},
  pages={251--257},
  year={1991},
  publisher={Elsevier}
}

@inproceedings{daniely2016toward,
  title={Toward deeper understanding of neural networks: The power of initialization and a dual view on expressivity},
  author={Daniely, Amit and Frostig, Roy and Singer, Yoram},
  booktitle={Advances In Neural Information Processing Systems},
  pages={2253--2261},
  year={2016}
}

@inproceedings{cohen2016expressive,
  title={On the expressive power of deep learning: A tensor analysis},
  author={Cohen, Nadav and Sharir, Or and Shashua, Amnon},
  booktitle={Conference on Learning Theory},
  pages={698--728},
  year={2016}
}
@inproceedings{cohen2016convolutional,
  title={Convolutional rectifier networks as generalized tensor decompositions},
  author={Cohen, Nadav and Shashua, Amnon},
  booktitle={International Conference on Machine Learning},
  pages={955--963},
  year={2016}
}

@article{raghu2016expressive,
  title={On the expressive power of deep neural networks},
  author={Raghu, Maithra and Poole, Ben and Kleinberg, Jon and Ganguli, Surya and Sohl-Dickstein, Jascha},
  journal={arXiv preprint arXiv:1606.05336},
  year={2016}
}
@inproceedings{poole2016exponential,
  title={Exponential expressivity in deep neural networks through transient chaos},
  author={Poole, Ben and Lahiri, Subhaneil and Raghu, Maithreyi and Sohl-Dickstein, Jascha and Ganguli, Surya},
  booktitle={Advances In Neural Information Processing Systems},
  pages={3360--3368},
  year={2016}
}
@inproceedings{montufar2014number,
  title={On the number of linear regions of deep neural networks},
  author={Montufar, Guido F and Pascanu, Razvan and Cho, Kyunghyun and Bengio, Yoshua},
  booktitle={Advances in neural information processing systems},
  pages={2924--2932},
  year={2014}
}

@article{vsima2006training,
  title={Training a single sigmoidal neuron is hard},
  author={{\v{S}}{\'\i}ma, Ji{\v{r}}{\'\i}},
  journal={Training},
  volume={14},
  number={11},
  year={2006},
  publisher={MIT Press}
}
@inproceedings{livni2014computational,
  title={On the computational efficiency of training neural networks},
  author={Livni, Roi and Shalev-Shwartz, Shai and Shamir, Ohad},
  booktitle={Advances in Neural Information Processing Systems},
  pages={855--863},
  year={2014}
}
@article{shamir2016distribution,
  title={Distribution-specific hardness of learning neural networks},
  author={Shamir, Ohad},
  journal={arXiv preprint arXiv:1609.01037},
  year={2016}
}
@inproceedings{shalev2017failures,
  title={Failures of gradient-based deep learning},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Shammah, Shaked},
  booktitle={International Conference on Machine Learning},
  pages={3067--3075},
  year={2017}
}
@article{shalev2017weight,
  title={Weight Sharing is Crucial to Succesful Optimization},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Shammah, Shaked},
  journal={arXiv preprint arXiv:1706.00687},
  year={2017}
}
@inproceedings{blum1989training,
  title={Training a 3-node neural network is NP-complete},
  author={Blum, Avrim and Rivest, Ronald L},
  booktitle={Advances in neural information processing systems},
  pages={494--501},
  year={1989}
}
@article{goel2016reliably,
  title={Reliably learning the relu in polynomial time},
  author={Goel, Surbhi and Kanade, Varun and Klivans, Adam and Thaler, Justin},
  journal={arXiv preprint arXiv:1611.10258},
  year={2016}
}
@article{zhang2015learning,
  title={Learning halfspaces and neural networks with random initialization},
  author={Zhang, Yuchen and Lee, Jason D and Wainwright, Martin J and Jordan, Michael I},
  journal={arXiv preprint arXiv:1511.07948},
  year={2015}
}
@article{sedghi2014provable,
  title={Provable methods for training neural networks with sparse connectivity},
  author={Sedghi, Hanie and Anandkumar, Anima},
  journal={arXiv preprint arXiv:1412.2693},
  year={2014}
}
@article{janzamin2015beating,
  title={Beating the perils of non-convexity: Guaranteed training of neural networks using tensor methods},
  author={Janzamin, Majid and Sedghi, Hanie and Anandkumar, Anima},
  journal={arXiv preprint arXiv:1506.08473},
  year={2015}
}
@inproceedings{ge2015escaping,
  title={Escaping from saddle points—online stochastic gradient for tensor decomposition},
  author={Ge, Rong and Huang, Furong and Jin, Chi and Yuan, Yang},
  booktitle={Conference on Learning Theory},
  pages={797--842},
  year={2015}
}
@inproceedings{jin2016provable,
  title={Provable efficient online matrix completion via non-convex stochastic gradient descent},
  author={Jin, Chi and Kakade, Sham M and Netrapalli, Praneeth},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4520--4528},
  year={2016}
}
@article{jin2017escape,
  title={How to Escape Saddle Points Efficiently},
  author={Jin, Chi and Ge, Rong and Netrapalli, Praneeth and Kakade, Sham M and Jordan, Michael I},
  journal={arXiv preprint arXiv:1703.00887},
  year={2017}
}
@inproceedings{kawaguchi2016deep,
  title={Deep learning without poor local minima},
  author={Kawaguchi, Kenji},
  booktitle={Advances in Neural Information Processing Systems},
  pages={586--594},
  year={2016}
}
@article{lu2017depth,
  title={Depth Creates No Bad Local Minima},
  author={Lu, Haihao and Kawaguchi, Kenji},
  journal={arXiv preprint arXiv:1702.08580},
  year={2017}
}


@article{taghvaei2017regularization,
  title={How regularization affects the critical points in linear networks},
  author={Taghvaei, Amirhossein and Kim, Jin W and Mehta, Prashant G},
  journal={arXiv preprint arXiv:1709.09625},
  year={2017}
}

@inproceedings{nguyen2017loss,
  title={The Loss Surface of Deep and Wide Neural Networks},
  author={Nguyen, Quynh and Hein, Matthias},
  booktitle={International Conference on Machine Learning},
  pages={2603--2612},
  year={2017}
}
@article{feizi2017porcupine,
  title={Porcupine Neural Networks:(Almost) All Local Optima are Global},
  author={Feizi, Soheil and Javadi, Hamid and Zhang, Jesse and Tse, David},
  journal={arXiv preprint arXiv:1710.02196},
  year={2017}
}



@article{carmon2017convex,
  title={" Convex Until Proven Guilty": Dimension-Free Acceleration of Gradient Descent on Non-Convex Functions},
  author={Carmon, Yair and Hinder, Oliver and Duchi, John C and Sidford, Aaron},
  journal={arXiv preprint arXiv:1705.02766},
  year={2017}
}


@inproceedings{choromanska2015loss,
  title={The loss surfaces of multilayer networks},
  author={Choromanska, Anna and Henaff, Mikael and Mathieu, Michael and Arous, G{\'e}rard Ben and LeCun, Yann},
  booktitle={Artificial Intelligence and Statistics},
  pages={192--204},
  year={2015}
}


@article{soudry2016no,
  title={No bad local minima: Data independent training error guarantees for multilayer neural networks},
  author={Soudry, Daniel and Carmon, Yair},
  journal={arXiv preprint arXiv:1605.08361},
  year={2016}
}
@article{arora2016provable,
  title={Provable learning of Noisy-or Networks},
  author={Arora, Sanjeev and Ge, Rong and Ma, Tengyu and Risteski, Andrej},
  journal={arXiv preprint arXiv:1612.08795},
  year={2016}
}

@inproceedings{zhang2017learnability,
  title={On the Learnability of Fully-Connected Neural Networks},
  author={Zhang, Yuchen and Lee, Jason and Wainwright, Martin and Jordan, Michael},
  booktitle={Artificial Intelligence and Statistics},
  pages={83--91},
  year={2017}
}
@inproceedings{jain2015fast,
  title={Fast exact matrix completion with finite samples},
  author={Jain, Prateek and Netrapalli, Praneeth},
  booktitle={Conference on Learning Theory},
  pages={1007--1034},
  year={2015}
}
@inproceedings{pan2016expressiveness,
  title={Expressiveness of rectifier networks},
  author={Pan, Xingyuan and Srikumar, Vivek},
  booktitle={International Conference on Machine Learning},
  pages={2427--2435},
  year={2016}
}

@article{barron1993universal,
  title={Universal approximation bounds for superpositions of a sigmoidal function},
  author={Barron, Andrew R},
  journal={IEEE Transactions on Information theory},
  volume={39},
  number={3},
  pages={930--945},
  year={1993},
  publisher={IEEE}
}

@article{mei2016landscape,
  title={The landscape of empirical risk for non-convex losses},
  author={Mei, Song and Bai, Yu and Montanari, Andrea},
  journal={arXiv preprint arXiv:1607.06534},
  year={2016}
}

@inproceedings{auer1996exponentially,
  title={Exponentially many local minima for single neurons},
  author={Auer, Peter and Herbster, Mark and Warmuth, Manfred K},
  booktitle={Advances in neural information processing systems},
  pages={316--322},
  year={1996}
}
@article{ge2017learning,
  title={Learning One-hidden-layer Neural Networks with Landscape Design},
  author={Ge, Rong and Lee, Jason D and Ma, Tengyu},
  journal={arXiv preprint arXiv:1711.00501},
  year={2017}
}
@inproceedings{kalai2009isotron,
  title={The Isotron Algorithm: High-Dimensional Isotonic Regression.},
  author={Kalai, Adam Tauman and Sastry, Ravi},
  booktitle={COLT},
  year={2009},
  organization={Citeseer}
}

@article{fu2018local,
  title={Local Geometry of One-Hidden-Layer Neural Networks for Logistic Regression},
  author={Fu, Haoyu and Chi, Yuejie and Liang, Yingbin},
  journal={arXiv preprint arXiv:1802.06463},
  year={2018}
}

@inproceedings{kakade2011efficient,
  title={Efficient learning of generalized linear and single index models with isotonic regression},
  author={Kakade, Sham M and Kanade, Varun and Shamir, Ohad and Kalai, Adam},
  booktitle={Advances in Neural Information Processing Systems},
  pages={927--935},
  year={2011}
}

@inproceedings{jain2013low,
  title={Low-rank matrix completion using alternating minimization},
  author={Jain, Prateek and Netrapalli, Praneeth and Sanghavi, Sujay},
  booktitle={Proceedings of the forty-fifth annual ACM symposium on Theory of computing},
  pages={665--674},
  year={2013},
  organization={ACM}
}

@inproceedings{sutskever2013importance,
  title={On the importance of initialization and momentum in deep learning},
  author={Sutskever, Ilya and Martens, James and Dahl, George and Hinton, Geoffrey},
  booktitle={International conference on machine learning},
  pages={1139--1147},
  year={2013}
}



@inproceedings{livni2014computational,
  title={On the computational efficiency of training neural networks},
  author={Livni, Roi and Shalev-Shwartz, Shai and Shamir, Ohad},
  booktitle={Advances in Neural Information Processing Systems},
  pages={855--863},
  year={2014}
}


@article{hornik1991approximation,
  title={Approximation capabilities of multilayer feedforward networks},
  author={Hornik, Kurt},
  journal={Neural networks},
  volume={4},
  number={2},
  pages={251--257},
  year={1991},
  publisher={Elsevier}
}


@article{barron1993universal,
  title={Universal approximation bounds for superpositions of a sigmoidal function},
  author={Barron, Andrew R},
  journal={IEEE Transactions on Information theory},
  volume={39},
  number={3},
  pages={930--945},
  year={1993},
  publisher={IEEE}
}





@article{neyshabur2014search,
  title={In search of the real inductive bias: On the role of implicit regularization in deep learning},
  author={Neyshabur, Behnam and Tomioka, Ryota and Srebro, Nathan},
  journal={arXiv preprint arXiv:1412.6614},
  year={2014}
}



@inproceedings{sutskever2013importance,
  title={On the importance of initialization and momentum in deep learning},
  author={Sutskever, Ilya and Martens, James and Dahl, George and Hinton, Geoffrey},
  booktitle={International conference on machine learning},
  pages={1139--1147},
  year={2013}
}

@inproceedings{blum1989training,
  title={Training a 3-node neural network is NP-complete},
  author={Blum, Avrim and Rivest, Ronald L},
  booktitle={Advances in neural information processing systems},
  pages={494--501},
  year={1989}
}

#########gaussian##############





@article{zhong2017learning,
  title={Learning Non-overlapping Convolutional Neural Networks with Multiple Kernels},
  author={Zhong, Kai and Song, Zhao and Dhillon, Inderjit S},
  journal={arXiv preprint arXiv:1711.03440},
  year={2017}
}


#############independent assumption 


@inproceedings{choromanska2015loss,
  title={The loss surfaces of multilayer networks},
  author={Choromanska, Anna and Henaff, Mikael and Mathieu, Michael and Arous, G{\'e}rard Ben and LeCun, Yann},
  booktitle={Artificial Intelligence and Statistics},
  pages={192--204},
  year={2015}
}



@inproceedings{bartlett2018gradient,
  title={Gradient descent with identity initialization efficiently learns positive definite linear transformations},
  author={Bartlett, Peter and Helmbold, Dave and Long, Phil},
  booktitle={International Conference on Machine Learning},
  pages={520--529},
  year={2018}
}

@inproceedings{langford2002not,
  title={(Not) bounding the true error},
  author={Langford, John and Caruana, Rich},
  booktitle={Advances in Neural Information Processing Systems},
  pages={809--816},
  year={2002}
}


###############generalization#################


@inproceedings{neyshabur2017exploring,
  title={Exploring generalization in deep learning},
  author={Neyshabur, Behnam and Bhojanapalli, Srinadh and McAllester, David and Srebro, Nati},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5947--5956},
  year={2017}
}


@book{anthony2009neural,
  title={Neural network learning: Theoretical foundations},
  author={Anthony, Martin and Bartlett, Peter L},
  year={2009},
  publisher={cambridge university press}
}

@book{vapnik2013nature,
  title={The nature of statistical learning theory},
  author={Vapnik, Vladimir},
  year={2013},
  publisher={Springer science \& business media}
}


@article{arora2016understanding,
  title={Understanding deep neural networks with rectified linear units},
  author={Arora, Raman and Basu, Amitabh and Mianjy, Poorya and Mukherjee, Anirbit},
  journal={arXiv preprint arXiv:1611.01491},
  year={2016}
}


###########more citation#################
@inproceedings{yarotsky2018optimal,
  title={Optimal approximation of continuous functions by very deep ReLU networks},
  author={Yarotsky, Dmitry},
  booktitle={Conference On Learning Theory},
  pages={639--649},
  year={2018}
}




@article{nacson2018convergence,
  title={Convergence of Gradient Descent on Separable Data},
  author={Nacson, Mor Shpigel and Lee, Jason and Gunasekar, Suriya and Srebro, Nathan and Soudry, Daniel},
  journal={arXiv preprint arXiv:1803.01905},
  year={2018}
}

@article{nag2018,
  title={Deterministic PAC-Bayesian generalization bounds for deep networks via generalizing noise-resilience. },
  author={Vaishnavh Nagarajan and J. Zico Kolter},
  journal={Integration of Deep Learning Theories, NeurIPS 2018},
  year={2018}
}

@article{minshuo2018,
  title={On Generalization Bounds for a Family of Recurrent Neural Networks. },
  author={Chen, Minshuo and Li, Xingguo and Zhao, Tuo},
  journal={Integration of Deep Learning Theories, NeurIPS 2018},
  year={2018}
}

@article{neyshabur2018towards,
  title={Towards Understanding the Role of Over-Parametrization in Generalization of Neural Networks},
  author={Neyshabur, Behnam and Li, Zhiyuan and Bhojanapalli, Srinadh and LeCun, Yann and Srebro, Nathan},
  journal={arXiv preprint arXiv:1805.12076},
  year={2018}
}


@article{li2018tighter,
  title={On Tighter Generalization Bound for Deep Neural Networks: {CNNs}, {ResNets}, and Beyond},
  author={Li, Xingguo and Lu, Junwei and Wang, Zhaoran and Haupt, Jarvis and Zhao, Tuo},
  journal={arXiv preprint arXiv:1806.05159},
  year={2018}
}

@article{bousquet2002stability,
  title={Stability and generalization},
  author={Bousquet, Olivier and Elisseeff, Andr{\'e}},
  journal={Journal of machine learning research},
  volume={2},
  number={Mar},
  pages={499--526},
  year={2002}
}

@article{mou2017generalization,
  title={Generalization bounds of SGLD for non-convex learning: Two theoretical viewpoints},
  author={Mou, Wenlong and Wang, Liwei and Zhai, Xiyu and Zheng, Kai},
  journal={arXiv preprint arXiv:1707.05947},
  year={2017}
}



@article{chen2018stability,
  title={Stability and Convergence Trade-off of Iterative Optimization Algorithms},
  author={Chen, Yuansi and Jin, Chi and Yu, Bin},
  journal={arXiv preprint arXiv:1804.01619},
  year={2018}
}

@article{zhou2018generalization,
  title={Generalization Error Bounds with Probabilistic Guarantee for SGD in Nonconvex Optimization},
  author={Zhou, Yi and Liang, Yingbin and Zhang, Huishuai},
  journal={arXiv preprint arXiv:1802.06903},
  year={2018}
}



@article{zagoruyko2016wide,
  title={Wide residual networks},
  author={Zagoruyko, Sergey and Komodakis, Nikos},
  journal={arXiv preprint arXiv:1605.07146},
  year={2016}
}

@inproceedings{glorot2010understanding,
  title={Understanding the difficulty of training deep feedforward neural networks},
  author={Glorot, Xavier and Bengio, Yoshua},
  booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
  pages={249--256},
  year={2010}
}




@article{haeffele2015global,
  title={Global optimality in tensor factorization, deep learning, and beyond},
  author={Haeffele, Benjamin D and Vidal, Ren{\'e}},
  journal={arXiv preprint arXiv:1506.07540},
  year={2015}
}





@article{soudry2017exponentially,
  title={Exponentially vanishing sub-optimal local minima in multilayer neural networks},
  author={Soudry, Daniel and Hoffer, Elad},
  journal={arXiv preprint arXiv:1702.05777},
  year={2017}
}





@inproceedings{lin2018resnet,
  title={{ResNet} with one-neuron hidden layers is a Universal Approximator},
  author={Lin, Hongzhou and Jegelka, Stefanie},
  booktitle={Advances in Neural Information Processing Systems},
  pages={6172--6181},
  year={2018}
}





@book{mohri2018foundations,
  title={Foundations of machine learning},
  author={Mohri, Mehryar and Rostamizadeh, Afshin and Talwalkar, Ameet},
  year={2018},
  publisher={MIT press}
}



@article{yang2019scaling,
  title={Scaling limits of wide neural networks with weight sharing: {Gaussian} process behavior, gradient independence, and neural tangent kernel derivation},
  author={Yang, Greg},
  journal={arXiv preprint arXiv:1902.04760},
  year={2019}
}



@inproceedings{rahimi2008random,
  title={Random features for large-scale kernel machines},
  author={Rahimi, Ali and Recht, Benjamin},
  booktitle={Advances in neural information processing systems},
  pages={1177--1184},
  year={2008}
}


@article{song2018mean,
  title={A mean field view of the landscape of two-layers neural networks},
  author={Song, Mei and Montanari, A and Nguyen, P},
  journal={PNAS},
  volume={115},
  pages={E7665--E7671},
  year={2018}
}





@inproceedings{hardt2015train,
  title={Train faster, generalize better: Stability of stochastic gradient descent},
  author={Hardt, Moritz and Recht, Ben and Singer, Yoram},
  booktitle={International Conference on Machine Learning},
  pages={1225--1234},
  year={2016}
}



@inproceedings{gao2019learning,
  title={Learning One-hidden-layer Neural Networks under General Input Distributions},
  author={Gao, Weihao and Makkuva, Ashok and Oh, Sewoong and Viswanath, Pramod},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={1950--1959},
  year={2019}
}


















@string{JASA = {Journal of the American Statistical Association}}
@string{JC  = {Journal of Classification}}
@string{JSPI  = {Journal of Statistical Planning and Inference}}
@string{JRSSB  = {Journal of the Royal Statistical Society, Series B}}
@string{SAGMB  = {Statistical Applications in Genetics and Molecular Biology}}
@string{NIPS  = {Advances in Neural Information Processing Systems}}
@string{AOS  = {Annals of Statistics}}
@string{AOAS  = {Annals of Applied Statistics}}
@string{JMLR  = {Journal of Machine Learning Research}}
@string{EJS  = {Electronic Journal of Statistics}}
@string{AISTATS  = {International Conference on Artificial Intelligence and Statistics}}
@string{UAI  = {Conference on Uncertainty in Artificial Intelligence}}
@string{NIPS  = {Advances in Neural Information Processing Systems}}
@string{ICML  = {International Conference on Machine Learning}}
@string{TIT = {IEEE Transactions on Information Theory}}


@inproceedings{harvey2017nearly,
  title={Nearly-tight VC-dimension bounds for piecewise linear neural networks},
  author={Harvey, Nick and Liaw, Christopher and Mehrabian, Abbas},
  booktitle={Conference on Learning Theory},
  pages={1064--1068},
  year={2017}
}

@inproceedings{bartlett1999almost,
  title={Almost linear VC dimension bounds for piecewise polynomial networks},
  author={Bartlett, Peter L and Maiorov, Vitaly and Meir, Ron},
  booktitle={Advances in Neural Information Processing Systems},
  pages={190--196},
  year={1999}
}

@inproceedings{du2017convolutional,
	title={When is a Convolutional Filter Easy To Learn?},
	author={Du, Simon S and Lee, Jason D and Tian, Yuandong},
	booktitle={International Conference on Learning Representations},
	year={2018}
}




@article{hsu2012tail,
  title={A tail inequality for quadratic forms of subgaussian random vectors},
  author={Hsu, Daniel and Kakade, Sham and Zhang, Tong and others},
  journal={Electronic Communications in Probability},
  volume={17},
  year={2012},
  publisher={The Institute of Mathematical Statistics and the Bernoulli Society}
}

@article{vershynin2010introduction,
  title={Introduction to the non-asymptotic analysis of random matrices},
  author={Vershynin, Roman},
  journal={arXiv preprint arXiv:1011.3027},
  year={2010}
}

@article{dally2015high,
  title={High-performance hardware for machine learning},
  author={Dally, William},
  journal={NIPS Tutorial},
  year={2015}
}


@inproceedings{szegedy2015going,
  title={Going deeper with convolutions},
  author={Szegedy, Christian and Liu, Wei and Jia, Yangqing and Sermanet, Pierre and Reed, Scott and Anguelov, Dragomir and Erhan, Dumitru and Vanhoucke, Vincent and Rabinovich, Andrew},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={1--9},
  year={2015}
}
@inproceedings{ren2015faster,
  title={Faster {R-CNN}: Towards real-time object detection with region proposal networks},
  author={Ren, Shaoqing and He, Kaiming and Girshick, Ross and Sun, Jian},
  booktitle={Advances in neural information processing systems},
  pages={91--99},
  year={2015}
}
@inproceedings{devlin2014fast,
  title={Fast and Robust Neural Network Joint Models for Statistical Machine Translation.},
  author={Devlin, Jacob and Zbib, Rabih and Huang, Zhongqiang and Lamar, Thomas and Schwartz, Richard M and Makhoul, John},
  booktitle={ACL (1)},
  pages={1370--1380},
  year={2014}
}
@article{bahdanau2014neural,
  title={Neural machine translation by jointly learning to align and translate},
  author={Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua},
  journal={arXiv preprint arXiv:1409.0473},
  year={2014}
}

@article{baum1990polynomial,
  title={A polynomial time algorithm that learns two hidden unit nets},
  author={Baum, Eric B},
  journal={Neural Computation},
  volume={2},
  number={4},
  pages={510--522},
  year={1990},
  publisher={MIT Press}
}
@inproceedings{zhang2016convexified,
  title={Convexified Convolutional Neural Networks},
  author={Zhang, Yuchen and Liang, Percy and Wainwright, Martin J},
  booktitle={International Conference on Machine Learning},
  pages={4044--4053},
  year={2017}
}


@article{nguyen2017bloss,
  title={The loss surface and expressivity of deep convolutional neural networks},
  author={Nguyen, Quynh and Hein, Matthias},
  journal={arXiv preprint arXiv:1710.10928},
  year={2017}
}




@article{tian2016symmetry,
  title={Symmetry-breaking convergence analysis of certain two-layered neural networks with ReLU nonlinearity},
  author={Tian, Yuandong},
  year={2016}
}


@incollection{klivans2009baum,
  title={Baum’s algorithm learns intersections of halfspaces with respect to log-concave distributions},
  author={Klivans, Adam R and Long, Philip M and Tang, Alex K},
  booktitle={Approximation, Randomization, and Combinatorial Optimization. Algorithms and Techniques},
  pages={588--600},
  year={2009},
  publisher={Springer}
}


@inproceedings{daniely2016toward,
  title={Toward deeper understanding of neural networks: The power of initialization and a dual view on expressivity},
  author={Daniely, Amit and Frostig, Roy and Singer, Yoram},
  booktitle={Advances In Neural Information Processing Systems},
  pages={2253--2261},
  year={2016}
}

@inproceedings{cohen2016expressive,
  title={On the expressive power of deep learning: A tensor analysis},
  author={Cohen, Nadav and Sharir, Or and Shashua, Amnon},
  booktitle={Conference on Learning Theory},
  pages={698--728},
  year={2016}
}


@article{raghu2016expressive,
  title={On the expressive power of deep neural networks},
  author={Raghu, Maithra and Poole, Ben and Kleinberg, Jon and Ganguli, Surya and Sohl-Dickstein, Jascha},
  journal={arXiv preprint arXiv:1606.05336},
  year={2016}
}
@inproceedings{poole2016exponential,
  title={Exponential expressivity in deep neural networks through transient chaos},
  author={Poole, Ben and Lahiri, Subhaneil and Raghu, Maithreyi and Sohl-Dickstein, Jascha and Ganguli, Surya},
  booktitle={Advances In Neural Information Processing Systems},
  pages={3360--3368},
  year={2016}
}
@inproceedings{montufar2014number,
  title={On the number of linear regions of deep neural networks},
  author={Montufar, Guido F and Pascanu, Razvan and Cho, Kyunghyun and Bengio, Yoshua},
  booktitle={Advances in neural information processing systems},
  pages={2924--2932},
  year={2014}
}

@article{vsima2006training,
  title={Training a single sigmoidal neuron is hard},
  author={{\v{S}}{\'\i}ma, Ji{\v{r}}{\'\i}},
  journal={Training},
  volume={14},
  number={11},
  year={2006},
  publisher={MIT Press}
}
@inproceedings{livni2014computational,
  title={On the computational efficiency of training neural networks},
  author={Livni, Roi and Shalev-Shwartz, Shai and Shamir, Ohad},
  booktitle={Advances in Neural Information Processing Systems},
  pages={855--863},
  year={2014}
}

@inproceedings{shalev2017failures,
  title={Failures of gradient-based deep learning},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Shammah, Shaked},
  booktitle={International Conference on Machine Learning},
  pages={3067--3075},
  year={2017}
}
@article{shalev2017weight,
  title={Weight Sharing is Crucial to Succesful Optimization},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Shammah, Shaked},
  journal={arXiv preprint arXiv:1706.00687},
  year={2017}
}

@article{goel2016reliably,
  title={Reliably learning the relu in polynomial time},
  author={Goel, Surbhi and Kanade, Varun and Klivans, Adam and Thaler, Justin},
  journal={arXiv preprint arXiv:1611.10258},
  year={2016}
}
@article{zhang2015learning,
  title={Learning halfspaces and neural networks with random initialization},
  author={Zhang, Yuchen and Lee, Jason D and Wainwright, Martin J and Jordan, Michael I},
  journal={arXiv preprint arXiv:1511.07948},
  year={2015}
}
@article{sedghi2014provable,
  title={Provable methods for training neural networks with sparse connectivity},
  author={Sedghi, Hanie and Anandkumar, Anima},
  journal={arXiv preprint arXiv:1412.2693},
  year={2014}
}

@inproceedings{ge2015escaping,
  title={Escaping from saddle points—online stochastic gradient for tensor decomposition},
  author={Ge, Rong and Huang, Furong and Jin, Chi and Yuan, Yang},
  booktitle={Conference on Learning Theory},
  pages={797--842},
  year={2015}
}
@inproceedings{jin2016provable,
  title={Provable efficient online matrix completion via non-convex stochastic gradient descent},
  author={Jin, Chi and Kakade, Sham M and Netrapalli, Praneeth},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4520--4528},
  year={2016}
}
@article{jin2017escape,
  title={How to Escape Saddle Points Efficiently},
  author={Jin, Chi and Ge, Rong and Netrapalli, Praneeth and Kakade, Sham M and Jordan, Michael I},
  journal={arXiv preprint arXiv:1703.00887},
  year={2017}
}

@article{lu2017depth,
  title={Depth Creates No Bad Local Minima},
  author={Lu, Haihao and Kawaguchi, Kenji},
  journal={arXiv preprint arXiv:1702.08580},
  year={2017}
}
@inproceedings{yun2017global,
  title={Global optimality conditions for deep neural networks},
  author={Yun, Chulhee and Sra, Suvrit and Jadbabaie, Ali},
  booktitle={International Conference on Learning Representations},
  year={2018}
}

@article{taghvaei2017regularization,
  title={How regularization affects the critical points in linear networks},
  author={Taghvaei, Amirhossein and Kim, Jin W and Mehta, Prashant G},
  journal={arXiv preprint arXiv:1709.09625},
  year={2017}
}

@article{feizi2017porcupine,
  title={Porcupine Neural Networks:(Almost) All Local Optima are Global},
  author={Feizi, Soheil and Javadi, Hamid and Zhang, Jesse and Tse, David},
  journal={arXiv preprint arXiv:1710.02196},
  year={2017}
}


@article{soltanolkotabi2017theoretical,
  title={Theoretical insights into the optimization landscape of over-parameterized shallow neural networks},
  author={Soltanolkotabi, Mahdi and Javanmard, Adel and Lee, Jason D},
  journal={IEEE Transactions on Information Theory},
  volume={65},
  number={2},
  pages={742--769},
  year={2018},
  publisher={IEEE}
}


@article{carmon2017convex,
  title={" Convex Until Proven Guilty": Dimension-Free Acceleration of Gradient Descent on Non-Convex Functions},
  author={Carmon, Yair and Hinder, Oliver and Duchi, John C and Sidford, Aaron},
  journal={arXiv preprint arXiv:1705.02766},
  year={2017}
}


@inproceedings{choromanska2015loss,
  title={The loss surfaces of multilayer networks},
  author={Choromanska, Anna and Henaff, Mikael and Mathieu, Michael and Arous, G{\'e}rard Ben and LeCun, Yann},
  booktitle={Artificial Intelligence and Statistics},
  pages={192--204},
  year={2015}
}

@inproceedings{hardt2016identity,
  title={Identity matters in deep learning},
  author={Hardt, Moritz and Ma, Tengyu},
  booktitle={International Conference on Learning Representations},
  year={2017}
}

@article{arora2016provable,
  title={Provable learning of Noisy-or Networks},
  author={Arora, Sanjeev and Ge, Rong and Ma, Tengyu and Risteski, Andrej},
  journal={arXiv preprint arXiv:1612.08795},
  year={2016}
}

@inproceedings{zhang2017learnability,
  title={On the Learnability of Fully-Connected Neural Networks},
  author={Zhang, Yuchen and Lee, Jason and Wainwright, Martin and Jordan, Michael},
  booktitle={Artificial Intelligence and Statistics},
  pages={83--91},
  year={2017}
}
@inproceedings{jain2015fast,
  title={Fast exact matrix completion with finite samples},
  author={Jain, Prateek and Netrapalli, Praneeth},
  booktitle={Conference on Learning Theory},
  pages={1007--1034},
  year={2015}
}
@inproceedings{pan2016expressiveness,
  title={Expressiveness of rectifier networks},
  author={Pan, Xingyuan and Srikumar, Vivek},
  booktitle={International Conference on Machine Learning},
  pages={2427--2435},
  year={2016}
}

@article{barron1993universal,
  title={Universal approximation bounds for superpositions of a sigmoidal function},
  author={Barron, Andrew R},
  journal={IEEE Transactions on Information theory},
  volume={39},
  number={3},
  pages={930--945},
  year={1993},
  publisher={IEEE}
}


@inproceedings{auer1996exponentially,
  title={Exponentially many local minima for single neurons},
  author={Auer, Peter and Herbster, Mark and Warmuth, Manfred K},
  booktitle={Advances in neural information processing systems},
  pages={316--322},
  year={1996}
}
@inproceedings{ge2017learning,
  title={Learning One-hidden-layer Neural Networks with Landscape Design},
  author={Ge, Rong and Lee, Jason D and Ma, Tengyu},
  booktitle={International Conference on Learning Representations},
  year={2017}
}
@inproceedings{kalai2009isotron,
  title={The Isotron Algorithm: High-Dimensional Isotonic Regression.},
  author={Kalai, Adam Tauman and Sastry, Ravi},
  booktitle={COLT},
  year={2009},
  organization={Citeseer}
}
\nshortmid




@inproceedings{kakade2011efficient,
  title={Efficient learning of generalized linear and single index models with isotonic regression},
  author={Kakade, Sham M and Kanade, Varun and Shamir, Ohad and Kalai, Adam},
  booktitle={Advances in Neural Information Processing Systems},
  pages={927--935},
  year={2011}
}



@inproceedings{yi2015regularized,
  title={Regularized em algorithms: A unified framework and statistical guarantees},
  author={Yi, Xinyang and Caramanis, Constantine},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1567--1575},
  year={2015}
}




@misc{hoeffding1940masstabinvariante,
  title={Masstabinvariante Korrelationtheorie, Schriften des Mathematis chen Instituts und des Instituts f{\"u}r Angewandte Mathematik der Universit{\"a}t Berlin 5, 181\# 233.(Translated in Fisher, NI and PK Sen (1994). The Collected Works of Wassily Hoeffding, New York},
  author={Hoeffding, W},
  year={1940},
  publisher={Springer-Verlag}
}


@article{cuadras2002covariance,
  title={On the covariance between functions},
  author={Cuadras, Carles M},
  journal={Journal of Multivariate Analysis},
  volume={81},
  number={1},
  pages={19--27},
  year={2002},
  publisher={Elsevier}
}


@incollection{sen1994impact,
  title={The impact of Wassily Hoeffding’s research on nonparametrics},
  author={Sen, Pranab K},
  booktitle={The Collected Works of Wassily Hoeffding},
  pages={29--55},
  year={1994},
  publisher={Springer}
}



@article{gordon1985some,
  title={Some inequalities for {Gaussian} processes and applications},
  author={Gordon, Yehoram},
  journal={Israel Journal of Mathematics},
  volume={50},
  number={4},
  pages={265--289},
  year={1985},
  publisher={Springer}
}


@inproceedings{goel2018learning,
  title={Learning One Convolutional Layer with Overlapping Patches},
  author={Goel, Surbhi and Klivans, Adam and Meka, Raghu},
  booktitle={International Conference on Machine Learning},
  pages={1778--1786},
  year={2018}
}


@article{du2018improved,
  title={Improved Learning of One-hidden-layer Convolutional Neural Networks with Overlaps},
  author={Du, Simon S and Goel, Surbhi},
  journal={arXiv preprint arXiv:1805.07798},
  year={2018}
}

@inproceedings{du2018many,
  title={How Many Samples are Needed to Learn a Convolutional Neural Network?},
  author={Du, Simon S and Wang, Yining and Zhai, Xiyu and Balakrishnan, Sivaraman and Salakhutdinov, Ruslan and Singh, Aarti},
  booktitle={Advances in Neural Information Processing Systems},
  year={2018}
}



@book{talagrand2014upper,
  title={Upper and lower bounds for stochastic processes: modern methods and classical problems},
  author={Talagrand, Michel},
  volume={60},
  year={2014},
  publisher={Springer Science \& Business Media}
}





@inproceedings{du2018gradient,
  title={Gradient Descent Provably Optimizes Over-parameterized Neural Networks},
  author={Du, Simon S and Zhai, Xiyu and Poczos, Barnabas and Singh, Aarti},
  booktitle={International Conference on Learning Representations},
  year={2019}
}
@inproceedings{li2018learning,
  title={Learning overparameterized neural networks via stochastic gradient descent on structured data},
  author={Li, Yuanzhi and Liang, Yingyu},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8157--8166},
  year={2018}
}
@inproceedings{cotter2011better,
  title={Better mini-batch algorithms via accelerated gradient methods},
  author={Cotter, Andrew and Shamir, Ohad and Srebro, Nati and Sridharan, Karthik},
  booktitle={Advances in neural information processing systems},
  pages={1647--1655},
  year={2011}
}
@article{shalev2013stochastic,
  title={Stochastic dual coordinate ascent methods for regularized loss minimization},
  author={Shalev-Shwartz, Shai and Zhang, Tong},
  journal={Journal of Machine Learning Research},
  volume={14},
  number={Feb},
  pages={567--599},
  year={2013}
}
@inproceedings{brutzkus2017sgd,
  title={SGD learns over-parameterized networks that provably generalize on linearly separable data},
  author={Brutzkus, Alon and Globerson, Amir and Malach, Eran and Shalev-Shwartz, Shai},
  booktitle = {International Conference on Learning Representations},
  year={2018}
}

@inproceedings{nacson2018stochastic,
  title={Stochastic Gradient Descent on Separable Data: Exact Convergence with a Fixed Learning Rate},
  author={Nacson, Mor Shpigel and Srebro, Nathan and Soudry, Daniel},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={3051--3059},
  year={2019}
}


@article{ji2018gradient,
  title={Gradient descent aligns the layers of deep linear networks},
  author={Ji, Ziwei and Telgarsky, Matus},
  journal={arXiv preprint arXiv:1810.02032},
  year={2018}
}



@article{slepian1962one,
  title={The one-sided barrier problem for {Gaussian} noise},
  author={Slepian, David},
  journal={Bell Labs Technical Journal},
  volume={41},
  number={2},
  pages={463--501},
  year={1962},
  publisher={Wiley Online Library}
}




######################## Teacher Network



######################### Deep Linear Network
@article{saxe2013exact,
  title={Exact solutions to the nonlinear dynamics of learning in deep linear neural networks},
  author={Saxe, Andrew M and McClelland, James L and Ganguli, Surya},
  journal={arXiv preprint arXiv:1312.6120},
  year={2013}
}


@inproceedings{bartlett2018gradient,
  title={Gradient descent with identity initialization efficiently learns positive definite linear transformations},
  author={Bartlett, Peter and Helmbold, Dave and Long, Phil},
  booktitle={International Conference on Machine Learning},
  pages={520--529},
  year={2018}
}

@inproceedings{arora2018convergence,
  title={A Convergence Analysis of Gradient Descent for Deep Linear Neural Networks},
  author={Arora, Sanjeev and Cohen, Nadav and Golowich, Noah and Hu, Wei},
  booktitle={International Conference on Learning Representations},
  year={2019}
}




#####################Optimization Landscape


######################Generalization



@inproceedings{li2017convergence,
  title={Convergence analysis of two-layer neural networks with relu activation},
  author={Li, Yuanzhi and Yuan, Yang},
  booktitle={Advances in Neural Information Processing Systems},
  pages={597--607},
  year={2017}
}



@inproceedings{zhong2017recovery,
  title={Recovery Guarantees for One-hidden-layer Neural Networks},
  author={Zhong, Kai and Song, Zhao and Jain, Prateek and Bartlett, Peter L and Dhillon, Inderjit S},
  booktitle={International Conference on Machine Learning},
  pages={4140--4149},
  year={2017}
}

@article{tian2017analytical,
  title={An Analytical Formula of Population Gradient for two-layered ReLU network and its Applications in Convergence and Critical Point Analysis},
  author={Tian, Yuandong},
  journal={arXiv preprint arXiv:1703.00560},
  year={2017}
}


@article{hsu2012tail,
  title={A tail inequality for quadratic forms of subgaussian random vectors},
  author={Hsu, Daniel and Kakade, Sham and Zhang, Tong and others},
  journal={Electronic Communications in Probability},
  volume={17},
  year={2012},
  publisher={The Institute of Mathematical Statistics and the Bernoulli Society}
}


@article{dally2015high,
  title={High-performance hardware for machine learning},
  author={Dally, William},
  journal={NIPS Tutorial},
  year={2015}
}

@inproceedings{sutskever2014sequence,
  title={Sequence to sequence learning with neural networks},
  author={Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V},
  booktitle={Advances in neural information processing systems},
  pages={3104--3112},
  year={2014}
}

@inproceedings{krizhevsky2012imagenet,
  title={Imagenet classification with deep convolutional neural networks},
  author={Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
  booktitle={Advances in neural information processing systems},
  pages={1097--1105},
  year={2012}
}
@inproceedings{szegedy2015going,
  title={Going deeper with convolutions},
  author={Szegedy, Christian and Liu, Wei and Jia, Yangqing and Sermanet, Pierre and Reed, Scott and Anguelov, Dragomir and Erhan, Dumitru and Vanhoucke, Vincent and Rabinovich, Andrew},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={1--9},
  year={2015}
}
@inproceedings{devlin2014fast,
  title={Fast and Robust Neural Network Joint Models for Statistical Machine Translation.},
  author={Devlin, Jacob and Zbib, Rabih and Huang, Zhongqiang and Lamar, Thomas and Schwartz, Richard M and Makhoul, John},
  booktitle={ACL (1)},
  pages={1370--1380},
  year={2014}
}
@article{bahdanau2014neural,
  title={Neural machine translation by jointly learning to align and translate},
  author={Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua},
  journal={arXiv preprint arXiv:1409.0473},
  year={2014}
}
@article{silver2016mastering,
  title={Mastering the game of Go with deep neural networks and tree search},
  author={Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
  journal={Nature},
  volume={529},
  number={7587},
  pages={484--489},
  year={2016},
  publisher={Nature Research}
}

@article{hornik1991approximation,
  title={Approximation capabilities of multilayer feedforward networks},
  author={Hornik, Kurt},
  journal={Neural networks},
  volume={4},
  number={2},
  pages={251--257},
  year={1991},
  publisher={Elsevier}
}

@inproceedings{daniely2016toward,
  title={Toward deeper understanding of neural networks: The power of initialization and a dual view on expressivity},
  author={Daniely, Amit and Frostig, Roy and Singer, Yoram},
  booktitle={Advances In Neural Information Processing Systems},
  pages={2253--2261},
  year={2016}
}

@inproceedings{lu2017expressive,
  title={The expressive power of neural networks: A view from the width},
  author={Lu, Zhou and Pu, Hongming and Wang, Feicheng and Hu, Zhiqiang and Wang, Liwei},
  booktitle={Advances in neural information processing systems},
  pages={6231--6239},
  year={2017}
}

@inproceedings{cohen2016expressive,
  title={On the expressive power of deep learning: A tensor analysis},
  author={Cohen, Nadav and Sharir, Or and Shashua, Amnon},
  booktitle={Conference on Learning Theory},
  pages={698--728},
  year={2016}
}
@inproceedings{cohen2016convolutional,
  title={Convolutional rectifier networks as generalized tensor decompositions},
  author={Cohen, Nadav and Shashua, Amnon},
  booktitle={International Conference on Machine Learning},
  pages={955--963},
  year={2016}
}
@inproceedings{telgarsky2016benefits,
  title={benefits of depth in neural networks},
  author={Telgarsky, Matus},
  booktitle={Conference on Learning Theory},
  pages={1517--1539},
  year={2016}
}
@article{raghu2016expressive,
  title={On the expressive power of deep neural networks},
  author={Raghu, Maithra and Poole, Ben and Kleinberg, Jon and Ganguli, Surya and Sohl-Dickstein, Jascha},
  journal={arXiv preprint arXiv:1606.05336},
  year={2016}
}
@inproceedings{poole2016exponential,
  title={Exponential expressivity in deep neural networks through transient chaos},
  author={Poole, Ben and Lahiri, Subhaneil and Raghu, Maithreyi and Sohl-Dickstein, Jascha and Ganguli, Surya},
  booktitle={Advances In Neural Information Processing Systems},
  pages={3360--3368},
  year={2016}
}
@inproceedings{montufar2014number,
  title={On the number of linear regions of deep neural networks},
  author={Montufar, Guido F and Pascanu, Razvan and Cho, Kyunghyun and Bengio, Yoshua},
  booktitle={Advances in neural information processing systems},
  pages={2924--2932},
  year={2014}
}

@article{vsima2006training,
  title={Training a single sigmoidal neuron is hard},
  author={{\v{S}}{\'\i}ma, Ji{\v{r}}{\'\i}},
  journal={Training},
  volume={14},
  number={11},
  year={2006},
  publisher={MIT Press}
}
@inproceedings{livni2014computational,
  title={On the computational efficiency of training neural networks},
  author={Livni, Roi and Shalev-Shwartz, Shai and Shamir, Ohad},
  booktitle={Advances in Neural Information Processing Systems},
  pages={855--863},
  year={2014}
}
@article{shamir2016distribution,
  title={Distribution-specific hardness of learning neural networks},
  author={Shamir, Ohad},
  journal={The Journal of Machine Learning Research},
  volume={19},
  number={1},
  pages={1135--1163},
  year={2018},
  publisher={JMLR. org}
}
@inproceedings{shalev2017failures,
  title={Failures of gradient-based deep learning},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Shammah, Shaked},
  booktitle={International Conference on Machine Learning},
  pages={3067--3075},
  year={2017}
}
@article{shalev2017weight,
  title={Weight Sharing is Crucial to Succesful Optimization},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Shammah, Shaked},
  journal={arXiv preprint arXiv:1706.00687},
  year={2017}
}

@article{goel2016reliably,
  title={Reliably learning the relu in polynomial time},
  author={Goel, Surbhi and Kanade, Varun and Klivans, Adam and Thaler, Justin},
  journal={arXiv preprint arXiv:1611.10258},
  year={2016}
}
@article{zhang2015learning,
  title={Learning halfspaces and neural networks with random initialization},
  author={Zhang, Yuchen and Lee, Jason D and Wainwright, Martin J and Jordan, Michael I},
  journal={arXiv preprint arXiv:1511.07948},
  year={2015}
}
@article{sedghi2014provable,
  title={Provable methods for training neural networks with sparse connectivity},
  author={Sedghi, Hanie and Anandkumar, Anima},
  journal={arXiv preprint arXiv:1412.2693},
  year={2014}
}
@article{janzamin2015beating,
  title={Beating the perils of non-convexity: Guaranteed training of neural networks using tensor methods},
  author={Janzamin, Majid and Sedghi, Hanie and Anandkumar, Anima},
  journal={arXiv preprint arXiv:1506.08473},
  year={2015}
}
@inproceedings{ge2015escaping,
  title={Escaping from saddle points—online stochastic gradient for tensor decomposition},
  author={Ge, Rong and Huang, Furong and Jin, Chi and Yuan, Yang},
  booktitle={Conference on Learning Theory},
  pages={797--842},
  year={2015}
}
@inproceedings{jin2016provable,
  title={Provable efficient online matrix completion via non-convex stochastic gradient descent},
  author={Jin, Chi and Kakade, Sham M and Netrapalli, Praneeth},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4520--4528},
  year={2016}
}
@article{jin2017escape,
  title={How to Escape Saddle Points Efficiently},
  author={Jin, Chi and Ge, Rong and Netrapalli, Praneeth and Kakade, Sham M and Jordan, Michael I},
  journal={arXiv preprint arXiv:1703.00887},
  year={2017}
}

@article{lu2017depth,
  title={Depth Creates No Bad Local Minima},
  author={Lu, Haihao and Kawaguchi, Kenji},
  journal={arXiv preprint arXiv:1702.08580},
  year={2017}
}

@article{taghvaei2017regularization,
  title={How regularization affects the critical points in linear networks},
  author={Taghvaei, Amirhossein and Kim, Jin W and Mehta, Prashant G},
  journal={arXiv preprint arXiv:1709.09625},
  year={2017}
}

@article{feizi2017porcupine,
  title={Porcupine Neural Networks:(Almost) All Local Optima are Global},
  author={Feizi, Soheil and Javadi, Hamid and Zhang, Jesse and Tse, David},
  journal={arXiv preprint arXiv:1710.02196},
  year={2017}
}



@article{carmon2017convex,
  title={" Convex Until Proven Guilty": Dimension-Free Acceleration of Gradient Descent on Non-Convex Functions},
  author={Carmon, Yair and Hinder, Oliver and Duchi, John C and Sidford, Aaron},
  journal={arXiv preprint arXiv:1705.02766},
  year={2017}
}


@inproceedings{safran2016quality,
  title={On the quality of the initial basin in overspecified neural networks},
  author={Safran, Itay and Shamir, Ohad},
  booktitle={International Conference on Machine Learning},
  pages={774--782},
  year={2016}
}


@article{arora2016provable,
  title={Provable learning of Noisy-or Networks},
  author={Arora, Sanjeev and Ge, Rong and Ma, Tengyu and Risteski, Andrej},
  journal={arXiv preprint arXiv:1612.08795},
  year={2016}
}
@inproceedings{xie2017diverse,
  title={Diverse Neural Network Learns True Target Functions},
  author={Xie, Bo and Liang, Yingyu and Song, Le},
  booktitle={Artificial Intelligence and Statistics},
  pages={1216--1224},
  year={2017}
}
@inproceedings{zhang2017learnability,
  title={On the Learnability of Fully-Connected Neural Networks},
  author={Zhang, Yuchen and Lee, Jason and Wainwright, Martin and Jordan, Michael},
  booktitle={Artificial Intelligence and Statistics},
  pages={83--91},
  year={2017}
}
@inproceedings{jain2015fast,
  title={Fast exact matrix completion with finite samples},
  author={Jain, Prateek and Netrapalli, Praneeth},
  booktitle={Conference on Learning Theory},
  pages={1007--1034},
  year={2015}
}
@inproceedings{pan2016expressiveness,
  title={Expressiveness of rectifier networks},
  author={Pan, Xingyuan and Srikumar, Vivek},
  booktitle={International Conference on Machine Learning},
  pages={2427--2435},
  year={2016}
}

@article{barron1993universal,
  title={Universal approximation bounds for superpositions of a sigmoidal function},
  author={Barron, Andrew R},
  journal={IEEE Transactions on Information theory},
  volume={39},
  number={3},
  pages={930--945},
  year={1993},
  publisher={IEEE}
}
@inproceedings{zhang2016understanding,
  title={Understanding deep learning requires rethinking generalization},
  author={Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
  booktitle={International Conference on Learning Representations},
  year={2017}
}
@inproceedings{mei2016landscape,
  title={The landscape of empirical risk for non-convex losses},
  author={Mei, Song and Bai, Yu and Montanari, Andrea},
  booktitle={The Annals of Statistics},
  year={2018}
}
@inproceedings{du2017gradient,
  title={Gradient Descent Learns One-hidden-layer {CNN}: Don’t be Afraid of Spurious Local Minima},
  author={Du, Simon S and Lee, Jason D and Tian, Yuandong and Singh, Aarti and Poczos, Barnabas},
  booktitle={International Conference on Machine Learning},
  pages={1338--1347},
  year={2018}
}
@inproceedings{auer1996exponentially,
  title={Exponentially many local minima for single neurons},
  author={Auer, Peter and Herbster, Mark and Warmuth, Manfred K},
  booktitle={Advances in neural information processing systems},
  pages={316--322},
  year={1996}
}

@inproceedings{kalai2009isotron,
  title={The Isotron Algorithm: High-Dimensional Isotonic Regression.},
  author={Kalai, Adam Tauman and Sastry, Ravi},
  booktitle={COLT},
  year={2009},
  organization={Citeseer}
}

@inproceedings{fu2018local,
  title={Local Geometry of Cross Entropy Loss in Learning One-Hidden-Layer Neural Networks},
  author={Fu, Haoyu and Chi, Yuejie and Liang, Yingbin},
  booktitle={2019 IEEE International Symposium on Information Theory (ISIT)},
  pages={1972--1976},
  year={2019},
  organization={IEEE}
}

@inproceedings{kakade2011efficient,
  title={Efficient learning of generalized linear and single index models with isotonic regression},
  author={Kakade, Sham M and Kanade, Varun and Shamir, Ohad and Kalai, Adam},
  booktitle={Advances in Neural Information Processing Systems},
  pages={927--935},
  year={2011}
}

@inproceedings{jain2013low,
  title={Low-rank matrix completion using alternating minimization},
  author={Jain, Prateek and Netrapalli, Praneeth and Sanghavi, Sujay},
  booktitle={Proceedings of the forty-fifth annual ACM symposium on Theory of computing},
  pages={665--674},
  year={2013},
  organization={ACM}
}

@inproceedings{sutskever2013importance,
  title={On the importance of initialization and momentum in deep learning},
  author={Sutskever, Ilya and Martens, James and Dahl, George and Hinton, Geoffrey},
  booktitle={International conference on machine learning},
  pages={1139--1147},
  year={2013}
}



@inproceedings{livni2014computational,
  title={On the computational efficiency of training neural networks},
  author={Livni, Roi and Shalev-Shwartz, Shai and Shamir, Ohad},
  booktitle={Advances in Neural Information Processing Systems},
  pages={855--863},
  year={2014}
}




@article{barron1993universal,
  title={Universal approximation bounds for superpositions of a sigmoidal function},
  author={Barron, Andrew R},
  journal={IEEE Transactions on Information theory},
  volume={39},
  number={3},
  pages={930--945},
  year={1993},
  publisher={IEEE}
}



@inproceedings{liang2016deep,
  title={Why deep neural networks for function approximation?},
  author={Liang, Shiyu and Srikant, R},
  booktitle={International Conference on Learning Representations},
  year={2017}
}

@article{yarotsky2017error,
  title={Error bounds for approximations with deep {ReLU} networks},
  author={Yarotsky, Dmitry},
  journal={Neural Networks},
  volume={94},
  pages={103--114},
  year={2017},
  publisher={Elsevier}
}

@article{neyshabur2014search,
  title={In search of the real inductive bias: On the role of implicit regularization in deep learning},
  author={Neyshabur, Behnam and Tomioka, Ryota and Srebro, Nathan},
  journal={arXiv preprint arXiv:1412.6614},
  year={2014}
}


@inproceedings{sutskever2013importance,
  title={On the importance of initialization and momentum in deep learning},
  author={Sutskever, Ilya and Martens, James and Dahl, George and Hinton, Geoffrey},
  booktitle={International conference on machine learning},
  pages={1139--1147},
  year={2013}
}

@inproceedings{blum1989training,
  title={Training a 3-node neural network is NP-complete},
  author={Blum, Avrim and Rivest, Ronald L},
  booktitle={Advances in neural information processing systems},
  pages={494--501},
  year={1989}
}

#########gaussian##############





@article{zhong2017learning,
  title={Learning Non-overlapping Convolutional Neural Networks with Multiple Kernels},
  author={Zhong, Kai and Song, Zhao and Dhillon, Inderjit S},
  journal={arXiv preprint arXiv:1711.03440},
  year={2017}
}


#############independent assumption 


@inproceedings{choromanska2015loss,
  title={The loss surfaces of multilayer networks},
  author={Choromanska, Anna and Henaff, Mikael and Mathieu, Michael and Arous, G{\'e}rard Ben and LeCun, Yann},
  booktitle={Artificial Intelligence and Statistics},
  pages={192--204},
  year={2015}
}

@inproceedings{du2018power,
  title={On the Power of Over-parametrization in Neural Networks with Quadratic Activation},
  author={Du, Simon S and Lee, Jason D},
  booktitle={International Conference on Machine Learning},
  pages={1328--1337},
  year={2018}
}


@inproceedings{allen2018convergence,
  title={A Convergence Theory for Deep Learning via Over-Parameterization},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  booktitle={International Conference on Machine Learning},
  pages={242--252},
  year={2019}
}


@inproceedings{du2018gradientdeep,
  title={Gradient Descent Finds Global Minima of Deep Neural Networks},
  author={Du, Simon and Lee, Jason and Li, Haochuan and Wang, Liwei and Zhai, Xiyu},
  booktitle={International Conference on Machine Learning},
  pages={1675--1685},
  year={2019}
}


@inproceedings{bartlett2018gradient,
  title={Gradient descent with identity initialization efficiently learns positive definite linear transformations},
  author={Bartlett, Peter and Helmbold, Dave and Long, Phil},
  booktitle={International Conference on Machine Learning},
  pages={520--529},
  year={2018}
}

@inproceedings{langford2002not,
  title={(Not) bounding the true error},
  author={Langford, John and Caruana, Rich},
  booktitle={Advances in Neural Information Processing Systems},
  pages={809--816},
  year={2002}
}


###############generalization#################
@inproceedings{neyshabur2017pac,
  title={A pac-bayesian approach to spectrally-normalized margin bounds for neural networks},
  author={Neyshabur, Behnam and Bhojanapalli, Srinadh and McAllester, David and Srebro, Nathan},
  booktitle={International Conference on Learning Representation},
  year={2018}
}

@inproceedings{neyshabur2017exploring,
  title={Exploring generalization in deep learning},
  author={Neyshabur, Behnam and Bhojanapalli, Srinadh and McAllester, David and Srebro, Nati},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5947--5956},
  year={2017}
}

@inproceedings{neyshabur2015norm,
  title={Norm-based capacity control in neural networks},
  author={Neyshabur, Behnam and Tomioka, Ryota and Srebro, Nathan},
  booktitle={Conference on Learning Theory},
  pages={1376--1401},
  year={2015}
}


@book{anthony2009neural,
  title={Neural network learning: Theoretical foundations},
  author={Anthony, Martin and Bartlett, Peter L},
  year={2009},
  publisher={cambridge university press}
}

@book{vapnik2013nature,
  title={The nature of statistical learning theory},
  author={Vapnik, Vladimir},
  year={2013},
  publisher={Springer science \& business media}
}

@inproceedings{golowich2017size,
  title={Size-Independent Sample Complexity of Neural Networks},
  author={Golowich, Noah and Rakhlin, Alexander and Shamir, Ohad},
  booktitle={Conference On Learning Theory},
  pages={297--299},
  year={2018}
}

@inproceedings{arora2018stronger,
  title={Stronger Generalization Bounds for Deep Nets via a Compression Approach},
  author={Arora, Sanjeev and Ge, Rong and Neyshabur, Behnam and Zhang, Yi},
  booktitle={International Conference on Machine Learning},
  pages={254--263},
  year={2018}
}

@article{arora2016understanding,
  title={Understanding deep neural networks with rectified linear units},
  author={Arora, Raman and Basu, Amitabh and Mianjy, Poorya and Mukherjee, Anirbit},
  journal={arXiv preprint arXiv:1611.01491},
  year={2016}
}


###########more citation#################


@article{hanin2017universal,
  title={Universal function approximation by deep neural nets with bounded width and relu activations},
  author={Hanin, Boris},
  journal={arXiv preprint arXiv:1708.02691},
  year={2017}
}

@article{hanin2017approximating,
  title={Approximating Continuous Functions by {ReLU} Nets of Minimal Width},
  author={Hanin, Boris and Sellke, Mark},
  journal={arXiv preprint arXiv:1710.11278},
  year={2017}
}


@inproceedings{allen2018rnn,
  title={On the Convergence Rate of Training Recurrent Neural Networks},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}

@inproceedings{zhang2018learning,
  title={Learning One-hidden-layer {ReLU} Networks via Gradient Descent},
  author={Zhang, Xiao and Yu, Yaodong and Wang, Lingxiao and Gu, Quanquan},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={1524--1534},
  year={2019}
}

@article{telgarsky2015representation,
  title={Representation benefits of deep feedforward networks},
  author={Telgarsky, Matus},
  journal={arXiv preprint arXiv:1509.08101},
  year={2015}
}


@inproceedings{allen2018learning,
  title={Learning and Generalization in Overparameterized Neural Networks, Going Beyond Two Layers},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Liang, Yingyu},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}


@inproceedings{nacson2018convergence,
  title={Convergence of Gradient Descent on Separable Data},
  author={Nacson, Mor Shpigel and Lee, Jason and Gunasekar, Suriya and Savarese, Pedro Henrique Pamplona and Srebro, Nathan and Soudry, Daniel},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={3420--3428},
  year={2019}
}



@article{nag2018,
  title={Deterministic PAC-Bayesian generalization bounds for deep networks via generalizing noise-resilience. },
  author={Vaishnavh Nagarajan and J. Zico Kolter},
  journal={Integration of Deep Learning Theories, NeurIPS 2018},
  year={2018}
}

@article{minshuo2018,
  title={On Generalization Bounds for a Family of Recurrent Neural Networks. },
  author={Chen, Minshuo and Li, Xingguo and Zhao, Tuo},
  journal={Integration of Deep Learning Theories, NeurIPS 2018},
  year={2018}
}



@article{bousquet2002stability,
  title={Stability and generalization},
  author={Bousquet, Olivier and Elisseeff, Andr{\'e}},
  journal={Journal of machine learning research},
  volume={2},
  number={Mar},
  pages={499--526},
  year={2002}
}

@article{mou2017generalization,
  title={Generalization bounds of SGLD for non-convex learning: Two theoretical viewpoints},
  author={Mou, Wenlong and Wang, Liwei and Zhai, Xiyu and Zheng, Kai},
  journal={arXiv preprint arXiv:1707.05947},
  year={2017}
}

@inproceedings{hardt2016train,
  title={Train faster, generalize better: stability of stochastic gradient descent},
  author={Hardt, Moritz and Recht, Benjamin and Singer, Yoram},
  booktitle={Proceedings of the 33rd International Conference on International Conference on Machine Learning-Volume 48},
  pages={1225--1234},
  year={2016},
  organization={JMLR. org}
}

@article{chen2018stability,
  title={Stability and Convergence Trade-off of Iterative Optimization Algorithms},
  author={Chen, Yuansi and Jin, Chi and Yu, Bin},
  journal={arXiv preprint arXiv:1804.01619},
  year={2018}
}

@article{zhou2018generalization,
  title={Generalization Error Bounds with Probabilistic Guarantee for SGD in Nonconvex Optimization},
  author={Zhou, Yi and Liang, Yingbin and Zhang, Huishuai},
  journal={arXiv preprint arXiv:1802.06903},
  year={2018}
}

@inproceedings{chizat2018global,
  title={On the global convergence of gradient descent for over-parameterized models using optimal transport},
  author={Chizat, Lenaic and Bach, Francis},
  booktitle={Advances in neural information processing systems},
  pages={3036--3046},
  year={2018}
}


@inproceedings{arora2018optimization,
  title={On the Optimization of Deep Networks: Implicit Acceleration by Overparameterization},
  author={Arora, Sanjeev and Cohen, Nadav and Hazan, Elad},
  booktitle={International Conference on Machine Learning},
  pages={244--253},
  year={2018}
}


@article{zagoruyko2016wide,
  title={Wide residual networks},
  author={Zagoruyko, Sergey and Komodakis, Nikos},
  journal={arXiv preprint arXiv:1605.07146},
  year={2016}
}

@inproceedings{glorot2010understanding,
  title={Understanding the difficulty of training deep feedforward neural networks},
  author={Glorot, Xavier and Bengio, Yoshua},
  booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
  pages={249--256},
  year={2010}
}

@inproceedings{he2015delving,
  title={Delving deep into rectifiers: Surpassing human-level performance on imagenet classification},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE international conference on computer vision},
  pages={1026--1034},
  year={2015}
}



@inproceedings{dziugaite2017computing,
  title={Computing nonvacuous generalization bounds for deep (stochastic) neural networks with many more parameters than training data},
  author={Dziugaite, Gintare Karolina and Roy, Daniel M},
  booktitle={Uncertainty in Artificial Intelligence},
  year={2017}
}

@inproceedings{gunasekar2018implicit,
  title={Implicit bias of gradient descent on linear convolutional networks},
  author={Gunasekar, Suriya and Lee, Jason D and Soudry, Daniel and Srebro, Nati},
  booktitle={Advances in Neural Information Processing Systems},
  pages={9461--9471},
  year={2018}
}


@inproceedings{gunasekar2018characterizing,
  title={Characterizing Implicit Bias in Terms of Optimization Geometry},
  author={Gunasekar, Suriya and Lee, Jason and Soudry, Daniel and Srebro, Nathan},
  booktitle={International Conference on Machine Learning},
  pages={1827--1836},
  year={2018}
}


@inproceedings{liang2018just,
  title={Just Interpolate: Kernel" Ridgeless" Regression Can Generalize},
  author={Liang, Tengyuan and Rakhlin, Alexander},
  booktitle={The Annals of Statistics},
  year={2019}
}



@article{mei2018mean,
  title={A mean field view of the landscape of two-layer neural networks},
  author={Mei, Song and Montanari, Andrea and Nguyen, Phan-Minh},
  journal={Proceedings of the National Academy of Sciences},
  volume={115},
  number={33},
  pages={E7665--E7671},
  year={2018},
  publisher={National Acad Sciences}
}

@inproceedings{li2018algorithmic,
  title={Algorithmic regularization in over-parameterized matrix sensing and neural networks with quadratic activations},
  author={Li, Yuanzhi and Ma, Tengyu and Zhang, Hongyang},
  booktitle={Conference On Learning Theory},
  pages={2--47},
  year={2018}
}

@inproceedings{belkin2018understand,
  title={To Understand Deep Learning We Need to Understand Kernel Learning},
  author={Belkin, Mikhail and Ma, Siyuan and Mandal, Soumik},
  booktitle={International Conference on Machine Learning},
  pages={540--548},
  year={2018}
}

@inproceedings{gunasekar2017implicit,
  title={Implicit regularization in matrix factorization},
  author={Gunasekar, Suriya and Woodworth, Blake E and Bhojanapalli, Srinadh and Neyshabur, Behnam and Srebro, Nati},
  booktitle={Advances in Neural Information Processing Systems},
  pages={6151--6159},
  year={2017}
}



@inproceedings{arora2019fine,
  title={Fine-Grained Analysis of Optimization and Generalization for Overparameterized Two-Layer Neural Networks},
  author={Arora, Sanjeev and Du, Simon and Hu, Wei and Li, Zhiyuan and Wang, Ruosong},
  booktitle={International Conference on Machine Learning},
  pages={322--332},
  year={2019}
}

@inproceedings{soltanolkotabi2017learning,
  title={Learning {ReLUs} via gradient descent},
  author={Soltanolkotabi, Mahdi},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2007--2017},
  year={2017}
}


@inproceedings{daniely2017sgd,
  title={SGD learns the conjugate kernel class of the network},
  author={Daniely, Amit},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2422--2430},
  year={2017}
}



@article{wei2018regularization,
  title={Regularization Matters: Generalization and Optimization of Neural Nets v.s. their Induced Kernel},
  author={Wei, Colin and Lee, Jason D and Liu, Qiang and Ma, Tengyu},
  journal={Advances in Neural Information Processing Systems},
  year={2019}
}

@article{soudry2017exponentially,
  title={Exponentially vanishing sub-optimal local minima in multilayer neural networks},
  author={Soudry, Daniel and Hoffer, Elad},
  journal={arXiv preprint arXiv:1702.05777},
  year={2017}
}

@article{zhou2017critical,
  title={Critical Points of Neural Networks: Analytical Forms and Landscape Properties},
  author={Zhou, Yi and Liang, Yingbin},
  journal={arXiv preprint arXiv:1710.11205},
  year={2017}
}

@inproceedings{yun2019small,
  title={Small nonlinearities in activation functions create bad local minima in neural networks},
  author={Yun, Chulhee and Sra, Suvrit and Jadbabaie, Ali},
  booktitle={International Conference on Learning Representation},
  year={2019}
}

@article{sirignano2019mean,
  title={Mean field analysis of neural networks: A central limit theorem},
  author={Sirignano, Justin and Spiliopoulos, Konstantinos},
  journal={Stochastic Processes and their Applications},
  year={2019},
  publisher={Elsevier}
}



@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@article{rotskoff2018neural,
  title={Neural networks as Interacting Particle Systems: Asymptotic convexity of the Loss Landscape and Universal Scaling of the Approximation Error},
  author={Rotskoff, Grant M and Vanden-Eijnden, Eric},
  journal={arXiv preprint arXiv:1805.00915},
  year={2018}
}

@inproceedings{jacot2018neural,
  title={Neural tangent kernel: Convergence and generalization in neural networks},
  author={Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
  booktitle={Advances in neural information processing systems},
  pages={8571--8580},
  year={2018}
}

@inproceedings{chizat2018note,
  title={On Lazy Training in Differentiable Programming},
  author={Chizat, Lenaic and Oyallon, Edouard and Bach, Francis},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}






@inproceedings{rahimi2009weighted,
  title={Weighted sums of random kitchen sinks: Replacing minimization with randomization in learning},
  author={Rahimi, Ali and Recht, Benjamin},
  booktitle={Advances in neural information processing systems},
  pages={1313--1320},
  year={2009}
}

@inproceedings{mendelson2014learning,
  title={Learning without concentration},
  author={Mendelson, Shahar},
  booktitle={Conference on Learning Theory},
  pages={25--39},
  year={2014}
}





@inproceedings{maurer2016vector,
  title={A vector-contraction inequality for rademacher complexities},
  author={Maurer, Andreas},
  booktitle={International Conference on Algorithmic Learning Theory},
  pages={3--17},
  year={2016},
  organization={Springer}
}



@article{cesa2004generalization,
  title={On the generalization ability of on-line learning algorithms},
  author={Cesa-Bianchi, Nicolo and Conconi, Alex and Gentile, Claudio},
  journal={IEEE Transactions on Information Theory},
  volume={50},
  number={9},
  pages={2050--2057},
  year={2004},
  publisher={IEEE}
}




@inproceedings{yehudai2019power,
  title={On the Power and Limitations of Random Features for Understanding Neural Networks},
  author={Yehudai, Gilad and Shamir, Ohad},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}




@article{e2019comparative,
  title={A Comparative Analysis of the Optimization and Generalization Property of Two-layer Neural Network and Random Feature Models Under Gradient Descent Dynamics},
  author={E, Weinan and Ma, Chao and Wu, Lei and others},
  journal={arXiv preprint arXiv:1904.04326},
  year={2019}
}



@inproceedings{rahimi2008random,
  title={Random features for large-scale kernel machines},
  author={Rahimi, Ali and Recht, Benjamin},
  booktitle={Advances in neural information processing systems},
  pages={1177--1184},
  year={2008}
}


@article{oymak2019towards,
  title={Towards moderate overparameterization: global convergence guarantees for training shallow neural networks},
  author={Oymak, Samet and Soltanolkotabi, Mahdi},
  journal={arXiv preprint arXiv:1902.04674},
  year={2019}
}

@inproceedings{neyshabur2018role,
  title={Towards Understanding the Role of Over-Parametrization in Generalization of Neural Networks},
  author={Neyshabur, Behnam and Li, Zhiyuan and Bhojanapalli, Srinadh and LeCun, Yann and Srebro, Nathan},
  booktitle={International Conference on Learning Representations},
  year={2019}
}


@inproceedings{arora2019exact,
  title={On exact computation with an infinitely wide neural net},
  author={Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and Salakhutdinov, Ruslan and Wang, Ruosong},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}

@inproceedings{lee2019wide,
  title={Wide neural networks of any depth evolve as linear models under gradient descent},
  author={Lee, Jaehoon and Xiao, Lechao and Schoenholz, Samuel S and Bahri, Yasaman and Sohl-Dickstein, Jascha and Pennington, Jeffrey},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}


@article{long2019size,
  title={Size-free generalization bounds for convolutional neural networks},
  author={Long, Philip M and Sedghi, Hanie},
  journal={arXiv preprint arXiv:1905.12600},
  year={2019}
}

@inproceedings{jain2019making,
  title={Making the Last Iterate of SGD Information Theoretically Optimal},
  author={Jain, Prateek and Nagaraj, Dheeraj and Netrapalli, Praneeth},
  booktitle={Conference on Learning Theory},
  year={2019}
}


@article{lecun1998gradient,
  title={Gradient-based learning applied to document recognition},
  author={LeCun, Yann and Bottou, L{\'e}on and Bengio, Yoshua and Haffner, Patrick and others},
  journal={Proceedings of the IEEE},
  volume={86},
  number={11},
  pages={2278--2324},
  year={1998},
  publisher={Taipei, Taiwan}
}

@inproceedings{zou2019improved,
  title={An Improved Analysis of Training Over-parameterized Deep Neural Networks},
  author={Zou, Difan and Gu, Quanquan},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}


@inproceedings{cao2019generalizationsgd,
  title={Generalization Bounds of Stochastic Gradient Descent for Wide and Deep Neural Networks},
  author={Cao, Yuan and Gu, Quanquan},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}




@article{fang2019convexformulation,
  title={Convex Formulation of Overparameterized Deep Neural Networks},
  author={Fang, Cong and Gu, Yihong and Zhang, Weizhong and Zhang, Tong},
  journal={arXiv preprint arXiv:1911.07626},
  year={2019}
}


@inproceedings{nguyen2019connected,
  title={On Connected Sublevel Sets in Deep Learning},
  author={Nguyen, Quynh},
  booktitle={International Conference on Machine Learning},
  pages={4790--4799},
  year={2019}
}



@inproceedings{cao2019tight,
  title={Tight Sample Complexity of Learning One-hidden-layer Convolutional Neural Networks},
  author={Cao, Yuan and Gu, Quanquan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={10611--10621},
  year={2019}
}




@inproceedings{frei2019algorithm,
  title={Algorithm-Dependent Generalization Bounds for Overparameterized Deep Residual Networks},
  author={Frei, Spencer and Cao, Yuan and Gu, Quanquan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={14769--14779},
  year={2019}
}


@inproceedings{nguyen2019connected,
  title={On Connected Sublevel Sets in Deep Learning},
  author={Nguyen, Quynh},
  booktitle={International Conference on Machine Learning},
  pages={4790--4799},
  year={2019}
}


@article{zou2018stochastic,
  title={Stochastic gradient descent optimizes over-parameterized deep relu networks},
  author={Zou, Difan and Cao, Yuan and Zhou, Dongruo and Gu, Quanquan},
  journal={arXiv preprint arXiv:1811.08888},
  year={2018}
}



@inproceedings{frei2019algorithm,
  title={Algorithm-Dependent Generalization Bounds for Overparameterized Deep Residual Networks},
  author={Frei, Spencer and Cao, Yuan and Gu, Quanquan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={14769--14779},
  year={2019}
}



@article{sirignano2019mean,
  title={Mean field analysis of neural networks: A central limit theorem},
  author={Sirignano, Justin and Spiliopoulos, Konstantinos},
  journal={Stochastic Processes and their Applications},
  year={2019},
  publisher={Elsevier}
}





@inproceedings{bietti2019inductive,
  title={On the inductive bias of neural tangent kernels},
  author={Bietti, Alberto and Mairal, Julien},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}




@inproceedings{basri2019convergence,
  title={The Convergence Rate of Neural Networks for Learned Functions of Different Frequencies},
  author={Basri, Ronen and Jacobs, David and Kasten, Yoni and Kritchman, Shira},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}




@inproceedings{nakkiran2019sgd,
  title={SGD on Neural Networks Learns Functions of Increasing Complexity},
  author={Nakkiran, Preetum and Kaplun, Gal and Kalimeris, Dimitris and Yang, Tristan and Edelman, Benjamin L and Zhang, Fred and Barak, Boaz},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}


@inproceedings{rahaman2019spectral,
  title={On the Spectral Bias of Neural Networks},
  author={Rahaman, Nasim and Baratin, Aristide and Arpit, Devansh and Draxler, Felix and Lin, Min and Hamprecht, Fred and Bengio, Yoshua and Courville, Aaron},
  booktitle={International Conference on Machine Learning},
  pages={5301--5310},
  year={2019}
}



@inproceedings{vempala2018gradient,
  title={Gradient Descent for One-Hidden-Layer Neural Networks: Polynomial Convergence and SQ Lower Bounds},
  author={Vempala, Santosh and Wilmes, John},
  booktitle={Conference on Learning Theory},
  year={2019}
}



@inproceedings{su2019learning,
  title={On Learning Over-parameterized Neural Networks: A Functional Approximation Prospective},
  author={Su, Lili and Yang, Pengkun},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}

@Article{zhang2019resnet,
author="Zhang, Huishuai
and Yu, Da
and Yi, Mingyang
and Chen, Wei
and Liu, Tie-Yan",
title="Convergence Theory of Learning Over-parameterized ResNet:
A Full Characterization",
journal="arXiv preprint arXiv:1903.07120",
year="2019",
month="Jul",
day="15"
}





@article{cao2019towards,
  title={Towards Understanding the Spectral Bias of Deep Learning},
  author={Cao, Yuan and Fang, Zhiying and Wu, Yue and Zhou, Ding-Xuan and Gu, Quanquan},
  journal={arXiv preprint arXiv:1912.01198},
  year={2019}
}



@article{chen2019much,
  title={How Much Over-parameterization Is Sufficient to Learn Deep ReLU Networks?},
  author={Chen, Zixiang and Cao, Yuan and Zou, Difan and Gu, Quanquan},
  journal={arXiv preprint arXiv:1911.12360},
  year={2019}
}



@article{allen2020backward,
  title={Backward Feature Correction: How Deep Learning Performs Deep Learning},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi},
  journal={arXiv preprint arXiv:2001.04413},
  year={2020}
}



@article{lyu2019gradient,
  title={Gradient Descent Maximizes the Margin of Homogeneous Neural Networks},
  author={Lyu, Kaifeng and Li, Jian},
  journal={arXiv preprint arXiv:1906.05890},
  year={2019}
}


@article{nacson2019lexicographic,
  title={Lexicographic and Depth-Sensitive Margins in Homogeneous and Non-Homogeneous Deep Models},
  author={Nacson, Mor Shpigel and Gunasekar, Suriya and Lee, Jason D and Srebro, Nathan and Soudry, Daniel},
  journal={arXiv preprint arXiv:1905.07325},
  year={2019}
}



@article{meir2003generalization,
  title={Generalization error bounds for Bayesian mixture algorithms},
  author={Meir, Ron and Zhang, Tong},
  journal={Journal of Machine Learning Research},
  volume={4},
  number={Oct},
  pages={839--860},
  year={2003}
}


@article{donsker1983asymptotic,
  title={Asymptotic evaluation of certain Markov process expectations for large time. IV},
  author={Donsker, Monroe D and Varadhan, SR Srinivasa},
  journal={Communications on Pure and Applied Mathematics},
  volume={36},
  number={2},
  pages={183--212},
  year={1983},
  publisher={Wiley Online Library}
}





@inproceedings{li2019towards,
  title={Towards explaining the regularization effect of initial large learning rate in training neural networks},
  author={Li, Yuanzhi and Wei, Colin and Ma, Tengyu},
  booktitle={Advances in Neural Information Processing Systems},
  pages={11669--11680},
  year={2019}
}




@book{bakry2013analysis,
  title={Analysis and geometry of Markov diffusion operators},
  author={Bakry, Dominique and Gentil, Ivan and Ledoux, Michel},
  volume={348},
  year={2013},
  publisher={Springer Science \& Business Media}
}




@article{tzen2020mean,
  title={A mean-field theory of lazy training in two-layer neural nets: entropic regularization and controlled McKean-Vlasov dynamics},
  author={Tzen, Belinda and Raginsky, Maxim},
  journal={arXiv preprint arXiv:2002.01987},
  year={2020}
}



@article{bach2017breaking,
  title={Breaking the curse of dimensionality with convex neural networks},
  author={Bach, Francis},
  journal={The Journal of Machine Learning Research},
  volume={18},
  number={1},
  pages={629--681},
  year={2017},
  publisher={JMLR. org}
}



@article{liu2020toward,
  title={Toward a theory of optimization for over-parameterized systems of non-linear equations: the lessons of deep learning},
  author={Liu, Chaoyue and Zhu, Libin and Belkin, Mikhail},
  journal={arXiv preprint arXiv:2003.00307},
  year={2020}
}





@article{wang2020benign,
  title={Benign Overfitting in Binary Classification of Gaussian Mixtures},
  author={Wang, Ke and Thrampoulidis, Christos},
  journal={arXiv preprint arXiv:2011.09148},
  year={2020}
}


@article{bartlett2020benign,
  title={Benign overfitting in linear regression},
  author={Bartlett, Peter L and Long, Philip M and Lugosi, G{\'a}bor and Tsigler, Alexander},
  journal={Proceedings of the National Academy of Sciences},
  year={2020},
  publisher={National Acad Sciences}
}



@article{muthukumar2020classification,
  title={Classification vs regression in overparameterized regimes: Does the loss function matter?},
  author={Muthukumar, Vidya and Narang, Adhyyan and Subramanian, Vignesh and Belkin, Mikhail and Hsu, Daniel and Sahai, Anant},
  journal={arXiv preprint arXiv:2005.08054},
  year={2020}
}



@article{chatterji2020finite,
  title={Finite-sample analysis of interpolating linear classifiers in the overparameterized regime},
  author={Chatterji, Niladri S and Long, Philip M},
  journal={arXiv preprint arXiv:2004.12019},
  year={2020}
}




@article{muthukumar2020harmless,
  title={Harmless interpolation of noisy data in regression},
  author={Muthukumar, Vidya and Vodrahalli, Kailas and Subramanian, Vignesh and Sahai, Anant},
  journal={IEEE Journal on Selected Areas in Information Theory},
  volume={1},
  number={1},
  pages={67--83},
  year={2020},
  publisher={IEEE}
}


@article{tsigler2020benign,
  title={Benign overfitting in ridge regression},
  author={Tsigler, Alexander and Bartlett, Peter L},
  journal={arXiv preprint arXiv:2009.14286},
  year={2020}
}




@article{wu2020optimal,
  title={On the Optimal Weighted $\ell_2$ Regularization in Overparameterized Linear Regression},
  author={Wu, Denny and Xu, Ji},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}


@inproceedings{liao2020random,
  title={A random matrix analysis of random Fourier features: beyond the Gaussian kernel, a precise phase transition, and the corresponding double descent},
  author={Liao, Zhenyu and Couillet, Romain and Mahoney, Michael},
  booktitle={34th Conference on Neural Information Processing Systems (NeurIPS 2020)},
  year={2020}
}


@article{montanari2020interpolation,
  title={The interpolation phase transition in neural networks: Memorization and generalization under lazy training},
  author={Montanari, Andrea and Zhong, Yiqiao},
  journal={arXiv preprint arXiv:2007.12826},
  year={2020}
}


@article{arora2019implicit,
  title={Implicit regularization in deep matrix factorization},
  author={Arora, Sanjeev and Cohen, Nadav and Hu, Wei and Luo, Yuping},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}