@article{jacot_neural_2020,
	title = {Neural Tangent Kernel: Convergence and Generalization in Neural Networks},
	shorttitle = {Neural Tangent Kernel},
	journaltitle = {{arXiv}:1806.07572 [cs, math, stat]},
	author = {Jacot, Arthur and Gabriel, Franck and Hongler, Clément},
	date = {2020-02-10},
	eprinttype = {arxiv},
	eprint = {1806.07572},
}

@article{jacot_neural_2018,
  title={Neural tangent kernel: Convergence and generalization in neural networks},
  author={Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
  journal={Advances in neural information processing systems},
  volume={31},
  year={2018}
}

@article{arora_exact_2019,
  title={On exact computation with an infinitely wide neural net},
  author={Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and Salakhutdinov, Russ R and Wang, Ruosong},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}


@article{zagoruyko_wide_2017,
	title = {Wide Residual Networks},
	abstract = {Deep residual networks were shown to be able to scale up to thousands of layers and still have improving performance. However, each fraction of a percent of improved accuracy costs nearly doubling the number of layers, and so training very deep residual networks has a problem of diminishing feature reuse, which makes these networks very slow to train. To tackle these problems, in this paper we conduct a detailed experimental study on the architecture of {ResNet} blocks, based on which we propose a novel architecture where we decrease depth and increase width of residual networks. We call the resulting network structures wide residual networks ({WRNs}) and show that these are far superior over their commonly used thin and very deep counterparts. For example, we demonstrate that even a simple 16-layer-deep wide residual network outperforms in accuracy and efficiency all previous deep residual networks, including thousand-layer-deep networks, achieving new state-of-the-art results on {CIFAR}, {SVHN}, {COCO}, and significant improvements on {ImageNet}. Our code and models are available at https://github.com/szagoruyko/wide-residual-networks},
	journaltitle = {{arXiv}:1605.07146 [cs]},
	author = {Zagoruyko, Sergey and Komodakis, Nikos},
	date = {2017-06-14},
	eprinttype = {arxiv},
	eprint = {1605.07146},
	keywords = {Computer Science - Machine Learning, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Neural and Evolutionary Computing},
}

@inproceedings{lee_wide_2019,
    title={Wide neural networks of any depth evolve as linear models under gradient descent},
    author={Lee, Jaehoon and Xiao, Lechao and Schoenholz, Samuel and Bahri, Yasaman and Novak, Roman and Sohl-Dickstein, Jascha and Pennington, Jeffrey},
    booktitle={Advances in neural information processing systems},
    pages={8570--8581},
    year={2019}
}


@inproceedings{chizat_lazy_2019,
	title = {On lazy training in differentiable programming},
	pages = {2933--2943},
	booktitle = {Advances in Neural Information Processing Systems},
	author = {Chizat, Lenaic and Oyallon, Edouard and Bach, Francis},
	year = {2019}
}

@article{zagoruyko_wide_2016,
    title={Wide residual networks},
    author={Zagoruyko, Sergey and Komodakis, Nikos},
    journal={arXiv preprint arXiv:1605.07146},
    year={2016}
}

@inproceedings{roy_effective_2007,
	title = {The effective rank: A measure of effective dimensionality},
	pages = {606--610},
	booktitle = {15th European Signal Processing Conference},
	publisher = {{IEEE}},
	author = {Roy, Olivier and Vetterli, Martin},
	year = {2007}
}

@article{yang_scaling_2020,
    title={Scaling limits of wide neural networks with weight sharing: Gaussian process behavior, gradient independence, and neural tangent kernel derivation},
    author={Yang, Greg},
    journal={arXiv preprint arXiv:1902.04760},
    year={2019}
}

@inproceedings{novak_neural_2020,
	title = {Neural Tangents},
	booktitle = {8th International Conference on Learning Representations, {ICLR}, Addis Ababa, Ethiopia, April 26-30},
	author = {Novak, Roman and Xiao, Lechao and Hron, Jiri and Lee, Jaehoon and Alemi, Alexander A. and Sohl-Dickstein, Jascha and Schoenholz, Samuel S.},
	year = {2020}
}

@inproceedings{he_delving_2015,
    title={Delving deep into rectifiers: Surpassing human-level performance on imagenet classification},
    author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
    booktitle={Proceedings of the IEEE international conference on computer vision},
    pages={1026--1034},
    year={2015}
    }


@article{huang_dynamics_2019,
    title={Dynamics of deep neural networks and neural tangent hierarchy},
    author={Huang, Jiaoyang and Yau, Horng-Tzer},
    journal={arXiv preprint arXiv:1909.08156},
    year={2019}
}

@inproceedings{chizat_global_2018,
	title = {On the global convergence of gradient descent for over-parameterized models using optimal transport},
	pages = {3036--3046},
	booktitle = {Advances in neural information processing systems},
	author = {Chizat, Lenaic and Bach, Francis},
	year = {2018}
}

@article{javanmard_analysis_2019,
    title={Analysis of a two-layer neural network via displacement convexity},
    author={Javanmard, Adel and Mondelli, Marco and Montanari, Andrea},
    journal={arXiv preprint arXiv:1901.01375},
    year={2019}
}

@inproceedings{mei_mean-field_2019,
	title = {Mean-field theory of two-layers neural networks: dimension-free bounds and kernel limit},
	volume = {99},
	pages = {2388--2464},
	booktitle = {Proceedings of the Thirty-Second Conference on Learning Theory},
	publisher = {{PMLR}},
	author = {Mei, Song and Misiakiewicz, Theodor and Montanari, Andrea},
	editor = {Beygelzimer, Alina and Hsu, Daniel},
	year = {2019}
}

@article{nguyen_mean_2019,
    title={Mean field limit of the learning dynamics of multilayer neural networks},
    author={Nguyen, Phan-Minh},
    journal={arXiv preprint arXiv:1902.02880},
    year={2019}
}

@inproceedings{rotskoff_global_2019,
	title = {Global convergence of neuron birth-death dynamics},
	pages = {9689--9698},
	booktitle = {36th International Conference on Machine Learning},
	author = {Rotskoff, Grant M. and Jelassi, Samy and Bruna, Joan and Vanden-Eijnden, Eric},
	year = {2019}
}


@article{hu_mean-field_2019,
    title={Mean-field Langevin dynamics and energy landscape of neural networks},
    author={Hu, Kaitong and Ren, Zhenjie and Siska, David and Szpruch, Lukasz},
    journal={arXiv preprint arXiv:1905.07769},
    year={2019}
    }


@article{lecun_gradient-based_1998,
	title = {Gradient-based learning applied to document recognition},
	volume = {86},
	pages = {2278--2324},
	number = {11},
	journaltitle = {Proceedings of the {IEEE}},
	author = {{LeCun}, Yann and Bottou, Léon and Bengio, Yoshua and Haffner, Patrick},
	date = {1998},
	note = {Publisher: Ieee}
}

@inproceedings{krizhevsky_imagenet_2012,
	title = {Imagenet classification with deep convolutional neural networks},
	pages = {1097--1105},
	booktitle = {Advances in neural information processing systems},
	author = {Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
	date = {2012}
}

@article{krizhevsky_one_2014,
	title = {One weird trick for parallelizing convolutional neural networks},
	journaltitle = {{arXiv} preprint {arXiv}:1404.5997},
	author = {Krizhevsky, Alex},
	date = {2014}
}

@article{krizhevsky_learning_2009,
	title = {Learning multiple layers of features from tiny images},
	author = {Krizhevsky, Alex and Hinton, Geoffrey and {others}},
	date = {2009},
	note = {Publisher: Citeseer}
}

@article{russakovsky_imagenet_2015,
	title = {{ImageNet} Large Scale Visual Recognition Challenge},
	volume = {115},
	doi = {10.1007/s11263-015-0816-y},
	pages = {211--252},
	number = {3},
	journaltitle = {International Journal of Computer Vision ({IJCV})},
	author = {Russakovsky, Olga and Deng, Jia and Su, Hao and Krause, Jonathan and Satheesh, Sanjeev and Ma, Sean and Huang, Zhiheng and Karpathy, Andrej and Khosla, Aditya and Bernstein, Michael and Berg, Alexander C. and Fei-Fei, Li},
	date = {2015}
}


@inproceedings{simonyan_very_2015,
  author    = {Karen Simonyan and
               Andrew Zisserman},
  editor    = {Yoshua Bengio and
               Yann LeCun},
  title     = {Very Deep Convolutional Networks for Large-Scale Image Recognition},
  booktitle = {3rd International Conference on Learning Representations, {ICLR} 2015,
               San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings},
  year      = {2015},
  timestamp = {Wed, 17 Jul 2019 10:40:54 +0200},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}




@inproceedings{he_deep_2016,
    title={Deep residual learning for image recognition},
    author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
    booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
    pages={770--778},
    year={2016}
}





@article{lenet_98,
    title={Gradient-based learning applied to document recognition},
    author={LeCun, Yann and Bottou, L{\'e}on and Bengio, Yoshua and Haffner, Patrick},
    journal={Proceedings of the IEEE},
    volume={86},
    number={11},
    pages={2278--2324},
    year={1998},
    publisher={Ieee}
}

@inproceedings{alexnet_2012,
    title={Imagenet classification with deep convolutional neural networks},
    author={Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
    booktitle={Advances in neural information processing systems},
    pages={1097--1105},
    year={2012}
}

@article{alexnetV2_2014,
    title={One weird trick for parallelizing convolutional neural networks},
    author={Krizhevsky, Alex},
    journal={arXiv preprint arXiv:1404.5997},
    year={2014}
}

@article{krizhevsky2009learning,
  added-at = {2021-01-21T03:01:11.000+0100},
  author = {Krizhevsky, Alex},
  interhash = {cc2d42f2b7ef6a4e76e47d1a50c8cd86},
  intrahash = {fe5248afe57647d9c85c50a98a12145c},
  keywords = {},
  pages = {32--33},
  timestamp = {2021-01-21T03:01:11.000+0100},
  title = {Learning Multiple Layers of Features from Tiny Images},
  year = 2009
}

@article{CIFAR_2009,
    title={Learning multiple layers of features from tiny images},
    author={Krizhevsky, Alex and Hinton, Geoffrey and others},
    year={2009},
    publisher={Citeseer},
    journal={Citeseer}
}

@article{ImageNet_2015,
    Author = {Olga Russakovsky and others},
    Title = {{ImageNet Large Scale Visual Recognition Challenge}},
    Year = {2015},
    journal   = {International Journal of Computer Vision (IJCV)},
    doi = {10.1007/s11263-015-0816-y},
    volume={115},
    number={3},
    pages={211-252}
}

@incollection{pytorch_2019,
title = {PyTorch: An Imperative Style, High-Performance Deep Learning Library},
author = {Paszke, Adam and others},
booktitle = {Advances in Neural Information Processing Systems 32},
editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
pages = {8024--8035},
year = {2019},
publisher = {Curran Associates, Inc.},
}


@article{adam_2014,
    title={Adam: A method for stochastic optimization},
    author={Kingma, Diederik P and Ba, Jimmy},
    journal={arXiv preprint arXiv:1412.6980},
    year={2014}
}

% new

@book{anderson1999lapack,
  title={LAPACK Users' guide},
  author={Anderson, Edward and Bai, Zhaojun and Bischof, Christian and Blackford, L Susan and Demmel, James and Dongarra, Jack and Du Croz, Jeremy and Greenbaum, Anne and Hammarling, Sven and McKenney, Alan and others},
  year={1999},
  publisher={SIAM}
}

@inproceedings{
du2018overparamerization,
title={Gradient Descent Provably Optimizes Over-parameterized Neural Networks},
author={Simon S. Du and Xiyu Zhai and Barnabas Poczos and Aarti Singh},
booktitle={International Conference on Learning Representations},
year={2019},
}

@misc{matthews2018gaussian,
      title={Gaussian Process Behaviour in Wide Deep Neural Networks}, 
      author={Alexander G. de G. Matthews and Mark Rowland and Jiri Hron and Richard E. Turner and Zoubin Ghahramani},
      year={2018},
      eprint={1804.11271},
      archivePrefix={arXiv},
      primaryClass={stat.ML}
}

@inproceedings{
lee2018deep,
title={Deep Neural Networks as Gaussian Processes},
author={Jaehoon Lee and Jascha Sohl-dickstein and Jeffrey Pennington and Roman Novak and Sam Schoenholz and Yasaman Bahri},
booktitle={International Conference on Learning Representations},
year={2018},
}

@inproceedings{
garriga-alonso2018cnngp,
title={Deep Convolutional Networks as shallow Gaussian Processes},
author={Adrià Garriga-Alonso and Carl Edward Rasmussen and Laurence Aitchison},
booktitle={International Conference on Learning Representations},
year={2019},
}

@article{yang2020tensor_ntk,
  title={Tensor programs ii: Neural tangent kernel for any architecture},
  author={Yang, Greg},
  journal={arXiv preprint arXiv:2006.14548},
  year={2020}
}

@article{alemohammad2020rntk,
  title={The recurrent neural tangent kernel},
  author={Alemohammad, Sina and Wang, Zichao and Balestriero, Randall and Baraniuk, Richard},
  journal={arXiv preprint arXiv:2006.10246},
  year={2020}
}

@article{huang2020deep,
  title={Why Do Deep Residual Networks Generalize Better than Deep Feedforward Networks?---A Neural Tangent Kernel Perspective},
  author={Huang, Kaixuan and Wang, Yuqing and Tao, Molei and Zhao, Tuo},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={2698--2709},
  year={2020}
}

@article{lee2020finite,
  title={Finite versus infinite neural networks: an empirical study},
  author={Lee, Jaehoon and Schoenholz, Samuel and Pennington, Jeffrey and Adlam, Ben and Xiao, Lechao and Novak, Roman and Sohl-Dickstein, Jascha},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={15156--15172},
  year={2020}
}

@inproceedings{huang2021flntk,
  title={Fl-ntk: A neural tangent kernel-based framework for federated learning analysis},
  author={Huang, Baihe and Li, Xiaoxiao and Song, Zhao and Yang, Xin},
  booktitle={International Conference on Machine Learning},
  pages={4423--4434},
  year={2021},
  organization={PMLR}
}

@inproceedings{
kanoh2022a,
title={A Neural Tangent Kernel Perspective of Infinite Tree Ensembles},
author={Ryuichi Kanoh and Mahito Sugiyama},
booktitle={International Conference on Learning Representations},
year={2022},
}

@misc{seleznova2022analyzing,
      title={Analyzing Finite Neural Networks: Can We Trust Neural Tangent Kernel Theory?}, 
      author={Mariia Seleznova and Gitta Kutyniok},
      year={2022},
      eprint={2012.04477},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@inproceedings{chen2020labelawareNTK,
  author={Shuxiao Chen and Hangfeng He and Weijie J. Su},
  title={Label-Aware Neural Tangent Kernel: Toward Better Generalization and Local Elasticity},
  year={2020},
  cdate={1577836800000},
  booktitle={NeurIPS}
}

@inproceedings{
Hanin2020Finite,
title={Finite Depth and Width Corrections to the Neural Tangent Kernel},
author={Boris Hanin and Mihai Nica},
booktitle={International Conference on Learning Representations},
year={2020},
}

@inproceedings{xiao2020disentangling,
  title={Disentangling trainability and generalization in deep neural networks},
  author={Xiao, Lechao and Pennington, Jeffrey and Schoenholz, Samuel},
  booktitle={International Conference on Machine Learning},
  pages={10462--10472},
  year={2020},
  organization={PMLR}
}

@misc{wei2020regularization,
      title={Regularization Matters: Generalization and Optimization of Neural Nets v.s. their Induced Kernel}, 
      author={Colin Wei and Jason D. Lee and Qiang Liu and Tengyu Ma},
      year={2020},
      eprint={1810.05369},
      archivePrefix={arXiv},
      primaryClass={stat.ML}
}

@article{Geiger_2020,
	doi = {10.1088/1742-5468/abc4de},
	year = 2020,
	month = {nov},
	publisher = {{IOP} Publishing},
	volume = {2020},
	number = {11},
	pages = {113301},
	author = {Mario Geiger and Stefano Spigler and Arthur Jacot and Matthieu Wyart},
	title = {Disentangling feature and lazy training in deep neural networks},
	journal = {Journal of Statistical Mechanics: Theory and Experiment},
}

@article{rahimi2007random,
  title={Random features for large-scale kernel machines},
  author={Rahimi, Ali and Recht, Benjamin},
  journal={Advances in neural information processing systems},
  volume={20},
  year={2007}
}

@inproceedings{alekseev2019gabornet,
  title={GaborNet: Gabor filters with learnable parameters in deep convolutional neural network},
  author={Alekseev, Andrey and Bobe, Anatoly},
  booktitle={2019 International Conference on Engineering and Telecommunication (EnT)},
  pages={1--4},
  year={2019},
  organization={IEEE}
}

@inproceedings{lee2009convdeepbeliefnet,
  title={Convolutional deep belief networks for scalable unsupervised learning of hierarchical representations},
  author={Lee, Honglak and Grosse, Roger and Ranganath, Rajesh and Ng, Andrew Y},
  booktitle={Proceedings of the 26th annual international conference on machine learning},
  pages={609--616},
  year={2009}
}

@article{bai2019deep,
  title={Deep equilibrium models},
  author={Bai, Shaojie and Kolter, J Zico and Koltun, Vladlen},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@article{tancik2020fourier,
  title={Fourier features let networks learn high frequency functions in low dimensional domains},
  author={Tancik, Matthew and Srinivasan, Pratul and Mildenhall, Ben and Fridovich-Keil, Sara and Raghavan, Nithin and Singhal, Utkarsh and Ramamoorthi, Ravi and Barron, Jonathan and Ng, Ren},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={7537--7547},
  year={2020}
}

@article{williams1996computing,
  title={Computing with infinite networks},
  author={Williams, Christopher},
  journal={Advances in neural information processing systems},
  volume={9},
  year={1996}
}

@book{neal1996bayesian,
  title={Bayesian learning for neural networks},
  author={Neal, Radford M},
  volume={118},
  year={1996},
  publisher={Lecture Notes in Statistics}
}

@article{ortiz2021linNNgeneralization,
  title={What can linearized neural networks actually say about generalization?},
  author={Ortiz-Jim{\'e}nez, Guillermo and Moosavi-Dezfooli, Seyed-Mohsen and Frossard, Pascal},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{fort2020deep,
  title={Deep learning versus kernel learning},
  author={Fort, Stanislav and Dziugaite, Gintare Karolina and Paul, Mansheej and Kharaghani, Sepideh and Roy, Daniel M and Ganguli, Surya},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={5850--5861},
  year={2020}
}

@inproceedings{kornblith2019similarity,
  title={Similarity of neural network representations revisited},
  author={Kornblith, Simon and Norouzi, Mohammad and Lee, Honglak and Hinton, Geoffrey},
  booktitle={International Conference on Machine Learning},
  pages={3519--3529},
  year={2019},
  organization={PMLR}
}

@inproceedings{yang2021tensorIV,
  title={Tensor programs iv: Feature learning in infinite-width neural networks},
  author={Yang, Greg and Hu, Edward J},
  booktitle={International Conference on Machine Learning},
  pages={11727--11737},
  year={2021},
  organization={PMLR}
}

@article{zou2020gradient,
  title={Gradient descent optimizes over-parameterized deep ReLU networks},
  author={Zou, Difan and Cao, Yuan and Zhou, Dongruo and Gu, Quanquan},
  journal={Machine Learning},
  volume={109},
  number={3},
  pages={467--492},
  year={2020},
  publisher={Springer}
}

@inproceedings{allen2019convergence,
  title={A convergence theory for deep learning via over-parameterization},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  booktitle={International Conference on Machine Learning},
  pages={242--252},
  year={2019},
  organization={PMLR}
}
