

@article{huang2019gpipe,
  title={Gpipe: Efficient training of giant neural networks using pipeline parallelism},
  author={Huang, Yanping and Cheng, Youlong and Bapna, Ankur and Firat, Orhan and Chen, Dehao and Chen, Mia and Lee, HyoukJoong and Ngiam, Jiquan and Le, Quoc V and Wu, Yonghui and others},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@inproceedings{langley00,
 author    = {P. Langley},
 title     = {Crafting Papers on Machine Learning},
 year      = {2000},
 pages     = {1207--1216},
 editor    = {Pat Langley},
 booktitle     = {Proceedings of the 17th International Conference
              on Machine Learning (ICML 2000)},
 address   = {Stanford, CA},
 publisher = {Morgan Kaufmann}
}

@TechReport{mitchell80,
  author = 	 "T. M. Mitchell",
  title = 	 "The Need for Biases in Learning Generalizations",
  institution =  "Computer Science Department, Rutgers University",
  year = 	 "1980",
  address =	 "New Brunswick, MA",
}

@phdthesis{kearns89,
  author = {M. J. Kearns},
  title =  {Computational Complexity of Machine Learning},
  school = {Department of Computer Science, Harvard University},
  year =   {1989}
}

@Book{MachineLearningI,
  editor = 	 "R. S. Michalski and J. G. Carbonell and T.
		  M. Mitchell",
  title = 	 "Machine Learning: An Artificial Intelligence
		  Approach, Vol. I",
  publisher = 	 "Tioga",
  year = 	 "1983",
  address =	 "Palo Alto, CA"
}

@Book{DudaHart2nd,
  author =       "R. O. Duda and P. E. Hart and D. G. Stork",
  title =        "Pattern Classification",
  publisher =    "John Wiley and Sons",
  edition =      "2nd",
  year =         "2000"
}

@misc{anonymous,
  title= {Suppressed for Anonymity},
  author= {Author, N. N.},
  year= {2021}
}

@InCollection{Newell81,
  author =       "A. Newell and P. S. Rosenbloom",
  title =        "Mechanisms of Skill Acquisition and the Law of
                  Practice", 
  booktitle =    "Cognitive Skills and Their Acquisition",
  pages =        "1--51",
  publisher =    "Lawrence Erlbaum Associates, Inc.",
  year =         "1981",
  editor =       "J. R. Anderson",
  chapter =      "1",
  address =      "Hillsdale, NJ"
}


@Article{Samuel59,
  author = 	 "A. L. Samuel",
  title = 	 "Some Studies in Machine Learning Using the Game of
		  Checkers",
  journal =	 "IBM Journal of Research and Development",
  year =	 "1959",
  volume =	 "3",
  number =	 "3",
  pages =	 "211--229"
}


@inproceedings{alexnet,
  title={Imagenet classification with deep convolutional neural networks},
  author={Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
  booktitle={Advances in neural information processing systems},
  pages={1097--1105},
  year={2012}
}

@inproceedings{decentralizedparallelsgd,
  title={Can decentralized algorithms outperform centralized algorithms? a case study for decentralized parallel stochastic gradient descent},
  author={Lian, Xiangru and Zhang, Ce and Zhang, Huan and Hsieh, Cho-Jui and Zhang, Wei and Liu, Ji},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5330--5340},
  year={2017}
}

@inproceedings{easgd,
  title={Deep learning with elastic averaging SGD},
  author={Zhang, Sixin and Choromanska, Anna E and LeCun, Yann},
  booktitle={Advances in neural information processing systems},
  pages={685--693},
  year={2015}
}

@inproceedings{parallelsgdanalysis,
  title={Distributed delayed stochastic optimization},
  author={Agarwal, Alekh and Duchi, John C},
  booktitle={Advances in Neural Information Processing Systems},
  pages={873--881},
  year={2011}
}

@inproceedings{parallelsgd,
  title={Parallelized stochastic gradient descent},
  author={Zinkevich, Martin and Weimer, Markus and Li, Lihong and Smola, Alex J},
  booktitle={Advances in neural information processing systems},
  pages={2595--2603},
  year={2010}
}

@inproceedings{stochdepth,
  title={Deep networks with stochastic depth},
  author={Huang, Gao and Sun, Yu and Liu, Zhuang and Sedra, Daniel and Weinberger, Kilian Q},
  booktitle={European conference on computer vision},
  pages={646--661},
  year={2016},
  organization={Springer}
}


@article{comm-comp,
  title={Communication compression for decentralized training},
  author={Tang, Hanlin and Gan, Shaoduo and Zhang, Ce and Zhang, Tong and Liu, Ji},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}


@article{low-prec-trn,
  title={High-accuracy low-precision training},
  author={De Sa, Christopher and Leszczynski, Megan and Zhang, Jian and Marzoev, Alana and Aberger, Christopher R and Olukotun, Kunle and R{\'e}, Christopher},
  journal={arXiv preprint arXiv:1803.03383},
  year={2018}
}


@inproceedings{sparse-comm,
  title={Sparse Communication for Distributed Gradient Descent},
  author={Aji, Alham Fikri and Heafield, Kenneth},
  booktitle={Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
  pages={440--445},
  year={2017}
}


@article{double-quant,
  title={Double quantization for communication-efficient distributed optimization},
  author={Yu, Yue and Wu, Jiaxiang and Huang, Longbo},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}


@article{linear-speed-quant,
  title={A linear speedup analysis of distributed deep learning with sparse and quantized communication},
  author={Jiang, Peng and Agrawal, Gagan},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}

@article{commeff-sgd,
  title={{QSGD}: Communication-efficient {SGD} via gradient quantization and encoding},
  author={Alistarh, Dan and Grubic, Demjan and Li, Jerry and Tomioka, Ryota and Vojnovic, Milan},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  year={2017}
}

@manual{pytorch-datadistrib,
  title  = "Getting Started with Distributed Data Parallel",
  author = "Shen Li",
  url    = "https://pytorch.org/tutorials/intermediate/ddp_tutorial.html",
  year   = "2017"
}

@manual{tensorflow-datadistrib,
  title = "Distributed Training with TensorFlow",
  author = "Google",
  url = "https://www.tensorflow.org/guide/distributed_training",
  year = "2020"
}

@ARTICLE{comm-eff-survey,
       author = {{Tang}, Zhenheng and {Shi}, Shaohuai and {Chu}, Xiaowen and {Wang}, Wei and
         {Li}, Bo},
        title = "{Communication-Efficient Distributed Deep Learning: A Comprehensive Survey}",
        
     keywords = {Computer Science - Distributed, Parallel, and Cluster Computing, Computer Science - Machine Learning, Electrical Engineering and Systems Science - Signal Processing},
         year = 2020,
        month = mar,
          eid = {arXiv:2003.06307},
        pages = {arXiv:2003.06307},
archivePrefix = {arXiv},
       eprint = {2003.06307},
 primaryClass = {cs.DC},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2020arXiv200306307T},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}


@inproceedings{distrib-benchmark,
  title={Performance modeling and evaluation of distributed deep learning frameworks on {GPU}s},
  author={Shi, Shaohuai and Wang, Qiang and Chu, Xiaowen},
  booktitle={2018 IEEE 16th Intl Conf on Dependable, Autonomic and Secure Computing},
  pages={949--957},
  year={2018},
  organization={IEEE}
}


@ARTICLE{benchmarkgpu,
       author = {{Wang}, Yuxin and {Wang}, Qiang and {Shi}, Shaohuai and {He}, Xin and
         {Tang}, Zhenheng and {Zhao}, Kaiyong and {Chu}, Xiaowen},
        title = "{Benchmarking the Performance and Energy Efficiency of AI Accelerators for AI Training}",
        
     keywords = {Computer Science - Distributed, Parallel, and Cluster Computing, Computer Science - Machine Learning},
         year = 2019,
        month = sep,
          eid = {arXiv:1909.06842},
        pages = {arXiv:1909.06842},
archivePrefix = {arXiv},
       eprint = {1909.06842},
 primaryClass = {cs.DC},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2019arXiv190906842W},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}


@inproceedings{BigT,
  title={Big Transfer (BiT): General Visual Representation Learning},
  author={A. Kolesnikov and Lucas Beyer and Xiaohua Zhai and Joan Puigcerver and Jessica Yung and S. Gelly and N. Houlsby},
  booktitle={ECCV},
  year={2020}
}

@ARTICLE{imagenetinmin,
       author = {{You}, Yang and {Zhang}, Zhao and {Hsieh}, Cho-Jui and {Demmel}, James and
         {Keutzer}, Kurt},
        title = "{ImageNet Training in Minutes}",
        
     keywords = {Computer Science - Computer Vision and Pattern Recognition},
         year = 2017,
        month = sep,
          eid = {arXiv:1709.05011},
        pages = {arXiv:1709.05011},
archivePrefix = {arXiv},
       eprint = {1709.05011},
 primaryClass = {cs.CV},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2017arXiv170905011Y},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@ARTICLE{16by16,
       author = {{Dosovitskiy}, Alexey and {Beyer}, Lucas and {Kolesnikov}, Alexander and
         {Weissenborn}, Dirk and {Zhai}, Xiaohua and {Unterthiner}, Thomas and
         {Dehghani}, Mostafa and {Minderer}, Matthias and {Heigold}, Georg and
         {Gelly}, Sylvain and {Uszkoreit}, Jakob and {Houlsby}, Neil},
        title = "{An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale}",
        
     keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
         year = 2020,
        month = oct,
          eid = {arXiv:2010.11929},
        pages = {arXiv:2010.11929},
archivePrefix = {arXiv},
       eprint = {2010.11929},
 primaryClass = {cs.CV},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2020arXiv201011929D},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@inproceedings{lamp,
  title={{Lamp}: Large deep nets with automated model parallelism for image segmentation},
  author={Zhu, Wentao and Zhao, Can and Li, Wenqi and Roth, Holger and Xu, Ziyue and Xu, Daguang},
  booktitle={International Conference on Medical Image Computing and Computer-Assisted Intervention},
  pages={374--384},
  year={2020},
  organization={Springer}
}

@ARTICLE{integrated_model_parallel,
       author = {{Gholami}, Amir and {Azad}, Ariful and {Jin}, Peter and {Keutzer}, Kurt and {Buluc}, Aydin},
        title = "{Integrated Model, Batch and Domain Parallelism in Training Neural Networks}",
        
     keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
         year = 2017,
        month = dec,
          eid = {arXiv:1712.04432},
        pages = {arXiv:1712.04432},
archivePrefix = {arXiv},
       eprint = {1712.04432},
 primaryClass = {cs.LG},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2017arXiv171204432G},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@article{parallelism_survey,
  title={Demystifying parallel and distributed deep learning: An in-depth concurrency analysis},
  author={Ben-Nun, Tal and Hoefler, Torsten},
  journal={ACM Computing Surveys (CSUR)},
  volume={52},
  number={4},
  pages={1--43},
  year={2019},
  publisher={ACM New York, NY, USA}
}

@ARTICLE{compiler_model_parallel,
       author = {{Tavarageri}, Sanket and {Sridharan}, Srinivas and {Kaul}, Bharat},
        title = "{Automatic Model Parallelism for Deep Neural Networks with Compiler and Hardware Support}",
        
     keywords = {Computer Science - Distributed, Parallel, and Cluster Computing, Computer Science - Machine Learning},
         year = 2019,
        month = jun,
          eid = {arXiv:1906.08168},
        pages = {arXiv:1906.08168},
archivePrefix = {arXiv},
       eprint = {1906.08168},
 primaryClass = {cs.DC},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2019arXiv190608168T},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@inproceedings{nonlinear_multigrid_layer_parallel,
  title={Layer-parallel training with gpu concurrency of deep residual neural networks via nonlinear multigrid},
  author={Kirby, Andrew and Samsi, Siddharth and Jones, Michael and Reuther, Albert and Kepner, Jeremy and Gadepally, Vijay},
  booktitle={2020 IEEE High Performance Extreme Computing Conference (HPEC)},
  pages={1--7},
  year={2020},
  organization={IEEE}
}


@article{pascalvoc,
  title={The pascal visual object classes ({VOC}) challenge},
  author={Everingham, Mark and Van Gool, Luc and Williams, Christopher KI and Winn, John and Zisserman, Andrew},
  journal={International journal of computer vision},
  volume={88},
  number={2},
  pages={303--338},
  year={2010},
  publisher={Springer}
}

@ARTICLE{kfac,
       author = {{Pauloski}, J. Gregory and {Zhang}, Zhao and {Huang}, Lei and {Xu}, Weijia and {Foster}, Ian T.},
        title = "{Convolutional Neural Network Training with Distributed K-FAC}",
        
     keywords = {Computer Science - Machine Learning, Computer Science - Distributed, Parallel, and Cluster Computing, Statistics - Machine Learning},
         year = 2020,
        month = jul,
          eid = {arXiv:2007.00784},
        pages = {arXiv:2007.00784},
archivePrefix = {arXiv},
       eprint = {2007.00784},
 primaryClass = {cs.LG},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2020arXiv200700784P},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}


@article{layer_parallel_resnet,
  title={Layer-parallel training of deep residual neural networks},
  author={Gunther, Stefanie and Ruthotto, Lars and Schroder, Jacob B and Cyr, Eric C and Gauger, Nicolas R},
  journal={SIAM Journal on Mathematics of Data Science},
  volume={2},
  number={1},
  pages={1--23},
  year={2020},
  publisher={SIAM}
}

@article{xpipe,
  title={{XPipe}: Efficient pipeline model parallelism for multi-{GPU DNN} training},
  author={Guan, Lei and Yin, Wotao and Li, Dongsheng and Lu, Xicheng},
  journal={arXiv preprint arXiv:1911.04610},
  year={2019}
}

@article{multi_gpu_model_parallel,
  title={Efficient and robust parallel dnn training through model parallelism on multi-gpu platform},
  author={Chen, Chi-Chung and Yang, Chia-Lin and Cheng, Hsiang-Yun},
  journal={arXiv preprint arXiv:1809.02839},
  year={2018}
}



@ARTICLE{adamw,
       author = {{Loshchilov}, Ilya and {Hutter}, Frank},
        title = "{Decoupled Weight Decay Regularization}",
        
     keywords = {Computer Science - Machine Learning, Computer Science - Neural and Evolutionary Computing, Mathematics - Optimization and Control},
         year = 2017,
        month = nov,
          eid = {arXiv:1711.05101},
        pages = {arXiv:1711.05101},
archivePrefix = {arXiv},
       eprint = {1711.05101},
 primaryClass = {cs.LG},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2017arXiv171105101L},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@ARTICLE{demon,
       author = {{Chen}, John and {Wolfe}, Cameron and {Li}, Zhao and {Kyrillidis}, Anastasios},
        title = "{Demon: Momentum Decay for Improved Neural Network Training}",
        
     keywords = {Computer Science - Machine Learning, Electrical Engineering and Systems Science - Image and Video Processing, Mathematics - Optimization and Control, Statistics - Machine Learning},
         year = 2019,
        month = oct,
          eid = {arXiv:1910.04952},
        pages = {arXiv:1910.04952},
archivePrefix = {arXiv},
       eprint = {1910.04952},
 primaryClass = {cs.LG},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2019arXiv191004952C},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@ARTICLE{adam,
       author = {{Kingma}, Diederik P. and {Ba}, Jimmy},
        title = "{Adam: A Method for Stochastic Optimization}",
        
     keywords = {Computer Science - Machine Learning},
         year = 2014,
        month = dec,
          eid = {arXiv:1412.6980},
        pages = {arXiv:1412.6980},
archivePrefix = {arXiv},
       eprint = {1412.6980},
 primaryClass = {cs.LG},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2014arXiv1412.6980K},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}



@article{asynch-summary,
  title={Advances in asynchronous parallel and distributed optimization},
  author={Assran, Mahmoud and Aytekin, Arda and Feyzmahdavian, Hamid Reza and Johansson, Mikael and Rabbat, Michael G},
  journal={Proceedings of the IEEE},
  volume={108},
  number={11},
  pages={2013--2031},
  year={2020},
  publisher={IEEE}
}

@article{grad-sparse,
  title={Gradient sparsification for communication-efficient distributed optimization},
  author={Wangni, Jianqiao and Wang, Jialei and Liu, Ji and Zhang, Tong},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}



@ARTICLE{2019arXiv190108924Y,
       author = {{Yu}, Yue and {Wu}, Jiaxiang and {Huang}, Junzhou},
        title = "{Exploring Fast and Communication-Efficient Algorithms in Large-scale Distributed Networks}",
        
     keywords = {Mathematics - Optimization and Control},
         year = 2019,
        month = jan,
          eid = {arXiv:1901.08924},
        pages = {arXiv:1901.08924},
archivePrefix = {arXiv},
       eprint = {1901.08924},
 primaryClass = {math.OC},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2019arXiv190108924Y},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@misc{dlcarbonemission,
    title={Quantifying the Carbon Emissions of Machine Learning},
    author={Alexandre Lacoste and Alexandra Luccioni and Victor Schmidt and Thomas Dandres},
    year={2019},
    eprint={1910.09700},
    archivePrefix={arXiv},
    primaryClass={cs.CY}
}

@ARTICLE{IST,
       author = {{Yuan}, Binhang and {Kyrillidis}, Anastasios and
         {Jermaine}, Christopher M.},
        title = "{Distributed Learning of Deep Neural Networks using Independent Subnet Training}",
        
     keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
         year = 2019,
        month = oct,
          eid = {arXiv:1910.02120},
        pages = {arXiv:1910.02120},
archivePrefix = {arXiv},
       eprint = {1910.02120},
 primaryClass = {cs.LG},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2019arXiv191002120Y},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@inproceedings{mobilenetv2,
  title={Mobilenetv2: Inverted residuals and linear bottlenecks},
  author={Sandler, Mark and Howard, Andrew and Zhu, Menglong and Zhmoginov, Andrey and Chen, Liang-Chieh},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={4510--4520},
  year={2018}
}

@article{4minimagenet,
  title={Highly scalable deep learning training system with mixed-precision: Training imagenet in four minutes},
  author={Jia, Xianyan and others},
  journal={arXiv preprint arXiv:1807.11205},
  year={2018}
}


@article{adascalesgd,
  title={AdaScale SGD: A User-Friendly Algorithm for Distributed Training},
  author={Johnson, Tyler B and Agrawal, Pulkit and Gu, Haijie and Guestrin, Carlos},
  journal={  arXiv:2007.05105},
  year={2020}
}

@inproceedings{yolov2,
  title={{YOLO9000}: better, faster, stronger},
  author={Redmon, Joseph and Farhadi, Ali},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={7263--7271},
  year={2017}
}


@inproceedings{imagenetinminutes,
  title={Imagenet training in minutes},
  author={You, Yang and Zhang, Zhao and Hsieh, Cho-Jui and Demmel, James and Keutzer, Kurt},
  booktitle={Proceedings of the 47th International Conference on Parallel Processing},
  pages={1--10},
  year={2018}
}

@article{32Kbatch,
  title={Scaling {SGD} batch size to 32{K} for {I}magenet training},
  author={You, Yang and Gitman, Igor and Ginsburg, Boris},
  journal={arXiv preprint arXiv:1708.03888},
  volume={6},
  number={12},
  pages={6},
  year={2017}
}

@article{15minimagenet,
  title={Extremely large minibatch {SGD}: Training {ResNet}-50 on imagenet in 15 minutes},
  author={Akiba, Takuya and Suzuki, Shuji and Fukuda, Keisuke},
  journal={  arXiv:1711.04325},
  year={2017}
}

@inproceedings{76minbert,
  author    = {Yang You and
               Jing Li and
               Sashank J. Reddi and
               Jonathan Hseu and
               Sanjiv Kumar and
               Srinadh Bhojanapalli and
               Xiaodan Song and
               James Demmel and
               Kurt Keutzer and
               Cho{-}Jui Hsieh},
  title     = {Large Batch Optimization for Deep Learning: Training {BERT} in 76 minutes},
  booktitle = {8th International Conference on Learning Representations, {ICLR} 2020}
}


@article{layeredsgd,
  title={Layered SGD: A Decentralized and Synchronous SGD Algorithm for Scalable Deep Neural Network Training},
  author={Yu, Kwangmin and Flynn, Thomas and Yoo, Shinjae and D'Imperio, Nicholas},
  journal={  arXiv:1906.05936},
  year={2019}
}

@article{increasebatchsize,
  title={Don't decay the learning rate, increase the batch size},
  author={Smith, Samuel L and Kindermans, Pieter-Jan and Ying, Chris and Le, Quoc V},
  journal={  arXiv:1711.00489},
  year={2017}
}

@article{1hrimagenet,
  title={Accurate, large minibatch {SGD}: Training {I}magenet in 1 hour},
  author={Goyal, Priya and Doll{\'a}r, Piotr and Girshick, Ross and Noordhuis, Pieter and Wesolowski, Lukasz and Kyrola, Aapo and Tulloch, Andrew and Jia, Yangqing and He, Kaiming},
  journal={  arXiv:1706.02677},
  year={2017}
}

@inproceedings{localsgdconverge,
title={Local {SGD} Converges Fast and Communicates Little},
author={Sebastian U. Stich},
booktitle={International Conference on Learning Representations},
year={2019}
}


@inproceedings{transferablefeats,
  title={How transferable are features in deep neural networks?},
  author={Yosinski, Jason and Clune, Jeff and Bengio, Yoshua and Lipson, Hod},
  booktitle={Advances in neural information processing systems},
  pages={3320--3328},
  year={2014}
}

@inproceedings{domainadapt,
  title={Unsupervised domain adaptation with residual transfer networks},
  author={Long, Mingsheng and Zhu, Han and Wang, Jianmin and Jordan, Michael I},
  booktitle={Advances in neural information processing systems},
  pages={136--144},
  year={2016}
}

@inproceedings{imgtransfer,
  title={Learning and transferring mid-level image representations using convolutional neural networks},
  author={Oquab, Maxime and Bottou, Leon and Laptev, Ivan and Sivic, Josef},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={1717--1724},
  year={2014}
}

@inproceedings{coco,
  title={Microsoft coco: Common objects in context},
  author={Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\'a}r, Piotr and Zitnick, C Lawrence},
  booktitle={European conference on computer vision},
  pages={740--755},
  year={2014},
  organization={Springer}
}

@ARTICLE{wideresnet,
       author = {{Zagoruyko}, Sergey and {Komodakis}, Nikos},
        title = "{Wide Residual Networks}",
        
     keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Neural and Evolutionary Computing},
         year = 2016,
        month = may,
          eid = {arXiv:1605.07146},
        pages = {arXiv:1605.07146},
archivePrefix = {arXiv},
       eprint = {1605.07146},
 primaryClass = {cs.CV},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2016arXiv160507146Z},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@inproceedings{imagenet,
  title={Imagenet: A large-scale hierarchical image database},
  author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Li, Kai and Fei-Fei, Li},
  booktitle={2009 IEEE conference on computer vision and pattern recognition},
  pages={248--255},
  year={2009},
  organization={Ieee}
}

@article{deeplimit,
  title={Deep limits of residual neural networks},
  author={Thorpe, Matthew and van Gennip, Yves},
  journal={  arXiv:1810.11741},
  year={2018}
}

@inproceedings{augmentedneuralode,
  title={Augmented neural {ODE}s},
  author={Dupont, Emilien and Doucet, Arnaud and Teh, Yee Whye},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3140--3150},
  year={2019}
}

@inproceedings{mfresnet,
  title={A mean field analysis of deep resnet and beyond: Towards provably optimization via overparameterization from depth},
  author={Lu, Yiping and Ma, Chao and Lu, Yulong and Lu, Jianfeng and Ying, Lexing},
  booktitle={International Conference on Machine Learning},
  pages={6426--6436},
  year={2020},
  organization={PMLR}
}


@misc{tensorflow,
title={{TensorFlow}: Large-Scale Machine Learning on Heterogeneous Systems},
author={
    Mart\'{\i}n~Abadi and others},
  year={2015},
}

@incollection{pytorch,
title = {PyTorch: An Imperative Style, High-Performance Deep Learning Library},
author = {Paszke, Adam and others},
booktitle = {Advances in Neural Information Processing Systems 32},
pages = {8024--8035},
year = {2019}
}

@ARTICLE{transformerode,
       author = {{Lu}, Yiping and {Li}, Zhuohan and {He}, Di and {Sun}, Zhiqing and
         {Dong}, Bin and {Qin}, Tao and {Wang}, Liwei and {Liu}, Tie-Yan},
        title = "{Understanding and Improving Transformer From a Multi-Particle Dynamic System Point of View}",
        
     keywords = {Computer Science - Machine Learning, Computer Science - Computation and Language, Statistics - Machine Learning},
         year = 2019,
        month = jun,
          eid = {arXiv:1906.02762},
        pages = {arXiv:1906.02762},
archivePrefix = {arXiv},
       eprint = {1906.02762},
 primaryClass = {cs.LG},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2019arXiv190602762L},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@inproceedings{beyondfinite,
  title={Beyond finite layer neural networks: Bridging deep architectures and numerical differential equations},
  author={Lu, Yiping and Zhong, Aoxiao and Li, Quanzheng and Dong, Bin},
  booktitle={International Conference on Machine Learning},
  pages={3276--3285},
  year={2018},
  organization={PMLR}
}

@inproceedings{continuumlimit,
  title={Double continuum limit of deep neural networks},
  author={Sonoda, Sho and Murata, Noboru},
  booktitle={ICML Workshop Principled Approaches to Deep Learning},
  volume={1740},
  year={2017}
}

@article{neuralode,
  title={Neural ordinary differential equations},
  author={Chen, Ricky TQ and Rubanova, Yulia and Bettencourt, Jesse and Duvenaud, David K},
  journal={Advances in neural information processing systems},
  volume={31},
  year={2018}
}


@inproceedings{filterprune,
  author    = {Hao Li and
               Asim Kadav and
               Igor Durdanovic and
               Hanan Samet and
               Hans Peter Graf},
  title     = {Pruning Filters for Efficient ConvNets},
  booktitle = {5th International Conference on Learning Representations, {ICLR} 2017},
  year      = {2017}
}

@article{shallowensemble,
  title={Residual networks behave like ensembles of relatively shallow networks},
  author={Veit, Andreas and Wilber, Michael J and Belongie, Serge},
  journal={Advances in neural information processing systems},
  volume={29},
  year={2016}
}

@article{rezero,
  title={Rezero is all you need: Fast convergence at large depth},
  author={Bachlechner, Thomas and Majumder, Bodhisattwa Prasad and Mao, Huanru Henry and Cottrell, Garrison W and McAuley, Julian},
  journal={  arXiv:2003.04887},
  year={2020}
}

@article{inceptionresnet,
  title={Inception-v4, inception-resnet and the impact of residual connections on learning},
  author={Szegedy, Christian and Ioffe, Sergey and Vanhoucke, Vincent and Alemi, Alex},
  journal={  arXiv:1602.07261},
  year={2016}
}

@article{bert,
  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={  arXiv:1810.04805},
  year={2018}
}

@inproceedings{transformer,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  booktitle={Advances in neural information processing systems},
  pages={5998--6008},
  year={2017}
}

@inproceedings{retinanet,
  title={Focal loss for dense object detection},
  author={Lin, Tsung-Yi and Goyal, Priya and Girshick, Ross and He, Kaiming and Doll{\'a}r, Piotr},
  booktitle={Proceedings of the IEEE international conference on computer vision},
  pages={2980--2988},
  year={2017}
}

@article{mobilenet,
  title={Mobilenets: Efficient convolutional neural networks for mobile vision applications},
  author={Howard, Andrew G and Zhu, Menglong and Chen, Bo and Kalenichenko, Dmitry and Wang, Weijun and Weyand, Tobias and Andreetto, Marco and Adam, Hartwig},
  journal={  arXiv:1704.04861},
  year={2017}
}

@inproceedings{maskrcnn,
  title={Mask r-cnn},
  author={He, Kaiming and Gkioxari, Georgia and Doll{\'a}r, Piotr and Girshick, Ross},
  booktitle={Proceedings of the IEEE international conference on computer vision},
  pages={2961--2969},
  year={2017}
}

@inproceedings{densenets,
  title={Densely connected convolutional networks},
  author={Huang, Gao and Liu, Zhuang and Van Der Maaten, Laurens and Weinberger, Kilian Q},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={4700--4708},
  year={2017}
}

@inproceedings{fasterrcnn,
  title={Faster r-cnn: Towards real-time object detection with region proposal networks},
  author={Ren, Shaoqing and He, Kaiming and Girshick, Ross and Sun, Jian},
  booktitle={Advances in neural information processing systems},
  pages={91--99},
  year={2015}
}

@inproceedings{preactres,
  title={Identity mappings in deep residual networks},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={European conference on computer vision},
  pages={630--645},
  year={2016},
  organization={Springer}
}

@ARTICLE{resnext,
       author = {{Xie}, Saining and {Girshick}, Ross and {Doll{\'a}r}, Piotr and
         {Tu}, Zhuowen and {He}, Kaiming},
        title = "{Aggregated Residual Transformations for Deep Neural Networks}",
        
     keywords = {Computer Science - Computer Vision and Pattern Recognition},
         year = 2016,
        month = nov,
          eid = {arXiv:1611.05431},
        pages = {arXiv:1611.05431},
archivePrefix = {arXiv},
       eprint = {1611.05431},
 primaryClass = {cs.CV},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2016arXiv161105431X},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@article{vgg,
  title={Very deep convolutional networks for large-scale image recognition},
  author={Simonyan, Karen and Zisserman, Andrew},
  journal={  arXiv:1409.1556},
  year={2014}
}

@inproceedings{googlenet,
  title={Going deeper with convolutions},
  author={Szegedy, Christian and Liu, Wei and Jia, Yangqing and Sermanet, Pierre and Reed, Scott and Anguelov, Dragomir and Erhan, Dumitru and Vanhoucke, Vincent and Rabinovich, Andrew},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={1--9},
  year={2015}
}

@article{batchnorm,
  title={Batch normalization: Accelerating deep network training by reducing internal covariate shift},
  author={Ioffe, Sergey and Szegedy, Christian},
  journal={  arXiv:1502.03167},
  year={2015}
}

@inproceedings{resnet,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@ARTICLE{zfnet,
       author = {{Zeiler}, Matthew D and {Fergus}, Rob},
        title = "{Visualizing and Understanding Convolutional Networks}",
        
     keywords = {Computer Science - Computer Vision and Pattern Recognition},
         year = 2013,
        month = nov,
          eid = {arXiv:1311.2901},
        pages = {arXiv:1311.2901},
archivePrefix = {arXiv},
       eprint = {1311.2901},
 primaryClass = {cs.CV},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2013arXiv1311.2901Z},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@ARTICLE{use_local_sgd,
       author = {{Lin}, Tao and {Stich}, Sebastian U. and {Kshitij Patel}, Kumar and {Jaggi}, Martin},
        title = "{Don't Use Large Mini-Batches, Use Local SGD}",
        
     keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
         year = 2018,
        month = aug,
          eid = {arXiv:1808.07217},
        pages = {arXiv:1808.07217},
archivePrefix = {arXiv},
       eprint = {1808.07217},
 primaryClass = {cs.LG},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2018arXiv180807217L},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}
@ARTICLE{2018arXiv180807217L,
       author = {{Lin}, Tao and {Stich}, Sebastian U. and {Kshitij Patel}, Kumar and {Jaggi}, Martin},
        title = "{Don't Use Large Mini-Batches, Use Local SGD}",
     keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
         year = 2018,
        month = aug,
          eid = {arXiv:1808.07217},
        pages = {arXiv:1808.07217},
archivePrefix = {arXiv},
       eprint = {1808.07217},
 primaryClass = {cs.LG},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2018arXiv180807217L},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@ARTICLE{parallel_sgd,
       author = {{Zhang}, Jian and {De Sa}, Christopher and {Mitliagkas}, Ioannis and {R{\'e}}, Christopher},
        title = "{Parallel SGD: When does averaging help?}",
        
     keywords = {Statistics - Machine Learning, Computer Science - Machine Learning},
         year = 2016,
        month = jun,
          eid = {arXiv:1606.07365},
        pages = {arXiv:1606.07365},
archivePrefix = {arXiv},
       eprint = {1606.07365},
 primaryClass = {stat.ML},
       adsurl = {https://ui.adsabs.harvard.edu/abs/2016arXiv160607365Z},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@inproceedings{fed_avg,
  title={Communication-efficient learning of deep networks from decentralized data},
  author={McMahan, Brendan and Moore, Eider and Ramage, Daniel and Hampson, Seth and y Arcas, Blaise Aguera},
  booktitle={Artificial intelligence and statistics},
  pages={1273--1282},
  year={2017},
  organization={PMLR}
}

@article{sergeev2018horovod,
  title={Horovod: fast and easy distributed deep learning in {T}ensorFlow},
  author={Sergeev, Alexander and Del Balso, Mike},
  journal={  arXiv:1802.05799},
  year={2018}
}


@inproceedings{li2014scaling,
  title={Scaling distributed machine learning with the parameter server},
  author={Li, Mu and Andersen, David G and Park, Jun Woo and Smola, Alexander J and Ahmed, Amr and Josifovski, Vanja and Long, James and Shekita, Eugene J and Su, Bor-Yiing},
  booktitle={11th $\{$USENIX$\}$ Symposium on Operating Systems Design and Implementation ($\{$OSDI$\}$ 14)},
  pages={583--598},
  year={2014}
}

@article{patarasuk2009bandwidth,
  title={Bandwidth optimal all-reduce algorithms for clusters of workstations},
  author={Patarasuk, Pitch and Yuan, Xin},
  journal={Journal of Parallel and Distributed Computing},
  volume={69},
  number={2},
  pages={117--124},
  year={2009},
  publisher={Elsevier}
}

@inproceedings{assran2019stochastic,
  title={Stochastic gradient push for distributed deep learning},
  author={Assran, Mahmoud and Loizou, Nicolas and Ballas, Nicolas and Rabbat, Mike},
  booktitle={International Conference on Machine Learning},
  pages={344--353},
  year={2019},
  organization={PMLR}
}

@article{chen2012diffusion,
  title={Diffusion adaptation strategies for distributed optimization and learning over networks},
  author={Chen, Jianshu and Sayed, Ali H},
  journal={IEEE Transactions on Signal Processing},
  volume={60},
  number={8},
  pages={4289--4305},
  year={2012},
  publisher={IEEE}
}

@article{lian2017can,
  title={Can decentralized algorithms outperform centralized algorithms? a case study for decentralized parallel stochastic gradient descent},
  author={Lian, Xiangru and Zhang, Ce and Zhang, Huan and Hsieh, Cho-Jui and Zhang, Wei and Liu, Ji},
  journal={  arXiv:1705.09056},
  year={2017}
}

@inproceedings{lian2018asynchronous,
  title={Asynchronous decentralized parallel stochastic gradient descent},
  author={Lian, Xiangru and Zhang, Wei and Zhang, Ce and Liu, Ji},
  booktitle={International Conference on Machine Learning},
  pages={3043--3052},
  year={2018},
  organization={PMLR}
}

@article{nedic2009distributed,
  title={Distributed subgradient methods for multi-agent optimization},
  author={Nedic, Angelia and Ozdaglar, Asuman},
  journal={IEEE Transactions on Automatic Control},
  volume={54},
  number={1},
  pages={48--61},
  year={2009},
  publisher={IEEE}
}

@inproceedings{recht2019imagenet,
  title={Do imagenet classifiers generalize to imagenet?},
  author={Recht, Benjamin and Roelofs, Rebecca and Schmidt, Ludwig and Shankar, Vaishaal},
  booktitle={International Conference on Machine Learning},
  pages={5389--5400},
  year={2019},
  organization={PMLR}
}


@inproceedings{koloskova2020unified,
  title={A unified theory of decentralized SGD with changing topology and local updates},
  author={Koloskova, Anastasia and Loizou, Nicolas and Boreiri, Sadra and Jaggi, Martin and Stich, Sebastian},
  booktitle={International Conference on Machine Learning},
  pages={5381--5393},
  year={2020},
  organization={PMLR}
}

@article{johansson2010randomized,
  title={A randomized incremental subgradient method for distributed optimization in networked systems},
  author={Johansson, Bj{\"o}rn and Rabi, Maben and Johansson, Mikael},
  journal={SIAM Journal on Optimization},
  volume={20},
  number={3},
  pages={1157--1170},
  year={2010},
  publisher={SIAM}
}

@inproceedings{lian2017can,
  title={Can decentralized algorithms outperform centralized algorithms? a case study for decentralized parallel stochastic gradient descent},
  author={Lian, Xiangru and Zhang, Ce and Zhang, Huan and Hsieh, Cho-Jui and Zhang, Wei and Liu, Ji},
  booktitle={Proceedings of the 31st International Conference on Neural Information Processing Systems},
  pages={5336--5346},
  year={2017}
}

@inproceedings{tang2018d,
  title={$D^2$: Decentralized training over decentralized data},
  author={Tang, Hanlin and Lian, Xiangru and Yan, Ming and Zhang, Ce and Liu, Ji},
  booktitle={International Conference on Machine Learning},
  pages={4848--4856},
  year={2018},
  organization={PMLR}
}

@article{assran2020asynchronous,
  title={Asynchronous gradient push},
  author={Assran, Mahmoud S and Rabbat, Michael G},
  journal={IEEE Transactions on Automatic Control},
  volume={66},
  number={1},
  pages={168--183},
  year={2020},
  publisher={IEEE}
}


@inproceedings{koloskova2019decentralized,
  title={Decentralized stochastic optimization and gossip algorithms with compressed communication},
  author={Koloskova, Anastasia and Stich, Sebastian and Jaggi, Martin},
  booktitle={International Conference on Machine Learning},
  pages={3478--3487},
  year={2019},
  organization={PMLR}
}

@inproceedings{you2018imagenet,
  title={Imagenet training in minutes},
  author={You, Yang and Zhang, Zhao and Hsieh, Cho-Jui and Demmel, James and Keutzer, Kurt},
  booktitle={Proceedings of the 47th International Conference on Parallel Processing},
  pages={1--10},
  year={2018}
}

@inproceedings{du2019gradient,
  title={Gradient descent finds global minima of deep neural networks},
  author={Du, Simon and Lee, Jason and Li, Haochuan and Wang, Liwei and Zhai, Xiyu},
  booktitle={International conference on machine learning},
  pages={1675--1685},
  year={2019},
  organization={PMLR}
}


@misc{liao2021convergence,
      title={On the Convergence of Shallow Neural Network Training with Randomly Masked Neurons}, 
      author={Fangshuo Liao and Anastasios Kyrillidis},
      year={2021},
      eprint={2112.02668},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{adaptive_fed_drop,
  doi = {10.48550/ARXIV.2011.04050},
  url = {https://arxiv.org/abs/2011.04050},
  
  author = {Bouacida, Nader and Hou, Jiahui and Zang, Hui and Liu, Xin},
  
  keywords = {Machine Learning (cs.LG), Distributed, Parallel, and Cluster Computing (cs.DC), FOS: Computer and information sciences, FOS: Computer and information sciences},
  
  title = {Adaptive Federated Dropout: Improving Communication Efficiency and Generalization for Federated Learning},
  
  publisher = {arXiv},
  
  year = {2020},
  
  copyright = {arXiv.org perpetual, non-exclusive license}
}
