@inproceedings{maranjyan2025ringmaster,
    title={{R}ingmaster {ASGD}: The First {A}synchronous {SGD} with Optimal Time Complexity}, 
    author={Artavazd Maranjyan and Alexander Tyurin and Peter Richt{\'a}rik},
    year={2025},
    booktitle={International Conference on Machine Learning},
}

@inproceedings{maranjyan2025ata,
    title={{ATA}: Adaptive Task Allocation for Efficient Resource Management in Distributed Machine Learning}, 
    author={Maranjyan, Artavazd and Saad, El Mehdi and Richt{\'a}rik, Peter and Orabona, Francesco},
    year={2025},
    booktitle={International Conference on Machine Learning},
}

@inproceedings{maranjyan2024mindflayer,
  title={{MindFlayer SGD}: Efficient Parallel {SGD} in the Presence of Heterogeneous and Random Worker Compute Times},
  author={Artavazd Maranjyan and Omar Shaikh Omar and Peter Richt{\'a}rik},
  booktitle={Conference on Uncertainty in Artificial Intelligence},
  year={2024},
}

@article{maranjyan2022gradskip,
  title = {{GradSkip}: Communication-accelerated local gradient methods with better computational complexity},
  author = {Maranjyan, Artavazd and Safaryan, Mher and Richt{\'a}rik, Peter},
  journal = {arXiv preprint arXiv:2210.16402},
  year = {2022},
}

@article{mania2017perturbed,
  title={Perturbed iterate analysis for asynchronous stochastic optimization},
  author={Mania, Horia and Pan, Xinghao and Papailiopoulos, Dimitris and Recht, Benjamin and Ramchandran, Kannan and Jordan, Michael I},
  journal={SIAM Journal on Optimization},
  volume={27},
  number={4},
  pages={2202--2229},
  year={2017},
  publisher={SIAM}
}

@article{agarwal2011distributed,
  title={Distributed delayed stochastic optimization},
  author={Agarwal, Alekh and Duchi, John C},
  journal={Advances in neural information processing systems},
  volume={24},
  year={2011}
}

@article{alahyane2025optimizing,
  title={Optimizing Asynchronous Federated Learning: A Delicate Trade-Off Between Model-Parameter Staleness and Update Frequency},
  author={Alahyane, Abdelkrim and Comte, C{\'e}line and Jonckheere, Matthieu and Moulines, {\'E}ric},
  journal={arXiv preprint arXiv:2502.08206},
  year={2025}
}

@inproceedings{Transformer2017,
 author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, \L ukasz and Polosukhin, Illia},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {I. Guyon and U. V. Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Attention is All you Need},
 url = {https://proceedings.neurips.cc/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf},
 volume = {30},
 year = {2017}
}

@InProceedings{Koloskova2019-DecentralizedEC-2019,
  author    = {Anastasia Koloskova and Sebastian U Stich and Martin Jaggi},
  booktitle = {International Conference on Machine Learning},
  title     = {Decentralized Stochastic Optimization and Gossip Algorithms with Compressed Communication},
  year      = {2019},
  journal   = {International Conference on Machine Learning},
}

@inproceedings{GPT3,
 author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 pages = {1877--1901},
 publisher = {Curran Associates, Inc.},
 title = {Language Models are Few-Shot Learners},
 url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},
 volume = {33},
 year = {2020}
}


@InProceedings{SPIDER,
  author    = {Cong Fang and Chris Junchi Li and Zhouchen Lin and Tong Zhang},
  booktitle = {NeurIPS Information Processing Systems},
  title     = {{SPIDER}: Near-Optimal Non-Convex Optimization via Stochastic Path Integrated Differential Estimator},
  year      = {2018},
}

@InProceedings{SARAH,
  author    = {Nguyen, Lam and Liu, Jie and Scheinberg, Katya and Tak{\'a}{\v{c}}, Martin},
  title     = {{SARAH}: A novel method for machine learning problems using stochastic recursive gradient},
  booktitle = {The 34th International Conference on Machine Learning},
  year      = {2017},
}

@Article{EF21,
  author  = {Peter Richt\'{a}rik and Igor Sokolov and Ilyas Fatkhullin},
  journal = {arXiv preprint arXiv:2106.05203},
  title   = {{EF21}: A new, simpler, theoretically better, and practically faster error feedback},
  year    = {2021},
}

@Article{FedNL,
  author  = {Safaryan, Mher and Islamov, Rustem and Qian, Xun and Richt\'{a}rik, Peter},
  journal = {arXiv preprint arXiv:2106.02969},
  title   = {{FedNL}: Making {N}ewton-type methods applicable to federated learning},
  year    = {2021},
}

@InProceedings{ADIANA,
  author    = {Zhize Li and Dmitry Kovalev and Xun Qian and Peter Richt\'{a}rik},
  booktitle = {International Conference on Machine Learning},
  title     = {Acceleration for Compressed Gradient Descent in Distributed and Federated Optimization},
  year      = {2020},
}

@Article{DCGD,
  author  = {Sarit Khirirat and Hamid Reza Feyzmahdavian and Mikael Johansson},
  journal = {arXiv preprint arXiv:1806.06573},
  title   = {Distributed learning with compressed gradients},
  year    = {2018},
}

@InProceedings{Alistarh-EF-NIPS2018,
  author    = {Alistarh, Dan and Hoefler, Torsten and Johansson,  Mikael and Khirirat,  Sarit and Konstantinov,  Nikola and Renggli,  C\'{e}dric},
    title     = {The convergence of sparsified gradient methods},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year      = {2018},
}

@inproceedings{Seide2014,
  author    = {Seide,  Frank and Fu,  Hao and Droppo,  Jasha and Li,  Gang and Yu,  Dong},
  title     = {1-bit stochastic gradient descent and its application to data-parallel distributed training of speech {DNN}s},
  booktitle = {Fifteenth Annual Conference of the International Speech Communication Association},
  year      = {2014},
}

@InProceedings{PowerSGD,
  author    = {Thijs Vogels and Sai Praneeth Karimireddy and Martin Jaggi},
  booktitle = {Neural Information Processing Systems},
  title     = {Power{SGD}: Practical Low-Rank Gradient Compression for Distributed Optimization},
  year      = {2019},
}

@InProceedings{MARINA,
  author    = {Eduard Gorbunov and Konstantin Burlachenko and Zhize Li and Peter Richt\'{a}rik},
  booktitle = {38th International Conference on Machine Learning},
  title     = {{MARINA}: {F}aster non-convex distributed learning with compression},
  year      = {2021},
}

@InProceedings{EC-SGD,
  author    = {Gorbunov, Eduard and Kovalev, Dmitry and Makarenko, Dmitry and Richt\'{a}rik, Peter},
  booktitle = {34th Conference on Neural Information Processing Systems (NeurIPS 2020)},
  title     = {Linearly Converging Error Compensated {SGD}},
  year      = {2020},
}

@Article{EC-LSVRG,
  author  = {Xun Qian and Hanze Dong and Peter Richt\'{a}rik and Tong Zhang},
  journal = {OPT2020: 12th Annual Workshop on Optimization for Machine Learning (NeurIPS 2020 Workshop)},
  title   = {Error compensated loopless {SVRG} for distributed optimization},
  year    = {2020},
}

@InProceedings{DoubleSqueeze2019,
  author    = {Tang, Hanlin and Yu, Chen and Lian, Xiangru and Zhang, Tong and Liu, Ji},
  booktitle = {Proceedings of the 36th International Conference on Machine Learning},
  title     = {DoubleSqueeze: Parallel Stochastic Gradient Descent with Double-pass Error-Compensated Compression},
  year      = {2019},
  address   = {Long Beach, California, USA},
  editor    = {Chaudhuri, Kamalika and Salakhutdinov, Ruslan},
  month     = {09--15 Jun},
  pages     = {6155--6165},
  publisher = {PMLR},
  series    = {Proceedings of Machine Learning Research},
  volume    = {97},
  abstract  = {A standard approach in large scale machine learning is distributed stochastic gradient training, which requires the computation of aggregated stochastic gradients over multiple nodes on a network. Communication is a major bottleneck in such applications, and in recent years, compressed stochastic gradient methods such as QSGD (quantized SGD) and sparse {SGD} have been proposed to reduce communication. It was also shown that error compensation can be combined with compression to achieve better convergence in a scheme that each node compresses its local stochastic gradient and broadcast the result to all other nodes over the network in a single pass. However, such a single pass broadcast approach is not realistic in many practical implementations. For example, under the popular parameter-server model for distributed learning, the worker nodes need to send the compressed local gradients to the parameter server, which performs the aggregation. The parameter server has to compress the aggregated stochastic gradient again before sending it back to the worker nodes. In this work, we provide a detailed analysis on this two-pass communication model, with error-compensated compression both on the worker nodes and on the parameter server. We show that the error-compensated stochastic gradient algorithm admits three very nice properties: 1) it is compatible with an \emph{arbitrary} compression technique; 2) it admits an improved convergence rate than the non error-compensated stochastic gradient method such as QSGD and sparse SGD; 3) it admits linear speedup with respect to the number of workers. The empirical study is also conducted to validate our theoretical results.},
  file      = {tang19d.pdf:http\://proceedings.mlr.press/v97/tang19d/tang19d.pdf:PDF},
  url       = {http://proceedings.mlr.press/v97/tang19d.html},
}

@InProceedings{errorSGD,
  author    = {Wu, Jiaxiang and Huang, Weidong and Huang, Junzhou and Zhang, Tong},
  title     = {Error Compensated Quantized {SGD} and its Applications to Large-scale Distributed Optimization},
  booktitle = {Proceedings of the 35th International Conference on Machine Learning},
  year      = {2018},
  editor    = {Dy, Jennifer and Krause, Andreas},
  volume    = {80},
  series    = {Proceedings of Machine Learning Research},
  pages     = {5325--5333},
  address   = {Stockholmsmässan, Stockholm Sweden},
  month     = {10--15 Jul},
  publisher = {PMLR},
}

@Article{Artemis2020,
  author  = {Constantin Philippenko and Aymeric Dieuleveut},
  journal = {arXiv preprint arXiv:2006.14591},
  title   = {Bidirectional compression in heterogeneous settings for distributed or federated learning with partial participation: tight convergence guarantees},
  year    = {2020},
}

@Article{DIANA,
  author  = {Mishchenko, Konstantin and Gorbunov, Eduard and Tak{\'a}{\v{c}}, Martin and Richt{\'a}rik, Peter},
  journal = {arXiv preprint arXiv:1901.09269},
  title   = {Distributed Learning with Compressed Gradient Differences},
  year    = {2019},
}

@incollection{Bengio+chapter2007,
author = {Bengio, Yoshua and LeCun, Yann},
booktitle = {Large Scale Kernel Machines},
publisher = {MIT Press},
title = {Scaling Learning Algorithms Towards {AI}},
year = {2007}
}

@article{Hinton06,
author = {Hinton, Geoffrey E. and Osindero, Simon and Teh, Yee Whye},
journal = {Neural Computation},
pages = {1527--1554},
title = {A Fast Learning Algorithm for Deep Belief Nets},
volume = {18},
year = {2006}
}

@book{goodfellow2016deep,
title={Deep learning},
author={Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron and Bengio, Yoshua},
volume={1},
year={2016},
publisher={MIT Press}
}

@article{khaled2020better,
	title={Better Theory for {SGD} in the Nonconvex World},
	author={Khaled,  Ahmed and Richt{\'a}rik,  Peter},
	journal={arXiv preprint arXiv:2002.03329},
	year={2020}
}

@article{stich2020biased,
	title={Analysis of {SGD} with Biased Gradient Estimators},
	author={Ajalloeian,  Ahmad and Stich,  Sebastian U},
	journal={arXiv preprint arXiv:2008.00051},
	year={2020}
}

% Provides with the rate of nonconvex {SGD} unbiased compressor

@article{beznosikov2020biased,
	title={On Biased Compression for Distributed Learning},
	author={Beznosikov,  Aleksandr and Horv{\'a}th,  Samuel and Richt{\'a}rik,  Peter and Safaryan,  Mher},
	journal={arXiv preprint arXiv:2002.12410},
	year={2020}
}

@Article{UP2021,
  author  = {Mher Safaryan and Egor Shulgin and Peter Richt\'{a}rik},
  journal = {Information and Inference: A Journal of the IMA},
  title   = {Uncertainty Principle for Communication Compression in Distributed and Federated Learning and the Search for an Optimal Compressor},
  year    = {2021},
}

@inproceedings{PAGE,
  title={{PAGE}: A simple and optimal probabilistic gradient estimator for nonconvex optimization},
  author={Li, Zhize and Bao, Hongyan and Zhang, Xiangliang and Richt{\'a}rik, Peter},
  booktitle={International Conference on Machine Learning},
  pages={6286--6295},
  year={2021},
  organization={PMLR}
}

%We use Lemma 2 (from Appendix A) in our work.


@article{Stich2019TheEF,
  title={The Error-Feedback Framework: Better Rates for {SGD} with Delayed Gradients and Compressed Communication},
  author={Stich,  Sebastian and Sai Praneeth Karimireddy},
  journal={arXiv preprint arXiv:1909.05350},
  year={2019},
}

@InProceedings{Karimireddy_SignSGD,
  title = 	 {Error Feedback Fixes {S}ign{SGD} and other Gradient Compression Schemes},
  author =       {Karimireddy,  Sai Praneeth and Rebjock,  Quentin and Stich,  Sebastian and Jaggi,  Martin},
 booktitle = 	 {36th International Conference on Machine Learning (ICML)},
   year = 	 {2019},
}

@inproceedings{A_better_alternative,
  title={A Better Alternative to Error Feedback for Communication-Efficient Distributed Learning},
  author={Horv\'{a}th,  Samuel and Richt\'{a}rik,  Peter},
  booktitle={9th International Conference on Learning Representations (ICLR)},
  year={2021},
}

@inproceedings{DoubleSqueeze,
  title={{D}ouble{S}queeze: {P}arallel Stochastic Gradient Descent with Double-Pass Error-Compensated Compression},
  author={Hanlin Tang and Xiangru Lian and Chen Yu and Tong Zhang and Ji Liu},
  booktitle = {Proceedings of the 36th International Conference on Machine Learning (ICML)},
  year={2020},
}


@InProceedings{Stich-EF-NIPS2018,
  author    = {Stich,  Sebastian U.  and Cordonnier,  J.-B.  and Jaggi,  Martin},
  title     = {Sparsified {SGD} with memory},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year      = {2018},
}

@inproceedings{mcmahan2017communication,
  title={Communication-efficient learning of deep networks from decentralized data},
  author={McMahan, Brendan and Moore, Eider and Ramage, Daniel and Hampson, Seth and y Arcas, Blaise Aguera},
  booktitle={Artificial intelligence and statistics},
  pages={1273--1282},
  year={2017},
  organization={PMLR}
}


@InProceedings{Lin_EC_SGD,
  author    = {Gorbunov,  Eduard and Kovalev,  Dmitry and Makarenko,  Dmitry and Richt\'{a}rik,  Peter},
  booktitle = {34th Conference on Neural Information Processing Systems (NeurIPS)},
  title     = {Linearly Converging Error Compensated {SGD}},
  year      = {2020},
}

@inproceedings{Koloskova2019DecentralizedDL,
  title={Decentralized Deep Learning with Arbitrary Communication Compression},
  author={Anastasia Koloskova and Tao Lin and S. Stich and Martin Jaggi},
  booktitle={International Conference on Learning Representations (ICLR)},
  year={2020},
}

@InProceedings{alistarh2017qsgd,
  author    = {Alistarh, Dan and Grubic, Demjan and Li, Jerry and Tomioka, Ryota and Vojnovic, Milan},
  title     = {{QSGD}: {C}ommunication-efficient {SGD} via gradient quantization and encoding},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  year      = {2017},
  pages     = {1709--1720},
}

@InProceedings{Qsparse_local_SGD,
  title={{Q}sparse-local-{SGD}: {D}istributed {SGD} with Quantization,  Sparsification,  and Local Computations},
  author={Debraj Basu and Deepesh Data and Can Karakus and Suhas Diggavi},
  booktitle={Advances in Neural Information Processing Systems (NeurIPS)},
  year={2019},
}


@InProceedings{CSER,
  title={{CSER:} {C}ommunication-efficient {SGD} with Error Reset},
  author={Cong Xie and Shuai Zheng and Oluwasanmi Koyejo and Indranil Gupta and Mu Li and Haibin Lin},
  booktitle={Advances in Neural Information Processing Systems (NeurIPS)},
  pages = {12593--12603},
  year={2020},
}


@InProceedings{induced,
  author    = {Horv\'{a}th, Samuel and Richt\'{a}rik, Peter},
  booktitle = {9th International Conference on Learning Representations (ICLR)},
  title     = {A better alternative to error feedback for communication-efficient distributed learning},
  year      = {2021},
}


@article{chang2011libsvm,
	title={{LIBSVM}: a library for support vector machines},
	author={Chang, Chih-Chung and Lin, Chih-Jen},
	journal={{ACM} {T}ransactions on {I}ntelligent {S}ystems and {T}echnology (TIST)},
	volume={2},
	number={3},
	pages={1--27},
	year={2011},
	publisher={ACM New York, NY, USA}
}
@article{tran2019hybrid,
	title={Hybrid stochastic gradient descent algorithms for stochastic nonconvex optimization},
	author={Tran-Dinh, Quoc and Pham, Nhan H and Phan, Dzung T and Nguyen, Lam M},
	journal={arXiv preprint arXiv:1905.05920},
	year={2019}
}

@book{nesterov2018lectures,
	title={Lectures on convex optimization},
	author={Nesterov, Yurii},
	volume={137},
	year={2018},
	publisher={Springer}
}

@inproceedings{he2016deep,
	title={Deep residual learning for image recognition},
	author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
	booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
	pages={770--778},
	year={2016}
}

@techreport{krizhevsky2009learning,
	title={Learning multiple layers of features from tiny images},
	author={Krizhevsky, Alex and Hinton, Geoffrey and others},
	year={2009},
	jnumber = {Technical Report TR-2009},
	institution = {University of Toronto,  Toronto}
}

@inproceedings{paszke2019pytorch,
	title={Pytorch: An imperative style, high-performance deep learning library},
	author={Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and others},
	booktitle={Advances in Neural Information Processing Systems (NeurIPS)},
	year={2019}
}

@article{lecun2010mnist,
  title={MNIST handwritten digit database},
  author={LeCun, Yann and Cortes, Corinna and Burges, CJ},
  journal={ATT Labs [Online]. Available: http://yann.lecun.com/exdb/mnist},
  volume={2},
  year={2010}
}

@book{knuth1997Art,
  author = {Knuth, Donald E.},
  title = {The Art of Computer Programming, Volume 2 (3rd Ed.): Seminumerical Algorithms},
  year = {1997},
  isbn = {0201896842},
  publisher = {Addison-Wesley Longman Publishing Co., Inc.},
  address = {USA}
}

@article{fisher1938statistical,
  title={Statistical tables for biological, agricultural aad medical research},
  author={Fisher, Ronald A and Yates, Frank},
  year={1938},
  publisher={Oliver and Boyd}
}

@article{khanduri2020distributed,
  title={Distributed stochastic non-convex optimization: Momentum-based variance reduction},
  author={Khanduri, Prashant and Sharma, Pranay and Kafle, Swatantra and Bulusu, Saikiran and Rajawat, Ketan and Varshney, Pramod K},
  journal={arXiv preprint arXiv:2005.00224},
  year={2020}
}

@article{sharma2019parallel,
  title={Parallel Restarted SPIDER--Communication Efficient Distributed Nonconvex Optimization with Optimal Computation Complexity},
  author={Sharma, Pranay and Kafle, Swatantra and Khanduri, Prashant and Bulusu, Saikiran and Rajawat, Ketan and Varshney, Pramod K},
  journal={arXiv preprint arXiv:1912.06036},
  year={2019}
}

@article{cutkosky2019momentum,
  title={Momentum-based variance reduction in non-convex {SGD}},
  author={Cutkosky, Ashok and Orabona, Francesco},
  journal={arXiv preprint arXiv:1905.10018},
  year={2019}
}

@techreport{xu2020compressed,
  title={Compressed communication for distributed deep learning: Survey and quantitative evaluation},
  author={Xu, Hang and Ho, Chen-Yu and Abdelmoniem, Ahmed M and Dutta, Aritra and Bergou, El Houcine and Karatsenidis, Konstantinos and Canini, Marco and Kalnis, Panos},
  year={2020}
}

@article{richtarik2021ef21,
  title={{EF21}: A New, Simpler, Theoretically Better, and Practically Faster Error Feedback},
  author={Richt{\'a}rik, Peter and Sokolov, Igor and Fatkhullin, Ilyas},
  journal={In Neural Information Processing Systems, 2021.},
  year={2021}
}

@article{horvath2019stochastic,
  title={Stochastic distributed learning with gradient quantization and variance reduction},
  author={Horv{\'a}th, Samuel and Kovalev, Dmitry and Mishchenko, Konstantin and Stich, Sebastian and Richt{\'a}rik, Peter},
  journal={Optimization Methods and Software},
  year={2022}
}

@inproceedings{haddadpour2021federated,
  title={Federated learning with compression: Unified analysis and sharp guarantees},
  author={Haddadpour, Farzin and Kamani, Mohammad Mahdi and Mokhtari, Aryan and Mahdavi, Mehrdad},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2350--2358},
  year={2021},
  organization={PMLR}
}

@article{das2020improved,
  title={Improved convergence rates for non-convex federated learning with compression},
  author={Das, Rudrajit and Hashemi, Abolfazl and Sanghavi, Sujay and Dhillon, Inderjit S},
  journal={arXiv e-prints},
  pages={arXiv--2012},
  year={2020}
}

@article{gorbunov2021marina,
  title={MARINA: Faster non-convex distributed learning with compression},
  author={Gorbunov, Eduard and Burlachenko, Konstantin and Li, Zhize and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2102.07845},
  year={2021}
}

@article{brown2020language,
  title={Language models are few-shot learners},
  author={Brown, Tom B and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  journal={arXiv preprint arXiv:2005.14165},
  year={2020}
}

@article{bubeck2021universal,
  title={A Universal Law of Robustness via Isoperimetry},
  author={Bubeck, S{\'e}bastien and Sellke, Mark},
  journal={arXiv preprint arXiv:2105.12806},
  year={2021}
}

@article{danilova2020recent,
  title={Recent theoretical advances in non-convex optimization},
  author={Danilova, Marina and Dvurechensky, Pavel and Gasnikov, Alexander and Gorbunov, Eduard and Guminov, Sergey and Kamzolov, Dmitry and Shibaev, Innokentiy},
  journal={arXiv preprint arXiv:2012.06188},
  year={2020}
}

@article{hestness2017deep,
  title={Deep learning scaling is predictable, empirically},
  author={Hestness, Joel and Narang, Sharan and Ardalani, Newsha and Diamos, Gregory and Jun, Heewoo and Kianinejad, Hassan and Patwary, Md and Ali, Mostofa and Yang, Yang and Zhou, Yanqi},
  journal={arXiv preprint arXiv:1712.00409},
  year={2017}
}

@article{ramesh2021zero,
  title={Zero-shot text-to-image generation},
  author={Ramesh, Aditya and Pavlov, Mikhail and Goh, Gabriel and Gray, Scott and Voss, Chelsea and Radford, Alec and Chen, Mark and Sutskever, Ilya},
  journal={arXiv preprint arXiv:2102.12092},
  year={2021}
}

@article{konevcny2016federated,
  title={Federated learning: Strategies for improving communication efficiency},
  author={Kone{\v{c}}n{\'y}, Jakub and McMahan, H Brendan and Yu, Felix X and Richt{\'a}rik, Peter and Suresh, Ananda Theertha and Bacon, Dave},
  journal={arXiv preprint arXiv:1610.05492},
  year={2016}
}

@article{wang2021field,
  title={A field guide to federated optimization},
  author={Wang, Jianyu and Charles, Zachary and Xu, Zheng and Joshi, Gauri and McMahan, H Brendan and Al-Shedivat, Maruan and Andrew, Galen and Avestimehr, Salman and Daly, Katharine and Data, Deepesh and others},
  journal={arXiv preprint arXiv:2107.06917},
  year={2021}
}

@article{bonawitz2019towards,
  title={Towards federated learning at scale: System design},
  author={Bonawitz, Keith and Eichner, Hubert and Grieskamp, Wolfgang and Huba, Dzmitry and Ingerman, Alex and Ivanov, Vladimir and Kiddon, Chloe and Kone{\v{c}}n{\'y}, Jakub and Mazzocchi, Stefano and McMahan, H Brendan and others},
  journal={arXiv preprint arXiv:1902.01046},
  year={2019}
}

@article{li2020federated,
  title={Federated optimization in heterogeneous networks},
  author={Li, Tian and Sahu, Anit Kumar and Zaheer, Manzil and Sanjabi, Maziar and Talwalkar, Ameet and Smith, Virginia},
  journal={Proceedings of Machine Learning and Systems},
  volume={2},
  pages={429--450},
  year={2020}
}

@article{vogels2021relaysum,
  title={{RelaySum} for decentralized deep learning on heterogeneous data},
  author={Vogels, Thijs and He, Lie and Koloskova, Anastasiia and Karimireddy, Sai Praneeth and Lin, Tao and Stich, Sebastian U and Jaggi, Martin},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{nedic2014distributed,
  title={Distributed optimization over time-varying directed graphs},
  author={Nedi{\'c}, Angelia and Olshevsky, Alex},
  journal={IEEE Transactions on Automatic Control},
  volume={60},
  number={3},
  pages={601--615},
  year={2014},
  publisher={IEEE}
}

@article{tran2021hybrid,
  title={A hybrid stochastic optimization framework for composite nonconvex optimization},
  author={Tran-Dinh, Quoc and Pham, Nhan H and Phan, Dzung T and Nguyen, Lam M},
  journal={Mathematical Programming},
  pages={1--67},
  year={2021},
  publisher={Springer}
}

@article{liu2020optimal,
  title={An optimal hybrid variance-reduced algorithm for stochastic composite nonconvex optimization},
  author={Liu, Deyi and Nguyen, Lam M and Tran-Dinh, Quoc},
  journal={arXiv preprint arXiv:2008.09055},
  year={2020}
}

@article{kairouz2021advances,
  title={Advances and open problems in federated learning},
  author={Kairouz, Peter and McMahan, H Brendan and Avent, Brendan and Bellet, Aur{\'e}lien and Bennis, Mehdi and Bhagoji, Arjun Nitin and Bonawitz, Kallista and Charles, Zachary and Cormode, Graham and Cummings, Rachel and others},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={14},
  number={1--2},
  pages={1--210},
  year={2021},
  publisher={Now Publishers, Inc.}
}

@article{tyurin2022dasha,
  title={{DASHA}: Distributed Nonconvex Optimization with Communication Compression, Optimal Oracle Complexity, and No Client Synchronization},
  author={Tyurin, Alexander and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2202.01268},
  year={2022}
}

@article{wangni2018gradient,
  title={Gradient sparsification for communication-efficient distributed optimization},
  author={Wangni, Jianqiao and Wang, Jialei and Liu, Ji and Zhang, Tong},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}

@inproceedings{horvath2019natural,
  title={Natural compression for distributed deep learning},
  author={Horv{\'a}th, Samuel and Ho, Chen-Yu and Horv\'{a}th, \v{L}udov\'{i}t and Sahu, Atal Narayan and Canini, Marco and Richt{\'a}rik, Peter},
    booktitle = {Mathematical and Scientific Machine Learning},
  note={arXiv preprint arXiv:1905.10988},
  year={2022}
}

@inproceedings{Cnat,
  author  = {Samuel Horv\'{a}th and Chen-Yu Ho and \v{L}udov\'{i}t Horv\'{a}th and Atal Narayan Sahu and Marco Canini and Peter Richt\'{a}rik},
  title   = {Natural compression for distributed deep learning},
  booktitle = {Mathematical and Scientific Machine Learning},
  year    = {2022},
  note = {arXiv preprint arXiv:1905.10988, 2019}
}

@article{stich2018sparsified,
  title={Sparsified {SGD} with memory},
  author={Stich, Sebastian U and Cordonnier, Jean-Baptiste and Jaggi, Martin},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}

@article{szlendak2021permutation,
  title={Permutation Compressors for Provably Faster Distributed Nonconvex Optimization},
  author={Szlendak, Rafa{\l} and Tyurin, Alexander and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2110.03300},
  year={2021}
}

@article{suresh2022correlated,
  title={Correlated quantization for distributed mean estimation and optimization},
  author={Suresh, Ananda Theertha and Sun, Ziteng and Ro, Jae Hun and Yu, Felix},
  journal={arXiv preprint arXiv:2203.04925},
  year={2022}
}

@article{fatkhullin2021ef21,
  title={{EF21} with bells \& whistles: Practical algorithmic extensions of modern error feedback},
  author={Fatkhullin, Ilyas and Sokolov, Igor and Gorbunov, Eduard and Li, Zhize and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2110.03294},
  year={2021}
}

@article{reddi2020adaptive,
  title={Adaptive federated optimization},
  author={Reddi, Sashank and Charles, Zachary and Zaheer, Manzil and Garrett, Zachary and Rush, Keith and Kone{\v{c}}n{\'y}, Jakub and Kumar, Sanjiv and McMahan, H Brendan},
  journal={arXiv preprint arXiv:2003.00295},
  year={2020}
}

@article{ramaswamy2019federated,
  title={Federated learning for emoji prediction in a mobile keyboard},
  author={Ramaswamy, Swaroop and Mathews, Rajiv and Rao, Kanishka and Beaufays, Fran{\c{c}}oise},
  journal={arXiv preprint arXiv:1906.04329},
  year={2019}
}

@article{lin2017deep,
  title={Deep gradient compression: Reducing the communication bandwidth for distributed training},
  author={Lin, Yujun and Han, Song and Mao, Huizi and Wang, Yu and Dally, William J},
  journal={arXiv preprint arXiv:1712.01887},
  year={2017}
}

@article{carmon2020lower,
  title={Lower bounds for finding stationary points I},
  author={Carmon, Yair and Duchi, John C and Hinder, Oliver and Sidford, Aaron},
  journal={Mathematical Programming},
  volume={184},
  number={1},
  pages={71--120},
  year={2020},
  publisher={Springer}
}

@inproceedings{ioffe2015batch,
  title={Batch normalization: Accelerating deep network training by reducing internal covariate shift},
  author={Ioffe, Sergey and Szegedy, Christian},
  booktitle={International Conference on Machine Learning},
  pages={448--456},
  year={2015},
  organization={PMLR}
}

@article{zhao2021faster,
  title={Faster Rates for Compressed Federated Learning with Client-Variance Reduction},
  author={Zhao, Haoyu and Burlachenko, Konstantin and Li, Zhize and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2112.13097},
  year={2021}
}

@article{zhao2021fedpage,
  title={{FedPAGE}: A fast local stochastic gradient method for communication-efficient federated learning},
  author={Zhao, Haoyu and Li, Zhize and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2108.04755},
  year={2021}
}

@article{li2021zerosarah,
  title={{ZeroSARAH}: Efficient nonconvex finite-sum optimization with zero full gradient computation},
  author={Li, Zhize and Hanzely, Slavom{\'\i}r and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2103.01447},
  year={2021}
}

@inproceedings{narayanan2019pipedream,
  title={{PipeDream}: generalized pipeline parallelism for DNN training},
  author={Narayanan, Deepak and Harlap, Aaron and Phanishayee, Amar and Seshadri, Vivek and Devanur, Nikhil R and Ganger, Gregory R and Gibbons, Phillip B and Zaharia, Matei},
  booktitle={Proceedings of the 27th ACM Symposium on Operating Systems Principles},
  pages={1--15},
  year={2019}
}

@article{sapio2019scaling,
  title={Scaling distributed machine learning with in-network aggregation},
  author={Sapio, Amedeo and Canini, Marco and Ho, Chen-Yu and Nelson, Jacob and Kalnis, Panos and Kim, Changhoon and Krishnamurthy, Arvind and Moshref, Masoud and Ports, Dan RK and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:1903.06701},
  year={2019}
}

@inproceedings{xu2021grace,
  title={Grace: A compressed communication framework for distributed machine learning},
  author={Xu, Hang and Ho, Chen-Yu and Abdelmoniem, Ahmed M and Dutta, Aritra and Bergou, El Houcine and Karatsenidis, Konstantinos and Canini, Marco and Kalnis, Panos},
  booktitle={2021 IEEE 41st International Conference on Distributed Computing Systems (ICDCS)},
  pages={561--572},
  year={2021},
  organization={IEEE}
}

@inproceedings{richtarik20223pc,
  title={{3PC}: Three Point Compressors for Communication-Efficient Distributed Training and a Better Theory for Lazy Aggregation},
  author={Richt{\'a}rik, Peter and Sokolov, Igor and Fatkhullin, Ilyas and Gasanov, Elnur and Li, Zhize and Gorbunov, Eduard},
  journal={International Conference on Machine Learning},
  year={2022},
  organization={PMLR}
}

@article{tyurin2022computation,
  title={A Computation and Communication Efficient Method for Distributed Nonconvex Problems in the Partial Participation Setting},
  author={Tyurin, Alexander and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2205.15580},
  year={2022}
}

@article{huang2022lower,
  title={Lower Bounds and Nearly Optimal Algorithms in Distributed Learning with Communication Compression},
  author={Huang, Xinmeng and Chen, Yiming and Yin, Wotao and Yuan, Kun},
  journal={arXiv preprint arXiv:2206.03665},
  year={2022}
}

@article{philippenko2021preserved,
  title={Preserved central model for faster bidirectional compression in distributed settings},
  author={Philippenko, Constantin and Dieuleveut, Aymeric},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={2387--2399},
  year={2021}
}

@inproceedings{wang2022communication,
  title={Communication-Compressed Adaptive Gradient Method for Distributed Nonconvex Optimization},
  author={Wang, Yujia and Lin, Lu and Chen, Jinghui},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={6292--6320},
  year={2022},
  organization={PMLR}
}

@inproceedings{tang2019doublesqueeze,
  title={Doublesqueeze: Parallel stochastic gradient descent with double-pass error-compensated compression},
  author={Tang, Hanlin and Yu, Chen and Lian, Xiangru and Zhang, Tong and Liu, Ji},
  booktitle={International Conference on Machine Learning},
  pages={6155--6165},
  year={2019},
  organization={PMLR}
}

@article{condat2022ef,
  title={{EF-BV}: A Unified Theory of Error Feedback and Variance Reduction Mechanisms for Biased and Unbiased Compression in Distributed Optimization},
  author={Condat, Laurent and Yi, Kai and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2205.04180},
  year={2022}
}

@article{sahu2021rethinking,
  title={Rethinking gradient sparsification as total error minimization},
  author={Sahu, Atal and Dutta, Aritra and M Abdelmoniem, Ahmed and Banerjee, Trambak and Canini, Marco and Kalnis, Panos},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={8133--8146},
  year={2021}
}

@article{li2021canita,
  title={{CANITA}: {F}aster rates for distributed convex optimization with communication compression},
  author={Li, Zhize and Richt{\'a}rik, Peter},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={13770--13781},
  year={2021}
}

@article{khaled2020unified,
  title={Unified analysis of stochastic gradient methods for composite convex and smooth optimization},
  author={Khaled, Ahmed and Sebbouh, Othmane and Loizou, Nicolas and Gower, Robert M and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2006.11573},
  year={2020}
}

@misc{PSGD,
  author = {Vardhan, Harsh and Stich, Sebastian U.},
  keywords = {Machine Learning (cs.LG), Machine Learning (stat.ML), FOS: Computer and information sciences, FOS: Computer and information sciences},
  title = {Tackling benign nonconvexity with smoothing and stochastic gradients},
  publisher = {arXiv},
  year = {2022},
  copyright = {arXiv.org perpetual, non-exclusive license}
}

@article{duchi2012randomized,
  title={Randomized smoothing for stochastic optimization},
  author={Duchi, John C and Bartlett, Peter L and Wainwright, Martin J},
  journal={SIAM Journal on Optimization},
  volume={22},
  number={2},
  pages={674--701},
  year={2012},
  publisher={SIAM}
}

@article{orvieto2022anticorrelated,
  title={Anticorrelated noise injection for improved generalization},
  author={Orvieto, Antonio and Kersting, Hans and Proske, Frank and Bach, Francis and Lucchi, Aurelien},
  journal={arXiv preprint arXiv:2202.02831},
  year={2022}
}

@inproceedings{liu2020double,
  title={A double residual compression algorithm for efficient distributed learning},
  author={Liu, Xiaorui and Li, Yao and Tang, Jiliang and Yan, Ming},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={133--143},
  year={2020},
  organization={PMLR}
}

@article{philippenko2020artemis,
  title={Artemis: tight convergence guarantees for bidirectional compression in federated learning},
  author={Philippenko, Constantin and Dieuleveut, Aymeric},
  journal={arXiv preprint arXiv:2006.14591},
  year={2020}
}

@article{safaryan2021fednl,
  title={FedNL: Making Newton-type methods applicable to federated learning},
  author={Safaryan, Mher and Islamov, Rustem and Qian, Xun and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2106.02969},
  year={2021}
}

@article{schmidt2013fast,
  title={Fast convergence of stochastic gradient descent under a strong growth condition},
  author={Schmidt, Mark and Roux, Nicolas Le},
  journal={arXiv preprint arXiv:1308.6370},
  year={2013}
}

@inproceedings{vaswani2019fast,
  title={Fast and faster convergence of {SGD} for over-parameterized models and an accelerated perceptron},
  author={Vaswani, Sharan and Bach, Francis and Schmidt, Mark},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={1195--1204},
  year={2019},
  organization={PMLR}
}

@inproceedings{meng2020fast,
  title={Fast and furious convergence: Stochastic second order methods under interpolation},
  author={Meng, Si Yi and Vaswani, Sharan and Laradji, Issam Hadj and Schmidt, Mark and Lacoste-Julien, Simon},
  booktitle={The 23rd International Conference on Artificial Intelligence and Statistics},
  pages={1375--1386},
  year={2020},
  organization={PMLR}
}

@article{recht2011hogwild,
  title={Hogwild!: A lock-free approach to parallelizing stochastic gradient descent},
  author={Recht, Benjamin and Re, Christopher and Wright, Stephen and Niu, Feng},
  journal={Advances in Neural Information Processing Systems},
  volume={24},
  year={2011}
}

@article{goyal2017accurate,
  title={Accurate, large minibatch {SGD}: Training imagenet in 1 hour},
  author={Goyal, Priya and Doll{\'a}r, Piotr and Girshick, Ross and Noordhuis, Pieter and Wesolowski, Lukasz and Kyrola, Aapo and Tulloch, Andrew and Jia, Yangqing and He, Kaiming},
  journal={arXiv preprint arXiv:1706.02677},
  year={2017}
}

@article{murata2021bias,
  title={Bias-variance reduced local {SGD} for less heterogeneous federated learning},
  author={Murata, Tomoya and Suzuki, Taiji},
  journal={arXiv preprint arXiv:2102.03198},
  year={2021}
}

@article{koloskova2022sharper,
  title={Sharper convergence guarantees for {A}synchronous {SGD} for distributed and federated learning},
  author={Koloskova, Anastasiia and Stich, Sebastian U and Jaggi, Martin},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={17202--17215},
  year={2022}
}
@article{xu2021deepreduce,
  title={DeepReduce: A Sparse-tensor Communication Framework for Federated Deep Learning},
  author={Xu, Hang and Kostopoulou, Kelly and Dutta, Aritra and Li, Xin and Ntoulas, Alexandros and Kalnis, Panos},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={21150--21163},
  year={2021}
}

@inproceedings{gorbunov2020unified,
  title={A unified theory of {SGD}: Variance reduction, sampling, quantization and coordinate descent},
  author={Gorbunov, Eduard and Hanzely, Filip and Richt{\'a}rik, Peter},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={680--690},
  year={2020},
  organization={PMLR}
}

@book{nesterov2003introductory,
  title={Introductory lectures on convex optimization: A basic course},
  author={Nesterov, Yurii},
  volume={87},
  year={2003},
  publisher={Springer Science \& Business Media}
}

@inproceedings{lu2021optimal,
  title={Optimal complexity in decentralized training},
  author={Lu, Yucheng and De Sa, Christopher},
  booktitle={International Conference on Machine Learning},
  pages={7111--7123},
  year={2021},
  organization={PMLR}
}

@book{lan2020first,
  title={First-order and stochastic optimization methods for machine learning},
  author={Lan, Guanghui},
  year={2020},
  publisher={Springer}
}

@article{arjevani2022lower,
  title={Lower bounds for non-convex stochastic optimization},
  author={Arjevani, Yossi and Carmon, Yair and Duchi, John C and Foster, Dylan J and Srebro, Nathan and Woodworth, Blake},
  journal={Mathematical Programming},
  pages={1--50},
  year={2022},
  publisher={Springer}
}

@article{krizhevsky2017imagenet,
  title={Imagenet classification with deep convolutional neural networks},
  author={Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
  journal={Communications of the ACM},
  volume={60},
  number={6},
  pages={84--90},
  year={2017},
  publisher={AcM New York, NY, USA}
}

@article{ghadimi2013stochastic,
  title={Stochastic first-and zeroth-order methods for nonconvex stochastic programming},
  author={Ghadimi, Saeed and Lan, Guanghui},
  journal={SIAM Journal on Optimization},
  volume={23},
  number={4},
  pages={2341--2368},
  year={2013},
  publisher={SIAM}
}

@article{woodworth2021even,
  title={An even more optimal stochastic optimization algorithm: minibatching and interpolation learning},
  author={Woodworth, Blake E and Srebro, Nathan},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={7333--7345},
  year={2021}
}

@article{nemirovskij1983problem,
  title={Problem complexity and method efficiency in optimization},
  author={Nemirovskij, Arkadij Semenovi{\v{c}} and Yudin, David Borisovich},
  year={1983},
  publisher={Wiley-Interscience}
}

@article{cotter2011better,
  title={Better mini-batch algorithms via accelerated gradient methods},
  author={Cotter, Andrew and Shamir, Ohad and Srebro, Nati and Sridharan, Karthik},
  journal={Advances in Neural Information Processing Systems},
  volume={24},
  year={2011}
}

@inproceedings{gower2019sgd,
  title={{SGD}: General analysis and improved rates},
  author={Gower, Robert Mansel and Loizou, Nicolas and Qian, Xun and Sailanbayev, Alibek and Shulgin, Egor and Richt{\'a}rik, Peter},
  booktitle={International Conference on Machine Learning},
  pages={5200--5209},
  year={2019},
  organization={PMLR}
}

@inproceedings{dutta2018slow,
  title={Slow and stale gradients can win the race: Error-runtime trade-offs in distributed {SGD}},
  author={Dutta, Sanghamitra and Joshi, Gauri and Ghosh, Soumyadip and Dube, Parijat and Nagpurkar, Priya},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={803--812},
  year={2018},
  organization={PMLR}
}

@article{chen2016revisiting,
  title={Revisiting distributed synchronous {SGD}},
  author={Chen, Jianmin and Pan, Xinghao and Monga, Rajat and Bengio, Samy and Jozefowicz, Rafal},
  journal={arXiv preprint arXiv:1604.00981},
  year={2016}
}

@inproceedings{nguyen2018sgd,
  title={{SGD} and Hogwild! Convergence without the bounded gradients assumption},
  author={Nguyen, Lam and Nguyen, Phuong Ha and Dijk, Marten and Richt{\'a}rik, Peter and Scheinberg, Katya and Tak{\'a}c, Martin},
  booktitle={International Conference on Machine Learning},
  pages={3750--3758},
  year={2018},
  organization={PMLR}
}

@inproceedings{arjevani2020tight,
  title={A tight convergence analysis for stochastic gradient descent with delayed updates},
  author={Arjevani, Yossi and Shamir, Ohad and Srebro, Nathan},
  booktitle={Algorithmic Learning Theory},
  pages={111--132},
  year={2020},
  organization={PMLR}
}

@article{feyzmahdavian2016asynchronous,
  title={An asynchronous mini-batch algorithm for regularized stochastic optimization},
  author={Feyzmahdavian, Hamid Reza and Aytekin, Arda and Johansson, Mikael},
  journal={IEEE Transactions on Automatic Control},
  volume={61},
  number={12},
  pages={3740--3754},
  year={2016},
  publisher={IEEE}
}

@article{mishchenko2022asynchronous,
  title={Asynchronous {SGD} beats minibatch {SGD} under arbitrary delays},
  author={Mishchenko, Konstantin and Bach, Francis and Even, Mathieu and Woodworth, Blake},
  journal={arXiv preprint arXiv:2206.07638},
  year={2022}
}

@inproceedings{woodworth2020local,
  title={Is local {SGD} better than minibatch {SGD}?},
  author={Woodworth, Blake and Patel, Kumar Kshitij and Stich, Sebastian and Dai, Zhen and Bullins, Brian and Mcmahan, Brendan and Shamir, Ohad and Srebro, Nathan},
  booktitle={International Conference on Machine Learning},
  pages={10334--10343},
  year={2020},
  organization={PMLR}
}

@article{wu2022delay,
  title={Delay-adaptive step-sizes for asynchronous learning},
  author={Wu, Xuyang and Magnusson, Sindri and Feyzmahdavian, Hamid Reza and Johansson, Mikael},
  journal={arXiv preprint arXiv:2202.08550},
  year={2022}
}

@article{woodworth2018graph,
  title={Graph oracle models, lower bounds, and gaps for parallel stochastic optimization},
  author={Woodworth, Blake E and Wang, Jialei and Smith, Adam and McMahan, Brendan and Srebro, Nati},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}

@article{kingma2014adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik P and Ba, Jimmy},
  journal={arXiv preprint arXiv:1412.6980},
  year={2014}
}

@article{cohen2021asynchronous,
  title={Asynchronous Stochastic Optimization Robust to Arbitrary Delays},
  author={Cohen, Alon and Daniely, Amit and Drori, Yoel and Koren, Tomer and Schain, Mariano},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={9024--9035},
  year={2021}
}

@article{peng2016arock,
  title={Arock: an algorithmic framework for asynchronous parallel coordinate updates},
  author={Peng, Zhimin and Xu, Yangyang and Yan, Ming and Yin, Wotao},
  journal={SIAM Journal on Scientific Computing},
  volume={38},
  number={5},
  pages={A2851--A2879},
  year={2016},
  publisher={SIAM}
}

@inproceedings{mishchenko2018delay,
  title={A delay-tolerant proximal-gradient algorithm for distributed learning},
  author={Mishchenko, Konstantin and Iutzeler, Franck and Malick, J{\'e}r{\^o}me and Amini, Massih-Reza},
  booktitle={International Conference on Machine Learning},
  pages={3587--3595},
  year={2018},
  organization={PMLR}
}

@article{aytekin2016analysis,
  title={Analysis and implementation of an asynchronous optimization algorithm for the parameter server},
  author={Aytekin, Arda and Feyzmahdavian, Hamid Reza and Johansson, Mikael},
  journal={arXiv preprint arXiv:1610.05507},
  year={2016}
}

@article{cannelli2016asynchronous,
  title={Asynchronous parallel algorithms for nonconvex big-data optimization: Model and convergence},
  author={Cannelli, Loris and Facchinei, Francisco and Kungurtsev, Vyacheslav and Scutari, Gesualdo},
  journal={arXiv preprint arXiv:1607.04818},
  year={2016}
}

@inproceedings{nguyen2022federated,
  title={Federated learning with buffered asynchronous aggregation},
  author={Nguyen, John and Malik, Kshitiz and Zhan, Hongyuan and Yousefpour, Ashkan and Rabbat, Mike and Malek, Mani and Huba, Dzmitry},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3581--3607},
  year={2022},
  organization={PMLR}
}

@article{sergeev2018horovod,
  Author = {Alexander Sergeev and Mike Del Balso},
  Journal = {arXiv preprint arXiv:1802.05799},
  Title = {Horovod: Fast and easy distributed deep learning in {TensorFlow}},
  Year = {2018}
}

@book{bauschke2011convex,
  title={Convex analysis and monotone operator theory in Hilbert spaces},
  author={Bauschke, Heinz H and Combettes, Patrick L and others},
  volume={408},
  year={2011},
  publisher={Springer}
}

@article{parikh2014proximal,
  title={Proximal algorithms},
  author={Parikh, Neal and Boyd, Stephen and others},
  journal={Foundations and trends{\textregistered} in Optimization},
  volume={1},
  number={3},
  pages={127--239},
  year={2014},
  publisher={Now Publishers, Inc.}
}

@inproceedings{carmon2022making,
  title={Making {SGD} parameter-free},
  author={Carmon, Yair and Hinder, Oliver},
  booktitle={Conference on Learning Theory},
  pages={2360--2389},
  year={2022},
  organization={PMLR}
}

@article{tyurin2024optimal,
  title={Optimal time complexities of parallel stochastic optimization methods under a fixed computation model},
  author={Tyurin, Alexander and Richt\'{a}rik, Peter},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2024}
}

@article{mcmahan2016federated,
  title={Federated learning of deep networks using model averaging},
  author={McMahan, H Brendan and Moore, Eider and Ramage, Daniel and y Arcas, Blaise Ag{\"u}era},
  journal={arXiv preprint arXiv:1602.05629},
  volume={2},
  pages={2},
  year={2016}
}

@misc{hanna2022adaptive,
      title={Adaptive Stochastic Gradient Descent for Fast and Communication-Efficient Distributed Learning}, 
      author={Serge Kas Hanna and Rawad Bitar and Parimal Parag and Venkat Dasari and Salim El Rouayheb},
      year={2022},
      eprint={2208.03134},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@INPROCEEDINGS{hanna2020strag,
  author={Hanna, Serge Kas and Bitar, Rawad and Parag, Parimal and Dasari, Venkat and El Rouayheb, Salim},
  booktitle={ICASSP 2020 - 2020 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, 
  title={Adaptive Distributed Stochastic Gradient Descent for Minimizing Delay in the Presence of Stragglers}, 
  year={2020},
  volume={},
  number={},
  pages={4262-4266},
  keywords={Adaptation models;Upper bound;Signal processing algorithms;Stochastic processes;Signal processing;Delays;Speech processing;Distributed SGD;adaptive policy;stragglers.},
  doi={10.1109/ICASSP40776.2020.9053961}
}

@inproceedings{islamov2023asgrad,
  title={{AsGrad}: A sharp unified analysis of asynchronous-{SGD} algorithms},
  author={Islamov, Rustem and Safaryan, Mher and Alistarh, Dan},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={649--657},
  year={2024},
  organization={PMLR}
}


@article{lecun1998gradient,
  title={Gradient-based learning applied to document recognition},
  author={LeCun, Yann and Bottou, L{\'e}on and Bengio, Yoshua and Haffner, Patrick},
  journal={Proceedings of the IEEE},
  volume={86},
  number={11},
  pages={2278--2324},
  year={1998},
  publisher={Ieee}
}

@article{zhu1997algorithm,
  title={Algorithm 778: L-BFGS-B: Fortran subroutines for large-scale bound-constrained optimization},
  author={Zhu, Ciyou and Byrd, Richard H and Lu, Peihuang and Nocedal, Jorge},
  journal={ACM Transactions on Mathematical Software (TOMS)},
  volume={23},
  number={4},
  pages={550--560},
  year={1997},
  publisher={ACM}
}
@article{robbins1951stochastic,
  title={A Stochastic Approximation Method},
  author={Robbins, Herbert and Monro, Sutton},
  journal={The Annals of Mathematical Statistics},
  volume={22},
  number={3},
  pages={400--407},
  year={1951},
  publisher={Institute of Mathematical Statistics},
  doi={10.1214/aoms/1177729586}
}
