@article{tyurin2022weightedab,
	title={Sharper Rates and Flexible Framework for Nonconvex {SGD} with Client and Data Sampling
	},
	author={Alexander Tyurin and Lukang Sun and Konstantin Burlachenko and Peter Richt{\'a}rik},
	year={2022},
	journal={arXiv preprint arXiv:2206.02275},
}

@Book{NesterovBook,
	title     = {Introductory lectures on convex optimization: a basic course (Applied Optimization)},
	publisher = {Kluwer Academic Publishers},
	year      = {2004},
	author    = {Yurii Nesterov},
}


@inproceedings{DoubleSqueeze,
	title={{D}ouble{S}queeze: {P}arallel Stochastic Gradient Descent with Double-Pass Error-Compensated Compression},
	author={Hanlin Tang and Xiangru Lian and Chen Yu and Tong Zhang and Ji Liu},
	booktitle = {Proceedings of the 36th International Conference on Machine Learning (ICML)},
	year={2020},
}

@article{ef21,
	title={{EF21}: A New, Simpler, Theoretically Better, and Practically Faster Error Feedback},
	author={Richt{\'a}rik, Peter and Sokolov, Igor and Fatkhullin, Ilyas},
	journal={arXiv preprint arXiv:2106.05203},
	year={2021}
}

@misc{vargaftik2021drive,
	title={{DRIVE}: One-bit Distributed Mean Estimation}, 
	author={Shay Vargaftik and Ran Ben Basat and Amit Portnoy and Gal Mendelson and Yaniv Ben-Itzhak and Michael Mitzenmacher},
	year={2021},
	eprint={2105.08339},
	archivePrefix={arXiv},
	primaryClass={cs.LG}
}

@misc{szlendak2021permutation,
	title={Permutation Compressors for Provably Faster Distributed Nonconvex Optimization}, 
	author={Rafa\l Szlendak and Alexander Tyurin and Peter Richt{\'a}rik},
	year={2021},
	eprint={2110.03300},
	archivePrefix={arXiv},
	primaryClass={cs.LG}
}

@misc{suresh2022correlated,
	title={Correlated quantization for distributed mean estimation and optimization}, 
	author={Ananda Theertha Suresh and Ziteng Sun and Jae Hun Ro and Felix Yu},
	year={2022},
	eprint={2203.04925},
	archivePrefix={arXiv},
	primaryClass={cs.LG}
}

@misc{gorbunov2022marina,
	title={{MARINA}: Faster Non-Convex Distributed Learning with Compression}, 
	author={Eduard Gorbunov and Konstantin Burlachenko and Zhize Li and Peter Richtárik},
	year={2022},
	eprint={2102.07845},
	archivePrefix={arXiv},
	primaryClass={cs.LG}
}
@article{arjevani2019lower,
	title={Lower bounds for non-convex stochastic optimization},
	author={Arjevani, Yossi and Carmon, Yair and Duchi, John C and Foster, Dylan J and Srebro, Nathan and Woodworth, Blake},
	journal={arXiv preprint arXiv:1912.02365},
	year={2019}
}

@article{YANG2019,
	title = {A survey of distributed optimization},
	journal = {Annual Reviews in Control},
	volume = {47},
	pages = {278-305},
	year = {2019},
	issn = {1367-5788},
	author = {Tao Yang and Xinlei Yi and Junfeng Wu and Ye Yuan and Di Wu and Ziyang Meng and Yiguang Hong and Hong Wang and Zongli Lin and Karl H. Johansson}
}
@ARTICLE{FLChallenges,
	author={Li, Tian and Sahu, Anit Kumar and Talwalkar, Ameet and Smith, Virginia},
	journal={IEEE Signal Processing Magazine}, 
	title={Federated Learning: Challenges, Methods, and Future Directions}, 
	year={2020},
	volume={37},
	number={3},
	pages={50-60}
}
@preprint{KairouzFL,
	title = {Advances and open problems in federated learning},
	author = {Peter Kairouz and et. al},
	journal = {arXiv preprint arXiv:1912.04977},
	year = {2019}
}

@article{BeckAccel,
	author = {Amir Beck and Marc Teboulle},
	title = {A fast iterative shrinkage-thresholding algorithm for linear inverse problems},
	journal = {SIAM Journal on Imaging Sciences},
	volume = {2},
	issue = {1},
	pages = {183-202},
	year = {2009}
}

@InProceedings{pmlr-v38-choromanska15,
	title = 	 {The Loss Surfaces of Multilayer Networks},
	author = 	 {Choromanska, Anna and Henaff, MIkael and Mathieu, Michael and Ben Arous, Gerard and LeCun, Yann},
	booktitle = 	 {Proceedings of the Eighteenth International Conference on Artificial Intelligence and Statistics},
	pages = 	 {192--204},
	year = 	 {2015},
	editor = 	 {Lebanon, Guy and Vishwanathan, S. V. N.},
	volume = 	 {38},
	series = 	 {Proceedings of Machine Learning Research},
	address = 	 {San Diego, California, USA},
	month = 	 {09--12 May},
	publisher =    {PMLR}
}
@InProceedings{KingBa15,
	author    = {Kingma, Diederik and Ba, Jimmy},
	booktitle = {International Conference on Learning Representations (ICLR)},
	title     = {Adam: A Method for Stochastic Optimization},
	year      = {2015},
	address   = {San Diega, CA, USA},
	optmonth  = {12},
}
@article{NesterovPaper,
	author = {Yurii Nesterov},
	title= {A method for unconstrained convex minimization problem with the rate of
	convergence $o(1/k^2)$},
	volume = {269},
	journal= {In Doklady AN USSR},
	pages={543–547},
	year={1983}
}

@inproceedings{LanLZ19-1,
	title = {A unified variance-reduced accelerated gradient method for convex optimization},
	author = {Guanghui Lan and Zhize Li and Yi Zhou},
	year = {2019},
	cites = {0},
	citedby = {0},
	pages = {10462-10472},
	booktitle = {Advances in Neural Information Processing Systems 32: Annual Conference on Neural Information Processing Systems 2019, NeurIPS 2019, 8-14 December 2019, Vancouver, BC, Canada},
	editor = {Hanna M. Wallach and Hugo Larochelle and Alina Beygelzimer and Florence d'Alché-Buc and Edward A. Fox and Roman Garnett},
}

@inproceedings{Zhu2017KatyushaTF,
	title={Katyusha: the first direct acceleration of stochastic gradient methods},
	author={Zeyuan Allen Zhu},
	booktitle={Symposium on the Theory of Computing},
	year={2017}
}
@inproceedings{Kovalev2019DontJT,
	title={Don't Jump Through Hoops and Remove Those Loops: {SVRG} and {Katyusha} are Better Without the Outer Loop},
	author={D. Kovalev and Samuel Horv{\'a}th and Peter Richt{\'a}rik},
	booktitle={International Conference on Algorithmic Learning Theory},
	year={2019}
}

@InProceedings{mishchenko22b,
	title = 	 {{P}rox{S}kip: Yes! {L}ocal Gradient Steps Provably Lead to Communication Acceleration! {F}inally!},
	author =       {Mishchenko, Konstantin and Malinovsky, Grigory and Stich, Sebastian and Richtarik, Peter},
	booktitle = 	 {Proceedings of the 39th International Conference on Machine Learning},
	pages = 	 {15750--15769},
	year = 	 {2022},
	editor = 	 {Chaudhuri, Kamalika and Jegelka, Stefanie and Song, Le and Szepesvari, Csaba and Niu, Gang and Sabato, Sivan},
	volume = 	 {162},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {17--23 Jul},
	publisher =    {PMLR}
}

@inproceedings{Seide20141bitSG,
	title={1-bit stochastic gradient descent and its application to data-parallel distributed training of speech DNNs},
	author={Frank Seide and Hao Fu and Jasha Droppo and Gang Li and Dong Yu},
	booktitle={Interspeech},
	year={2014}
}

@InProceedings{mcmahan17a,
	title = 	 {Communication-Efficient Learning of Deep Networks from Decentralized Data},
	author = 	 {McMahan, Brendan and Moore, Eider and Ramage, Daniel and Hampson, Seth and Arcas, Blaise Aguera y},
	booktitle = 	 {Proceedings of the 20th International Conference on Artificial Intelligence and Statistics},
	pages = 	 {1273--1282},
	year = 	 {2017},
	editor = 	 {Singh, Aarti and Zhu, Jerry},
	volume = 	 {54},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {20--22 Apr},
	publisher =    {PMLR},
	pdf = 	 {http://proceedings.mlr.press/v54/mcmahan17a/mcmahan17a.pdf},
}
@article{Povey2014ParallelTO,
	title={Parallel training of DNNs with Natural Gradient and Parameter Averaging},
	author={Daniel Povey and Xiaohui Zhang and Sanjeev Khudanpur},
	journal={arXiv: Neural and Evolutionary Computing},
	year={2014},
}
@inproceedings{Moritz16,
	author       = {Philipp Moritz and
	Robert Nishihara and
	Ion Stoica and
	Michael I. Jordan},
	editor       = {Yoshua Bengio and
	Yann LeCun},
	title        = {{SparkNet}: Training Deep Networks in Spark},
	booktitle    = {4th International Conference on Learning Representations, {ICLR} 2016,
	San Juan, Puerto Rico, May 2-4, 2016, Conference Track Proceedings},
	year         = {2016}
}
@inproceedings{Li20,
	author = {Li, Zhize and Kovalev, Dmitry and Qian, Xun and Richt\'arik, Peter},
	title = {Acceleration for Compressed Gradient Descent in Distributed and Federated Optimization},
	year = {2020},
	publisher = {JMLR.org},
	booktitle = {Proceedings of the 37th International Conference on Machine Learning},
	articleno = {547},
	numpages = {10},
	series = {ICML'20}
}
@article{lecun2015deep,
	author = {LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
	journal = {Nature},
	number = 7553,
	pages = 436,
	publisher = {Nature Publishing Group},
	title = {Deep learning},
	volume = 521,
	year = 2015
}
@inproceedings{Attention,
	author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, \L ukasz and Polosukhin, Illia},
	booktitle = {Advances in Neural Information Processing Systems},
	editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
	pages = {},
	publisher = {Curran Associates, Inc.},
	title = {Attention is All you Need},
	volume = {30},
	year = {2017}
}
@inproceedings{LMFSL20,
	author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},
	booktitle = {Advances in Neural Information Processing Systems},
	editor = {H. Larochelle and M. Ranzato and R. Hadsell and M.F. Balcan and H. Lin},
	pages = {1877--1901},
	publisher = {Curran Associates, Inc.},
	title = {Language Models are Few-Shot Learners},
	volume = {33},
	year = {2020}
}
@inproceedings{QSGD17,
	author = {Alistarh, Dan and Grubic, Demjan and Li, Jerry and Tomioka, Ryota and Vojnovic, Milan},
	booktitle = {Advances in Neural Information Processing Systems},
	editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
	publisher = {Curran Associates, Inc.},
	title = {{QSGD}: Communication-Efficient {SGD} via Gradient Quantization and Encoding},
	volume = {30},
	year = {2017}
}
@INPROCEEDINGS{HorvathNat,
	author = {Horv\'ath, Samuel and Ho, Chen-Yu and Horv\'ath, \v{L}udov\'it and Sahu, Atal Narayan and Canini, Marco and Richt\'arik, Peter},
	title = {Natural Compression for Distributed Deep Learning},
	booktitle = {Proceedings of MSML'22},
	year = {2022},
	month = {Aug}
}
@article{DianaGeneral,
	author = {Samuel Horv\'ath and Dmitry Kovalev and Konstantin Mishchenko and Peter Richt\'arik and Sebastian Stich},
	title = {Stochastic distributed learning with gradient quantization and double-variance reduction},
	journal = {Optimization Methods and Software},
	volume = {38},
	number = {1},
	pages = {91-106},
	year = {2023},
	publisher = {Taylor & Francis}
}
@article{Diana,
	title={Distributed Learning with Compressed Gradient Differences},
	author={Konstantin Mishchenko and Eduard A. Gorbunov and Martin Tak\'ac and Peter Richt\'arik},
	journal={ArXiv},
	year={2019},
	volume={abs/1901.09269}
}

@inproceedings{demidovich2023guide,
 author = {Demidovich, Yury and Malinovsky, Grigory and Sokolov, Igor and Richt{\'a}rik, Peter},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine},
 pages = {23158--23171},
 publisher = {Curran Associates, Inc.},
 title = {A Guide Through the Zoo of Biased SGD},
 volume = {36},
 year = {2023}
}

@article{beznosikov2020biased,
	title={On Biased Compression for Distributed Learning},
	author={Beznosikov,  Aleksandr and Horv{\'a}th,  Samuel and Richt{\'a}rik,  Peter and Safaryan,  Mher},
	journal={arXiv preprint arXiv:2002.12410},
	year={2020}
}

@inproceedings{suresh2017meanestimation,
	author = {Suresh, Ananda Theertha and Yu, Felix X. and Kumar, Sanjiv and McMahan, H. Brendan},
	title = {Distributed Mean Estimation with Limited Communication},
	year = {2017},
	publisher = {JMLR.org},
	booktitle = {Proceedings of the 34th International Conference on Machine Learning - Volume 70},
	pages = {3329–3337},
	numpages = {9},
	location = {Sydney, NSW, Australia},
	series = {ICML'17}
}
@inproceedings{terngrad,
	author = {Wen, Wei and Xu, Cong and Yan, Feng and Wu, Chunpeng and Wang, Yandan and Chen, Yiran and Li, Hai},
	booktitle = {Advances in Neural Information Processing Systems},
	editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
	publisher = {Curran Associates, Inc.},
	title = {{TernGrad}: Ternary Gradients to Reduce Communication in Distributed Deep Learning},
	volume = {30},
	year = {2017}
}
@InProceedings{Alistarh-EF-NIPS2018,
	author    = {Alistarh, Dan and Hoefler, Torsten and Johansson,  Mikael and Khirirat,  Sarit and Konstantinov,  Nikola and Renggli,  C\'{e}dric},
	title     = {The convergence of sparsified gradient methods},
	booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
	year      = {2018},
}
@article{CC01a,
	author = {Chang, Chih-Chung and Lin, Chih-Jen},
	title = {{LIBSVM}: A library for support vector machines},
	journal = {ACM Transactions on Intelligent Systems and Technology},
	volume = {2},
	issue = {3},
	year = {2011},
	pages = {27:1--27:27},
	note =	 {Software available at \url{http://www.csie.ntu.edu.tw/~cjlin/libsvm}}
}

@misc{Condat2023TAMUNADA,
      title={TAMUNA: Doubly Accelerated Distributed Optimization with Local Training, Compression, and Partial Participation}, 
      author={Laurent Condat and Ivan Agarský and Grigory Malinovsky and Peter Richtárik},
      year={2024},
      eprint={2302.09832},
      archivePrefix={arXiv}
}
@article{Grudzien2023ImprovingAF,
	title={Improving Accelerated Federated Learning with Compression and Importance Sampling},
	author={Michal Grudzie\'n and Grigory Malinovsky and Peter Richt\'arik},
	journal={ArXiv},
	year={2023},
	volume={abs/2306.03240}
}

@misc{mayekar2019ratq,
	title={RATQ: A Universal Fixed-Length Quantizer for Stochastic Optimization}, 
	author={Prathamesh Mayekar and Himanshu Tyagi},
	year={2019},
	eprint={1908.08200},
	archivePrefix={arXiv},
	primaryClass={cs.LG}
}
@article{Safaryan_2021,
	doi = {10.1093/imaiai/iaab006},
	url = {https://doi.org/10.1093%2Fimaiai%2Fiaab006},
	year = 2021,
	month = {apr},
	publisher = {Oxford University Press ({OUP})},
	volume = {11},
	number = {2},
	pages = {557--580},
	author = {Mher Safaryan and Egor Shulgin and Peter Richt{\'{a}}rik},
	title = {Uncertainty principle for communication compression in distributed and federated learning and the search for an optimal compressor},
	journal = {Information and Inference: A Journal of the {IMA}}
}
@InProceedings{pmlr-v202-wang23t,
	title = 	 {{C}ocktail{SGD}: Fine-tuning Foundation Models over 500{M}bps Networks},
	author =       {Wang, Jue and Lu, Yucheng and Yuan, Binhang and Chen, Beidi and Liang, Percy and De Sa, Christopher and Re, Christopher and Zhang, Ce},
	booktitle = 	 {Proceedings of the 40th International Conference on Machine Learning},
	pages = 	 {36058--36076},
	year = 	 {2023},
	editor = 	 {Krause, Andreas and Brunskill, Emma and Cho, Kyunghyun and Engelhardt, Barbara and Sabato, Sivan and Scarlett, Jonathan},
	volume = 	 {202},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {23--29 Jul},
	publisher =    {PMLR},
	pdf = 	 {https://proceedings.mlr.press/v202/wang23t/wang23t.pdf},
	url = 	 {https://proceedings.mlr.press/v202/wang23t.html},
	abstract = 	 {Distributed training of foundation models, especially large language models (LLMs), is communication-intensive and so has heavily relied on centralized data centers with fast interconnects. Can we train on slow networks and unlock the potential of decentralized infrastructure for foundation models? In this paper, we propose CocktailSGD, a novel communication-efficient training framework that combines three distinct compression techniques – random sparsification, top-K sparsification, and quantization – to achieve much greater compression than each individual technique alone. We justify the benefit of such a hybrid approach through a theoretical analysis of convergence. Empirically, we show that CocktailSGD achieves up to 117$\times$ compression in fine-tuning LLMs up to 20 billion parameters without hurting convergence. On a 500Mbps network, CocktailSGD only incurs $\sim$1.2$\times$ slowdown compared with data center networks.}
}

@InProceedings{LiAccDCGD,
	title = 	 {Acceleration for Compressed Gradient Descent in Distributed and Federated Optimization},
	author =       {Li, Zhize and Kovalev, Dmitry and Qian, Xun and Richtarik, Peter},
	booktitle = 	 {Proceedings of the 37th International Conference on Machine Learning},
	pages = 	 {5895--5904},
	year = 	 {2020},
	editor = 	 {III, Hal Daumé and Singh, Aarti},
	volume = 	 {119},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {13--18 Jul},
	publisher =    {PMLR}
}


@InProceedings{PAGE2021,
	title = 	 {PAGE: A Simple and Optimal Probabilistic Gradient Estimator for Nonconvex Optimization},
	author =       {Li, Zhize and Bao, Hongyan and Zhang, Xiangliang and Richtarik, Peter},
	booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
	pages = 	 {6286--6295},
	year = 	 {2021},
	editor = 	 {Meila, Marina and Zhang, Tong},
	volume = 	 {139},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {18--24 Jul},
	publisher =    {PMLR},
	pdf = 	 {http://proceedings.mlr.press/v139/li21a/li21a.pdf},
	url = 	 {https://proceedings.mlr.press/v139/li21a.html},
}

@article{Lim20183LCLA,
	title={3LC: Lightweight and Effective Traffic Compression for Distributed Machine Learning},
	author={Hyeontaek Lim and David G. Andersen and Michael Kaminsky},
	journal={ArXiv},
	year={2018},
	volume={abs/1802.07389}
}

@article{Lin2017DeepGC,
	title={Deep Gradient Compression: Reducing the Communication Bandwidth for Distributed Training},
	author={Yujun Lin and Song Han and Huizi Mao and Yu Wang and William J. Dally},
	journal={ArXiv},
	year={2017},
	volume={abs/1712.01887}
}


@InProceedings{pmlr-v70-zhang17e,
	title = 	 {{Z}ip{ML}: Training Linear Models with End-to-End Low Precision, and a Little Bit of Deep Learning},
	author =       {Hantian Zhang and Jerry Li and Kaan Kara and Dan Alistarh and Ji Liu and Ce Zhang},
	booktitle = 	 {Proceedings of the 34th International Conference on Machine Learning},
	pages = 	 {4035--4043},
	year = 	 {2017},
	editor = 	 {Precup, Doina and Teh, Yee Whye},
	volume = 	 {70},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {06--11 Aug},
	publisher =    {PMLR},
	pdf = 	 {http://proceedings.mlr.press/v70/zhang17e/zhang17e.pdf},
	url = 	 {https://proceedings.mlr.press/v70/zhang17e.html},
}

@inproceedings{NEURIPS2018WangAtomo,
	author = {Wang, Hongyi and Sievert, Scott and Liu, Shengchao and Charles, Zachary and Papailiopoulos, Dimitris and Wright, Stephen},
	booktitle = {Advances in Neural Information Processing Systems},
	editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
	pages = {},
	publisher = {Curran Associates, Inc.},
	title = {ATOMO: Communication-efficient Learning via Atomic Sparsification},
	volume = {31},
	year = {2018}
}

@article{demidovich2024methods,
  title={Methods with local steps and random reshuffling for generally smooth non-convex federated optimization},
  author={Demidovich, Yury and Ostroukhov, Petr and Malinovsky, Grigory and Horv{\'a}th, Samuel and Tak{\'a}{\v{c}}, Martin and Richt{\'a}rik, Peter and Gorbunov, Eduard},
  journal={arXiv preprint arXiv:2412.02781},
  year={2024}
}

@article{demidovich2023mast,
  title={MAST: Model-agnostic sparsified training},
  author={Demidovich, Yury and Malinovsky, Grigory and Shulgin, Egor and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2311.16086},
  year={2023}
}

@article{sokolov2024marina,
  title={MARINA-P: Superior Performance in Non-smooth Federated Optimization with Adaptive Stepsizes},
  author={Sokolov, Igor and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2412.17082},
  year={2024}
}

@article{demidovich2024streamlining,
  title={Streamlining in the Riemannian Realm: Efficient Riemannian Optimization with Loopless Variance Reduction},
  author={Demidovich, Yury and Malinovsky, Grigory and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2403.06677},
  year={2024}
}