@article{einstein,
author =       "Albert Einstein",
title =        "{Zur Elektrodynamik bewegter Körper}",
journal =      "Annalen der Physik",
volume =       "322",
number =       "10",
pages =        "891--921",
year =         "1905"}

@book{latexcompanion,
author    = "Michel Goossens and Frank Mittelbach and Alexander Samarin",
title     = "The \LaTeX\ Companion",
year      = "1993",
publisher = "Addison-Wesley"}



@article{stich2019error,
  author  = {Sebastian U. Stich and Sai Praneeth Karimireddy},
  title   = {The Error-Feedback framework: {SGD} with Delayed Gradients},
  journal = {Journal of Machine Learning Research},
  year    = {2020},
  volume  = {21},
  number  = {237},
  pages   = {1-36},
  url     = {http://jmlr.org/papers/v21/19-748.html}
}


@InProceedings{sigma_k,
  author    = {Gorbunov, Eduard and Hanzely, Filip and Richt\'{a}rik, Peter},
  title     = {A Unified Theory of {SGD}: Variance Reduction, Sampling, Quantization and Coordinate Descent},
  booktitle = {The 23rd International Conference on Artificial Intelligence and Statistics},
  year      = {2020},
}


@article{stich2019unified,
  title={Unified optimal analysis of the (stochastic) gradient method},
  author={Stich, Sebastian U},
  journal={arXiv preprint arXiv:1907.04232},
  year={2019}
}

@article{gower2019sgd,
  title={{SGD}: General analysis and improved rates},
  author={Gower, Robert Mansel and Loizou, Nicolas and Qian, Xun and Sailanbayev, Alibek and Shulgin, Egor and Richt{\'a}rik, Peter},
  journal={Proceedings of the 36th International Conference on Machine
Learning, Long Beach, California},
  year={2019}
}

@inproceedings{karimi2016linear,
  title={Linear convergence of gradient and proximal-gradient methods under the Polyak-{\L}ojasiewicz condition},
  author={Karimi, Hamed and Nutini, Julie and Schmidt, Mark},
  booktitle={Joint European Conference on Machine Learning and Knowledge Discovery in Databases},
  pages={795--811},
  year={2016},
  organization={Springer}
}

@article{necoara2019linear,
  title={Linear convergence of first order methods for non-strongly convex optimization},
  author={Necoara, Ion and Nesterov, Yu and Glineur, Francois},
  journal={Mathematical Programming},
  volume={175},
  number={1-2},
  pages={69--107},
  year={2019},
  publisher={Springer}
}

@article{bottou2018optimization,
  title={Optimization methods for large-scale machine learning},
  author={Bottou, L{\'e}on and Curtis, Frank E and Nocedal, Jorge},
  journal={Siam Review},
  volume={60},
  number={2},
  pages={223--311},
  year={2018},
  publisher={SIAM}
}

@inproceedings{stich2018sparsified,
  title={Sparsified {SGD} with memory},
  author={Stich, Sebastian U and Cordonnier, Jean-Baptiste and Jaggi, Martin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4447--4458},
  year={2018}
}

@article{lacoste2012simpler,
  title={A simpler approach to obtaining an $\mathcal{O}(1/t)$ convergence rate for the projected stochastic subgradient method},
  author={Lacoste-Julien, Simon and Schmidt, Mark and Bach, Francis},
  journal={arXiv preprint arXiv:1212.2002},
  year={2012}
}

@article{grimmer2019convergence,
  title={Convergence rates for deterministic and stochastic subgradient methods without {L}ipschitz continuity},
  author={Grimmer, Benjamin},
  journal={SIAM Journal on Optimization},
  volume={29},
  number={2},
  pages={1350--1365},
  year={2019},
  publisher={SIAM}
}

@article{karimireddy2019scaffold,
  title={SCAFFOLD: Stochastic controlled averaging for on-device federated learning},
  author={Karimireddy, Sai Praneeth and Kale, Satyen and Mohri, Mehryar and Reddi, Sashank J and Stich, Sebastian U and Suresh, Ananda Theertha},
  journal={arXiv preprint arXiv:1910.06378},
  year={2019}
}

@inproceedings{wangni2018gradient,
  title={Gradient sparsification for communication-efficient distributed optimization},
  author={Wangni, Jianqiao and Wang, Jialei and Liu, Ji and Zhang, Tong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1299--1309},
  year={2018}
}

@article{beznosikov2020biased,
  title={On Biased Compression for Distributed Learning},
  author={Beznosikov, Aleksandr and Horv{\'a}th, Samuel and Richt{\'a}rik, Peter and Safaryan, Mher},
  journal={arXiv preprint arXiv:2002.12410},
  year={2020}
}

@article{eichner2019semi,
  title={Semi-cyclic stochastic gradient descent},
  author={Eichner, Hubert and Koren, Tomer and McMahan, H Brendan and Srebro, Nathan and Talwar, Kunal},
  journal={arXiv preprint arXiv:1904.10120},
  year={2019}
}

@article{horvath2018nonconvex,
  title={Nonconvex variance reduced optimization with arbitrary sampling},
  author={Horv{\'a}th, Samuel and Richt{\'a}rik, Peter},
  journal={Proceedings of the 36th International Conference on Machine Learning},
  year={2019}
}

@inproceedings{qu2015quartz,
  title={Quartz: Randomized dual coordinate ascent with arbitrary sampling},
  author={Qu, Zheng and Richt{\'a}rik, Peter and Zhang, Tong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={865--873},
  year={2015}
}

@article{chambolle2018stochastic,
  title={Stochastic primal-dual hybrid gradient algorithm with arbitrary sampling and imaging applications},
  author={Chambolle, Antonin and Ehrhardt, Matthias J and Richt{\'a}rik, Peter and Schonlieb, Carola-Bibiane},
  journal={SIAM Journal on Optimization},
  volume={28},
  number={4},
  pages={2783--2808},
  year={2018},
  publisher={SIAM}
}

@article{PCDM,
  title={Parallel coordinate descent methods for big data optimization},
  author={Richt{\'a}rik, Peter and Tak{\'a}{\v{c}}, Martin},
  journal={Mathematical Programming},
  volume={156},
  number={1-2},
  pages={433--484},
  year={2016},
  publisher={Springer}
}

@article{horvath2019stochastic,
  title={Stochastic distributed learning with gradient quantization and variance reduction},
  author={Horv{\'a}th, Samuel and Kovalev, Dmitry and Mishchenko, Konstantin and Stich, Sebastian and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:1904.05115},
  year={2019}
}

@article{mishchenko2019distributed,
  title={Distributed Learning with Compressed Gradient Differences},
  author={Mishchenko, Konstantin and Gorbunov, Eduard and Tak{\'a}{\v{c}}, Martin and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:1901.09269},
  year={2019}
}

@article{kingma2014adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik P and Ba, Jimmy},
  journal={	Published as a conference paper at the 3rd International Conference for Learning Representations, San Diego},
  year={2015}
}

@article{reddi2019convergence,
  title={On the convergence of {A}dam and beyond},
  author={Reddi, Sashank J and Kale, Satyen and Kumar, Sanjiv},
  journal={ICLR 2018 - International Conference on Learning Representations},
  year={2018}
}

@article{li2020acceleration,
  title={Acceleration for Compressed Gradient Descent in Distributed and Federated Optimization},
  author={Li, Zhize and Kovalev, Dmitry and Qian, Xun and Richt{\'a}rik, Peter},
  journal={Proceedings of the 37th International Conference on Machine Learning},
  year={2020}
}



@techreport{cordonnier2018convex,
  title={Convex optimization using sparsified stochastic gradient descent with memory},
  author={Cordonnier, Jean-Baptiste},
  year={2018}
}


@article{koloskova2019decentralized,
  title={Decentralized stochastic optimization and gossip algorithms with compressed communication},
  author={Koloskova, Anastasia and Stich, Sebastian U and Jaggi, Martin},
  journal={arXiv preprint arXiv:1902.00340},
  year={2019}
}

@inproceedings{alistarh2018convergence,
  title={The convergence of sparsified gradient methods},
  author={Alistarh, Dan and Hoefler, Torsten and Johansson, Mikael and Konstantinov, Nikola and Khirirat, Sarit and Renggli, C{\'e}dric},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5973--5983},
  year={2018}
}

@inproceedings{alistarh2017qsgd,
     author = {Alistarh, Dan and Grubic, Demjan and Li, Jerry and Tomioka, Ryota and Vojnovic, Milan},
     booktitle = {Advances in Neural Information Processing Systems},
     publisher = {Curran Associates, Inc.},
     title = {{QSGD}: Communication-Efficient {SGD} via Gradient Quantization and Encoding},
     url = {https://proceedings.neurips.cc/paper/2017/file/6c340f25839e6acdc73414517203f5f0-Paper.pdf},
     volume = {30},
     year = {2017}
}

@InProceedings{mcmahan2017communication, 
    title = {{Communication-Efficient Learning of Deep Networks from Decentralized Data}}, 
    author = {Brendan McMahan and Eider Moore and Daniel Ramage and Seth Hampson and Blaise Aguera y Arcas}, 
    booktitle = {Proceedings of the 20th International Conference on Artificial Intelligence and Statistics}, 
    pages = {1273--1282}, 
    year = {2017}, 
    volume = {54}, 
    series = {Proceedings of Machine Learning Research},
    month = {20--22 Apr}, 
    pdf = {http://proceedings.mlr.press/v54/mcmahan17a/mcmahan17a.pdf}, 
    url = {http://proceedings.mlr.press/v54/mcmahan17a.html}
}

@article{konecny2017federated,
      title={Federated Learning: Strategies for Improving Communication Efficiency}, 
      author={Jakub Konečný and H. Brendan McMahan and Felix X. Yu and Peter Richt{\'a}rik and Ananda Theertha Suresh and Dave Bacon},
      year={2016},
      journal={NIPS Private Multi-Party Machine Learning Workshop}
}

@article{kairouz2021advances,
    url = {http://dx.doi.org/10.1561/2200000083},
    year = {2021},
    volume = {14},
    journal = {Foundations and Trends® in Machine Learning},
    title = {Advances and Open Problems in Federated Learning},
    doi = {10.1561/2200000083},
    issn = {1935-8237},
    number = {1},
    author = {Edited by: Peter Kairouz and H. Brendan McMahan}
}

@inproceedings{khaled2020tighter,
  title={Tighter theory for local {SGD} on identical and heterogeneous data},
  author={Khaled, Ahmed and Mishchenko, Konstantin and Richt{\'a}rik, Peter},
  booktitle={The 23rd International Conference on Artificial Intelligence and Statistics (AISTATS 2020)},
  year={2020}
}

@article{lin2018don,
  title={Don't Use Large Mini-Batches, Use Local {SGD}},
  author={Lin, Tao and Stich, Sebastian U and Patel, Kumar Kshitij and Jaggi, Martin},
  journal={arXiv preprint arXiv:1808.07217},
  year={2018}
}

@article{stich2018local,
  title={Local {SGD} converges fast and communicates little},
  author={Stich, Sebastian U},
  journal={ICLR 2019 - International Conference on Learning Representations},
  year={2019}
}


@article{woodworth2020local,
  title={Is Local {SGD} Better than Minibatch {SGD}?},
  author={Woodworth, Blake and Patel, Kumar Kshitij and Stich, Sebastian U and Dai, Zhen and Bullins, Brian and McMahan, H Brendan and Shamir, Ohad and Srebro, Nathan},
  journal={arXiv preprint arXiv:2002.07839},
  year={2020}
}

@article{goodall1951television,
  title={Television by pulse code modulation},
  author={Goodall, WM},
  journal={Bell System Technical Journal},
  volume={30},
  number={1},
  pages={33--49},
  year={1951},
  publisher={Wiley Online Library}
}

@article{roberts1962picture,
  title={Picture coding using pseudo-random noise},
  author={Roberts, Lawrence},
  journal={IRE Transactions on Information Theory},
  volume={8},
  number={2},
  pages={145--154},
  year={1962},
  publisher={IEEE}
}

@inproceedings{wen2017terngrad,
  title={Terngrad: Ternary gradients to reduce communication in distributed deep learning},
  author={Wen, Wei and Xu, Cong and Yan, Feng and Wu, Chunpeng and Wang, Yandan and Chen, Yiran and Li, Hai},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1509--1519},
  year={2017}
}

@inproceedings{zhang2017zipml,
  title={ZIPML: Training linear models with end-to-end low precision, and a little bit of deep learning},
  author={Zhang, Hantian and Li, Jerry and Kara, Kaan and Alistarh, Dan and Liu, Ji and Zhang, Ce},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={4035--4043},
  year={2017},
  organization={JMLR. org}
}

@article{horvath2019natural,
  title={Natural Compression for Distributed Deep Learning},
  author={Horv{\'a}th, Samuel and Ho, Chen-Yu and Horv{\'a}th, {\v{L}}udovit and Sahu, Atal Narayan and Canini, Marco and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:1905.10988},
  year={2019}
}

@article{ramezani2019nuqsgd,
  title={{NUQSGD}: Improved Communication Efficiency for Data-parallel {SGD} via Nonuniform Quantization},
  author={Ramezani-Kebrya, Ali and Faghri, Fartash and Roy, Daniel M},
  journal={arXiv preprint arXiv:1908.06077},
  year={2019}
}

@inproceedings{vogels2019powersgd,
  title={Power{SGD}: Practical low-rank gradient compression for distributed optimization},
  author={Vogels, Thijs and Karimireddy, Sai Praneeth and Jaggi, Martin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={14236--14245},
  year={2019}
}

@inproceedings{yu2019double,
  title={Double Quantization for Communication-Efficient Distributed Optimization},
  author={Yu, Yue and Wu, Jiaxiang and Huang, Longbo},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4440--4451},
  year={2019}
}

@inproceedings{basu2019qsparse,
  title={Qsparse-local-{SGD}: Distributed {SGD} with Quantization, Sparsification and Local Computations},
  author={Basu, Debraj and Data, Deepesh and Karakus, Can and Diggavi, Suhas},
  booktitle={Advances in Neural Information Processing Systems},
  pages={14668--14679},
  year={2019}
}

@article{karimireddy2019error,
  title={Error feedback fixes Sign{SGD} and other gradient compression schemes},
  author={Karimireddy, Sai Praneeth and Rebjock, Quentin and Stich, Sebastian U and Jaggi, Martin},
  journal={arXiv preprint arXiv:1901.09847},
  year={2019}
}

@article{tang2019doublesqueeze,
  title={Doublesqueeze: Parallel stochastic gradient descent with double-pass error-compensated compression},
  author={Tang, Hanlin and Lian, Xiangru and Zhang, Tong and Liu, Ji},
  journal={arXiv preprint arXiv:1905.05957},
  year={2019}
}


@article{konevcny2018randomized,
  title={Randomized Distributed Mean Estimation: Accuracy vs. Communication},
  author={Kone{\v{c}}n{\'y}, Jakub and Richt{\'a}rik, Peter},
  journal={Frontiers in Applied Mathematics and Statistics},
  volume={4},
  pages={62},
  year={2018},
  publisher={Frontiers}
}

@article{mishchenko201999,
  title={99\% of Parallel Optimization is Inevitably a Waste of Time},
  author={Mishchenko, Konstantin and Hanzely, Filip and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:1901.09437},
  year={2019}
}

@article{vaswani2018fast,
  title={Fast and faster convergence of {SGD} for over-parameterized models and an accelerated perceptron},
  author={Vaswani, Sharan and Bach, Francis and Schmidt, Mark},
  journal={Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics (AISTATS) 2019, Naha, Okinawa, Japan},
  year={2019}
}

@inproceedings{seide20141,
  title={1-bit stochastic gradient descent and its application to data-parallel distributed training of speech DNNs},
  author={Seide, Frank and Fu, Hao and Droppo, Jasha and Li, Gang and Yu, Dong},
  booktitle={Fifteenth Annual Conference of the International Speech Communication Association},
  year={2014}
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
  pages={770--778},
  year={2016}
}

@article{simonyan2014very,
  title={Very deep convolutional networks for large-scale image recognition},
  author={Simonyan, Karen and Zisserman, Andrew},
  journal={ICLR 2015 - International Conference on Learning Representations},
  year={2015}
}

@article{dutta2019discrepancy,
  title={On the Discrepancy between the Theoretical Analysis and Practical Implementations of Compressed Communication for Distributed Deep Learning},
  author={Dutta, Aritra and Bergou, El Houcine and Abdelmoniem, Ahmed M and Ho, Chen-Yu and Sahu, Atal Narayan and Canini, Marco and Kalnis, Panos},
  journal={arXiv preprint arXiv:1911.08250},
  year={2019}
}

@article{gadat2018stochastic,
  title={Stochastic heavy ball},
  author={Gadat, S{\'e}bastien and Panloup, Fabien and Saadane, Sofiane and others},
  journal={Electronic Journal of Statistics},
  volume={12},
  number={1},
  pages={461--529},
  year={2018},
  publisher={The Institute of Mathematical Statistics and the Bernoulli Society}
}

@article{yang2016unified,
  title={Unified convergence analysis of stochastic momentum methods for convex and non-convex optimization},
  author={Yang, Tianbao and Lin, Qihang and Li, Zhe},
  journal={arXiv preprint arXiv:1604.03257},
  year={2016}
}

@article{loizou2017momentum,
  title={Momentum and stochastic momentum for stochastic gradient, {N}ewton, proximal point and subspace descent methods},
  author={Loizou, Nicolas and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:1712.09677},
  year={2017}
}

@article{lim20183lc,
  title={{3LC}: {L}ightweight and Effective Traffic Compression for Distributed Machine Learning},
  author={Lim, Hyeontaek and Andersen, David G and Kaminsky, Michael},
  journal={arXiv preprint arXiv:1802.07389},
  year={2018}
}

@article{aji2017sparse,
  title={Sparse communication for distributed gradient descent},
  author={Aji, Alham Fikri and Heafield, Kenneth},
  journal={Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing},
  year={2017}
}

@inproceedings{hsieh2017gaia,
  title={Gaia: Geo-distributed machine learning approaching {LAN} speeds},
  author={Hsieh, Kevin and Harlap, Aaron and Vijaykumar, Nandita and Konomis, Dimitris and Ganger, Gregory R and Gibbons, Phillip B and Mutlu, Onur},
  booktitle={14th  Symposium on Networked Systems Design and Implementation},
  pages={629--647},
  year={2017}
}

@inproceedings{li2014scaling,
  title={Scaling distributed machine learning with the parameter server},
  author={Li, Mu and Andersen, David G and Park, Jun Woo and Smola, Alexander J and Ahmed, Amr and Josifovski, Vanja and Long, James and Shekita, Eugene J and Su, Bor-Yiing},
  booktitle={11th $\{$USENIX$\}$ Symposium on Operating Systems Design and Implementation ($\{$OSDI$\}$ 14)},
  pages={583--598},
  year={2014}
}

@article{lin2017deep,
  title={Deep gradient compression: Reducing the communication bandwidth for distributed training},
  author={Lin, Yujun and Han, Song and Mao, Huizi and Wang, Yu and Dally, William J},
  journal={ICLR 2018 - International Conference on Learning Representations},
  year={2018}
}

@inproceedings{wei2015managed,
  title={Managed communication and consistency for fast data-parallel iterative analytics},
  author={Wei, Jinliang and Dai, Wei and Qiao, Aurick and Ho, Qirong and Cui, Henggang and Ganger, Gregory R and Gibbons, Phillip B and Gibson, Garth A and Xing, Eric P},
  booktitle={Proceedings of the Sixth ACM Symposium on Cloud Computing},
  pages={381--394},
  year={2015}
}

@article{khaled2019gradient,
      title={Gradient Descent with Compressed Iterates}, 
      author={Ahmed Khaled and Peter Richt{\'a}rik},
      year={2019},
      journal={NeurIPS 2019 Workshop on Federated Learning for Data Privacy and Confidentiality},
      eprint={1909.04716},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@inproceedings{horvath2021better,
    title={A Better Alternative to Error Feedback for Communication-Efficient Distributed Learning},
    author={Samuel Horv{\'a}th and Peter Richt{\'a}rik},
    booktitle={International Conference on Learning Representations},
    year={2021},
    url={https://openreview.net/forum?id=vYVI1CHPaQg}
}

@article{goyal2018accurate,
      title={Accurate, Large Minibatch {SGD}: Training ImageNet in 1 Hour}, 
      author={Priya Goyal and Piotr Dollár and Ross Girshick and Pieter Noordhuis and Lukasz Wesolowski and Aapo Kyrola and Andrew Tulloch and Yangqing Jia and Kaiming He},
      year={2018},
      journal={arXiv preprint arXiv:1706.02677},
      eprint={1706.02677},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

@article{xu2020compressed,
	year = {2020},
	title = {Compressed Communication for Distributed Deep Learning: Survey and Quantitative Evaluation},
	author = {Xu, Hang and Ho, Chen-Yu and Abdelmoniem, Ahmed M. and Dutta, Aritra and Bergou, El Houcine and Karatsenidis, Konstantinos and Canini, Marco and Kalnis, Panos},
	url = {http://hdl.handle.net/10754/662495},
	journal = {Technical report},
	abstract = {Powerful computer clusters are used nowadays to train complex deep neural networks (DNN) on large datasets. Distributed training workloads increasingly become communication bound. For this reason, many lossy compression techniques have been proposed to reduce the volume of transferred data. Unfortunately, it is difficult to argue about the behavior of compression methods, because existing work relies on inconsistent evaluation testbeds and largely ignores the performance impact of practical system configurations.In this paper, we present a comprehensive survey of the most influential compressed communication methods for DNN training, together with an intuitive classification (i.e., quantization, sparsification, hybrid and low-rank). We also propose a unified framework and API that allows for consistent and easy implementation of compressed communication on popular machine learning toolkits. We instantiate our API on TensorFlow and PyTorch, and implement 16 such methods. Finally, we present a thorough quantitative evaluation with a variety of DNNs (convolutional and recurrent), datasets and system configurations. We show that the DNN architecture affects the relative performance among methods. Interestingly, depending on the underlying communication library and computational cost of compression/decompression, we demonstrate that some methods may be impractical.}
}

@article{safaryan2021uncertainty,
    author = {Safaryan, Mher and Shulgin, Egor and Richt{\'a}rik, Peter},
    title = "{Uncertainty principle for communication compression in distributed and federated learning and the search for an optimal compressor}",
    journal = {Information and Inference: A Journal of the IMA},
    year = {2021},
    month = {04},
    issn = {2049-8772},
    doi = {10.1093/imaiai/iaab006},
    url = {https://doi.org/10.1093/imaiai/iaab006},
    note = {iaab006},
    eprint = {https://academic.oup.com/imaiai/advance-article-pdf/doi/10.1093/imaiai/iaab006/37022980/iaab006.pdf}
}

@article{albasyoni2020optimal,
      title={Optimal Gradient Compression for Distributed and Federated Learning}, 
      author={Alyazeed Albasyoni and Mher Safaryan and Laurent Condat and Peter Richt{\'a}rik},
      year={2020},
      journal={arXiv preprint arXiv:2010.03246},
      eprint={2010.03246},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@InProceedings{mishchenko2021intsgd,
      title={Int{SGD}: Floatless Compression of Stochastic Gradients}, 
      author={Konstantin Mishchenko and Bokun Wang and Dmitry Kovalev and Peter Richt{\'a}rik},
      year={2022},
      booktitle = {International Conference on Learning Representations},
      journal={arXiv preprint arXiv:2102.08374},
      eprint={2102.08374},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}


@InProceedings{gandikota2021vqsgd,
  title = 	 {vq{SGD}: Vector Quantized Stochastic Gradient Descent },
  author =       {Gandikota, Venkata and Kane, Daniel and Kumar Maity, Raj and Mazumdar, Arya},
  booktitle = 	 {Proceedings of The 24th International Conference on Artificial Intelligence and Statistics},
  pages = 	 {2197--2205},
  year = 	 {2021},
  volume = 	 {130},
  series = 	 {Proceedings of Machine Learning Research},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v130/gandikota21a/gandikota21a.pdf},
  url = 	 {http://proceedings.mlr.press/v130/gandikota21a.html},
  abstract = 	 { In this work, we present a family of vector quantization schemes vqSGD (Vector-Quantized Stochastic Gradient Descent) that provide an asymptotic reduction in the communication cost with convergence guarantees in first-order distributed optimization. In the process we derive the following fundamental information theoretic fact: $\Theta(\frac{d}{R^2})$ bits are necessary and sufficient (up to an additive $O(\log d)$ term) to describe an unbiased estimator $\hat{g}(g)$ for any $g$ in the $d$-dimensional unit sphere, under the constraint that $\|\hat{g}(g)\|_2\le R$ almost surely. In particular, we consider a randomized scheme based on the convex hull of a point set, that returns an unbiased estimator of a $d$-dimensional gradient vector with almost surely bounded norm. We provide multiple efficient instances of our scheme, that are near optimal, and require only $o(d)$ bits of communication at the expense of tolerable increase in error. The instances of our quantization scheme are obtained using the properties of binary error-correcting codes and provide a smooth tradeoff between the communication and the estimation error of quantization. Furthermore, we show that vqSGD also offers some automatic privacy guarantees. }
}

@inproceedings{wang2018atomo,
     author = {Wang, Hongyi and Sievert, Scott and Liu, Shengchao and Charles, Zachary and Papailiopoulos, Dimitris and Wright, Stephen},
     booktitle = {Advances in Neural Information Processing Systems},
     pages = {},
     publisher = {Curran Associates, Inc.},
     title = {ATOMO: Communication-efficient Learning via Atomic Sparsification},
     url = {https://proceedings.neurips.cc/paper/2018/file/33b3214d792caf311e1f00fd22b392c5-Paper.pdf},
     volume = {31},
     year = {2018}
}

@inproceedings{vogels2002powergossip,
     author = {Vogels, Thijs and Karimireddy, Sai Praneeth and Jaggi, Martin},
     booktitle = {Advances in Neural Information Processing Systems},
     pages = {14171--14181},
     publisher = {Curran Associates, Inc.},
     title = {Practical Low-Rank Communication Compression in Decentralized Deep Learning},
     url = {https://proceedings.neurips.cc/paper/2020/file/a376802c0811f1b9088828288eb0d3f0-Paper.pdf},
     volume = {33},
     year = {2020}
}

@inproceedings{bernstein2018signsgd, 
    title = {sign{SGD}: compressed optimisation for non-convex problems}, author = {Bernstein, Jeremy and Wang, Yu-Xiang and Azizzadenesheli, Kamyar and Anandkumar, Animashree},
    booktitle = {International Conference on Machine Learning}, 
    year = {2018} 
}

@inproceedings{bernstein2019signsgd, 
    title = {sign{SGD} with majority vote is communication efficient and fault tolerant}, 
    author = {Bernstein, Jeremy and Wang, Yu-Xiang and Azizzadenesheli, Kamyar and Anandkumar, Animashree}, 
    booktitle = {International Conference on Learning Representations}, year = {2019} 
}

@inproceedings{safaryan2021stochastic,
  title={Stochastic sign descent methods: New algorithms and better theory},
  author={Safaryan, Mher and Richt{\'a}rik, Peter},
  booktitle={International Conference on Machine Learning},
  pages={9224--9234},
  year={2021},
  organization={PMLR}
}

@article{khirirat2018distributed,
      title={Distributed learning with compressed gradients}, 
      author={Sarit Khirirat and Hamid Reza Feyzmahdavian and Mikael Johansson},
      year={2018},
      journal={arXiv preprint arXiv:1806.06573},
      eprint={1806.06573},
      archivePrefix={arXiv},
      primaryClass={math.OC}
}

@article{nesterov1983method,
      title={A method of solving a convex programming problem with convergence rate $O(1/k^2)$}, 
      author={Yurii Nesterov},
      year={1983},
      journal={Doklady Akademii Nauk USSR},
      volume={269},
      number={3},
      pages={543--547},
}

@inproceedings{gorbunov2020linearly,
     author = {Gorbunov, Eduard and Kovalev, Dmitry and Makarenko, Dmitry and Richt{\'a}rik, Peter},
     booktitle = {Advances in Neural Information Processing Systems},
     pages = {20889--20900},
     publisher = {Curran Associates, Inc.},
     title = {Linearly Converging Error Compensated {SGD}},
     url = {https://proceedings.neurips.cc/paper/2020/file/ef9280fbc5317f17d480e4d4f61b3751-Paper.pdf},
     volume = {33},
     year = {2020}
}

@article{safaryan2021smoothness,
  title={Smoothness matrices beat smoothness constants: better communication compression techniques for distributed optimization},
  author={Safaryan, Mher and Hanzely, Filip and Richt{\'a}rik, Peter},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{chraibi2019distributed,
      title={Distributed Fixed Point Methods with Compressed Iterates}, 
      author={Sélim Chraibi and Ahmed Khaled and Dmitry Kovalev and Peter Richt{\'a}rik and Adil Salim and Martin Takáč},
      year={2019},
      journal={arXiv preprint arXiv:2102.07245},
      eprint={1912.09925},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}


@inproceedings{reisizadeh2020fedpaq,
      title =    {FedPAQ: A Communication-Efficient Federated Learning Method with Periodic Averaging and Quantization},
      author =   {Reisizadeh, Amirhossein and Mokhtari, Aryan and Hassani, Hamed and Jadbabaie, Ali and Pedarsani, Ramtin},
      booktitle = 	 {Proceedings of the Twenty Third International Conference on Artificial Intelligence and Statistics},
      pages = 	 {2021--2031},
      year = 	 {2020},
      volume = 	 {108},
      series = 	 {Proceedings of Machine Learning Research},
      month = 	 {26--28 Aug},
      publisher =    {PMLR},
      pdf = 	 {http://proceedings.mlr.press/v108/reisizadeh20a/reisizadeh20a.pdf},
      url = 	 {http://proceedings.mlr.press/v108/reisizadeh20a.html},
      abstract = 	 {Federated learning is a distributed framework according to which  a model is trained over a set of devices, while keeping data localized. This framework  faces several systems-oriented challenges which include (i) communication bottleneck since a large number of devices upload their local updates to a parameter server, and (ii) scalability as the federated network consists of millions of devices. Due to these systems challenges as well as issues related to statistical heterogeneity of data and privacy concerns, designing a provably efficient federated learning method is of significant importance yet it remains challenging. In this paper, we present FedPAQ, a communication-efficient Federated Learning method with Periodic Averaging and Quantization. FedPAQ relies on three key features: (1) periodic averaging where models are updated locally at devices and only periodically averaged at the server; (2) partial device participation where only a fraction of devices participate in each round of the training; and (3) quantized message-passing where the edge nodes quantize their updates before uploading to the parameter server. These features address the communications and scalability challenges in federated learning. We also show that FedPAQ achieves near-optimal theoretical guarantees for strongly convex and non-convex loss functions and empirically demonstrate the communication-computation tradeoff provided by our method.}
    }

@inproceedings{hanzely2018sega,
     author = {Hanzely, Filip and Mishchenko, Konstantin and Richt{\'a}rik, Peter},
     booktitle = {Advances in Neural Information Processing Systems},
     pages = {},
     publisher = {Curran Associates, Inc.},
     title = {SEGA: Variance Reduction via Gradient Sketching},
     url = {https://proceedings.neurips.cc/paper/2018/file/fc2c7c47b918d0c2d792a719dfb602ef-Paper.pdf},
     volume = {31},
     year = {2018}
}

@inproceedings{luo2018phub,
  author    = {Liang Luo and
               Jacob Nelson and
               Luis Ceze and
               Amar Phanishayee and
               Arvind Krishnamurthy},
  title     = {Parameter Hub: a Rack-Scale Parameter Server for Distributed Deep
               Neural Network Training},
  booktitle = {Proceedings of the {ACM} Symposium on Cloud Computing, SoCC 2018},
  pages     = {41--54},
  year      = {2018},
  url       = {https://doi.org/10.1145/3267809.3267840},
  doi       = {10.1145/3267809.3267840},
  timestamp = {Wed, 21 Nov 2018 12:44:07 +0100},
  biburl    = {https://dblp.org/rec/conf/cloud/LuoNCPK18.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{peng2019generic,
  author    = {Yanghua Peng and
               Yibo Zhu and
               Yangrui Chen and
               Yixin Bao and
               Bairen Yi and
               Chang Lan and
               Chuan Wu and
               Chuanxiong Guo},
  title     = {A generic communication scheduler for distributed {DNN} training acceleration},
  booktitle = {Proceedings of the 27th {ACM} Symposium on Operating Systems Principles,
               {SOSP} 2019},
  pages     = {16--29},
  year      = {2019},
  url       = {https://doi.org/10.1145/3341301.3359642},
  doi       = {10.1145/3341301.3359642},
  timestamp = {Thu, 18 Feb 2021 10:45:51 +0100},
  biburl    = {https://dblp.org/rec/conf/sosp/PengZCBYLWG19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{sapio2021switchml,
  author    = {Amedeo Sapio and
               Marco Canini and
               Chen{-}Yu Ho and
               Jacob Nelson and
               Panos Kalnis and
               Changhoon Kim and
               Arvind Krishnamurthy and
               Masoud Moshref and
               Dan R. K. Ports and
               Peter Richt{\'{a}}rik},
  title     = {Scaling Distributed Machine Learning with In-Network Aggregation},
  booktitle = {18th {USENIX} Symposium on Networked Systems Design and Implementation,
               {NSDI} 2021, April 12-14},
  pages     = {785--808},
  publisher = {{USENIX} Association},
  year      = {2021},
  url       = {https://www.usenix.org/conference/nsdi21/presentation/sapio},
  timestamp = {Tue, 04 May 2021 13:05:21 +0200},
  biburl    = {https://dblp.org/rec/conf/nsdi/SapioC0NKKKMPR21.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{scikit-learn,
 title={Scikit-learn: Machine Learning in {P}ython},
 author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
 journal={Journal of Machine Learning Research},
 volume={12},
 pages={2825--2830},
 year={2011}
}

@article{chang2011libsvm,
 author = {Chang, Chih-Chung and Lin, Chih-Jen},
 title = {{LIBSVM}: A library for support vector machines},
 journal = {ACM Transactions on Intelligent Systems and Technology},
 volume = {2},
 issue = {3},
 year = {2011},
 pages = {27:1--27:27},
 note =	 {Software available at \url{http://www.csie.ntu.edu.tw/~cjlin/libsvm}}
}

@InProceedings{kovalev2020loopless, 
    title = {Don’t Jump Through Hoops and Remove Those Loops: SVRG and Katyusha are Better Without the Outer Loop}, 
    author = {Kovalev, Dmitry and Horv{\'a}th, Samuel and Richt{\'a}rik, Peter}, 
    booktitle = {Proceedings of the 31st International Conference on Algorithmic Learning Theory}, 
    pages = {451--467}, 
    year = {2020}, 
    volume = {117}, 
    series = {Proceedings of Machine Learning Research}, 
    publisher = {PMLR}, 
    pdf = {http://proceedings.mlr.press/v117/kovalev20a/kovalev20a.pdf},
    url = {http://proceedings.mlr.press/v117/kovalev20a.html}
}

@article{safaryan2021fednl,
  title={FedNL: Making Newton-Type Methods Applicable to Federated Learning},
  author={Safaryan, Mher and Islamov, Rustem and Qian, Xun and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2106.02969},
  year={2021}
}

@article{condat2022ef,
  title={{EF-BV}: A Unified Theory of Error Feedback and Variance Reduction Mechanisms for Biased and Unbiased Compression in Distributed Optimization},
  author={Condat, Laurent and Yi, Kai and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2205.04180},
  year={2022}
}

@article{condat2021murana,
  title={{MURANA}: A generic framework for stochastic variance-reduced optimization},
  author={Condat, Laurent and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2106.03056},
  year={2021}
}