@article{ghosh2012universally,
  title={Universally utility-maximizing privacy mechanisms},
  author={Ghosh, Arpita and Roughgarden, Tim and Sundararajan, Mukund},
  journal={SIAM Journal on Computing},
  volume={41},
  number={6},
  pages={1673--1693},
  year={2012},
  publisher={SIAM}
}

@inproceedings{cutkosky2019momentum,
  title={Momentum-based variance reduction in non-convex SGD},
  author={Cutkosky, Ashok and Orabona, Francesco},
  booktitle={Advances in Neural Information Processing Systems},
  pages={15236--15245},
  year={2019}
}
@article{pu2020distributed,
  title={Distributed stochastic gradient tracking methods},
  author={Pu, Shi and Nedi{\'c}, Angelia},
  journal={Mathematical Programming},
  pages={1--49},
  year={2020},
  publisher={Springer}
}
@inproceedings{reisizadeh2020fedpaq,
  title={Fedpaq: A communication-efficient federated learning method with periodic averaging and quantization},
  author={Reisizadeh, Amirhossein and Mokhtari, Aryan and Hassani, Hamed and Jadbabaie, Ali and Pedarsani, Ramtin},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2021--2031},
  year={2020}
}

@article{liu2020optimal,
  title={An Optimal Hybrid Variance-Reduced Algorithm for Stochastic Composite Nonconvex Optimization},
  author={Liu, Deyi and Nguyen, Lam M and Tran-Dinh, Quoc},
  journal={arXiv preprint arXiv:2008.09055},
  year={2020}
}

@article{li2018federated,
  title={Federated optimization in heterogeneous networks},
  author={Li, Tian and Sahu, Anit Kumar and Zaheer, Manzil and Sanjabi, Maziar and Talwalkar, Ameet and Smith, Virginia},
  journal={arXiv preprint arXiv:1812.06127},
  year={2018}
}

@inproceedings{basu2019qsparse,
  title={Qsparse-local-SGD: Distributed SGD with quantization, sparsification and local computations},
  author={Basu, Debraj and Data, Deepesh and Karakus, Can and Diggavi, Suhas},
  booktitle={Advances in Neural Information Processing Systems},
  pages={14695--14706},
  year={2019}
}

@inproceedings{yin2018gradient,
  title={Gradient diversity: a key ingredient for scalable distributed learning},
  author={Yin, Dong and Pananjady, Ashwin and Lam, Max and Papailiopoulos, Dimitris and Ramchandran, Kannan and Bartlett, Peter},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1998--2007},
  year={2018},
  organization={PMLR}
}

@inproceedings{vaswani2019fast,
  title={Fast and faster convergence of SGD for over-parameterized models and an accelerated perceptron},
  author={Vaswani, Sharan and Bach, Francis and Schmidt, Mark},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={1195--1204},
  year={2019},
  organization={PMLR}
}

@article{schmidt2013fast,
  title={Fast convergence of stochastic gradient descent under a strong growth condition},
  author={Schmidt, Mark and Roux, Nicolas Le},
  journal={arXiv preprint arXiv:1308.6370},
  year={2013}
}

@article{karimireddy2020mime,
  title={Mime: Mimicking Centralized Stochastic Algorithms in Federated Learning},
  author={Karimireddy, Sai Praneeth and Jaggi, Martin and Kale, Satyen and Mohri, Mehryar and Reddi, Sashank J and Stich, Sebastian U and Suresh, Ananda Theertha},
  journal={arXiv preprint arXiv:2008.03606},
  year={2020}
}

@article{haddadpour2020federated,
  title={Federated learning with compression: Unified analysis and sharp guarantees},
  author={Haddadpour, Farzin and Kamani, Mohammad Mahdi and Mokhtari, Aryan and Mahdavi, Mehrdad},
  journal={arXiv preprint arXiv:2007.01154},
  year={2021}
}

@article{karimireddy2019scaffold,
  title={Scaffold: Stochastic controlled averaging for federated learning},
  author={Karimireddy, Sai Praneeth and Kale, Satyen and Mohri, Mehryar and Reddi, Sashank J and Stich, Sebastian U and Suresh, Ananda Theertha},
  journal={arXiv preprint arXiv:1910.06378},
  year={2019}
}

@article{liang2019variance,
  title={Variance reduced local SGD with lower communication complexity},
  author={Liang, Xianfeng and Shen, Shuheng and Liu, Jingchang and Pan, Zhen and Chen, Enhong and Cheng, Yifei},
  journal={arXiv preprint arXiv:1912.12844},
  year={2019}
}
@article{chen2021communication,
  title={Communication-Efficient Variance-Reduced Decentralized Stochastic Optimization over Time-Varying Directed Graphs},
  author={Chen, Yiyue and Hashemi, Abolfazl and Vikalo, Haris},
  journal={IEEE Transactions on Automatic Control},
  year={2021}
}
@article{hashemi2020delicoco,
  title={On the Benefits of Multiple Gossip Steps in Communication-Constrained Decentralized Federated Learning},
  author={Hashemi, Abolfazl and Acharya, Anish and Das, Rudrajit and Vikalo, Haris and Sanghavi, Sujay and Dhillon, Inderjit S},
  journal={IEEE Transactions on Parallel and Distributed Systems},
  year={2021},
  publisher={IEEE}
}
@article{chen2020communication,
  title={Communication-Efficient Algorithms for Decentralized Optimization Over Directed Graphs},
  author={Chen, Yiuye and Hashemi, Abolfazl and Vikalo, Haris},
  journal={arXiv preprint arXiv:2005.13189},
  year={2020}
}
@article{li2019convergence,
  title={On the convergence of fedavg on non-iid data},
  author={Li, Xiang and Huang, Kaixuan and Yang, Wenhao and Wang, Shusen and Zhang, Zhihua},
  journal={arXiv preprint arXiv:1907.02189},
  year={2019}
}
@inproceedings{mcmahan2017communication,
  title={Communication-efficient learning of deep networks from decentralized data},
  author={McMahan, Brendan and Moore, Eider and Ramage, Daniel and Hampson, Seth and y Arcas, Blaise Aguera},
  booktitle={Artificial Intelligence and Statistics},
  pages={1273--1282},
  year={2017},
  organization={PMLR}
}

@article{bassily2018exponential,
  title={On exponential convergence of sgd in non-convex over-parametrized learning},
  author={Bassily, Raef and Belkin, Mikhail and Ma, Siyuan},
  journal={arXiv preprint arXiv:1811.02564},
  year={2018}
}

@inproceedings{alistarh2017qsgd,
  title={QSGD: Communication-efficient SGD via gradient quantization and encoding},
  author={Alistarh, Dan and Grubic, Demjan and Li, Jerry and Tomioka, Ryota and Vojnovic, Milan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1709--1720},
  year={2017}
}

@article{polyak1963gradient,
  title={Gradient methods for minimizing functionals},
  author={Polyak, Boris Teodorovich},
  journal={Zhurnal Vychislitel'noi Matematiki i Matematicheskoi Fiziki},
  volume={3},
  number={4},
  pages={643--653},
  year={1963},
  publisher={Russian Academy of Sciences, Branch of Mathematical Sciences}
}

@inproceedings{karimi2016linear,
  title={Linear convergence of gradient and proximal-gradient methods under the polyak-{\l}ojasiewicz condition},
  author={Karimi, Hamed and Nutini, Julie and Schmidt, Mark},
  booktitle={Joint European Conference on Machine Learning and Knowledge Discovery in Databases},
  pages={795--811},
  year={2016},
  organization={Springer}
}

@article{liu2020toward,
  title={Toward a theory of optimization for over-parameterized systems of non-linear equations: the lessons of deep learning},
  author={Liu, Chaoyue and Zhu, Libin and Belkin, Mikhail},
  journal={arXiv preprint arXiv:2003.00307},
  year={2020}
}

@article{stich2018local,
  title={Local SGD converges fast and communicates little},
  author={Stich, Sebastian U},
  journal={arXiv preprint arXiv:1805.09767},
  year={2018}
}

@article{wang2018cooperative,
  title={Cooperative SGD: A unified framework for the design and analysis of communication-efficient SGD algorithms},
  author={Wang, Jianyu and Joshi, Gauri},
  journal={arXiv preprint arXiv:1808.07576},
  year={2018}
}

@article{yu2018parallel,
  title={Parallel restarted SGD for non-convex optimization with faster convergence and less communication},
  author={Yu, Hao and Yang, Sen and Zhu, Shenghuo},
  journal={arXiv preprint arXiv:1807.06629},
  volume={2},
  number={4},
  pages={7},
  year={2018}
}

@article{zinkevich2010parallelized,
  title={Parallelized stochastic gradient descent},
  author={Zinkevich, Martin and Weimer, Markus and Li, Lihong and Smola, Alex},
  journal={Advances in neural information processing systems},
  volume={23},
  pages={2595--2603},
  year={2010}
}

@article{wang2019slowmo,
  title={SlowMo: Improving communication-efficient distributed SGD with slow momentum},
  author={Wang, Jianyu and Tantia, Vinayak and Ballas, Nicolas and Rabbat, Michael},
  journal={arXiv preprint arXiv:1910.00643},
  year={2019}
}

@article{huo2020faster,
  title={Faster On-Device Training Using New Federated Momentum Algorithm},
  author={Huo, Zhouyuan and Yang, Qian and Gu, Bin and Huang, Lawrence Carin and others},
  journal={arXiv preprint arXiv:2002.02090},
  year={2020}
}

@article{qu2020federated,
  title={Federated Learning's Blessing: FedAvg has Linear Speedup},
  author={Qu, Zhaonan and Lin, Kaixiang and Kalagnanam, Jayant and Li, Zhaojian and Zhou, Jiayu and Zhou, Zhengyuan},
  journal={arXiv preprint arXiv:2007.05690},
  year={2020}
}

@article{reddi2020adaptive,
  title={Adaptive Federated Optimization},
  author={Reddi, Sashank and Charles, Zachary and Zaheer, Manzil and Garrett, Zachary and Rush, Keith and Kone{\v{c}}n{\`y}, Jakub and Kumar, Sanjiv and McMahan, H Brendan},
  journal={arXiv preprint arXiv:2003.00295},
  year={2020}
}

@article{arjevani2019lower,
  title={Lower bounds for non-convex stochastic optimization},
  author={Arjevani, Yossi and Carmon, Yair and Duchi, John C and Foster, Dylan J and Srebro, Nathan and Woodworth, Blake},
  journal={arXiv preprint arXiv:1912.02365},
  year={2019}
}

@article{stich2019error,
  title={The error-feedback framework: Better rates for SGD with delayed gradients and compressed communication},
  author={Stich, Sebastian U and Karimireddy, Sai Praneeth},
  journal={arXiv preprint arXiv:1909.05350},
  year={2019}
}

@article{patel2019communication,
  title={Communication trade-offs for synchronized distributed SGD with large step size},
  author={Patel, Kumar Kshitij and Dieuleveut, Aymeric},
  journal={arXiv preprint arXiv:1904.11325},
  year={2019}
}

@article{woodworth2020local,
  title={Is Local SGD Better than Minibatch SGD?},
  author={Woodworth, Blake and Patel, Kumar Kshitij and Stich, Sebastian U and Dai, Zhen and Bullins, Brian and McMahan, H Brendan and Shamir, Ohad and Srebro, Nathan},
  journal={arXiv preprint arXiv:2002.07839},
  year={2020}
}

@inproceedings{bayoumi2020tighter,
  title={Tighter theory for local SGD on identical and heterogeneous data},
  author={Bayoumi, Ahmed Khaled Ragab and Mishchenko, Konstantin and Richt{\'a}rik, Peter},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={4519--4529},
  year={2020}
}

@article{bernstein2018signsgd,
  title={signSGD: Compressed optimisation for non-convex problems},
  author={Bernstein, Jeremy and Wang, Yu-Xiang and Azizzadenesheli, Kamyar and Anandkumar, Anima},
  journal={arXiv preprint arXiv:1802.04434},
  year={2018}
}

@inproceedings{suresh2017distributed,
  title={Distributed mean estimation with limited communication},
  author={Suresh, Ananda Theertha and Felix, X Yu and Kumar, Sanjiv and McMahan, H Brendan},
  booktitle={International Conference on Machine Learning},
  pages={3329--3337},
  year={2017}
}

@inproceedings{tang2018communication,
  title={Communication compression for decentralized training},
  author={Tang, Hanlin and Gan, Shaoduo and Zhang, Ce and Zhang, Tong and Liu, Ji},
  booktitle={Advances in Neural Information Processing Systems},
  pages={7652--7662},
  year={2018}
}

@article{wu2018error,
  title={Error compensated quantized SGD and its applications to large-scale distributed optimization},
  author={Wu, Jiaxiang and Huang, Weidong and Huang, Junzhou and Zhang, Tong},
  journal={arXiv preprint arXiv:1806.08054},
  year={2018}
}

@inproceedings{alistarh2018convergence,
  title={The convergence of sparsified gradient methods},
  author={Alistarh, Dan and Hoefler, Torsten and Johansson, Mikael and Konstantinov, Nikola and Khirirat, Sarit and Renggli, C{\'e}dric},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5973--5983},
  year={2018}
}

@article{lin2017deep,
  title={Deep gradient compression: Reducing the communication bandwidth for distributed training},
  author={Lin, Yujun and Han, Song and Mao, Huizi and Wang, Yu and Dally, William J},
  journal={arXiv preprint arXiv:1712.01887},
  year={2017}
}

@inproceedings{stich2018sparsified,
  title={Sparsified SGD with memory},
  author={Stich, Sebastian U and Cordonnier, Jean-Baptiste and Jaggi, Martin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4447--4458},
  year={2018}
}

@inproceedings{fang2018spider,
  title={Spider: Near-optimal non-convex optimization via stochastic path-integrated differential estimator},
  author={Fang, Cong and Li, Chris Junchi and Lin, Zhouchen and Zhang, Tong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={689--699},
  year={2018}
}

@article{zhou2018stochastic,
  title={Stochastic nested variance reduced gradient descent for nonconvex optimization},
  author={Zhou, Dongruo and Xu, Pan and Gu, Quanquan},
  journal={Advances in neural information processing systems},
  year={2018}
}

@article{konevcny2016federated,
  title={Federated optimization: Distributed machine learning for on-device intelligence},
  author={Kone{\v{c}}n{\`y}, Jakub and McMahan, H Brendan and Ramage, Daniel and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:1610.02527},
  year={2016}
}

@article{konevcny2016federated2,
  title={Federated learning: Strategies for improving communication efficiency},
  author={Kone{\v{c}}n{\`y}, Jakub and McMahan, H Brendan and Yu, Felix X and Richt{\'a}rik, Peter and Suresh, Ananda Theertha and Bacon, Dave},
  journal={arXiv preprint arXiv:1610.05492},
  year={2016}
}

@article{kairouz2019advances,
  title={Advances and open problems in federated learning},
  author={Kairouz, Peter and McMahan, H Brendan and Avent, Brendan and Bellet, Aur{\'e}lien and Bennis, Mehdi and Bhagoji, Arjun Nitin and Bonawitz, Keith and Charles, Zachary and Cormode, Graham and Cummings, Rachel and others},
  journal={arXiv preprint arXiv:1912.04977},
  year={2019}
}

@inproceedings{arjevani2015communication,
  title={Communication complexity of distributed convex learning and optimization},
  author={Arjevani, Yossi and Shamir, Ohad},
  booktitle={Advances in neural information processing systems},
  pages={1756--1764},
  year={2015}
}

@article{xiao2017fashion,
  title={Fashion-mnist: a novel image dataset for benchmarking machine learning algorithms},
  author={Xiao, Han and Rasul, Kashif and Vollgraf, Roland},
  journal={arXiv preprint arXiv:1708.07747},
  year={2017}
}

@incollection{bottou2012stochastic,
  title={Stochastic gradient descent tricks},
  author={Bottou, L{\'e}on},
  booktitle={Neural networks: Tricks of the trade},
  pages={421--436},
  year={2012},
  publisher={Springer}
}

@inproceedings{koloskova2020unified,
  title={A unified theory of decentralized SGD with changing topology and local updates},
  author={Koloskova, Anastasia and Loizou, Nicolas and Boreiri, Sadra and Jaggi, Martin and Stich, Sebastian},
  booktitle={International Conference on Machine Learning},
  pages={5381--5393},
  year={2020},
  organization={PMLR}
}


@article{horvath2019stochastic,
  title={Stochastic distributed learning with gradient quantization and variance reduction},
  author={Horv{\'a}th, Samuel and Kovalev, Dmitry and Mishchenko, Konstantin and Stich, Sebastian and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:1904.05115},
  year={2019}
}

@article{gorbunov2021marina,
  title={MARINA: Faster Non-Convex Distributed Learning with Compression},
  author={Gorbunov, Eduard and Burlachenko, Konstantin and Li, Zhize and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2102.07845},
  year={2021}
}

@article{roux2012stochastic,
  title={A stochastic gradient method with an exponential convergence \_rate for finite training sets},
  author={Roux, Nicolas and Schmidt, Mark and Bach, Francis},
  journal={Advances in neural information processing systems},
  volume={25},
  year={2012}
}

@article{defazio2014saga,
  title={SAGA: A fast incremental gradient method with support for non-strongly convex composite objectives},
  author={Defazio, Aaron and Bach, Francis and Lacoste-Julien, Simon},
  journal={Advances in neural information processing systems},
  volume={27},
  year={2014}
}

@article{johnson2013accelerating,
  title={Accelerating stochastic gradient descent using predictive variance reduction},
  author={Johnson, Rie and Zhang, Tong},
  journal={Advances in neural information processing systems},
  volume={26},
  year={2013}
}

@inproceedings{nguyen2017sarah,
  title={SARAH: A novel method for machine learning problems using stochastic recursive gradient},
  author={Nguyen, Lam M and Liu, Jie and Scheinberg, Katya and Tak{\'a}{\v{c}}, Martin},
  booktitle={International Conference on Machine Learning},
  pages={2613--2621},
  year={2017},
  organization={PMLR}
}

@article{li2021zerosarah,
  title={ZeroSARAH: Efficient nonconvex finite-sum optimization with zero full gradient computation},
  author={Li, Zhize and Hanzely, Slavom{\'\i}r and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2103.01447},
  year={2021}
}

@article{khanduri2021stem,
  title={Stem: A stochastic two-sided momentum algorithm achieving near-optimal sample and communication complexities for federated learning},
  author={Khanduri, Prashant and Sharma, Pranay and Yang, Haibo and Hong, Mingyi and Liu, Jia and Rajawat, Ketan and Varshney, Pramod},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{lee2019wide,
  title={Wide neural networks of any depth evolve as linear models under gradient descent},
  author={Lee, Jaehoon and Xiao, Lechao and Schoenholz, Samuel and Bahri, Yasaman and Novak, Roman and Sohl-Dickstein, Jascha and Pennington, Jeffrey},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@article{jacot2018neural,
  title={Neural tangent kernel: Convergence and generalization in neural networks},
  author={Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
  journal={Advances in neural information processing systems},
  volume={31},
  year={2018}
}

@article{mu2020gradients,
  title={Gradients as features for deep representation learning},
  author={Mu, Fangzhou and Liang, Yingyu and Li, Yin},
  journal={arXiv preprint arXiv:2004.05529},
  year={2020}
}
