@article{wang2021field,
  title={A Field Guide to Federated Optimization},
  author={Wang, Jianyu and Charles, Zachary and Xu, Zheng and Joshi, Gauri and McMahan, H Brendan and Al-Shedivat, Maruan and Andrew, Galen and Avestimehr, Salman and Daly, Katharine and Data, Deepesh and others},
  journal={arXiv preprint arXiv:2107.06917},
  year={2021}
}



@inproceedings{krizhevsky2012imagenet,
  title={Imagenet classification with deep convolutional neural networks},
  author={Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1097--1105},
  year={2012}
}

@book{gallager2013stochastic,
 	author = {R. Gallager},
 	title = {Stochastic Processes: Theory for Applications},
	year = {2013},
	publisher = {Cambridge University Press},
	edition = {1st}
 }
 
 @misc{gelfandcmu,
author={{Carnegie Mellon University}},
title={Gelfand Summer Outreach Classes},
howpublished={\url{https://www.cmu.edu/gelfand/gelfand-outreach/gelfandoutreach_summer2020.pdf}},
year={2020}
 }
 
  @ARTICLE{dav2009domain,
  author={S. Ben-David and J. Blitzer and K. Crammer and A. Kulesza and F. Pereira and J. W. Vaughan},
  journal={Machine Learning}, 
  title={A Theory of Learning from Different Domains}, 
   year={2009},
  volume={79},
  number={1-2},
  pages={151-175}
  }
 
   @ARTICLE{alb1998mp,
  author={Ya.I. Alber and A.N. Iusem and M.V. Solodovz},
  journal={Mathematical Programming}, 
  title={ON THE PROJECTED SUBGRADIENT METHOD FOR NONSMOOTH CONVEX OPTIMIZATION IN A HILBERT SPACE}, 
   year={1998},
  volume={81},
  number={1},
  pages={23-25}
  }
 
 
  @inproceedings{bist2020distdis,
  author={I. Bistritz and A. J. Mann and N. Bambos},
   year={2020},
   booktitle={Advances in Neural Information Processing Systems},
   title = {Distributed Distillation for On-Device Learning}
  }
 
 @ARTICLE{shag2020codist,
  author={S. Sodhani and O. Delalleau and M. Assran and K. Sinha and N. Ballas and M. Rabbat},
  title={A Closer Look at Codistillation for Distributed Training}, 
   journal   = {CoRR},
  volume    = {abs/2010.02838},
  year      = {2020},
  url       = {http://arxiv.org/abs/2010.02838},
  archivePrefix = {arXiv}
}

@inproceedings{zhan2018dml,
author = {Y. Zhang and T. Xiang and T. M. Hospedales and H. Lu},
title = {Deep mutual learning},
 booktitle={In IEEE Conf. on
Computer Vision and Pattern Recognition (CVPR)},
pages={4320–4328}, 
year={2018}
}

@article{zhan2021adcodis,
author = {Haoran Zhang and Zhenzhen Hu and Wei Qin and Mingliang Xu and Meng Wang},
title = {Adversarial co-distillation learning for image recognition},
journal = {Pattern Recognition},
volume = {111},
pages = {107659},
year = {2021}
}


 @ARTICLE{rohan2018dist,
  author={R. Anil and G. Pereyra and A. Passos and R. Ormandi and G. E. Dahl and G. E. Hinton},
  title={Large Scale Distributed Neural Network Training Through Online Distillation}, 
   journal={International Conference on Learning Representations (ICLR)},
  year={2018}
}

@article{hadda2021comp,
 author = {Farzin Haddadpour and Mohammad Mahdi Kamani and Aryan Mokhtari and Mehrdad Mahdavi},
 title = "{Federated Learning with Compression: Unified Analysis and Sharp Guarantees}",
 journal = {International Conference on Artificial Intelligence and Statistics (AISTATS)},
 year = {2021}
}

 @inproceedings{liang2020thinklocal,
  author={Paul Pu Liang and Terrance Liu and Liu Ziyin and and Nicholas B. Allen and Randy P. Auerbach and David Brent and Ruslan Salakhutdinov and Louis-Philippe Morency},
  title={Think Locally, Act Globally: Federated Learning with Local and Global Representations}, 
   booktitle   = {nternational Workshop on Feder-ated Learning for User Privacy and Data Confidentiality inConjunction with NeurIPS 2019 (FL-NeurIPS’19)},
  year      = {2019},
  url       = {http://arxiv.org/abs/2001.01523},
}

 @inproceedings{ghosh2020cfl,
  author={Avishek Ghosh and Jichan Chung and Dong Yin and Kannan Ramchandran},
  title={An Efficient Framework for Clustered Federated Learning}, 
   booktitle   = {Advances in Neural Information Processing Systems},
  year      = {2020},
  url       = {http://arxiv.org/abs/2006.04088},
}

 @ARTICLE{ziyi2020tpfl,
  author={Alysa Ziying Tan and Han Yu and Lizhen Cui and Qiang Yang},
  title={Towards Personalized Federated Learning}, 
   journal   = {CoRR},
  volume    = {abs/2103.00710},
  year      = {2020},
  url       = {http://arxiv.org/abs/2103.00710},
  archivePrefix = {arXiv}
}

 @ARTICLE{eun2018dist,
  author={Eunjeong Jeong and Seungeun Oh and Hyesung Kim and Jihong Park and Mehdi Bennis and Seong-Lyun Kim},
  journal={International Workshop on Machine Learning on the Phone and other Consumer Devices in Conjunction with NeurIPS (NeurIPS-MLPCD)},
  year={2018}
  }
  
 @ARTICLE{sha2020privacy,
  author={S. Asoodeh and F. {du Pin Calmon}},
  journal={International Workshop on Federated Learning for User Privacy and Data Confidentiality in Conjunction with ICML (ICML-FL)}, 
  title={Differentially Private Federated Learning: An Information-Theoretic Perspective}, 
  year={2020}
  }
 
 @ARTICLE{liao2019tunable,
  author={J. {Liao} and O. {Kosut} and L. {Sankar} and F. {du Pin Calmon}},
  journal={IEEE Transactions on Information Theory}, 
  title={Tunable Measures for Information Leakage and Applications to Privacy-Utility Tradeoffs}, 
  year={2019},
  volume={65},
  number={12},
  pages={8043-8066}
  }
  
  @ARTICLE{diaz2020robustness,
  author={M. {Diaz} and H. {Wang} and F. P. {Calmon} and L. {Sankar}},
  journal={IEEE Transactions on Information Theory}, 
  title={On the Robustness of Information-Theoretic Privacy Measures and Mechanisms}, 
  year={2020},
  volume={66},
  number={4},
  pages={1949-1978}
  }
 
 @ARTICLE{wei2020federated,
  author={K. {Wei} and J. {Li} and M. {Ding} and C. {Ma} and H. H. {Yang} and F. {Farokhi} and S. {Jin} and T. Q. S. {Quek} and H. {Vincent Poor}},
  journal={IEEE Transactions on Information Forensics and Security}, 
  title={Federated Learning With Differential Privacy: Algorithms and Performance Analysis}, 
  year={2020},
  volume={15},
  number={},
  pages={3454-3469}
  }
  
 @article{kairouz2016extremal,
 author = {Kairouz, Peter and Oh, Sewoong and Viswanath, Pramod},
 title = {Extremal Mechanisms for Local Differential Privacy},
 journal = {Journal of Machine Learning Research (JMLR)},
 volume = {17},
 number = {1},
 month = jan,
 year = {2016},
 issn = {1532-4435},
 pages = {492--542},
 numpages = {51}
} 

 @inproceedings{kairouz2016discrete,
 author = {Kairouz, Peter and Bonawitz, Keith and Ramage, Daniel},
 title = {Discrete Distribution Estimation Under Local Privacy},
 booktitle = {Proceedings of the International Conference on Machine Learning (ICML)},
 year = {2016},
 location = {New York, NY, USA},
 pages = {2436--2444},
 numpages = {9},
 url = {http://dl.acm.org/citation.cfm?id=3045390.3045647},
 acmid = {3045647},
 publisher = {JMLR.org}
} 

@incollection{agarwal2018cpsgd,
title = {cpSGD: Communication-efficient and differentially-private distributed SGD},
author = {Agarwal, Naman and Suresh, Ananda Theertha and Yu, Felix Xinnan X and Kumar, Sanjiv and McMahan, Brendan},
booktitle = {Advances in Neural Information Processing Systems},
pages = {7564--7575},
year = {2018},
url = {http://papers.nips.cc/paper/7984-cpsgd-communication-efficient-and-differentially-private-distributed-sgd.pdf}
}

@ARTICLE{ye2018optimal, 
author={M. {Ye} and A. {Barg}}, 
journal={IEEE Transactions on Information Theory}, 
title={Optimal Schemes for Discrete Distribution Estimation Under Locally Differential Privacy}, 
year={2018}, 
volume={64}, 
number={8}, 
pages={5662-5676}, 
doi={10.1109/TIT.2018.2809790}, 
ISSN={0018-9448}, 
month={Aug}
}

@article{acharya2018communication,
  author    = {Jayadev Acharya and Ziteng Sun},
  title     = {Communication Complexity in Locally Private Distribution Estimation and Heavy Hitters},
  journal   = {CoRR},
  volume    = {abs/1905.11888},
  year      = {2019},
  url       = {http://arxiv.org/abs/1905.11888},
  archivePrefix = {arXiv},
  eprint    = {1905.11888},
  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1905-11888},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{erlingsson2014rappor,
 author = {Erlingsson, \'{U}lfar and Pihur, Vasyl and Korolova, Aleksandra},
 title = {RAPPOR: Randomized Aggregatable Privacy-Preserving Ordinal Response},
 booktitle = {Proceedings of ACM SIGSAC Conference on Computer and Communications Security},
 series = {CCS '14},
 year = {2014},
 isbn = {978-1-4503-2957-6},
 pages = {1054--1067},
 numpages = {14},
 url = {http://doi.acm.org/10.1145/2660267.2660348},
 doi = {10.1145/2660267.2660348},
 acmid = {2660348},
 publisher = {ACM}
} 

@article{caldas2018leaf,
  author    = {Sebastian Caldas and Peter Wu and Tian Li and Jakub Konecn{\'{y}} and H. Brendan McMahan and Virginia Smith and Ameet Talwalkar},
  title     = {{LEAF:} {A} Benchmark for Federated Settings},
  journal   = {CoRR},
  volume    = {abs/1812.01097},
  year      = {2018},
  url       = {http://arxiv.org/abs/1812.01097},
  archivePrefix = {arXiv},
  eprint    = {1812.01097},
  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1812-01097}
}

@book{bollobas2013modern,
  title={Modern graph theory},
  author={Bollob{\'a}s, B{\'e}la},
  volume={184},
  year={2013},
  publisher={Springer Science \& Business Media}
}

@article{kar2008sensor,
  title={Sensor networks with random links: Topology design for distributed consensus},
  author={Kar, Soummya and Moura, Jos{\'e} MF},
  journal={IEEE Transactions on Signal Processing},
  volume={56},
  number={7},
  pages={3315--3326},
  year={2008},
  publisher={IEEE}
}
 
@article{towfic2016excess,
  title={Excess-risk of distributed stochastic learners},
  author={Towfic, Zaid J and Chen, Jianshu and Sayed, Ali H},
  journal={IEEE Transactions on Information Theory},
  volume={62},
  number={10},
  pages={5753--5785},
  year={2016},
  publisher={IEEE}
}

 @InProceedings{assran2019stochastic,
  title = 	 {Stochastic Gradient Push for Distributed Deep Learning},
  author = 	 {Assran, Mahmoud and Loizou, Nicolas and Ballas, Nicolas and Rabbat, Mike},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {344--353},
  year = 	 {2019},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--15 Jun},
  publisher = 	 {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/assran19a/assran19a.pdf},
  url = 	 {http://proceedings.mlr.press/v97/assran19a.html}}
}

@inproceedings{jhunjhunwala2021adaptive,
  title={Adaptive quantization of model updates for communication-efficient federated learning},
  author={Jhunjhunwala, Divyansh and Gadhikar, Advait and Joshi, Gauri and Eldar, Yonina C},
  booktitle={ICASSP 2021-2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
  pages={3110--3114},
  year={2021},
  organization={IEEE}
}



@InProceedings{lian2018asynchronous,
  title = 	 {Asynchronous Decentralized Parallel Stochastic Gradient Descent},
  author = 	 {Lian, Xiangru and Zhang, Wei and Zhang, Ce and Liu, Ji},
  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
  pages = 	 {3043--3052},
  year = 	 {2018},
  volume = 	 {80},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {10--15 Jul},
  publisher = 	 {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v80/lian18a/lian18a.pdf},
  url = 	 {http://proceedings.mlr.press/v80/lian18a.html}
}

@article{jhunjhunwala2021leveraging,
  title={Leveraging Spatial and Temporal Correlations in Sparsified Mean Estimation},
  author={Jhunjhunwala, Divyansh and Mallick, Ankur and Gadhikar, Advait and Kadhe, Swanand and Joshi, Gauri},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@inproceedings{yjc2020bandit,
  title={Bandit-based Communication-Efficient Client Selection Strategies for Federated Learning},
  author={Cho, Yae Jee and Gupta, Samarth and Joshi, Gauri and Ya{\u{g}}an, Osman},
  booktitle={2020 54th Asilomar Conference on Signals, Systems, and Computers},
  pages={1066--1069},
  year={2020},
  organization={IEEE}
}

@article{smith2017dont,
  title={Don't Decay the Learning Rate, Increase the Batch Size},
  author={Samuel L. Smith and Pieter-Jan Kindermans and Quoc V. Le},
  journal={International Conference on Learning Representations (ICLR)},
  year={2017},
  url={https://arxiv.org/abs/1711.00489}
}

@article{cohen2017emnist,
  title={{EMNIST}: an extension of {MNIST} to handwritten letters},
  author={Cohen, Gregory and Afshar, Saeed and Tapson, Jonathan  and Andr\'e van Schaik},
  journal={arXiv preprint arXiv:1702.05373},
  year = {2017}
}

@article{sam2020errfdbk,
  title={A Better Alternative to Error Feedback for Communication-Efficient Distributed Learning},
  author={Horv\'ath, Samuel and Richt\'arik, Peter},
  journal={arXiv preprint arXiv:2006.11077},
  year = {2020}
}


@InProceedings{chen2016scalable,
author = {Chen, Kai and Huo, Qiang},
title = {Scalable Training of Deep Learning Machines by Incremental Block Training with Intra-block Parallel Optimization and Blockwise Model-Update Filtering},
booktitle = {Proceedings of the International Conference on Acoustics, Speech, and Signal Processing (ICASSP)},
year = {2016},
month = {March}
}

@InProceedings{yu2019linear,
author = {Hao Yu and Rong Jin and Sen Yang},
title = {On the Linear Speedup Analysis of Communication Efficient Momentum SGD for Distributed Non-Convex Optimization},
booktitle = {Proceedings of the International Conference on Machine Learning (ICML)},
year = {2019},
month = {jun}
}


@inproceedings{tang2018communication,
  title={Communication compression for decentralized training},
  author={Tang, Hanlin and Gan, Shaoduo and Zhang, Ce and Zhang, Tong and Liu, Ji},
  booktitle={Advances in Neural Information Processing Systems},
  pages={7652--7662},
  year={2018}
}

@article{tang2018d,
  title={D$^{2}$: Decentralized Training over Decentralized Data},
  author={Tang, Hanlin and Lian, Xiangru and Yan, Ming and Zhang, Ce and Liu, Ji},
  journal={arXiv preprint arXiv:1803.07068},
  year={2018}
}


@inproceedings{defazio2014saga,
title = {{SAGA}: A Fast Incremental Gradient Method With Support for Non-Strongly Convex Composite Objectives},
author = {Defazio, Aaron and Bach, Francis and Lacoste-Julien, Simon},
booktitle = {Advances in Neural Information Processing Systems 27},
pages = {1646--1654},
year = {2014}
}

@inproceedings{reddi2020adaptive,
  title={Adaptive Federated Optimization},
  author={Reddi, Sashank J and Charles, Zachary and Zaheer, Manzil and Garrett, Zachary and Rush, Keith and Kone{\v{c}}n{\`y}, Jakub and Kumar, Sanjiv and McMahan, Hugh Brendan},
  booktitle={International Conference on Learning Representations},
  year={2021}
}

@article{wang2020tackling,
  title={Tackling the objective inconsistency problem in heterogeneous federated optimization},
  author={Wang, Jianyu and Liu, Qinghua and Liang, Hao and Joshi, Gauri and Poor, H Vincent},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={7611--7623},
  year={2020}
}

@article{yang2021achieving,
  title={Achieving linear speedup with partial worker participation in non-iid federated learning},
  author={Yang, Haibo and Fang, Minghong and Liu, Jia},
  journal={arXiv preprint arXiv:2101.11203},
  year={2021}
}

@article{li2020federated,
  title={Federated optimization in heterogeneous networks},
  author={Li, Tian and Sahu, Anit Kumar and Zaheer, Manzil and Sanjabi, Maziar and Talwalkar, Ameet and Smith, Virginia},
  journal={Proceedings of Machine Learning and Systems},
  volume={2},
  pages={429--450},
  year={2020}
}



@article{zhang2020fedpd,
  title={Fedpd: A federated learning framework with optimal rates and adaptivity to non-iid data},
  author={Zhang, Xinwei and Hong, Mingyi and Dhople, Sairaj and Yin, Wotao and Liu, Yang},
  journal={arXiv preprint arXiv:2005.11418},
  year={2020}
}

@InProceedings{shamir2014communication,
  title = 	 {Communication-Efficient Distributed Optimization using an Approximate Newton-type Method},
  author = 	 {Ohad Shamir and Nati Srebro and Tong Zhang},
  booktitle = 	 {Proceedings of the 31st International Conference on Machine Learning},
  pages = 	 {1000--1008},
  year = 	 {2014},
  editor = 	 {Eric P. Xing and Tony Jebara},
  volume = 	 {32},
  number =    {2},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 jun,
  url = 	 {http://proceedings.mlr.press/v32/shamir14.pdf}
}


@article{shi2015extra,
  title={Extra: An exact first-order algorithm for decentralized consensus optimization},
  author={Shi, Wei and Ling, Qing and Wu, Gang and Yin, Wotao},
  journal={SIAM Journal on Optimization},
  volume={25},
  number={2},
  pages={944--966},
  year={2015},
  publisher={SIAM}
}

@article{xin2019variance,
  title={Variance-Reduced Decentralized Stochastic Optimization with Accelerated Convergence},
  author={Xin, Ran and Khan, Usman A and Kar, Soummya},
  journal={arXiv preprint arXiv:1912.04230},
  year={2019}
}


@article{cen2019convergence,
  title={Convergence of Distributed Stochastic Variance Reduced Methods without Sampling Extra Data},
  author={Shicong Cen and Huishuai Zhang and Yuejie Chi and Wei Chen and Tie-Yan Liu},
  journal = {arXiV preprint 1905.12648},
  year={2019},
  url={https://arxiv.org/abs/1905.12648}
}


@inproceedings{jia2014caffe,
 author = {Jia, Yangqing and Shelhamer, Evan and Donahue, Jeff and Karayev, Sergey and Long, Jonathan and Girshick, Ross and Guadarrama, Sergio and Darrell, Trevor},
 title = {Caffe: Convolutional Architecture for Fast Feature Embedding},
 booktitle = {Proceedings of the ACM International Conference on Multimedia},
 year = {2014},
 pages = {675--678},
 numpages = {4}
} 

@article{paszke2017automatic,
  title={Automatic differentiation in PyTorch},
  author={Paszke, Adam and Gross, Sam and Chintala, Soumith and Chanan, Gregory and Yang, Edward and DeVito, Zachary and Lin, Zeming and Desmaison, Alban and Antiga, Luca and Lerer, Adam},
  year={2017}
}

@inproceedings{zhang2017understanding,
title	= {Understanding deep learning requires rethinking generalization},
author	= {Chiyuan Zhang and Samy Bengio and Moritz Hardt and Benjamin Recht and Oriol Vinyals},
year	= {2017},
URL	= {https://arxiv.org/abs/1611.03530}
}

@article{azizan2019stochastic,
  author    = {Navid Azizan Ruhi and Sahin Lale and Babak Hassibi},
  title     = {Stochastic Mirror Descent on Overparameterized Nonlinear Models: Convergence,
               Implicit Regularization, and Generalization},
  journal   = {CoRR},
  volume    = {abs/1906.03830},
  year      = {2019},
  url       = {http://arxiv.org/abs/1906.03830},
  archivePrefix = {arXiv}
}

 @article{ma2017implicit,
  author    = {Cong Ma and
               Kaizheng Wang and
               Yuejie Chi and
               Yuxin Chen},
  title     = {Implicit Regularization in Nonconvex Statistical Estimation: Gradient
               Descent Converges Linearly for Phase Retrieval, Matrix Completion
               and Blind Deconvolution},
  journal   = {CoRR},
  volume    = {abs/1711.10467},
  year      = {2017},
  url       = {http://arxiv.org/abs/1711.10467},
  archivePrefix = {arXiv},
  eprint    = {1711.10467}
}

@inproceedings{hardt2016train,
 author = {Hardt, Moritz and Recht, Benjamin and Singer, Yoram},
 title = {Train Faster, Generalize Better: Stability of Stochastic Gradient Descent},
 booktitle = {Proceedings of the International Conference on Machine Learning (ICML)},
 series = {ICML'16},
 year = {2016},
 location = {New York, NY, USA},
 pages = {1225--1234},
 numpages = {10},
 url = {http://dl.acm.org/citation.cfm?id=3045390.3045520},
 acmid = {3045520},
 publisher = {JMLR.org}
} 

@article{neyshabur2017geometry,
  author    = {Behnam Neyshabur and Ryota Tomioka and Ruslan Salakhutdinov and Nathan Srebro},
  title     = {Geometry of Optimization and Implicit Regularization in Deep Learning},
  journal   = {CoRR},
  volume    = {abs/1705.03071},
  year      = {2017},
  url       = {http://arxiv.org/abs/1705.03071},
  archivePrefix = {arXiv},
  eprint    = {1705.03071}
}

@article{shwartz2017opening,
  author    = {Ravid Shwartz{-}Ziv and
               Naftali Tishby},
  title     = {Opening the Black Box of Deep Neural Networks via Information},
  journal   = {CoRR},
  volume    = {abs/1703.00810},
  year      = {2017},
  url       = {http://arxiv.org/abs/1703.00810},
  archivePrefix = {arXiv},
  eprint    = {1703.00810}
}

@article{chaudhari2017stochastic,
  author    = {Pratik Chaudhari and
               Stefano Soatto},
  title     = {Stochastic gradient descent performs variational inference, converges to limit cycles for deep networks},
  journal   = {CoRR},
  volume    = {abs/1710.11029},
  year      = {2017},
  url       = {http://arxiv.org/abs/1710.11029},
  archivePrefix = {arXiv},
  eprint    = {1710.11029}
}

@inproceedings{abadi2016tensorflow,
title = {TensorFlow: A system for large-scale machine learning},
author  = {{The TensorFlow development team}},
year  = {2016},
booktitle = {Proceedings of the USENIX Symposium on Operating Systems Design and Implementation (OSDI)},
pages = {265--283}
}

@ARTICLE{theano2016theano,
   author = {{Theano Development Team}},
    title = "{Theano: A {Python} framework for fast computation of mathematical expressions}",
  journal = {arXiv e-prints},
   volume = {abs/1605.02688},
 primaryClass = "cs.SC",
     year = 2016,
    month = may,
   url = {http://arxiv.org/abs/1605.02688}
}


@ARTICLE{tf_federated,
   author = {{Google Research}},
    title = "{TensorFlow Federated}",
     year = 2018,
     url = {https://www.tensorflow.org/federated}
}

@ARTICLE{course18847,
   author = {{Gauri Joshi}},
    title = "{18-847F Course Website}",
     year = 2018,
     url = {http://www.andrew.cmu.edu/user/gaurij/18-847F-Fall-2018.html}
}

@ARTICLE{course18661,
   author = {{Gauri Joshi}},
    title = "{18-461/661 Course Website}",
     year = 2019,
     url = {https://www.andrew.cmu.edu/course/18-661/}
}

@ARTICLE{wisdm_2019,
   author = {{ICERM}},
    title = "{WiSDM Workshop}",
     year = 2019,
     url = {https://icerm.brown.edu/topical_workshops/tw19-5-wisdm/}
}

@ARTICLE{wisdm_2017,
   author = {{ICERM}},
    title = "{WiSDM Workshop}",
     year = 2017,
     url = {https://icerm.brown.edu/topical_workshops/tw17-3-wisdm/}
}



@ARTICLE{zhang2016parallel,
   author = {{Jian Zhang, Christopher De Sa, Ioannis Mitliagkas, Christopher Re}},
    title = "{Parallel SGD: When does averaging help?}",
  journal = {arXiv e-prints},
   volume = {abs/1606.07365},
 primaryClass = "stat.ML",
     year = 2016,
    month = jun,
      url = {https://arxiv.org/abs/1606.07365},
}

@ARTICLE{jin2016how,
   author = {{Peter H. Jin, Qiaochu Yuan, Forrest Iandola, Kurt Keutzer}},
    title = "{How to scale distributed deep learning?}",
  journal = {arXiv e-prints},
   volume = {abs/1611.04581},
 primaryClass = "cs.LG",
     year = 2016,
    month = nov,
      url = {https://arxiv.org/abs/1611.04581},
}


@INPROCEEDINGS{szegedy2015going, 
author={C. Szegedy and Wei Liu and Yangqing Jia and P. Sermanet and S. Reed and D. Anguelov and D. Erhan and V. Vanhoucke and A. Rabinovich}, 
booktitle={IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 
title={Going deeper with convolutions}, 
year={2015},
pages={1-9},
month= jun}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@INPROCEEDINGS{simonyan2015very, 
author={Karen Simonyan and Andrew Zisserman}, 
booktitle={International Conference on Learning Representations (ICLR)}, 
title={Very Deep Convolutional Networks for Large-Scale Image Recognition}, 
year={2015}
}

@article{krizhevsky2009learning,
  title={Learning multiple layers of features from tiny images},
  author={Krizhevsky, Alex and Hinton, Geoffrey},
  year={2009},
  journal={Technical report, University of Toronto}
}


@article{google2017tensor,
  title={Tensor Processing Units},
  author={Google},
  year={2017},
  url={https://cloud.google.com/tpu/docs/tpus}
}

@article{google2017tensor,
  title={Tensor Processing Units},
  author={Google},
  year={2017},
  url={https://cloud.google.com/tpu/docs/tpus}
}

@article{simonyan2014very,
  author    = {Karen Simonyan and Andrew Zisserman},
  title     = {Very Deep Convolutional Networks for Large-Scale Image Recognition},
  journal   = {CoRR},
  volume    = {abs/1409.1556},
  year      = {2014},
  url       = {http://arxiv.org/abs/1409.1556},
  archivePrefix = {arXiv},
  eprint    = {1409.1556}
}

@inproceedings{gupta2018active,
 author = {Samarth Gupta and Gauri Joshi and Osman Ya\u{g}an},
 title = {Active Distribution Learning Using Indirect Samples},
 booktitle = {Allerton Conference on Communication, Control and Computing},
 month = oct,
 year = {2018}
} 


@article{gupta2018correlated,
 author = {Samarth Gupta and Gauri Joshi and Osman Ya\u{g}an},
 title = {Correlated Multi-armed Bandits with a Latent Random Source},
 journal = {ArXiv},
 month = aug,
 year = 2018,
 url = {https://arxiv.org/abs/1808.05904}
} 

@article{gupta2018exploiting,
 author = {Samarth Gupta and Gauri Joshi and Osman Ya\u{g}an},
 title = {Exploiting Correlation in Finite-Armed Structured Bandits},
 journal = {ArXiv},
 month = oct,
 year = 2018,
 url = {https://arxiv.org/abs/1810.08164}
} 

@article{lecun1998mnist,
  title={The {MNIST} database of handwritten digits},
  author={LeCun, Yann},
  journal={http://yann. lecun. com/exdb/mnist/},
  year={1998},
}

@article{lecun2015lenet,
  title={LeNet-5, convolutional neural networks},
  author={LeCun, Yann and others},
  journal={URL: http://yann. lecun. com/exdb/lenet},
  volume={20},
  number={5},
  pages={14},
  year={2015}
}

@article{robbins1951stochastic,
  title={A stochastic approximation method},
  author={Robbins, Herbert and Monro, Sutton},
  journal={The annals of mathematical statistics},
  pages={400--407},
  year={1951},
  publisher={JSTOR}
}

@article{krizhevsky2009cifar,
title= {Learning Multiple Layers of Features from Tiny Images},
journal= {CIFAR-10 (Canadian Institute for Advanced Research)},
author= {Alex Krizhevsky and Vinod Nair and Geoffrey Hinton},
url= {http://www.cs.toronto.edu/~kriz/cifar.html},
year = {2009}
}



@inproceedings{sha2017largenn,
 author = {Noam Shazeer and Azalia Mirhoseini and Krzysztof Maziarz and Andy Davis and Quoc V. Le and Geoffrey E. Hinton and Jeff Dean},
 title = {Outrageously large neural networks: The sparsely-gated mixture-of-experts layer},
 booktitle = {International Conference on Learning Respresentations (ICLR)},
 year = {2017}
} 

@article{blum2021flincentive,
  title={One for One, or All for All: Equilibria and Optimality of Collaboration in Federated Learning},
  author={Avrim Blum and Nika Haghtalab and Richard Lanas Phillips and Han Shao},
  journal={International Conference on Machine Learning (ICML)},
  year={2021}
}


@article{don2021flincentive,
  title={Optimality and Stability in Federated Learning:
A Game-theoretic Approach},
  author={Kate Donahue and Jon Kleinberg},
  journal={Advances in Neural Information Processing Systems (NeurIPS)},
  year={2021}
}

@article{alec2009sent,
title= {Twitter Sentiment Classification using Distant Supervision},
journal= {CS224N Project Report, Stanford},
author= {Alec Go and Richa Bhayani and Lei Huang},
url= {https://www-cs.stanford.edu/people/alecmgo/papers/TwitterDistantSupervision09.pdf},
year = {2009}
}

@article{russakovsky2015imagenet,
 author = {Russakovsky, Olga and Deng, Jia and Su, Hao and Krause, Jonathan and Satheesh, Sanjeev and Ma, Sean and Huang, Zhiheng and Karpathy, Andrej and Khosla, Aditya and Bernstein, Michael and Berg, Alexander C. and Fei-Fei, Li},
 title = {ImageNet Large Scale Visual Recognition Challenge},
 journal = {International Journal of Computer Vision},
 volume = {115},
 number = {3},
 month = dec,
 year = {2015},
 pages = {211--252},
 numpages = {42}
} 

@inproceedings{gupta2015deep,
 author = {Gupta, Suyog and Agrawal, Ankur and Gopalakrishnan, Kailash and Narayanan, Pritish},
 title = {Deep Learning with Limited Numerical Precision},
 booktitle = {Proceedings of the International Conference on International Conference on Machine Learning (ICML)},
 year = {2015},
 pages = {1737--1746},
 numpages = {10}
} 

@inproceedings{agarwal2011distributed,
 author = {Agarwal, Alekh and Duchi, John C.},
 title = {Distributed Delayed Stochastic Optimization},
 booktitle = {Proceedings of theInternational Conference on Neural Information Processing Systems},
 year = {2011},
 pages = {873--881},
 numpages = {9}
}

@INPROCEEDINGS{feyzmahdavian2015asynchronous, 
author={H. R. Feyzmahdavian and A. Aytekin and M. Johansson}, 
booktitle={Proceedings of the IEEE Conference on Decision and Control (CDC)}, 
title={An asynchronous mini-batch algorithm for regularized stochastic optimization}, 
year={2015}, 
pages={1384-1389}, 
month={Dec}
}

@book{gray1989source,
 author = {Gray, Robert M.},
 title = {Source Coding Theory},
 year = {1989},
 isbn = {0792390482},
 publisher = {Kluwer Academic Publishers},
 address = {Norwell, MA, USA}
} 

@book{du1993combinatorial,
  title={Combinatorial Group Testing and Its Applications},
  author={Ding-Zu Du and Frank K. Hwang},
  year={1993},
  publisher={World Scientific}
}

@book{sheldon2002first,
  title={A first course in probability},
  author={Sheldon, Ross },
  year={2002},
  publisher={Pearson Education India}
}

@book{shalev-shwartz2014understanding,
 author = {Shalev-Shwartz, Shai and Ben-David, Shai},
 title = {Understanding Machine Learning: From Theory to Algorithms},
 year = {2014},
 publisher = {Cambridge University Press},
 address = {New York, NY, USA}
} 

@book{kreps1990course,
  title={A course in microeconomic theory},
  author={Kreps, David M},
  volume={41},
  year={1990},
  publisher={JSTOR}
}


@book{sedgewick2011algorithms,
  title={Algorithms},
  author={Sedgewick, Robert and Wayne, Kevin},
  year={2011},
  publisher={Addison-Wesley Professional}
}


@article{nguyen2017sarah,
  title={SARAH: A novel method for machine learning problems using stochastic recursive gradient},
  author={Nguyen, Lam and Liu, Jie and Scheinberg, Katya and Tak{\'a}{\v{c}}, Martin},
  journal={arXiv preprint arXiv:1703.00102},
  year={2017}
}


@inproceedings{johnson2013accelerating,
  title={Accelerating stochastic gradient descent using predictive variance reduction},
  author={Johnson, Rie and Zhang, Tong},
  booktitle={Advances in neural information processing systems},
  pages={315--323},
  year={2013}
}

@inproceedings{roux2012stochastic,
  title={A stochastic gradient method with an exponential convergence rate for finite training sets},
  author={Roux, Nicolas L and Schmidt, Mark and Bach, Francis R},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2663--2671},
  year={2012}
}

@article{dekel2012optimal,
 author = {Dekel, Ofer and Gilad-Bachrach, Ran and Shamir, Ohad and Xiao, Lin},
 title = {Optimal Distributed Online Prediction Using Mini-batches},
 journal = {Journal of Machine Learning Research},
 volume = {13},
 number = {1},
 month = jan,
 year = {2012},
 pages = {165--202},
 numpages = {38}
} 

@incollection{shamir2016without,
title = {Without-Replacement Sampling for Stochastic Gradient Methods},
author = {Shamir, Ohad},
booktitle = {Advances in Neural Information Processing Systems 29},
pages = {46--54},
year = {2016},
url = {http://papers.nips.cc/paper/6245-without-replacement-sampling-for-stochastic-gradient-methods.pdf}
}

@InProceedings{haochen2019random,
  title = 	 {Random Shuffling Beats {SGD} after Finite Epochs},
  author = 	 {Haochen, Jeff and Sra, Suvrit},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {2624--2633},
  year = 	 {2019},
  volume = 	 {97},
  month = 	 {09--15 Jun},
  publisher = 	 {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/haochen19a/haochen19a.pdf},
  url = 	 {http://proceedings.mlr.press/v97/haochen19a.html}
}

@inproceedings{hsieh2017gaia,
 author = {Hsieh, Kevin and Harlap, Aaron and Vijaykumar, Nandita and Konomis, Dimitris and Ganger, Gregory R. and Gibbons, Phillip B. and Mutlu, Onur},
 title = {Gaia: Geo-distributed Machine Learning Approaching LAN Speeds},
 booktitle = {Proceedings of the 14th USENIX Conference on Networked Systems Design and Implementation (NSDI)},
 year = {2017},
 pages = {629--647},
 numpages = {19},
 url = {http://dl.acm.org/citation.cfm?id=3154630.3154682},
 acmid = {3154682},
 publisher = {USENIX Association}
} 

@article{sattler2019robust,
  author    = {Felix Sattler and Simon Wiedemann and Klaus{-}Robert M{\"{u}}ller and Wojciech Samek},
  title     = {Robust and Communication-Efficient Federated Learning from Non-IID
               Data},
  journal   = {CoRR arXiv},
  volume    = {abs/1903.02891},
  year      = {2019},
  url       = {http://arxiv.org/abs/1903.02891}
}



@InProceedings{bhagoji2019analyzing,
  title = 	 {Analyzing Federated Learning through an Adversarial Lens},
  author = 	 {Bhagoji, Arjun Nitin and Chakraborty, Supriyo and Mittal, Prateek and Calo, Seraphin},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {634--643},
  year = 	 {2019},
  volume = 	 {97},
  month = 	 {09--15 Jun},
  publisher = 	 {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/bhagoji19a/bhagoji19a.pdf},
  url = 	 {http://proceedings.mlr.press/v97/bhagoji19a.html}}
}

@article{ghosh2019robust,
    title={Robust Federated Learning in a Heterogeneous Environment},
    author={Avishek Ghosh and Justin Hong and Dong Yin and Kannan Ramchandran},
    month = {jun},
    year={2019},
    journal={arXiv 1906.06629},
   url = {https://arxiv.org/abs/1906.06629}
}

@article{so2020turboaggregate,
    title={Turbo-Aggregate: Breaking the Quadratic Aggregation Barrier in Secure Federated Learning},
    author={Jinhyun So and Basak Guler and A. Salman Avestimehr},
    year={2020},
    month  = {feb},
    journal ={arXiv cs.LG 2002.04156}
}

@article{pillutla2019robust,
    title={Robust Aggregation for Federated Learning},
    author={Krishna Pillutla and Sham M. Kakade and Zaid Harchaoui},
    year={2019},
    journal ={arXiv preprint 1912.13445},
    url = {https://arxiv.org/abs/1912.13445}
}

@inproceedings{wallach2019detox,
title = {DETOX: A Redundancy-based Framework for Faster and More Robust Gradient Aggregation},
author = {Rajput, Shashank and Wang, Hongyi and Charles, Zachary and Papailiopoulos, Dimitris},
booktitle = {Advances in Neural Information Processing Systems 32},
pages = {10320--10330},
year = {2019},
url = {http://papers.nips.cc/paper/9220-detox-a-redundancy-based-framework-for-faster-and-more-robust-gradient-aggregation.pdf}
}

@article{zhao2018federated,
  author    = {Yue Zhao and
               Meng Li and
               Liangzhen Lai and
               Naveen Suda and
               Damon Civin and
               Vikas Chandra},
  title     = {Federated Learning with Non-IID Data},
  journal   = {CoRR arXiv},
  volume    = {abs/1806.00582},
  year      = {2018},
  url       = {http://arxiv.org/abs/1806.00582}
}

@inproceedings{li2014efficient,
 author = {Li, Mu and Zhang, Tong and Chen, Yuqiang and Smola, Alexander J.},
 title = {Efficient Mini-batch Training for Stochastic Optimization},
 booktitle = {Proceedings of the ACM SIGKDD International Conference on Knowledge Discovery and Data Mining},
 year = {2014},
 pages = {661--670},
 numpages = {10}
} 

@article{turing1950computing,
 author = {Alan M. Turing},
 title = {Computing Machinery and Intelligence},
 journal = {Mind},
 volume = {49},
 number  = {236},
 year = {1950},
 pages = {433--460}
} 


@article{rumelhart1986learning,
 author = {David E. Rumelhart and Geoffrey E. Hinton and Ronald J. Williams},
 title = {Learning representations by back-propagating errors},
 journal = {Nature},
 volume = {323},
 month = oct,
 year = {1986},
 pages = {533-536}
} 

@article{rosenblatt1958perceptron,
 author = {Frank Rosenblatt},
 title = {The Perceptron: A Probabilistic Model for Information Storage and Organization in the Brain},
 journal = {Psychological Review},
 volume = {65},
 number  = {6},
 year = {1958},
 pages = {386--408}
} 

@book{minsky69perceptrons,
  title = {Perceptrons: An Introduction to Computational Geometry},
  publisher = {MIT Press},
  address = {Cambridge, MA, USA},
  author = {Minsky, Marvin and Papert, Seymour},
  year = {1969}
}

@article{kingma2015adam,
  author    = {Diederik P. Kingma and
               Jimmy Ba},
  title     = {Adam: {A} Method for Stochastic Optimization},
  journal   = {International Conference on Learning Representations (ICLR)},
  year      = {2015}
}

@book{boyd2004convex,
  title={Convex optimization},
  author={Boyd, Stephen and Vandenberghe, Lieven},
  year={2004},
  publisher={Cambridge university press}
}

@article{bottou2018optimization,
  title={Optimization Methods for Large-scale Machine Learning},
  author={Bottou, L{\'e}on and Curtis, Frank E and Nocedal, Jorge},
  journal={arXiv preprint arXiv:1606.04838},
  month = feb,
  year={2018}
}

@inproceedings{lian2015asynchronous,
  title={Asynchronous Parallel Stochastic Gradient for Nonconvex Optimization},
  author={Lian, Xiangru and Huang, Yijun and Li, Yuncheng and Liu, Ji},
  booktitle={Proceedings of the International Conference on Neural Information Processing Systems},
  pages={2737--2745},
  location={Montreal, Canada},
  year={2015}
}

@inproceedings{lian2017can,
  title={Can Decentralized Algorithms Outperform Centralized Algorithms? A Case Study for Decentralized Parallel Stochastic Gradient Descent},
  author={Xiangru Lian and Ce Zhang and Huan Zhang and Cho-Jui Hsieh and Wei Zhang and Ji Liu},
  booktitle={Proceedings of the International Conference on Neural Information Processing Systems},
pages = {5330--5340},
  location={Montreal, Canada},
  year={2017}
}

@inproceedings{jiang2017collaborative,
  title={Collaborative Deep Learning in Fixed Topology Networks},
  author={Zhanhong Jiang and Aditya Balu and Chinmay Hegde and Soumik Sarkar},
  booktitle={NIPS},
  year={2017}
}

@article{hannah2017more,
  title={More Iterations per Second, Same Quality -- Why Asynchronous Algorithms may Drastically Outperform Traditional Ones},
  author={Robert Hannah and Wotao Yin},
  journal ={arXiv preprint arXiv:1708.05136},
  year={2017},
  month = aug
}

@article{lian2018asynchronous,
  title={Asynchronous Decentralized Parallel Stochastic Gradient Descent},
  author={Xiangru Lian and Wei Zhang and Ce Zhang and Ji Liu},
  journal ={arXiv preprint arXiv:1710.06952},
  year={2018},
  month = feb
}

@article{doan2017convergence,
 author = {Doan, Thinh T.  and Beck, Carolyn L. and Srikant, R.},
 title = {On the Convergence Rate of Distributed Gradient Methods for Finite-Sum Optimization Under Communication Delays},
 journal = {Proc. ACM Measurement and Analysis of Computer Systems},
 issue_date = {December 2017},
 volume = {1},
 number = {2},
 month = dec,
 year = {2017},
 issn = {2476-1249},
 pages = {37:1--37:27},
 articleno = {37},
 numpages = {27},
 url = {http://doi.acm.org/10.1145/3154496},
 doi = {10.1145/3154496},
 acmid = {3154496},
 publisher = {ACM},
 address = {New York, NY, USA}
} 

@article{jin2016how,
  title={How to scale distributed deep learning?},
  author={Peter H. Jin and Qiaochu Yuan and Forrest Iandola and Kurt Keutzer},
  journal={arXiv preprint arXiv:1611.04581},
  year={2016},
  month = nov
}


@inproceedings{gupta2016model,
  title={Model Accuracy and Runtime Tradeoff in Distributed Deep Learning: A Systematic Study},
  author={Gupta, Suyog and Zhang, Wei and Wang, Fei},
  booktitle={IEEE International Conference on Data Mining (ICDM)},
  pages={171--180},
  year={2016},
  location={Barcelona, Spain},
  organization={IEEE}
}

@inproceedings{zhang2015deep,
  title={Deep learning with elastic averaging {SGD}},
  author={Zhang, Sixin and Choromanska, Anna E and LeCun, Yann},
  booktitle={NIPS'15 Proceedings of the 28th International Conference on Neural Information Processing Systems},
  pages={685--693},
  location={Montreal, Canada},
  year={2015}
}

@article{zhang2015staleness,
  title={Staleness-aware Async-SGD for Distributed Deep Learning},
  author={Zhang, Wei and Gupta, Suyog and Lian, Xiangru and Liu, Ji},
  journal={arXiv preprint arXiv:1511.05950},
  year={2015}
}

@inproceedings{cui2014exploiting,
  title={Exploiting Bounded Staleness to Speed Up Big Data Analytics},
  author={Cui, Henggang and Cipar, James and Ho, Qirong and Kim, Jin Kyu and Lee, Seunghak and Kumar, Abhimanu and Wei, Jinliang and Dai, Wei and Ganger, Gregory R and Gibbons, Phillip B and Gibson, Garth A. and Xing, Eric P.},
  booktitle={Proceedings of the {USENIX} Annual Technical Conference},
  pages={37--48},
  location = {Philadelphia, PA},
  year={2014}
}


@article{moritz2015sparknet,
  title={SparkNet: Training deep networks in Spark},
  author={Moritz, Philipp and Nishihara, Robert and Stoica, Ion and Jordan, Michael I},
  journal={arXiv preprint arXiv:1511.06051},
  year={2015}
}


@article{nedic2009distributed,
  title={Distributed subgradient methods for multi-agent optimization},
  author={Nedic, Angelia and Ozdaglar, Asuman},
  journal={IEEE Transactions on Automatic Control},
  volume={54},
  number={1},
  pages={48--61},
  year={2009},
  publisher={IEEE}
}


@article{zeng2016nonconvex,
  title={On nonconvex decentralized gradient descent},
  author={Zeng, Jinshan and Yin, Wotao},
  journal={arXiv preprint arXiv:1608.05766},
  year={2016}
}

@article{sirb2018decentralized,
  title={Decentralized consensus algorithm with delayed and stochastic gradients},
  author={Sirb, Benjamin and Ye, Xiaojing},
  journal={SIAM Journal on Optimization},
  volume={28},
  number={2},
  pages={1232--1254},
  year={2018},
  publisher={SIAM}
}

@article{duchi2012dual,
  title={Dual averaging for distributed optimization: Convergence analysis and network scaling},
  author={Duchi, John C and Agarwal, Alekh and Wainwright, Martin J},
  journal={IEEE Transactions on Automatic control},
  volume={57},
  number={3},
  pages={592--606},
  year={2012},
  publisher={IEEE}
}

@article{jin2016scale,
  title={How to scale distributed deep learning?},
  author={Jin, Peter H and Yuan, Qiaochu and Iandola, Forrest and Keutzer, Kurt},
  journal={arXiv preprint arXiv:1611.04581},
  year={2016}
}


@article{lobel2010distributed,
  title={Distributed subgradient methods for convex optimization over random networks},
  author={Lobel, Ilan and Ozdaglar, Asuman},
  journal={IEEE Transactions on Automatic Control},
  volume={56},
  number={6},
  pages={1291--1306},
  year={2010},
  publisher={IEEE}
}

@article{boyd2006randomized,
  title={Randomized gossip algorithms},
  author={Boyd, Stephen and Ghosh, Arpita and Prabhakar, Balaji and Shah, Devavrat},
  journal={IEEE/ACM Transactions on Networking (TON)},
  volume={14},
  number={SI},
  pages={2508--2530},
  year={2006},
  publisher={IEEE Press}
}


@inproceedings{tsianos2012communication,
  title={Communication/computation tradeoffs in consensus-based distributed optimization},
  author={Tsianos, Konstantinos and Lawlor, Sean and Rabbat, Michael G},
  booktitle={Advances in neural information processing systems},
  pages={1943--1951},
  year={2012}
}

@article{berahas2021convergence_TSP,
  title={On the convergence of nested decentralized gradient methods with multiple consensus and gradient steps},
  author={Berahas, Albert S and Bollapragada, Raghu and Wei, Ermin},
  journal={IEEE Transactions on Signal Processing},
  volume={69},
  pages={4192--4203},
  year={2021},
  publisher={IEEE}
}

@article{iakovidou2022NearDGD_TAC,
  title={{S-NEAR-DGD}: A flexible distributed stochastic gradient method for inexact communication},
  author={Iakovidou, Charikleia and Wei, Ermin},
  journal={IEEE Transactions on Automatic Control},
  year={2022},
  publisher={IEEE}
}

@article{yuan2016convergence,
  title={On the convergence of decentralized gradient descent},
  author={Yuan, Kun and Ling, Qing and Yin, Wotao},
  journal={SIAM Journal on Optimization},
  volume={26},
  number={3},
  pages={1835--1854},
  year={2016},
  publisher={SIAM}
}

@article{woodworth2020minibatch,
  title={Minibatch vs local sgd for heterogeneous distributed learning},
  author={Woodworth, Blake E and Patel, Kumar Kshitij and Srebro, Nati},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={6281--6292},
  year={2020}
}

@article{khanduri2021stem,
  title={Stem: A stochastic two-sided momentum algorithm achieving near-optimal sample and communication complexities for federated learning},
  author={Khanduri, Prashant and Sharma, Pranay and Yang, Haibo and Hong, Mingyi and Liu, Jia and Rajawat, Ketan and Varshney, Pramod},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@inproceedings{li2014scaling,
  title={Scaling distributed machine learning with the parameter server},
  author={Li, Mu and Andersen, David G and Park, Jun Woo and Smola, Alexander J and Ahmed, Amr and Josifovski, Vanja and Long, James and Shekita, Eugene J and Su, Bor-Yiing},
  booktitle={11th USENIX Symposium on Operating Systems Design and Implementation (OSDI 14)},
  pages={583--598},
  year={2014}
}

@article{konecny2016federated,
  title={Federated learning: Strategies for improving communication efficiency},
  author={Kone{\v{c}}n{\`y}, Jakub and McMahan, H Brendan and Yu, Felix X and Richt{\'a}rik, Peter and Suresh, Ananda Theertha and Bacon, Dave},
  journal={arXiv preprint arXiv:1610.05492},
  year={2016}
}

@inproceedings{jakovetic2018convergence,
  title={Convergence rates for distributed stochastic optimization over random networks},
  author={Jakovetic, Dusan and Bajovic, Dragana and Sahu, Anit Kumar and Kar, Soummya},
  booktitle={2018 IEEE Conference on Decision and Control (CDC)},
  pages={4238--4245},
  year={2018},
  organization={IEEE}
}


@inproceedings{misra1992constructive,
  title={A constructive proof of Vizing’s theorem},
  author={Misra, Jayadev and Gries, David},
  booktitle={Information Processing Letters},
  year={1992},
  organization={Citeseer}
}


@inproceedings{scaman2018optimal,
  title={Optimal algorithms for non-smooth distributed optimization in networks},
  author={Scaman, Kevin and Bach, Francis and Bubeck, S{\'e}bastien and Massouli{\'e}, Laurent and Lee, Yin Tat},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2740--2749},
  year={2018}
}

@inproceedings{wang2018atomo,
  title={Atomo: Communication-efficient learning via atomic sparsification},
  author={Wang, Hongyi and Sievert, Scott and Liu, Shengchao and Charles, Zachary and Papailiopoulos, Dimitris and Wright, Stephen},
  booktitle={Advances in Neural Information Processing Systems},
  pages={9850--9861},
  year={2018}
}

@inproceedings{chao2020GKT,
  title={Group Knowledge Transfer: Federated Learning of Large CNNs at the Edge},
  author={Chaoyang He and Murali Annavaram and Salman Avestimehr},
  booktitle={Advances in Neural Information Processing Systems},
  year={2020}
}

@InProceedings{yin2018gradient,
  title = 	 {Gradient Diversity: a Key Ingredient for Scalable Distributed Learning},
  author = 	 {Dong Yin and Ashwin Pananjady and Max Lam and Dimitris Papailiopoulos and Kannan Ramchandran and Peter Bartlett},
  booktitle = 	 {Proceedings of the Twenty-First International Conference on Artificial Intelligence and Statistics},
  pages = 	 {1998--2007},
  year = 	 {2018},
  volume = 	 {84},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--11 Apr},
  pdf = 	 {http://proceedings.mlr.press/v84/yin18a/yin18a.pdf},
  url = 	 {http://proceedings.mlr.press/v84/yin18a.html}
}

@inproceedings{li2019on,
  title={On the Convergence of FedAvg on Non-IID Data},
  author={Xiang Li and Kaixuan Huang and Wenhao Yang and Shusen Wang and Zhihua Zhang},
  booktitle = {International Conference on Learning Representations (ICLR)},
  url={https://arxiv.org/abs/1907.02189},
  month = jul,
  year={2020}
}

@inproceedings{sahu2019federated,
  title={Federated Optimization for Heterogeneous Networks},
  author={Anit Kumar Sahu and Tian Li and Maziar Sanjabi and Manzil Zaheer and Ameet Talwalkar and Virginia Smith},
  booktitle={Proceedings of the 3rd MLSys Conference},
  month = jan,
  year={2020}
}



@inproceedings{hsu2019noniid,
title={Measuring the Effects of Non-Identical Data
Distribution for Federated Visual Classification},
author={Tzu-Ming Harry Hsu and Hang Qi and Matthew Brown},
booktitle={International Workshop on Federated Learning for User Privacy and Data Confidentiality
in Conjunction with NeurIPS 2019 (FL-NeurIPS'19)},
month=dec,
year={2019}
}

@article{xiao2017fmnist,
  author       = {Han Xiao and Kashif Rasul and Roland Vollgraf},
  title        = {Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms},
  year         = {2017},
  journal={https://arxiv.org/abs/1708.07747},
  month = {aug}
}

@inproceedings{pennington2014glove,
  author = {Jeffrey Pennington and Richard Socher and Christopher D. Manning},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {GloVe: Global Vectors for Word Representation},
  year = {2014},
  pages = {1532--1543},
  url = {http://www.aclweb.org/anthology/D14-1162},
}

@ARTICLE{bijral2017data, 
author={A. S. {Bijral} and A. D. {Sarwate} and N. {Srebro}}, 
journal={IEEE Transactions on Automatic Control}, 
title={Data-Dependent Convergence for Consensus Stochastic Optimization}, 
year={2017}, 
volume={62}, 
number={9}, 
pages={4483-4498}, 
ISSN={0018-9286}, 
month= sep
}

@article{wangni2017gradient,
  title={Gradient sparsification for communication-efficient distributed optimization},
  author={Wangni, Jianqiao and Wang, Jialei and Liu, Ji and Zhang, Tong},
  journal={arXiv preprint arXiv:1710.09854},
  year={2017}
}

@article{bernstein2018signsgd,
  title={signSGD: compressed optimisation for non-convex problems},
  author={Bernstein, Jeremy and Wang, Yu-Xiang and Azizzadenesheli, Kamyar and Anandkumar, Anima},
  journal={arXiv preprint arXiv:1802.04434},
  year={2018}
}

@misc{rothchild2020fetchsgd,
    title={FetchSGD: Communication-Efficient Federated Learning with Sketching},
    author={Daniel Rothchild and Ashwinee Panda and Enayat Ullah and Nikita Ivkin and Ion Stoica and Vladimir Braverman and Joseph Gonzalez and Raman Arora},
    year={2020},
    eprint={2007.07682},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}

@inproceedings{ivkin2019communication,
title = {Communication-efficient Distributed SGD with Sketching},
author = {Ivkin, Nikita and Rothchild, Daniel and Ullah, Enayat and braverman, Vladimir and Stoica, Ion and Arora, Raman},
booktitle = {Advances in Neural Information Processing Systems 32},
pages = {13144--13154},
year = {2019},
url = {http://papers.nips.cc/paper/9473-communication-efficient-distributed-sgd-with-sketching.pdf}
}


@article{wen2017terngrad,
  title={TernGrad: Ternary Gradients to Reduce Communication in Distributed Deep Learning},
  author={Wen, Wei and Xu, Cong and Yan, Feng and Wu, Chunpeng and Wang, Yandan and Chen, Yiran and Li, Hai},
  journal={arXiv preprint arXiv:1705.07878},
  month = may,
  year={2017}
}

@article{lin2017deep,
  title={Deep gradient compression: Reducing the communication bandwidth for distributed training},
  author={Lin, Yujun and Han, Song and Mao, Huizi and Wang, Yu and Dally, William J},
  journal={arXiv preprint arXiv:1712.01887},
  year={2017}
}

@article{mania2015perturbed,
  title={Perturbed Iterate Analysis for Asynchronous Stochastic Optimization},
  author={Horia Mania and Xinghao Pan and Dimitris Papailiopoulos and Benjamin Recht and Kannan Ramchandran and Michael I. Jordan},
  journal={arXiv preprint arXiv:1507.06970},
  month = jul,
  year={2015}
}

@article{polyak1964some,
  title={Some methods of speeding up the convergence of iteration methods},
  author={Boris T Polyak},
  journal={USSR Computational Mathematics and Mathematical Physics},
volume = {4},
number = {5},
pages = {1--17},
  year={1964}
}

@article{nesterov1983method,
  title={A method of solving a convex programming problem with convergence rate $o(1/k^2)$},
  author={Yurii Nesterov},
  journal={Soviet Mathematics Doklady},
volume = {27},
number = {5},
pages = {372--376},
  year={1983}
}

@inproceedings{allen-zhu12017katyusha,
  title={Katyusha: The First Direct Acceleration of Stochastic Gradient Methods},
  author={Zeyuan Allen-Zhu},
 booktitle={Proceedings of Symposium on Theory of Computing (STOC)},
 pages = {1200--1205},
  year={2017}
}

@article{zhou2017convergence,
  title={On the Convergence Properties of a $ K $-step Averaging Stochastic Gradient Descent Algorithm for Nonconvex Optimization},
  author={Zhou, Fan and Cong, Guojing},
  journal={arXiv preprint arXiv:1708.01012},
  year={2017}
}


@article{chaudhari2017parle,
  title={Parle: parallelizing stochastic gradient descent},
  author={Chaudhari, Pratik and Baldassi, Carlo and Zecchina, Riccardo and Soatto, Stefano and Talwalkar, Ameet and Oberman, Adam},
  journal={arXiv preprint arXiv:1707.00424},
  year={2017}
}

@article{ruder2016overview,
  title={An Overview of Gradient Descent Optimization Algorithms},
  author={Ruder, Sebastian},
  journal={arXiv preprint arXiv:1609.04747},
  year={2016}
}

@article{haddadpour2019convergence,
  title={On the Convergence of Local Descent Methods in Federated Learning},
  author={Haddadpour, Farzin and Mahdavi, Mehrdad},
  journal={arXiv preprint arXiv:1910.14425},
  year={2019}
}

@inproceedings{haddadpour2019local,
  title={Local {SGD} with periodic averaging: Tighter analysis and adaptive synchronization},
  author={Haddadpour, Farzin and Kamani, Mohammad Mahdi and Mahdavi, Mehrdad and Cadambe, Viveck},
  booktitle={Advances in Neural Information Processing Systems},
  pages={11080--11092},
  year={2019}
}


@InProceedings{haddadpour2019trading,
  title = 	 {Trading Redundancy for Communication: Speeding up Distributed {SGD} for Non-convex Optimization},
  author = 	 {Haddadpour, Farzin and Kamani, Mohammad Mahdi and Mahdavi, Mehrdad and Cadambe, Viveck},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {2545--2554},
  year = 	 {2019},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--15 Jun},
  publisher = 	 {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/haddadpour19a/haddadpour19a.pdf},
  url = 	 {http://proceedings.mlr.press/v97/haddadpour19a.html}
}

@article{boyd2011distributed,
 author = {Boyd, Stephen and Parikh, Neal and Chu, Eric and Peleato, Borja and Eckstein, Jonathan},
 title = {Distributed Optimization and Statistical Learning via the Alternating Direction Method of Multipliers},
 journal = {Foundations and Trends in Machine Learning},
 volume = {3},
 number = {1},
 month = jan,
 year = {2011},
 pages = {1--122},
 numpages = {122}
}

@article{parikh2014proximal,
 author = {Parikh, Neal and Boyd, Stephen},
 title = {Proximal Algorithms},
 journal = {Foundations and Trends in Optimization},
 volume = {1},
 number = {3},
 month = jan,
 year = {2014},
 pages = {127--239},
 numpages = {113},
 publisher = {Now Publishers Inc.},
 address = {Hanover, MA, USA}
} 

@inproceedings{mitliagkas2016asynchrony,
  title={Asynchrony begets momentum, with an application to deep learning},
  author={Mitliagkas, Ioannis and Zhang, Ce and Hadjis, Stefan and R{\'e}, Christopher},
  booktitle={54th Annual Allerton Conference on Communication, Control, and Computing (Allerton)},
  pages={997--1004},
  year={2016},
  organization={IEEE}
}

@InProceedings{tandon2017gradient,
  title =    {Gradient Coding: Avoiding Stragglers in Distributed Learning},
  author =   {Rashish Tandon and Qi Lei and Alexandros G. Dimakis and Nikos Karampatziakis},
  booktitle =    {Proceedings of the 34th International Conference on Machine Learning},
  pages =    {3368--3376},
  year =   {2017},
  volume =   {70},
  series =   {Proceedings of Machine Learning Research}
}

@INPROCEEDINGS{karakus2017encoded, 
author={C. Karakus and Y. Sun and S. Diggavi}, 
booktitle={IEEE International Symposium on Information Theory (ISIT)}, 
title={Encoded distributed optimization}, 
year={2017}, 
volume={}, 
number={}, 
pages={2890-2894},
month={June}
}

@InProceedings{raviv2018gradient,
  title = 	 {Gradient Coding from Cyclic {MDS} Codes and Expander Graphs},
  author = 	 {Raviv, Netanel and Tandon, Rashish and Dimakis, Alex and Tamo, Itzhak},
  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
  pages = 	 {4302--4310},
  year = 	 {2018},
  volume = 	 {80},
  month = 	 {10--15 Jul},
  publisher = 	 {PMLR},
  url = 	 {http://proceedings.mlr.press/v80/raviv18a.html}
}

@InProceedings{maity2018robust,
  title = 	 {Robust Gradient Descent via Moment Encoding with LDPC Codes},
  author = 	 {Raj Kumar Maity and  Ankit Singh Rawat and Arya Mazumdar},
  booktitle = 	 {Proceedings of the SysML conference},
  year = 	 {2018},
  month = 	 {feb}
}


@inproceedings{chen2014dadiannao,
 author = {Chen, Yunji and Luo, Tao and Liu, Shaoli and Zhang, Shijin and He, Liqiang and Wang, Jia and Li, Ling and Chen, Tianshi and Xu, Zhiwei and Sun, Ninghui and Temam, Olivier},
 title = {DaDianNao: A Machine-Learning Supercomputer},
 booktitle = {Proceedings of the IEEE/ACM International Symposium on Microarchitecture},
 year = {2014},
 pages = {609--622},
 numpages = {14}
} 

@article{pan2016hemingway,
  title         = {Hemingway: Modeling Distributed Optimization Algorithms},
  author        = {Pan, Xinghao
                   and Venkataraman, Shivaram
                   and Tai, Zizheng
                   and Gonzalez, Joseph},
  booktitle     = {NIPS Workshop on Machine Learning Systems (MLSys)},
  year          = {2016},
  month         = {December}
}

@incollection{recht2011hogwild,
title = {Hogwild: A Lock-Free Approach to Parallelizing Stochastic Gradient Descent},
author = {Recht, Benjamin and Re, Christopher and Wright, Stephen and Feng Niu},
booktitle = {Proceedings of the International Conference on Neural Information Processing Systems},
pages = {693--701},
year = {2011}
}

@inproceedings{sa2015taming,
 author = {Sa, Christopher De and Zhang, Ce and Olukotun, Kunle and R{\'e}, Christopher},
 title = {Taming the Wild: A Unified Analysis of HOG WILD! -style Algorithms},
 booktitle = {Proceedings of the 28th International Conference on Neural Information Processing Systems},
 year = {2015},
 pages = {2674--2682},
 numpages = {9},
 url = {http://dl.acm.org/citation.cfm?id=2969442.2969538}
} 

@inproceedings{dean2012large,
 author = {Dean, Jeffrey and Corrado, Greg S. and Monga, Rajat and Chen, Kai and Devin, Matthieu and Le, Quoc V. and Mao, Mark Z. and Ranzato, Marc'Aurelio and Senior, Andrew and Tucker, Paul and Yang, Ke and Ng, Andrew Y.},
 title = {Large Scale Distributed Deep Networks},
 booktitle = {Proceedings of the International Conference on Neural Information Processing Systems},
 year = {2012},
 pages = {1223--1231}
}


@article{duchi2011adaptive,
 author = {John Duchi and Elad Hazan and Yoram Singer},
 title = "{Adaptive Subgradient Methods for Online Learning and Stochastic Optimization}",
 journal = {Journal of Machine Learning Research},
 volume = {12},
 month = jul,
 year = {2011},
 pages = {2121--2159},
 numpages = {39}
}


@inproceedings{wang2019osp,
  title={OSP: Overlapping Computation and Communication in Parameter Server for Fast Machine Learning},
  author={Wang, Haozhao and Guo, Song and Li, Ruixuan},
  booktitle={Proceedings of the 48th International Conference on Parallel Processing},
  pages={1--10},
  year={2019}
}


@article{dutta2018slow,
 author = {Sanghamitra Dutta and Gauri Joshi and Soumyadip Ghosh and Parijat Dube and Priya Nagpurkar},
 title = "{Slow and Stale Gradients Can Win the Race: Error-Runtime Trade-offs in Distributed SGD}",
 journal = {International Conference on Artificial Intelligence and Statistics (AISTATS)},
 year = {2018},
 month = apr,
 url = {https://arxiv.org/abs/1803.01113}
}

@article{wang2018cooperative,
  title={Cooperative {SGD}: A unified framework for the design and analysis of communication-efficient {SGD} algorithms},
  author={Wang, Jianyu and Joshi, Gauri},
  journal={Journal of Machine Learning Research (JMLR)},
  year={2021},
  url  = {https://arxiv.org/abs/1808.07576}
}


@inproceedings{wang2020slowmo,
title={{SlowMo}: Improving Communication-Efficient Distributed {SGD} with Slow Momentum},
author={Jianyu Wang and Vinayak Tantia and Nicolas Ballas and Michael Rabbat},
booktitle={International Conference on Learning Representations},
year={2020},
url={https://openreview.net/forum?id=SkxJ8REYPH}
}

@inproceedings{zhang2020machine,
  	author        = {Xiaoxi Zhang and Jianyu Wang and Gauri Joshi and Carlee Joe-Wong},
  	title         = "{Machine Learning on Volatile Instances}",
  	booktitle = "{IEEE International Conference on Computer Communications (INFOCOM)}",
	month         =  apr,
	year          = 2020,
    keywords = {aconference}
}

@inproceedings{li2019fedmd,
    title={FedMD: Heterogenous Federated Learning via Model Distillation},
    author={Daliang Li and Junpu Wang},
    year={2019},
    booktitle={International Workshop on Feder-ated Learning for User Privacy and Data Confidentiality inConjunction with NeurIPS 2019 (FL-NeurIPS’19)}
}

@misc{to2019stabi,
    title={Online Learning, Stability, and Stochastic Gradient Descent},
    author={Tomaso Poggio and Stephen Voinea and Lorenzo Rosasco},
    year={2019},
    eprint={1105.4701},
    archivePrefix={arXiv}
}


@inproceedings{lin2020ensemble,
    title={Ensemble Distillation for Robust Model Fusion in Federated Learning},
    author={Tao Lin and Lingjing Kong and Sebastian U. Stich and Martin Jaggi},
    year={2020},
    booktitle={Advances in Neural Information Processing Systems},
    url = {https://arxiv.org/abs/2006.07242}
}

@InProceedings{ham2020fedboost,
    title={FedBoost: Communication-Efficient Algorithms for Federated Learning},
    author={Jenny Hamer and Mehryar Mohri and Ananda Theertha Suresh},
    year={2020},
    booktitle = 	 {Proceedings of the International Conference on Machine Learning},
  pdf = 	 {https://proceedings.icml.cc/static/paper_files/icml/2020/5967-Paper.pdf}
}




@inproceedings{vogels2019powersgd,
    title={{PowerSGD}: Practical Low-Rank Gradient Compression for Distributed Optimization},
    author={Vogels, Thijs and Karimireddy, Sai Praneeth and Jaggi, Martin},
    booktitle={Advances in Neural Information Processing Systems},
    year={2019}
}


@inproceedings{wang2020overlap,
	title = "{Overlap Local-SGD: An Algorithmic Approach to Hide Communication Delays in Distributed SGD}",
	booktitle   = "{Proceedings of the International Conference on Acoustics, Speech, and Signal Processing (ICASSP)}",
	url         = {"https://arxiv.org/abs/1808.07576"},
	author      = {Jianyu Wang and Hao Liang and Gauri Joshi},
	month       = may,
	year        = "2020"
}


@article{wang2019matcha,
 author = {Jianyu Wang and Anit Sahu and Gauri Joshi and Soummya Kar},
 title = "{MATCHA: Speeding Up Decentralized SGD via Matching Decomposition Sampling}",
 journal = {preprint},
 year = {2019},
 month = may,
  url = {https://arxiv.org/abs/1905.09435}
}

@article{lin2021flpos,
 author = {Xinyang Lin and and Hanting Chen and Yixing Xu and Chao Xu and Xiaolin Gui and Yiping Deng and Yunhe Wang},
 title = "{Federated Learning with Positive and Unlabeled Data}",
 journal = {preprint},
 year = {2021},
  url = {https://arxiv.org/abs/2106.10904}
}


@inproceedings{wang2018adaptive,
 author = {Jianyu Wang and Gauri Joshi},
 title = "{Adaptive Communication Strategies for Best Error-Runtime Trade-offs in Communication-Efficient Distributed SGD}",
 booktitle = {Proceedings of the SysML Conference},
 year = {2019},
 month = apr,
 url = {https://arxiv.org/abs/1810.08313}
}


@ARTICLE{course18847_2017,
   author = {{Gauri Joshi}},
    title = "{18-847F Course Website}",
     year = 2017,
     url = {http://www.andrew.cmu.edu/user/gaurij/18-847F-Fall-2017.html},
}

@ARTICLE{course18847_2018,
   author = {{Gauri Joshi}},
    title = "{18-847F Course Website}",
     year = 2017,
     url = {http://www.andrew.cmu.edu/user/gaurij/18-847F-Fall-2018.html},
}

@article{hart1979kmeans,
 author = {J. A. Hartigan and M. A. Wong},
 journal = {Journal of the Royal Statistical Society. Series C (Applied Statistics)},
 number = {1},
 pages = {100--108},
 publisher = {[Wiley, Royal Statistical Society]},
 title = {Algorithm AS 136: A K-Means Clustering Algorithm},
 volume = {28},
 year = {1979}
}




@article{nedic2009distributed,
  title={Distributed subgradient methods for multi-agent optimization},
  author={Nedic, Angelia and Ozdaglar, Asuman},
  journal={IEEE Transactions on Automatic Control},
  volume={54},
  number={1},
  pages={48--61},
  year={2009},
  publisher={IEEE}
}


@article{zeng2016nonconvex,
  title={On nonconvex decentralized gradient descent},
  author={Zeng, Jinshan and Yin, Wotao},
  journal={arXiv preprint arXiv:1608.05766},
  year={2016}
}

@article{sirb2018decentralized,
  title={Decentralized consensus algorithm with delayed and stochastic gradients},
  author={Sirb, Benjamin and Ye, Xiaojing},
  journal={SIAM Journal on Optimization},
  volume={28},
  number={2},
  pages={1232--1254},
  year={2018},
  publisher={SIAM}
}

@article{duchi2012dual,
  title={Dual averaging for distributed optimization: Convergence analysis and network scaling},
  author={Duchi, John C and Agarwal, Alekh and Wainwright, Martin J},
  journal={IEEE Transactions on Automatic control},
  volume={57},
  number={3},
  pages={592--606},
  year={2012},
  publisher={IEEE}
}

@article{jin2016scale,
  title={How to scale distributed deep learning?},
  author={Jin, Peter H and Yuan, Qiaochu and Iandola, Forrest and Keutzer, Kurt},
  journal={arXiv preprint arXiv:1611.04581},
  year={2016}
}


@inproceedings{tsianos2012communication,
  title={Communication/computation tradeoffs in consensus-based distributed optimization},
  author={Tsianos, Konstantinos and Lawlor, Sean and Rabbat, Michael G},
  booktitle={Advances in neural information processing systems},
  pages={1943--1951},
  year={2012}
}

@article{yuan2016convergence,
  title={On the convergence of decentralized gradient descent},
  author={Yuan, Kun and Ling, Qing and Yin, Wotao},
  journal={SIAM Journal on Optimization},
  volume={26},
  number={3},
  pages={1835--1854},
  year={2016},
  publisher={SIAM}
}

@article{zagoruyko2016wide,
  title={Wide residual networks},
  author={Zagoruyko, Sergey and Komodakis, Nikos},
  journal={arXiv preprint arXiv:1605.07146},
  year={2016}
}

@article{mcmahan2017communication,
 author = {H. Brendan McMahan and Eider Moore and Daniel Ramage and Seth Hampson and Blaise Ag\o{u}ra y Arcas},
 title = "{Communication-Efficient Learning of Deep Networks from Decentralized Data}",
 journal = {International Conference on Artificial Intelligenece and Statistics (AISTATS)},
 year = {2017},
 month = apr,
 url = {https://arxiv.org/abs/1602.05629}
}



@article{kairouz2019advances,
  title={Advances and open problems in federated learning},
  author={Peter Kairouz and H. Brendan McMahan and Brendan Avent and Aurelien Bellet et. al.},
  journal={arXiv preprint arXiv:1912.04977},
  year={2019}
}

@article{bonawitz2019towards,
 author = {Keith Bonawitz and Hubert Eichner and Wolfgang Grieskamp and Dzmitry Huba and Alex Ingerman and Vladimir Ivanov and Chloe Kiddon and Jakub Konecny and Stefano Mazzocchi and H. Brendan McMahan and Timon Van Overveldt and David Petrou and Daniel Ramage and Jason Roselander},
 title = "{Towards Federated Learning at Scale: System Design}",
 journal = {SysML},
 year = {2019},
 month = apr,
 url = {https://www.sysml.cc/doc/2019/193.pdf}
}

@inproceedings{bonawitz2016practical,
title	= {Practical Secure Aggregation for Federated Learning on User-Held Data},
author	= {Keith Bonawitz and Vladimir Ivanov and Ben Kreuter and Antonio Marcedone and H. Brendan McMahan and Sarvar Patel and Daniel Ramage and Aaron Segal and Karn Seth},
year	= {2016},
booktitle	= {NIPS Workshop on Private Multi-Party Machine Learning}
}



@incollection{smith2017federated,
title = {Federated Multi-Task Learning},
author = {Smith, Virginia and Chiang, Chao-Kai and Sanjabi, Maziar and Talwalkar, Ameet S},
booktitle = {Advances in Neural Information Processing Systems},
pages = {4424--4434},
year = {2017},
url = {http://papers.nips.cc/paper/7029-federated-multi-task-learning.pdf}
}

@article{smith2018cocoa,
  title={CoCoA: A general framework for communication-efficient distributed optimization},
  author={Smith, Virginia and Forte, Simone and Chenxin, Ma and Tak{\'a}{\v{c}}, Martin and Jordan, Michael I and Jaggi, Martin},
  journal={Journal of Machine Learning Research},
  volume={18},
  pages={230},
  year={2018},
  publisher={MIT press}
}

@article{smith2016cocoa,
  author    = {Virginia Smith and Simone Forte and Chenxin Ma and Martin Tak{\'{a}}c and Michael I. Jordan and Martin Jaggi},
  title     = {{CoCoA}: {A} General Framework for Communication-Efficient Distributed
               Optimization},
  journal   = {CoRR},
  volume    = {abs/1611.02189},
  year      = {2016},
  url       = {http://arxiv.org/abs/1611.02189}
}


@article{yu2018parallel,
  title={Parallel Restarted {SGD} for Non-Convex Optimization with Faster Convergence and Less Communication},
  author={Yu, Hao and Yang, Sen and Zhu, Shenghuo},
  journal={The Thirty-Third AAAI Conference on Artificial Intelligence (AAAI-19)},
  year={2019}
}

@inproceedings{stich2018sparsified,
  title={Sparsified sgd with memory},
  author={Stich, Sebastian U and Cordonnier, Jean-Baptiste and Jaggi, Martin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4447--4458},
  year={2018}
}

@inproceedings{sun2021fldistprivacy,
  title={Federated Model Distillation with Noise-Free Differential Privacy},
  author={Lichao Sun and Lingjuan Lyu},
  booktitle={Proceedings of the Thirtieth International Joint Conference on Artificial Intelligence (IJCAI-21)},
  year={2021}
}

@article{lyu2020flprivacy,
  title={Privacy and Robustness in Federated Learning: Attacks and Defenses},
  author={Lingjuan Lyu and Han Yu and Xingjun Ma and Lichao Sun and Jun Zhao and Qiang Yang and Philip S. Yu},
  journal={arXiv preprint arXiv:2012.06337},
  year={2020}
}

@article{huo2020faster,
  title={Faster On-Device Training Using New Federated Momentum Algorithm},
  author={Zhouyuan Huo and Qian Yang and Bin Gu and Lawrence Carin and Heng Huang},
  journal={arXiv preprint arXiv:2002.02090},
  year={2020}
}


@article{zhou2019distributed,
  title={A Distributed Hierarchical {SGD} Algorithm with Sparse Global Reduction},
  author={Zhou, Fan and Cong, Guojing},
  journal={arXiv preprint arXiv:1903.05133},
  year={2019}
}


@inproceedings{malinovsky2020local,
  title={From Local {SGD} to Local Fixed Point Methods for Federated Learning},
  author={Malinovsky, Grigory and Kovalev, Dmitry and Gasanov, Elnur and Condat, Laurent and Richt\'arik, Peter},
  booktitle={Proceedings of the 37th International Conference on Machine Learning},
  year={2020}
}

@misc{charles2020outsized,
    title={On the Outsized Importance of Learning Rates in Local Update Methods},
    author={Zachary Charles and Jakub Konecny},
    year={2020},
    eprint={2007.00878},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}


@article{liang2019variance,
  title={Variance Reduced Local {SGD} with Lower Communication Complexity},
  author={Liang, Xianfeng and Shen, Shuheng and Liu, Jingchang and Pan, Zhen and Chen, Enhong and Cheng, Yifei},
  journal={arXiv preprint arXiv:1912.12844},
  year={2019}
}

@article{karimireddy2019scaffold,
  title={{SCAFFOLD}: Stochastic controlled averaging for on-device federated learning},
  author={Karimireddy, Sai Praneeth and Kale, Satyen and Mohri, Mehryar and Reddi, Sashank J and Stich, Sebastian U and Suresh, Ananda Theertha},
  journal={arXiv preprint arXiv:1910.06378},
  year={2019}
}

@inproceedings{finn2017model,
author = {Finn, Chelsea and Abbeel, Pieter and Levine, Sergey},
title = {Model-Agnostic Meta-Learning for Fast Adaptation of Deep Networks},
year = {2017},
publisher = {JMLR.org},
booktitle = {Proceedings of the 34th International Conference on Machine Learning - Volume 70},
pages = {1126–1135},
numpages = {10},
location = {Sydney, NSW, Australia},
series = {ICML’17}
}

@article{jiang2019improving,
    title={Improving Federated Learning Personalization via Model Agnostic Meta Learning},
    author={Yihan Jiang and Jakub Konecny and Keith Rush and Sreeram Kannan},
    year={2019},
    journal ={arXiv 1909.12488}
}

@inproceedings{fallah2020personalized,
    title={Personalized Federated Learning with Theoretical Guarantees: A Model-Agnostic Meta-Learning Approach},
    author={Alireza Fallah and Aryan Mokhtari and Asuman Ozdaglar},
    year={2020},
    booktitle ={Advances in Neural Information Processing Systems},
}




@misc{huang2020personalized,
    title={Personalized Federated Learning: An Attentive Collaboration Approach},
    author={Yutao Huang and Lingyang Chu and Zirui Zhou and Lanjun Wang and Jiangchuan Liu and Jian Pei and Yong Zhang},
    year={2020},
    eprint={2007.03797},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}

@misc{deng2020adaptive,
    title={Adaptive Personalized Federated Learning},
    author={Yuyang Deng and Mohammad Mahdi Kamani and Mehrdad Mahdavi},
    year={2020},
    eprint={2003.13461},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}

@article{mansour2020approaches,
    title={Three Approaches for Personalization with Applications to Federated Learning},
    author={Yishay Mansour and Mehryar Mohri and Jae Ro and Ananda Theertha Suresh},
    year={2020},
    journal={arXiV preprint cs.LG 2002.10619},
    url = {https://arxiv.org/abs/2002.10619}
}

@inproceedings{li2021ditto,
title = {Ditto: Fair and Robust Federated Learning Through Personalization},
author={Tian Li and Shengyuan Hu and Ahmad Beirami and Virginia Smith},
year = {2021},
booktitle={Proceedings of the 38th International Conference on Machine Learning},
    url = {https://arxiv.org/abs/2012.04221}
}



@article{yjc2020csfl,
  author    = {Yae Jee Cho and Jianyu Wang and Gauri Joshi},
  title     = {Client Selection in Federated Learning: Convergence Analysis and Power-of-Choice Selection Strategies},
   journal={to appear in The 25rd International Conference on Artificial Intelligence and Statistics (AISTATS)},
   year = {2022}
}

@article{yjc2021perfl,
  author    = {Yae Jee Cho and Jianyu Wang and Tarun Chiruvolu and Gauri Joshi},
  title     = {Personalized Federated Learning for Heterogeneous Clients with Clustered Knowledge Transfer},
  journal={arXiv preprint arXiv:2109.08119},
  year = {2021},
  url       = {http://arxiv.org/abs/2109.08119}
}

@inProceedings{mohri2019agnostic,
  title = 	 {Agnostic Federated Learning},
  author = 	 {Mohri, Mehryar and Sivek, Gary and Suresh, Ananda Theertha},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {4615--4625},
  year = 	 {2019},
  editor = 	 {Chaudhuri, Kamalika and Salakhutdinov, Ruslan},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Long Beach, California, USA},
  month = 	 {09--15 Jun},
  publisher = 	 {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/mohri19a/mohri19a.pdf}
}

@misc{li2020feddane,
    title={FedDANE: A Federated Newton-Type Method},
    author={Tian Li and Anit Kumar Sahu and Manzil Zaheer and Maziar Sanjabi and Ameet Talwalkar and Virginia Smith},
    year={2020},
    eprint={2001.01920},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}

@misc{li2020tilted,
    title={Tilted Empirical Risk Minimization},
    author={Tian Li and Ahmad Beirami and Maziar Sanjabi and Virginia Smith},
    year={2020},
    eprint={2007.01162},
    archivePrefix={arXiv},
    url ={https://arxiv.org/abs/2007.01162},
    primaryClass={cs.LG}
}

@inproceedings{li2019fair,
  title={Fair resource allocation in federated learning},
  author={Li, Tian and Sanjabi, Maziar and Smith, Virginia},
   booktitle = {International Conference on Learning Representations (ICLR)},
  year={2020}
}


@inproceedings{woodworth2020local,
  title={Is Local {SGD} Better than Minibatch {SGD}?},
  author={Woodworth, Blake and Patel, Kumar Kshitij and Stich, Sebastian U and Dai, Zhen and Bullins, Brian and McMahan, H Brendan and Shamir, Ohad and Srebro, Nathan},
  booktitle={Proceedings of the 37th International Conference on Machine Learning},
  year={2020}
}


@inproceedings{dinh2020pfedme,
  title={Personalized Federated Learning with Moreau Envelopes},
  author={Canh T. Dinh and Nguten H. Tran and Tuan Dung Nguyen},
  booktitle={Advances in Neural Information Processing Systems},
  year={2020}
}

@inproceedings{doso2021vit,
  title={An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale},
  author={Alexey Dosovitskiy and Lucas Beyer and Alexander Kolesnikov and Dirk Weissenborn and Xiaohua Zhai and Thomas Unterthiner and Mostafa Dehghani and Matthias Minderer and Georg Heigold and Sylvain Gelly and Jakob Uszkoreit and Neil Houlsby},
  booktitle={ICLR},
  year={2021}
}

@inproceedings{zha2021fedfomo,
  title={Personalized Federated Learning with
First Order Model Optimization},
  author={M. Zhang and K. Sapra and S. Fidler and S. Yeung and J. M. Alvarez},
  booktitle={International Conference on Learning Representations (ICLR)},
  year={2021}
}

@inproceedings{dieuleveut2019communication,
  title={Communication trade-offs for Local-{SGD} with large step size},
  author={Dieuleveut, Aymeric and Patel, Kumar Kshitij},
  booktitle={Advances in Neural Information Processing Systems},
  pages={13579--13590},
  year={2019}
}

@inproceedings{pathak2020fedsplit,
  title={{FedSplit}: An algorithmic framework for fast federated optimization},
  author={Pathak, Reese and Wainwright, Martin J},
  booktitle={Advances in Neural Information Processing Systems},
  year={2020}
}

@inproceedings{khaled2020tighter,
  title={Tighter theory for local {SGD} on identical and heterogeneous data},
  author={Khaled, A and Mishchenko, K and Richt{\'a}rik, P},
  booktitle={The 23rd International Conference on Artificial Intelligence and Statistics (AISTATS 2020)},
  year={2020}
}

@article{li2021destress_arxiv,
  title={{DESTRESS}: Computation-Optimal and Communication-Efficient Decentralized Nonconvex Finite-Sum Optimization},
  author={Li, Boyue and Li, Zhize and Chi, Yuejie},
  journal={arXiv preprint arXiv:2110.01165},
  year={2021}
}

@inproceedings{koloskova2020unified,
  title={A Unified Theory of Decentralized {SGD} with Changing Topology and Local Updates},
  author={Koloskova, Anastasia and Loizou, Nicolas and Boreiri, Sadra and Jaggi, Martin and Stich, Sebastian U},
  booktitle={Proceedings of 37th International Conference on Machine Learning},
  year={2020}
}

@article{koloskova2019decentralized,
  title={Decentralized Stochastic Optimization and Gossip Algorithms with Compressed Communication},
  author={Koloskova, Anastasia and Stich, Sebastian U and Jaggi, Martin},
  journal={arXiv preprint arXiv:1902.00340},
  year={2019}
}









@InProceedings{resisizadeh2020fedpaq,
  title = 	 {FedPAQ: A Communication-Efficient Federated Learning Method with Periodic Averaging and Quantization},
  author = 	 {Reisizadeh, Amirhossein and Mokhtari, Aryan and Hassani, Hamed and Jadbabaie, Ali and Pedarsani, Ramtin},
  booktitle = 	 {Proceedings of the Twenty Third International Conference on Artificial Intelligence and Statistics (AISTATS)},
  pages = 	 {2021--2031},
  year = 	 {2020},
  volume = 	 {108},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Online},
  month = 	 {26--28 Aug},
  publisher = 	 {PMLR},
  url = 	 {http://proceedings.mlr.press/v108/reisizadeh20a.html},
}

@misc{barnes2020rtopk,
    title={rTop-k: A Statistical Estimation Approach to Distributed SGD},
    author={Leighton Pate Barnes and Huseyin A. Inan and Berivan Isik and Ayfer Ozgur},
    year={2020},
    eprint={2005.10761},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}

@misc{gandikota2019vqsgd,
    title={vqSGD: Vector Quantized Stochastic Gradient Descent},
    author={Venkata Gandikota and Daniel Kane and Raj Kumar Maity and Arya Mazumdar},
    year={2019},
    eprint={1911.07971},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}


@article{stich2019error,
  title={The Error-Feedback Framework: Better Rates for {SGD} with Delayed Gradients and Compressed Communication},
  author={Stich, Sebastian U and Karimireddy, Sai Praneeth},
  journal={Journal of Machine Learning Research (JMLR)},
  year={2020}
}


@inproceedings{stich2018local,
  title={Local {SGD} Converges Fast and Communicates Little},
  author={Stich, Sebastian U},
  booktitle={International Conference on Learning Representations (ICLR)},
  year={2019}
}

@article{lin2018don,
  title={Don't Use Large Mini-Batches, Use Local {SGD}},
  author={Lin, Tao and Stich, Sebastian U and Jaggi, Martin},
  journal={arXiv preprint arXiv:1808.07217},
  year={2018}
}

@article{yu2020fair,
 author = {Yu, Han and Liu, Zelei and Liu, Yang and Chen, Tianjian and Cong, Mingshu and Weng, Xi and Niyato, Dusit and Yang, Qiang},
 title = {A Fairness-aware Incentive Scheme for Federated Learning},
 journal = {Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society},
 year = {2020},
 pages = {393–399}
} 

@article{lyu2020fair,
  title="{Towards Fair and Privacy-Preserving Federated Deep Models}",
  author={Lyu, Lingjuan and Yu, Jiangshan and Nandakumar, Karthik and Li, Yitong and Ma, Xingjun and Jin, Jiong and Yu, Han and Ng, Kee Siong},
  journal={IEEE Transactions on Parallel and Distributed Systems},
  year = {2020},
  month = May
}

@article{xie2019local,
  title={Local {AdaAlter}: Communication-Efficient Stochastic Gradient Descent with Adaptive Learning Rates},
  author={Xie, Cong and Koyejo, Oluwasanmi and Gupta, Indranil and Lin, Haibin},
  journal={arXiv preprint arXiv:1911.09030},
  year={2019}
}

@article{povey2014parallel,
  title={Parallel training of DNNs with natural gradient and parameter averaging},
  author={Povey, Daniel and Zhang, Xiaohui and Khudanpur, Sanjeev},
  journal={arXiv preprint arXiv:1410.7455},
  year={2014}
}

@article{su2015experiments,
  title={Experiments on parallel training of deep neural network using model averaging},
  author={Su, Hang and Chen, Haoyu},
  journal={arXiv preprint arXiv:1507.01239},
  year={2015}
}

@article{revisiting_sgd,
  author    = {Jianmin Chen and Rajat Monga and Samy Bengio and Rafal J{\'{o}}zefowicz},
  title     = {Revisiting Distributed Synchronous {SGD}},
  journal   = {CoRR},
  volume    = {abs/1604.00981},
  year      = {2016},
  url       = {http://arxiv.org/abs/1604.00981}
}

@incollection{cyclades,
title = {Cyclades: Conflict-free Asynchronous Machine Learning},
author = {Pan, Xinghao and Lam, Maximilian and Tu, Stephen and Papailiopoulos, Dimitris and Zhang, Ce and Jordan, Michael I and Ramchandran, Kannan and R\'{e}, Christopher},
booktitle = {Proceedings of the International Conference on Neural Information Processing Systems},
pages = {2568--2576},
year = {2016}
}

@article{zhang2017yellowfin,
  author    = {Jian Zhang and Ioannis Mitliagkas and Christopher Re},
  title     = {YellowFin and the Art of Momentum Tuning},
  journal   = {CoRR},
  volume    = { arXiv:1706.03471},
  month = jun,
  year      = {2017},
  url       = {http://arxiv.org/abs/1706.03471}
}

@inproceedings{langford2009slow,
 author = {Langford, John and Smola, Alexander J. and Zinkevich, Martin},
 title = {Slow Learners Are Fast},
 booktitle = {Proceedings of the International Conference on Neural Information Processing Systems (NIPS)},
 year = {2009},
 pages = {2331--2339},
 numpages = {9}
} 

@article{wibisono2016variational,
author = {Wibisono, Andre and Wilson, Ashia C. and Jordan, Michael I.}, 
title = {A variational perspective on accelerated methods in optimization},
volume = {113}, 
number = {47}, 
pages = {E7351-E7358}, 
year = {2016}, 
journal = {Proceedings of the National Academy of Sciences} 
}

@inproceedings{sun2020improving_ICML,
  title={Improving the sample and communication complexity for decentralized non-convex optimization: Joint gradient estimation and tracking},
  author={Sun, Haoran and Lu, Songtao and Hong, Mingyi},
  booktitle={International conference on machine learning},
  pages={9217--9228},
  year={2020},
  organization={PMLR}
}

@article{tsitsiklis1986distributed,
  title={Distributed asynchronous deterministic and stochastic gradient optimization algorithms},
  author={Tsitsiklis, John and Bertsekas, Dimitri and Athans, Michael},
  journal={IEEE Transactions on Automatic Control},
  volume={31},
  number={9},
  pages={803--812},
  year={1986},
  publisher={IEEE}
}


@InProceedings{zheng2017asynchronous,
  title = 	 {Asynchronous Stochastic Gradient Descent with Delay Compensation},
  author = 	 {Shuxin Zheng and Qi Meng and Taifeng Wang and Wei Chen and Nenghai Yu and Zhi-Ming Ma and Tie-Yan Liu},
  booktitle = 	 {Proceedings of the International Conference on Machine Learning},
  pages = 	 {4120--4129},
  year = 	 {2017},
  volume = 	 {70},
  month = 	 {06--11 Aug},
  pdf = 	 {http://proceedings.mlr.press/v70/zheng17b/zheng17b.pdf}
}
%% Redundant Requests

@article{dean2013tail,
  title={The tail at scale},
  author={Dean, Jeffrey and Barroso, Luiz Andr{\'e}},
  journal={Communications of the ACM},
  volume={56},
  number={2},
  pages={74--80},
  year={2013},
  publisher={ACM}
}

@inproceedings{liang2014tofec,
 author = {Guangfeng Liang and Ulas Kozat},
 title = "{TOFEC: Achieving Optimal Throughput-Delay Trade-off of Cloud Storage Using Erasure Codes}",
booktitle = {Proceedings of the IEEE International Conference on Communications (INFOCOM)},
 month = apr,
 year = {2014},
 pages={826-834}
}

@inproceedings{gardner2016s&x,
  title = {A Better Model for Job Redundancy: Decoupling Server Slowdown and Job Size},
  author = {K. Gardner and  M. Harchol-Balter and A. Scheller-Wolf},
  booktitle = {Proceedings of IEEE MASCOTS},
  month = sep,
  year = {2016}
}

@inproceedings{gardner2015reducing,
  title = {Reducing Latency via Redundant Requests: Exact Analysis},
  author = {K. Gardner and S. Zbarsky and S. Doroudi and M. Harchol-Balter and E. Hyyti{\"a} and A. Scheller-Wolf},
  booktitle = {Proceedings of the {ACM} {SIGMETRICS}},
  month = {Jun.},
  year = {2015}
}

@inproceedings{gardner2015analyzing,
  title = {Analyzing Response Time in the Redundancy-d System},
  author = {K. Gardner and S. Zbarsky  and M. Harchol-Balter and A. Scheller-Wolf},
  booktitle = {CMU-CS-15-141 archive},
  month = dec,
  year = {2015}
 }
  
 @article{gardner2017redundancyd,
  title={Redundancy-d: The Power of d Choices for Redundancy},
  author={K. Gardner and Mor Harchol-Balter and Alan Scheller-Wolf and Mark Velednitsky and Samuel Zbarsky},
  journal={Operations Research},
  year={2017},
  volume={65},
  pages={1078-1094}
}

@inproceedings{sun2015provably,
  author    = {Yin Sun and Zizhan Zheng and Can Emre Koksal and Kyu{-}Han Kim and Ness B. Shroff},
  title     = {Provably Delay Efficient Data Retrieving in Storage Clouds},
  booktitle  = {Proceedings of the IEEE Conference on Computer Communications (INFOCOM)},
  month = apr,
  year      = {2015}
}

@article{sun_shroff_2016,
  author    = {Yin Sun and Can Emre Koksal and Ness B. Shroff},
  title     = {On Delay-Optimal Scheduling in Queueing Systems with Replications},
  journal = {arXiv:1603.07322},
  month = mar,
  year      = {2016}
}

@INPROCEEDINGS{rawat2014locality, 
author={Rawat, A.S. and Papailiopoulos, D.S. and Dimakis, A.G. and Vishwanath, S.}, 
booktitle={Proceedings of IEEE International Symposium on Information Theory (ISIT)}, 
title={Locality and availability in distributed storage}, 
year={2014}, 
pages={681-685},
month={June}
}


@article{nelson_tantawi,
 author = {R. Nelson and A. Tantawi},
 title = {Approximate Analysis of Fork/Join Synchronization in Parallel Queues},
 journal = {IEEE Transactions on Computers},
 volume = {37},
 number = {6},
 month = {Jun.},
 year = {1988},
 pages = {739--743}
}

@article{varki_merc_chen,
    author = {Varki, Elizabeth and Merchant, Arif and Chen, Hui},
    title = {The {M/M/1} fork-join queue with variable sub-tasks},
    journal = {{unpublished, available online}},
    year = 2008
}

@article{kim_agarwal,
 author = {Kim, C. and Agrawala, A. K.},
 title = "{Analysis of the Fork-Join Queue}",
 journal = {IEEE Transactions on Computers},
 volume = {38},
 number = {2},
 month = {Feb.},
 year = {1989},
 pages = {250--255}
}
% Swanand, Emina, Alex
@inproceedings{kadhe2015analyzing,
 	author = {Swanand Kadhe and Emina Soljanin and Alex Sprintson},
 	title = {Analyzing the Download Time of Availability Codes},
	booktitle = {Proceedings of IEEE International Symposium on Information Theory (ISIT)},
	year = {2015},
	month = {Jun},
	pages={1467-1471}
}

@article{gu21mifa_neurips,
  title={Fast federated learning in the presence of arbitrary device unavailability},
  author={Gu, Xinran and Huang, Kaixuan and Zhang, Jingzhao and Huang, Longbo},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@inproceedings{chen2014when,
 author = {Shengbo Chen and Ulas C Kozat and Longbo Huang and Prasun Sinha and Guanfeng Liang and Xin Liu and Yin Sun and Ness B. Shroff},
 title = "{When Queueing Meets Coding: Optimal-Latency Data Retrieving Scheme in Storage Clouds}",
booktitle= {Proceedings of the IEEE International Conference on Communications (INFOCOM)},
 month = apr,
 pages={1042-1050},
 year = {2014}
}

@inproceedings{ananthanarayanan2013effective,
	title = {Effective Straggler Mitigation: Attack of the Clones},
	booktitle = {Proceedings of the 10th {USENIX} Conference on Networked Systems Design and Implementation},
	author = {Ananthanarayanan, Ganesh and Ghodsi, Ali and Shenker, Scott and Stoica, Ion},
	month = apr,
	year = {2013},
	pages = {185--198}
},


@inproceedings{aktas2017service,
  title={On the Service Capacity of Accessing Erasure Coded Content},
  author={Mehmet Aktas and Sarah E. Anderson and Ann Johnston and Gauri Joshi and Swanand Kadhe and Gretchen L. Matthews and Carolyn Mayer and Emina Soljanin},
  booktitle = {Proc. Allerton Conf.  Commun., Control and Computing},
  year = {2017},
  month = Oct
}

@inproceedings{anderson2018service,
  title={Service Rate Region of Content Access from Erasure Coded Storage},
  author={Sarah E. Anderson and Ann Johnston and Gauri Joshi and Gretchen L. Matthews and Carolyn Mayer and Emina Soljanin},
  booktitle = {Proc. Allerton Conf.  Commun., Control and Computing},
  year = {2017},
  month = Oct
}


@article{joshi2017boosting,
 author = {Joshi, Gauri},
 title = {Boosting Service Capacity via Adaptive Task Replication},
 journal = {SIGMETRICS Perform. Eval. Rev.},
 issue_date = {Setember 2017},
 volume = {45},
 number = {2},
 month = oct,
 year = {2017},
 issn = {0163-5999},
 pages = {9--11},
 numpages = {3},
 url = {http://doi.acm.org/10.1145/3152042.3152046},
 doi = {10.1145/3152042.3152046},
 acmid = {3152046},
 publisher = {ACM},
 address = {New York, NY, USA}
},

@article{joshi2018synergy,
 author = {Joshi, Gauri},
 title = {Synergy via Redundancy: Boosting Service Capacity with Adaptive Replication},
 journal = {SIGMETRICS Performance Evaluation Review},
 issue_date = {December 2017},
 volume = {45},
 number = {3},
 month = mar,
 year = {2018},
 issn = {0163-5999},
 pages = {21--28},
 numpages = {8},
 url = {http://doi.acm.org/10.1145/3199524.3199530}
} 

@inproceedings{joshi2012coding,
  title={Coding for fast content download},
  author={Joshi, Gauri and Liu, Yanpei and Soljanin, Emina},
  booktitle ={Proceedings of the Allerton Conference on Communication, Control and Computing},
  year = {2012},
  month = oct,
  pages = {326--333}
}

@article{joshi2014delay,
  title="{On the Delay-storage Trade-off in Content Download from Coded Distributed Storage}",
  author={Joshi, Gauri and Liu, Yanpei and Soljanin, Emina},
  journal={IEEE Journal on Selected Areas on Communications},
  year = {2014},
  month = may
}


@inproceedings{dutta2016short,
  title={Short-dot: Computing large linear transforms distributedly using coded short dot products},
  author={Dutta, Sanghamitra and Cadambe, Viveck and Grover, Pulkit},
  booktitle={Advances In Neural Information Processing Systems},
  pages={2100--2108},
  year={2016}
}


@article{kosaian2018learning,
  author    = {Jack Kosaian and K. V. Rashmi and Shivaram Venkataraman},
  title     = {Learning a Code: Machine Learning for Approximate Non-Linear Coded Computation},
  journal   = {CoRR},
  volume    = {abs/1806.01259},
  year      = {2018},
  url       = {http://arxiv.org/abs/1806.01259},
  archivePrefix = {arXiv},
  eprint    = {1806.01259}
}

@article{kosaian2019parity,
  author    = {Jack Kosaian and K. V. Rashmi and Shivaram Venkataraman},
  title     = {Parity Models: {A} General Framework for Coding-Based Resilience in {ML} Inference},
  journal   = {CoRR},
  volume    = {abs/1905.00863},
  year      = {2019},
  url       = {http://arxiv.org/abs/1905.00863},
  archivePrefix = {arXiv},
  eprint    = {1905.00863}
}


@article{severinson2017block,
  title={Block-Diagonal and LT Codes for Distributed Computing With Straggling Servers},
  author={Albin Severinson and Alexandre Graell i Amat and Eirik Rosnes},
  journal={arXiv preprint arXiv:1712.08230},
  month ={dec},
  year={2017}
}

@inproceedings{yu2017polynomial,
  title={Polynomial codes: an optimal design for high-dimensional coded matrix multiplication},
  author={Yu, Qian and Maddah-Ali, Mohammad and Avestimehr, Salman},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4406--4416},
  year={2017}
}

@article{wang2018coded,
  title={Coded Sparse Matrix Multiplication},
  author={Wang, Sinong and Liu, Jiashang and Shroff, Ness},
  journal={arXiv preprint arXiv:1802.03430},
  year={2018}
}


@article{tandon2017gradient,
  title={Gradient Coding: Avoiding Stragglers in Synchronous Gradient Descent},
  author={Tandon, Rashish and Lei, Qi and Dimakis, Alexandros G and Karampatziakis, Nikos},
  journal={stat},
  volume={1050},
  pages={8},
  year={2017}
}

@article{dutta2018optimal,
  title={On the Optimal Recovery Threshold of Coded Matrix Multiplication},
  author={Dutta, Sanghamitra and Fahim, Mohammad and Haddadpour, Farzin and Jeong, Haewon and Cadambe, Viveck and Grover, Pulkit},
  journal={arXiv preprint arXiv:1801.10292},
  year={2018}
}

@inproceedings{yang2017coded,
  title={Coded Distributed Computing for Inverse Problems},
  author={Yang, Yaoqing and Grover, Pulkit and Kar, Soummya},
  booktitle={Advances in Neural Information Processing Systems},
  pages={709--719},
  year={2017}
}


@inproceedings{luby2002lt,
  title={LT codes},
  author={Luby, Michael},
  booktitle={Proceedings of the IEEE Symposium on Foundations of Computer Science},
  pages={271},
  year={2002},
  organization={IEEE}
}


@article{shokrollahi2006raptor,
  title={Raptor codes},
  author={Shokrollahi, Amin},
  journal={IEEE/ACM Transactions on Networking (TON)},
  volume={14},
  number={SI},
  pages={2551--2567},
  year={2006},
  publisher={IEEE Press}
}

@article{shokrollahi2011raptor,
  title={Raptor codes},
  author={Shokrollahi, Amin and Luby, Michael and others},
  journal={Foundations and trends{\textregistered} in communications and information theory},
  volume={6},
  number={3--4},
  pages={213--322},
  year={2011},
  publisher={Now Publishers, Inc.}
}

@article{joshi2010fountain,
  title={Fountain codes},
  author={Joshi, Gauri and Rhim, Joong Bum and Sun, John and Wang, Da},
  journal={Technical Report},
  year={2010}
}

@book{berkelamp1968algebraic,
 	author = "{E. Berkelamp}",
 	title = "{Algebraic coding theory}",
	year = {1968},
	publisher = "{McGraw-Hill}",
	address = "{New York, USA}"
}

@inproceedings{mallick2019rateless,
  title={Rateless Codes for Distributed Computations with Sparse Compressed Matrices},
  author={Ankur Mallick and Gauri Joshi},
  booktitle = {IEEE International Symposium on Information Theory (ISIT)},
  month = {jul},
  year={2019}
}

@inproceedings{mallick2019fast,
  title={Fast and Efficient Distributed Matrix-Vector Multiplication Using Rateless Fountain Codes},
  author={Ankur Mallick and Malhar Chaudhari and Gauri Joshi},
  booktitle = {Proceedings of the International Conference on Acoustics, Speech, and Signal Processing (ICASSP)},
  month = {may},
  year={2019}
}

@inproceedings{mallick2020rateless,
  	author   = {Ankur Mallick and Utsav Sheth and Ganesh Palanikumar and Malhar Chaudhari and Gauri Joshi},
  	title = "{Rateless Codes for Near-Perfect Load Balancing in Distributed Matrix-Vector Multiplication}",
	booktitle  = "{Proceedings of ACM Sigmetrics 2020}",
	url           = {"https://arxiv.org/abs/1804.10331"},
	month         =  may,
	year          = 2020
}


@article{lee2017speeding,
  title={Speeding up distributed machine learning using codes},
  author={Lee, Kangwook and Lam, Maximilian and Pedarsani, Ramtin and Papailiopoulos, Dimitris and Ramchandran, Kannan},
  journal={IEEE Transactions on Information Theory},
  year={2017}, 
volume={PP}, 
number={99}, 
pages={1-1}, 
}

@article{wang2015using,
  title={Using straggler replication to reduce latency in large-scale parallel computing},
  author={Wang, Da and Joshi, Gauri and Wornell, Gregory},
  journal={ACM SIGMETRICS Performance Evaluation Review},
  volume={43},
  number={3},
  pages={7--11},
  year={2015},
  publisher={ACM}
}

@article{wang2014efficient,
  title={Efficient Straggler Replication for fast response times in parallel computation},
  author={Wang, Da and Joshi, Gauri and Wornell, Gregory},
  journal={ACM SIGMETRICS Performance Evaluation Review},
  volume = {42},
 number = {1},
 month = jun,
 year = {2014},
 pages = {599--600}
}

@article{wang2019efficient,
author = {Wang, Da and Joshi, Gauri and W. Wornell, Gregory},
year = {2019},
month = {04},
pages = {1-23},
title = {Efficient Straggler Replication in Large-Scale Parallel Computing},
volume = {4},
journal = {ACM Transactions on Modeling and Performance Evaluation of Computing Systems},
doi = {10.1145/3310336}
}

@inproceedings{sahu2019convergence,
  title={Federated Optimization in Heterogeneous Networks},
  author={Sahu, Anit Kumar and Li, Tian and Sanjabi, Maziar and Zaheer, Manzil and Talwalkar, Ameet and Smith, Virginia},
  booktitle={Proceedings of the Machine Learning and Systems (MLSys) Conference},
  month = mar,
  year={2019}
}

@inproceedings{harlap2016addressing,
 author = {Harlap, Aaron and Cui, Henggang and Dai, Wei and Wei, Jinliang and Ganger, Gregory R. and Gibbons, Phillip B. and Gibson, Garth A. and Xing, Eric P.},
 title = {Addressing the Straggler Problem for Iterative Convergent Parallel {ML}},
 booktitle = {Proceedings of the ACM Symposium on Cloud Computing (SoCC)},
 year = {2016},
 pages = {98--111},
 numpages = {14}
} 

@inproceedings {cipar2013solving,
author = {James Cipar and Qirong Ho and Jin Kyu Kim and Seunghak Lee and Gregory R. Ganger and Garth Gibson and Kimberly Keeton and Eric Xing},
title = {Solving the Straggler Problem with Bounded Staleness},
booktitle = {Proceedings of the Workshop on Hot Topics in Operating Systems},
year = {2013}
}

@inproceedings{ho2013more,
 author = {Ho, Qirong and Cipar, James and Cui, Henggang and Kim, Jin Kyu and Lee, Seunghak and Gibbons, Phillip B. and Gibson, Garth A. and Ganger, Gregory R. and Xing, Eric P.},
 title = {More Effective Distributed ML via a Stale Synchronous Parallel Parameter Server},
 booktitle = {Proceedings of the International Conference on Neural Information Processing Systems},
 year = {2013},
 pages = {1223--1231},
 numpages = {9}
} 

@article{dean2008mapreduce,
 author = {Dean, Jeffrey and Ghemawat, Sanjay},
 title = "{MapReduce: simplified data processing on large clusters}",
 journal = {ACM Commun.\ Mag.},
 volume = {51},
 number = {1},
 month = {Jan.},
 year = {2008},
 pages = {107--113}
}

@inproceedings{zaharia2010spark,
  title={Spark: cluster computing with working sets},
  author={Zaharia, Matei and Chowdhury, Mosharaf and Franklin, Michael J and Shenker, Scott and Stoica, Ion},
  booktitle={Proceedings of the 2nd USENIX conference on Hot topics in cloud computing},
  volume={10},
  pages={10},
  year={2010}
}


@inproceedings{li2015coded,
  author    = {Songze Li and Mohammad Maddah-Ali and A. S. Avestimehr},
  title     = {Coded MapReduce},
  booktitle  = {Proceedings of the Allerton Conference on Communication, Control and Computing},
  month = oct,
  year      = {2015}
}

@inproceedings{lee2016speeding,
  author    = "{K. Lee, M. Lam, R. Pedarsani, D. Papailiopoulos, and K. Ramchandran}",
  title     = {Speeding Up Distributed Machine Learning Using Codes},
  booktitle  = {Proceedings of the IEEE International Symposium on Information Theory (ISIT)},
  month = jul,
  year      = {2016}
}

@inproceedings{joshi2015queues,
	author = {Gauri Joshi and Emina Soljanin and Gregory Wornell},
	title = {Queues with Redundancy: Latency-Cost Analysis},
	booktitle = {Proceedings of the ACM SIGMETRICS Workshop on Mathematical Modeling and Analysis},
	month = jun,
	year = {2015}
}
  
@inproceedings{london2017distributed,
	author = {Palma London and Niangjun Chen and Shai Vardi and Adam Wierman},
	title = {Distributed Optimization via Local Computation Algorithms},
	booktitle = {Proceedings of the ACM SIGMETRICS Workshop on Mathematical Modeling and Analysis},
	month = jun,
	year = {2017}
}

 @inproceedings{joshi2015efficient,
  title={Efficient Replication of Queued Tasks for Latency Reduction in Cloud Systems},
  author={Gauri Joshi and Emina Soljanin and Gregory Wornell},
  booktitle = {Proceedings of the Allerton Conference},
  year = {2015},
  month = oct,
}

@article{joshi2017efficient,
	title = {Efficient Redundancy Techniques for Latency Reduction in Cloud Systems},
	journal = {ACM Trans. on Performance Evaluation of Computer Systems},
	author = {Joshi, Gauri and Soljanin, Emina and Wornell, Gregory W.},
	volume = {2},
  number = {2},
 month = apr,
 year = {2017},
 issn = {2376-3639},
 pages = {1--30},
 articleno = {12}
}


@phdthesis{mitzenmacher1996power,
  title= "{The power of two choices in randomized load balancing}",
  author={M. Mitzenmacher},
  year={1996},
  school={University of California Berkeley, CA}
}

@inproceedings{azar1994balanced,
title = "Balanced allocations",
author = "Y. Azar and Broder, {A. Z.} and Karlin, {A. R.} and E. Upfal",
year = "1994",
pages = "593--602",
booktitle = {Proceedings of the ACM Symposium on Theory of Computing}
}

@inproceedings{basu2019qsparse,
  title={Qsparse-local-SGD: Distributed SGD with Quantization, Sparsification, and Local Computations},
  author={Basu, Debraj and Data, Deepesh and Karakus, Can and Diggavi, Suhas},
  booktitle={Advances in Neural Information Processing Systems},
  pages={14695--14706},
  year={2019}
}

@inproceedings{jiang2018linear,
  title={A linear speedup analysis of distributed deep learning with sparse and quantized communication},
  author={Jiang, Peng and Agrawal, Gagan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2525--2536},
  year={2018}
}

@inproceedings{alistarh2017qsgd,
  title={QSGD: Communication-efficient SGD via gradient quantization and encoding},
  author={Alistarh, Dan and Grubic, Demjan and Li, Jerry and Tomioka, Ryota and Vojnovic, Milan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1709--1720},
  year={2017}
}

@article{sattler2018sparse,
  title={Sparse binary compression: Towards distributed deep learning with minimal communication},
  author={Sattler, Felix and Wiedemann, Simon and M{\"u}ller, Klaus-Robert and Samek, Wojciech},
  journal={arXiv preprint arXiv:1805.08768},
  year={2018}
}


%%% Preemptible or Volatile Instances

@misc{spot,
author={{Amazon EC2}},
title={Amazon EC2 Spot Instances},
howpublished={\url{https://aws.amazon.com/ec2/spot/}},
year={2019}
}

@misc{gcp,
author={{Google Cloud Platform}},
title={Preemptible Virtual Machines},
howpublished={\url{https://cloud.google.com/preemptible-vms/}},
year={2019}
}
@misc{gcp_preemptible,
author={{Google Cloud Platform}},
title={Preemptible VM Instances},
howpublished={\url{https://cloud.google.com/compute/docs/instances/preemptible}},
year={2019}
}

@misc{azure,
author={{Microsoft Azure}},
title={Batch VMs},
howpublished={\url{https://azure.microsoft.com/en-us/pricing/details/batch/}},
year={2019}
}

@misc{low_priority_azure,
author={{Microsoft Azure}},
title={Announcing low-priority VMs on scale sets now in public preview},
howpublished={\url{https://azure.microsoft.com/en-us/blog/low-priority-scale-sets/}},
year={2018}
}


%% Importance Sampling

@INPROCEEDINGS{kawaguchi2018ordered,
author={K. Kawaguchi and H. Lu},
booktitle={Proceedings of the 23rd International Conference on Artificial Intelligence and Statistics (AISTATS 2020)},
title={Ordered SGD: A New Stochastic Optimization Framework for Empirical Risk Minimization},
year={2020},
volume={108},
url = {https://arxiv.org/abs/1907.04371}
}

@phdthesis{powerof2,
  title= "{The power of two choices in randomized load balancing}",
  author={M. Mitzenmacher},
  year={1996},
  school={University of California Berkeley, CA}
}

@INPROCEEDINGS{shah2020choosing,
title={Choosing the Sample with Lowest Loss makes SGD Robust},
author={Vatsal Shah and Xiaoxia Wu and Sujay Sanghavi},
 year={2020},
booktitle={Proceedings of the 23rd International Conference on Artificial Intelligence and Statistics (AISTATS 2020)},
url = {https://arxiv.org/abs/2001.03316}
}


@inproceedings{katharopoulos2018not,
    title={Not All Samples Are Created Equal: Deep Learning with Importance Sampling},
    author={Katharopoulos, A. and Fleuret, F.},
    booktitle={Proceedings of the International Conference on Machine Learning (ICML)},
    pages = 	 {2525--2534},
  year = 	 {2018},
  volume = 	 {80},
  series = 	 {Proceedings of Machine Learning Research}
}

@inproceedings{ruan2020flexible,
    title={Towards Flexible Device Participation in Federated Learning for Non-IID Data},
    author={Yichen Ruan and Xiaoxi Zhang and Shu-Che Liang and Carlee Joe-Wong},
    year={2020},
    booktitle={The 23rd International Conference on Artificial Intelligence and Statistics (AISTATS 2020)}
}

@article{jack2019afl,
    title={Active Federated Learning},
    author={Jack Goetz and Kshitiz Malik and Duc Bui and Seungwhan Moon and Honglei Liu and Anuj Kumar},
     journal={arXiv preprint arXiv:1909.12641},
    year ={2019}
}

@article{moni2020opticlient,
    title={Communication-Efficient Federated Learning via Optimal Client Sampling},
    author={M\'onica Ribero and Haris Vikalo},
    year={2020},
    journal={arXiv preprint arXiv:2007.15197}
    }

@article{yass2020devhetsuper,
    title={Device Heterogeneity in Federated Learning: A Superquantile Approach},
    author={Yassine Laguel and Krishna Pillutla and J\'er\^ome Malick and Zaid Harchaoui},
    year={2020},
     journal = {ArXiv},
    eprint = {2002.11223},
    archivePrefix={arXiv},
    primaryClass={stat.ML}
    }
    
@inproceedings{tiff2020noisyfed,
 author = {Tiffany Tuor and Shiqiang Wang and and Bong Jun Ko and Changchang Liu and Kin K. Leung},
 title = {Overcoming Noisy and Irrelevant Data in
Federated Learning},
 booktitle = {International Conference on Pattern Recognition},
 month = {June},
 year = {2020},
 url = {https://arxiv.org/pdf/2001.08300}
} 

@inproceedings{tak2019clisel,
 author = {Takayuki Nishio and Ryo Yonetani},
 title = {Client Selection for Federated Learning with Heterogeneous Resources in Mobile Edge},
 booktitle = {{IEEE} International Conference on Communications},
 month = {May},
 year = {2019},
 pages = {1--7}
} 

@inproceedings{stich2017safeis,
author={Sebastian U. Stich and Anant Raj and Martin Jaggi},
title={Safe Adaptive Importance Sampling},
booktitle = {Advances in Neural Information Processing Systems 31},
pages = {4381--4391},
year ={2017}
}

@inproceedings{pezhao2015stochasticis,
author = {Peilin Zhao and Tong Zhang},
  title = 	 {Stochastic Optimization with Importance Sampling for Regularized Loss Minimization},
  booktitle = 	 {Proceedings of the 32th International Conference on Machine Learning},
  pages = 	 {1--9},
  year = 	 {2015},
  volume = 	 {37},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {07--09 Jul}
  }
  
  @
  
  
@incollection{salehi2018coordinate,
title = {Coordinate Descent with Bandit Sampling},
author = {Salehi, Farnood and Thiran, Patrick and Celis, Elisa},
booktitle = {Advances in Neural Information Processing Systems 31},
editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
pages = {9247--9257},
year = {2018},
url = {http://papers.nips.cc/paper/8137-coordinate-descent-with-bandit-sampling.pdf}
}

@misc{jiang2019accelerating,
    title={Accelerating Deep Learning by Focusing on the Biggest Losers},
    author={Angela H. Jiang and Daniel L. -K. Wong and Giulio Zhou and David G. Andersen and Jeffrey Dean and Gregory R. Ganger and Gauri Joshi and Michael Kaminksy and Michael Kozuch and Zachary C. Lipton and Padmanabhan Pillai},
    year={2019},
    eprint={1910.00762},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}


@article{bubeck2012regret,
  title={Regret analysis of stochastic and nonstochastic multi-armed bandit problems},
  author={Bubeck, S{\'e}bastien and Cesa-Bianchi, Nicolo},
  journal={Foundations and Trends in Machine Learning},
  volume={5},
  number={1},
  pages={1--122},
  year={2012},
  publisher={Now Publishers, Inc.}
}

@Book{lattimore2018bandit,
  Title = {Bandit Algorithms},
  Author  = {Tor Lattimore and Csaba Szepesvari},
  year= {2018},
  url = {http://downloads.tor-lattimore.com/banditbook/book.pdf}
}

@article{auer2002finite,
  title={Finite-time analysis of the multiarmed bandit problem},
  author={Auer, Peter and Cesa-Bianchi, Nicolo and Fischer, Paul},
  journal={Machine learning},
  volume={47},
  number={2-3},
  pages={235--256},
  year={2002},
  publisher={Springer}
}


@article{thompson1933likelihood,
    author = {Thompson, W. R.},
    journal = {Biometrika},
    month = dec,
    number = {3-4},
    pages = {285--294},
    posted-at = {2012-06-08 07:10:29},
    priority = {2},
    publisher = {Biometrika Trust},
    title = {{On the Likelihood that One Unknown Probability Exceeds Another in View of the Evidence of Two Samples}},
    volume = {25},
    year = {1933}
}


@article{lai1985asymptotically,
  title={Asymptotically efficient adaptive allocation rules},
  author={Lai, Tze Leung and Robbins, Herbert},
  journal={Advances in applied mathematics},
  volume={6},
  number={1},
  pages={4--22},
  year={1985},
  publisher={Elsevier}
}

@article{yang2021anarchic,
  title={Anarchic Federated Learning},
  author={Yang, Haibo and Zhang, Xin and Khanduri, Prashant and Liu, Jia},
  journal={arXiv preprint arXiv:2108.09875},
  year={2021}
}

@article{yan2020distributed,
  title={Distributed non-convex optimization with sublinear speedup under intermittent client availability},
  author={Yan, Yikai and Niu, Chaoyue and Ding, Yucheng and Zheng, Zhenzhe and Wu, Fan and Chen, Guihai and Tang, Shaojie and Wu, Zhihua},
  journal={arXiv preprint arXiv:2002.07399},
  year={2020}
}

@article{hu2020fedmgda+,
  title={Fedmgda+: Federated learning meets multi-objective optimization},
  author={Hu, Zeou and Shaloudegi, Kiarash and Zhang, Guojun and Yu, Yaoliang},
  journal={arXiv preprint arXiv:2006.11489},
  year={2020}
}


@inproceedings{ruan2021towards,
  title={Towards flexible device participation in federated learning},
  author={Ruan, Yichen and Zhang, Xiaoxi and Liang, Shu-Che and Joe-Wong, Carlee},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3403--3411},
  year={2021},
  organization={PMLR}
}

@article{raj2018k,
  title={k-SVRG: Variance Reduction for Large Scale Optimization},
  author={Raj, Anant and Stich, Sebastian U},
  journal={arXiv preprint arXiv:1805.00982},
  year={2018}
}


@article{allen2016exploiting,
  title={Exploiting the structure: Stochastic gradient methods using raw clusters},
  author={Allen-Zhu, Zeyuan and Yuan, Yang and Sridharan, Karthik},
  journal={Advances in Neural Information Processing Systems},
  volume={29},
  year={2016}
}



@article{hofmann2015variance,
  title={Variance reduced stochastic gradient descent with neighbors},
  author={Hofmann, Thomas and Lucchi, Aurelien and Lacoste-Julien, Simon and McWilliams, Brian},
  journal={Advances in Neural Information Processing Systems},
  volume={28},
  year={2015}
}


@article{lei2017non,
  title={Non-convex finite-sum optimization via scsg methods},
  author={Lei, Lihua and Ju, Cheng and Chen, Jianbo and Jordan, Michael I},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  year={2017}
}

@inproceedings{fraboni2021clustered,
  title={Clustered Sampling: Low-Variance and Improved Representativity for Clients Selection in Federated Learning},
  author={Fraboni, Yann and Vidal, Richard and Kameni, Laetitia and Lorenzi, Marco},
  booktitle={International Conference on Machine Learning},
  pages={3407--3416},
  year={2021},
  organization={PMLR}
}

@article{sattler2020clustered,
  title={Clustered federated learning: Model-agnostic distributed multitask optimization under privacy constraints},
  author={Sattler, Felix and M{\"u}ller, Klaus-Robert and Samek, Wojciech},
  journal={IEEE transactions on neural networks and learning systems},
  volume={32},
  number={8},
  pages={3710--3722},
  year={2020},
  publisher={IEEE}
}
@inproceedings{jalalirad2019simple,
  title={A simple and efficient federated recommender system},
  author={Jalalirad, Amir and Scavuzzo, Marco and Capota, Catalin and Sprague, Michael},
  booktitle={Proceedings of the 6th IEEE/ACM International Conference on Big Data Computing, Applications and Technologies},
  pages={53--58},
  year={2019}
}

@article{cifar10,
title= {CIFAR-10 (Canadian Institute for Advanced Research)},
journal= {},
author= {Alex Krizhevsky and Vinod Nair and Geoffrey Hinton},
year= {},
url= {http://www.cs.toronto.edu/~kriz/cifar.html},
}
@inproceedings{hsieh2020non,
  title={The non-iid data quagmire of decentralized machine learning},
  author={Hsieh, Kevin and Phanishayee, Amar and Mutlu, Onur and Gibbons, Phillip},
  booktitle={International Conference on Machine Learning},
  pages={4387--4398},
  year={2020},
  organization={PMLR}
}
@book{shakespeare2014complete,
  title={The complete works of William Shakespeare},
  author={Shakespeare, William},
  year={2014},
  publisher={Race Point Publishing}
}

@inproceedings{wu2018group,
  title={Group normalization},
  author={Wu, Yuxin and He, Kaiming},
  booktitle={Proceedings of the European conference on computer vision (ECCV)},
  pages={3--19},
  year={2018}
}
@misc{laguel2020device,
      title={Device Heterogeneity in Federated Learning: A Superquantile Approach}, 
      author={Yassine Laguel and Krishna Pillutla and Jérôme Malick and Zaid Harchaoui},
      year={2020},
      eprint={2002.11223},
      archivePrefix={arXiv},
      primaryClass={stat.ML}
}


@article{hard2018federated,
  title={Federated learning for mobile keyboard prediction},
  author={Hard, Andrew and Rao, Kanishka and Mathews, Rajiv and Ramaswamy, Swaroop and Beaufays, Fran{\c{c}}oise and Augenstein, Sean and Eichner, Hubert and Kiddon, Chlo{\'e} and Ramage, Daniel},
  journal={arXiv preprint arXiv:1811.03604},
  year={2018}
}

@article{tran2019hybrid,
  title={Hybrid stochastic gradient descent algorithms for stochastic nonconvex optimization},
  author={Tran-Dinh, Quoc and Pham, Nhan H and Phan, Dzung T and Nguyen, Lam M},
  journal={arXiv preprint arXiv:1905.05920},
  year={2019}
}


@article{cutkosky2019momentum,
  title={Momentum-based variance reduction in non-convex sgd},
  author={Cutkosky, Ashok and Orabona, Francesco},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}


@article{fang2018spider,
  title={Spider: Near-optimal non-convex optimization via stochastic path-integrated differential estimator},
  author={Fang, Cong and Li, Chris Junchi and Lin, Zhouchen and Zhang, Tong},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}

@article{cho2020client,
  title={Client selection in federated learning: Convergence analysis and power-of-choice selection strategies},
  author={Cho, Yae Jee and Wang, Jianyu and Joshi, Gauri},
  journal={arXiv preprint arXiv:2010.01243},
  year={2020}
}

@article{chen2020optimal,
  title={Optimal client sampling for federated learning},
  author={Chen, Wenlin and Horvath, Samuel and Richtarik, Peter},
  journal={arXiv preprint arXiv:2010.13723},
  year={2020}
}

@inproceedings{nishio2019client,
  title={Client selection for federated learning with heterogeneous resources in mobile edge},
  author={Nishio, Takayuki and Yonetani, Ryo},
  booktitle={ICC 2019-2019 IEEE international conference on communications (ICC)},
  pages={1--7},
  year={2019},
  organization={IEEE}
}


@article{acar2021federated,
  title={Federated learning based on dynamic regularization},
  author={Acar, Durmus Alp Emre and Zhao, Yue and Navarro, Ramon Matas and Mattina, Matthew and Whatmough, Paul N and Saligrama, Venkatesh},
  journal={arXiv preprint arXiv:2111.04263},
  year={2021}
}


@article{bauschke2008proximal,
  title={The proximal average: basic theory},
  author={Bauschke, Heinz H and Goebel, Rafal and Lucet, Yves and Wang, Xianfu},
  journal={SIAM Journal on Optimization},
  volume={19},
  number={2},
  pages={766--785},
  year={2008},
  publisher={SIAM}
}

@inproceedings{allen2016improved,
  title={Improved SVRG for non-strongly-convex or sum-of-non-convex objectives},
  author={Allen-Zhu, Zeyuan and Yuan, Yang},
  booktitle={International conference on machine learning},
  pages={1080--1089},
  year={2016},
  organization={PMLR}
}


@inproceedings{li2020convergence,
  title={On the convergence of sarah and beyond},
  author={Li, Bingcong and Ma, Meng and Giannakis, Georgios B},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={223--233},
  year={2020},
  organization={PMLR}
}
@article{li2021zerosarah,
  title={ZeroSARAH: Efficient nonconvex finite-sum optimization with zero full gradient computation},
  author={Li, Zhize and Hanzely, Slavom{\'\i}r and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2103.01447},
  year={2021}
}

@article{das2020faster,
  title={Faster non-convex federated learning via global and local momentum},
  author={Das, Rudrajit and Acharya, Anish and Hashemi, Abolfazl and Sanghavi, Sujay and Dhillon, Inderjit S and Topcu, Ufuk},
  journal={arXiv preprint arXiv:2012.04061},
  year={2020}
}


@article{charles2021large,
  title={On large-cohort training for federated learning},
  author={Charles, Zachary and Garrett, Zachary and Huo, Zhouyuan and Shmulyian, Sergei and Smith, Virginia},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{reddi2016fast,
  title={Fast incremental method for nonconvex optimization},
  author={Reddi, Sashank J and Sra, Suvrit and P{\'o}czos, Barnab{\'a}s and Smola, Alex},
  journal={arXiv preprint arXiv:1603.06159},
  year={2016}
}


@article{schmidt2017minimizing,
  title={Minimizing finite sums with the stochastic average gradient},
  author={Schmidt, Mark and Le Roux, Nicolas and Bach, Francis},
  journal={Mathematical Programming},
  volume={162},
  number={1},
  pages={83--112},
  year={2017},
  publisher={Springer}
}


@inproceedings{gupta2018active,
 author = {Samarth Gupta and Gauri Joshi and Osman Ya\u{g}an},
 title = {Active Distribution Learning using Indirect Samples},
 booktitle = {Allerton conference on Communication, Control and Computing},
 month = oct,
 year = 2018,
 url = {https://arxiv.org/abs/1808.05334}
} 

@article{gupta2018correlated,
 author = {Samarth Gupta and Shreyas Chaudhari and Gauri Joshi and Osman Ya\u{g}an},
 title = {Multi-Armed Bandits with Correlated Arms},
 journal = {ArXiv},
 month = nov,
 year = 2019,
 url = {https://arxiv.org/abs/1911.03959}
} 

@article{hinton2015distillknow,
 author = {Geoffrey Hinton and Oriol Vinyals and Jeff Dean},
 title = {Distilling the Knowledge in a Neural Network},
 journal = {ArXiv},
 month = Mar,
 year = 2015,
 url = {https://arxiv.org/abs/1503.02531}
} 

@article{sangho2020distilledge,
 author = {Sangho Lee and Kiyoon Yoo and Nojun Kwak},
 title = {Edge Bias in Federated Learning and its Solution by Buffered Knowledge Distillation},
 journal = {ArXiv},
 month = Feb,
 year = 2021,
 url = {https://arxiv.org/abs/2010.10338}
} 


@article{zhouyan2021distilloneshot,
 author = {Yanlin Zhou and George Pu and Xiyao Ma and Xiaolin Li and Dapeng Wu},
 title = {Distilled One-Shot Federated Learning},
 journal = {ArXiv},
 month = Jun,
 year = 2021,
 url = {https://arxiv.org/abs/2009.07999}
} 

@inproceedings{qinli2021distillone,
 author = {Qinbin Li and Bingsheng He and Dawn Song},
 title = {Practical One-Shot Federated Learning for Cross-Silo Setting},
 booktitle = {Proceedings of the Thirtieth International Joint Conference on Artificial Intelligence (IJCAI-21)},
 month = May,
 year = 2021,
 url = {https://arxiv.org/abs/2010.01017}
} 

@inproceedings{sun2021distillpriv,
 author = {Lichao Sun and Lingjuan Lyu},
 title = {Federated Model Distillation with Noise-Free Differential Privacy},
 booktitle = {Proceedings of the Thirtieth International Joint Conference on Artificial Intelligence (IJCAI-21)},
 month = May,
 year = 2021,
 url = {https://arxiv.org/abs/2009.05537}
} 

@inproceedings{lan2018distillfly,
 author = {X. Lan and X. Zhu and S. Gong},
 title = {Knowledge distillation by on-the-fly native ensemble},
 booktitle = {Proceedings of the 32nd International Conference on Neural Information Processing Systems.},
 year = {2018},
 pages = {7528--7538}
}


@inproceedings{ma2020distillhet,
 author = {Jiaxin Ma and Ryo Yonetani and Zahid Iqbal},
 title = {Adaptive Distillation for Decentralized Learning
from Heterogeneous Clients},
 booktitle = {2020 25th International Conference on Pattern Recognition (ICPR)},
 month = Aug,
 year = 2020,
 url = {https://arxiv.org/abs/2008.07948}
} 

@article{ita2021distillsemisup,
 author = {Sohei Itahara and Takayuki Nishio and Yusuke Koda and Masahiro Morikura and Koji Yamamoto},
 title = {Distillation-Based Semi-Supervised Federated Learning
for Communication-Efficient Collaborative Training
with Non-IID Private Data},
 journal = {IEEE Transactions on Mobile Computing},
 month = Aug,
 year = 2020,
 url = {https://arxiv.org/abs/2008.06180}
} 

@inproceedings{anil2020codistill,
 author = {Rohan Anil and Gabriel Pereyra and Alexandre Passos and Robert Ormandi and George E. Dahl and Geoffrey E. Hinton},
 title = {Large Scale Distributed Neural Network Training Through Online Distillation},
 booktitle = {International Conference on Learning Representations (ICLR)},
 year = 2018,
 url = {https://arxiv.org/abs/1804.03235}
} 

@article{gupta2018exploiting,
 author = {Samarth Gupta and Shreyas Chaudhari and Subhojyoti Mukherjeeand and Gauri Joshi and Osman Ya\u{g}an},
 title = {A Unified Approach to Translate Classical Bandit Algorithms to the Structured Bandit Setting
},
 journal = {ArXiv},
 month = oct,
 year = 2018,
 url = {https://arxiv.org/abs/1810.08164}
} 

@inproceedings{li2020massively,
  author    = {Liam Li and Kevin G. Jamieson and Afshin Rostamizadeh and Ekaterina Gonina and Moritz Hardt and Benjamin Recht and Ameet Talwalkar},
  title     = {Massively Parallel Hyperparameter Tuning},
  booktitle   = {Machine Learning and Systems Conference 2020},
  year      = {2020},
  url       = {http://arxiv.org/abs/1810.05934}
}

@article{li2017hyperband,
 author = {Li, Lisha and Jamieson, Kevin and DeSalvo, Giulia and Rostamizadeh, Afshin and Talwalkar, Ameet},
 title = {Hyperband: A Novel Bandit-based Approach to Hyperparameter Optimization},
 journal = {Journal of Machine Learning Research},
 volume = {18},
 number = {1},
 month = jan,
 year = {2017},
 issn = {1532-4435},
 pages = {6765--6816},
 numpages = {52},
 url = {http://dl.acm.org/citation.cfm?id=3122009.3242042}
} 


@inproceedings{golovin2017vizier,
 author = {Golovin, Daniel and Solnik, Benjamin and Moitra, Subhodeep and Kochanski, Greg and Karro, John and Sculley, D.},
 title = {Google Vizier: A Service for Black-Box Optimization},
 booktitle = {ACM SIGKDD Internationl Conference on Knowledge Discovery and Data Mining},
 year = {2017},
 pages = {1487--1495},
 numpages = {9}
}

@inproceedings{snoek2012practical,
title = {Practical Bayesian Optimization of Machine Learning Algorithms},
author = {Snoek, Jasper and Larochelle, Hugo and Adams, Ryan P},
booktitle = {Advances in Neural Information Processing Systems 25},
pages = {2951--2959},
year = {2012},
url = {http://papers.nips.cc/paper/4522-practical-bayesian-optimization-of-machine-learning-algorithms.pdf}
}

@article{jamieson2015non,
  author    = {Kevin G. Jamieson and Ameet Talwalkar},
  title     = {Non-stochastic Best Arm Identification and Hyperparameter Optimization},
  journal   = {CoRR},
  volume    = {abs/1502.07943},
  year      = {2015},
  url       = {http://arxiv.org/abs/1502.07943}
}

@inproceedings{audibert2010best,
  title={Best Arm Identification in Multi-Armed Bandits},
  author={Jean-Yves Audibert and
Sebastien Bubeck and R\'emi Munosr},
  booktitle={Proceedings of the annual Conference On Learning Theory (COLT)},
  pages={359--376},
  year={2010}
}

@article{brochu2010tutorial,
  title={A tutorial on Bayesian optimization of expensive cost functions, with application to active user modeling and hierarchical reinforcement learning},
  author={Brochu, Eric and Cora, Vlad M and De Freitas, Nando},
  journal={arXiv preprint arXiv:1012.2599},
  year={2010}
}

