@article{ying2016stochastic,
  title={Stochastic online AUC maximization},
  author={Ying, Yiming and Wen, Longyin and Lyu, Siwei},
  journal={Advances in neural information processing systems},
  volume={29},
  pages={451--459},
  year={2016}
}
@inproceedings{
Liu2020Stochastic,
title={Stochastic AUC Maximization with Deep Neural Networks},
author={Mingrui Liu and Zhuoning Yuan and Yiming Ying and Tianbao Yang},
booktitle={International Conference on Learning Representations},
year={2020},
}
@inproceedings{gao2013one,
  title={One-pass AUC optimization},
  author={Gao, Wei and Jin, Rong and Zhu, Shenghuo and Zhou, Zhi-Hua},
  booktitle={International conference on machine learning},
  pages={906--914},
  year={2013},
  organization={PMLR}
}


@incollection{Bengio+chapter2007,
author = {Bengio, Yoshua and LeCun, Yann},
booktitle = {Large Scale Kernel Machines},
publisher = {MIT Press},
title = {Scaling Learning Algorithms Towards {AI}},
year = {2007}
}


@inproceedings{kendall2018multi,
  title={Multi-task learning using uncertainty to weigh losses for scene geometry and semantics},
  author={Kendall, Alex and Gal, Yarin and Cipolla, Roberto},
  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
  pages={7482--7491},
  year={2018}
}

@article{chen2017gradnorm,
  title={Gradnorm: Gradient normalization for adaptive loss balancing in deep multitask networks},
  author={Chen, Zhao and Badrinarayanan, Vijay and Lee, Chen-Yu and Rabinovich, Andrew},
  journal={arXiv preprint arXiv:1711.02257},
  year={2017}
}



@article{nguyen2019toward,
  title={Toward understanding catastrophic forgetting in continual learning},
  author={Nguyen, Cuong V and Achille, Alessandro and Lam, Michael and Hassner, Tal and Mahadevan, Vijay and Soatto, Stefano},
  journal={arXiv preprint arXiv:1908.01091},
  year={2019}
}

@article{li2019learn,
  title={Learn to grow: A continual structure learning framework for overcoming catastrophic forgetting},
  author={Li, Xilai and Zhou, Yingbo and Wu, Tianfu and Socher, Richard and Xiong, Caiming},
  journal={arXiv preprint arXiv:1904.00310},
  year={2019}
}
@article{titsias2019functional,
  title={Functional regularisation for continual learning using gaussian processes},
  author={Titsias, Michalis K and Schwarz, Jonathan and Matthews, Alexander G de G and Pascanu, Razvan and Teh, Yee Whye},
  journal={arXiv preprint arXiv:1901.11356},
  year={2019}
}

@inproceedings{shin2017continual,
  title={Continual learning with deep generative replay},
  author={Shin, Hanul and Lee, Jung Kwon and Kim, Jaehong and Kim, Jiwon},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2990--2999},
  year={2017}
}

@inproceedings{lee2017overcoming,
  title={Overcoming catastrophic forgetting by incremental moment matching},
  author={Lee, Sang-Woo and Kim, Jin-Hwa and Jun, Jaehyun and Ha, Jung-Woo and Zhang, Byoung-Tak},
  booktitle={Advances in neural information processing systems},
  pages={4652--4662},
  year={2017}
}

@article{yoon2017lifelong,
  title={Lifelong learning with dynamically expandable networks},
  author={Yoon, Jaehong and Yang, Eunho and Lee, Jeongtae and Hwang, Sung Ju},
  journal={arXiv preprint arXiv:1708.01547},
  year={2017}
}

@article{adel2019continual,
  title={Continual Learning with Adaptive Weights (CLAW)},
  author={Adel, Tameem and Zhao, Han and Turner, Richard E},
  journal={arXiv preprint arXiv:1911.09514},
  year={2019}
}

@inproceedings{rolnick2019experience,
  title={Experience replay for continual learning},
  author={Rolnick, David and Ahuja, Arun and Schwarz, Jonathan and Lillicrap, Timothy and Wayne, Gregory},
  booktitle={Advances in Neural Information Processing Systems},
  pages={348--358},
  year={2019}

}

@article{parisi2019continual,
  title={Continual lifelong learning with neural networks: A review},
  author={Parisi, German I and Kemker, Ronald and Part, Jose L and Kanan, Christopher and Wermter, Stefan},
  journal={Neural Networks},
  year={2019},
  publisher={Elsevier}
}

@article{farajtabar2019orthogonal,
  title={Orthogonal Gradient Descent for Continual Learning},
  author={Farajtabar, Mehrdad and Azizan, Navid and Mott, Alex and Li, Ang},
  journal={arXiv preprint arXiv:1910.07104},
  year={2019}
}

@article{he2018overcoming,
  title={Overcoming catastrophic interference using conceptor-aided backpropagation},
  author={He, Xu and Jaeger, Herbert},
  year={2018}
}


@article{zeng2019continual,
  title={Continual learning of context-dependent processing in neural networks},
  author={Zeng, Guanxiong and Chen, Yang and Cui, Bo and Yu, Shan},
  journal={Nature Machine Intelligence},
  volume={1},
  number={8},
  pages={364--372},
  year={2019},
  publisher={Nature Publishing Group}
}

@article{guo2019new,
  title={A New Benchmark for Evaluation of Cross-Domain Few-Shot Learning},
  author={Guo, Yunhui and Codella, Noel CF and Karlinsky, Leonid and Smith, John R and Rosing, Tajana and Feris, Rogerio},
  journal={arXiv preprint arXiv:1912.07200},
  year={2019}
}

@inproceedings{finn2017model,
  title={Model-agnostic meta-learning for fast adaptation of deep networks},
  author={Finn, Chelsea and Abbeel, Pieter and Levine, Sergey},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={1126--1135},
  year={2017},
  organization={JMLR. org}
}

@inproceedings{vinyals2016matching,
  title={Matching networks for one shot learning},
  author={Vinyals, Oriol and Blundell, Charles and Lillicrap, Timothy and Wierstra, Daan and others},
  booktitle={Advances in neural information processing systems},
  pages={3630--3638},
  year={2016}
}

@inproceedings{snell2017prototypical,
  title={Prototypical networks for few-shot learning},
  author={Snell, Jake and Swersky, Kevin and Zemel, Richard},
  booktitle={Advances in neural information processing systems},
  pages={4077--4087},
  year={2017}
}

@article{toneva2018empirical,
  title={An empirical study of example forgetting during deep neural network learning},
  author={Toneva, Mariya and Sordoni, Alessandro and Combes, Remi Tachet des and Trischler, Adam and Bengio, Yoshua and Gordon, Geoffrey J},
  journal={arXiv preprint arXiv:1812.05159},
  year={2018}
}

@inproceedings{kemker2018measuring,
  title={Measuring catastrophic forgetting in neural networks},
  author={Kemker, Ronald and McClure, Marc and Abitino, Angelina and Hayes, Tyler L and Kanan, Christopher},
  booktitle={Thirty-second AAAI conference on artificial intelligence},
  year={2018}
}

@inproceedings{hayes2018new,
  title={New metrics and experimental paradigms for continual learning},
  author={Hayes, Tyler L and Kemker, Ronald and Cahill, Nathan D and Kanan, Christopher},
  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition Workshops},
  pages={2031--2034},
  year={2018}
}

@article{farquhar2019unifying,
  title={A unifying bayesian view of continual learning},
  author={Farquhar, Sebastian and Gal, Yarin},
  journal={arXiv preprint arXiv:1902.06494},
  year={2019}
}


@article{farquhar2018towards,
  title={Towards robust evaluations of continual learning},
  author={Farquhar, Sebastian and Gal, Yarin},
  journal={arXiv preprint arXiv:1805.09733},
  year={2018}

}

@article{crammer2006online,
  title={Online passive-aggressive algorithms},
  author={Crammer, Koby and Dekel, Ofer and Keshet, Joseph and Shalev-Shwartz, Shai and Singer, Yoram},
  journal={Journal of Machine Learning Research},
  volume={7},
  number={Mar},
  pages={551--585},
  year={2006}
}

@inproceedings{sener2018multi,
  title={Multi-task learning as multi-objective optimization},
  author={Sener, Ozan and Koltun, Vladlen},
  booktitle={Advances in Neural Information Processing Systems},
  pages={527--538},
  year={2018}
}

@article{yu2020gradient,
  title={Gradient surgery for multi-task learning},
  author={Yu, Tianhe and Kumar, Saurabh and Gupta, Abhishek and Levine, Sergey and Hausman, Karol and Finn, Chelsea},
  journal={arXiv preprint arXiv:2001.06782},
  year={2020}
}

@inproceedings{aljundi2019task,
  title={Task-free continual learning},
  author={Aljundi, Rahaf and Kelchtermans, Klaas and Tuytelaars, Tinne},
  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
  pages={11254--11263},
  year={2019}
}
@article{chaudhry2019tiny,
  title={On tiny episodic memories in continual learning},
  author={Chaudhry, Arslan and Rohrbach, Marcus and Elhoseiny, Mohamed and Ajanthan, Thalaiyasingam and Dokania, Puneet K and Torr, Philip HS and Ranzato, Marc'Aurelio},
  journal={arXiv preprint arXiv:1902.10486},
  year={2019}
}

@article{bradley1997use,
  title={The use of the area under the ROC curve in the evaluation of machine learning algorithms},
  author={Bradley, Andrew P},
  journal={Pattern recognition},
  volume={30},
  number={7},
  pages={1145--1159},
  year={1997},
  publisher={Elsevier}
}

@article{tasar2019incremental,
  title={Incremental learning for semantic segmentation of large-scale remote sensing data},
  author={Tasar, Onur and Tarabalka, Yuliya and Alliez, Pierre},
  journal={IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing},
  volume={12},
  number={9},
  pages={3524--3537},
  year={2019},
  publisher={IEEE}
}

@article{hu2022continual,
  title={Continual Learning for CTR Prediction: A Hybrid Approach},
  author={Hu, Ke and Qi, Yi and Huang, Jianqiang and Cheng, Jia and Lei, Jun},
  journal={arXiv preprint arXiv:2201.06886},
  year={2022}
}

@inproceedings{chrysakis2020online,
  title={Online continual learning from imbalanced data},
  author={Chrysakis, Aristotelis and Moens, Marie-Francine},
  booktitle={International Conference on Machine Learning},
  pages={1952--1961},
  year={2020},
  organization={PMLR}
}

@inproceedings{aljundi2019gradient,
  title={Gradient based sample selection for online continual learning},
  author={Aljundi, Rahaf and Lin, Min and Goujaud, Baptiste and Bengio, Yoshua},
  booktitle={Advances in Neural Information Processing Systems},
  pages={11816--11825},
  year={2019}
}


@inproceedings{de2019episodic,
  title={Episodic Memory in Lifelong Language Learning},
  author={de Masson d'Autume, Cyprien and Ruder, Sebastian and Kong, Lingpeng and Yogatama, Dani},
  booktitle={Advances in Neural Information Processing Systems},
  pages={13122--13131},
  year={2019}
}


@article{weston2014memory,
  title={Memory networks},
  author={Weston, Jason and Chopra, Sumit and Bordes, Antoine},
  journal={arXiv preprint arXiv:1410.3916},
  year={2014}
}

@article{santoro2016one,
  title={One-shot learning with memory-augmented neural networks},
  author={Santoro, Adam and Bartunov, Sergey and Botvinick, Matthew and Wierstra, Daan and Lillicrap, Timothy},
  journal={arXiv preprint arXiv:1605.06065},
  year={2016}
}

@article{pritzel2017neural,
  title={Neural episodic control},
  author={Pritzel, Alexander and Uria, Benigno and Srinivasan, Sriram and Puigdomenech, Adria and Vinyals, Oriol and Hassabis, Demis and Wierstra, Daan and Blundell, Charles},
  journal={arXiv preprint arXiv:1703.01988},
  year={2017}
}

@article{de2019continual,
  title={Continual learning: A comparative study on how to defy forgetting in classification tasks},
  author={De Lange, Matthias and Aljundi, Rahaf and Masana, Marc and Parisot, Sarah and Jia, Xu and Leonardis, Ales and Slabaugh, Gregory and Tuytelaars, Tinne},
  journal={arXiv preprint arXiv:1909.08383},
  year={2019}
}

@InProceedings{Rannen_2017_ICCV,
author = {Rannen, Amal and Aljundi, Rahaf and Blaschko, Matthew B. and Tuytelaars, Tinne},
title = {Encoder Based Lifelong Learning},
booktitle = {The IEEE International Conference on Computer Vision (ICCV)},
month = {Oct},
year = {2017}
}

@article{ebrahimi2019uncertainty,
  title={Uncertainty-guided continual learning with bayesian neural networks},
  author={Ebrahimi, Sayna and Elhoseiny, Mohamed and Darrell, Trevor and Rohrbach, Marcus},
  journal={arXiv preprint arXiv:1906.02425},
  year={2019}
}

@inproceedings{ritter2018online,
  title={Online structured laplace approximations for overcoming catastrophic forgetting},
  author={Ritter, Hippolyt and Botev, Aleksandar and Barber, David},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3738--3748},
  year={2018}
}



@inproceedings{lee2019overcoming,
  title={Overcoming Catastrophic Forgetting With Unlabeled Data in the Wild},
  author={Lee, Kibok and Lee, Kimin and Shin, Jinwoo and Lee, Honglak},
  booktitle={Proceedings of the IEEE International Conference on Computer Vision},
  pages={312--321},
  year={2019}
}

@inproceedings{aljundi2018memory,
  title={Memory aware synapses: Learning what (not) to forget},
  author={Aljundi, Rahaf and Babiloni, Francesca and Elhoseiny, Mohamed and Rohrbach, Marcus and Tuytelaars, Tinne},
  booktitle={Proceedings of the European Conference on Computer Vision (ECCV)},
  pages={139--154},
  year={2018}
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@inproceedings{lampert2009learning,
  title={Learning to detect unseen object classes by between-class attribute transfer},
  author={Lampert, Christoph H and Nickisch, Hannes and Harmeling, Stefan},
  booktitle={2009 IEEE Conference on Computer Vision and Pattern Recognition},
  pages={951--958},
  year={2009},
  organization={IEEE}
}

@article{wah2011caltech,
  title={The caltech-ucsd birds-200-2011 dataset},
  author={Wah, Catherine and Branson, Steve and Welinder, Peter and Perona, Pietro and Belongie, Serge},
  year={2011},
  publisher={California Institute of Technology}
}

@techreport{krizhevsky2009learning,
  title={Learning multiple layers of features from tiny images},
  author={Krizhevsky, Alex and others},
  year={2009},
  institution={Citeseer}
}

@article{lecun1998mnist,
  title={The MNIST database},
  author={LeCun, Yann and Cortes, Corinna and Burges, Christopher JC},
  journal={URL http://yann. lecun. com/exdb/mnist},
  year={1998}
}

@article{Hinton06,
author = {Hinton, Geoffrey E. and Osindero, Simon and Teh, Yee Whye},
journal = {Neural Computation},
pages = {1527--1554},
title = {A Fast Learning Algorithm for Deep Belief Nets},
volume = {18},
year = {2006}
}

@book{goodfellow2016deep,
title={Deep learning},
author={Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron and Bengio, Yoshua},
volume={1},
year={2016},
publisher={MIT Press}
}

@inproceedings{rebuffi2017icarl,
  title={icarl: Incremental classifier and representation learning},
  author={Rebuffi, Sylvestre-Alvise and Kolesnikov, Alexander and Sperl, Georg and Lampert, Christoph H},
  booktitle={Proceedings of the IEEE conference on Computer Vision and Pattern Recognition},
  pages={2001--2010},
  year={2017}
}

@article{chaudhry2018efficient,
  title={Efficient lifelong learning with a-gem},
  author={Chaudhry, Arslan and Ranzato, Marc'Aurelio and Rohrbach, Marcus and Elhoseiny, Mohamed},
  journal={arXiv preprint arXiv:1812.00420},
  year={2018}
}

@article{ramyachitra2014imbalanced,
  title={Imbalanced dataset classification and solutions: a review},
  author={Ramyachitra, D and Manikandan, P},
  journal={International Journal of Computing and Business Research (IJCBR)},
  volume={5},
  number={4},
  pages={1--29},
  year={2014}
}

@inproceedings{lopez2017gradient,
  title={Gradient episodic memory for continual learning},
  author={Lopez-Paz, David and others},
  booktitle={Advances in Neural Information Processing Systems},
  pages={6467--6476},
  year={2017}
}

@article{guo2020improved,
  title={Improved Schemes for Episodic Memory-based Lifelong Learning},
  author={Guo, Yunhui and Liu, Mingrui and Yang, Tianbao and Rosing, Tajana},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}

@article{kirkpatrick2017overcoming,
  title={Overcoming catastrophic forgetting in neural networks},
  author={Kirkpatrick, James and Pascanu, Razvan and Rabinowitz, Neil and Veness, Joel and Desjardins, Guillaume and Rusu, Andrei A and Milan, Kieran and Quan, John and Ramalho, Tiago and Grabska-Barwinska, Agnieszka and others},
  journal={Proceedings of the national academy of sciences},
  volume={114},
  number={13},
  pages={3521--3526},
  year={2017},
  publisher={National Acad Sciences}
}

@article{rusu2016progressive,
  title={Progressive neural networks},
  author={Rusu, Andrei A and Rabinowitz, Neil C and Desjardins, Guillaume and Soyer, Hubert and Kirkpatrick, James and Kavukcuoglu, Koray and Pascanu, Razvan and Hadsell, Raia},
  journal={arXiv preprint arXiv:1606.04671},
  year={2016}
}

@inproceedings{zenke2017continual,
  title={Continual learning through synaptic intelligence},
  author={Zenke, Friedemann and Poole, Ben and Ganguli, Surya},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={3987--3995},
  year={2017},
  organization={JMLR. org}
}

@article{nguyen2017variational,
  title={Variational continual learning},
  author={Nguyen, Cuong V and Li, Yingzhen and Bui, Thang D and Turner, Richard E},
  journal={arXiv preprint arXiv:1710.10628},
  year={2017}
}
@article{french1999catastrophic,
  title={Catastrophic forgetting in connectionist networks},
  author={French, Robert M},
  journal={Trends in cognitive sciences},
  volume={3},
  number={4},
  pages={128--135},
  year={1999},
  publisher={Elsevier}
}
@article{riemer2018learning,
  title={Learning to learn without forgetting by maximizing transfer and minimizing interference},
  author={Riemer, Matthew and Cases, Ignacio and Ajemian, Robert and Liu, Miao and Rish, Irina and Tu, Yuhai and Tesauro, Gerald},
  journal={arXiv preprint arXiv:1810.11910},
  year={2018}
}

@inproceedings{chaudhry2018riemannian,
  title={Riemannian walk for incremental learning: Understanding forgetting and intransigence},
  author={Chaudhry, Arslan and Dokania, Puneet K and Ajanthan, Thalaiyasingam and Torr, Philip HS},
  booktitle={Proceedings of the European Conference on Computer Vision (ECCV)},
  pages={532--547},
  year={2018}
}

@article{thrun1995lifelong,
  title={Lifelong robot learning},
  author={Thrun, Sebastian and Mitchell, Tom M},
  journal={Robotics and autonomous systems},
  volume={15},
  number={1-2},
  pages={25--46},
  year={1995},
  publisher={Elsevier}
}

@article{ratcliff1990connectionist,
  title={Connectionist models of recognition memory: constraints imposed by learning and forgetting functions.},
  author={Ratcliff, Roger},
  journal={Psychological review},
  volume={97},
  number={2},
  pages={285},
  year={1990},
  publisher={American Psychological Association}
}

@article{robins1995catastrophic,
  title={Catastrophic forgetting, rehearsal and pseudorehearsal},
  author={Robins, Anthony},
  journal={Connection Science},
  volume={7},
  number={2},
  pages={123--146},
  year={1995},
  publisher={Taylor \& Francis}
}
@article{cesa2004generalization,
  title={On the generalization ability of on-line learning algorithms},
  author={Cesa-Bianchi, Nicolo and Conconi, Alex and Gentile, Claudio},
  journal={IEEE Transactions on Information Theory},
  volume={50},
  number={9},
  pages={2050--2057},
  year={2004},
  publisher={IEEE}
}


@article{baxter2000model,
  title={A model of inductive bias learning},
  author={Baxter, Jonathan},
  journal={Journal of artificial intelligence research},
  volume={12},
  pages={149--198},
  year={2000}
}

@book{nesterov2018lectures,
  title={Lectures on convex optimization},
  author={Nesterov, Yurii},
  volume={137},
  year={2018},
  publisher={Springer}
}
@article{nesterov1998introductory,
  title={Introductory lectures on convex programming volume i: Basic course},
  author={Nesterov, Yurii},
  journal={Lecture notes},
  volume={3},
  number={4},
  pages={5},
  year={1998}
}


@article{helber2019eurosat,
  title={Eurosat: A novel dataset and deep learning benchmark for land use and land cover classification},
  author={Helber, Patrick and Bischke, Benjamin and Dengel, Andreas and Borth, Damian},
  journal={IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing},
  year={2019},
  publisher={IEEE}
}

@article{hinton2015distilling,
  title={Distilling the knowledge in a neural network},
  author={Hinton, Geoffrey and Vinyals, Oriol and Dean, Jeff},
  journal={arXiv preprint arXiv:1503.02531},
  year={2015}
}

@article{DBLP:journals/corr/GutmanCCHMMH16,
  author    = {David A. Gutman and
               Noel C. F. Codella and
               M. Emre Celebi and
               Brian Helba and
               Michael A. Marchetti and
               Nabin K. Mishra and
               Allan Halpern},
  title     = {Skin Lesion Analysis toward Melanoma Detection: {A} Challenge at the
               International Symposium on Biomedical Imaging {(ISBI)} 2016, hosted
               by the International Skin Imaging Collaboration {(ISIC)}},
  journal   = {CoRR},
  volume    = {abs/1605.01397},
  year      = {2016},
  eprinttype = {arXiv},
  eprint    = {1605.01397},
  timestamp = {Tue, 29 Jun 2021 15:47:35 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/GutmanCCHMMH16.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{yuan2021robust,
	title={Large-scale Robust Deep AUC Maximization: A New Surrogate Loss and Empirical Studies on Medical Image Classification},
	author={Yuan, Zhuoning and Yan, Yan and Sonka, Milan and Yang, Tianbao},
	booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
	year={2021}
}

@article{prabhu2020greedy,
  title={GDumb: A Simple Approach that Questions Our Progress in Continual Learning},
  author={Prabhu, Ameya and Torr, Philip and Dokania, Puneet},
  booktitle={The European Conference on Computer Vision (ECCV)},
  month={August},
  year={2020}
}



@article{zhao2011online,
  title={Online AUC maximization},
  author={Zhao, Peilin and Hoi, Steven CH and Jin, Rong and Yang, Tianbao},
  booktitle={International Conference on Machine Learning},
  year={2011}
}

@inproceedings{yu_linear,
  author    = {Hao Yu and
               Rong Jin and
               Sen Yang},
  title     = {On the Linear Speedup Analysis of Communication Efficient Momentum
               {SGD} for Distributed Non-Convex Optimization},
  booktitle = {Proceedings of the 36th International Conference on Machine Learning,
               {ICML} 2019, 9-15 June 2019, Long Beach, California, {USA}},
  pages     = {7184--7193},
  year      = {2019}
}


@inproceedings{DBLP:conf/nips/Yuan0JY19,
  author    = {Zhuoning Yuan and
               Yan Yan and
               Rong Jin and
               Tianbao Yang},
  title     = {Stagewise Training Accelerates Convergence of Testing Error Over {SGD}},
  booktitle = {Advances in Neural Information Processing Systems 32: Annual Conference
               on Neural Information Processing Systems 2019, NeurIPS 2019, 8-14
               December 2019, Vancouver, BC, Canada},
  pages     = {2604--2614},
  year      = {2019},
}

@inproceedings{yu_dynamic,
  author    = {Hao Yu and
               Rong Jin},
  title     = {On the Computation and Communication Complexity of Parallel {SGD}
               with Dynamic Batch Sizes for Stochastic Non-Convex Optimization},
  booktitle = {Proceedings of the 36th International Conference on Machine Learning,
               {ICML} 2019, 9-15 June 2019, Long Beach, California, {USA}},
  pages     = {7174--7183},
  year      = {2019}
}

@inproceedings{haddadpour2019trading,
  title={Trading Redundancy for Communication: Speeding up Distributed SGD for Non-convex Optimization},
  author={Haddadpour, Farzin and Kamani, Mohammad Mahdi and Mahdavi, Mehrdad and Cadambe, Viveck},
  booktitle={International Conference on Machine Learning},
  pages={2545--2554},
  year={2019}
}

@article{zhang2019distributed,
  title={Distributed Optimization for Over-Parameterized Learning},
  author={Zhang, Chi and Li, Qianxiao},
  journal={arXiv preprint arXiv:1906.06205},
  year={2019}
}

@book{DBLP:books/sp/Nesterov04,
  author    = {Yurii E. Nesterov},
  title     = {Introductory Lectures on Convex Optimization - {A} Basic Course},
  series    = {Applied Optimization},
  volume    = {87},
  publisher = {Springer},
  year      = {2004}
}

@article{liu2019stochastic,
  title={Stochastic AUC Maximization with Deep Neural Networks},
  author={Liu, Mingrui and Yuan, Zhuoning and Ying, Yiming and Yang, Tianbao},
  journal={ICLR},
  year={2020}
}

@inproceedings{DBLP:conf/iclr/Stich19,
  author    = {Sebastian U. Stich},
  title     = {Local {SGD} Converges Fast and Communicates Little},
  booktitle = {7th International Conference on Learning Representations, {ICLR} 2019, New Orleans, LA, USA, May 6-9, 2019},
  year = {2019}
}
@inproceedings{li2014scaling,
  title={Scaling distributed machine learning with the parameter server},
  author={Li, Mu and Andersen, David G and Park, Jun Woo and Smola, Alexander J and Ahmed, Amr and Josifovski, Vanja and Long, James and Shekita, Eugene J and Su, Bor-Yiing},
  booktitle={11th $\{$USENIX$\}$ Symposium on Operating Systems Design and Implementation ($\{$OSDI$\}$ 14)},
  pages={583--598},
  year={2014}
}
@inproceedings{dean2012large,
  title={Large scale distributed deep networks},
  author={Dean, Jeffrey and Corrado, Greg and Monga, Rajat and Chen, Kai and Devin, Matthieu and Mao, Mark and Ranzato, Marc'aurelio and Senior, Andrew and Tucker, Paul and Yang, Ke and others},
  booktitle={Advances in neural information processing systems},
  pages={1223--1231},
  year={2012}
}
@article{goyal2017accurate,
  title={Accurate, large minibatch sgd: Training imagenet in 1 hour},
  author={Goyal, Priya and Doll{\'a}r, Piotr and Girshick, Ross and Noordhuis, Pieter and Wesolowski, Lukasz and Kyrola, Aapo and Tulloch, Andrew and Jia, Yangqing and He, Kaiming},
  journal={arXiv preprint arXiv:1706.02677},
  year={2017}
}
@article{brock2018large,
  title={Large scale gan training for high fidelity natural image synthesis},
  author={Brock, Andrew and Donahue, Jeff and Simonyan, Karen},
  journal={arXiv preprint arXiv:1809.11096},
  year={2018}
}
@article{silver2017mastering,
  title={Mastering the game of go without human knowledge},
  author={Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and others},
  journal={Nature},
  volume={550},
  number={7676},
  pages={354--359},
  year={2017},
  publisher={Nature Publishing Group}
}
@article{silver2016mastering,
  title={Mastering the game of Go with deep neural networks and tree search},
  author={Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
  journal={nature},
  volume={529},
  number={7587},
  pages={484},
  year={2016},
  publisher={Nature Publishing Group}
}
@inproceedings{yang2019xlnet,
  title={Xlnet: Generalized autoregressive pretraining for language understanding},
  author={Yang, Zhilin and Dai, Zihang and Yang, Yiming and Carbonell, Jaime and Salakhutdinov, Russ R and Le, Quoc V},
  booktitle={Advances in neural information processing systems},
  pages={5754--5764},
  year={2019}
}
@article{yu2019linear,
  title={On the linear speedup analysis of communication efficient momentum sgd for distributed non-convex optimization},
  author={Yu, Hao and Jin, Rong and Yang, Sen},
  journal={arXiv preprint arXiv:1905.03817},
  year={2019}
}
@inproceedings{yu2019parallel,
  title={Parallel restarted SGD with faster convergence and less communication: Demystifying why model averaging works for deep learning},
  author={Yu, Hao and Yang, Sen and Zhu, Shenghuo},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={33},
  pages={5693--5700},
  year={2019}
}
@inproceedings{jiang2018linear,
  title={A linear speedup analysis of distributed deep learning with sparse and quantized communication},
  author={Jiang, Peng and Agrawal, Gagan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2525--2536},
  year={2018}
}
@article{stich2018local,
  title={Local SGD converges fast and communicates little},
  author={Stich, Sebastian U},
  journal={arXiv preprint arXiv:1805.09767},
  year={2018}
}
@article{wang2018cooperative,
  title={Cooperative SGD: A unified framework for the design and analysis of communication-efficient SGD algorithms},
  author={Wang, Jianyu and Joshi, Gauri},
  journal={arXiv preprint arXiv:1808.07576},
  year={2018}
}
@inproceedings{basu2019qsparse,
  title={Qsparse-local-SGD: Distributed SGD with Quantization, Sparsification and Local Computations},
  author={Basu, Debraj and Data, Deepesh and Karakus, Can and Diggavi, Suhas},
  booktitle={Advances in Neural Information Processing Systems},
  pages={14668--14679},
  year={2019}
}
@article{zhou2017convergence,
  title={On the convergence properties of a $ K $-step averaging stochastic gradient descent algorithm for nonconvex optimization},
  author={Zhou, Fan and Cong, Guojing},
  journal={arXiv preprint arXiv:1708.01012},
  year={2017}
}
@inproceedings{haddadpour2019local,
  title={Local SGD with periodic averaging: Tighter analysis and adaptive synchronization},
  author={Haddadpour, Farzin and Kamani, Mohammad Mahdi and Mahdavi, Mehrdad and Cadambe, Viveck},
  booktitle={Advances in Neural Information Processing Systems},
  pages={11080--11092},
  year={2019}
}
@article{wang2018adaptive,
  title={Adaptive communication strategies to achieve the best error-runtime trade-off in local-update SGD},
  author={Wang, Jianyu and Joshi, Gauri},
  journal={arXiv preprint arXiv:1810.08313},
  year={2018}
}
@inproceedings{stich2018sparsified,
  title={Sparsified SGD with memory},
  author={Stich, Sebastian U and Cordonnier, Jean-Baptiste and Jaggi, Martin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4447--4458},
  year={2018}
}
@inproceedings{wangni2018gradient,
  title={Gradient sparsification for communication-efficient distributed optimization},
  author={Wangni, Jianqiao and Wang, Jialei and Liu, Ji and Zhang, Tong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1299--1309},
  year={2018}
}
This file was created with JabRef 1.8.1.
Encoding: Cp1252

@STRING{aaai = {Proceedings of \ AAAI Conference on Artificial Intelligence}}

@STRING{colt = {Proceedings \ Annual Conf.\ on Learning Theory}}

@STRING{cvpr = {Proc.\ CVPR}}

@STRING{ecml = {Proc.\ European Conf.\ on Mach. Learn.}}

@STRING{ewcbr = {Proc.\ the European Workshop on Advances in Case-Based Reasoning}}

@STRING{fg = {Proc.\ Int.\ Conf.\ on Face and Gesture Recognition}}

@STRING{iccv = {Proc.\ Int.\ Conf.\ on Computer Vision}}

@STRING{icdm = {Proc.\ Int.\ Conf.\ on Data Mining}}

@STRING{icml = {Proceedings of the International Conference on Machine Learning}}

@STRING{icpr = {Proc.\ Int.\ Conf.\ on Pattern Recognition}}

@STRING{icvp = {Proc.\ Int.\ Conf.\ Vision and Pattern Recognition}}

@STRING{ijcai = {Proc.\ Int.\ Joint Conf.\ on Artificial Intelligence}}

@STRING{ijcv = {Int.\ J.\ of Computer Vision}}

@STRING{kdd = {Proc.\ ACM SIGKDD Int.\ Conf.\ on Knowledge Discovery and Data Mining}}

@STRING{mm = {Proc.\ ACM Multimedia}}

@STRING{nips = {NIPS}}

@STRING{pami = {IEEE Trans.\ Pattern Anal.\ Mach.\ Intell.}}

@STRING{sigir = {Proc.\ ACM SIGIR}}

@STRING{tvcg = {IEEE Trans. Visualization and Computer Graphics}}

@STRING{uai = {Proc.\ Uncertainty in Artificial Intelligence}}

@STRING{pcm = {Proc.\ Pacific-Rim Conf.\ on Multimedia}}

@STRING{nips = {Proc.\ Conf.\ on Advances in Neural Information Processing Systems}}


This file was created with JabRef 1.8.1.
Encoding: Cp1252

@STRING{aaai = {Proceedings of \ AAAI Conference on Artificial Intelligence}}

@STRING{colt = {Proceedings \ Annual Conf.\ on Learning Theory}}

@STRING{cvpr = {Proc.\ CVPR}}

@STRING{ecml = {Proc.\ European Conf.\ on Mach. Learn.}}

@STRING{ewcbr = {Proc.\ the European Workshop on Advances in Case-Based Reasoning}}

@STRING{fg = {Proc.\ Int.\ Conf.\ on Face and Gesture Recognition}}

@STRING{iccv = {Proc.\ Int.\ Conf.\ on Computer Vision}}

@STRING{icdm = {Proc.\ Int.\ Conf.\ on Data Mining}}

@STRING{icml = {Proceedings of the International Conference on Machine Learning}}

@STRING{icpr = {Proc.\ Int.\ Conf.\ on Pattern Recognition}}

@STRING{icvp = {Proc.\ Int.\ Conf.\ Vision and Pattern Recognition}}

@STRING{ijcai = {Proc.\ Int.\ Joint Conf.\ on Artificial Intelligence}}

@STRING{ijcv = {Int.\ J.\ of Computer Vision}}

@STRING{kdd = {Proc.\ ACM SIGKDD Int.\ Conf.\ on Knowledge Discovery and Data Mining}}

@STRING{mm = {Proc.\ ACM Multimedia}}

@STRING{nips = {NIPS}}

@STRING{pami = {IEEE Trans.\ Pattern Anal.\ Mach.\ Intell.}}

@STRING{sigir = {Proc.\ ACM SIGIR}}

@STRING{tvcg = {IEEE Trans. Visualization and Computer Graphics}}

@STRING{uai = {Proc.\ Uncertainty in Artificial Intelligence}}

@STRING{pcm = {Proc.\ Pacific-Rim Conf.\ on Multimedia}}

@STRING{nips = {Proc.\ Conf.\ on Advances in Neural Information Processing Systems}}


@article{nemirovski2009robust,
	title={Robust stochastic approximation approach to stochastic programming},
	author={Nemirovski, Arkadi and Juditsky, Anatoli and Lan, Guanghui and Shapiro, Alexander},
	journal={SIAM Journal on optimization},
	volume={19},
	number={4},
	pages={1574--1609},
	year={2009},
	publisher={SIAM}
}

@article{hazan2014beyond,
	title={Beyond the regret minimization barrier: optimal algorithms for stochastic strongly-convex optimization.},
	author={Hazan, Elad and Kale, Satyen},
	journal={Journal of Machine Learning Research},
	volume={15},
	number={1},
	pages={2489--2512},
	year={2014}
}


@article{li2013global,
	title={Global error bounds for piecewise convex polynomials},
	author={Li, Guoyin},
	journal={Mathematical Programming},
	pages={1--28},
	year={2013},
	publisher={Springer}
}
@article{clemenccon2008ranking,
	title={Ranking and empirical minimization of U-statistics},
	author={Cl{\'e}men{\c{c}}on, St{\'e}phan and Lugosi, G{\'a}bor and Vayatis, Nicolas},
	journal={The Annals of Statistics},
	pages={844--874},
	year={2008},
	publisher={JSTOR}
}


@article{powers2011evaluation,
	title={Evaluation: from precision, recall and F-measure to ROC, informedness, markedness and correlation},
	author={Powers, David Martin},
	year={2011},
	publisher={Bioinfo Publications}
}
@article{vitter1985random,
	title={Random sampling with a reservoir},
	author={Vitter, Jeffrey S},
	journal={ACM Transactions on Mathematical Software (TOMS)},
	volume={11},
	number={1},
	pages={37--57},
	year={1985},
	publisher={ACM}
}
@article{nemirovski2004prox,
	title={Prox-method with rate of convergence O (1/t) for variational inequalities with Lipschitz continuous monotone operators and smooth convex-concave saddle point problems},
	author={Nemirovski, Arkadi},
	journal={SIAM Journal on Optimization},
	volume={15},
	number={1},
	pages={229--251},
	year={2004},
	publisher={SIAM}
}
@article{korpelevich1976extragradient,
	title={The extragradient method for finding saddle points and other problems},
	author={Korpelevich, GM},
	journal={Matecon},
	volume={12},
	pages={747--756},
	year={1976}
}
@article{juditsky2011solving,
	title={Solving variational inequalities with stochastic mirror-prox algorithm},
	author={Juditsky, Anatoli and Nemirovski, Arkadi and Tauvel, Claire and others},
	journal={Stochastic Systems},
	volume={1},
	number={1},
	pages={17--58},
	year={2011},
	publisher={INFORMS Applied Probability Society}
}
@inproceedings{zinkevich2003online,
	title={Online convex programming and generalized infinitesimal gradient ascent},
	author={Zinkevich, Martin},
	booktitle={Proceedings of the 20th International Conference on Machine Learning (ICML-03)},
	pages={928--936},
	year={2003}
}
@article{nemirovskii1983problem,
	title={Problem complexity and method efficiency in optimization},
	author={Nemirovskii, Arkadii and Yudin, David Borisovich and Dawson, Edgar Ronald},
	year={1983},
	publisher={Wiley}
}
@book{shalev2014understanding,
	title={Understanding machine learning: From theory to algorithms},
	author={Shalev-Shwartz, Shai and Ben-David, Shai},
	year={2014},
	publisher={Cambridge university press}
}
@book{vapnik1998statistical,
	title={Statistical learning theory},
	author={Vapnik, Vladimir Naumovich and Vapnik, Vlamimir},
	volume={1},
	year={1998},
	publisher={Wiley New York}
}
@article{shalev2010learnability,
	title={Learnability, stability and uniform convergence},
	author={Shalev-Shwartz, Shai and Shamir, Ohad and Srebro, Nathan and Sridharan, Karthik},
	journal={Journal of Machine Learning Research},
	volume={11},
	number={Oct},
	pages={2635--2670},
	year={2010}
}
@book{nesterov2013introductory,
	title={Introductory lectures on convex optimization: A basic course},
	author={Nesterov, Yurii},
	volume={87},
	year={2013},
	publisher={Springer Science \& Business Media}
}
@incollection{bottou2010large,
	title={Large-scale machine learning with stochastic gradient descent},
	author={Bottou, L{\'e}on},
	booktitle={Proceedings of COMPSTAT'2010},
	pages={177--186},
	year={2010},
	publisher={Springer}
}
@inproceedings{defazio2014saga,
	title={Saga: A fast incremental gradient method with support for non-strongly convex composite objectives},
	author={Defazio, Aaron and Bach, Francis and Lacoste-Julien, Simon},
	booktitle={Advances in Neural Information Processing Systems},
	pages={1646--1654},
	year={2014}
}
@article{allen2016katyusha,
	title={Katyusha: Accelerated variance reduction for faster SGD},
	author={Allen-Zhu, Zeyuan},
	journal={ArXiv e-prints, abs/1603.05953},
	year={2016}
}
@article{lan2015optimal,
	title={An optimal randomized incremental gradient method},
	author={Lan, Guanghui and Zhou, Yi},
	journal={arXiv preprint arXiv:1507.02000},
	year={2015}
}
@inproceedings{zhang2015stochastic,
	title={Stochastic Primal-Dual Coordinate Method for Regularized Empirical Risk Minimization.},
	author={Zhang, Yuchen and Xiao, Lin},
	booktitle={ICML},
	pages={353--361},
	year={2015}
}
@inproceedings{lin2014accelerated,
	title={An accelerated proximal coordinate gradient method},
	author={Lin, Qihang and Lu, Zhaosong and Xiao, Lin},
	booktitle={Advances in Neural Information Processing Systems},
	pages={3059--3067},
	year={2014}
}
@inproceedings{shalev2014accelerated,
	title={Accelerated proximal stochastic dual coordinate ascent for regularized loss minimization.},
	author={Shalev-Shwartz, Shai and Zhang, Tong},
	booktitle={ICML},
	pages={64--72},
	year={2014}
}
@article{xiao2014proximal,
	title={A proximal stochastic gradient method with progressive variance reduction},
	author={Xiao, Lin and Zhang, Tong},
	journal={SIAM Journal on Optimization},
	volume={24},
	number={4},
	pages={2057--2075},
	year={2014},
	publisher={SIAM}
}
@inproceedings{shalev2013accelerated,
	title={Accelerated mini-batch stochastic dual coordinate ascent},
	author={Shalev-Shwartz, Shai and Zhang, Tong},
	booktitle={Advances in Neural Information Processing Systems},
	pages={378--385},
	year={2013}
}
@article{ouyang2013stochastic,
	title={Stochastic Alternating Direction Method of Multipliers.},
	author={Ouyang, Hua and He, Niao and Tran, Long and Gray, Alexander G},
	journal={ICML (1)},
	volume={28},
	pages={80--88},
	year={2013}
}
@inproceedings{suzuki2013dual,
	title={Dual Averaging and Proximal Gradient Descent for Online Alternating Direction Multiplier Method.},
	author={Suzuki, Taiji and others},
	booktitle={ICML (1)},
	pages={392--400},
	year={2013}
}
@inproceedings{recht2011hogwild,
	title={Hogwild: A lock-free approach to parallelizing stochastic gradient descent},
	author={Recht, Benjamin and Re, Christopher and Wright, Stephen and Niu, Feng},
	booktitle={Advances in Neural Information Processing Systems},
	pages={693--701},
	year={2011}
}
@inproceedings{yang2013trading,
	title={Trading computation for communication: Distributed stochastic dual coordinate ascent},
	author={Yang, Tianbao},
	booktitle={Advances in Neural Information Processing Systems},
	pages={629--637},
	year={2013}
}
@inproceedings{zhang2014asynchronous,
	title={Asynchronous Distributed ADMM for Consensus Optimization.},
	author={Zhang, Ruiliang and Kwok, James T},
	booktitle={ICML},
	pages={1701--1709},
	year={2014}
}
@inproceedings{roux2012stochastic,
	title={A stochastic gradient method with an exponential convergence \_rate for finite training sets},
	author={Roux, Nicolas L and Schmidt, Mark and Bach, Francis R},
	booktitle={Advances in Neural Information Processing Systems},
	pages={2663--2671},
	year={2012}
}
@inproceedings{defazio2016simple,
	title={A simple practical accelerated method for finite sums},
	author={Defazio, Aaron},
	booktitle={Advances In Neural Information Processing Systems},
	pages={676--684},
	year={2016}
}
@article{yang2013analysis,
	title={Analysis of distributed stochastic dual coordinate ascent},
	author={Yang, Tianbao and Zhu, Shenghuo and Jin, Rong and Lin, Yuanqing},
	journal={arXiv preprint arXiv:1312.1031},
	year={2013}
}
@inproceedings{recht2011hogwild,
	title={Hogwild: A lock-free approach to parallelizing stochastic gradient descent},
	author={Recht, Benjamin and Re, Christopher and Wright, Stephen and Niu, Feng},
	booktitle={Advances in Neural Information Processing Systems},
	pages={693--701},
	year={2011}
}
@inproceedings{agarwal2011distributed,
	title={Distributed delayed stochastic optimization},
	author={Agarwal, Alekh and Duchi, John C},
	booktitle={Advances in Neural Information Processing Systems},
	pages={873--881},
	year={2011}
}
@article{gabay1976dual,
	title={A dual algorithm for the solution of nonlinear variational problems via finite element approximation},
	author={Gabay, Daniel and Mercier, Bertrand},
	journal={Computers \& Mathematics with Applications},
	volume={2},
	number={1},
	pages={17--40},
	year={1976},
	publisher={Elsevier}
}
@inproceedings{lian2015asynchronous,
	title={Asynchronous parallel stochastic gradient for nonconvex optimization},
	author={Lian, Xiangru and Huang, Yijun and Li, Yuncheng and Liu, Ji},
	booktitle={Advances in Neural Information Processing Systems},
	pages={2737--2745},
	year={2015}
}
@inproceedings{shalev2014accelerated,
	title={Accelerated proximal stochastic dual coordinate ascent for regularized loss minimization},
	author={Shalev-Shwartz, Shai and Zhang, Tong},
	booktitle={International Conference on Machine Learning},
	pages={64--72},
	year={2014}
}
@inproceedings{johnson2013accelerating,
	title={Accelerating stochastic gradient descent using predictive variance reduction},
	author={Johnson, Rie and Zhang, Tong},
	booktitle={Advances in neural information processing systems},
	pages={315--323},
	year={2013}
}
@inproceedings{defazio2014saga,
	title={Saga: A fast incremental gradient method with support for non-strongly convex composite objectives},
	author={Defazio, Aaron and Bach, Francis and Lacoste-Julien, Simon},
	booktitle={Advances in Neural Information Processing Systems},
	pages={1646--1654},
	year={2014}
}
@article{shalev2013stochastic,
	title={Stochastic dual coordinate ascent methods for regularized loss minimization},
	author={Shalev-Shwartz, Shai and Zhang, Tong},
	journal={Journal of Machine Learning Research},
	volume={14},
	number={Feb},
	pages={567--599},
	year={2013}
}
@incollection{bottou2010large,
	title={Large-scale machine learning with stochastic gradient descent},
	author={Bottou, L{\'e}on},
	booktitle={Proceedings of COMPSTAT'2010},
	pages={177--186},
	year={2010},
	publisher={Springer}
}
@article{bottou2016optimization,
	title={Optimization methods for large-scale machine learning},
	author={Bottou, L{\'e}on and Curtis, Frank E and Nocedal, Jorge},
	journal={arXiv preprint arXiv:1606.04838},
	year={2016}
}
@book{cesa2006prediction,
	title={Prediction, learning, and games},
	author={Cesa-Bianchi, Nicolo and Lugosi, G{\'a}bor},
	year={2006},
	publisher={Cambridge university press}
}
@book{boyd2004convex,
	title={Convex optimization},
	author={Boyd, Stephen and Vandenberghe, Lieven},
	year={2004},
	publisher={Cambridge university press}
}
@inproceedings{busa2015online,
	title={Online F-measure optimization},
	author={Busa-Fekete, R{\'o}bert and Sz{\"o}r{\'e}nyi, Bal{\'a}zs and Dembczynski, Krzysztof and H{\"u}llermeier, Eyke},
	booktitle={Advances in Neural Information Processing Systems},
	pages={595--603},
	year={2015}
}
@article{hardt2015train,
	title={Train faster, generalize better: Stability of stochastic gradient descent},
	author={Hardt, Moritz and Recht, Benjamin and Singer, Yoram},
	journal={arXiv preprint arXiv:1509.01240},
	year={2015}
}
@article{lecun2015deep,
	title={Deep learning},
	author={LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
	journal={Nature},
	volume={521},
	number={7553},
	pages={436--444},
	year={2015},
	publisher={Nature Research}
}
@inproceedings{nesterov1983method,
	title={A method of solving a convex programming problem with convergence rate O (1/k2)},
	author={Nesterov, Yurii},
	booktitle={Soviet Mathematics Doklady},
	volume={27},
	number={2},
	pages={372--376},
	year={1983}
}
@book{hestenes1952methods,
	title={Methods of conjugate gradients for solving linear systems},
	author={Hestenes, Magnus Rudolph and Stiefel, Eduard},
	volume={49},
	number={1},
	year={1952},
	publisher={NBS}
}
@article{cauchy1847methode,
	title={M{\'e}thode g{\'e}n{\'e}rale pour la r{\'e}solution des systemes d’{\'e}quations simultan{\'e}es},
	author={Cauchy, Augustin},
	journal={Comp. Rend. Sci. Paris},
	volume={25},
	number={1847},
	pages={536--538},
	year={1847}
}
@article{shalev2007online,
	title={Online learning: Theory, algorithms, and applications},
	author={Shalev-Shwartz, Shai and Singer, Yoram},
	year={2007},
	publisher={Citeseer}
}
@article{cesa2004generalization,
	title={On the generalization ability of on-line learning algorithms},
	author={Cesa-Bianchi, Nicolo and Conconi, Alex and Gentile, Claudio},
	journal={IEEE Transactions on Information Theory},
	volume={50},
	number={9},
	pages={2050--2057},
	year={2004},
	publisher={IEEE}
}
@inproceedings{shamir2013stochastic,
	title={Stochastic gradient descent for non-smooth optimization: Convergence results and optimal averaging schemes},
	author={Shamir, Ohad and Zhang, Tong},
	booktitle={International Conference on Machine Learning},
	pages={71--79},
	year={2013}
}
@article{hazan2014beyond,
	title={Beyond the regret minimization barrier: optimal algorithms for stochastic strongly-convex optimization.},
	author={Hazan, Elad and Kale, Satyen},
	journal={Journal of Machine Learning Research},
	volume={15},
	number={1},
	pages={2489--2512},
	year={2014}
}
@inproceedings{hazan2006logarithmic,
	title={Logarithmic regret algorithms for online convex optimization},
	author={Hazan, Elad and Kalai, Adam and Kale, Satyen and Agarwal, Amit},
	booktitle={COLT},
	volume={4005},
	pages={499--513},
	year={2006},
	organization={Springer}
}
@inproceedings{he2016deep,
	title={Deep residual learning for image recognition},
	author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
	booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
	pages={770--778},
	year={2016}
}
@inproceedings{girshick2015fast,
	title={Fast r-cnn},
	author={Girshick, Ross},
	booktitle={Proceedings of the IEEE international conference on computer vision},
	pages={1440--1448},
	year={2015}
}
@inproceedings{ren2015faster,
	title={Faster r-cnn: Towards real-time object detection with region proposal networks},
	author={Ren, Shaoqing and He, Kaiming and Girshick, Ross and Sun, Jian},
	booktitle={Advances in neural information processing systems},
	pages={91--99},
	year={2015}
}
@inproceedings{liu2018fast,
	title={Fast stochastic AUC maximization with O (1/n)-convergence rate},
	author={Liu, Mingrui and Zhang, Xiaoxuan and Chen, Zaiyi and Wang, Xiaoyu and Yang, Tianbao},
	booktitle={International Conference on Machine Learning},
	pages={3195--3203},
	year={2018}
}
@article{rafique2018non,
	title={Non-convex min-max optimization: Provable algorithms and applications in machine learning},
	author={Rafique, Hassan and Liu, Mingrui and Lin, Qihang and Yang, Tianbao},
	journal={arXiv preprint arXiv:1810.02060},
	year={2018}
}
@article{davis2017proximally,
	title={Proximally Guided Stochastic Subgradient Method for Nonsmooth, Nonconvex Problems},
	author={Davis, Damek and Grimmer, Benjamin},
	journal={arXiv preprint arXiv:1707.03505},
	year={2017}
}
@book{nesterov2013introductory,
	title={Introductory lectures on convex optimization: A basic course},
	author={Nesterov, Yurii},
	volume={87},
	year={2013},
	publisher={Springer Science \& Business Media}
}
@article{duchi2011adaptive,
	title={Adaptive subgradient methods for online learning and stochastic optimization},
	author={Duchi, John and Hazan, Elad and Singer, Yoram},
	journal={Journal of Machine Learning Research},
	volume={12},
	number={Jul},
	pages={2121--2159},
	year={2011}
}
@article{hanley1982meaning,
	title={The meaning and use of the area under a receiver operating characteristic (ROC) curve.},
	author={Hanley, James A and McNeil, Barbara J},
	journal={Radiology},
	volume={143},
	number={1},
	pages={29--36},
	year={1982}
}
@article{hanley1983method,
	title={A method of comparing the areas under receiver operating characteristic curves derived from the same cases.},
	author={Hanley, James A and McNeil, Barbara J},
	journal={Radiology},
	volume={148},
	number={3},
	pages={839--843},
	year={1983}
}
@inproceedings{elkan2001foundations,
	title={The foundations of cost-sensitive learning},
	author={Elkan, Charles},
	booktitle={International joint conference on artificial intelligence},
	volume={17},
	number={1},
	pages={973--978},
	year={2001},
	organization={Lawrence Erlbaum Associates Ltd}
}
@inproceedings{natole2018stochastic,
	title={Stochastic proximal algorithms for AUC maximization},
	author={Natole, Michael and Ying, Yiming and Lyu, Siwei},
	booktitle={International Conference on Machine Learning},
	pages={3707--3716},
	year={2018}
}
@inproceedings{krizhevsky2012imagenet,
	title={Imagenet classification with deep convolutional neural networks},
	author={Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
	booktitle={Advances in neural information processing systems},
	pages={1097--1105},
	year={2012}
}
@article{simonyan2014very,
	title={Very deep convolutional networks for large-scale image recognition},
	author={Simonyan, Karen and Zisserman, Andrew},
	journal={arXiv preprint arXiv:1409.1556},
	year={2014}
}
@inproceedings{karimi2016linear,
	title={Linear convergence of gradient and proximal-gradient methods under the polyak-{\l}ojasiewicz condition},
	author={Karimi, Hamed and Nutini, Julie and Schmidt, Mark},
	booktitle={Joint European Conference on Machine Learning and Knowledge Discovery in Databases},
	pages={795--811},
	year={2016},
	organization={Springer}
}
@article{polyak1963gradient,
	title={Gradient methods for minimizing functionals},
	author={Polyak, Boris Teodorovich},
	journal={Zhurnal Vychislitel'noi Matematiki i Matematicheskoi Fiziki},
	volume={3},
	number={4},
	pages={643--653},
	year={1963},
	publisher={Russian Academy of Sciences, Branch of Mathematical Sciences}
}
@inproceedings{lei2017non,
	title={Non-convex finite-sum optimization via scsg methods},
	author={Lei, Lihua and Ju, Cheng and Chen, Jianbo and Jordan, Michael I},
	booktitle={Advances in Neural Information Processing Systems},
	pages={2348--2358},
	year={2017}
}
@inproceedings{reddi2016stochastic,
	title={Stochastic variance reduction for nonconvex optimization},
	author={Reddi, Sashank J and Hefny, Ahmed and Sra, Suvrit and Poczos, Barnabas and Smola, Alex},
	booktitle={International conference on machine learning},
	pages={314--323},
	year={2016}
}
@inproceedings{allen2016variance,
	title={Variance reduction for faster non-convex optimization},
	author={Allen-Zhu, Zeyuan and Hazan, Elad},
	booktitle={International conference on machine learning},
	pages={699--707},
	year={2016}
}
@article{hardt2016identity,
	title={Identity matters in deep learning},
	author={Hardt, Moritz and Ma, Tengyu},
	journal={arXiv preprint arXiv:1611.04231},
	year={2016}
}
@article{zhou2017characterization,
	title={Characterization of gradient dominance and regularity conditions for neural networks},
	author={Zhou, Yi and Liang, Yingbin},
	journal={arXiv preprint arXiv:1710.06910},
	year={2017}
}
@article{charles2017stability,
	title={Stability and generalization of learning algorithms that converge to global optima},
	author={Charles, Zachary and Papailiopoulos, Dimitris},
	journal={arXiv preprint arXiv:1710.08402},
	year={2017}
}
@article{arora2018convergence,
	title={A convergence analysis of gradient descent for deep linear neural networks},
	author={Arora, Sanjeev and Cohen, Nadav and Golowich, Noah and Hu, Wei},
	journal={arXiv preprint arXiv:1810.02281},
	year={2018}
}
@inproceedings{li2017convergence,
	title={Convergence analysis of two-layer neural networks with relu activation},
	author={Li, Yuanzhi and Yuan, Yang},
	booktitle={Advances in Neural Information Processing Systems},
	pages={597--607},
	year={2017}
}
@article{allen2018convergence,
	title={A convergence theory for deep learning via over-parameterization},
	author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
	journal={arXiv preprint arXiv:1811.03962},
	year={2018}
}
@article{du2018gradient,
	title={Gradient descent provably optimizes over-parameterized neural networks},
	author={Du, Simon S and Zhai, Xiyu and Poczos, Barnabas and Singh, Aarti},
	journal={arXiv preprint arXiv:1810.02054},
	year={2018}
}
@inproceedings{li2018learning,
	title={Learning overparameterized neural networks via stochastic gradient descent on structured data},
	author={Li, Yuanzhi and Liang, Yingyu},
	booktitle={Advances in Neural Information Processing Systems},
	pages={8157--8166},
	year={2018}
}
@inproceedings{duchi2010composite,
	title={Composite Objective Mirror Descent.},
	author={Duchi, John C and Shalev-Shwartz, Shai and Singer, Yoram and Tewari, Ambuj},
	booktitle={COLT},
	pages={14--26},
	year={2010}
}
@article{juditsky2011solving,
	title={Solving variational inequalities with stochastic mirror-prox algorithm},
	author={Juditsky, Anatoli and Nemirovski, Arkadi and Tauvel, Claire},
	journal={Stochastic Systems},
	volume={1},
	number={1},
	pages={17--58},
	year={2011},
	publisher={INFORMS}
}
@article{rockafellar1976monotone,
	title={Monotone operators and the proximal point algorithm},
	author={Rockafellar, R Tyrrell},
	journal={SIAM journal on control and optimization},
	volume={14},
	number={5},
	pages={877--898},
	year={1976},
	publisher={SIAM}
}
@article{devlin2018bert,
	title={Bert: Pre-training of deep bidirectional transformers for language understanding},
	author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
	journal={arXiv preprint arXiv:1810.04805},
	year={2018}
}
@article{hand2001simple,
	title={A simple generalisation of the area under the ROC curve for multiple class classification problems},
	author={Hand, David J and Till, Robert J},
	journal={Machine learning},
	volume={45},
	number={2},
	pages={171--186},
	year={2001},
	publisher={Springer}
}
@article{graves2013generating,
	title={Generating sequences with recurrent neural networks},
	author={Graves, Alex},
	journal={arXiv preprint arXiv:1308.0850},
	year={2013}
}
@article{bahdanau2014neural,
	title={Neural machine translation by jointly learning to align and translate},
	author={Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua},
	journal={arXiv preprint arXiv:1409.0473},
	year={2014}
}
@inproceedings{sutskever2014sequence,
	title={Sequence to sequence learning with neural networks},
	author={Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V},
	booktitle={Advances in neural information processing systems},
	pages={3104--3112},
	year={2014}
}

@article{hinton2012deep,
	title={Deep neural networks for acoustic modeling in speech recognition},
	author={Hinton, Geoffrey and Deng, Li and Yu, Dong and Dahl, George and Mohamed, Abdel-rahman and Jaitly, Navdeep and Senior, Andrew and Vanhoucke, Vincent and Nguyen, Patrick and Kingsbury, Brian and others},
	journal={IEEE Signal processing magazine},
	volume={29},
	year={2012}
}
@article{mohamed2012acoustic,
	title={Acoustic modeling using deep belief networks},
	author={Mohamed, Abdel-rahman and Dahl, George E and Hinton, Geoffrey},
	journal={IEEE Transactions on Audio, Speech, and Language Processing},
	volume={20},
	number={1},
	pages={14--22},
	year={2012},
	publisher={IEEE}
}
@inproceedings{graves2013speech,
	title={Speech recognition with deep recurrent neural networks},
	author={Graves, Alex and Mohamed, Abdel-rahman and Hinton, Geoffrey},
	booktitle={2013 IEEE international conference on acoustics, speech and signal processing},
	pages={6645--6649},
	year={2013},
	organization={IEEE}
}
@article{kleinberg2018alternative,
	title={An Alternative View: When Does SGD Escape Local Minima?},
	author={Kleinberg, Robert and Li, Yuanzhi and Yuan, Yang},
	journal={arXiv preprint arXiv:1802.06175},
	year={2018}
}
@article{lin2018solving,
	title={Solving weakly-convex-weakly-concave saddle-point problems as weakly-monotone variational inequality},
	author={Lin, Qihang and Liu, Mingrui and Rafique, Hassan and Yang, Tianbao},
	journal={arXiv preprint arXiv:1810.10207},
	year={2018}
}
@article{sanjabi2018solving,
	title={Solving Non-Convex Non-Concave Min-Max Games Under Polyak-L ojasiewicz Condition},
	author={Sanjabi, Maziar and Razaviyayn, Meisam and Lee, Jason D},
	journal={arXiv preprint arXiv:1812.02878},
	year={2018}
}
@article{lu2019hybrid,
	title={Hybrid Block Successive Approximation for One-Sided Non-Convex Min-Max Problems: Algorithms and Applications},
	author={Lu, Songtao and Tsaknakis, Ioannis and Hong, Mingyi and Chen, Yongxin},
	journal={arXiv preprint arXiv:1902.08294},
	year={2019}
}
@article{jin2019minmax,
	title={Minmax Optimization: Stable Limit Points of Gradient Descent Ascent are Locally Optimal},
	author={Jin, Chi and Netrapalli, Praneeth and Jordan, Michael I},
	journal={arXiv preprint arXiv:1902.00618},
	year={2019}
}

@inproceedings{
	chen2018universal,
	title={Universal  Stagewise Learning for Non-Convex Problems with  Convergence on  Averaged Solutions},
	author={Zaiyi Chen and Zhuoning Yuan and Jinfeng Yi and Bowen Zhou and Enhong Chen and Tianbao Yang},
	booktitle={International Conference on Learning Representations},
	year={2019},
}


@article{girosi1995regularization,
	title={Regularization theory and neural networks architectures},
	author={Girosi, Federico and Jones, Michael and Poggio, Tomaso},
	journal={Neural computation},
	volume={7},
	number={2},
	pages={219--269},
	year={1995},
	publisher={MIT Press}
}
@inproceedings{zhou2018stochastic,
	title={Stochastic nested variance reduced gradient descent for nonconvex optimization},
	author={Zhou, Dongruo and Xu, Pan and Gu, Quanquan},
	booktitle={Advances in Neural Information Processing Systems},
	pages={3921--3932},
	year={2018}
}
@inproceedings{li2018simple,
	title={A Simple Proximal Stochastic Gradient Method for Nonsmooth Nonconvex Optimization},
	author={Li, Zhize and Li, Jian},
	booktitle={Advances in Neural Information Processing Systems},
	pages={5564--5574},
	year={2018}
}
@article{wang2018spiderboost,
	title={SpiderBoost: A class of faster variance-reduced algorithms for nonconvex optimization},
	author={Wang, Zhe and Ji, Kaiyi and Zhou, Yi and Liang, Yingbin and Tarokh, Vahid},
	journal={arXiv preprint arXiv:1810.10690},
	year={2018}
}
@inproceedings{nguyen2017sarah,
	title={SARAH: A novel method for machine learning problems using stochastic recursive gradient},
	author={Nguyen, Lam M and Liu, Jie and Scheinberg, Katya and Takac, Martin},
	booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
	pages={2613--2621},
	year={2017},
	organization={JMLR. org}
}
@article{nguyen2017stochastic,
	title={Stochastic recursive gradient algorithm for nonconvex optimization},
	author={Nguyen, Lam M and Liu, Jie and Scheinberg, Katya and Takac, Martin},
	journal={arXiv preprint arXiv:1705.07261},
	year={2017}
}

@article{zou2018stochastic,
	title={Stochastic gradient descent optimizes over-parameterized deep relu networks},
	author={Zou, Difan and Cao, Yuan and Zhou, Dongruo and Gu, Quanquan},
	journal={arXiv preprint arXiv:1811.08888},
	year={2018}
}
@article{du2018gradient1,
	title={Gradient descent finds global minima of deep neural networks},
	author={Du, Simon S and Lee, Jason D and Li, Haochuan and Wang, Liwei and Zhai, Xiyu},
	journal={arXiv preprint arXiv:1811.03804},
	year={2018}
}

@article{zou2019improved,
	title={An Improved Analysis of Training Over-parameterized Deep Neural Networks},
	author={Zou, Difan and Gu, Quanquan},
	journal={arXiv preprint arXiv:1906.04688},
	year={2019}
}


@article{code,
	title={\url{https://drive.google.com/drive/folders/1nPM6fmvN5fTsSaWsOcGFbhMVW7Fxso-Y?usp=sharing}},
	author={Anonymous},
	journal={ },
	year={ }
}

@misc{code_v2,
	author = {Anonymous.},
	title = {{}},
	howpublished ="\url{https://drive.google.com/drive/folders/1nPM6fmvN5fTsSaWsOcGFbhMVW7Fxso-Y?usp=sharing}",
	year = {}, 
	note = ""
}
@inproceedings{zinkevich2010parallelized,
  title={Parallelized stochastic gradient descent},
  author={Zinkevich, Martin and Weimer, Markus and Li, Lihong and Smola, Alex J},
  booktitle={Advances in neural information processing systems},
  pages={2595--2603},
  year={2010}
}
@inproceedings{mcdonald2010distributed,
  title={Distributed training strategies for the structured perceptron},
  author={McDonald, Ryan and Hall, Keith and Mann, Gideon},
  booktitle={Human language technologies: The 2010 annual conference of the North American chapter of the association for computational linguistics},
  pages={456--464},
  year={2010},
  organization={Association for Computational Linguistics}
}
@article{zhang2013communication,
  title={Communication-efficient algorithms for statistical optimization},
  author={Zhang, Yuchen and Duchi, John C and Wainwright, Martin J},
  journal={The Journal of Machine Learning Research},
  volume={14},
  number={1},
  pages={3321--3363},
  year={2013},
  publisher={JMLR. org}
}
@article{zhang2016parallel,
  title={Parallel SGD: When does averaging help?},
  author={Zhang, Jian and De Sa, Christopher and Mitliagkas, Ioannis and R{\'e}, Christopher},
  journal={arXiv preprint arXiv:1606.07365},
  year={2016}
}
@inproceedings{shamir2014distributed,
  title={Distributed stochastic optimization and learning},
  author={Shamir, Ohad and Srebro, Nathan},
  booktitle={2014 52nd Annual Allerton Conference on Communication, Control, and Computing (Allerton)},
  pages={850--857},
  year={2014},
  organization={IEEE}
}
@article{godichon2017rates,
  title={On the rates of convergence of parallelized averaged stochastic gradient algorithms},
  author={Godichon-Baggioni, Antoine and Saadane, Sofiane},
  journal={arXiv preprint arXiv:1710.07926},
  year={2017}
}
@article{jain2017parallelizing,
  title={Parallelizing stochastic gradient descent for least squares regression: mini-batching, averaging, and model misspecification},
  author={Jain, Prateek and Netrapalli, Praneeth and Kakade, Sham M and Kidambi, Rahul and Sidford, Aaron},
  journal={The Journal of Machine Learning Research},
  volume={18},
  number={1},
  pages={8258--8299},
  year={2017},
  publisher={JMLR. org}
}
@article{povey2014parallel,
  title={Parallel training of DNNs with natural gradient and parameter averaging},
  author={Povey, Daniel and Zhang, Xiaohui and Khudanpur, Sanjeev},
  journal={arXiv preprint arXiv:1410.7455},
  year={2014}
}
@article{bernstein2018signsgd,
  title={signSGD: Compressed optimisation for non-convex problems},
  author={Bernstein, Jeremy and Wang, Yu-Xiang and Azizzadenesheli, Kamyar and Anandkumar, Anima},
  journal={arXiv preprint arXiv:1802.04434},
  year={2018}
}
@inproceedings{chen2016scalable,
  title={Scalable training of deep learning machines by incremental block training with intra-block parallel optimization and blockwise model-update filtering},
  author={Chen, Kai and Huo, Qiang},
  booktitle={2016 ieee international conference on acoustics, speech and signal processing (icassp)},
  pages={5880--5884},
  year={2016},
  organization={IEEE}
}
@article{mcmahan2016communication,
  title={Communication-efficient learning of deep networks from decentralized data},
  author={McMahan, H Brendan and Moore, Eider and Ramage, Daniel and Hampson, Seth and others},
  journal={arXiv preprint arXiv:1602.05629},
  year={2016}
}
@article{su2015experiments,
  title={Experiments on parallel training of deep neural network using model averaging},
  author={Su, Hang and Chen, Haoyu},
  journal={arXiv preprint arXiv:1507.01239},
  year={2015}
}
@inproceedings{kamp2018efficient,
  title={Efficient decentralized deep learning by dynamic model averaging},
  author={Kamp, Michael and Adilova, Linara and Sicking, Joachim and H{\"u}ger, Fabian and Schlicht, Peter and Wirtz, Tim and Wrobel, Stefan},
  booktitle={Joint European Conference on Machine Learning and Knowledge Discovery in Databases},
  pages={393--409},
  year={2018},
  organization={Springer}
}
@article{lin2018don,
  title={Don't Use Large Mini-Batches, Use Local SGD},
  author={Lin, Tao and Stich, Sebastian U and Patel, Kumar Kshitij and Jaggi, Martin},
  journal={arXiv preprint arXiv:1808.07217},
  year={2018}
}

@inproceedings{
liu2020towards,
title={Towards Better Understanding of Adaptive Gradient Algorithms in Generative Adversarial Nets},
author={Mingrui Liu and Youssef Mroueh and Jerret Ross and Wei Zhang and Xiaodong Cui and Payel Das and Tianbao Yang},
booktitle={International Conference on Learning Representations},
year={2020},
}
@article{lin2019gradient,
  title={On gradient descent ascent for nonconvex-concave minimax problems},
  author={Lin, Tianyi and Jin, Chi and Jordan, Michael I},
  journal={arXiv preprint arXiv:1906.00331},
  year={2019}
}
@inproceedings{paszke2019pytorch,
  title={PyTorch: An imperative style, high-performance deep learning library},
  author={Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and others},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8024--8035},
  year={2019}
}

@inproceedings{DBLP:conf/nips/JaggiSTTKHJ14,
  author    = {Martin Jaggi and
               Virginia Smith and
               Martin Tak{\'{a}}c and
               Jonathan Terhorst and
               Sanjay Krishnan and
               Thomas Hofmann and
               Michael I. Jordan},
  title     = {Communication-Efficient Distributed Dual Coordinate Ascent},
  booktitle = {Advances in Neural Information Processing Systems 27: Annual Conference
               on Neural Information Processing Systems 2014, December 8-13 2014,
               Montreal, Quebec, Canada},
  pages     = {3068--3076},
  year      = {2014}
}

@inproceedings{
koloskova*2020decentralized,
title={Decentralized Deep Learning with Arbitrary Communication Compression},
author={Anastasia Koloskova* and Tao Lin* and Sebastian U Stich and Martin Jaggi},
booktitle={International Conference on Learning Representations},
year={2020},
url={https://openreview.net/forum?id=SkgGCkrKvH}
}

@inproceedings{DBLP:conf/icml/KoloskovaSJ19,
  author    = {Anastasia Koloskova and
               Sebastian U. Stich and
               Martin Jaggi},
  title     = {Decentralized Stochastic Optimization and Gossip Algorithms with Compressed
               Communication},
  booktitle = {Proceedings of the 36th International Conference on Machine Learning,
               {ICML} 2019, 9-15 June 2019, Long Beach, California, {USA}},
  pages     = {3478--3487},
  year      = {2019}
}
@article{lei2021stochastic,
  title={Stochastic proximal AUC maximization},
  author={Lei, Yunwen and Ying, Yiming},
  journal={Journal of Machine Learning Research},
  volume={22},
  number={61},
  pages={1--45},
  year={2021},
  publisher={Journal of Machine Learning Research}
}
@inproceedings{dan2021variance,
  title={Variance reduced stochastic proximal algorithm for auc maximization},
  author={Dan, Soham and Sahoo, Dushyant},
  booktitle={Joint European Conference on Machine Learning and Knowledge Discovery in Databases},
  pages={184--199},
  year={2021},
  organization={Springer}
}

@inproceedings{KhoslaYaoJayadevaprakashFeiFei_FGVC2011,
author = "Aditya Khosla and Nityananda Jayadevaprakash and Bangpeng Yao and Li Fei-Fei",
title = "Novel Dataset for Fine-Grained Image Categorization",
booktitle = "First Workshop on Fine-Grained Visual Categorization, IEEE Conference on Computer Vision and Pattern Recognition",
year = "2011",
month = "June",
address = "Colorado Springs, CO",
}

@article{yang2021learning,
  title={Learning with Multiclass AUC: Theory and Algorithms},
  author={Yang, Zhiyong and Xu, Qianqian and Bao, Shilong and Cao, Xiaochun and Huang, Qingming},
  journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
  year={2021},
  publisher={IEEE}
}

@article{buzzega2020dark,
  title={Dark experience for general continual learning: a strong, simple baseline},
  author={Buzzega, Pietro and Boschini, Matteo and Porrello, Angelo and Abati, Davide and Calderara, Simone},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={15920--15930},
  year={2020}
}

@article{esteva2017dermatologist,
  title={Dermatologist-level classification of skin cancer with deep neural networks},
  author={Esteva, Andre and Kuprel, Brett and Novoa, Roberto A and Ko, Justin and Swetter, Susan M and Blau, Helen M and Thrun, Sebastian},
  journal={nature},
  volume={542},
  number={7639},
  pages={115--118},
  year={2017},
  publisher={Nature Publishing Group}
}

@inproceedings{irvin2019chexpert,
  title={Chexpert: A large chest radiograph dataset with uncertainty labels and expert comparison},
  author={Irvin, Jeremy and Rajpurkar, Pranav and Ko, Michael and Yu, Yifan and Ciurea-Ilcus, Silviana and Chute, Chris and Marklund, Henrik and Haghgoo, Behzad and Ball, Robyn and Shpanskaya, Katie and others},
  booktitle={Proceedings of the AAAI conference on artificial intelligence},
  volume={33},
  number={01},
  pages={590--597},
  year={2019}
}

@inproceedings{ebrahimi2020adversarial,
  title={Adversarial continual learning},
  author={Ebrahimi, Sayna and Meier, Franziska and Calandra, Roberto and Darrell, Trevor and Rohrbach, Marcus},
  booktitle={European Conference on Computer Vision},
  pages={386--402},
  year={2020},
  organization={Springer}
}

@inproceedings{mallya2018packnet,
  title={Packnet: Adding multiple tasks to a single network by iterative pruning},
  author={Mallya, Arun and Lazebnik, Svetlana},
  booktitle={Proceedings of the IEEE conference on Computer Vision and Pattern Recognition},
  pages={7765--7773},
  year={2018}
}




@inproceedings{bang2021rainbow,
  title={Rainbow memory: Continual learning with a memory of diverse samples},
  author={Bang, Jihwan and Kim, Heesu and Yoo, YoungJoon and Ha, Jung-Woo and Choi, Jonghyun},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={8218--8227},
  year={2021}
}

@inproceedings{
koh2022online,
title={Online Continual Learning on Class Incremental Blurry Task Configuration with Anytime Inference},
author={Hyunseo Koh and Dahyun Kim and Jung-Woo Ha and Jonghyun Choi},
booktitle={International Conference on Learning Representations},
year={2022},
url={https://openreview.net/forum?id=nrGGfMbY_qK}
}

@inproceedings{kim2020imbalanced,
  title={Imbalanced continual learning with partitioning reservoir sampling},
  author={Kim, Chris Dongjoo and Jeong, Jinseo and Kim, Gunhee},
  booktitle={European Conference on Computer Vision},
  pages={411--428},
  year={2020},
  organization={Springer}
}

@inproceedings{de2021continual,
  title={Continual prototype evolution: Learning online from non-stationary data streams},
  author={De Lange, Matthias and Tuytelaars, Tinne},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={8250--8259},
  year={2021}
}

@article{mai2022online,
  title={Online continual learning in image classification: An empirical survey},
  author={Mai, Zheda and Li, Ruiwen and Jeong, Jihwan and Quispe, David and Kim, Hyunwoo and Sanner, Scott},
  journal={Neurocomputing},
  volume={469},
  pages={28--51},
  year={2022},
  publisher={Elsevier}
}

@article{borsos2020coresets,
  title={Coresets via bilevel optimization for continual learning and streaming},
  author={Borsos, Zal{\'a}n and Mutny, Mojmir and Krause, Andreas},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={14879--14890},
  year={2020}
}

@article{hayes2021replay,
  title={Replay in deep learning: Current approaches and missing biological elements},
  author={Hayes, Tyler L and Krishnan, Giri P and Bazhenov, Maxim and Siegelmann, Hava T and Sejnowski, Terrence J and Kanan, Christopher},
  journal={Neural Computation},
  volume={33},
  number={11},
  pages={2908--2950},
  year={2021},
  publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…}
}

@inproceedings{shim2021online,
  title={Online class-incremental continual learning with adversarial shapley value},
  author={Shim, Dongsub and Mai, Zheda and Jeong, Jihwan and Sanner, Scott and Kim, Hyunwoo and Jang, Jongseong},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={35},
  number={11},
  pages={9630--9638},
  year={2021}
}


@inproceedings{mai2021supervised,
  title={Supervised contrastive replay: Revisiting the nearest class mean classifier in online class-incremental continual learning},
  author={Mai, Zheda and Li, Ruiwen and Kim, Hyunwoo and Sanner, Scott},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={3589--3599},
  year={2021}
}

@inproceedings{verwimp2021rehearsal,
  title={Rehearsal revealed: The limits and merits of revisiting samples in continual learning},
  author={Verwimp, Eli and De Lange, Matthias and Tuytelaars, Tinne},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={9385--9394},
  year={2021}
}

@article{jin2021gradient,
  title={Gradient-based Editing of Memory Examples for Online Task-free Continual Learning},
  author={Jin, Xisen and Sadhu, Arka and Du, Junyi and Ren, Xiang},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={29193--29205},
  year={2021}
}

@inproceedings{pham2020contextual,
  title={Contextual transformation networks for online continual learning},
  author={Pham, Quang and Liu, Chenghao and Sahoo, Doyen and Steven, HOI},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@inproceedings{Ahrens_2021,
  title={DRILL: Dynamic Representations for Imbalanced Lifelong Learning},
  author={Ahrens, Kyra and Abawi, Fares and Wermter, Stefan},
  booktitle={Artificial Neural Networks and Machine Learning--ICANN 2021: 30th International Conference on Artificial Neural Networks, Bratislava, Slovakia, September 14--17, 2021, Proceedings, Part II 30},
  pages={409--420},
  year={2021},
  organization={Springer}
}

@misc{zhou2022forgetting,
      title={Forgetting and Imbalance in Robot Lifelong Learning with Off-policy Data}, 
      author={Wenxuan Zhou and Steven Bohez and Jan Humplik and Abbas Abdolmaleki and Dushyant Rao and Markus Wulfmeier and Tuomas Haarnoja and Nicolas Heess},
      year={2022},
      eprint={2204.05893},
      archivePrefix={arXiv},
      primaryClass={cs.RO}
}

@article{CanoK22,
  author       = {Alberto Cano and
                  Bartosz Krawczyk},
  title        = {{ROSE:} robust online self-adjusting ensemble for continual learning
                  on imbalanced drifting data streams},
  journal      = {Mach. Learn.},
  volume       = {111},
  number       = {7},
  pages        = {2561--2599},
  year         = {2022},
  url          = {https://doi.org/10.1007/s10994-022-06168-x},
  doi          = {10.1007/s10994-022-06168-x},
  timestamp    = {Mon, 08 Aug 2022 21:24:10 +0200},
  biburl       = {https://dblp.org/rec/journals/ml/CanoK22.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}