@inproceedings{perezProgress,
title={Progress in self-certified neural networks},
author={Mar\'ia P\'erez-Ortiz and Omar Rivasplata and Emilio Parrado-Hernandez and Benjamin Guedj and John Shawe-Taylor},
year={2021},
booktitle={{NeurIPS 2021 workshop Bayesian Deep Learning [BDL]}},
eprint={2111.07737},
archivePrefix={arXiv},
primaryClass={cs.LG},
}

@article{bartlett2002rademacher,
  author    = {Peter Bartlett and
               Shahar Mendelson},
  title     = {Rademacher and Gaussian Complexities: Risk Bounds and Structural
               Results},
  journal   = {Journal of Machine Learning Research},
  volume    = {3},
  year      = {2002},
  pages     = {463-482},
  ee        = {http://www.jmlr.org/papers/v3/bartlett02a.html},
  bibsource = {DBLP, http://dblp.uni-trier.de}
}
@inproceedings{lei2023generalization,
  title={Generalization analysis for contrastive representation learning},
  author={Lei, Yunwen and Yang, Tianbao and Ying, Yiming and Zhou, Ding-Xuan},
  booktitle={International Conference on Machine Learning},
  pages={19200--19227},
  year={2023},
  organization={PMLR}
}
@inproceedings{jin2009regularized,
  title={Regularized distance metric learning: Theory and algorithm},
  author={Jin, Rong and Wang, Shijun and Zhou, Yang},
  booktitle={Advances in Neural Information Processing Systems},
  pages={862--870},
  year={2009}
}
@inproceedings{wang2019multitask,
  title={Multitask metric learning: Theory and algorithm},
  author={Wang, Boyu and Zhang, Hejia and Liu, Peng and Shen, Zebang and Pineau, Joelle},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3362--3371},
  year={2019}
}
@inproceedings{mucke2019beating,
  title={Beating SGD Saturation with Tail-Averaging and Minibatching},
  author={M{\"u}cke, Nicole and Neu, Gergely and Rosasco, Lorenzo},
  booktitle={Advances in Neural Information Processing Systems},
  pages={12568--12577},
  year={2019}
}
@article{liang2020exploring,
  title={Exploring private federated learning with laplacian smoothing},
  author={Liang, Zhicong and Wang, Bao and Gu, Quanquan and Osher, Stanley and Yao, Yuan},
  journal={arXiv preprint arXiv:2005.00218},
  year={2020}
}
@inproceedings{mironov2017renyi,
  title={R{\'e}nyi differential privacy},
  author={Mironov, Ilya},
  booktitle={2017 IEEE 30th computer security foundations symposium (CSF)},
  pages={263--275},
  year={2017},
  organization={IEEE}
}
@inproceedings{dwork2008differential,
  title={Differential privacy: A survey of results},
  author={Dwork, Cynthia},
  booktitle={International conference on theory and applications of models of computation},
  pages={1--19},
  year={2008},
  organization={Springer}
}
@article{klochkov2021stability,
  title={Stability and Deviation Optimal Risk Bounds with Convergence Rate $ O (1/n) $},
  author={Klochkov, Yegor and Zhivotovskiy, Nikita},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}


@article{vitale2000some,
  title={Some comparisons for Gaussian processes},
  author={Vitale, Richard A},
  journal={Proceedings of the American Mathematical Society},
  pages={3043--3046},
  year={2000},
  publisher={JSTOR}
}
@inproceedings{allen2016even,
  title={Even faster accelerated coordinate descent using non-uniform sampling},
  author={Allen-Zhu, Zeyuan and Qu, Zheng and Richt{\'a}rik, Peter and Yuan, Yang},
  booktitle={International Conference on Machine Learning},
  pages={1110--1119},
  year={2016},
  organization={PMLR}
}
@article{nikolakakis2022beyond,
  title={Beyond Lipschitz: sharp generalization and excess risk bounds for full-batch GD},
  author={Nikolakakis, Konstantinos E and Haddadpour, Farzin and Karbasi, Amin and Kalogerias, Dionysios S},
  journal={arXiv preprint arXiv:2204.12446},
  year={2022}
}
@inproceedings{raj2023algorithmic,
  title={Algorithmic stability of heavy-tailed SGD with general loss functions},
  author={Raj, Anant and Zhu, Lingjiong and Gurbuzbalaban, Mert and Simsekli, Umut},
  booktitle={International Conference on Machine Learning},
  pages={28578--28597},
  year={2023}
}
@article{viallard2023learning,
  title={Learning via Wasserstein-Based High Probability Generalisation Bounds},
  author={Viallard, Paul and Haddouche, Maxime and Simsekli, Umut and Guedj, Benjamin},
  journal={arXiv preprint arXiv:2306.04375},
  year={2023}
}
@article{zhou2021towards,
  title={Towards understanding why lookahead generalizes better than sgd and beyond},
  author={Zhou, Pan and Yan, Hanshu and Yuan, Xiaotong and Feng, Jiashi and Yan, Shuicheng},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={27290--27304},
  year={2021}
}
@article{taheri2023generalization,
  title={Generalization and Stability of Interpolating Neural Networks with Minimal Width},
  author={Taheri, Hossein and Thrampoulidis, Christos},
  journal={arXiv preprint arXiv:2302.09235},
  year={2023}
}
@article{nikolakakis2022black,
  title={Black-Box Generalization: Stability of Zeroth-Order Learning},
  author={Nikolakakis, Konstantinos and Haddadpour, Farzin and Kalogerias, Dionysis and Karbasi, Amin},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={31525--31541},
  year={2022}
}
@article{chen2023stability,
  title={On the Stability and Generalization of Triplet Learning},
  author={Chen, Jun and Chen, Hong and Jiang, Xue and Gu, Bin and Li, Weifu and Gong, Tieliang and Zheng, Feng},
  journal={arXiv preprint arXiv:2302.09815},
  year={2023}
}
@article{nesterov2009primal,
  title={Primal-dual subgradient methods for convex problems},
  author={Nesterov, Yurii},
  journal={Mathematical Programming},
  volume={120},
  number={1},
  pages={221--259},
  year={2009},
  publisher={Springer}
}
@InProceedings{jain19a,
  title = 	 {Making the Last Iterate of {SGD} Information Theoretically Optimal},
  author = 	 {Jain, Prateek and Nagaraj, Dheeraj and Netrapalli, Praneeth},
  booktitle = 	 {Conference on Learning Theory},
  pages = 	 {1752--1755},
  year = 	 {2019},
  editor = 	 {Beygelzimer, Alina and Hsu, Daniel}
}
%,
%  volume = 	 {99},
%  series = 	 {Proceedings of Machine Learning Research},
%  publisher = 	 {PMLR}
@article{hansen2012air,
  title={AIR tools—a MATLAB package of algebraic iterative reconstruction methods},
  author={Hansen, Per Christian and Saxild-Hansen, Maria},
  journal={Journal of Computational and Applied Mathematics},
  volume={236},
  number={8},
  pages={2167--2178},
  year={2012},
  publisher={Elsevier}
}
@article{shalev2011pegasos,
  title={Pegasos: Primal estimated sub-gradient solver for svm},
  author={Shalev-Shwartz, Shai and Singer, Yoram and Srebro, Nathan and Cotter, Andrew},
  journal={Mathematical programming},
  volume={127},
  number={1},
  pages={3--30},
  year={2011},
  publisher={Springer}
}
@article{soltanolkotabi2018theoretical,
  title={Theoretical insights into the optimization landscape of over-parameterized shallow neural networks},
  author={Soltanolkotabi, Mahdi and Javanmard, Adel and Lee, Jason D},
  journal={IEEE Transactions on Information Theory},
  volume={65},
  number={2},
  pages={742--769},
  year={2018},
  publisher={IEEE}
}

@article{robbins1951stochastic,
  title={A stochastic approximation method},
  author={Robbins, Herbert and Monro, Sutton},
  journal={Annals of Mathematical Statistics},
  pages={400--407},
  year={1951},
  publisher={JSTOR}
}
@inproceedings{lian2015asynchronous,
  title={Asynchronous parallel stochastic gradient for nonconvex optimization},
  author={Lian, Xiangru and Huang, Yijun and Li, Yuncheng and Liu, Ji},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2737--2745},
  year={2015}
}
@inproceedings{jaggi2013revisiting,
  title={Revisiting Frank-Wolfe: Projection-Free Sparse Convex Optimization},
  author={Jaggi, Martin},
  booktitle={International Conference on Machine Learning},
  pages={427--435},
  year={2013}
}
@inproceedings{allen2018natasha,
  title={Natasha 2: Faster non-convex optimization than sgd},
  author={Allen-Zhu, Zeyuan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2675--2686},
  year={2018}
}
@inproceedings{de2015taming,
  title={Taming the wild: A unified analysis of hogwild-style algorithms},
  author={De Sa, Christopher M and Zhang, Ce and Olukotun, Kunle and R{\'e}, Christopher},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2674--2682},
  year={2015}
}
@inproceedings{recht2011hogwild,
  title={Hogwild: A lock-free approach to parallelizing stochastic gradient descent},
  author={Recht, Benjamin and Re, Christopher and Wright, Stephen and Niu, Feng},
  booktitle={Advances in Neural Information Processing Systems},
  pages={693--701},
  year={2011}
}
@inproceedings{hazan2016variance,
  title={Variance-reduced and projection-free stochastic optimization},
  author={Hazan, Elad and Luo, Haipeng},
  booktitle={International Conference on Machine Learning},
  pages={1263--1271},
  year={2016}
}
@inproceedings{reddi2016stochasticb,
  title={Stochastic frank-wolfe methods for nonconvex optimization},
  author={Reddi, Sashank J and Sra, Suvrit and P{\'o}czos, Barnab{\'a}s and Smola, Alex},
  booktitle={Annual Allerton Conference on Communication, Control, and Computing},
  pages={1244--1251},
  year={2016},
  organization={IEEE}
}

@article{polyak1992acceleration,
  title={Acceleration of stochastic approximation by averaging},
  author={Polyak, Boris T and Juditsky, Anatoli B},
  journal={SIAM Journal on Control and Optimization},
  volume={30},
  number={4},
  pages={838--855},
  year={1992},
  publisher={SIAM}
}
@inproceedings{xu2017stochastic,
  title={Stochastic convex optimization: Faster local growth implies faster global convergence},
  author={Xu, Yi and Lin, Qihang and Yang, Tianbao},
  booktitle={International Conference on Machine Learning},
  pages={3821--3830},
  year={2017}
}

@inproceedings{harvey2018tight,
  title = 	 {Tight analyses for non-smooth stochastic gradient descent},
  author = 	 {Harvey, Nicholas J.~A. and Liaw, Christopher and Plan, Yaniv and Randhawa, Sikander},
  booktitle = 	 {Conference on Learning Theory},
  pages = 	 {1579--1613},
  year = 	 {2019},
  editor = 	 {Beygelzimer, Alina and Hsu, Daniel}
}

@inproceedings{allen2016variance,
  title={Variance reduction for faster non-convex optimization},
  author={Allen-Zhu, Zeyuan and Hazan, Elad},
  booktitle={International Conference on Machine Learning},
  pages={699--707},
  year={2016}
}
@article{nguyen2017stochastic,
  title={Stochastic recursive gradient algorithm for nonconvex optimization},
  author={Nguyen, Lam M and Liu, Jie and Scheinberg, Katya and Tak{\'a}{\v{c}}, Martin},
  journal={arXiv preprint arXiv:1705.07261},
  year={2017}
}
@article{liu2016algorithm,
  title={Algorithm-dependent generalization bounds for multi-task learning},
  author={Liu, Tongliang and Tao, Dacheng and Song, Mingli and Maybank, Stephen J},
  journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
  volume={39},
  number={2},
  pages={227--241},
  year={2016},
  publisher={IEEE}
}
@article{zhang2017sparse,
  title={Sparse learning with stochastic composite optimization},
  author={Zhang, Weizhong and Zhang, Lijun and Jin, Zhongming and Jin, Rong and Cai, Deng and Li, Xuelong and Liang, Ronghua and He, Xiaofei},
  journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
  volume={39},
  number={6},
  pages={1223--1236},
  year={2017},
  publisher={IEEE}
}
@article{zhou2019understanding,
  title={Generalization Error Bounds with Probabilistic Guarantee for {SGD} in Nonconvex Optimization},
  author={Zhou, Yi and Liang, Yingbin and Zhang, Huishuai},
  journal={arXiv preprint arXiv:1802.06903},
  year={2018}
}
@article{JMLR:v20:17-750,
  author  = {John Duchi and Hongseok Namkoong},
  title   = {Variance-based Regularization with Convex Objectives},
  journal = {Journal of Machine Learning Research},
  year    = {2019},
  volume  = {20},
  number  = {68},
  pages   = {1-55},
  url     = {http://jmlr.org/papers/v20/17-750.html}
}
@article{shalev2010learnability,
  title={Learnability, stability and uniform convergence},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Srebro, Nathan and Sridharan, Karthik},
  journal={Journal of Machine Learning Research},
  volume={11},
  number={Oct},
  pages={2635--2670},
  year={2010}
}
@article{rogers1978finite,
  title={A finite sample distribution-free performance bound for local discrimination rules},
  author={Rogers, William H and Wagner, Terry J},
  journal={The Annals of Statistics},
  pages={506--514},
  year={1978},
  publisher={JSTOR}
}
@article{devroye1979distribution,
  title={Distribution-free inequalities for the deleted and holdout error estimates},
  author={Devroye, Luc and Wagner, Terry},
  journal={IEEE Transactions on Information Theory},
  volume={25},
  number={2},
  pages={202--207},
  year={1979},
  publisher={IEEE}
}

@inproceedings{london2013collective,
  title={Collective stability in structured prediction: Generalization from one example},
  author={London, Ben and Huang, Bert and Taskar, Ben and Getoor, Lise},
  booktitle={International Conference on Machine Learning},
  pages={828--836},
  year={2013}
}
@inproceedings{dwork2018privacy,
  title={Privacy-preserving Prediction},
  author={Dwork, Cynthia and Feldman, Vitaly},
  booktitle={Conference on Learning Theory},
  pages={1693--1702},
  year={2018}
}

@inproceedings{nesterov1983method,
  title={A method for solving the convex programming problem with convergence rate O ($1/k^2$)},
  author={Nesterov, Yurii E},
  booktitle={Dokl. akad. nauk Sssr},
  volume={269},
  pages={543--547},
  year={1983}
}
@InProceedings{kingma2015adam,
author = {Kingma, Diederick P and Ba, Jimmy},
title = {Adam: A method for stochastic optimization},
booktitle = {International Conference on Learning Representations},
year = {2015}
}
@inproceedings{sutskever2013importance,
  title={On the importance of initialization and momentum in deep learning},
  author={Sutskever, Ilya and Martens, James and Dahl, George and Hinton, Geoffrey},
  booktitle={International Conference on Machine Learning},
  pages={1139--1147},
  year={2013}
}
@article{polyak1964some,
  title={Some methods of speeding up the convergence of iteration methods},
  author={Polyak, Boris T},
  journal={USSR Computational Mathematics and Mathematical Physics},
  volume={4},
  number={5},
  pages={1--17},
  year={1964},
  publisher={Elsevier}
}
@book{boucheron2013concentration,
  title={Concentration Inequalities: A Nonasymptotic Theory of Independence},
  author={Boucheron, St{\'e}phane and Lugosi, G{\'a}bor and Massart, Pascal},
  year={2013},
  publisher={Oxford university press}
}
@book{shawe2004kernel,
  title={Kernel Methods for Pattern Analysis},
  author={Shawe-Taylor, John and Cristianini, Nello},
  year={2004},
  publisher={Cambridge university press}
}
@inproceedings{mou2018generalization,
  title={Generalization Bounds of SGLD for Non-convex Learning: Two Theoretical Viewpoints},
  author={Mou, Wenlong and Wang, Liwei and Zhai, Xiyu and Zheng, Kai},
  booktitle={Conference on Learning Theory},
  pages={605--638},
  year={2018}
}
@inproceedings{pillaud2018exponential,
  title={Exponential Convergence of Testing Error for Stochastic Gradient Methods},
  author={Pillaud-Vivien, Loucas and Rudi, Alessandro and Bach, Francis},
  booktitle={Conference on Learning Theory},
  pages={250--296},
  year={2018}
}
@article{maurer2005algorithmic,
  title={Algorithmic stability and meta-learning},
  author={Maurer, Andreas},
  journal={Journal of Machine Learning Research},
  volume={6},
  number={Jun},
  pages={967--994},
  year={2005}
}
@article{xing2021algorithmic,
  title={On the Algorithmic Stability of Adversarial Training},
  author={Xing, Yue and Song, Qifan and Cheng, Guang},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}
@inproceedings{
li2019generalization,
title={On Generalization Error Bounds of Noisy Gradient Methods for Non-Convex Learning},
author={Jian Li and Xuanyuan Luo and Mingda Qiao},
booktitle={International Conference on Learning Representations},
year={2020}
}
@inproceedings{bousquet2020sharper,
  title={Sharper bounds for uniformly stable algorithms},
  author={Bousquet, Olivier and Klochkov, Yegor and Zhivotovskiy, Nikita},
  booktitle={Conference on Learning Theory},
  pages={610--626},
  year={2020}
}
@inproceedings{csiba2015stochastic,
  title={Stochastic dual coordinate ascent with adaptive probabilities},
  author={Csiba, Dominik and Qu, Zheng and Richt{\'a}rik, Peter},
  booktitle={International Conference on Machine Learning},
  pages={674--683},
  year={2015},
  organization={PMLR}
}
@article{chen2018stability,
  title={Stability and convergence trade-off of iterative optimization algorithms},
  author={Chen, Yuansi and Jin, Chi and Yu, Bin},
  journal={arXiv preprint arXiv:1804.01619},
  year={2018}
}
@inproceedings{london2017pac,
  title={A {PAC}-bayesian analysis of randomized learning with application to stochastic gradient descent},
  author={London, Ben},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2931--2940},
  year={2017}
}
@inproceedings{charles2018stability,
  title={Stability and Generalization of Learning Algorithms that Converge to Global Optima},
  author={Charles, Zachary and Papailiopoulos, Dimitris},
  booktitle={International Conference on Machine Learning},
  pages={744--753},
  year={2018}
}
@inproceedings{kuzborskij2018data,
  title={Data-Dependent Stability of Stochastic Gradient Descent},
  author={Kuzborskij, Ilja and Lampert, Christoph},
  booktitle={International Conference on Machine Learning},
  pages={2820--2829},
  year={2018}
}
@article{steinhardt2014statistics,
  title={The statistics of streaming sparse regression},
  author={Steinhardt, Jacob and Wager, Stefan and Liang, Percy},
  journal={arXiv preprint arXiv:1412.4182},
  year={2014}
}
@article{schmidt2017minimizing,
  title={Minimizing finite sums with the stochastic average gradient},
  author={Schmidt, Mark and Le Roux, Nicolas and Bach, Francis},
  journal={Mathematical Programming},
  volume={162},
  number={1-2},
  pages={83--112},
  year={2017},
  publisher={Springer}
}
@article{cherkassky2009another,
  title={Another look at statistical learning theory and regularization},
  author={Cherkassky, Vladimir and Ma, Yunqian},
  journal={Neural Networks},
  volume={22},
  number={7},
  pages={958--969},
  year={2009},
  publisher={Elsevier}
}
@article{slepian1962one,
  title={The One-Sided Barrier Problem for Gaussian Noise},
  author={Slepian, David},
  journal={Bell System Technical Journal},
  volume={41},
  number={2},
  pages={463--501},
  year={1962},
  publisher={Wiley Online Library}
}

@article{alber2017distributed,
  title={Distributed optimization of multi-class SVMs},
  author={Alber, Maximilian and Zimmert, Julian and Dogan, Urun and Kloft, Marius},
  journal={PloS one},
  volume={12},
  number={6},
  pages={e0178161},
  year={2017},
  publisher={Public Library of Science}
}

@inproceedings{johnson2013accelerating,
  title={Accelerating stochastic gradient descent using predictive variance reduction},
  author={Johnson, Rie and Zhang, Tong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={315--323},
  year={2013}
}
@article{xiao2014proximal,
  title={A proximal stochastic gradient method with progressive variance reduction},
  author={Xiao, Lin and Zhang, Tong},
  journal={SIAM Journal on Optimization},
  volume={24},
  number={4},
  pages={2057--2075},
  year={2014},
  publisher={SIAM}
}
@inproceedings{lapin2015top,
  title={Top-k Multiclass {SVM}},
  author={Lapin, Maksim and Hein, Matthias and Schiele, Bernt},
  booktitle={Advances in Neural Information Processing Systems},
  pages={325--333},
  year={2015}
}
@inproceedings{ngiam2011optimization,
  title={On optimization methods for deep learning},
  author={Ngiam, Jiquan and Coates, Adam and Lahiri, Ahbik and Prochnow, Bobby and Le, Quoc V and Ng, Andrew Y},
  booktitle={International Conference on Machine Learning},
  pages={265--272},
  year={2011}
}
@article{bottou1991stochastic,
  title={Stochastic gradient learning in neural networks},
  author={Bottou, L{\'e}on},
  journal={Proceedings of Neuro-N{\i}mes},
  volume={91},
  number={8},
  year={1991}
}
@article{sun2015sparse,
  title={Sparse representation in kernel machines},
  author={Sun, Hongwei and Wu, Qiang},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  volume={26},
  number={10},
  pages={2576--2582},
  year={2015},
  publisher={IEEE}
}
@article{chang2018generalization,
  title={On the Generalization Ability of Online Gradient Descent Algorithm Under the Quadratic Growth Condition},
  author={Chang, Daqing and Lin, Ming and Zhang, Changshui},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  number={99},
  pages={1--12},
  year={2018},
  publisher={IEEE}
}
@article{schmidt2013fast,
  title={Fast convergence of stochastic gradient descent under a strong growth condition},
  author={Schmidt, Mark and Roux, Nicolas Le},
  journal={arXiv preprint arXiv:1308.6370},
  year={2013}
}
@article{mcallester1999some,
  title={Some pac-bayesian theorems},
  author={McAllester, David A},
  journal={Machine Learning},
  volume={37},
  number={3},
  pages={355--363},
  year={1999},
  publisher={Springer}
}
@article{london2016stability,
  title={Stability and generalization in structured prediction},
  author={London, Ben and Huang, Bert and Getoor, Lise},
  journal={The Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={7808--7859},
  year={2016},
  publisher={JMLR. org}
}
@inproceedings{defazio2014saga,
  title={{SAGA}: A fast incremental gradient method with support for non-strongly convex composite objectives},
  author={Defazio, Aaron and Bach, Francis and Lacoste-Julien, Simon},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1646--1654},
  year={2014}
}
@incollection{lei2017nonconvex,
title = {Non-convex Finite-Sum Optimization Via SCSG Methods},
author = {Lei, Lihua and Ju, Cheng and Chen, Jianbo and Jordan, Michael I},
booktitle = {Advances in Neural Information Processing Systems 30},
pages = {2348--2358},
year = {2017}
}

@inproceedings{nguyen2017sarah,
  title={{SARAH}: A novel method for machine learning problems using stochastic recursive gradient},
  author={Nguyen, Lam M and Liu, Jie and Scheinberg, Katya and Tak{\'a}{\v{c}}, Martin},
  booktitle={International Conference on Machine Learning},
  pages={2613--2621},
  year={2017},
  organization={JMLR. org}
}
@inproceedings{nguyen2018sgd,
  title={{SGD} and Hogwild! Convergence Without the Bounded Gradients Assumption},
  author={Nguyen, Lam and Nguyen, Phuong and van Dijk, Marten and Richt{\'a}rik, Peter and Scheinberg, Katya and Tak{\'a}{\v{c}}, Martin},
  booktitle={International Conference on Machine Learning},
  pages={3747--3755},
  year={2018}
}
@inproceedings{zhang2013linear,
  title={Linear convergence with condition number independent access of full gradients},
  author={Zhang, Lijun and Mahdavi, Mehrdad and Jin, Rong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={980--988},
  year={2013}
}
@inproceedings{keerthi2008sequential,
  title={A sequential dual method for large scale multi-class linear SVMs},
  author={Keerthi, S Sathiya and Sundararajan, Sellamanickam and Chang, Kai-Wei and Hsieh, Cho-Jui and Lin, Chih-Jen},
  booktitle={14th ACM SIGKDD},
  pages={408--416},
  year={2008}
}
@article{bottou2018optimization,
  title={Optimization methods for large-scale machine learning},
  author={Bottou, L{\'e}on and Curtis, Frank E and Nocedal, Jorge},
  journal={SIAM Review},
  volume={60},
  number={2},
  pages={223--311},
  year={2018},
  publisher={SIAM}
}
@article{mcsvmlinsolver,
  title={Accelerated proximal stochastic dual coordinate ascent for regularized loss minimization},
  author={Shalev-Shwartz, Shai and Zhang, Tong},
  journal={Mathematical Programming SERIES A and B},
  volume={5},
  year={2016}
}
@article{hill2007framework,
  title={A Framework for Kernel-Based Multi-Category Classification.},
  author={Hill, Simon I and Doucet, Arnaud},
  journal={J. Artif. Intell. Res.(JAIR)},
  volume={30},
  pages={525--564},
  year={2007}
}
@article{zhang2004statistical,
  title={Statistical analysis of some multi-category large margin classification methods},
  author={Zhang, Tong},
  journal={Journal of Machine Learning Research},
  volume={5},
  pages={1225--1251},
  year={2004},
  publisher={JMLR. org}
}
@inproceedings{zhang2004class,
  title={Class-size Independent Generalization Analsysis of Some Discriminative Multi-Category Classification},
  author={Zhang, Tong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1625--1632},
  year={2004}
}
@article{guermeur2002combining,
  title={Combining discriminant models with new multi-class SVMs},
  author={Guermeur, Yann},
  journal={Pattern Analysis \& Applications},
  volume={5},
  number={2},
  pages={168--179},
  year={2002},
  publisher={Springer}
}
@article{koltchinskii2002empirical,
  title={Empirical margin distributions and bounding the generalization error of combined classifiers},
  author={Koltchinskii, Vladimir and Panchenko, Dmitry},
  journal={Annals of Statistics},
  pages={1--50},
  year={2002},
  publisher={JSTOR}
}
@inproceedings{maurer2016vector,
  title={A vector-contraction inequality for Rademacher complexities},
  author={Maurer, Andreas},
  booktitle={International Conference on Algorithmic Learning Theory},
  pages={3--17},
  year={2016}
}
@inproceedings{cortes2016structured,
  title={Structured Prediction Theory Based on Factor Graph Complexity},
  author={Cortes, Corinna and Kuznetsov, Vitaly and Mohri, Mehryar and Yang, Scott},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2514--2522},
  year={2016}
}
@inproceedings{deng2009imagenet,
  title={Imagenet: A large-scale hierarchical image database},
  author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Li, Kai and Fei-Fei, Li},
  booktitle={IEEE Conference on Computer Vision and Pattern Recognition},
  pages={248--255},
  year={2009}
}
@inproceedings{cortes2013multi,
  title={Multi-class classification with maximum margin multiple kernel},
  author={Cortes, Corinna and Mohri, Mehryar and Rostamizadeh, Afshin},
  booktitle={International Conference on Machine Learning},
  pages={46--54},
  year={2013}
}
@article{zhang2002covering,
  title={Covering number bounds of certain regularized linear function classes},
  author={Zhang, Tong},
  journal={Journal of Machine Learning Research},
  volume={2},
  pages={527--550},
  year={2002},
  publisher={JMLR. org}
}
@book{bishop2006pattern,
  title={Pattern Recognition and Machine Learning},
  author={Bishop, Christopher M},
  year={2006},
  publisher={Springer}
}
@article{wawrzynski2017asd,
  title={ASD+M: Automatic parameter tuning in stochastic optimization and on-line learning},
  author={Wawrzy{\'n}ski, Pawe{\l}},
  journal={Neural Networks},
  volume={96},
  pages={1--10},
  year={2017},
  publisher={Elsevier}
}
@article{schmidhuber2015deep,
  title={Deep learning in neural networks: An overview},
  author={Schmidhuber, J{\"u}rgen},
  journal={Neural networks},
  volume={61},
  pages={85--117},
  year={2015},
  publisher={Elsevier}
}
@article{crammer2002algorithmic,
  title={On the algorithmic implementation of multiclass kernel-based vector machines},
  author={Crammer, Koby and Singer, Yoram},
  journal={Journal of Machine Learning Research},
  volume={2},
  pages={265--292},
  year={2002},
  publisher={JMLR. org}
}

@book{mohri2012foundations,
  title={Foundations of Machine Learning},
  author={Mohri, Mehryar and Rostamizadeh, Afshin and Talwalkar, Ameet},
  year={2012},
  publisher={MIT press}
}

@article{shi2011concentration,
  title={Concentration estimates for learning with $\ell_1$-regularizer and data dependent hypothesis spaces},
  author={Shi, Lei and Feng, Yun-Long and Zhou, Ding-Xuan},
  journal={Applied and Computational Harmonic Analysis},
  volume={31},
  number={2},
  pages={286--302},
  year={2011},
  publisher={Elsevier}
}
@article{rejchel2015fast,
  title={Fast rates for ranking with large families},
  author={Rejchel, Wojciech},
  journal={Neurocomputing},
  volume={168},
  pages={1104--1110},
  year={2015},
  publisher={Elsevier}
}
@article{clemenccon2016scaling,
  title={Scaling-up empirical risk minimization: optimization of incomplete U-statistics},
  author={Cl{\'e}men{\c{c}}on, Stephan and Colin, Igor and Bellet, Aur{\'e}lien},
  journal={The Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={2682--2717},
  year={2016},
  publisher={JMLR. org}
}
@article{zhou2002covering,
  title={The covering number in learning theory},
  author={Zhou, Ding-Xuan},
  journal={Journal of Complexity},
  volume={18},
  number={3},
  pages={739--767},
  year={2002},
  publisher={Elsevier}
}
@article{rejchel2012ranking,
  title={On ranking and generalization bounds},
  author={Rejchel, Wojciech},
  journal={Journal of Machine Learning Research},
  volume={13},
  number={May},
  pages={1373--1392},
  year={2012}
}
@article{bellet2015robustness,
  title={Robustness and generalization for metric learning},
  author={Bellet, Aur{\'e}lien and Habrard, Amaury},
  journal={Neurocomputing},
  volume={151},
  pages={259--267},
  year={2015},
  publisher={Elsevier}
}
@article{celisse2016stability,
  title={Stability revisited: new generalisation bounds for the leave-one-out},
  author={Celisse, Alain and Guedj, Benjamin},
  journal={arXiv preprint arXiv:1608.06412},
  year={2016}
}
@article{gonen2017average,
  title={Average stability is invariant to data preconditioning: Implications to exp-concave empirical risk minimization},
  author={Gonen, Alon and Shalev-Shwartz, Shai},
  journal={The Journal of Machine Learning Research},
  volume={18},
  number={1},
  pages={8245--8257},
  year={2017},
  publisher={JMLR. org}
}
@article{zhou2015generalization,
  title={Generalization performance of regularized ranking with multiscale kernels},
  author={Zhou, Yicong and Chen, Hong and Lan, Rushi and Pan, Zhibin},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  volume={27},
  number={5},
  pages={993--1002},
  year={2015},
  publisher={IEEE}
}
@article{ying2016online,
  title={Online pairwise learning algorithms},
  author={Ying, Yiming and Zhou, Ding-Xuan},
  journal={Neural computation},
  volume={28},
  number={4},
  pages={743--777},
  year={2016},
  publisher={MIT Press}
}
@inproceedings{kuznetsov2014multi,
  title={Multi-Class Deep Boosting},
  author={Kuznetsov, Vitaly and Mohri, Mehryar and Syed, Umar},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2501--2509},
  year={2014}
}
@article{lecun2015deep,
  title={Deep learning},
  author={LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
  journal={Nature},
  volume={521},
  number={7553},
  pages={436--444},
  year={2015},
  publisher={Nature Publishing Group}
}

@article{osher2005iterative,
  title={An iterative regularization method for total variation-based image restoration},
  author={Osher, Stanley and Burger, Martin and Goldfarb, Donald and Xu, Jinjun and Yin, Wotao},
  journal={Multiscale Modeling \& Simulation},
  volume={4},
  number={2},
  pages={460--489},
  year={2005},
  publisher={SIAM}
}
@article{lin2017distributed,
  title={Distributed Kernel-Based Gradient Descent Algorithms},
  author={Lin, Shao-Bo and Zhou, Ding-Xuan},
  journal={Constructive Approximation},
  pages={1--28},
  year={2017},
  publisher={Springer}
}
@inproceedings{schliserman2022stability,
  title={Stability vs implicit bias of gradient methods on separable data and beyond},
  author={Schliserman, Matan and Koren, Tomer},
  booktitle={Conference on Learning Theory},
  pages={3380--3394},
  year={2022},
  organization={PMLR}
}
@inproceedings{koren2022benign,
  title={Benign Underfitting of Stochastic Gradient Descent},
  author={Koren, Tomer and Livni, Roi and Mansour, Yishay and Sherman, Uri},
  booktitle={Advances in Neural Information Processing Systems},
  year={2022}
}
@inproceedings{lowy2023private,
  title={Private Stochastic Optimization with Large Worst-Case Lipschitz Parameter: Optimal Rates for (Non-Smooth) Convex Losses and Extension to Non-Convex Losses},
  author={Lowy, Andrew and Razaviyayn, Meisam},
  booktitle={International Conference on Algorithmic Learning Theory},
  pages={986--1054},
  year={2023},
  organization={PMLR}
}
@inproceedings{wang2022stability,
  title={Stability and Generalization for Markov Chain Stochastic Gradient Methods},
  author={Wang, Puyu and Lei, Yunwen and Ying, Yiming and Zhou, Ding-Xuan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={37735--37748},
  year={2022}
}
@article{guo2017thresholded,
  title={Thresholded spectral algorithms for sparse approximations},
  author={Guo, Zheng-Chu and Xiang, Dao-Hong and Guo, Xin and Zhou, Ding-Xuan},
  journal={Analysis and Applications},
  volume={15},
  number={03},
  pages={433--455},
  year={2017},
  publisher={World Scientific}
}
@article{sun2018markov,
  title={On markov chain gradient descent},
  author={Sun, Tao and Sun, Yuejiao and Yin, Wotao},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}
@inproceedings{ying2009sparse,
  title={Sparse metric learning via smooth optimization},
  author={Ying, Yiming and Huang, Kaizhu and Campbell, Colin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2214--2222},
  year={2009}
}
@article{kivinen2004online,
  title={Online learning with kernels},
  author={Kivinen, Jyrki and Smola, Alexander J and Williamson, Robert C},
  journal={IEEE transactions on signal processing},
  volume={52},
  number={8},
  pages={2165--2176},
  year={2004},
  publisher={IEEE}
}
@article{feng2015learning,
  title={Learning with the maximum correntropy criterion induced losses for regression.},
  author={Feng, Yunlong and Huang, Xiaolin and Shi, Lei and Yang, Yuning and Suykens, Johan AK},
  journal={Journal of Machine Learning Research},
  volume={16},
  pages={993--1034},
  year={2015}
}
@article{lin2016iterative,
  title={Iterative regularization for learning with convex loss functions},
  author={Lin, Junhong and Rosasco, Lorenzo and Zhou, Ding-Xuan},
  journal={Journal of Machine Learning Research},
  volume={17},
  number={77},
  pages={1--38},
  year={2016}
}
@article{zou2005regularization,
  title={Regularization and variable selection via the elastic net},
  author={Zou, Hui and Hastie, Trevor},
  journal={Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
  volume={67},
  number={2},
  pages={301--320},
  year={2005},
  publisher={Wiley Online Library}
}
@book{cucker2007learning,
  title={Learning Theory: an Approximation Theory Viewpoint},
  author={Cucker, Felipe and Zhou, Ding-Xuan},
  year={2007},
  publisher={Cambridge University Press}
}
%volume={24},
@article{duchi2012dual,
  title={Dual averaging for distributed optimization: Convergence analysis and network scaling},
  author={Duchi, John C and Agarwal, Alekh and Wainwright, Martin J},
  journal={IEEE Transactions on Automatic control},
  volume={57},
  number={3},
  pages={592--606},
  year={2012},
  publisher={IEEE}
}
@inproceedings{lei2021stability,
  title = 	 {Stability and Generalization of Stochastic Gradient Methods for Minimax Problems},
  author =       {Lei, Yunwen and Yang, Zhenhuan and Yang, Tianbao and Ying, Yiming},
  booktitle = 	 {International Conference on Machine Learning},
  pages = 	 {6175--6186},
  year = 	 {2021}
}
@article{dong2008learning,
  title={Learning gradients by a gradient descent algorithm},
  author={Dong, Xuemei and Zhou, Ding-Xuan},
  journal={Journal of Mathematical Analysis and Applications},
  volume={341},
  number={2},
  pages={1018--1027},
  year={2008},
  publisher={Elsevier}
}
@article{ying2007convergence,
  title={Convergence analysis of online algorithms},
  author={Ying, Yiming},
  journal={Advances in Computational Mathematics},
  volume={27},
  number={3},
  pages={273--291},
  year={2007},
  publisher={Springer}
}
@article{ying2017unregularized,
  title={Unregularized Online Learning Algorithms with General Loss Functions},
  author={Ying, Yiming and Zhou, Ding-Xuan},
  journal={Applied and Computational Harmonic Analysis},
  volume={42},
  number={2},
  pages={224--244},
  year={2017}
}
@article{davis2019stochastic,
  title={Stochastic model-based minimization of weakly convex functions},
  author={Davis, Damek and Drusvyatskiy, Dmitriy},
  journal={SIAM Journal on Optimization},
  volume={29},
  number={1},
  pages={207--239},
  year={2019},
  publisher={SIAM}
}
@article{davis2018graphical,
  title={Graphical convergence of subgradients in nonconvex optimization and learning},
  author={Davis, Damek and Drusvyatskiy, Dmitriy},
  journal={arXiv preprint arXiv:1810.07590},
  year={2018}
}
@article{shalev2013stochastic,
  title={Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization.},
  author={Shalev-Shwartz, Shai and Zhang, Tong},
  journal={Journal of Machine Learning Research},
  volume={14},
  number={2},
  year={2013}
}
@book{de2012decoupling,
  title={Decoupling: from dependence to independence},
  author={De la Pena, Victor and Gin{\'e}, Evarist},
  year={2012},
  publisher={Springer Science \& Business Media}
}
@article{duchi2012randomized,
  title={Randomized smoothing for stochastic optimization},
  author={Duchi, John C and Bartlett, Peter L and Wainwright, Martin J},
  journal={SIAM Journal on Optimization},
  volume={22},
  number={2},
  pages={674--701},
  year={2012},
  publisher={SIAM}
}
@inproceedings{
lei2021sharper,
title={Sharper Generalization Bounds for Learning with Gradient-dominated Objective Functions},
author={Yunwen Lei and Yiming Ying},
booktitle={International Conference on Learning Representations},
year={2021}
}
@inproceedings{li2019convergence,
  title={On the convergence of stochastic gradient descent with adaptive stepsizes},
  author={Li, Xiaoyu and Orabona, Francesco},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={983--992},
  year={2019},
  organization={PMLR}
}
@inproceedings{richards2021learning,
  title={Learning with Gradient Descent and Weakly Convex Losses},
  author={Richards, Dominic and Rabbat, Mike},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1990--1998},
  year={2021},
  organization={PMLR}
}
@InProceedings{gergely2021,
  title = 	 {Information-Theoretic Generalization Bounds for Stochastic Gradient Descent},
  author =       {Neu, Gergely and Dziugaite, Gintare Karolina and Haghifam, Mahdi and Roy, Daniel M.},
  booktitle = 	 {Annual Conference on Learning Theory},
  pages = 	 {3526--3545},
  year = 	 {2021},
  volume = 	 {134}
}

@article{xu2017information,
  title={Information-theoretic analysis of generalization capability of learning algorithms},
  author={Xu, Aolin and Raginsky, Maxim},
  journal={Advances in Neural Information Processing Systems},
  volume={2017},
  pages={2525--2534},
  year={2017}
}
@article{kearns1999algorithmic,
  title={Algorithmic stability and sanity-check bounds for leave-one-out cross-validation},
  author={Kearns, Michael and Ron, Dana},
  journal={Neural computation},
  volume={11},
  number={6},
  pages={1427--1453},
  year={1999}
}
@book{qiu2014cognitive,
  title={Cognitive networked sensing and big data},
  author={Qiu, Robert and Wicks, Michael},
  year={2014},
  publisher={Springer}
}

@techreport{bertsekas2011incremental,
            title={Incremental gradient, subgradient, and proximal methods for
                        convex optimization: A survey},
            author={Bertsekas, Dimitri P},
            number={Report LIDS-P-2848},
            institution={Laboratory for Information and Decision Systems, MIT},
            year={2010},
            address={Cambridge, MA}
        }

  %Report LIDS-P-2848, Laboratory for Information and Decision Systems, MIT, Cambridge, MA, 2010

@incollection{bottou1998online,
author = {Bottou, L{\'e}on},
title = {On-line Learning and Stochastic Approximations},
booktitle = {On-line Learning in Neural Networks},
editor = {Saad, David},
year = {1998},
pages = {9--42},
publisher = {Cambridge University Press},
address = {New York, NY, USA},
}
@inproceedings{guyon2005result,
  title={Result analysis of the NIPS 2003 feature selection challenge},
  author={Guyon, Isabelle and Gunn, Steve and Ben-Hur, Asa and Dror, Gideon},
  booktitle={Advances in Neural Information Processing Systems},
  pages={545--552},
  year={2005}
}
@article{lewis2004rcv1,
  title={Rcv1: A new benchmark collection for text categorization research},
  author={Lewis, David D and Yang, Yiming and Rose, Tony G and Li, Fan},
  journal={Journal of Machine Learning Research},
  volume={5},
  number={Apr},
  pages={361--397},
  year={2004}
}
@misc{Dua:2017,
author = "Dheeru, Dua and Karra Taniskidou, Efi",
year = "2017",
title = "{UCI} Machine Learning Repository",
url = "http://archive.ics.uci.edu/ml",
institution = "University of California, Irvine, School of Information and Computer Sciences" }
@incollection{platt1999fast,
  title={Fast training of support vector machines using sequential minimal optimization},
  booktitle={Advances in Kernel Methods—Support Vector Learning},
  pages = {185--208},
  editor={Sch\"olkopf, Bernhard and Burges, Christopher and Smola, Alexander},
  author={Platt, John},
  publisher={MIT Press},
  address={Cambridge, MA},
  year={1999}
}
@article{schopfer2016linear,
title={Linear convergence of the randomized sparse {K}aczmarz method},
author={Sch{\"o}pfer, Frank and Lorenz, Dirk A.},
journal={Math. Program.},
year={2018},
issn={1436-4646},
doi={10.1007/s10107-017-1229-1},
url={https://doi.org/10.1007/s10107-017-1229-1}
}

@article{cortes1995support,
  title={Support-vector networks},
  author={Cortes, Corinna and Vapnik, Vladimir},
  journal={Machine learning},
  volume={20},
  number={3},
  pages={273--297},
  year={1995},
  publisher={Springer Netherlands}
}
@article{tibshirani1996regression,
  title={Regression shrinkage and selection via the lasso},
  author={Tibshirani, Robert},
  journal={Journal of the Royal Statistical Society. Series B (Methodological)},
  pages={267--288},
  year={1996},
  publisher={JSTOR}
}
@book{friedman2001elements,
  title={The Elements of Statistical Learning},
  author={Friedman, Jerome and Hastie, Trevor and Tibshirani, Robert},
  volume={1},
  year={2001},
  publisher={Springer}
}
@article{kaczmarz1937angenaherte,
  title={Angen{\"a}herte aufl{\"o}sung von systemen linearer gleichungen},
  author={Kaczmarz, Stefan},
  journal={Bulletin International de l’Academie Polonaise des Sciences et des Lettres},
  volume={35},
  pages={355--357},
  year={1937}
}
@inproceedings{juditsky2006generalization,
  title={Generalization error bounds for aggregation by mirror descent with averaging},
  author={Juditsky, Anatoli and Nazin, Alexander and Tsybakov, Alexandre and Vayatis, Nicolas},
  booktitle={Advances in Neural Information Processing Systems},
  pages={603--610},
  year={2006}
}
@article{cai2010singular,
  title={A singular value thresholding algorithm for matrix completion},
  author={Cai, Jian-Feng and Cand{\`e}s, Emmanuel J and Shen, Zuowei},
  journal={SIAM Journal on Optimization},
  volume={20},
  number={4},
  pages={1956--1982},
  year={2010},
  publisher={SIAM}
}
@article{cox1958regression,
  title={The regression analysis of binary sequences},
  author={Cox, David R},
  journal={Journal of the Royal Statistical Society. Series B (Methodological)},
  pages={215--242},
  year={1958},
  publisher={JSTOR}
}
@book{borwein2010convex,
  title={Convex Analysis and Nonlinear Optimization: Theory and Examples},
  author={Borwein, Jonathan M and Lewis, Adrian S},
  year={2010},
  publisher={Springer Science \& Business Media}
}
@inproceedings{shamir2013stochastic,
  title={Stochastic Gradient Descent for Non-smooth Optimization Convergence Results and Optimal Averaging Schemes},
  author={Shamir, Ohad and Zhang, Tong},
  booktitle={International Conference on Machine Learning},
  pages={71--79},
  year={2013}
}

 % publisher={JMLR},
%  address={Atlanta}
@article{ball1994sharp,
  title={Sharp uniform convexity and smoothness inequalities for trace norms},
  author={Ball, Keith and Carlen, Eric A and Lieb, Elliott H},
  journal={Inventiones mathematicae},
  volume={115},
  number={1},
  pages={463--482},
  year={1994},
  publisher={Springer}
}
@article{parikh2014proximal,
  title={Proximal Algorithms},
  author={Parikh, Neal and Boyd, Stephen P},
  journal={Foundations and Trends in optimization},
  volume={1},
  number={3},
  pages={127--239},
  year={2014}
}
@inproceedings{yuan2019stagewise,
  title={Stagewise training accelerates convergence of testing error over SGD},
  author={Yuan, Zhuoning and Yan, Yan and Jin, Rong and Yang, Tianbao},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2604--2614},
  year={2019}
}
@inproceedings{deng2021toward,
  title={Toward better generalization bounds with locally elastic stability},
  author={Deng, Zhun and He, Hangfeng and Su, Weijie},
  booktitle={International Conference on Machine Learning},
  pages={2590--2600},
  year={2021},
  organization={PMLR}
}
@article{zhou2021understanding,
  title={Understanding generalization error of sgd in nonconvex optimization},
  author={Zhou, Yi and Liang, Yingbin and Zhang, Huishuai},
  journal={Machine Learning},
  pages={1--31},
  year={2021},
  publisher={Springer}
}
@inproceedings{rakhlin2012making,
  title={Making Gradient Descent Optimal for Strongly Convex Stochastic Optimization},
  author={Rakhlin, Alexander and Shamir, Ohad and Sridharan, Karthik},
  booktitle={International Conference on Machine Learning},
  pages={449--456},
  year={2012}
}
@inproceedings{bousquet2008tradeoffs,
  title={The tradeoffs of large scale learning},
  author={Bousquet, Olivier and Bottou, L{\'e}on},
  booktitle={Advances in Neural Information Processing Systems},
  pages={161--168},
  year={2008}
}
@inproceedings{rosasco2015learning,
  title={Learning with incremental iterative regularization},
  author={Rosasco, Lorenzo and Villa, Silvia},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1630--1638},
  year={2015}
}
@inproceedings{lin2016optimal,
  title={Optimal learning for multi-pass stochastic gradient methods},
  author={Lin, Junhong and Rosasco, Lorenzo},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4556--4564},
  year={2016}
}
@inproceedings{hardt2016train,
  title={Train faster, generalize better: Stability of stochastic gradient descent},
  author={Hardt, Moritz and Recht, Ben and Singer, Yoram},
  booktitle={International Conference on Machine Learning},
  pages={1225--1234},
  year={2016}
}
@article{lin2015iterative,
  title={Iterative regularization for learning with convex loss functions},
  author={Lin, Junhong and Rosasco, Lorenzo and Zhou, Ding-Xuan},
  journal={Journal of Machine Learning Research},
  volume={17},
  number={77},
  pages={1--38},
  year={2016}
}
@article{lin2015learning,
  title={Learning Theory of Randomized {K}aczmarz Algorithm},
  author={Lin, Junhong and Zhou, Ding-Xuan},
  journal={Journal of Machine Learning Research},
  volume={16},
  pages={3341--3365},
  year={2015}
}
@article{shalev2011stochastic,
  title={Stochastic methods for $\ell_1$-regularized loss minimization},
  author={Shalev-Shwartz, Shai and Tewari, Ambuj},
  journal={Journal of Machine Learning Research},
  volume={12},
  pages={1865--1892},
  year={2011},
  publisher={JMLR. org}
}
@article{smale2009online,
  title={Online learning with Markov sampling},
  author={Smale, Steve and Zhou, Ding-Xuan},
  journal={Analysis and Applications},
  volume={7},
  number={01},
  pages={87--113},
  year={2009},
  publisher={World Scientific}
}
@article{agarwal2013generalization,
  title={The generalization ability of online algorithms for dependent data},
  author={Agarwal, Alekh and Duchi, John C},
  journal={IEEE Transactions on Information Theory},
  volume={59},
  number={1},
  pages={573--587},
  year={2013},
  publisher={IEEE}
}

@article{ying2008online,
  title={Online gradient descent learning algorithms},
  author={Ying, Yiming and Pontil, Massimiliano},
  journal={Foundations of Computational Mathematics},
  volume={8},
  number={5},
  pages={561--596},
  year={2008},
  publisher={Springer}
}
@inproceedings{srebro2011universality,
  title={On the universality of online mirror descent},
  author={Srebro, Nathan and Sridharan, Karthik and Tewari, Ambuj},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2645--2653},
  year={2011}
}
@article{dang2015stochastic,
  title={Stochastic block mirror descent methods for nonsmooth and stochastic optimization},
  author={Dang, Cong D and Lan, Guanghui},
  journal={SIAM Journal on Optimization},
  volume={25},
  number={2},
  pages={856--881},
  year={2015},
  publisher={SIAM}
}
@article{juditsky2005recursive,
  title={Recursive aggregation of estimators by the mirror descent algorithm with averaging},
  author={Juditsky, AB and Nazin, AV and Tsybakov, AB and Vayatis, Nicolas},
  journal={Problems of Information Transmission},
  volume={41},
  number={4},
  pages={368--384},
  year={2005},
  publisher={Springer}
}
@article{nemirovski2009robust,
  title={Robust stochastic approximation approach to stochastic programming},
  author={Nemirovski, Arkadi and Juditsky, Anatoli and Lan, Guanghui and Shapiro, Alexander},
  journal={SIAM Journal on Optimization},
  volume={19},
  number={4},
  pages={1574--1609},
  year={2009},
  publisher={SIAM}
}

@article{yin2008bregman,
  title={Bregman iterative algorithms for $\ell_1$-minimization with applications to compressed sensing},
  author={Yin, Wotao and Osher, Stanley and Goldfarb, Donald and Darbon, Jerome},
  journal={SIAM Journal on Imaging sciences},
  volume={1},
  number={1},
  pages={143--168},
  year={2008},
  publisher={SIAM}
}
@book{deimling1985nonlinear,
  title={Nonlinear functional analysis},
  author={Deimling, Klaus},
  year={1985},
  publisher={Springer}
}
@article{hu2015regularization,
  title={Regularization schemes for minimum error entropy principle},
  author={Hu, Ting and Fan, Jun and Wu, Qiang and Zhou, Ding-Xuan},
  journal={Analysis and Applications},
  volume={13},
  number={04},
  pages={437--455},
  year={2015},
  publisher={World Scientific}
}
@article{nesterov2013gradient,
  title={Gradient methods for minimizing composite functions},
  author={Nesterov, Yu},
  journal={Mathematical Programming},
  volume={140},
  number={1},
  pages={125--161},
  year={2013},
  publisher={Springer}
}

@article{lan2012optimal,
  title={An optimal method for stochastic composite optimization},
  author={Lan, Guanghui},
  journal={Mathematical Programming},
  volume={133},
  number={1-2},
  pages={365--397},
  year={2012},
  publisher={Springer}
}
@article{agarwal2012information,
  title={Information-Theoretic Lower Bounds on the Oracle Complexity of Stochastic Convex Optimization},
  author={Agarwal, Alekh and Bartlett, Peter L and Ravikumar, Pradeep and Wainwright, Martin J},
  journal={IEEE Transactions on Information Theory},
  volume={5},
  number={58},
  pages={3235--3249},
  year={2012}
}
@article{smale2006online,
  title={Online learning algorithms},
  author={Smale, Steve and Yao, Yuan},
  journal={Foundations of computational mathematics},
  volume={6},
  number={2},
  pages={145--170},
  year={2006},
  publisher={Springer}
}
@inproceedings{gao2011consistency,
  title={On the Consistency of Multi-Label Learning.},
  author={Gao, Wei and Zhou, Zhi-Hua},
  booktitle={Conference on Learning Theory},
  volume={19},
  pages={341--358},
  year={2011}
}
@inproceedings{ying2016stochastic,
  title={Stochastic Online {AUC} Maximization},
  author={Ying, Yiming and Wen, Longyin and Lyu, Siwei},
  booktitle={Advances in Neural Information Processing Systems},
  pages={451--459},
  year={2016}
}
@article{fan2016consistency,
  title={Consistency analysis of an empirical minimum error entropy algorithm},
  author={Fan, Jun and Hu, Ting and Wu, Qiang and Zhou, Ding-Xuan},
  journal={Applied and Computational Harmonic Analysis},
  volume={41},
  number={1},
  pages={164--189},
  year={2016},
  publisher={Elsevier}
}
@article{christmann2016robustness,
  title={On the robustness of regularized pairwise learning methods based on kernels},
  author={Christmann, Andreas and Zhou, Ding-Xuan},
  journal={Journal of Complexity},
  volume={37},
  pages={1--33},
  year={2016},
  publisher={Elsevier}
}
@article{szabo2016learning,
  title={Learning theory for distribution regression},
  author={Szab{\'o}, Zolt{\'a}n and Sriperumbudur, Bharath K and P{\'o}czos, Barnab{\'a}s and Gretton, Arthur},
  journal={The Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={5272--5311},
  year={2016},
  publisher={JMLR. org}
}
@inproceedings{verma2015sample,
  title={Sample complexity of learning mahalanobis distance metrics},
  author={Verma, Nakul and Branson, Kristin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2584--2592},
  year={2015}
}
@article{shen2019stability,
  title={Stability and optimization error of stochastic gradient descent for pairwise learning},
  author={Shen, Wei and Yang, Zhenhuan and Ying, Yiming and Yuan, Xiaoming},
  journal={Analysis and Applications},
  pages={1--41},
  year={2019},
  publisher={World Scientific}
}
@article{rakhlin2005stability,
  title={Stability results in learning theory},
  author={Rakhlin, Alexander and Mukherjee, Sayan and Poggio, Tomaso},
  journal={Analysis and Applications},
  volume={3},
  number={04},
  pages={397--417},
  year={2005},
  publisher={World Scientific}
}
@article{ye2019fast,
  title={Fast generalization rates for distance metric learning},
  author={Ye, Han-Jia and Zhan, De-Chuan and Jiang, Yuan},
  journal={Machine Learning},
  volume={108},
  number={2},
  pages={267--295},
  year={2019},
  publisher={Springer}
}
@article{cao2012generalization,
  title={Generalization bounds for metric and similarity learning},
  author={Cao, Qiong and Guo, Zheng-Chu and Ying, Yiming},
  journal={Machine Learning},
  volume={102},
  number={1},
  pages={115--132},
  year={2016},
  publisher={Springer}
}
@article{mukherjee2006learning,
  title={Learning coordinate covariances via gradients},
  author={Mukherjee, Sayan and Zhou, Ding-Xuan},
  journal={Journal of Machine Learning Research},
  volume={7},
  pages={519--549},
  year={2006},
  publisher={JMLR. org}
}
@article{hardt2016identity,
  title={Identity matters in deep learning},
  author={Hardt, Moritz and Ma, Tengyu},
  journal={arXiv preprint arXiv:1611.04231},
  year={2016}
}
@inproceedings{
Long2020Generalization,
title={Generalization bounds for deep convolutional neural networks},
author={Philip M. Long and Hanie Sedghi},
booktitle={International Conference on Learning Representations},
year={2020}
}
@inproceedings{bassily2019private,
  title={Private stochastic convex optimization with optimal rates},
  author={Bassily, Raef and Feldman, Vitaly and Talwar, Kunal and Thakurta, Abhradeep Guha},
  booktitle={Advances in Neural Information Processing Systems},
  pages={11279--11288},
  year={2019}
}
@inproceedings{rivasplata2018pac,
  title={PAC-Bayes bounds for stable algorithms with instance-dependent priors},
  author={Rivasplata, Omar and Parrado-Hern{\'a}ndez, Emilio and Shawe-Taylor, John S and Sun, Shiliang and Szepesv{\'a}ri, Csaba},
  booktitle={Advances in Neural Information Processing Systems},
  pages={9214--9224},
  year={2018}
}
@article{clemenccon2008ranking,
  title={Ranking and empirical minimization of {U}-statistics},
  author={Cl{\'e}men{\c{c}}on, St{\'e}phan and Lugosi, Gabor and Vayatis, Nicolas},
  journal={The Annals of Statistics},
  pages={844--874},
  year={2008},
  publisher={JSTOR}
}
@article{hu2013learning,
  title={Learning theory approach to minimum error entropy criterion},
  author={Hu, Ting and Fan, Jun and Wu, Qiang and Zhou, Ding-Xuan},
  journal={Journal of Machine Learning Research},
  volume={14},
  number={Feb},
  pages={377--397},
  year={2013}
}
@article{smale2007learning,
  title={Learning theory estimates via integral operators and their approximations},
  author={Smale, Steve and Zhou, Ding-Xuan},
  journal={Constructive approximation},
  volume={26},
  number={2},
  pages={153--172},
  year={2007},
  publisher={Springer}
}
@article{agarwal2010information,
  title={Information-theoretic lower bounds on the oracle complexity of stochastic convex optimization},
  author={Agarwal, Alekh and Bartlett, Peter L and Ravikumar, Pradeep and Wainwright, Martin J},
  journal={IEEE Transactions on Information Theory},
  volume={99},
  pages={1--1},
  year={2010}
}
@inproceedings{krizhevsky2012imagenet,
  title={Imagenet classification with deep convolutional neural networks},
  author={Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1097--1105},
  year={2012}
}
@article{hoffman1952approximate,
  title={On Approximate Solutions of Systems of Linear Inequalities},
  author={Hoffman, Alan J},
  journal={Journal of Research of the National Bureau of Standards},
  volume={49},
  pages={263--265},
  number={4},
  year={1952}
}
@article{necoara2018linear,
  title={Linear convergence of first order methods for non-strongly convex optimization},
  author={Necoara, Ion and Nesterov, Yu and Glineur, Francois},
  journal={Mathematical Programming},
  pages={1--39},
  year={2018},
  publisher={Springer}
}
@article{cai2009blinearized,
  title={Linearized Bregman iterations for frame-based image deblurring},
  author={Cai, Jian-Feng and Osher, Stanley and Shen, Zuowei},
  journal={SIAM Journal on Imaging Sciences},
  volume={2},
  number={1},
  pages={226--252},
  year={2009},
  publisher={SIAM}
}
@misc{gong2015globally,
  title={On Globally Linear Convergence of Dual Gradient Descent Method for Sparse Solutions},
  author={Gong, Pinghua and Lai, Ming-Jun and Ye, Jieping},
  year={2015},
  journal={manuscript},
  howpublished ={\url{http://alpha.math.uga.edu/~mjlai/papers/Lai2015N.pdf}}
}
@article{hoffman2003approximate,
  title={On approximate solutions of systems of linear inequalities},
  author={Hoffman, Alan J},
  journal={Selected Papers Of Alan J Hoffman: With Commentary},
  pages={174--176},
  year={2003}
}
@article{agarwal2009generalization,
  title={Generalization bounds for ranking algorithms via algorithmic stability},
  author={Agarwal, Shivani and Niyogi, Partha},
  journal={Journal of Machine Learning Research},
  volume={10},
  number={Feb},
  pages={441--474},
  year={2009}
}
@article{zhao2017learning,
  title={Learning rates for regularized least squares ranking algorithm},
  author={Zhao, Yulong and Fan, Jun and Shi, Lei},
  journal={Analysis and Applications},
  pages={1--22},
  year={2017},
  publisher={World Scientific}
}

@article{caponnetto2007optimal,
  title={Optimal rates for the regularized least-squares algorithm},
  author={Caponnetto, Andrea and De Vito, Ernesto},
  journal={Foundations of Computational Mathematics},
  volume={7},
  number={3},
  pages={331--368},
  year={2007},
  publisher={Springer}
}
@article{shi2013learning,
  title={Learning theory estimates for coefficient-based regularized regression},
  author={Shi, Lei},
  journal={Applied and Computational Harmonic Analysis},
  volume={34},
  number={2},
  pages={252--265},
  year={2013},
  publisher={Elsevier}
}
@article{yao2010complexity,
  title={On complexity issues of online learning algorithms},
  author={Yao, Yuan},
  journal={IEEE Transactions on Information Theory},
  volume={56},
  number={12},
  pages={6470--6481},
  year={2010},
  publisher={IEEE}
}
@article{chaudhuri2011differentially,
  title={Differentially private empirical risk minimization.},
  author={Chaudhuri, Kamalika and Monteleoni, Claire and Sarwate, Anand D},
  journal={Journal of Machine Learning Research},
  volume={12},
  number={3},
  year={2011}
}
@inproceedings{bassily2014private,
  title={Private empirical risk minimization: Efficient algorithms and tight error bounds},
  author={Bassily, Raef and Smith, Adam and Thakurta, Abhradeep},
  booktitle={2014 IEEE 55th Annual Symposium on Foundations of Computer Science},
  pages={464--473},
  year={2014},
  organization={IEEE}
}
@inproceedings{shalev2009stochasticcolt,
  title={Stochastic Convex Optimization.},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Srebro, Nathan and Sridharan, Karthik},
  booktitle={COLT},
  year={2009}
}
@inproceedings{wang2019subsampled,
  title={Subsampled R{\'e}nyi differential privacy and analytical moments accountant},
  author={Wang, Yu-Xiang and Balle, Borja and Kasiviswanathan, Shiva Prasad},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={1226--1235},
  year={2019},
  organization={PMLR}
}
@inproceedings{feldman2018privacy,
  title={Privacy amplification by iteration},
  author={Feldman, Vitaly and Mironov, Ilya and Talwar, Kunal and Thakurta, Abhradeep},
  booktitle={2018 IEEE 59th Annual Symposium on Foundations of Computer Science (FOCS)},
  pages={521--532},
  year={2018},
  organization={IEEE}
}
@inproceedings{abadi2016deep,
  title={Deep learning with differential privacy},
  author={Abadi, Martin and Chu, Andy and Goodfellow, Ian and McMahan, H Brendan and Mironov, Ilya and Talwar, Kunal and Zhang, Li},
  booktitle={Proceedings of the 2016 ACM SIGSAC Conference on Computer and Communications Security},
  pages={308--318},
  year={2016}
}
@inproceedings{pillaud2018statistical,
  title={Statistical optimality of stochastic gradient descent on hard learning problems through multiple passes},
  author={Pillaud-Vivien, Loucas and Rudi, Alessandro and Bach, Francis},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8114--8124},
  year={2018}
}
@inproceedings{marteau2019beyond,
  title={Beyond Least-Squares: Fast Rates for Regularized Empirical Risk Minimization through Self-Concordance},
  author={Marteau-Ferey, Ulysse and Ostrovskii, Dmitrii and Bach, Francis and Rudi, Alessandro},
  booktitle={Conference on Learning Theory},
  pages={2294--2340},
  year={2019}
}
@article{caponnetto2010cross,
  title={Cross-validation based adaptation for regularization operators in learning theory},
  author={Caponnetto, Andrea and Yao, Yuan},
  journal={Analysis and Applications},
  volume={8},
  number={02},
  pages={161--183},
  year={2010},
  publisher={World Scientific}
}

@article{bach2014adaptivity,
  title={Adaptivity of averaged stochastic gradient descent to local strong convexity for logistic regression},
  author={Bach, Francis},
  journal={The Journal of Machine Learning Research},
  volume={15},
  number={1},
  pages={595--627},
  year={2014},
  publisher={JMLR. org}
}
@article{bauer2007regularization,
  title={On regularization algorithms in learning theory},
  author={Bauer, Frank and Pereverzev, Sergei and Rosasco, Lorenzo},
  journal={Journal of complexity},
  volume={23},
  number={1},
  pages={52--72},
  year={2007},
  publisher={Elsevier}
}
@article{vito2004some,
  title={Some properties of regularized kernel methods},
  author={Vito, Ernesto De and Rosasco, Lorenzo and Caponnetto, Andrea and Piana, Michele and Verri, Alessandro},
  journal={Journal of Machine Learning Research},
  volume={5},
  number={Oct},
  pages={1363--1390},
  year={2004}
}
@article{yao2007early,
  title={On early stopping in gradient descent learning},
  author={Yao, Yuan and Rosasco, Lorenzo and Caponnetto, Andrea},
  journal={Constructive Approximation},
  volume={26},
  number={2},
  pages={289--315},
  year={2007},
  publisher={Springer}
}
@article{devolder2014first,
  title={First-order methods of smooth convex optimization with inexact oracle},
  author={Devolder, Olivier and Glineur, Fran{\c{c}}ois and Nesterov, Yurii},
  journal={Mathematical Programming},
  volume={146},
  number={1-2},
  pages={37--75},
  year={2014},
  publisher={Springer}
}
@article{nesterov2015universal,
  title={Universal gradient methods for convex optimization problems},
  author={Nesterov, Yu},
  journal={Mathematical Programming},
  volume={152},
  number={1-2},
  pages={381--404},
  year={2015},
  publisher={Springer}
}
@article{lin2018modified,
  title={Modified {F}ej{\'e}r sequences and applications},
  author={Lin, Junhong and Rosasco, Lorenzo and Villa, Silvia},
  journal={Computational Optimization and Applications},
  volume={71},
  number={1},
  pages={95--113},
  year={2018},
  publisher={Springer}
}
@article{yashtini2016global,
  title={On the global convergence rate of the gradient descent method for functions with H{\"o}lder continuous gradients},
  author={Yashtini, Maryam},
  journal={Optimization Letters},
  volume={10},
  number={6},
  pages={1361--1370},
  year={2016},
  publisher={Springer}
}
@article{zhang2017restricted,
  title={The restricted strong convexity revisited: analysis of equivalence to error bound and quadratic growth},
  author={Zhang, Hui},
  journal={Optimization Letters},
  volume={11},
  number={4},
  pages={817--833},
  year={2017},
  publisher={Springer}
}
@article{lei2020convergence,
  title={Convergence of online mirror descent},
  author={Lei, Yunwen and Zhou, Ding-Xuan},
  journal={Applied and Computational Harmonic Analysis},
  volume={48},
  number={1},
  pages={343--373},
  year={2020},
  publisher={Elsevier}
}
@article{lei2019stochastic,
  title={Stochastic gradient descent for nonconvex learning without bounded gradient assumptions},
  author={Lei, Yunwen and Hu, Ting and Li, Guiying and Tang, Ke},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  year={2019},
  doi={10.1109/TNNLS.2019.2952219}
}
@article{lin2017generalization,
  title={Generalization Properties of Doubly Online Learning Algorithms},
  author={Lin, Junhong and Rosasco, Lorenzo},
  journal={arXiv preprint arXiv:1707.00577},
  year={2017}
}

@article{lin2018online,
  title={Online learning algorithms can converge comparably fast as batch learning},
  author={Lin, Junhong and Zhou, Ding-Xuan},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  volume={29},
  number={6},
  pages={2367--2378},
  year={2018},
  publisher={IEEE}
}

@article{zhang2015restricted,
  title={Restricted strong convexity and its applications to convergence analysis of gradient-type methods in convex optimization},
  author={Zhang, Hui and Cheng, Lizhi},
  journal={Optimization Letters},
  volume={9},
  number={5},
  pages={961--979},
  year={2015},
  publisher={Springer}
}
@inproceedings{kar2013generalization,
  title={On the Generalization Ability of Online Learning Algorithms for Pairwise Loss Functions},
  author={Kar, Purushottam and Sriperumbudur, Bharath and Jain, Prateek and Karnick, Harish},
  booktitle={International Conference on Machine Learning},
  pages={441--449},
  year={2013}
}
@inproceedings{wang2012generalization,
  title={Generalization Bounds for Online Learning Algorithms with Pairwise Loss Functions.},
  author={Wang, Yuyang and Khardon, Roni and Pechyony, Dmitry and Jones, Rosie},
  booktitle={Conference on Learning Theory},
  volume={23},
  pages={13--1},
  year={2012}
}
@article{ying2006online,
  title={Online regularized classification algorithms},
  author={Ying, Yiming and Zhou, Ding-Xuan},
  journal={IEEE Transactions on Information Theory},
  volume={52},
  number={11},
  pages={4775--4788},
  year={2006},
  publisher={IEEE}
}
@article{tarres2014online,
  title={Online learning as stochastic approximation of regularization paths: optimality and almost-sure convergence},
  author={Tarres, Pierre and Yao, Yuan},
  journal={IEEE Transactions on Information Theory},
  volume={60},
  number={9},
  pages={5716--5735},
  year={2014},
  publisher={IEEE}
}
@inproceedings{neyshabur2017exploring,
  title={Exploring generalization in deep learning},
  author={Neyshabur, Behnam and Bhojanapalli, Srinadh and McAllester, David and Srebro, Nati},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5947--5956},
  year={2017}
}
@inproceedings{ZhangBHRV17,
  author    = {Chiyuan Zhang and
               Samy Bengio and
               Moritz Hardt and
               Benjamin Recht and
               Oriol Vinyals},
  title     = {Understanding deep learning requires rethinking generalization},
  booktitle = {International Conference on Learning Representations},
  year      = {2017},
}
@article{lin2017does,
  title={Why does deep and cheap learning work so well?},
  author={Lin, Henry W and Tegmark, Max and Rolnick, David},
  journal={Journal of Statistical Physics},
  volume={168},
  number={6},
  pages={1223--1247},
  year={2017},
  publisher={Springer}
}
@inproceedings{bartlett2017spectrally,
  title={Spectrally-normalized margin bounds for neural networks},
  author={Bartlett, Peter L and Foster, Dylan J and Telgarsky, Matus J},
  booktitle={Advances in Neural Information Processing Systems},
  pages={6240--6249},
  year={2017}
}
@article{hu2009online,
  title={Online learning with samples drawn from non-identical distributions},
  author={Hu, Ting and Zhou, Ding-Xuan},
  journal={Journal of Machine Learning Research},
  volume={10},
  number={Dec},
  pages={2873--2898},
  year={2009}
}
@article{hong2018distributed,
  title={A Distributed, Asynchronous, and Incremental Algorithm for Nonconvex Optimization: An ADMM Approach},
  author={Hong, Mingyi},
  journal={IEEE Transactions on Control of Network Systems},
  volume={5},
  number={3},
  pages={935--945},
  year={2018},
  publisher={IEEE}
}
@article{bertsekas2000gradient,
  title={Gradient convergence in gradient methods with errors},
  author={Bertsekas, Dimitri P and Tsitsiklis, John N},
  journal={SIAM Journal on Optimization},
  volume={10},
  number={3},
  pages={627--642},
  year={2000},
  publisher={SIAM}
}
@article{gaivoronski1994convergence,
  title={Convergence analysis of parallel backpropagation algorithm for neural networks},
  author={Gaivoronski, AA},
  journal={Optimization Methods and Software},
  volume={4},
  pages={117--134},
  year={1994}
}
@article{smale2003estimating,
  title={Estimating the approximation error in learning theory},
  author={Smale, Steve and Zhou, Ding-Xuan},
  journal={Analysis and Applications},
  volume={1},
  number={01},
  pages={17--41},
  year={2003},
  publisher={World Scientific}
}


@article{ball1994norm,
  title={Sharp uniform convexity and smoothness inequalities for trace norms},
  author={Ball, Keith and Carlen, Eric A and Lieb, Elliott H},
  journal={Inventiones Mathematicae},
  volume={115},
  number={1},
  pages={463--482},
  year={1994},
  publisher={Springer}
}
@inproceedings{moulines2011non,
  title={Non-asymptotic analysis of stochastic approximation algorithms for machine learning},
  author={Bach, Francis and Moulines, Eric},
  booktitle={Advances in Neural Information Processing Systems},
  pages={451--459},
  year={2011}
}
@article{dieuleveut2016nonparametric,
  title={Nonparametric stochastic approximation with large step-sizes},
  author={Dieuleveut, Aymeric and Bach, Francis},
  journal={Annals of Statistics},
  volume={44},
  number={4},
  pages={1363--1399},
  year={2016},
  publisher={Institute of Mathematical Statistics}
}
@inproceedings{zhou2017stochastic,
  title={Stochastic Mirror Descent in Variationally Coherent Optimization Problems},
  author={Zhou, Zhengyuan and Mertikopoulos, Panayotis and Bambos, Nicholas and Boyd, Stephen and Glynn, Peter W},
  booktitle={Advances in Neural Information Processing Systems},
  pages={7043--7052},
  year={2017}
}
@article{nie2015constructive,
  title={Constructive analysis for coefficient regularization regression algorithms},
  author={Nie, Weilin and Wang, Cheng},
  journal={Journal of Mathematical Analysis and Applications},
  volume={431},
  number={2},
  pages={1153--1171},
  year={2015},
  publisher={Elsevier}
}
@article{yin2010analysis,
  title={Analysis and generalizations of the linearized Bregman method},
  author={Yin, Wotao},
  journal={SIAM Journal on Imaging Sciences},
  volume={3},
  number={4},
  pages={856--877},
  year={2010},
  publisher={SIAM}
}
@inproceedings{duchi2010composite,
  title={Composite Objective Mirror Descent},
  author={Duchi, John and Shalev-Shwartz, Shai and Singer, Yoram and Tewari, Ambuj},
  booktitle={Conference on Learning Theory},
  pages={14--26},
  year={2010}
}
@article{lin2015modified,
  title={Modified {F}ej\'er sequences and applications},
  author={Lin, Junhong and Rosasco, Lorenzo and Villa, Silvia and Zhou, Ding-Xuan},
  journal={arXiv preprint arXiv:1510.04641},
  year={2015}
}
@book{nemirovsky1983problem,
  title={Problem Complexity and Method Efficiency in Optimization.},
  author={Nemirovsky, A-S and Yudin, D-B },
  year={1983},
  publisher={John Wiley \& Sons}
}
@article{beck2003mirror,
  title={Mirror descent and nonlinear projected subgradient methods for convex optimization},
  author={Beck, Amir and Teboulle, Marc},
  journal={Operations Research Letters},
  volume={31},
  number={3},
  pages={167--175},
  year={2003},
  publisher={Elsevier}
}
@article{zhou2013grouping,
  title={On grouping effect of elastic net},
  author={Zhou, Ding-Xuan},
  journal={Statistics \& Probability Letters},
  volume={83},
  number={9},
  pages={2108--2112},
  year={2013},
  publisher={Elsevier}
}
@inproceedings{palaniswami2002adaptive,
  title={Adaptive support vector machines for regression},
  author={Palaniswami, M and Shilton, A},
  booktitle={International Conference on Neural Information Processing},
  volume={2},
  pages={1043--1049},
  year={2002}
}
@article{cai2009linearized,
  title={Linearized Bregman iterations for compressed sensing},
  author={Cai, Jian-Feng and Osher, Stanley and Shen, Zuowei},
  journal={Mathematics of Computation},
  volume={78},
  number={267},
  pages={1515--1536},
  year={2009}
}
@article{cai2009convergence,
  title={Convergence of the linearized Bregman iteration for $\ell_1$-norm minimization},
  author={Cai, Jian-Feng and Osher, Stanley and Shen, Zuowei},
  journal={Mathematics of Computation},
  volume={78},
  number={268},
  pages={2127--2136},
  year={2009}
}
@article{lai2013augmented,
  title={Augmented $\ell_1$ and nuclear-norm models with a globally linearly convergent algorithm},
  author={Lai, Ming-Jun and Yin, Wotao},
  journal={SIAM Journal on Imaging Sciences},
  volume={6},
  number={2},
  pages={1059--1091},
  year={2013},
  publisher={Society for Industrial and Applied Mathematics}
}
@book{nesterov2013introductory,
  title={Introductory Lectures on Convex Optimization: A Basic Course},
  author={Nesterov, Yurii},
  volume={87},
  year={2013},
  publisher={Springer Science \& Business Media}
}
@article{wang2014iteration,
  title={Iteration complexity of feasible descent methods for convex optimization.},
  author={Wang, Po-Wei and Lin, Chih-Jen},
  journal={Journal of Machine Learning Research},
  volume={15},
  number={1},
  pages={1523--1548},
  year={2014}
}
@article{kunapuli2012mirror,
  title={Mirror descent for metric learning: A unified approach},
  author={Kunapuli, Gautam and Shavlik, Jude},
  journal={Machine Learning and Knowledge Discovery in Databases},
  pages={859--874},
  year={2012},
  publisher={Springer}
}
@article{bach2015duality,
  title={Duality between subgradient and conditional gradient methods},
  author={Bach, Francis},
  journal={SIAM Journal on Optimization},
  volume={25},
  number={1},
  pages={115--129},
  year={2015},
  publisher={SIAM}
}
@article{rosasco2014convergence,
  title={Convergence of stochastic proximal gradient algorithm},
  author={Rosasco, Lorenzo and Villa, Silvia and V{\~u}, Bang C{\^o}ng},
  journal={arXiv preprint arXiv:1403.5074},
  year={2014}
}
@article{bubeck2014theory,
  title={Theory of convex optimization for machine learning},
  author={Bubeck, S{\'e}bastien},
  journal={arXiv preprint arXiv:1405.4980},
  year={2014}
}
@article{zhao2006model,
  title={On model selection consistency of Lasso},
  author={Zhao, Peng and Yu, Bin},
  journal={Journal of Machine Learning Research},
  volume={7},
  pages={2541--2563},
  year={2006},
  publisher={JMLR. org}
}
@article{xiao2010dual,
  title={Dual Averaging Methods for Regularized Stochastic Learning and Online Optimization},
  author={Xiao, Lin},
  journal={Journal of Machine Learning Research},
  volume={11},
  pages={2543--2596},
  year={2010}
}
@article{lions1979splitting,
  title={Splitting algorithms for the sum of two nonlinear operators},
  author={Lions, Pierre-Louis and Mercier, Bertrand},
  journal={SIAM Journal on Numerical Analysis},
  volume={16},
  number={6},
  pages={964--979},
  year={1979},
  publisher={SIAM}
}
@article{chen1997convergence,
  title={Convergence rates in forward--backward splitting},
  author={Chen, George HG and Rockafellar, RT},
  journal={SIAM Journal on Optimization},
  volume={7},
  number={2},
  pages={421--444},
  year={1997},
  publisher={SIAM}
}
@article{tseng2000modified,
  title={A modified forward-backward splitting method for maximal monotone mappings},
  author={Tseng, Paul},
  journal={SIAM Journal on Control and Optimization},
  volume={38},
  number={2},
  pages={431--446},
  year={2000},
  publisher={SIAM}
}
@inproceedings{duchi2009efficient,
  title={Efficient Online and Batch Learning Using Forward Backward Splitting},
  author={Duchi, John and Singer, Yoram},
  booktitle={Advances in Neural Information Processing Systems},
  pages={495--503},
  year={2009}
}
@article{lei2021generalizationb,
  title={Generalization Guarantee of SGD for Pairwise Learning},
  author={Lei, Yunwen and Liu, Mingrui and Ying, Yiming},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={21216--21228},
  year={2021}
}
@inproceedings{feldman2020private,
  title={Private stochastic convex optimization: optimal rates in linear time},
  author={Feldman, Vitaly and Koren, Tomer and Talwar, Kunal},
  booktitle={Proceedings of the 52nd Annual ACM SIGACT Symposium on Theory of Computing},
  pages={439--449},
  year={2020}
}
@inproceedings{karimi2016linear,
  title={Linear convergence of gradient and proximal-gradient methods under the polyak-{\l}ojasiewicz condition},
  author={Karimi, Hamed and Nutini, Julie and Schmidt, Mark},
  booktitle={European Conference on Machine Learning},
  pages={795--811},
  year={2016}
}
@InProceedings{feldman2019high,
  title={High probability generalization bounds for uniformly stable algorithms with nearly optimal rate},
  author={Feldman, Vitaly and Vondrak, Jan},
  booktitle = 	 {Conference on Learning Theory},
  pages = 	 {1270--1279},
  year = 	 {2019}
}


@inproceedings{fang2018spider,
  title={Spider: Near-optimal non-convex optimization via stochastic path-integrated differential estimator},
  author={Fang, Cong and Li, Chris Junchi and Lin, Zhouchen and Zhang, Tong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={689--699},
  year={2018}
}
@misc{nguyen2019finitesum,
    title={Finite-Sum Smooth Optimization with {SARAH}},
    author={Lam M. Nguyen and Marten van Dijk and Dzung T. Phan and Phuong Ha Nguyen and Tsui-Wei Weng and Jayant R. Kalagnanam},
    year={2019},
    eprint={1901.07648},
    archivePrefix={arXiv},
    primaryClass={math.OC}
}
@incollection{zhou2018stochastic,
title = {Stochastic Nested Variance Reduced Gradient Descent for Nonconvex Optimization},
author = {Zhou, Dongruo and Xu, Pan and Gu, Quanquan},
booktitle = {Advances in Neural Information Processing Systems},
pages = {3921--3932},
year = {2018}
}
%editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
@incollection{wang2019spiderboost,
title = {SpiderBoost and Momentum: Faster Variance Reduction Algorithms},
author = {Wang, Zhe and Ji, Kaiyi and Zhou, Yi and Liang, Yingbin and Tarokh, Vahid},
booktitle = {Advances in Neural Information Processing Systems},
pages = {2403--2413},
year = {2019}
}
%editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d'Alch\'{e}-Buc and E. Fox and R. Garnett},

%Joint European Conference on Machine Learning and Knowledge Discovery
@article{polyak1963gradient,
  title={Gradient methods for minimizing functionals},
  author={Polyak, Boris Teodorovich},
  journal={Zhurnal Vychislitel'noi Matematiki i Matematicheskoi Fiziki},
  volume={3},
  number={4},
  pages={643--653},
  year={1963},
  publisher={Russian Academy of Sciences, Branch of Mathematical Sciences}
}
@inproceedings{bach2013non,
  title={Non-strongly-convex smooth stochastic approximation with convergence rate {O}(1/n)},
  author={Bach, Francis and Moulines, Eric},
  booktitle={Advances in Neural Information Processing Systems},
  pages={773--781},
  year={2013}
}
@article{duchi2011adaptive,
  title={Adaptive subgradient methods for online learning and stochastic optimization},
  author={Duchi, John and Hazan, Elad and Singer, Yoram},
  journal={Journal of Machine Learning Research},
  volume={12},
  pages={2121--2159},
  year={2011},
  publisher={JMLR. org}
}
@article{duchi2010adaptive,
  title={Adaptive Subgradient Methods for Online Learning and Stochastic Optimization},
  author={Duchi, John and Hazan, Elad and Singer, Yoram},
  journal={Conference on Learning Theory},
  pages={257},
  year={2010}
}
@inproceedings{zhao2015stochastic,
  title={Stochastic Optimization with Importance Sampling for Regularized Loss Minimization},
  author={Zhao, Peilin and Zhang, Tong},
  booktitle={International Conference on Machine Learning},
  pages={1--9},
  year={2015}
}
@article{nedic2014stochastic,
  title={On stochastic subgradient mirror-descent algorithm with weighted averaging},
  author={Nedic, Angelia and Lee, Soomin},
  journal={SIAM Journal on Optimization},
  volume={24},
  number={1},
  pages={84--107},
  year={2014},
  publisher={SIAM}
}
@article{duchi2012ergodic,
  title={Ergodic mirror descent},
  author={Duchi, John C and Agarwal, Alekh and Johansson, Mikael and Jordan, Michael I},
  journal={SIAM Journal on Optimization},
  volume={22},
  number={4},
  pages={1549--1578},
  year={2012},
  publisher={SIAM}
}
@article{daubechies2004iterative,
  title={An iterative thresholding algorithm for linear inverse problems with a sparsity constraint},
  author={Daubechies, Ingrid and Defrise, Michel and De Mol, Christine},
  journal={Communications on pure and applied mathematics},
  volume={57},
  number={11},
  pages={1413--1457},
  year={2004},
  publisher={Wiley Online Library}
}
@inproceedings{langford2009sparse,
  title={Sparse online learning via truncated gradient},
  author={Langford, John and Li, Lihong and Zhang, Tong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={905--912},
  year={2009}
}
@inproceedings{shalev2009stochastic,
  title={Stochastic methods for $\ell_1$ regularized loss minimization},
  author={Shalev-Shwartz, Shai and Tewari, Ambuj},
  booktitle={International Conference on Machine Learning},
  pages={929--936},
  year={2009}
}
@article{hoeffding1963probability,
  title={Probability inequalities for sums of bounded random variables},
  author={Hoeffding, Wassily},
  journal={Journal of the American Statistical Association},
  volume={58},
  number={301},
  pages={13--30},
  year={1963},
  publisher={Taylor \& Francis Group}
}
@inproceedings{zhang2005data,
  title={Data Dependent Concentration Bounds for Sequential Prediction Algorithms.},
  author={Zhang, Tong},
  booktitle={Conference on Learning Theory},
  pages={173--187},
  year={2005}
}
@inproceedings{lin2016generalization,
  title={Generalization properties and implicit regularization for multiple passes {SGM}},
  author={Lin, Junhong and Camoriano, Raffaello and Rosasco, Lorenzo},
  booktitle={International Conference on Machine Learning},
  pages={2340--2348},
  year={2016}
}
@inproceedings{agarwal2009information,
  title={Information-theoretic lower bounds on the oracle complexity of convex optimization},
  author={Agarwal, Alekh and Wainwright, Martin J and Bartlett, Peter L and Ravikumar, Pradeep K},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1--9},
  year={2009}
}
@article{chen2004support,
  title={Support vector machine soft margin classifiers: error analysis},
  author={Chen, Di-Rong and Wu, Qiang and Ying, Yiming and Zhou, Ding-Xuan},
  journal={Journal of Machine Learning Research},
  volume={5},
  pages={1143--1175},
  year={2004},
  publisher={JMLR. org}
}
@book{steinwart2008support,
  title={Support Vector Machines},
  author={Steinwart, Ingo and Christmann, Andreas},
  year={2008},
  publisher={Springer Science \& Business Media}
}
@inproceedings{xiao2009dual,
  title={Dual averaging method for regularized stochastic learning and online optimization},
  author={Xiao, Lin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2116--2124},
  year={2009}
}



@article{kakade2012regularization,
  title={Regularization techniques for learning with matrices},
  author={Kakade, Sham M and Shalev-Shwartz, Shai and Tewari, Ambuj},
  journal={Journal of Machine Learning Research},
  volume={13},
  pages={1865--1890},
  year={2012},
  publisher={JMLR. org}
}

}@article{srebro2010optimistic,
  title={Optimistic Rates for Learning with a Smooth Loss},
  author={Srebro, Nathan and Sridharan, Karthik and Tewari, Ambuj},
  journal={arXiv preprint arXiv:1009.3896},
  year={2010}
}
@inproceedings{srebro2010smoothness,
  title={Smoothness, low noise and fast rates},
  author={Srebro, Nathan and Sridharan, Karthik and Tewari, Ambuj},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2199--2207},
  year={2010}
}
@article{cesa2004generalization,
  title={On the generalization ability of on-line learning algorithms},
  author={Cesa-Bianchi, Nicolo and Conconi, Alex and Gentile, Claudio},
  journal={IEEE Transactions on Information Theory},
  volume={50},
  number={9},
  pages={2050--2057},
  year={2004},
  publisher={IEEE}
}
@inproceedings{kakade2009generalization,
  title={On the generalization ability of online strongly convex programming algorithms},
  author={Kakade, Sham M and Tewari, Ambuj},
  booktitle={Advances in Neural Information Processing Systems},
  pages={801--808},
  year={2009}
}
@article{freedman1975tail,
  title={On tail probabilities for martingales},
  author={Freedman, David A},
  journal={the Annals of Probability},
  pages={100--118},
  year={1975},
  publisher={JSTOR}
}
@inproceedings{sridharan2009fast,
  title={Fast rates for regularized objectives},
  author={Sridharan, Karthik and Shalev-Shwartz, Shai and Srebro, Nathan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1545--1552},
  year={2009}
}
@book{de1999decoupling,
  title={Decoupling: From Dependence to Independence},
  author={De la Pe{\~n}a, V. and Gin{\'e}, E.},
  year={1999},
  address={New York},
  publisher={Springer-Verlag}
}
@incollection{mcdiarmid1989method,
  title={On the method of bounded differences},
  author={McDiarmid, Colin},
  booktitle={Surveys in combinatorics},
  pages={148--188},
  address={Cambridge},
  publisher={Cambridge Univ. Press},
  editor={Siemous, J. },
  year={1989}
}
@book{scholkopf2001learning,
  title={Learning with Kernels: Support vector machines, Regularization, Optimization, and Beyond},
  author={Sch\"olkopf, Bernhard and Smola, Alexander J},
  year={2001},
  publisher={MIT press}
}
@article{aronszajn1950theory,
  title={Theory of reproducing kernels},
  author={Aronszajn, Nachman},
  journal={Transactions of the American Mathematical Society},
  volume={68},
  number={3},
  pages={337--404},
  year={1950},
  publisher={JSTOR}
}
@article{kakade2009duality,
  title={On the duality of strong convexity and strong smoothness: Learning applications and matrix regularization},
  author={Kakade, Sham M and Shalev-Shwartz, Shai and Tewari, Ambuj},
  journal={Unpublished Manuscript, available at http://ttic. uchicago. edu/shai/papers/KakadeShalevTewari09. pdf},
  year={2009}
}
@book{boyd2004convex,
  title={Convex Optimization},
  author={Boyd, Stephen Poythress and Vandenberghe, Lieven},
  year={2004},
  publisher={Cambridge Univ. Press},
  address={New York},
}
@inproceedings{kakade2008complexity,
  title={On the complexity of linear prediction: Risk bounds, margin bounds, and regularization},
  author={Kakade, Sham M and Sridharan, Karthik and Tewari, Ambuj},
  booktitle={Advances in the 21st Neural Information Processing Systems},
  series = {NIPS '08},
  pages={793--800},
  editor={Daphne Koller and Dale Schuurmans and Yoshua Bengio and L{\'e}on Bottou},
  address={Vancouver, British Columbia, Canada},
  year={2008}
}
@book{ledoux1991probability,
  title={Probability in Banach Spaces: Isoperimetry and Processes},
  author={Ledoux, Michel and Talagrand, Michel},
  volume={23},
  year={1991},
  address={Berlin},
  publisher={Springer}
}

@article{guo2014guaranteed,
  title={Guaranteed Classification via Regularized Similarity Learning},
  author={Guo, Zheng-Chu and Ying, Yiming},
  year={2014},
  volume={26},
  number={3},
  pages={497--522},
  journal={Neural Computation},
  publisher={MIT Press}
}

@article{guo2016online,
  title={Online regularized learning with pairwise loss functions},
  author={Guo, Zheng-Chu and Ying, Yiming and Zhou, Ding-Xuan},
  journal={Advances in Computational Mathematics},
  pages={1--24},
  year={2016},
  publisher={Springer}
}
@article{guo2016sparsity,
  title={Sparsity and error analysis of empirical feature-based regularization schemes},
  author={Guo, Xin and Fan, Jun and Zhou, Ding-Xuan},
  journal={Journal of Machine Learning Research},
  volume={17},
  number={89},
  pages={1--34},
  year={2016}
}
@article{huber1964,
  title={Robust estimation of a location parameter},
  author={Huber, Peter J},
  journal={The Annals of Mathematical Statistics},
  volume={35},
  number={1},
  pages={73--101},
  year={1964},
  publisher={Institute of Mathematical Statistics}
}

@article{SmaleYao,
  title={Online learning algorithms},
  author={Smale, Steve and Yao, Yuan},
  journal={Foundations of computational mathematics},
  volume={6},
  number={2},
  pages={145--170},
  year={2006},
  publisher={Springer}
}
@inproceedings{shamir2012open,
  title={Open Problem: Is Averaging Needed for Strongly Convex Stochastic Gradient Descent?},
  author={Shamir, Ohad},
  year={2012},
  journal={Open problem presented at Conference on Learning Theory}
}
@inproceedings{zinkevich2003online,
  title={Online Convex Programming and Generalized Infinitesimal Gradient Ascent},
  author={Zinkevich, Martin},
  booktitle={International Conference on Machine Learning},
  pages={928--936},
  year={2003}
}
@article{kivinen1997exponentiated,
  title={Exponentiated gradient versus gradient descent for linear predictors},
  author={Kivinen, Jyrki and Warmuth, Manfred K},
  journal={Information and Computation},
  volume={132},
  number={1},
  pages={1--63},
  year={1997},
  publisher={Elsevier}
}
@article{hazan2014beyond,
  title={Beyond the regret minimization barrier: optimal algorithms for stochastic strongly-convex optimization},
  author={Hazan, Elad and Kale, Satyen},
  journal={Journal of Machine Learning Research},
  volume={15},
  number={1},
  pages={2489--2512},
  year={2014}
}
@article{lacoste2012simpler,
  title={A simpler approach to obtaining an {O}(1/t) convergence rate for the projected stochastic subgradient method},
  author={Lacoste-Julien, Simon and Schmidt, Mark and Bach, Francis},
  journal={arXiv preprint arXiv:1212.2002},
  year={2012}
}


@article{shalev2012online,
  title={Online learning and online convex optimization},
  author={Shalev-Shwartz, Shai},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={4},
  number={2},
  pages={107--194},
  year={2012},
  publisher={Now Publishers, Inc.}
}
@inproceedings{boissier2016fast,
  title={Fast Convergence of Online Pairwise Learning Algorithms},
  author={Boissier, Martin and Lyu, Siwei and Ying, Yiming and Zhou, Ding-Xuan},
  booktitle={Proceedings of the 19th International Conference on Artificial Intelligence and Statistics},
  pages={204--212},
  year={2016}
}

@article{hazan2007logarithmic,
  title={Logarithmic regret algorithms for online convex optimization},
  author={Hazan, Elad and Agarwal, Amit and Kale, Satyen},
  journal={Machine Learning},
  volume={69},
  number={2},
  pages={169--192},
  year={2007},
  publisher={Springer}
}
@article{beck2009fast,
  title={A fast iterative shrinkage-thresholding algorithm for linear inverse problems},
  author={Beck, Amir and Teboulle, Marc},
  journal={SIAM journal on imaging sciences},
  volume={2},
  number={1},
  pages={183--202},
  year={2009},
  publisher={SIAM}
}
@inproceedings{zhang2004solving,
  title={Solving large scale linear prediction problems using stochastic gradient descent algorithms},
  author={Zhang, Tong},
  booktitle={International Conference on Machine Learning},
  pages={919--926},
  year={2004}
}
@article{steinwart2007fast,
  title={Fast rates for support vector machines using Gaussian kernels},
  author={Steinwart, Ingo and Scovel, Clint},
  journal={Annals of Statistics},
  volume={35},
  number={2},
  pages={575--607},
  year={2007},
  publisher={Institute of Mathematical Statistics}
}
@article{DBLP:journals/corr/HuangZ13,
  author    = {Sheng{-}Jun Huang and
               Zhi{-}Hua Zhou},
  title     = {Fast Multi-Instance Multi-Label Learning},
  journal   = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
  Doi = {10.1109/TPAMI.2018.2860995},
  year={to appear}
}
@inproceedings{lei2018generalization,
  title={Generalization Bounds for Regularized Pairwise Learning.},
  author={Lei, Yunwen and Lin, Shao-Bo and Tang, Ke},
  booktitle={International Joint Conference on Artificial Intelligence},
  pages={2376--2382},
  year={2018}
}
@inproceedings{feldman2016generalization,
  title={Generalization of erm in stochastic convex optimization: The dimension strikes back},
  author={Feldman, Vitaly},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3576--3584},
  year={2016}
}

@article{wright2015coordinate,
  title={Coordinate descent algorithms},
  author={Wright, Stephen J},
  journal={Mathematical Programming},
  volume={151},
  number={1},
  pages={3--34},
  year={2015},
  publisher={Springer}
}
@article{koren2015fast,
  title={Fast rates for exp-concave empirical risk minimization},
  author={Koren, Tomer and Levy, Kfir},
  journal={Advances in Neural Information Processing Systems},
  volume={28},
  year={2015}
}
@article{xiao2019dscovr,
  title={DSCOVR: Randomized Primal-Dual Block Coordinate Algorithms for Asynchronous Distributed Optimization.},
  author={Xiao, Lin and Yu, Adams Wei and Lin, Qihang and Chen, Weizhu},
  journal={Journal of Machine Learning Research},
  volume={20},
  pages={43--1},
  year={2019}
}
@article{li2020complexity,
  title={On the Complexity Analysis of the Primal Solutions for the Accelerated Randomized Dual Coordinate Ascent.},
  author={Li, Huan and Lin, Zhouchen},
  journal={Journal of Machine Learning Research},
  volume={21},
  number={33},
  pages={1--45},
  year={2020}
}
@inproceedings{gu2018accelerated,
  title={Accelerated Asynchronous Greedy Coordinate Descent Algorithm for SVMs.},
  author={Gu, Bin and Shan, Yingying and Geng, Xiang and Zheng, Guansheng},
  booktitle={IJCAI},
  pages={2170--2176},
  year={2018}
}
@article{yang2013trading,
  title={Trading computation for communication: Distributed stochastic dual coordinate ascent},
  author={Yang, Tianbao},
  journal={Advances in Neural Information Processing Systems},
  volume={26},
  pages={629--637},
  year={2013}
}
@inproceedings{ren2017distributed,
  title={Distributed Accelerated Proximal Coordinate Gradient Methods.},
  author={Ren, Yong and Zhu, Jun},
  booktitle={IJCAI},
  pages={2655--2661},
  year={2017}
}
@inproceedings{qu2016sdna,
  title={SDNA: Stochastic dual Newton ascent for empirical risk minimization},
  author={Qu, Zheng and Richt{\'a}rik, Peter and Tak{\'a}c, Martin and Fercoq, Olivier},
  booktitle={International Conference on Machine Learning},
  pages={1823--1832},
  year={2016}
}
@article{richtarik2016distributed,
  title={Distributed coordinate descent method for learning with big data},
  author={Richt{\'a}rik, Peter and Tak{\'a}{\v{c}}, Martin},
  journal={The Journal of Machine Learning Research},
  volume={17},
  number={1},
  pages={2657--2681},
  year={2016},
  publisher={JMLR. org}
}
@article{lu2015complexity,
  title={On the complexity analysis of randomized block-coordinate descent methods},
  author={Lu, Zhaosong and Xiao, Lin},
  journal={Mathematical Programming},
  volume={152},
  number={1-2},
  pages={615--642},
  year={2015},
  publisher={Springer}
}
@article{hu2014scalable,
  title={Scalable nonparametric low-rank kernel learning using block coordinate descent},
  author={Hu, En-Liang and Kwok, James T},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  volume={26},
  number={9},
  pages={1927--1938},
  year={2014},
  publisher={IEEE}
}
@article{hu2019low,
  title={Low-Rank Matrix Learning Using Biconvex Surrogate Minimization},
  author={Hu, En-Liang and Kwok, James T},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  volume={30},
  number={11},
  pages={3517--3527},
  year={2019},
  publisher={IEEE}
}
@article{bian2019parallel,
  title={Parallel Coordinate Descent Newton Method for Efficient $\ell_1$-Regularized Loss Minimization},
  author={Bian, Yatao An and Li, Xiong and Liu, Yuncai and Yang, Ming-Hsuan},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  volume={30},
  number={11},
  pages={3233--3245},
  year={2019},
  publisher={IEEE}
}
@article{chang2008coordinate,
  title={Coordinate descent method for large-scale l2-loss linear support vector machines},
  author={Chang, Kai-Wei and Hsieh, Cho-Jui and Lin, Chih-Jen},
  journal={Journal of Machine Learning Research},
  volume={9},
  number={Jul},
  pages={1369--1398},
  year={2008}
}
@article{richtarik2014iteration,
  title={Iteration complexity of randomized block-coordinate descent methods for minimizing a composite function},
  author={Richt{\'a}rik, Peter and Tak{\'a}{\v{c}}, Martin},
  journal={Mathematical Programming},
  volume={144},
  number={1-2},
  pages={1--38},
  year={2014},
  publisher={Springer}
}
@article{fangfast,
  title={FAST CONVERGENCE OF STOCHASTIC SUBGRADIENT METHOD UNDER INTERPOLATION},
  author={Fang, Huang and Fan, Zhenan and Friedlander, Michael P}
}
@article{rockafellar1976monotone,
  title={Monotone operators and the proximal point algorithm},
  author={Rockafellar, R Tyrrell},
  journal={SIAM journal on control and optimization},
  volume={14},
  number={5},
  pages={877--898},
  year={1976},
  publisher={SIAM}
}
@book{vershynin2018high,
  title={High-dimensional probability: An introduction with applications in data science},
  author={Vershynin, Roman},
  volume={47},
  year={2018},
  publisher={Cambridge university press}
}
@inproceedings{DBLP:conf/colt/ZhangZ19,
  author    = {Lijun Zhang and
               Zhi-Hua Zhou},
  title     = {Stochastic Approximation of Smooth and Strongly Convex Functions:
               Beyond the {\textdollar}O(1/T){\textdollar} Convergence Rate},
  booktitle = {Conference on Learning Theory},
  pages     = {3160--3179},
  year      = {2019}
}
@ARTICLE{8792163,
  author={Zhou, Pan and Yuan, Xiao-Tong and Yan, Shuicheng and Feng, Jiashi},
  journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
  title={Faster First-Order Methods for Stochastic Non-Convex Optimization on Riemannian Manifolds},
  year={2021},
  volume={43},
  number={2},
  pages={459-472}}
@article{fang2018near,
  title={Near-optimal non-convex optimization via stochastic path integrated differential estimator},
  author={Fang, C and Li, CJ and Lin, Z and Zhang, T},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  pages={689},
  year={2018}
}
@article{zhang2019mcdiarmid,
  title={McDiarmid-Type Inequalities for Graph-Dependent Variables and Stability Bounds},
  author={Zhang, Rui and Liu, Xingwu and Wang, Yuyi and Wang, Liwei},
  journal={Advances in Neural Information Processing Systems 32},
  pages={10889--10899},
  year={2019},
  publisher={Curran}
}
@article{dekel2012optimal,
  title={Optimal Distributed Online Prediction Using Mini-Batches},
  author={Dekel, Ofer and Gilad-Bachrach, Ran and Shamir, Ohad and Xiao, Lin},
  journal={Journal of Machine Learning Research},
  volume={13},
  number={1},
  year={2012}
}
@article{woodworth2020minibatch,
  title={Minibatch vs local sgd for heterogeneous distributed learning},
  author={Woodworth, Blake and Patel, Kumar Kshitij and Srebro, Nathan},
  journal={arXiv preprint arXiv:2006.04735},
  year={2020}
}
@inproceedings{khaled2020tighter,
  title={Tighter theory for local SGD on identical and heterogeneous data},
  author={Khaled, Ahmed and Mishchenko, Konstantin and Richt{\'a}rik, Peter},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={4519--4529},
  year={2020},
  organization={PMLR}
}

@inproceedings{mcmahan2017communication,
  title={Communication-efficient learning of deep networks from decentralized data},
  author={McMahan, Brendan and Moore, Eider and Ramage, Daniel and Hampson, Seth and y Arcas, Blaise Aguera},
  booktitle={Artificial intelligence and statistics},
  pages={1273--1282},
  year={2017}
}

@inproceedings{lin2019don,
  title={Don't Use Large Mini-batches, Use Local SGD},
  author={Lin, Tao and Stich, Sebastian U and Patel, Kumar Kshitij and Jaggi, Martin},
  booktitle={International Conference on Learning Representations},
  year={2019}
}
@inproceedings{shamir2014distributed,
  title={Distributed stochastic optimization and learning},
  author={Shamir, Ohad and Srebro, Nathan},
  booktitle={Annual Allerton Conference on Communication, Control, and Computing },
  pages={850--857},
  year={2014},
  organization={IEEE}
}
@inproceedings{stich2018local,
  title={Local SGD Converges Fast and Communicates Little},
  author={Stich, Sebastian U},
  booktitle={International Conference on Learning Representations},
  year={2018}
}
@article{richards2021stability,
  title={Stability \& Generalisation of Gradient Descent for Shallow Neural Networks without the Neural Tangent Kernel},
  author={Richards, Dominic and Kuzborskij, Ilja},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}
@inproceedings{zhang2021generalization,
  title={Generalization bounds for stochastic saddle point problems},
  author={Zhang, Junyu and Hong, Mingyi and Wang, Mengdi and Zhang, Shuzhong},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={568--576},
  year={2021},
  organization={PMLR}
}
@inproceedings{woodworth2020local,
  title={Is local SGD better than minibatch SGD?},
  author={Woodworth, Blake and Patel, Kumar Kshitij and Stich, Sebastian and Dai, Zhen and Bullins, Brian and Mcmahan, Brendan and Shamir, Ohad and Srebro, Nathan},
  booktitle={International Conference on Machine Learning},
  pages={10334--10343},
  year={2020},
  organization={PMLR}
}
@article{woodworth2021even,
  title={An Even More Optimal Stochastic Optimization Algorithm: Minibatching and Interpolation Learning},
  author={Woodworth, Blake and Srebro, Nathan},
  journal={arXiv preprint arXiv:2106.02720},
  year={2021}
}
@inproceedings{mou2018dropout,
  title={Dropout training, data-dependent regularization, and generalization bounds},
  author={Mou, Wenlong and Zhou, Yuchen and Gao, Jun and Wang, Liwei},
  booktitle={International Conference on Machine Learning},
  pages={3645--3653},
  year={2018},
  organization={PMLR}
}
@article{zhou2018convergenceb,
  title={On the convergence of adaptive gradient methods for nonconvex optimization},
  author={Zhou, Dongruo and Chen, Jinghui and Cao, Yuan and Tang, Yiqi and Yang, Ziyan and Gu, Quanquan},
  journal={arXiv preprint arXiv:1808.05671},
  year={2018}
}
@inproceedings{DBLP:conf/colt/SteinwartHS09,
  author    = {Ingo Steinwart and
               Don R. Hush and
               Clint Scovel},
  title     = {Optimal Rates for Regularized Least Squares Regression},
  booktitle = {Conference on Learning Theory},
  year      = {2009},
  pages = {79--93}
}
@inproceedings{steinwart2009fast,
  title={Fast learning from non-iid observations},
  author={Steinwart, Ingo and Christmann, Andreas},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1768--1776},
  year={2009}
}
@incollection{mendelson2004geometric,
  title={Geometric parameters in learning theory},
  author={Mendelson, Shahar},
  booktitle={Geometric aspects of functional analysis},
  pages={193--235},
  year={2004},
  publisher={Springer}
}
@inproceedings{orabona2014simultaneous,
  title={Simultaneous model selection and optimization through parameter-free stochastic learning},
  author={Orabona, Francesco},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1116--1124},
  year={2014}
}
@inproceedings{carratino2018learning,
  title={Learning with sgd and random features},
  author={Carratino, Luigi and Rudi, Alessandro and Rosasco, Lorenzo},
  booktitle={Advances in Neural Information Processing Systems},
  pages={10213--10224},
  year={2018}
}
@inproceedings{zinkevich2010parallelized,
  title={Parallelized stochastic gradient descent},
  author={Zinkevich, Martin A and Weimer, Markus and Smola, Alex and Li, Lihong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2595--2603},
  year={2010}
}
@inproceedings{gower2021sgd,
  title={Sgd for structured nonconvex functions: Learning rates, minibatching and interpolation},
  author={Gower, Robert and Sebbouh, Othmane and Loizou, Nicolas},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1315--1323},
  year={2021},
  organization={PMLR}
}
@inproceedings{li2014efficient,
  title={Efficient mini-batch training for stochastic optimization},
  author={Li, Mu and Zhang, Tong and Chen, Yuqiang and Smola, Alexander J},
  booktitle={Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining},
  pages={661--670},
  year={2014}
}
@inproceedings{lin2018optimal,
  title={Optimal distributed learning with multi-pass stochastic gradient methods},
  author={Lin, Junhong and Cevher, Volkan},
  booktitle={International Conference on Machine Learning},
  pages={3098--3107},
  year={2018}
}
@article{wu2007multi,
  title={Multi-kernel regularized classifiers},
  author={Wu, Qiang and Ying, Yiming and Zhou, Ding-Xuan},
  journal={Journal of Complexity},
  volume={23},
  number={1},
  pages={108--134},
  year={2007},
  publisher={Elsevier}
}
@article{tsybakov2004optimal,
  title={Optimal aggregation of classifiers in statistical learning},
  author={Tsybakov, Alexander},
  journal={Annals of Statistics},
  volume={32},
  number={1},
  pages={135--166},
  year={2004},
  publisher={Institute of Mathematical Statistics}
}
@article{bartlett2006convexity,
  title={Convexity, classification, and risk bounds},
  author={Bartlett, Peter and Jordan, Michael and McAuliffe, Jon},
  journal={Journal of the American Statistical Association},
  volume={101},
  number={473},
  pages={138--156},
  year={2006},
  publisher={Taylor \& Francis}
}
@book{doob1994graduate,
  title={Measure Theory},
  author={Doob, Joseph L},
  year={1994},
  publisher={Springer}
}
@book{pisier1999volume,
  title={The Volume of Convex Bodies and Banach Space Geometry},
  author={Pisier, Gilles},
  volume={94},
  year={1999},
  publisher={Cambridge University Press}
}
@article{wang2019distributed,
  title={Distributed pairwise algorithms with gradient descent methods},
  author={Wang, Baobin and Hu, Ting},
  journal={Neurocomputing},
  volume={333},
  pages={364--373},
  year={2019},
  publisher={Elsevier}
}

@InProceedings{pmlr-v70-allen-zhu17a,
  title = 	 {Natasha: Faster Non-Convex Stochastic Optimization via Strongly Non-Convex Parameter},
  author = 	 {Zeyuan Allen-Zhu},
  booktitle = 	 {International Conference on Machine Learning},
  pages = 	 {89--97},
  year = 	 {2017},
  volume = 	 {70}
}

@article{katyusha18,
  author  = {Zeyuan Allen-Zhu},
  title   = {Katyusha: The First Direct Acceleration of Stochastic Gradient Methods},
  journal = {Journal of Machine Learning Research},
  year    = {2018},
  volume  = {18},
  number  = {221},
  pages   = {1-51}
}
@article{loh15a,
  author  = {Po-Ling Loh and Martin J. Wainwright},
  title   = {Regularized M-estimators with Nonconvexity: Statistical and Algorithmic Theory for Local Optima},
  journal = {Journal of Machine Learning Research},
  year    = {2015},
  volume  = {16},
  pages   = {559-616}
}

@article{lei2018convergenceb,
  title={Convergence of online mirror descent},
  author={Lei, Yunwen and Zhou, Ding-Xuan},
  journal={Applied and Computational Harmonic Analysis},
  year={2020},
  volume={48},
  number={1},
  pages={343--373},
  publisher={Elsevier}
}
@article{guo2017learning,
  title={Learning theory of distributed spectral algorithms},
  author={Guo, Zheng-Chu and Lin, Shao-Bo and Zhou, Ding-Xuan},
  journal={Inverse Problems},
  volume={33},
  number={7},
  pages={074009},
  year={2017},
  publisher={IOP Publishing}
}
@incollection{mendelson2003few,
  title={A few notes on statistical learning theory},
  author={Mendelson, Shahar},
  booktitle={Advanced lectures on machine learning},
  pages={1--40},
  year={2003},
  publisher={Springer}
}
@article{blanchard2008statistical,
  title={Statistical performance of support vector machines},
  author={Blanchard, Gilles and Bousquet, Olivier and Massart, Pascal},
  journal={The Annals of Statistics},
  volume={36},
  number={2},
  pages={489--531},
  year={2008},
  publisher={Institute of Mathematical Statistics}
}
@article{blanchard2003rate,
  title={On the rate of convergence of regularized boosting classifiers},
  author={Blanchard, Gilles and Lugosi, G{\'a}bor and Vayatis, Nicolas},
  journal={Journal of Machine Learning Research},
  volume={4},
  number={Oct},
  pages={861--894},
  year={2003}
}
@book{DBLP:books/daglib/0097035,
  author    = {Vladimir Vapnik},
  title     = {Statistical Learning Theory},
  publisher = {Wiley},
  year      = {1998}
}
@article{yan2018fast,
  title={A fast Uyghur text detector for complex background images},
  author={Yan, Chenggang and Xie, Hongtao and Chen, Jianjun and Zha, Zhengjun and Hao, Xinhong and Zhang, Yongdong and Dai, Qionghai},
  journal={IEEE Transactions on Multimedia},
  volume={20},
  number={12},
  pages={3389--3398},
  year={2018},
  publisher={IEEE}
}
@book{cristianini2000introduction,
  title={An Introduction to Support Vector Machines and other Kernel-based Learning Methods},
  author={Cristianini, Nello and Shawe-Taylor, John},
  year={2000},
  publisher={Cambridge university press}
}
@article{lin2018distributed,
  title={Distributed kernel-based gradient descent algorithms},
  author={Lin, Shao-Bo and Zhou, Ding-Xuan},
  journal={Constructive Approximation},
  volume={47},
  number={2},
  pages={249--276},
  year={2018},
  publisher={Springer}
}
@article{lei2015rbf,
  title={Generalization performance of radial basis function networks},
  author={Lei, Yunwen and Ding, Lixin and Zhang, Wensheng},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  volume={26},
  number={3},
  pages={551--564},
  year={2015},
  publisher={IEEE}
}
@article{lei2017SVT,
  title={Analysis of Singular Value Thresholding Algorithm for Matrix Completion},
  author={Lei, Yunwen and Zhou, Ding-Xuan},
  journal={under review},
  year={2016}
}
@article{lei2017analysis,
  title={Analysis of online composite mirror descent algorithm},
  author={Lei, Yunwen and Zhou, Ding-Xuan},
  journal={Neural Computation},
  volume={29},
  number={3},
  pages={825--860},
  year={2017},
  publisher={MITP}
}
@article{lei2016generalization,
  title={Generalization analysis of multi-modal metric learning},
  author={Lei, Yunwen and Ying, Yiming},
  journal={Analysis and Applications},
  volume={14},
  number={04},
  pages={503--521},
  year={2016},
  publisher={World Scientific}
}
@inproceedings{lei2015multi,
  title={Multi-class SVMs: From Tighter Data-Dependent Generalization Bounds to Novel Algorithms},
  author={Lei, Yunwen and Dogan, Urun and Binder, Alexander and Kloft, Marius},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2026--2034},
  year={2015}
}
@inproceedings{massart2000some,
  title={Some applications of concentration inequalities to statistics},
  author={Massart, Pascal},
  booktitle={Annales de la Facult{\'e} des sciences de Toulouse: Math{\'e}matiques},
  volume={9},
  number={2},
  pages={245--303},
  year={2000}
}
@inproceedings{suzuki2013dual,
  title={Dual averaging and proximal gradient descent for online alternating direction multiplier method},
  author={Suzuki, Taiji},
  booktitle={International Conference on Machine Learning},
  pages={392--400},
  year={2013}
}
@inproceedings{chen2012optimal,
  title={Optimal regularized dual averaging methods for stochastic optimization},
  author={Chen, Xi and Lin, Qihang and Pena, Javier},
  booktitle={Advances in Neural Information Processing Systems},
  pages={395--403},
  year={2012}
}
@article{davis2018uniform,
  title={Uniform graphical convergence of subgradients in nonconvex optimization and learning},
  author={Davis, Damek and Drusvyatskiy, Dmitriy},
  journal={arXiv preprint arXiv:1810.07590},
  year={2018}
}
@inproceedings{feldman2018generalization,
  title={Generalization bounds for uniformly stable algorithms},
  author={Feldman, Vitaly and Vondrak, Jan},
  booktitle={Advances in Neural Information Processing Systems},
  pages={9747--9757},
  year={2018}
}
@article{bassily2020stability,
  title={Stability of Stochastic Gradient Descent on Nonsmooth Convex Losses},
  author={Bassily, Raef and Feldman, Vitaly and Guzm{\'a}n, Crist{\'o}bal and Talwar, Kunal},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}
@inproceedings{bousquet2019sharper,
  title={Sharper bounds for uniformly stable algorithms},
  author={Bousquet, Olivier and Klochkov, Yegor and Zhivotovskiy, Nikita},
  booktitle={Conference on Learning Theory},
  pages={610--626},
  year={2020}
}
@article{dudley1967sizes,
  title={The sizes of compact subsets of Hilbert space and continuity of Gaussian processes},
  author={Dudley, Richard M},
  journal={Journal of Functional Analysis},
  volume={1},
  number={3},
  pages={290--330},
  year={1967},
  publisher={Elsevier}
}
@article{lee2004multicategory,
  title={Multicategory support vector machines: Theory and application to the classification of microarray data and satellite radiance data},
  author={Lee, Yoonkyung and Lin, Yi and Wahba, Grace},
  journal={Journal of the American Statistical Association},
  volume={99},
  number={465},
  pages={67--81},
  year={2004},
  publisher={Taylor \& Francis}
}
@techreport{weston1998multi,
  title={Multi-class support vector machines},
  author={Weston, Jason and Watkins, Chris},
  year={1998},
  institution={Citeseer}
}
@article{zhang2017survey,
  title={A survey on multi-task learning},
  author={Zhang, Yu and Yang, Qiang},
  journal={arXiv preprint arXiv:1707.08114},
  year={2017}
}
@inproceedings{xu2020partial,
  title={Partial Multi-Label Learning with Label Distribution.},
  author={Xu, Ning and Liu, Yun-Peng and Geng, Xin},
  booktitle={AAAI},
  pages={6510--6517},
  year={2020}
}
@ARTICLE{8263148,
  author={W. {Liu} and D. {Xu} and I. W. {Tsang} and W. {Zhang}},
  journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
  title={Metric Learning for Multi-Output Tasks},
  year={2019},
  volume={41},
  number={2},
  pages={408-422}
}
@article{lei2019data,
  title={Data-dependent Generalization Bounds for Multi-class Classification},
  author={Lei, Yunwen and Dogan, {\"U}r{\"u}n and Zhou, Ding-Xuan and Kloft, Marius},
  journal={IEEE Transactions on Information Theory},
  volume={65},
  number={5},
  pages={2995--3021},
  year={2019},
  publisher={IEEE}
}
@article{lei2017generalization,
  title={Generalization Error Bounds for Extreme Multi-class Classification},
  author={Lei, Yunwen and Dogan, Urun and Zhou, Ding-Xuan and Kloft, Marius},
  journal={arXiv preprint arXiv:1706.09814},
  year={2017}
}
@inproceedings{lei2016localized,
  title={Localized Multiple Kernel Learning—A Convex Approach},
  author={Lei, Yunwen and Binder, Alexander and Dogan, Urun and Kloft, Marius},
  booktitle={Asian Conference on Machine Learning},
  pages={81--96},
  year={2016}
}
@inproceedings{lei2018stochastic,
  title={Stochastic Composite Mirror Descent: Optimal Bounds with High Probabilities},
  author={Lei, Yunwen and Tang, Ke},
  booktitle={Advance in Neural Information Processing Systems},
  pages={1524--1534},
  year={2018}
}
@inproceedings{lei2020fine,
  title={Fine-Grained Analysis of Stability and Generalization for Stochastic Gradient Descent},
  author={Lei, Yunwen and Ying, Yiming},
  booktitle={International Conference on Machine Learning},
  pages={5809--5819},
  year={2020}
}
@inproceedings{lei2020sharper,
  title={Sharper Generalization Bounds for Pairwise Learning},
  author={Lei, Yunwen and Ledent, Antoine and Kloft, Marius},
  booktitle={Advances in Neural Information Processing Systems},
  volume={33},
  pages={21236--21246},
  year={2020}
}
@article{ward2020adagrad,
  title={AdaGrad stepsizes: Sharp convergence over nonconvex landscapes},
  author={Ward, Rachel and Wu, Xiaoxia and Bottou, L{\'e}on},
  journal={Journal of Machine Learning Research},
  volume={21},
  pages={1--30},
  year={2020}
}
@article{lin2017pairwise,
   author = {Lin, Junhong and Lei, Yunwen and Zhang, Bo and Zhou, Ding-Xuan},
   title = {Online Pairwise Learning Algorithms with Convex Loss Functions},
   journal = {Information Sciences},
   year = {2017},
   pages = {57--70},
   volume={406--407},
   type = {Journal Article}
}
@article{lei2014refined,
  title={Refined Rademacher Chaos Complexity Bounds with Applications to the Multikernel Learning Problem},
  author={Lei, Yunwen and Ding, Lixin},
  journal={Neural Computation},
  volume={26},
  number={4},
  pages={1--22},
  year={2014},
  publisher={MIT Press}
}

@article{mei2018landscape,
  title={The landscape of empirical risk for nonconvex losses},
  author={Mei, Song and Bai, Yu and Montanari, Andrea},
  journal={The Annals of Statistics},
  volume={46},
  number={6A},
  pages={2747--2774},
  year={2018},
  publisher={Institute of Mathematical Statistics}
}
@inproceedings{gao2013uniform,
  title={Uniform convergence, stability and learnability for ranking problems},
  author={Gao, Wei and Zhou, Zhi-Hua},
  booktitle={International Joint Conference on Artificial Intelligence},
  pages={1337--1343},
  year={2013},
  organization={AAAI Press}
}
@inproceedings{cortes2004auc,
  title={AUC optimization vs. error rate minimization},
  author={Cortes, Corinna and Mohri, Mehryar},
  booktitle={Advances in Neural Information Processing Systems},
  pages={313--320},
  year={2004}
}
@inproceedings{zhang2017empirical,
  title={Empirical Risk Minimization for Stochastic Convex Optimization: {O}$  (1/n) $-and {O}$ (1/n^2) $-type of Risk Bounds},
  author={Zhang, Lijun and Yang, Tianbao and Jin, Rong},
  booktitle={Conference on Learning Theory},
  pages={1954--1979},
  year={2017}
}
@article{ying2010rademacher,
  title={Rademacher chaos complexities for learning the kernel problem},
  author={Ying, Yiming and Campbell, Colin},
  journal={Neural computation},
  volume={22},
  number={11},
  pages={2858--2886},
  year={2010},
  publisher={MIT Press}
}
@article{grimmer2019convergence,
  title={Convergence rates for deterministic and stochastic subgradient methods without Lipschitz continuity},
  author={Grimmer, Benjamin},
  journal={SIAM Journal on Optimization},
  volume={29},
  number={2},
  pages={1350--1365},
  year={2019},
  publisher={SIAM}
}
@article{li2016distributed,
  title={Distributed mirror descent method for multi-agent optimization with delay},
  author={Li, Jueyou and Chen, Guo and Dong, Zhaoyang and Wu, Zhiyou},
  journal={Neurocomputing},
  volume={177},
  pages={643--650},
  year={2016},
  publisher={Elsevier}
}
@article{LIN2018516,
title = "Stochastic Primal-Dual Proximal ExtraGradient descent for compositely regularized optimization",
journal = "Neurocomputing",
volume = "273",
pages = "516 - 525",
year = "2018",
author = "Tianyi Lin and Linbo Qiao and Teng Zhang and Jiashi Feng and Bofeng Zhang"
}
@inproceedings{liu2018fastb,
  title={Fast rates of erm and stochastic approximation: Adaptive to error bound conditions},
  author={Liu, Mingrui and Zhang, Xiaoxuan and Zhang, Lijun and Jin, Rong and Yang, Tianbao},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4678--4689},
  year={2018}
}
@incollection{lei2019optimal,
title = {Optimal Stochastic and Online Learning with Individual Iterates},
author = {Lei, Yunwen and Yang, Peng and Tang, Ke and Zhou, Ding-Xuan},
booktitle = {Advances in Neural Information Processing Systems},
pages = {5415--5425},
year = {2019}
}
@article{lei2021generalization,
  title={Generalization Performance of Multi-pass Stochastic Gradient Descent with Convex Loss Functions},
  author={Lei, Yunwen and Hu, Ting and Tang, Ke},
  journal={Journal of Machine Learning Research},
  volume={22},
  issue={25},
  pages={1--41},
  year={2021}
}
@article{lin2019boosted,
  title={Boosted Kernel Ridge Regression: Optimal Learning Rates and Early Stopping.},
  author={Lin, Shao-Bo and Lei, Yunwen and Zhou, Ding-Xuan},
  journal={Journal of Machine Learning Research},
  volume={20},
  issue={46},
  pages={1--36},
  year={2019}
}
@article{lei2018convergence,
  title={Convergence of unregularized online learning algorithms},
  author={Lei, Yunwen and Shi, Lei and Guo, Zheng-Chu},
  journal={Journal of Machine Learning Research},
  volume={18},
  number={171},
  pages={1-33},
  year={2018}
}
@article{lei2018learning,
  title={Learning Theory of Randomized Sparse {K}aczmarz Method},
  author={Lei, Yunwen and Zhou, Ding-Xuan},
  journal={SIAM Journal on Imaging Sciences},
  volume={11},
  number={1},
  pages={547--574},
  year={2018},
  publisher={SIAM}
}
@inproceedings{NguyenNDRST18,
  author    = {Lam M. Nguyen and
               Phuong Ha Nguyen and
               Marten van Dijk and
               Peter Richt{\'{a}}rik and
               Katya Scheinberg and
               Martin Tak{\'{a}}c},
  title     = {{SGD} and Hogwild! Convergence Without the Bounded Gradients Assumption},
  booktitle = {International Conference on Machine Learning},
  pages     = {3747--3755},
  year      = {2018}
}
@article{lin2017optimal,
  title={Optimal rates for multi-pass stochastic gradient methods},
  author={Lin, Junhong and Rosasco, Lorenzo},
  journal={Journal of Machine Learning Research},
  volume={18},
  number={1},
  pages={3375--3421},
  year={2017},
  publisher={JMLR. org}
}
@article{michaelides2009snap,
  title={SNAP: Fault tolerant event location estimation in sensor networks using binary data},
  author={Michaelides, Michalis P and Panayiotou, Christos G},
  journal={IEEE Transactions on Computers},
  volume={58},
  number={9},
  pages={1185--1197},
  year={2009},
  publisher={IEEE}
}
@article{foster2019ell,
  title={$\ell_\infty$ Vector Contraction for Rademacher Complexity},
  author={Foster, Dylan J and Rakhlin, Alexander},
  journal={arXiv preprint arXiv:1911.06468},
  year={2019}
}
@article{maurer2016bounds,
  title={Bounds for vector-valued function estimation},
  author={Maurer, Andreas and Pontil, Massimiliano},
  journal={arXiv preprint arXiv:1606.01487},
  year={2016}
}
@article{xu2016local,
  title={Local rademacher complexity for multi-label learning},
  author={Xu, Chang and Liu, Tongliang and Tao, Dacheng and Xu, Chao},
  journal={IEEE Transactions on Image Processing},
  volume={25},
  number={3},
  pages={1495--1507},
  year={2016},
  publisher={IEEE}
}
@article{guermeur2017lp,
  title={Lp-norm Sauer--Shelah lemma for margin multi-category classifiers},
  author={Guermeur, Yann},
  journal={Journal of Computer and System Sciences},
  volume={89},
  pages={450--473},
  year={2017},
  publisher={Elsevier}
}
@article{guermeur2007vc,
  title={VC theory of large margin multi-category classifiers},
  author={Guermeur, Yann},
  journal={Journal of Machine Learning Research},
  volume={8},
  number={Nov},
  pages={2551--2594},
  year={2007}
}
@article{musayeva2019rademacher,
  title={Rademacher complexity and generalization performance of multi-category margin classifiers},
  author={Musayeva, Khadija and Lauer, Fabien and Guermeur, Yann},
  journal={Neurocomputing},
  volume={342},
  pages={6--15},
  year={2019},
  publisher={Elsevier}
}
@inproceedings{li2018multi,
  title={Multi-class learning: from theory to algorithm},
  author={Li, Jian and Liu, Yong and Yin, Rong and Zhang, Hua and Ding, Lizhong and Wang, Weiping},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1586--1595},
  year={2018}
}
@article{alvarez2012kernels,
  title={Kernels for Vector-Valued Functions: A Review},
  author={{\'A}lvarez, Mauricio A and Rosasco, Lorenzo and Lawrence, Neil D},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={4},
  number={3},
  pages={195--266},
  year={2012},
  publisher={Now Publishers Inc. Hanover, MA, USA}
}
@article{zhang2013review,
  title={A review on multi-label learning algorithms},
  author={Zhang, Min-Ling and Zhou, Zhi-Hua},
  journal={IEEE Transactions on Knowledge and Data Engineering},
  volume={26},
  number={8},
  pages={1819--1837},
  year={2013},
  publisher={IEEE}
}
@article{xu2019survey,
  title={Survey on multi-output learning},
  author={Xu, Donna and Shi, Yaxin and Tsang, Ivor W and Ong, Yew-Soon and Gong, Chen and Shen, Xiaobo},
  journal={IEEE Transactions on Neural Networks and Learning Systems},
  year={2019},
  publisher={IEEE}
}
@inproceedings{yu2014large,
  title={Large-scale multi-label learning with missing labels},
  author={Yu, Hsiang-Fu and Jain, Prateek and Kar, Purushottam and Dhillon, Inderjit},
  booktitle={International Conference on Machine Learning},
  pages={593--601},
  year={2014}
}
@inproceedings{shen2018compact,
  title={Compact multi-label learning},
  author={Shen, Xiaobo and Liu, Weiwei and Tsang, Ivor W and Sun, Quan-Sen and Ong, Yew-Soon},
  booktitle={Thirty-Second AAAI Conference on Artificial Intelligence},
  year={2018}
}
@article{li2019learning,
  title={Learning Vector-valued Functions with Local Rademacher Complexity},
  author={Li, Jian and Liu, Yong and Wang, Weiping},
  journal={arXiv preprint arXiv:1909.04883},
  year={2019}
}
@article{anonymous,
  title = {The {R}ademacher complexity of quadratic classifiers with a nuclear norm constraint}
}
@article{lust1991non,
  title={Non commutative Khintchine and Paley inequalities},
  author={Lust-Piquard, Fran{\c{c}}oise and Pisier, Gilles},
  journal={Arkiv f{\"o}r Matematik},
  volume={29},
  number={1},
  pages={241--260},
  year={1991},
  publisher={Springer}
}
@article{tropp2015introduction,
  title={An Introduction to Matrix Concentration Inequalities},
  author={Tropp, Joel A},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={8},
  number={1-2},
  pages={1--230},
  year={2015},
  publisher={Now Publishers Inc. Hanover, MA, USA}
}
@article{nesterov2012efficiency,
  title={Efficiency of coordinate descent methods on huge-scale optimization problems},
  author={Nesterov, Yu},
  journal={SIAM Journal on Optimization},
  volume={22},
  number={2},
  pages={341--362},
  year={2012},
  publisher={SIAM}
}
@inproceedings{wang2012evolutionary,
  title={Evolutionary study of web spam: Webb spam corpus 2011 versus webb spam corpus 2006},
  author={Wang, De and Irani, Danesh and Pu, Calton},
  booktitle={8th International Conference on Collaborative Computing: Networking, Applications and Worksharing},
  pages={40--49},
  year={2012},
  organization={IEEE}
}
@article{zhou2020towards,
  title={Towards Better Generalization of Adaptive Gradient Methods},
  author={Zhou, Yingxue and Karimi, Belhal and Yu, Jinxing and Xu, Zhiqiang and Li, Ping},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}
@article{ghadimi2013stochastic,
  title={Stochastic first-and zeroth-order methods for nonconvex stochastic programming},
  author={Ghadimi, Saeed and Lan, Guanghui},
  journal={SIAM Journal on Optimization},
  volume={23},
  number={4},
  pages={2341--2368},
  year={2013},
  publisher={SIAM}
}
@inproceedings{zhang2019stochastic,
  title={Stochastic Approximation of Smooth and Strongly Convex Functions: Beyond the $ O (1/T) $ Convergence Rate},
  author={Zhang, Lijun and Zhou, Zhi-Hua},
  booktitle={Conference on Learning Theory},
  pages={3160--3179},
  year={2019},
  organization={PMLR}
}
@article{bao2021stability,
  title={Stability and Generalization of Bilevel Programming in Hyperparameter Optimization},
  author={Bao, Fan and Wu, Guoqiang and Li, Chongxuan and Zhu, Jun and Zhang, Bo},
  journal={arXiv preprint arXiv:2106.04188},
  year={2021}
}
@article{cotter2011better,
  title={Better Mini-Batch Algorithms via Accelerated Gradient Methods},
  author={Cotter, Andrew and Shamir, Ohad and Srebro, Nati and Sridharan, Karthik},
  journal={Advances in Neural Information Processing Systems},
  volume={24},
  pages={1647--1655},
  year={2011}
}
@article{lei2021learning,
  author={Lei, Yunwen and Tang, Ke},
  journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
  title={Learning Rates for Stochastic Gradient Descent With Nonconvex Objectives},
  year={2021},
  volume={43},
  number={12},
  pages={4505-4511}
}
@book{vapnik2013nature,
  title={The nature of statistical learning theory},
  author={Vapnik, Vladimir},
  year={2013},
  publisher={Springer}
}
@article{davis2021graphical,
  title={Graphical convergence of subgradients in nonconvex optimization and learning},
  author={Davis, Damek and Drusvyatskiy, Dmitriy},
  journal={Mathematics of Operations Research},
  year={2021},
  publisher={INFORMS}
}
@inproceedings{tripuraneni2018stochastic,
  title={Stochastic cubic regularization for fast nonconvex optimization},
  author={Tripuraneni, Nilesh and Stern, Mitchell and Jin, Chi and Regier, Jeffrey and Jordan, Michael I},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2904--2913},
  year={2018}
}
@article{ghadimi2016mini,
  title={Mini-batch stochastic approximation methods for nonconvex stochastic composite optimization},
  author={Ghadimi, Saeed and Lan, Guanghui and Zhang, Hongchao},
  journal={Mathematical Programming},
  volume={155},
  number={1-2},
  pages={267--305},
  year={2016},
  publisher={Springer}
}
@inproceedings{zhou2018convergence,
  title={Convergence of Cubic Regularization for Nonconvex Optimization under KL Property},
  author={Zhou, Yi and Wang, Zhe and Liang, Yingbin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3764--3773},
  year={2018}
}
@inproceedings{foster2018uniform,
  title={Uniform convergence of gradients for non-convex learning and optimization},
  author={Foster, Dylan J and Sekhari, Ayush and Sridharan, Karthik},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8759--8770},
  year={2018}
}
@article{lojasiewicz1963topological,
  title={A topological property of real analytic subsets},
  author={Lojasiewicz, S},
  journal={Coll. du CNRS, Les {\'e}quations aux d{\'e}riv{\'e}es partielles},
  volume={117},
  pages={87--89},
  year={1963}
}
@inproceedings{reddi2016stochastic,
  title={Stochastic variance reduction for nonconvex optimization},
  author={Reddi, Sashank and Hefny, Ahmed and Sra, Suvrit and Poczos, Barnabas and Smola, Alex},
  booktitle={International Conference on Machine Learning},
  pages={314--323},
  year={2016}
}
@inproceedings{ShwartzSS07,
  author    = {Shai Shalev{-}Shwartz and
               Yoram Singer and
               Nathan Srebro},
  title     = {Pegasos: Primal Estimated sub-GrAdient SOlver for {SVM}},
  booktitle = {International Conference
               Machine Learning},
  pages     = {807--814},
  year      = {2007}
}
@inproceedings{natole2018stochastic,
  title={Stochastic proximal algorithms for {AUC} maximization},
  author={Natole, Michael and Ying, Yiming and Lyu, Siwei},
  booktitle={International Conference on Machine Learning},
  pages={3707--3716},
  year={2018}
}
@article{lei2021stochastic,
  title={Stochastic Proximal {AUC} Maximization},
  author={Lei, Yunwen and Ying, Yiming},
  journal={JMLR},
  volume={22},
  number={61},
  pages={1--45},
  year={2021}
}
@inproceedings{gao2013one,
  title={One-pass {AUC} optimization},
  author={Gao, Wei and Jin, Rong and Zhu, Shenghuo and Zhou, Zhi-Hua},
  booktitle={International Conference on Machine Learning},
  pages={906--914},
  year={2013}
}

@inproceedings{zhao2011online,
  title={Online {AUC} maximization},
  author={Zhao, Peilin and Hoi, Steven CH and Jin, Rong and Yang, Tianbao},
  booktitle={International Conference on Machine Learning},
  pages={233--240},
  year={2011},
  organization={Omnipress}
}
@inproceedings{liu2018fast,
  title={Fast stochastic {AUC} maximization with {O}(1/n)-convergence rate},
  author={Liu, Mingrui and Zhang, Xiaoxuan and Chen, Zaiyi and Wang, Xiaoyu and Yang, Tianbao},
  booktitle={International Conference on Machine Learning},
  pages={3195--3203},
  year={2018}
}
@inproceedings{foster2019hypothesis,
title = {Hypothesis Set Stability and Generalization},
author = {Foster, Dylan J and Greenberg, Spencer and Kale, Satyen and Luo, Haipeng and Mohri, Mehryar and Sridharan, Karthik},
booktitle = {Advances in Neural Information Processing Systems},
pages = {6726--6736},
year = {2019}
}
%editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
@article{schopfer2018linear,
  title={Linear convergence of the randomized sparse {K}aczmarz method},
  author={Sch{\"o}pfer, Frank and Lorenz, Dirk A},
  journal={Mathematical Programming},
  pages={1--28},
  year={2018},
  publisher={Springer}
}
@InProceedings{pmlr-v65-maurer17a,
  title = 	 {A Second-order Look at Stability and Generalization},
  author = 	 {Andreas Maurer},
  booktitle = 	 {Conference on Learning Theory},
  pages = 	 {1461--1475},
  year = 	 {2017}
}
@inproceedings{liu2017algorithmic,
  title={Algorithmic stability and hypothesis complexity},
  author={Liu, Tongliang and Lugosi, G{\'a}bor and Neu, Gergely and Tao, Dacheng},
  booktitle={International Conference on Machine Learning},
  pages={2159--2167},
  year={2017}
}
@inproceedings{maurer2017second,
  title={A second-order look at stability and generalization},
  author={Maurer, Andreas},
  booktitle={Conference on Learning Theory},
  pages={1461--1475},
  year={2017}
}
@article{zhou2018generalization,
  title={Generalization error bounds with probabilistic guarantee for {SGD} in nonconvex optimization},
  author={Zhou, Yi and Liang, Yingbin and Zhang, Huishuai},
  journal={arXiv preprint arXiv:1802.06903},
  year={2018}
}
@article{chang2011libsvm,
  title={LIBSVM: a library for support vector machines},
  author={Chang, Chih-Chung and Lin, Chih-Jen},
  journal={ACM Transactions on Intelligent Systems and Technology},
  volume={2},
  number={3},
  pages={27},
  year={2011},
  publisher={Acm}
}
@article{prokhorov2001ijcnn,
  title={IJCNN 2001 neural network competition},
  author={Prokhorov, Danil},
  journal={Slide presentation in IJCNN},
  volume={1},
  pages={97},
  year={2001}
}
@article{bousquet2002stability,
  title={Stability and generalization},
  author={Bousquet, Olivier and Elisseeff, Andr{\'e}},
  journal={Journal of Machine Learning Research},
  volume={2},
  number={Mar},
  pages={499--526},
  year={2002}
}
@inproceedings{liu2018dual,
  title={Dual set multi-label learning},
  author={Liu, Chong and Zhao, Peng and Huang, Sheng-Jun and Jiang, Yuan and Zhou, Zhi-Hua},
  booktitle={Thirty-Second AAAI Conference on Artificial Intelligence},
  year={2018}
}
@article{zhou2012multi,
  title={Multi-instance multi-label learning},
  author={Zhou, Zhi-Hua and Zhang, Min-Ling and Huang, Sheng-Jun and Li, Yu-Feng},
  journal={Artificial Intelligence},
  volume={176},
  number={1},
  pages={2291--2320},
  year={2012},
  publisher={Elsevier}
}
@article{reeve2020optimistic,
  title={Optimistic bounds for multi-output prediction},
  author={Reeve, Henry WJ and Kaban, Ata},
  journal={arXiv preprint arXiv:2002.09769},
  year={2020}
}
@article{wang2019online,
  title={Online regularized pairwise learning with least squares loss},
  author={Wang, Cheng and Hu, Ting},
  journal={Analysis and Applications},
  pages={1--30},
  year={2019},
  publisher={World Scientific}
}
@article{elisseeff2005stability,
  title={Stability of randomized learning algorithms},
  author={Elisseeff, Andre and Evgeniou, Theodoros and Pontil, Massimiliano},
  journal={Journal of Machine Learning Research},
  volume={6},
  number={Jan},
  pages={55--79},
  year={2005}
}
@inproceedings{li2017convergence,
  title={Convergence analysis of two-layer neural networks with relu activation},
  author={Li, Yuanzhi and Yuan, Yang},
  booktitle={Advances in Neural Information Processing Systems},
  pages={597--607},
  year={2017}
}
@article{necoara2019linear,
  title={Linear convergence of first order methods for non-strongly convex optimization},
  author={Necoara, Ion and Nesterov, Yu and Glineur, Francois},
  journal={Mathematical Programming},
  volume={175},
  number={1-2},
  pages={69--107},
  year={2019},
  publisher={Springer}
}

@inproceedings{vaswani2019fast,
  title={Fast and Faster Convergence of SGD for Over-Parameterized Models and an Accelerated Perceptron},
  author={Vaswani, Sharan and Bach, Francis and Schmidt, Mark},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1195--1204},
  year={2019}
}
@article{bassily2018exponential,
  title={On exponential convergence of sgd in non-convex over-parametrized learning},
  author={Bassily, Raef and Belkin, Mikhail and Ma, Siyuan},
  journal={arXiv preprint arXiv:1811.02564},
  year={2018}
}
@inproceedings{ma2018power,
  title={The Power of Interpolation: Understanding the Effectiveness of SGD in Modern Over-parametrized Learning},
  author={Ma, Siyuan and Bassily, Raef and Belkin, Mikhail},
  booktitle={International Conference on Machine Learning},
  pages={3325--3334},
  year={2018}
}

@inproceedings{DBLP:conf/aistats/NitandaS19,
  author    = {Atsushi Nitanda and
               Taiji Suzuki},
  editor    = {Kamalika Chaudhuri and
               Masashi Sugiyama},
  title     = {Stochastic Gradient Descent with Exponential Convergence Rates of
               Expected Classification Errors},
  booktitle = {International Conference on Artificial Intelligence and Statistics},
  pages     = {1417--1426},
  year = {2019}
}

@InProceedings{pmlr-v75-pillaud-vivien18a,
  title = 	 {Exponential Convergence of Testing Error for Stochastic Gradient Methods},
  author = 	 {Pillaud-Vivien, Loucas and Rudi, Alessandro and Bach, Francis},
  booktitle = 	 {Conference On Learning Theory},
  pages = 	 {250--296},
  year = 	 {2018}
}
@article{bartlett2005local,
  title={Local {R}ademacher complexities},
  author={Bartlett, P.L. and Bousquet, O. and Mendelson, S.},
  journal={Annals of Statistics},
  volume={33},
  number={4},
  pages={1497--1537},
  year={2005},
  publisher={Institute of Mathematical Statistics}
}
@inproceedings{yu2019parallel,
  title={Parallel restarted sgd with faster convergence and less communication: Demystifying why model averaging works for deep learning},
  author={Yu, Hao and Yang, Sen and Zhu, Shenghuo},
  booktitle={AAAI Conference on Artificial Intelligence},
  pages={5693--5700},
  year={2019}
}
@inproceedings{kuzborskij2013stability,
  title={Stability and hypothesis transfer learning},
  author={Kuzborskij, Ilja and Orabona, Francesco},
  booktitle={International Conference on Machine Learning},
  pages={942--950},
  year={2013},
  organization={PMLR}
}
@article{spiridonoff2021communication,
  title={Communication-efficient SGD: From Local SGD to One-Shot Averaging},
  author={Spiridonoff, Artin and Olshevsky, Alex and Paschalidis, Ioannis Ch},
  journal={arXiv preprint arXiv:2106.04759},
  year={2021}
}
@article{oneto2020randomized,
  title={Randomized learning and generalization of fair and private classifiers: From PAC-Bayes to stability and differential privacy},
  author={Oneto, Luca and Donini, Michele and Pontil, Massimiliano and Shawe-Taylor, John},
  journal={Neurocomputing},
  volume={416},
  pages={231--243},
  year={2020},
  publisher={Elsevier}
}
@inproceedings{wu2017bolt,
  title={Bolt-on differential privacy for scalable stochastic gradient descent-based analytics},
  author={Wu, Xi and Li, Fengan and Kumar, Arun and Chaudhuri, Kamalika and Jha, Somesh and Naughton, Jeffrey},
  booktitle={ACM International Conference on Management of Data},
  pages={1307--1322},
  year={2017}
}
@article{yun2021minibatch,
  title={Minibatch vs Local SGD with Shuffling: Tight Convergence Bounds and Beyond},
  author={Yun, Chulhee and Rajput, Shashank and Sra, Suvrit},
  journal={arXiv preprint arXiv:2110.10342},
  year={2021}
}
@article{madden2020high,
  title={High probability convergence bounds for stochastic gradient descent assuming the Polyak-Lojasiewicz inequality},
  author={Madden, Liam and Dall'Anese, Emiliano and Becker, Stephen},
  journal={arXiv preprint arXiv:2006.05610},
  year={2020}
}
@article{keskar2016large,
  title={On large-batch training for deep learning: Generalization gap and sharp minima},
  author={Keskar, Nitish Shirish and Mudigere, Dheevatsa and Nocedal, Jorge and Smelyanskiy, Mikhail and Tang, Ping Tak Peter},
  journal={arXiv preprint arXiv:1609.04836},
  year={2016}
}
@inproceedings{yin2018gradient,
  title={Gradient diversity: a key ingredient for scalable distributed learning},
  author={Yin, Dong and Pananjady, Ashwin and Lam, Max and Papailiopoulos, Dimitris and Ramchandran, Kannan and Bartlett, Peter},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1998--2007},
  year={2018},
  organization={PMLR}
}
@inproceedings{zhang2017efficient,
  title={Efficient private {ERM} for smooth objectives},
  author={Zhang, Jiaqi and Zheng, Kai and Mou, Wenlong and Wang, Liwei},
  booktitle={International Joint Conference on Artificial Intelligence},
  pages={3922--3928},
  year={2017}
}
@article{zhou2020private,
  title={Private stochastic non-convex optimization: Adaptive algorithms and tighter generalization bounds},
  author={Zhou, Yingxue and Chen, Xiangyi and Hong, Mingyi and Wu, Zhiwei Steven and Banerjee, Arindam},
  journal={arXiv preprint arXiv:2006.13501},
  year={2020}
}
@inproceedings{wang2019differentially,
  title={Differentially private empirical risk minimization with smooth non-convex loss functions: A non-stationary view},
  author={Wang, Di and Xu, Jinhui},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={33},
  number={01},
  pages={1182--1189},
  year={2019}
}
@article{nagarajan2019uniform,
  title={Uniform convergence may be unable to explain generalization in deep learning},
  author={Nagarajan, Vaishnavh and Kolter, J Zico},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}
@article{yuan2022boosting,
  title={Boosting the Confidence of Generalization for $ L\_2 $-Stable Randomized Learning Algorithms},
  author={Yuan, Xiao-Tong and Li, Ping},
  journal={arXiv preprint arXiv:2206.03834},
  year={2022}
}

@article{donsker1975asymptotic,
  title={Asymptotic evaluation of certain Markov process expectations for large time, I},
  author={Donsker, Monroe D and Varadhan, SR Srinivasa},
  journal={Communications on Pure and Applied Mathematics},
  volume={28},
  number={1},
  pages={1--47},
  year={1975},
  publisher={Wiley Online Library}
}
@article{davis2022graphical,
  title={Graphical convergence of subgradients in nonconvex optimization and learning},
  author={Davis, Damek and Drusvyatskiy, Dmitriy},
  journal={Mathematics of Operations Research},
  volume={47},
  number={1},
  pages={209--231},
  year={2022}
}
@article{laurent2000adaptive,
  title={Adaptive estimation of a quadratic functional by model selection},
  author={Laurent, Beatrice and Massart, Pascal},
  journal={Annals of Statistics},
  pages={1302--1338},
  year={2000},
  publisher={JSTOR}
}
@inproceedings{yuan2021stability,
  title={Stability and Risk Bounds of Iterative Hard Thresholding},
  author={Yuan, Xiaotong and Li, Ping},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1702--1710},
  year={2021},
  organization={PMLR}
}
@article{zhou2020convergence,
  title={On the convergence of mirror descent beyond stochastic convex programming},
  author={Zhou, Zhengyuan and Mertikopoulos, Panayotis and Bambos, Nicholas and Boyd, Stephen P and Glynn, Peter W},
  journal={SIAM Journal on Optimization},
  volume={30},
  number={1},
  pages={687--716},
  year={2020},
  publisher={SIAM}
}
@article{rosasco2010learning,
  title={On learning with integral operators.},
  author={Rosasco, Lorenzo and Belkin, Mikhail and De Vito, Ernesto},
  journal={Journal of Machine Learning Research},
  volume={11},
  number={2},
  year={2010}
}

@inproceedings{russo2016controlling,
  title={Controlling bias in adaptive data analysis using information theory},
  author={Russo, Daniel and Zou, James},
  booktitle={Artificial Intelligence and Statistics},
  pages={1232--1240},
  year={2016},
  organization={PMLR}
}

@article{nesterov2006cubic,
  title={Cubic regularization of Newton method and its global performance},
  author={Nesterov, Yurii and Polyak, Boris T},
  journal={Mathematical Programming},
  volume={108},
  number={1},
  pages={177--205},
  year={2006},
  publisher={Springer}
}
@book{shalev2014understanding,
  title={Understanding machine learning: From theory to algorithms},
  author={Shalev-Shwartz, Shai and Ben-David, Shai},
  year={2014},
  publisher={Cambridge university press}
}
@inproceedings{wang2021stability,
  title={Stability and Generalization for Randomized Coordinate Descent},
  author={Wang, Puyu and Wu, Liang and Lei, Yunwen},
  booktitle = {International Joint Conference on Artificial Intelligence},
  pages     = {3104--3110},
  year      = {2021}
}
@inproceedings{zhang2020local,
  title={Local regularizer improves generalization},
  author={Zhang, Yikai and Qu, Hui and Metaxas, Dimitris and Chen, Chao},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={6861--6868},
  year={2020}
}
@inproceedings{shawe1997pac,
  title={A PAC analysis of a Bayesian estimator},
  author={Shawe-Taylor, John and Williamson, Robert C},
  booktitle={Proceedings of the Tenth Annual Conference on Computational Learning Theory},
  pages={2--9},
  year={1997}
}
@article{langford2005tutorial,
  title={Tutorial on Practical Prediction Theory for Classification.},
  author={Langford, John and Schapire, Robert},
  journal={Journal of Machine Learning Research},
  volume={6},
  number={3},
  year={2005}
}
@article{catoni2003pac,
  title={A PAC-Bayesian approach to adaptive classification},
  author={Catoni, Olivier},
  journal={preprint},
  volume={840},
  year={2003}
}
@article{ralaivola2010chromatic,
  title={Chromatic PAC-Bayes bounds for non-iid data: Applications to ranking and stationary $\beta$-mixing processes},
  author={Ralaivola, Liva and Szafranski, Marie and Stempfel, Guillaume},
  journal={The Journal of Machine Learning Research},
  volume={11},
  pages={1927--1956},
  year={2010}
}
@article{seldin2010pac,
  title={A PAC-Bayesian analysis of graph clustering and pairwise clustering},
  author={Seldin, Yevgeny},
  journal={arXiv preprint arXiv:1009.0499},
  year={2010}
}
@article{li2018quasi,
  title={A quasi-Bayesian perspective to online clustering},
  author={Li, Le and Guedj, Benjamin and Loustau, S{\'e}bastien},
  journal={Electronic journal of statistics},
  volume={12},
  number={2},
  pages={3071--3113},
  year={2018},
  publisher={Institute of Mathematical Statistics and Bernoulli Society}
}
@article{ambroladze2006tighter,
  title={Tighter pac-bayes bounds},
  author={Ambroladze, Amiran and Parrado-Hern{\'a}ndez, Emilio and Shawe-Taylor, John},
  journal={Advances in Neural Information Processing Systems},
  volume={19},
  year={2006}
}
@article{parrado2012pac,
  title={PAC-Bayes bounds with data dependent priors},
  author={Parrado-Hern{\'a}ndez, Emilio and Ambroladze, Amiran and Shawe-Taylor, John and Sun, Shiliang},
  journal={The Journal of Machine Learning Research},
  volume={13},
  number={1},
  pages={3507--3531},
  year={2012},
  publisher={JMLR. org}
}
@article{perez2021tighter,
  title={Tighter risk certificates for neural networks},
  author={P{\'e}rez-Ortiz, Mar{\'\i}a and Rivasplata, Omar and Shawe-Taylor, John and Szepesv{\'a}ri, Csaba},
  journal={Journal of Machine Learning Research},
  volume={22},
  year={2021}
}
@article{negrea2019information,
  title={Information-theoretic generalization bounds for SGLD via data-dependent estimates},
  author={Negrea, Jeffrey and Haghifam, Mahdi and Dziugaite, Gintare Karolina and Khisti, Ashish and Roy, Daniel M},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}
@article{sun2022stability,
  title={Stability-based PAC-Bayes analysis for multi-view learning algorithms},
  author={Sun, Shiliang and Yu, Mengran and Shawe-Taylor, John and Mao, Liang},
  journal={Information Fusion},
  volume={86},
  pages={76--92},
  year={2022},
  publisher={Elsevier}
}
@inproceedings{dziugaite2017computing,
  title={Computing nonvacuous generalization bounds for deep (stochastic) neural networks with many more parameters than training data},
  author={Dziugaite, Gintare Karolina and Roy, Daniel M},
  booktitle={33-rd Conference on Uncertainty in Artificial Intelligence},
  publisher = {{AUAI} Press},
  year={2017}
}
@inproceedings{begin2016pac,
  title={PAC-Bayesian bounds based on the R{\'e}nyi divergence},
  author={B{\'e}gin, Luc and Germain, Pascal and Laviolette, Fran{\c{c}}ois and Roy, Jean-Francis},
  booktitle={Artificial Intelligence and Statistics},
  pages={435--444},
  year={2016},
  organization={PMLR}
}
@inproceedings{blundell2015weight,
  title={Weight uncertainty in neural network},
  author={Blundell, Charles and Cornebise, Julien and Kavukcuoglu, Koray and Wierstra, Daan},
  booktitle={International Conference on Machine Learning},
  pages={1613--1622},
  year={2015},
  organization={PMLR}
}
@inproceedings{dziugaite2021role,
  title={On the role of data in PAC-Bayes bounds},
  author={Dziugaite, Gintare Karolina and Hsu, Kyle and Gharbieh, Waseem and Arpino, Gabriel and Roy, Daniel},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={604--612},
  year={2021},
  organization={PMLR}
}
@inproceedings{dziugaite2018data,
  title={Data-dependent PAC-Bayes priors via differential privacy},
  author={Dziugaite, Gintare Karolina and Roy, Daniel M},
  booktitle={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}
@inproceedings{zhou2023toward,
  title={Toward Better PAC-Bayes Bounds for Uniformly Stable Algorithms},
  author={Zhou, Sijia and Lei, Yunwen and Kab\'an, Ata},
  booktitle={Advances in Neural Information Processing Systems},
  volume={36},
  year={2023}
}

@inproceedings{harvey2019tight,
  title={Tight analyses for non-smooth stochastic gradient descent},
  author={Harvey, Nicholas JA and Liaw, Christopher and Plan, Yaniv and Randhawa, Sikander},
  booktitle={Conference on Learning Theory},
  pages={1579--1613},
  year={2019},
  organization={PMLR}
}
@article{peel2010empirical,
  title={Empirical Bernstein inequalities for u-statistics},
  author={Peel, Thomas and Anthoine, Sandrine and Ralaivola, Liva},
  journal={Advances in Neural Information Processing Systems},
  volume={23},
  year={2010}
}
@article{perez2021learning,
  title={Learning PAC-Bayes priors for probabilistic neural networks},
  author={P{\'e}rez-Ortiz, Mar{\'\i}a and Rivasplata, Omar and Guedj, Benjamin and Gleeson, Matthew and Zhang, Jingyu and Shawe-Taylor, John and Bober, Miroslaw and Kittler, Josef},
  journal={arXiv preprint arXiv:2109.10304},
  year={2021}
}
@article{alquier2021user,
  title={User-friendly introduction to PAC-Bayes bounds},
  author={Alquier, Pierre},
  journal={arXiv preprint arXiv:2110.11216},
  year={2021}
}
@article{van2014probability,
  title={Probability in high dimension},
  author={Van Handel, Ramon},
  journal={Lecture Notes (Princeton University)},
  year={2014}
}
@article{salehi2017stochastic,
  title={Stochastic optimization with bandit sampling},
  author={Salehi, Farnood and Celis, L Elisa and Thiran, Patrick},
  journal={arXiv preprint arXiv:1708.02544},
  year={2017}
}
@phdthesis{borsos2021data,
  title={Data Summarization in Modern Machine Learning},
  author={Borsos, Zal{\'a}n},
  year={2021},
  school={ETH Zurich}
}
@inproceedings{baltieri20113dpes,
  title={3dpes: 3d people dataset for surveillance and forensics},
  author={Baltieri, Davide and Vezzani, Roberto and Cucchiara, Rita},
  booktitle={Proceedings of the 2011 joint ACM workshop on Human gesture and behavior understanding},
  pages={59--64},
  year={2011}
}
@inproceedings{cheng2011custom,
  title={Custom pictorial structures for re-identification.},
  author={Cheng, Dong Seon and Cristani, Marco and Stoppa, Michele and Bazzani, Loris and Murino, Vittorio and others},
  booktitle={Bmvc},
  volume={1},
  pages={6},
  year={2011},
  organization={Citeseer}
}
@inproceedings{gray2008viewpoint,
  title={Viewpoint invariant pedestrian recognition with an ensemble of localized features},
  author={Gray, Douglas and Tao, Hai},
  booktitle={European Conference on Computer Vision},
  pages={262--275},
  year={2008},
  organization={Springer}
}
@inproceedings{Zheng2009AssociatingGO,
  title={Associating Groups of People},
  author={Weishi Zheng and Shaogang Gong and Tao Xiang},
  booktitle={British Machine Vision Conference},
  year={2009}
}
@inproceedings{koestinger2012large,
  title={Large scale metric learning from equivalence constraints},
  author={Koestinger, Martin and Hirzer, Martin and Wohlhart, Paul and Roth, Peter M and Bischof, Horst},
  booktitle={Computer Vision and Pattern Recognition},
  pages={2288--2295},
  year={2012},
  organization={IEEE}
}
@inproceedings{xiong2014person,
  title={Person re-identification using kernel-based metric learning methods},
  author={Xiong, Fei and Gou, Mengran and Camps, Octavia and Sznaier, Mario},
  booktitle={European Conference Computer Vision},
  pages={1--16},
  year={2014},
  organization={Springer}
}
@inproceedings{zhang2022stability,
  title={Stability of sgd: Tightness analysis and improved bounds},
  author={Zhang, Yikai and Zhang, Wenjia and Bald, Sammy and Pingali, Vamsi and Chen, Chao and Goswami, Mayank},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={2364--2373},
  year={2022},
  organization={PMLR}
}
@article{orabona2019modern,
  title={A modern introduction to online learning},
  author={Orabona, Francesco},
  journal={arXiv preprint arXiv:1912.13213},
  year={2019}
}
@article{hellstrom2023generalization,
  title={Generalization Bounds: Perspectives from Information Theory and PAC-Bayes},
  author={Hellstr{\"o}m, Fredrik and Durisi, Giuseppe and Guedj, Benjamin and Raginsky, Maxim},
  journal={arXiv preprint arXiv:2309.04381},
  year={2023}
}
@article{germain2015risk,
  title={Risk Bounds for the Majority Vote: From a PAC-Bayesian Analysis to a Learning Algorithm},
  author={Germain, Pascal and Lacasse, Alexandre and Laviolette, Francois and March, Mario and Roy, Jean-Francis},
  journal={Journal of Machine Learning Research},
  volume={16},
  number={26},
  pages={787--860},
  year={2015}
}
@article{guedj2019primer,
  title={A primer on PAC-Bayesian learning},
  author={Guedj, Benjamin},
  journal={arXiv preprint arXiv:1901.05353},
  year={2019}
}
@inproceedings{cao2013similarity,
  title={Similarity metric learning for face recognition},
  author={Cao, Qiong and Ying, Yiming and Li, Peng},
  booktitle={International Conference on Computer Vision},
  pages={2408--2415},
  year={2013}
}
@article{stich2017safe,
  title={Safe adaptive importance sampling},
  author={Stich, Sebastian U and Raj, Anant and Jaggi, Martin},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  year={2017}
}
@article{pitcan2017note,
  title={A note on concentration inequalities for u-statistics},
  author={Pitcan, Yannik},
  journal={arXiv preprint arXiv:1712.06160},
  year={2017}
}

@article{wang2023generalization,
  title={Generalization Guarantees of Gradient Descent for Multi-Layer Neural Networks},
  author={Wang, Puyu and Lei, Yunwen and Wang, Di and Ying, Yiming and Zhou, Ding-Xuan},
  journal={arXiv preprint arXiv:2305.16891},
  year={2023}
}
@article{foong2021tight,
  title={How Tight Can PAC-Bayes be in the Small Data Regime?},
  author={Foong, Andrew and Bruinsma, Wessel and Burt, David and Turner, Richard},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={4093--4105},
  year={2021}
}
@article{Deora2023OnTO,
      title={On the Optimization and Generalization of Multi-head Attention}, 
      author={Puneesh Deora and Rouzbeh Ghaderi and Hossein Taheri and Christos Thrampoulidis},
      year={2023},
  journal={arXiv preprint arXiv:2310.12680},
}
@ARTICLE{kaban2013Lq,
  author={Kab\'an, Ata},
  journal={IEEE Transactions on Neural Networks and Learning Systems}, 
  title={Fractional Norm Regularization: Learning With Very Few Relevant Features}, 
  year={2013},
  volume={24},
  number={6},
  pages={953-963},
}
@article{raghavan1988probabilistic,
  title={Probabilistic construction of deterministic algorithms: approximating packing integer programs},
  author={Raghavan, Prabhakar},
  journal={Journal of Computer and System Sciences},
  volume={37},
  number={2},
  pages={130--143},
  year={1988},
  publisher={Elsevier}
}
@inproceedings{katharopoulos2018not,
  title={Not all samples are created equal: Deep learning with importance sampling},
  author={Katharopoulos, Angelos and Fleuret, Fran{\c{c}}ois},
  booktitle={International conference on machine learning},
  pages={2525--2534},
  year={2018},
  organization={PMLR}
}
@inproceedings{lv2018unsupervised,
  title={Unsupervised cross-dataset person re-identification by transfer learning of spatial-temporal patterns},
  author={Lv, Jianming and Chen, Weihang and Li, Qing and Yang, Can},
  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
  pages={7948--7956},
  year={2018}
}
@inproceedings{zheng2015scalable,
  title={Scalable person re-identification: A benchmark},
  author={Zheng, Liang and Shen, Liyue and Tian, Lu and Wang, Shengjin and Wang, Jingdong and Tian, Qi},
  booktitle={Proceedings of the IEEE international conference on computer vision},
  pages={1116--1124},
  year={2015}
}
@inproceedings{li2013locally,
  title={Locally aligned feature transforms across views},
  author={Li, Wei and Wang, Xiaogang},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={3594--3601},
  year={2013}
}
@inproceedings{loy2009multi,
  title={Multi-camera activity correlation analysis},
  author={Loy, Chen Change and Xiang, Tao and Gong, Shaogang},
  booktitle={2009 IEEE Conference on Computer Vision and Pattern Recognition},
  pages={1988--1995},
  year={2009},
  organization={IEEE}
}
@article{zheng2017discriminatively,
  title={A discriminatively learned cnn embedding for person reidentification},
  author={Zheng, Zhedong and Zheng, Liang and Yang, Yi},
  journal={ACM transactions on multimedia computing, communications, and applications (TOMM)},
  volume={14},
  number={1},
  pages={1--20},
  year={2017},
  publisher={ACM New York, NY, USA}
}
@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}
@article{katharopoulos2017biased,
  title={Biased importance sampling for deep neural network training},
  author={Katharopoulos, Angelos and Fleuret, Fran{\c{c}}ois},
  journal={arXiv preprint arXiv:1706.00043},
  year={2017}
}
@article{johnson2018training,
  title={Training deep models faster with robust, approximate importance sampling},
  author={Johnson, Tyler B and Guestrin, Carlos},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}
@inproceedings{wu2017sampling,
  title={Sampling matters in deep embedding learning},
  author={Wu, Chao-Yuan and Manmatha, R and Smola, Alexander J and Krahenbuhl, Philipp},
  booktitle={Proceedings of the IEEE international conference on computer vision},
  pages={2840--2848},
  year={2017}
}
@article{salehi2018coordinate,
  title={Coordinate descent with bandit sampling},
  author={Salehi, Farnood and Thiran, Patrick and Celis, Elisa},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}
@article{liu2020adam,
  title={Adam with bandit sampling for deep learning},
  author={Liu, Rui and Wu, Tianyi and Mozafari, Barzan},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={5393--5404},
  year={2020}
}
@inproceedings{ristani2016performance,
  title={Performance measures and a data set for multi-target, multi-camera tracking},
  author={Ristani, Ergys and Solera, Francesco and Zou, Roger and Cucchiara, Rita and Tomasi, Carlo},
  booktitle={European conference on computer vision},
  pages={17--35},
  year={2016},
  organization={Springer}
}
@inproceedings{hirzer2012relaxed,
  title={Relaxed pairwise learned metric for person re-identification},
  author={Hirzer, Martin and Roth, Peter M and K{\"o}stinger, Martin and Bischof, Horst},
  booktitle={Computer Vision--ECCV 2012: 12th European Conference on Computer Vision, Florence, Italy, October 7-13, 2012, Proceedings, Part VI 12},
  pages={780--793},
  year={2012},
  organization={Springer}
}
@inproceedings{yi2014deep,
  title={Deep metric learning for person re-identification},
  author={Yi, Dong and Lei, Zhen and Liao, Shengcai and Li, Stan Z},
  booktitle={2014 22nd international conference on pattern recognition},
  pages={34--39},
  year={2014},
  organization={IEEE}
}
@article{bromley1993signature,
  title={Signature verification using a" siamese" time delay neural network},
  author={Bromley, Jane and Guyon, Isabelle and LeCun, Yann and S{\"a}ckinger, Eduard and Shah, Roopak},
  journal={Advances in neural information processing systems},
  volume={6},
  year={1993}
}
@inproceedings{hadsell2006dimensionality,
  title={Dimensionality reduction by learning an invariant mapping},
  author={Hadsell, Raia and Chopra, Sumit and LeCun, Yann},
  booktitle={2006 IEEE computer society conference on computer vision and pattern recognition (CVPR'06)},
  volume={2},
  pages={1735--1742},
  year={2006},
  organization={IEEE}
}
@inproceedings{guillaumin2009you,
  title={Is that you? Metric learning approaches for face identification},
  author={Guillaumin, Matthieu and Verbeek, Jakob and Schmid, Cordelia},
  booktitle={2009 IEEE 12th international conference on computer vision},
  pages={498--505},
  year={2009},
  organization={IEEE}
}
@article{bar2005learning,
  title={Learning a Mahalanobis metric from equivalence constraints.},
  author={Bar-Hillel, Aharon and Hertz, Tomer and Shental, Noam and Weinshall, Daphna and Ridgeway, Greg},
  journal={Journal of machine learning research},
  volume={6},
  number={6},
  year={2005}
}
@article{han2022rethinking,
  title={Rethinking sampling strategies for unsupervised person re-identification},
  author={Han, Xumeng and Yu, Xuehui and Li, Guorong and Zhao, Jian and Pan, Gang and Ye, Qixiang and Jiao, Jianbin and Han, Zhenjun},
  journal={IEEE Transactions on Image Processing},
  volume={32},
  pages={29--42},
  year={2022},
  publisher={IEEE}
}
@inproceedings{li2014deepreid,
  title={Deepreid: Deep filter pairing neural network for person re-identification},
  author={Li, Wei and Zhao, Rui and Xiao, Tong and Wang, Xiaogang},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={152--159},
  year={2014}
}
@inproceedings{zhong2017re,
  title={Re-ranking person re-identification with k-reciprocal encoding},
  author={Zhong, Zhun and Zheng, Liang and Cao, Donglin and Li, Shaozi},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={1318--1327},
  year={2017}
}
@inproceedings{gray2007evaluating,
  title={Evaluating appearance models for recognition, reacquisition, and tracking},
  author={Gray, Douglas and Brennan, Shane and Tao, Hai},
  booktitle={Proc. IEEE international workshop on performance evaluation for tracking and surveillance (PETS)},
  volume={3},
  number={5},
  pages={1--7},
  year={2007}
}
@book{serfling2009approximation,
  title={Approximation theorems of mathematical statistics},
  author={Serfling, Robert J},
  year={2009},
  publisher={John Wiley \& Sons}
}
@article{langford2001not,
  title={(Not) bounding the true error},
  author={Langford, John and Caruana, Rich},
  journal={Advances in Neural Information Processing Systems},
  volume={14},
  year={2001}
}
@article{zhang2021beyond,
  title={Beyond triplet loss: Meta prototypical N-tuple loss for person re-identification},
  author={Zhang, Zhizheng and Lan, Cuiling and Zeng, Wenjun and Chen, Zhibo and Chang, Shih-Fu},
  journal={IEEE Transactions on Multimedia},
  volume={24},
  pages={4158--4169},
  year={2021},
  publisher={IEEE}
}
@inproceedings{schroff2015facenet,
  title={Facenet: A unified embedding for face recognition and clustering},
  author={Schroff, Florian and Kalenichenko, Dmitry and Philbin, James},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={815--823},
  year={2015}
}
@inproceedings{farnia2021train,
  title={Train simultaneously, generalize better: Stability of gradient-based minimax learners},
  author={Farnia, Farzan and Ozdaglar, Asuman},
  booktitle={International Conference on Machine Learning},
  pages={3174--3185},
  year={2021},
  organization={PMLR}
}
@inproceedings{beznosikov2023stochastic,
  title={Stochastic gradient descent-ascent: Unified theory and new efficient methods},
  author={Beznosikov, Aleksandr and Gorbunov, Eduard and Berard, Hugo and Loizou, Nicolas},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={172--235},
  year={2023},
  organization={PMLR}
}
@article{goodfellow2014generative,
  title={Generative adversarial nets},
  author={Goodfellow, Ian and Pouget-Abadie, Jean and Mirza, Mehdi and Xu, Bing and Warde-Farley, David and Ozair, Sherjil and Courville, Aaron and Bengio, Yoshua},
  journal={Advances in neural information processing systems},
  volume={27},
  year={2014}
}
@article{madry2017towards,
  title={Towards deep learning models resistant to adversarial attacks},
  author={Madry, Aleksander and Makelov, Aleksandar and Schmidt, Ludwig and Tsipras, Dimitris and Vladu, Adrian},
  journal={arXiv preprint arXiv:1706.06083},
  year={2017}
}
@article{guedj2021still,
  title={Still no free lunches: the price to pay for tighter PAC-Bayes bounds},
  author={Guedj, Benjamin and Pujol, Louis},
  journal={Entropy},
  volume={23},
  number={11},
  pages={1529},
  year={2021},
  publisher={MDPI}
}
@inproceedings{zhou2024self,
  title={Self-certified Tuple-wise Deep Learning},
  author={Zhou, Sijia and Lei, Yunwen and Kaban, Ata},
  booktitle={European Conference on Machine Learning and Principles and Practice of Knowledge Discovery in Databases (ECML PKDD)},
  year={2024}
}
@inproceedings{agarwal2022estimating,
  title={Estimating example difficulty using variance of gradients},
  author={Agarwal, Chirag and D'souza, Daniel and Hooker, Sara},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={10368--10378},
  year={2022}
}
@inproceedings{shah2020choosing,
  title={Choosing the sample with lowest loss makes sgd robust},
  author={Shah, Vatsal and Wu, Xiaoxia and Sanghavi, Sujay},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2120--2130},
  year={2020},
  organization={PMLR}
}
@inproceedings{yang2020robustness,
  title={Robustness for non-parametric classification: A generic attack and defense},
  author={Yang, Yao-Yuan and Rashtchian, Cyrus and Wang, Yizhen and Chaudhuri, Kamalika},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={941--951},
  year={2020},
  organization={PMLR}
}
@article{zhang2021understanding,
  title={Understanding deep learning (still) requires rethinking generalization},
  author={Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
  journal={Communications of the ACM},
  volume={64},
  number={3},
  pages={107--115},
  year={2021},
  publisher={ACM New York, NY, USA}
}
@article{goodfellow2014explaining,
  title={Explaining and harnessing adversarial examples},
  author={Goodfellow, Ian J and Shlens, Jonathon and Szegedy, Christian},
  journal={arXiv preprint arXiv:1412.6572},
  year={2014}
}
@article{wen2024generalization,
  title={Generalization analysis of adversarial pairwise learning},
  author={Wen, Wen and Li, Han and Wu, Rui and Wu, Lingjuan and Chen, Hong},
  journal={Neural Networks},
  pages={106955},
  year={2024},
  publisher={Elsevier}
}
@inproceedings{liu2019geo,
  title={Geo-ALM: POI Recommendation by Fusing Geographical Information and Adversarial Learning Mechanism.},
  author={Liu, Wei and Wang, Zhi-Jie and Yao, Bin and Yin, Jian},
  booktitle={IJCAI},
  volume={7},
  pages={1807--1813},
  year={2019}
}
@article{zhang2020advkin,
  title={AdvKin: Adversarial convolutional network for kinship verification},
  author={Zhang, Lei and Duan, Qingyan and Zhang, David and Jia, Wei and Wang, Xizhao},
  journal={IEEE transactions on cybernetics},
  volume={51},
  number={12},
  pages={5883--5896},
  year={2020},
  publisher={IEEE}
}
@inproceedings{zhou2022enhancing,
  title={Enhancing adversarial robustness for deep metric learning},
  author={Zhou, Mo and Patel, Vishal M},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={15325--15334},
  year={2022}
}
@article{catoni2007pac,
  title={PAC-Bayesian supervised classification: the thermodynamics of statistical learning},
  author={Catoni, Olivier},
  journal={arXiv preprint arXiv:0712.0248},
  year={2007}
}
@inproceedings{mbacke2023pac,
  title={PAC-Bayesian generalization bounds for adversarial generative models},
  author={Mbacke, Sokhna Diarra and Clerc, Florence and Germain, Pascal},
  booktitle={International Conference on Machine Learning},
  pages={24271--24290},
  year={2023},
  organization={PMLR}
}
@article{viallard2021pac,
  title={A pac-bayes analysis of adversarial robustness},
  author={Viallard, Paul and VIDOT, Eric Guillaume and Habrard, Amaury and Morvant, Emilie},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={14421--14433},
  year={2021}
}
@article{xiao2022stability,
  title={Stability analysis and generalization bounds of adversarial training},
  author={Xiao, Jiancong and Fan, Yanbo and Sun, Ruoyu and Wang, Jue and Luo, Zhi-Quan},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={15446--15459},
  year={2022}
}
@article{qin2023large,
  title={Large language models are effective text rankers with pairwise ranking prompting},
  author={Qin, Zhen and Jagerman, Rolf and Hui, Kai and Zhuang, Honglei and Wu, Junru and Yan, Le and Shen, Jiaming and Liu, Tianqi and Liu, Jialu and Metzler, Donald and others},
  journal={arXiv preprint arXiv:2306.17563},
  year={2023}
}
@inproceedings{zantedeschi2017efficient,
  title={Efficient defenses against adversarial attacks},
  author={Zantedeschi, Valentina and Nicolae, Maria-Irina and Rawat, Ambrish},
  booktitle={Proceedings of the 10th ACM workshop on artificial intelligence and security},
  pages={39--49},
  year={2017}
}
@article{xiao2023pac,
  title={PAC-bayesian spectrally-normalized bounds for adversarially robust generalization},
  author={Xiao, Jiancong and Sun, Ruoyu and Luo, Zhi-Quan},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={36305--36323},
  year={2023}
}
@article{kurakin2016adversarial,
  title={Adversarial machine learning at scale},
  author={Kurakin, Alexey and Goodfellow, Ian and Bengio, Samy},
  journal={arXiv preprint arXiv:1611.01236},
  year={2016}
}
@article{nouiehed2019solving,
  title={Solving a class of non-convex min-max games using iterative first order methods},
  author={Nouiehed, Maher and Sanjabi, Maziar and Huang, Tianjian and Lee, Jason D and Razaviyayn, Meisam},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}
@article{zhang2020single,
  title={A single-loop smoothed gradient descent-ascent algorithm for nonconvex-concave min-max problems},
  author={Zhang, Jiawei and Xiao, Peijun and Sun, Ruoyu and Luo, Zhiquan},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={7377--7389},
  year={2020}
}
@article{szegedy2013intriguing,
  title={Intriguing properties of neural networks},
  author={Szegedy, C},
  journal={arXiv preprint arXiv:1312.6199},
  year={2013}
}
@article{gu2014towards,
  title={Towards deep neural network architectures robust to adversarial examples},
  author={Gu, Shixiang and Rigazio, Luca},
  journal={arXiv preprint arXiv:1412.5068},
  year={2014}
}
@inproceedings{zhang2019theoretically,
  title={Theoretically principled trade-off between robustness and accuracy},
  author={Zhang, Hongyang and Yu, Yaodong and Jiao, Jiantao and Xing, Eric and El Ghaoui, Laurent and Jordan, Michael},
  booktitle={International conference on machine learning},
  pages={7472--7482},
  year={2019},
  organization={PMLR}
}
@article{sinha2017certifying,
  title={Certifying some distributional robustness with principled adversarial training},
  author={Sinha, Aman and Namkoong, Hongseok and Volpi, Riccardo and Duchi, John},
  journal={arXiv preprint arXiv:1710.10571},
  year={2017}
}
@article{sanjabi2018convergence,
  title={On the convergence and robustness of training gans with regularized optimal transport},
  author={Sanjabi, Maziar and Ba, Jimmy and Razaviyayn, Meisam and Lee, Jason D},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}
@article{namkoong2016stochastic,
  title={Stochastic gradient methods for distributionally robust optimization with f-divergences},
  author={Namkoong, Hongseok and Duchi, John C},
  journal={Advances in neural information processing systems},
  volume={29},
  year={2016}
}
@inproceedings{dai2018sbeed,
  title={SBEED: Convergent reinforcement learning with nonlinear function approximation},
  author={Dai, Bo and Shaw, Albert and Li, Lihong and Xiao, Lin and He, Niao and Liu, Zhen and Chen, Jianshu and Song, Le},
  booktitle={International conference on machine learning},
  pages={1125--1134},
  year={2018},
  organization={PMLR}
}
@inproceedings{mbacke2023pac,
  title={PAC-Bayesian generalization bounds for adversarial generative models},
  author={Mbacke, Sokhna Diarra and Clerc, Florence and Germain, Pascal},
  booktitle={International Conference on Machine Learning},
  pages={24271--24290},
  year={2023},
  organization={PMLR}
}
@article{viallard2021pac,
  title={A pac-bayes analysis of adversarial robustness},
  author={Viallard, Paul and VIDOT, Eric Guillaume and Habrard, Amaury and Morvant, Emilie},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={14421--14433},
  year={2021}
}
@article{xiao2022stability,
  title={Stability analysis and generalization bounds of adversarial training},
  author={Xiao, Jiancong and Fan, Yanbo and Sun, Ruoyu and Wang, Jue and Luo, Zhi-Quan},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={15446--15459},
  year={2022}
}
@article{qin2023large,
  title={Large language models are effective text rankers with pairwise ranking prompting},
  author={Qin, Zhen and Jagerman, Rolf and Hui, Kai and Zhuang, Honglei and Wu, Junru and Yan, Le and Shen, Jiaming and Liu, Tianqi and Liu, Jialu and Metzler, Donald and others},
  journal={arXiv preprint arXiv:2306.17563},
  year={2023}
}
@inproceedings{zantedeschi2017efficient,
  title={Efficient defenses against adversarial attacks},
  author={Zantedeschi, Valentina and Nicolae, Maria-Irina and Rawat, Ambrish},
  booktitle={Proceedings of the 10th ACM workshop on artificial intelligence and security},
  pages={39--49},
  year={2017}
}
@article{xiao2023pac,
  title={PAC-bayesian spectrally-normalized bounds for adversarially robust generalization},
  author={Xiao, Jiancong and Sun, Ruoyu and Luo, Zhi-Quan},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={36305--36323},
  year={2023}
}
@inproceedings{yin2019rademacher,
  title={Rademacher complexity for adversarially robust generalization},
  author={Yin, Dong and Kannan, Ramchandran and Bartlett, Peter},
  booktitle={International conference on machine learning},
  pages={7085--7094},
  year={2019},
  organization={PMLR}
}
@article{bamber1969reaction,
  title={Reaction times and error rates for “same”-“different” judgments of multidimensional stimull},
  author={Bamber, Donald},
  journal={Perception \& Psychophysics},
  volume={6},
  pages={169--174},
  year={1969},
  publisher={Springer}
}
@article{chen2024nrat,
  title={Nrat: towards adversarial training with inherent label noise},
  author={Chen, Zhen and Wang, Fu and Mu, Ronghui and Xu, Peipei and Huang, Xiaowei and Ruan, Wenjie},
  journal={Machine Learning},
  volume={113},
  number={6},
  pages={3589--3610},
  year={2024},
  publisher={Springer}
}
@InProceedings{pmlr-v272-haddouche25a,
  title = 	 {A PAC-Bayesian Link Between Generalisation and Flat Minima},
  author =       {Haddouche, Maxime and Viallard, Paul and Simsekli, Umut and Guedj, Benjamin},
  booktitle = 	 {Proceedings of The 36th International Conference on Algorithmic Learning Theory},
  pages = 	 {481--511},
  year = 	 {2025},
  editor = 	 {Kamath, Gautam and Loh, Po-Ling},
  volume = 	 {272},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {24--27 Feb},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v272/main/assets/haddouche25a/haddouche25a.pdf},
  url = 	 {https://proceedings.mlr.press/v272/haddouche25a.html},
}
@article{alquier2016properties,
  title={On the properties of variational approximations of Gibbs posteriors},
  author={Alquier, Pierre and Ridgway, James and Chopin, Nicolas},
  journal={Journal of Machine Learning Research},
  volume={17},
  number={236},
  pages={1--41},
  year={2016}
}
@article{WEN2025106955,
title = {Generalization analysis of adversarial pairwise learning},
journal = {Neural Networks},
volume = {183},
pages = {106955},
year = {2025},
issn = {0893-6080},
doi = {https://doi.org/10.1016/j.neunet.2024.106955},
url = {https://www.sciencedirect.com/science/article/pii/S0893608024008840},
author = {Wen Wen and Han Li and Rui Wu and Lingjuan Wu and Hong Chen}
}
@inproceedings{wang2019improving,
  title={Improving adversarial robustness requires revisiting misclassified examples},
  author={Wang, Yisen and Zou, Difan and Yi, Jinfeng and Bailey, James and Ma, Xingjun and Gu, Quanquan},
  booktitle={International conference on learning representations},
  year={2019}
}
@article{zhou2025randomized,
  title={Randomized Pairwise Learning with Adaptive Sampling: A PAC-Bayes Analysis},
  author={Zhou, Sijia and Lei, Yunwen and Kab{\'a}n, Ata},
  journal={arXiv preprint arXiv:2504.02957},
  year={2025}
}