@article{wedin1972perturbation,
  title={Perturbation bounds in connection with singular value decomposition},
  author={Wedin, Per-{\AA}ke},
  journal={BIT Numerical Mathematics},
  volume={12},
  pages={99--111},
  year={1972},
  publisher={Springer}
}

@article{hajek1968asymptotic,
  title={Asymptotic normality of simple linear rank statistics under alternatives},
  author={H{\'a}jek, Jaroslav},
  journal={The Annals of Mathematical Statistics},
  pages={325--346},
  year={1968},
  publisher={JSTOR}
}

@book{stewart1990matrix,
  author    = {G. W. Stewart and Ji-Guang Sun},
  title     = {Matrix Perturbation Theory},
  publisher = {Academic Press},
  year      = {1990}
}

@article{cape2019signal,
  title={Signal-plus-noise matrix models: eigenvector deviations and fluctuations},
  author={Cape, Joshua and Tang, Minh and Priebe, Carey E},
  journal={Biometrika},
  volume={106},
  number={1},
  pages={243--250},
  year={2019},
  publisher={Oxford University Press}
}
@inproceedings{SGD_lee2022fast,
  title={Fast and robust online inference with stochastic gradient descent via random scaling},
  author={Lee, Sokbae and Liao, Yuan and Seo, Myung Hwan and Shin, Youngki},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  pages={7381--7389},
  year={2022}
}
@article{chen2020SGD,
author = "Chen, Xi and Lee, Jason D. and Tong, Xin T. and Zhang, Yichen",
doi = "10.1214/18-AOS1801",
fjournal = "Annals of Statistics",
journal = "Ann. Statist.",
month = "02",
number = "1",
pages = "251--273",
publisher = "The Institute of Mathematical Statistics",
title = "Statistical inference for model parameters in stochastic gradient descent",
url = "https://doi.org/10.1214/18-AOS1801",
volume = "48",
year = "2020"
}
@inproceedings{cheng2020stochastic,
      title={Stochastic Gradient and Langevin Processes}, 
      author={Xiang Cheng and Dong Yin and Peter L. Bartlett and Michael I. Jordan},
      year={2020},
      eprint={1907.03215},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}
@InProceedings{pmlr-v99-anastasiou19a,
  title = 	 {Normal Approximation for Stochastic Gradient Descent via Non-Asymptotic Rates of Martingale CLT},
  author =       {Anastasiou, Andreas and Balasubramanian, Krishnakumar and Erdogdu, Murat A.},
  pages = 	 {115--137},
  year = 	 {2019},
  editor = 	 {Alina Beygelzimer and Daniel Hsu},
  volume = 	 {99},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Phoenix, USA},
  month = 	 {25--28 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v99/anastasiou19a/anastasiou19a.pdf},
  url = 	 {http://proceedings.mlr.press/v99/anastasiou19a.html},
  abstract = 	 {We provide non-asymptotic convergence rates of the Polyak-Ruppert averaged stochastic gradient descent (SGD) to a normal random vector for a class of twice-differentiable test functions. A crucial intermediate step is proving a non-asymptotic martingale central limit theorem (CLT), i.e., establishing the rates of convergence of a multivariate martingale difference sequence to a normal random vector, which might be of independent interest. We obtain the explicit rates for the multivariate martingale CLT using a combination of Stein?s method and Lindeberg?s argument, which is then used in conjunction with a non-asymptotic analysis of averaged SGD proposed in [PJ92]. Our results have potentially interesting consequences for computing confidence intervals for parameter estimation with SGD and constructing hypothesis tests with SGD that are valid in a non-asymptotic sense}
}
@article{SGD_JMLR:v19:17-370,
  author  = {Yixin Fang and Jinfeng Xu and Lei Yang},
  title   = {Online Bootstrap Confidence Intervals for the Stochastic Gradient Descent Estimator},
  journal = {Journal of Machine Learning Research},
  year    = {2018},
  volume  = {19},
  number  = {78},
  pages   = {1--21},
  url     = {http://jmlr.org/papers/v19/17-370.html}
}
@inproceedings{SGD_conf/aaai/LiLKC18,
  added-at = {2023-09-04T00:00:00.000+0200},
  author = {Li, Tianyang and Liu, Liu and Kyrillidis, Anastasios and Caramanis, Constantine},
  biburl = {https://www.bibsonomy.org/bibtex/23aae70c57ea99a6888083fb4de687fd4/dblp},
  booktitle = {AAAI},
  editor = {McIlraith, Sheila A. and Weinberger, Kilian Q.},
  ee = {https://doi.org/10.1609/aaai.v32i1.11686},
  interhash = {5863a3fe5a67ed6cfe12e29fd5d245fa},
  intrahash = {3aae70c57ea99a6888083fb4de687fd4},
  keywords = {dblp},
  pages = {3571-3578},
  publisher = {AAAI Press},
  timestamp = {2024-04-09T18:09:31.000+0200},
  title = {Statistical Inference Using SGD.},
  url = {http://dblp.uni-trier.de/db/conf/aaai/aaai2018.html#LiLKC18},
  year = 2018
}
@inproceedings{SGD_bather1989stochastic,
  title={Stochastic approximation: a generalisation of the Robbins-Monro procedure},
  author={Bather, J.A.},
  booktitle={Proceedings of the Fourth Prague Symposium on Asymptotic Statistics (Prague, 1988)},
  pages={13--27},
  year={1989},
  publisher={Charles University, Prague}
}
@article{robbins1951,
	author = "Robbins, Herbert and Monro, Sutton",
	doi = "10.1214/aoms/1177729586",
	fjournal = "Annals of Mathematical Statistics",
	journal = "Ann. Math. Statist.",
	month = "09",
	number = "3",
	pages = "400--407",
	publisher = "The Institute of Mathematical Statistics",
	title = "A Stochastic Approximation Method",
	url = "https://doi.org/10.1214/aoms/1177729586",
	volume = "22",
	year = "1951"
}
@article{efron1979bootstrap,
author = {B. Efron},
title = {{Bootstrap Methods: Another Look at the Jackknife}},
volume = {7},
journal = {The Annals of Statistics},
number = {1},
publisher = {Institute of Mathematical Statistics},
pages = {1 -- 26},
keywords = {bootstrap, discriminant analysis, error rate estimation, jackknife, Nonlinear regression, nonparametric variance estimation, Resampling, subsample values},
year = {1979},
doi = {10.1214/aos/1176344552},
URL = {https://doi.org/10.1214/aos/1176344552}
}

@book{hall1992bootstrap,
  title={The Bootstrap and Edgeworth Expansion},
  author={Peter Hall},
  year={1992},
  publisher={Springer}
}
@article{Politis_10.1093/biomet/asad021,
    author = {Politis, Dimitris N},
    title = "{Scalable subsampling: computation, aggregation and inference}",
    journal = {Biometrika},
    volume = {111},
    number = {1},
    pages = {347-354},
    year = {2023},
    month = {03},
    abstract = "{Subsampling has seen a resurgence in the big data era where the standard, full-resample size bootstrap can be infeasible to compute. Nevertheless, even choosing a single random subsample of size b can be computationally challenging with both b and the sample size n being very large. This paper shows how a set of appropriately chosen, nonrandom subsamples can be used to conduct effective, and computationally feasible, subsampling distribution estimation. Furthermore, the same set of subsamples can be used to yield a procedure for subsampling aggregation, also known as subagging, that is scalable with big data. Interestingly, the scalable subagging estimator can be tuned to have the same, or better, rate of convergence than that of θ^n. Statistical inference could then be based on the scalable subagging estimator instead of the original θ^n.}",
    issn = {1464-3510},
    doi = {10.1093/biomet/asad021},
    url = {https://doi.org/10.1093/biomet/asad021},
    eprint = {https://academic.oup.com/biomet/article-pdf/111/1/347/56665622/asad021.pdf},
}

@article{Bickel-m-out-of-n,
	author = "Peter J. Bickel and Friedrich G{\"o}tze and Willem van Zwet",
	title = "Resampling fewer than n obersvations:
	Gains, Losses, and Remedies for Losses",
	journal = "Statistica Sinica",
	volume = {},
	year = 1997,
	pages = "1-31"
}

@article{andrews2010asymptotic,
  title={Asymptotic size and a problem with subsampling and with the m out of n bootstrap},
  author={Andrews, Donald WK and Guggenberger, Patrik},
  journal={Econometric Theory},
  volume={26},
  number={2},
  pages={426--468},
  year={2010},
  publisher={Cambridge University Press}
}

@book{sakov1998using,
  title={Using the m out of n bootstrap in hypothesis testing},
  author={Sakov, Anat},
  year={1998},
  publisher={University of California, Berkeley}
}

@article{nemirovskij1983problem,
  title={Problem complexity and method efficiency in optimization},
  author={Nemirovskij, Arkadij Semenovi{\v{c}} and Yudin, David Borisovich},
  year={1983},
  publisher={Wiley-Interscience}
}

@book{politis1999subsampling,
keywords = {Statistics },
language = {eng},
isbn = {1-4612-1554-4},
publisher = {Springer New York},
series = {Springer Series in Statistics},
title = {Subsampling  / by Dimitris N. Politis, Joseph P. Romano, Michael Wolf.},
year = {1999},
abstract = {Since Efron's profound paper on the bootstrap, an enormous amount of effort has been spent on the development of bootstrap, jacknife, and other resampling methods. The primary goal of these computer-intensive methods has been to provide statistical tools that work in complex situations without imposing unrealistic or unverifiable assumptions about the data generating mechanism. The primary goal of this book is to lay some of the foundation for subsampling methodology and related methods.},
author = {Politis, Dimitris N. and Romano, Joseph P. and Wolf, Michael.},
address = {New York, NY},
booktitle = {Subsampling},
edition = {1st ed. 1999.},
}

@article{wang2018optimal,
  title={Optimal subsampling for large sample logistic regression},
  author={Wang, HaiYing and Zhu, Rong and Ma, Ping},
  journal={Journal of the American Statistical Association},
  volume={113},
  number={522},
  pages={829--844},
  year={2018},
  publisher={Taylor \& Francis}
}

@article{levina2017subsampling,
  title={Subsampling scaling},
  author={Levina, Anna and Priesemann, Viola},
  journal={Nature communications},
  volume={8},
  number={1},
  pages={15140},
  year={2017},
  publisher={Nature Publishing Group UK London}
}

@article{chua2024scalable,
  title={Scalable DP-SGD: Shuffling vs. poisson subsampling},
  author={Chua, Lynn and Ghazi, Badih and Kamath, Pritish and Kumar, Ravi and Manurangsi, Pasin and Sinha, Amer and Zhang, Chiyuan},
  journal={Advances in Neural Information Processing Systems},
  volume={37},
  pages={70026--70047},
  year={2024}
}

@article{bertail1999subsampling,
  title={On subsampling estimators with unknown rate of convergence},
  author={Bertail, Patrice and Politis, Dimitris N and Romano, Joseph P},
  journal={Journal of the American Statistical Association},
  volume={94},
  number={446},
  pages={569--579},
  year={1999},
  publisher={Taylor \& Francis}
}

@article{keshavan2010completion,
author = {Keshavan, Raghunandan and Montanari, Andrea and Oh, Sewoong},
year = {2010},
month = {07},
pages = {2980 - 2998},
title = {Matrix Completion From a Few Entries},
volume = {56},
journal = {Information Theory, IEEE Transactions on},
doi = {10.1109/TIT.2010.2046205}
}

@article{ramprasad2023online,
  title={Online bootstrap inference for policy evaluation in reinforcement learning},
  author={Ramprasad, Pratik and Li, Yuantong and Yang, Zhuoran and Wang, Zhaoran and Sun, Will Wei and Cheng, Guang},
  journal={Journal of the American Statistical Association},
  volume={118},
  number={544},
  pages={2901--2914},
  year={2023},
  publisher={Taylor \& Francis}
}

@inproceedings{chaudhuri2024differentially,
 author = {Chaudhuri, Kamalika and Loh, Po-Ling and Pandey, Shourya and Sarkar, Purnamrita},
 booktitle = {Advances in Neural Information Processing Systems},
 title = {On Differentially Private U Statistics},
 volume = {37},
 year = {2024}
}


@article{zhu2023online,
  title={Online covariance matrix estimation in stochastic gradient descent},
  author={Zhu, Wanrong and Chen, Xi and Wu, Wei Biao},
  journal={Journal of the American Statistical Association},
  volume={118},
  number={541},
  pages={393--404},
  year={2023},
  publisher={Taylor \& Francis}
}

@article{ibragimov2010t,
  title={t-Statistic based correlation and heterogeneity robust inference},
  author={Ibragimov, Rustam and M{\"u}ller, Ulrich K},
  journal={Journal of Business \& Economic Statistics},
  volume={28},
  number={4},
  pages={453--468},
  year={2010},
  publisher={Taylor \& Francis}
}

@article{zhu2024high,
  title={High confidence level inference is almost free using parallel stochastic optimization},
  author={Zhu, Wanrong and Lou, Zhipeng and Wei, Ziyang and Wu, Wei Biao},
  journal={arXiv preprint arXiv:2401.09346},
  year={2024}
}

@inproceedings{yu2020nonconvexAnalysis,
author = {Yu, Lu and Balasubramanian, Krishnakumar and Volgushev, Stanislav and Erdogdu, Murat A.},
title = {An analysis of constant step size SGD in the non-convex regime: asymptotic normality and bias},
year = {2021},
isbn = {9781713845393},
publisher = {Curran Associates Inc.},
address = {Red Hook, NY, USA},
booktitle = {Proceedings of the 35th International Conference on Neural Information Processing Systems},
articleno = {324},
numpages = {15},
series = {NIPS '21}
}

@inproceedings{lee2022fast,
  title={Fast and robust online inference with stochastic gradient descent via random scaling},
  author={Lee, Sokbae and Liao, Yuan and Seo, Myung Hwan and Shin, Youngki},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={36},
  number={7},
  pages={7381--7389},
  year={2022}
}

@article{kuchibhotla2024hulc,
  title={The HulC: confidence regions from convex hulls},
  author={Kuchibhotla, Arun Kumar and Balakrishnan, Sivaraman and Wasserman, Larry},
  journal={Journal of the Royal Statistical Society Series B: Statistical Methodology},
  volume={86},
  number={3},
  pages={586--622},
  year={2024},
  publisher={Oxford University Press UK}
}

@InProceedings{pmlr-v151-balzano22a,
  title = 	 { On the equivalence of Oja’s algorithm and GROUSE },
  author =       {Balzano, Laura},
  booktitle = 	 {Proceedings of The 25th International Conference on Artificial Intelligence and Statistics},
  pages = 	 {7014--7030},
  year = 	 {2022},
  editor = 	 {Camps-Valls, Gustau and Ruiz, Francisco J. R. and Valera, Isabel},
  volume = 	 {151},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {28--30 Mar},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v151/balzano22a/balzano22a.pdf},
  url = 	 {https://proceedings.mlr.press/v151/balzano22a.html},
  abstract = 	 { The analysis of streaming PCA has gained significant traction through the analysis of an early simple variant: Oja’s algorithm, which implements online projected gradient descent for the trace objective. Several other streaming PCA algorithms have been developed, each with their own performance guarantees or empirical studies, and the question arises whether there is a relationship between the algorithms. We show that the Grassmannian Rank-One Subspace Estimation (GROUSE) algorithm is indeed equivalent to Oja’s algorithm in the sense that, at each iteration, given a step size for one of the algorithms, we may construct a step size for the other algorithm that results in an identical update. This allows us to apply all results on one algorithm to the other. In particular, we have (1) better global convergence guarantees of GROUSE to the global minimizer of the PCA objective with full data; and (2) local convergence guarantees for Oja’s algorithm with incomplete or compressed data. }
}
@inproceedings{jain2013matcompl,
author = {Jain, Prateek and Netrapalli, Praneeth and Sanghavi, Sujay},
title = {Low-Rank Matrix Completion Using Alternating Minimization},
year = {2013},
isbn = {9781450320290},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/2488608.2488693},
doi = {10.1145/2488608.2488693},
abstract = {Alternating minimization represents a widely applicable and empirically successful approach for finding low-rank matrices that best fit the given data. For example, for the problem of low-rank matrix completion, this method is believed to be one of the most accurate and efficient, and formed a major component of the winning entry in the Netflix Challenge [17].In the alternating minimization approach, the low-rank target matrix is written in a bi-linear form, i.e. X = UV†; the algorithm then alternates between finding the best U and the best V. Typically, each alternating step in isolation is convex and tractable. However the overall problem becomes non-convex and is prone to local minima. In fact, there has been almost no theoretical understanding of when this approach yields a good result.In this paper we present one of the first theoretical analyses of the performance of alternating minimization for matrix completion, and the related problem of matrix sensing. For both these problems, celebrated recent results have shown that they become well-posed and tractable once certain (now standard) conditions are imposed on the problem. We show that alternating minimization also succeeds under similar conditions. Moreover, compared to existing results, our paper shows that alternating minimization guarantees faster (in particular, geometric) convergence to the true matrix, while allowing a significantly simpler analysis.},
booktitle = {Proceedings of the Forty-Fifth Annual ACM Symposium on Theory of Computing},
pages = {665–674},
numpages = {10},
keywords = {matrix completion, matrix sensing, alternating minimization},
location = {Palo Alto, California, USA},
series = {STOC '13}
}

@inproceedings{balzano2010sstracking,
  title={Online identification and tracking of subspaces from highly incomplete information},
  author={Balzano, Laura and Nowak, Robert and Recht, Benjamin},
  booktitle={2010 48th Annual allerton conference on communication, control, and computing},
  pages={704--711},
  year={2010},
  organization={IEEE}
}

@Book{efron1993introduction,
  Title                    = {An Introduction to the Bootstrap},
  Author                   = {Bradley Efron and Robert J. Tibshirani},
  Publisher                = {Chapman \& Hall/CRC},
  Year                     = {1993},

  Address                  = {Boca Raton, Florida, USA},
  Number                   = {57},
  Series                   = {Monographs on Statistics and Applied Probability}
}

@book{ljung1992plusminusref,
author = {Ljung, Lennart and Pflug, Georg and Walk, Harro},
title = {Stochastic approximation and optimization of random systems},
year = {1992},
isbn = {3764327332},
publisher = {Birkhauser Verlag},
address = {CHE}
}
@article{blbjrssb,
	author = {Kleiner, Ariel and Talwalkar, Ameet and Sarkar, Purnamrita and Jordan, Michael I.},
	title = {A scalable bootstrap for massive data},
	journal = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
	volume = {76},
	number = {4},
	pages = {795-816},
	year = {2014}
}

@techreport{ruppert1988efficient,
  title={Efficient estimations from a slowly convergent Robbins-Monro process},
  author={Ruppert, David},
  year={1988},
  institution={Cornell University Operations Research and Industrial Engineering}
}

@article{Davydov1973,
	author = "Yu. A. Davydov",
	title = "Mixing conditions for {Markov} chains",
	journal = "Theory of Probability and Its Applications",
	volume = 18,
	year = 1973,
	pages = "312--328",
	doi = "10.1137/1118033"}

@article{fabian1968,
	author = "Fabian, Vaclav",
	doi = "10.1214/aoms/1177698258",
	fjournal = "Annals of Mathematical Statistics",
	journal = "Ann. Math. Statist.",
	month = "08",
	number = "4",
	pages = "1327--1332",
	publisher = "The Institute of Mathematical Statistics",
	title = "On Asymptotic Normality in Stochastic Approximation",
	url = "https://doi.org/10.1214/aoms/1177698258",
	volume = "39",
	year = "1968"
}

@incollection{bach2011sgd,
	title = {Non-Asymptotic Analysis of Stochastic Approximation Algorithms for Machine Learning},
	author = {Moulines, Eric and Francis R. Bach},
	booktitle = {Advances in Neural Information Processing Systems 24},
	editor = {J. Shawe-Taylor and R. S. Zemel and P. L. Bartlett and F. Pereira and K. Q. Weinberger},
	pages = {451--459},
	year = {2011},
	publisher = {Curran Associates, Inc.},
	url = {http://papers.nips.cc/paper/4316-non-asymptotic-analysis-of-stochastic-approximation-algorithms-for-machine-learning.pdf}
}

@article{toulis2017SGD,
author = "Toulis, Panos and Airoldi, Edoardo M.",
doi = "10.1214/16-AOS1506",
fjournal = "Annals of Statistics",
journal = "Ann. Statist.",
month = "08",
number = "4",
pages = "1694--1727",
publisher = "The Institute of Mathematical Statistics",
title = "Asymptotic and finite-sample properties of estimators based on stochastic gradients",
url = "https://doi.org/10.1214/16-AOS1506",
volume = "45",
year = "2017"
}
@inproceedings{rakhlin2012asgd,
	author = {Rakhlin, Alexander and Shamir, Ohad and Sridharan, Karthik},
	title = {Making Gradient Descent Optimal for Strongly Convex Stochastic Optimization},
	year = {2012},
	isbn = {9781450312851},
	publisher = {Omnipress},
	address = {Madison, WI, USA},
	booktitle = {Proceedings of the 29th International Coference on International Conference on Machine Learning},
	pages = {1571–1578},
	numpages = {8},
	location = {Edinburgh, Scotland},
	series = {ICML'12}
}

@article{nemirovsky2009,
	author = {Nemirovski, Arkadi and Juditsky, Anatoli B. and Lan, Guanghui and Shapiro, Alexander},
	biburl = {https://www.bibsonomy.org/bibtex/2f848ce1c6f71793028500df74ca58745/dblp},
	ee = {https://www.wikidata.org/entity/Q57392911},
	interhash = {c8e4843656e58e7ac0999f5b8e07dac1},
	intrahash = {f848ce1c6f71793028500df74ca58745},
	keywords = {dblp},
	number = 4,
	pages = {1574-1609},
	timestamp = {2020-05-23T11:58:25.000+0200},
	title = {Robust Stochastic Approximation Approach to Stochastic Programming.},
	url = {http://dblp.uni-trier.de/db/journals/siamjo/siamjo19.html#NemirovskiJLS09},
	volume = 19,
	year = 2009
}

@article{zhong2023online,
  title={Online bootstrap inference with nonconvex stochastic gradient descent estimator},
  author={Zhong, Yanjie and Kuffner, Todd and Lahiri, Soumendra},
  journal={arXiv preprint arXiv:2306.02205},
  year={2023}
}


@InProceedings{pmlr-v206-chee23a,
  title = 	 {“Plus/minus the learning rate”: Easy and Scalable Statistical Inference with SGD},
  author =       {Chee, Jerry and Kim, Hwanwoo and Toulis, Panos},
  booktitle = 	 {Proceedings of The 26th International Conference on Artificial Intelligence and Statistics},
  pages = 	 {2285--2309},
  year = 	 {2023},
  editor = 	 {Ruiz, Francisco and Dy, Jennifer and van de Meent, Jan-Willem},
  volume = 	 {206},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {25--27 Apr},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v206/chee23a/chee23a.pdf},
  url = 	 {https://proceedings.mlr.press/v206/chee23a.html},
  abstract = 	 {In this paper, we develop a statistical inference procedure using stochastic gradient descent (SGD)-based confidence intervals. These intervals are of the simplest possible form: $\theta_{N,j} \pm 2\sqrt{}(\gamma/N)$ , where $\theta_N$ is the SGD estimate of model parameters $\theta$ over N data points, and $\gamma$ is the learning rate. This construction relies only on a proper selection of the learning rate to ensure the standard SGD conditions for O(1/n) convergence. The procedure performs well in our empirical evaluations, achieving near-nominal coverage intervals scaling up to 20$\times$ as many parameters as other SGD-based inference methods. We also demonstrate our method’s  practical significance on modeling adverse events in emergency general surgery patients using a novel dataset from the Hospital of the University of Pennsylvania.}
}


@article{polyak1992sgd,
author = {Polyak, B. T. and Juditsky, A. B.},
title = {Acceleration of Stochastic Approximation by Averaging},
year = {1992},
issue_date = {July 1992},
publisher = {Society for Industrial and Applied Mathematics},
address = {USA},
volume = {30},
number = {4},
issn = {0363-0129},
url = {https://doi.org/10.1137/0330046},
doi = {10.1137/0330046},
journal = {SIAM J. Control Optim.},
month = jul,
pages = {838–855},
numpages = {18},
keywords = {stochastic approximation, stochastic optimization, optimal algorithms, recursive estimation}
}


@misc{su2018uncertainty,
title={Uncertainty Quantification for Online Learning and Stochastic Approximation via Hierarchical Incremental Gradient Descent}, 
author={Weijie J. Su and Yuancheng Zhu},
year={2018},
eprint={1802.04876},
archivePrefix={arXiv},
primaryClass={stat.ML}
}
@article{fang2018boot,
	author = {Fang, Yixin and Xu, Jinfeng and Yang, Lei},
	title = {Online Bootstrap Confidence Intervals for the Stochastic Gradient Descent Estimator},
	year = {2018},
	issue_date = {January 2018},
	publisher = {JMLR.org},
	volume = {19},
	number = {1},
	issn = {1532-4435},
	journal = {J. Mach. Learn. Res.},
	month = jan,
	pages = {3053–3073},
	numpages = {21},
	keywords = {resampling methods, interval estimation, M-estimators, generalized linear models, quantile regression, stochastic gradient descent, large datasets, bootstrap}
}
@article{cape2017singular,
author = {Joshua Cape and Minh Tang and Carey E. Priebe},
title = {{The two-to-infinity norm and singular subspace geometry with applications to high-dimensional statistics}},
volume = {47},
journal = {The Annals of Statistics},
number = {5},
publisher = {Institute of Mathematical Statistics},
pages = {2405 -- 2439},
keywords = {eigenvector perturbation, High-dimensional statistics, Principal Component Analysis, Procrustes analysis, Singular value decomposition, spectral methods},
year = {2019},
doi = {10.1214/18-AOS1752},
URL = {https://doi.org/10.1214/18-AOS1752}
}
@article{carter2025statistical,
  title={Statistical Inference for Online Algorithms},
  author={Carter, Selina and Kuchibhotla, Arun K},
  journal={arXiv preprint arXiv:2505.17300},
  year={2025}
}
@inproceedings{eldridge2018unperturbed,
  title={Unperturbed: spectral analysis beyond Davis-Kahan},
  author={Eldridge, Justin and Belkin, Mikhail and Wang, Yusu},
  booktitle={Algorithmic learning theory},
  pages={321--358},
  year={2018},
  organization={PMLR}
}



@article{Mao02102021,
author = {Xueyu Mao and Purnamrita Sarkar and Deepayan Chakrabarti},
title = {Estimating Mixed Memberships With Sharp Eigenvector Deviations},
journal = {Journal of the American Statistical Association},
volume = {116},
number = {536},
pages = {1928--1940},
year = {2021},
publisher = {ASA Website},
doi = {10.1080/01621459.2020.1751645},
URL = { 
        https://doi.org/10.1080/01621459.2020.1751645
},
eprint = { 
    
        https://doi.org/10.1080/01621459.2020.1751645
    
    

}

}
@article{davis1970rotation,
  author    = {Chandler Davis and William M. Kahan},
  title     = {The Rotation of Eigenvectors by a Perturbation. {III}},
  journal   = {SIAM Journal on Numerical Analysis},
  volume    = {7},
  number    = {1},
  pages     = {1--46},
  year      = {1970},
  publisher = {Society for Industrial and Applied Mathematics}
}
@article{abbe2020entrywise,
  author    = {Emmanuel Abbe and Jianqing Fan and Kaizheng Wang and Yiqiao Zhong},
  title     = {Entrywise Eigenvector Analysis of Random Matrices with Low Expected Rank},
  journal   = {Annals of Statistics},
  volume    = {48},
  number    = {3},
  pages     = {1452--1474},
  year      = {2020},
  publisher = {Institute of Mathematical Statistics}
}
@article{abbe2022lptheory,
author = {Emmanuel Abbe and Jianqing Fan and Kaizheng Wang},
title = {{An ${\ell _{p}}$ theory of PCA and spectral clustering}},
volume = {50},
journal = {The Annals of Statistics},
number = {4},
publisher = {Institute of Mathematical Statistics},
pages = {2359 -- 2385},
keywords = {Community detection, contextual network models, eigenvector perturbation, Mixture models, Phase transitions, Principal Component Analysis, spectral clustering},
year = {2022},
doi = {10.1214/22-AOS2196},
URL = {https://doi.org/10.1214/22-AOS2196}
}

@article{lunde2021bootstrapping,
  title={Bootstrapping the error of Oja's algorithm},
  author={Lunde, Robert and Sarkar, Purnamrita and Ward, Rachel},
  journal={Advances in neural information processing systems},
  volume={34},
  pages={6240--6252},
  year={2021}
}
@article{raic2019multivariate,
  title={A multivariate Berry--Esseen theorem with explicit constants},
  author={Raič, Martin},
  journal={Bernoulli},
  volume={25},
  number={4A},
  pages={2824--2853},
  year={2019},
  publisher={Bernoulli Society for Mathematical Statistics and Probability},
  doi={10.3150/18-BEJ1072},
  url={https://arxiv.org/abs/1802.06475}
}
@article{bentkus1986dependence,
  title={Dependence of the Berry--Esseen estimate on the dimension},
  author={Bentkus, V. Yu.},
  journal={Lithuanian Mathematical Journal},
  volume={26},
  number={2},
  pages={110--114},
  year={1986},
  publisher={Springer},
  doi={10.1007/BF00966143},
  mrnumber={862741}
}

@article{bhattacharya2010exposition,
  title={An Exposition of Götze's Estimation of the Rate of Convergence in the Multivariate Central Limit Theorem},
  author={Bhattacharya, Rabi and Holmes, Susan},
  journal={arXiv preprint arXiv:1003.4254},
  year={2010},
  url={https://arxiv.org/abs/1003.4254}
}
@incollection{nagaev1976estimate,
  title={An estimate of the remainder term in the multidimensional central limit theorem},
  author={Nagaev, S.V.},
  booktitle={Proceedings of the Third Japan–USSR Symposium on Probability Theory (Tashkent, 1975)},
  series={Lecture Notes in Mathematics},
  volume={550},
  pages={419--438},
  year={1976},
  publisher={Springer},
  address={Berlin},
  mrnumber={0443043}
}
@article{gotze1991rate,
  title={On the rate of convergence in the multivariate CLT},
  author={Götze, Friedrich},
  journal={The Annals of Probability},
  volume={19},
  number={2},
  pages={724--739},
  year={1991},
  publisher={Institute of Mathematical Statistics},
  doi={10.1214/aop/1176990448},
  mrnumber={1106283}
}
@article{huang2022matrix,
  title={Matrix concentration for products},
  author={Huang, De and Niles-Weed, Jonathan and Tropp, Joel A and Ward, Rachel},
  journal={Foundations of Computational Mathematics},
  volume={22},
  number={6},
  pages={1767--1799},
  year={2022},
  publisher={Springer}
}
@incollection{nazarov2003maximal,
  author    = {Fedor Nazarov},
  title     = {On the Maximal Perimeter of a Convex Set in \( \mathbb{R}^n \) with Respect to a Gaussian Measure},
  booktitle = {Geometric Aspects of Functional Analysis (GAFA) 2001-2002},
  series    = {Lecture Notes in Mathematics},
  volume    = {1807},
  pages     = {169--187},
  year      = {2003},
  publisher = {Springer}
}
@InProceedings{jain2016streaming,
author = {Jain, Prateek and Jin, Chi and Kakade, Sham and Netrapalli, Praneeth and Sidford, Aaron},
title = {Streaming {PCA}: Matching Matrix Bernstein and Near-Optimal Finite Sample Guarantees for {Oja's} Algorithm},
booktitle = {Proceedings of The 29th Conference on Learning Theory (COLT)},
year = {2016},
month = {June},
edition = {Proceedings of The 29th Conference on Learning Theory (COLT)},
}
@article{oja1982simplified,
  title={Simplified neuron model as a principal component analyzer},
  author={Oja, Erkki},
  journal={Journal of mathematical biology},
  volume={15},
  pages={267--273},
  year={1982},
  publisher={Springer}
}

@article{boucheron2003concentration,
  title={Concentration inequalities using the entropy method},
  author={Boucheron, St{\'e}phane and Lugosi, G{\'a}bor and Massart, Pascal},
  journal={The Annals of Probability},
  volume={31},
  number={3},
  pages={1583--1614},
  year={2003},
  publisher={Institute of Mathematical Statistics}
}

@book{wainwright2019high,
  title={High-dimensional statistics: A non-asymptotic viewpoint},
  author={Wainwright, Martin J},
  volume={48},
  year={2019},
  publisher={Cambridge university press}
}

@misc{chernozhukov2017detailed,
      title={Detailed proof of Nazarov's inequality}, 
      author={Victor Chernozhukov and Denis Chetverikov and Kengo Kato},
      year={2017},
      eprint={1711.10696},
      archivePrefix={arXiv},
      primaryClass={math.ST},
      url={https://arxiv.org/abs/1711.10696}, 
}

@article{ChernoCLT2015,
author = {Victor Chernozhukov and Denis Chetverikov and Kengo Kato},
title = {{Central limit theorems and bootstrap in high dimensions}},
volume = {45},
journal = {The Annals of Probability},
number = {4},
publisher = {Institute of Mathematical Statistics},
pages = {2309 -- 2352},
keywords = {bootstrap limit theorems, central limit theorem, high dimensions, hyperrectangles, sparsely convex sets},
year = {2017},
doi = {10.1214/16-AOP1113},
URL = {https://doi.org/10.1214/16-AOP1113}
}

@inproceedings{kumarsarkar2024sparse,
  title={Oja's Algorithm for Streaming Sparse PCA},
  author={Kumar, Syamantak and Sarkar, Purnamrita},
  booktitle={The Thirty-eighth Annual Conference on Neural Information Processing Systems},
    year={2024},
}

@article{mendelson2020robust,
  title={Robust covariance estimation under L\_4-L\_2 norm equivalence},
  author={Mendelson, Shahar and Zhivotovskiy, Nikita},
  year={2020}
}

@article{jambulapati2020robust,
  title={Robust sub-gaussian principal component analysis and width-independent schatten packing},
  author={Jambulapati, Arun and Li, Jerry and Tian, Kevin},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={15689--15701},
  year={2020}
}


@InProceedings{pmlr-v247-jambulapati24a,
  title = 	 {Black-Box k-to-1-PCA Reductions: Theory and Applications},
  author =       {Jambulapati, Arun and Kumar, Syamantak and Li, Jerry and Pandey, Shourya and Pensia, Ankit and Tian, Kevin},
  booktitle = 	 {Proceedings of Thirty Seventh Conference on Learning Theory},
  pages = 	 {2564--2607},
  year = 	 {2024},
  editor = 	 {Agrawal, Shipra and Roth, Aaron},
  volume = 	 {247},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {30 Jun--03 Jul},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v247/jambulapati24a/jambulapati24a.pdf},
  url = 	 {https://proceedings.mlr.press/v247/jambulapati24a.html},
  abstract = 	 {The $k$-principal component analysis ($k$-PCA) problem is a fundamental algorithmic primitive that is widely-used in data analysis and dimensionality reduction applications. In statistical settings, the goal of $k$-PCA is to identify a top eigenspace of the covariance matrix of a distribution, which we only have black-box access to via samples. Motivated by these  settings, we analyze black-box deflation methods as a framework for designing $k$-PCA algorithms, where we model access to the unknown target matrix via a black-box $1$-PCA oracle which returns an approximate top eigenvector, under two popular notions of approximation. Despite being arguably the most natural reduction-based approach to $k$-PCA algorithm design, such black-box methods, which recursively call a $1$-PCA oracle $k$ times, were previously poorly-understood.  Our main contribution is significantly sharper bounds on the approximation parameter degradation of deflation methods for $k$-PCA. For a quadratic form notion of approximation we term ePCA (energy PCA), we show deflation methods suffer no parameter loss. For an alternative well-studied approximation notion we term cPCA (correlation PCA), we tightly characterize the parameter regimes where deflation methods are feasible. Moreover, we show that in all feasible regimes, $k$-cPCA deflation algorithms suffer no asymptotic parameter loss for any constant $k$. We apply our framework to obtain state-of-the-art $k$-PCA algorithms robust to dataset contamination, improving prior work in sample complexity by a $\mathsf{poly}(k)$ factor.}
}



@book{vershynin2018high,
  title={High-dimensional probability: An introduction with applications in data science},
  author={Vershynin, Roman},
  volume={47},
  year={2018},
  publisher={Cambridge university press}
}


@incollection{vershynin2010introduction,
  added-at = {2013-10-18T03:34:57.000+0200},
  author = {Vershynin, Roman},
  booktitle = {Compressed Sensing: Theory and Practice},
  doi = {10.1017/CBO9780511794308.006},
  editor = {Eldar, Yonina C. and Kutyniok, GittaEditors},
  interhash = {3d1ac0c7fc7743dda68253a52385642b},
  intrahash = {7c4db02a502bcc13f88942d308532d29},
  isbn = {9780511794308},
  keywords = {no.pdf random_matrix tutorial},
  pages = {210--268},
  publisher = {Cambridge University Press},
  timestamp = {2015-12-12T14:11:58.000+0100},
  title = {Introduction to the Non-Asymptotic Analysis of Random Matrices},
  year = 2012
}


@inproceedings{diakonikolas2023nearly,
  title={Nearly-linear time and streaming algorithms for outlier-robust PCA},
  author={Diakonikolas, Ilias and Kane, Daniel and Pensia, Ankit and Pittas, Thanasis},
  booktitle={International Conference on Machine Learning},
  pages={7886--7921},
  year={2023},
  organization={PMLR}
}

@article{ziegel2003principal,
  title={Principal component analysis},
  author={Ziegel, Eric R},
  journal={Technometrics},
  volume={45},
  number={3},
  pages={276--277},
  year={2003},
  publisher={American Statistical Association}
}

@article{pearson1901liii,
  title={Principal components analysis},
  author={Pearson, Karl},
  journal={The London, Edinburgh, and Dublin Philosophical Magazine and Journal of Science},
  volume={6},
  number={2},
  pages={559},
  year={1901}
}

@inproceedings{allenzhu2017efficient,
  title={First efficient convergence for streaming k-pca: a global, gap-free, and near-optimal rate},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi},
  booktitle={2017 IEEE 58th Annual Symposium on Foundations of Computer Science (FOCS)},
  pages={487--492},
  year={2017},
  organization={IEEE}
}

@article{kumarsarkar2024markovoja,
  title={Streaming PCA for Markovian data},
  author={Kumar, Syamantak and Sarkar, Purnamrita},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2024}
}

@book{hebb2005organization,
  title={The organization of behavior: A neuropsychological theory},
  author={Hebb, Donald Olding},
  year={2005},
  publisher={Psychology press}
}

@article{oja1985stochastic,
  title={On stochastic approximation of the eigenvectors and eigenvalues of the expectation of a random matrix},
  author={Oja, Erkki and Karhunen, Juha},
  journal={Journal of mathematical analysis and applications},
  volume={106},
  number={1},
  pages={69--84},
  year={1985},
  publisher={Elsevier}
}

@inproceedings{hall1998incremental,
  title={Incremental eigenanalysis for classification.},
  author={Hall, Peter M and Marshall, A David and Martin, Ralph R},
  booktitle={BMVC},
  volume={98},
  pages={286--295},
  year={1998},
  organization={Citeseer}
}

@incollection{gen-oja,
	title = {Gen-Oja: Simple \&amp; Efficient Algorithm for Streaming Generalized Eigenvector Computation},
	author = {Bhatia, Kush and Pacchiano, Aldo and Flammarion, Nicolas and Bartlett, Peter L and Jordan, Michael I},
	booktitle = {Advances in Neural Information Processing Systems 31},
	editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
	pages = {7016--7025},
	year = {2018},
	publisher = {Curran Associates, Inc.},
	url = {http://papers.nips.cc/paper/7933-gen-oja-simple-efficient-algorithm-for-streaming-generalized-eigenvector-computation.pdf}
}

@article{chen2018dimensionality,
  title={Dimensionality reduction for stationary time series via stochastic nonconvex optimization},
  author={Chen, Minshuo and Yang, Lin and Wang, Mengdi and Zhao, Tuo},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}

@article{yang2018history,
  title={History PCA: A new algorithm for streaming PCA},
  author={Yang, Puyudi and Hsieh, Cho-Jui and Wang, Jane-Ling},
  journal={arXiv preprint arXiv:1802.05447},
  year={2018}
}

@INPROCEEDINGS{mouzakis2022spectral,
  author={Price, Eric and Xun, Zhiyang},
  booktitle={2024 IEEE 65th Annual Symposium on Foundations of Computer Science (FOCS)}, 
  title={Spectral Guarantees for Adversarial Streaming PCA}, 
  year={2024},
  volume={},
  number={},
  pages={1768-1785},
  keywords={Computer science;Tail;Vectors;Covariance matrices;Principal component analysis;Principal Component Analysis;Streaming Algorithm},
  doi={10.1109/FOCS61266.2024.00108}}

@misc{henriksen2019adaoja,
      title={AdaOja: Adaptive Learning Rates for Streaming PCA}, 
      author={Amelia Henriksen and Rachel Ward},
      year={2019},
      eprint={1905.12115},
      archivePrefix={arXiv},
      primaryClass={stat.ML},
      url={https://arxiv.org/abs/1905.12115}, 
}
@article{kathuria2020concentration,
  title={On concentration inequalities for random matrix products},
  author={Kathuria, Tarun and Mukherjee, Satyaki and Srivastava, Nikhil},
  journal={arXiv preprint arXiv:2003.06319},
  year={2020}
}
@article{DBLP:journals/corr/abs-2102-03646,
  author    = {De Huang and
               Jonathan Niles{-}Weed and
               Rachel Ward},
  title     = {Streaming k-PCA: Efficient guarantees for Oja's algorithm, beyond
               rank-one updates},
  journal   = {CoRR},
  volume    = {abs/2102.03646},
  year      = {2021},
  url       = {https://arxiv.org/abs/2102.03646},
  eprinttype = {arXiv},
  eprint    = {2102.03646},
  timestamp = {Wed, 10 Feb 2021 15:24:32 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-2102-03646.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@article{monnez2022stochastic,
  title={Stochastic approximation of eigenvectors and eigenvalues of the Q-symmetric expectation of a random matrix},
  author={Monnez, Jean-Marie},
  journal={Communications in Statistics-Theory and Methods},
  pages={1--15},
  year={2022},
  publisher={Taylor \& Francis}
}

@book{vandervaart-asymptotic,
title={Asymptotic statistics},
author = {Aad {van der Vaart}},
year= {2000},
publisher ={{Cambridge University Press}}
}

@book{lee1990u,
  author    = {Lee, A. J.},
  title     = {U-Statistics: Theory and Practice},
  year      = {1990},
  edition   = {1st},
  publisher = {Routledge},
  doi       = {10.1201/9780203734520}
}

@article{bickel2008choice,
  title={On the choice of m in the m out of n bootstrap and confidence bounds for extrema},
  author={Bickel, Peter J and Sakov, Anat},
  journal={Statistica Sinica},
  pages={967--985},
  year={2008},
  publisher={JSTOR}
}

@inproceedings{anguita2013public,
  title        = {A public domain dataset for human activity recognition using smartphones},
  author       = {Anguita, Davide and Ghio, Alessandro and Oneto, Luca and Parra, Xavier and Reyes-Ortiz, Juan-Luis},
  booktitle    = {Proceedings of the 21st European Symposium on Artificial Neural Networks, Computational Intelligence and Machine Learning (ESANN)},
  pages        = {437--442},
  year         = {2013}
}

@article{lecun1998gradient,
  title        = {Gradient-based learning applied to document recognition},
  author       = {LeCun, Yann and Bottou, Le{\'o}n and Bengio, Yoshua and Haffner, Patrick},
  journal      = {Proceedings of the IEEE},
  volume       = {86},
  number       = {11},
  pages        = {2278--2324},
  year         = {1998},
  doi          = {10.1109/5.726791},
  publisher    = {IEEE}
}