@InProceedings{Bottou2010,
author="Bottou, L{\'e}on",
editor="Lechevallier, Yves
and Saporta, Gilbert",
title="Large-Scale Machine Learning with Stochastic Gradient Descent",
booktitle="Proc. COMPSTAT",
year="2010",
pages="177--186",
}

@article{watkins1992q,
  title={Q-learning},
  author={Watkins, Christopher JCH and Dayan, Peter},
  journal={Machine learning},
  volume={8},
  number={3-4},
  pages={279--292},
  year={1992},
  publisher={Springer}
}

@incollection{baird1995residual,
  title={Residual algorithms: Reinforcement learning with function approximation},
  author={Baird, Leemon},
  booktitle={Machine Learning Proceedings},
  pages={30--37},
  year={1995},
  publisher={Elsevier}
}

@article{gordon1996chattering,
  title={Chattering in {SARSA} ($\lambda$)},
  author={Gordon, Geoffrey J},
  journal={CMU Learning Lab Technical Report},
  year={1996}
}

@article{Marti2017,
author = {Marti, Kurt},
title = {Stochastic Optimization of Regulators},
year = {2017},
address = {USA},
volume = {180},
issue = {C},
journal = {Computers and Structures},
month = feb,
pages = {40–51},
numpages = {12},
}



@article{tao2015random,
  title={Random matrices: universality of local spectral statistics of non-Hermitian matrices},
  author={Tao, Terence and Vu, Van and others},
  journal={Annals of probability},
  volume={43},
  number={2},
  pages={782--874},
  year={2015},
}

@book{sutton2018,
author = {Sutton, Richard S. and Barto, Andrew G.},
title = {Reinforcement Learning: An Introduction},
year = {2018},
publisher = {Bradford},
}


@book{Goodfellow-et-al-2016,
    title={Deep Learning},
    author={Ian Goodfellow and Yoshua Bengio and Aaron Courville},
    publisher={MIT Press},
    year={2016}
}

@article{Kotsalis2020,
author = {Kotsalis, Georgios and Lan, Guanghui and Li, Tianjiao},
year = {2020},
month = {11},
title = {Simple and optimal methods for stochastic variational inequalities, II: Markovian noise and policy evaluation in reinforcement learning},
journal = {arXiv:2011.02987},
}

@inproceedings{Nagaraj2020,
 author = {Nagaraj, Dheeraj and Wu, Xian and Bresler, Guy and Jain, Prateek and Netrapalli, Praneeth},
 booktitle = {Proc. Advances in Neural Information Processing Systems},
 title = {Least Squares Regression with Markovian Data: Fundamental Limits and Algorithms},
 volume = {33},
 year = {2020},
}



@article{delyon2009exponential,
	title={Exponential inequalities for sums of weakly dependent variables},
	author={Delyon, Bernard and others},
	journal={Electronic Journal of Probability},
	volume={14},
	pages={752--779},
	year={2009},
}

@article{agarwal2012generalization,
	title={The generalization ability of online algorithms for dependent data},
	author={Agarwal, Alekh and Duchi, John C},
	journal={IEEE Transactions on Information Theory},
	volume={59},
	number={1},
	pages={573--587},
	year={2012},
}

@article{delyon2015exponential,
  title={Exponential inequalities for dependent processes},
	journal={hal-01072019},
  author={Delyon, Bernard},
  year={2015}
}

@article{jarner2002polynomial,
  title={Polynomial convergence rates of Markov chains},
  author={Jarner, S{\o}ren F and Roberts, Gareth O},
  journal={The Annals of Applied Probability},
  volume={12},
  number={1},
  pages={224--247},
  year={2002},
}

@article{modha1996minimum,
  title={Minimum complexity regression estimation with weakly dependent observations},
  author={Modha, Dharmendra S and Masry, Elias},
  journal={IEEE Transactions on Information Theory},
  volume={42},
  number={6},
  pages={2133--2145},
  year={1996},
}

@book{meyn2012markov,
  title={Markov chains and stochastic stability},
  author={Meyn, Sean P and Tweedie, Richard L},
  year={2012},
  publisher={Springer Science \& Business Media}
}


% in RL section
@article{mnih2013playing,
  title={Playing atari with deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},
  journal={arXiv:1312.5602},
  year={2013}
}

@article{chen2019finite,
  title={Finite-sample analysis of nonlinear stochastic approximation with applications in reinforcement learning},
  author={Chen, Zaiwei and Zhang, Sheng and Doan, Thinh T and Clarke, John-Paul and Maguluri, Siva Theja},
  journal={arXiv:1905.11425},
  year={2019}
}

@article{mnih2015human,
  title={Human-level control through deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and others},
  journal={nature},
  volume={518},
  number={7540},
  pages={529--533},
  year={2015},
}

@inproceedings{melo2008analysis,
  title={An analysis of reinforcement learning with function approximation},
  author={Melo, Francisco S and Meyn, Sean P and Ribeiro, M Isabel},
  booktitle={Proc. International Conference on Machine Learning},
  pages={664--671},
  year={2008},
}

@inproceedings{xu2020finite,
  title={A finite-time analysis of Q-learning with neural network function approximation},
  author={Xu, Pan and Gu, Quanquan},
  booktitle={Proc. International Conference on Machine Learning},
  pages={10555--10565},
  year={2020},
}

@article{agarwal2021rlbook,
  author = {Agarwal, Alekh and  Jiang, Nan and Kakade Sham M and  Sun, Wen},
  title = {Reinforcement Learning: Theory and Algorithms},
  year = 2021,
  journal = {https://rltheorybook.github.io/},
}
% in Related Work
@inproceedings{karimi2019non,
  title={Non-asymptotic analysis of biased stochastic approximation scheme},
  author={Karimi, Belhal and Miasojedow, Blazej and Moulines, Eric and Wai, Hoi-To},
  booktitle={Proc. Conference on Learning Theory},
  pages={1944--1974},
  year={2019},
}

@article{debavelaere2021,
author = {Vianney Debavelaere and Stanley Durrleman and Stéphanie Allassonnière},
title = {{On the convergence of stochastic approximations under a subgeometric ergodic Markov dynamic}},
volume = {15},
journal = {Electronic Journal of Statistics},
number = {1},
pages = {1583 -- 1609},
year = {2021},
}

@article{godichon2021non,
  title={Non-Asymptotic Analysis of Stochastic Approximation Algorithms for Streaming Data},
  author={Godichon-Baggioni, Antoine and Werge, Nicklas and Wintenberger, Olivier},
  journal={arXiv:2109.07117},
  year={2021}
}
 
@Book{hazan2019introduction,
 author = {Hazan, Elad},
 title = {Introduction to Online Convex Optimization},
 publisher = {Elad Hazan},
 year = {2017},
 address = {Erscheinungsort nicht ermittelbar},
 isbn = {1521133301}
 }

@article{xiao2009dual,
  title={Dual averaging method for regularized stochastic learning and online optimization},
  author={Xiao, Lin},
  journal={Proc. Advances in Neural Information Processing Systems},
  volume={22},
  pages={2116--2124},
  year={2009},
}

@article{kuhn2020properties,
  title={Properties of the stochastic approximation EM algorithm with mini-batch sampling},
  author={Kuhn, Estelle and Matias, Catherine and Rebafka, Tabea},
  journal={Statistics and Computing},
  volume={30},
  number={6},
  pages={1725--1739},
  year={2020},
}

@inproceedings{dalal2018finite,
  title={Finite sample analyses for TD (0) with function approximation},
  author={Dalal, Gal and Sz{\"o}r{\'e}nyi, Bal{\'a}zs and Thoppe, Gugan and Mannor, Shie},
  booktitle={Proc. AAAI conference on artificial intelligence},
  year={2018},
}

@article{szepesvari1998asymptotic,
  title={The asymptotic convergence-rate of Q-learning},
  author={Szepesv{\'a}ri, Csaba and others},
  journal={Proc. Advances in neural information processing systems},
  pages={1064--1070},
  year={1998},
}

@article{even2003learning,
  title={Learning Rates for Q-learning.},
  author={Even-Dar, Eyal and Mansour, Yishay and Bartlett, Peter},
  journal={Journal of machine learning Research},
  volume={5},
  number={1},
  year={2003},
}

@inproceedings{qu2020finite,
  title={Finite-Time Analysis of Asynchronous Stochastic Approximation and $ Q $-Learning},
  author={Qu, Guannan and Wierman, Adam},
  booktitle={Proc. Conference on Learning Theory},
  pages={3185--3205},
  year={2020},
}

@book{douc2018markov,
  title={Markov chains},
  author={Douc, Randal and Moulines, Eric and Priouret, Pierre and Soulier, Philippe},
  year={2018},
  publisher={Springer},
}

@article{glynn1996poisson,
author = {Peter W. Glynn and Sean P. Meyn},
title = {{A Lyapunov bound for solutions of the Poisson equation}},
volume = {24},
journal = {The Annals of Probability},
number = {2},
pages = {916 -- 931},
year = {1996},
}

 
@inproceedings{li2021q,
  author    = {Gen Li and
               Changxiao Cai and
               Yuxin Chen and
               Yuantao Gu and
               Yuting Wei and
               Yuejie Chi},
  title     = {Tightening the Dependence on Horizon in the Sample Complexity of Q-Learning},
  booktitle = {{ICML}},
  series    = {Proceedings of Machine Learning Research},
  volume    = {139},
  pages     = {6296--6306},
  publisher = {{PMLR}},
  year      = {2021}
}


@inproceedings{wang2020finite,
  title={Finite-sample analysis of greedy-gq with linear function approximation under markovian noise},
  author={Wang, Yue and Zou, Shaofeng},
  booktitle={Proc. Conference on Uncertainty in Artificial Intelligence},
  pages={11--20},
  year={2020},
}

@inproceedings{kaledin2020finite,
  title={Finite time analysis of linear two-timescale stochastic approximation with Markovian noise},
  author={Kaledin, Maxim and Moulines, Eric and Naumov, Alexey and Tadic, Vladislav and Wai, Hoi-To},
  booktitle={Conference on Learning Theory},
  pages={2144--2203},
  year={2020},
  organization={PMLR}
}

@inproceedings{xu2019two,
	title={Two Time-scale Off-Policy {TD} Learning: Non-asymptotic Analysis over {Markovian} Samples},
	author={Xu, Tengyu and Zou, Shaofeng and Liang, Yingbin},
	booktitle={Proc. Advances in Neural Information Processing Systems (NeurIPS)},
	year={2019}
}

@inproceedings{xu2020improving,
	title={Improving Sample Complexity Bounds for (Natural) Actor-Critic Algorithms},
	author={Xu, Tengyu and Wang, Zhe and Liang, Yingbin},
	booktitle={Proc. Advances in Neural Information Processing Systems (NeurIPS)},
	volume={33},
	year={2020},
}

@InProceedings{wu2020finite,
	title = {A Finite-Time Analysis of Two Time-Scale Actor-Critic Methods},
    author = {Wu, Yue Frank and ZHANG, Weitong and Xu, Pan and Gu, Quanquan},
	booktitle = {Proc. Advances in Neural Information Processing Systems (NeurIPS)},
	pages = {17617--17628},
	volume = {33},
	year = {2020}
} 

@inproceedings{qiu2019finite,
	title={On the finite-time convergence of actor-critic algorithm},
	author={Qiu, Shuang and Yang, Zhuoran and Ye, Jieping and Wang, Zhaoran},
	booktitle={NeurIPS Optimization Foundations for Reinforcement Learning Workshop},
	year={2019}
}

@article{kumar2019sample,
  title={On the sample complexity of actor-critic method for reinforcement learning with function approximation},
  author={Kumar, Harshat and Koppel, Alec and Ribeiro, Alejandro},
  journal={ArXiv:1910.08412},
  year={2019}
}
 
 @ARTICLE{lecun98,
  author={Lecun, Y. and Bottou, L. and Bengio, Y. and Haffner, P.},
  journal={Proceedings of the IEEE}, 
  title={Gradient-based learning applied to document recognition}, 
  year={1998},
  volume={86},
  number={11},
  pages={2278-2324},
  }
 
@inproceedings{wang2019neural,
  title={Neural Policy Gradient Methods: Global Optimality and Rates of Convergence},
  author={Wang, Lingxiao and Cai, Qi and Yang, Zhuoran and Wang, Zhaoran},
  booktitle={International Conference on Learning Representations},
  year={2019}
}

@inproceedings{yang2019provably,
  title={Provably global convergence of actor-critic: A case for linear quadratic regulator with ergodic cost},
  author={Yang, Zhuoran and Chen, Yongxin and Hong, Mingyi and Wang, Zhaoran},
  booktitle={Proc. Advances in Neural Information Processing Systems (NeurIPS)},
  year={2019}
}

@inproceedings{zou2019finite,
  title={Finite-sample analysis for {SARSA} with linear function approximation},
  author={Zou, Shaofeng and Xu, Tengyu and Liang, Yingbin},
  booktitle={Proc. Advances in Neural Information Processing Systems},
  pages={8665--8675},
  year={2019},
}

@inproceedings{duchi2011ergodic,
  title={Ergodic subgradient descent},
  author={Duchi, John and Agarwal, Alekh and Johansson, Mikael and Jordan, Michael},
  booktitle={Allerton Conference},
  year={2011}
}
 

@article{steinwart2009fast,
  title={Fast learning from non-iid observations},
  author={Steinwart, Ingo and Christmann, Andreas},
  journal={Advances in neural information processing systems},
  volume={22},
  pages={1768--1776},
  year={2009}
}

@article{fort2005subgeometric,
  title={Subgeometric ergodicity of strong Markov processes},
  author={Fort, Gersende and Roberts, Gareth O},
  journal={The Annals of Applied Probability},
  volume={15},
  number={2},
  pages={1565--1589},
  year={2005},
  publisher={Institute of Mathematical Statistics}
}

@inproceedings{onalan2009financial,
  title={Financial modelling with Ornstein-Uhlenbeck processes driven by L{\'e}vy process},
  author={Onalan, Omer},
  booktitle={Proceedings of the World Congress on Engineering},
  volume={2},
  pages={1--3},
  year={2009}
}