@inproceedings{langley00,
 author    = {P. Langley},
 title     = {Crafting Papers on Machine Learning},
 year      = {2000},
 pages     = {1207--1216},
 editor    = {Pat Langley},
 booktitle     = {Proceedings of the 17th International Conference
              on Machine Learning (ICML 2000)},
 address   = {Stanford, CA},
 publisher = {Morgan Kaufmann}
}

@inproceedings{wang2023model,
  title={Model-free robust average-reward reinforcement learning},
  author={Wang, Yue and Velasquez, Alvaro and Atia, George K and Prater-Bennette, Ashley and Zou, Shaofeng},
  booktitle={International Conference on Machine Learning},
  pages={36431--36469},
  year={2023},
  organization={PMLR}
}

@inproceedings{blanchet2015unbiased,
  title={Unbiased {M}onte {C}arlo for optimization and functions of expectations via multi-level randomization},
  author={Blanchet, Jose H and Glynn, Peter W},
  booktitle={2015 Winter Simulation Conference (WSC)},
  pages={3656--3667},
  year={2015},
  organization={IEEE}
}

@article{wang2022unbiased,
  title={Unbiased Multilevel {M}onte {C}arlo methods for intractable distributions: MLMC meets MCMC},
  author={Wang, Guanyang and Wang, Tianze},
  journal={arXiv preprint arXiv:2204.04808},
  year={2022}
}

@article{blanchet2019unbiased,
  title={Unbiased multilevel Monte Carlo: Stochastic optimization, steady-state simulation, quantiles, and other applications},
  author={Blanchet, Jose H and Glynn, Peter W and Pei, Yanan},
  journal={arXiv preprint arXiv:1904.09929},
  year={2019}
}

@inproceedings{liu2022distributionally,
  title={Distributionally Robust $ Q $-Learning},
  author={Liu, Zijian and Bai, Qinxun and Blanchet, Jose and Dong, Perry and Xu, Wei and Zhou, Zhengqing and Zhou, Zhengyuan},
  booktitle={International Conference on Machine Learning},
  pages={13623--13643},
  year={2022},
  organization={PMLR}
}


@article{levy2020large,
  title={Large-scale methods for distributionally robust optimization},
  author={Levy, Daniel and Carmon, Yair and Duchi, John C and Sidford, Aaron},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={8847--8860},
  year={2020}
}

@article{wang2023achieving,
  title={Achieving Minimax Optimal Sample Complexity of Offline Reinforcement Learning: A DRO-Based Approach},
  author={Wang, Yue and Xiong, Jinjun and Zou, Shaofeng},
  year={2023}
}

@inproceedings{zhou2021finite,
  title={Finite-sample regret bound for distributionally robust offline tabular reinforcement learning},
  author={Zhou, Zhengqing and Zhou, Zhengyuan and Bai, Qinxun and Qiu, Linhai and Blanchet, Jose and Glynn, Peter},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3331--3339},
  year={2021},
  organization={PMLR}
}

@article{shi2022distributionally,
  title={Distributionally robust model-based offline reinforcement learning with near-optimal sample complexity},
  author={Shi, Laixi and Chi, Yuejie},
  journal={arXiv preprint arXiv:2208.05767},
  year={2022}
}

@article{gao2023distributionally,
  title={Distributionally robust stochastic optimization with Wasserstein distance},
  author={Gao, Rui and Kleywegt, Anton},
  journal={Mathematics of Operations Research},
  volume={48},
  number={2},
  pages={603--655},
  year={2023},
  publisher={INFORMS}
}

@article{duchi2018learning,
  title={Learning models with uniform performance via distributionally robust optimization},
  author={Duchi, John and Namkoong, Hongseok},
  journal={arXiv preprint arXiv:1810.08750},
  year={2018}
}

@book{vershynin2018high,
  title={High-dimensional probability: An introduction with applications in data science},
  author={Vershynin, Roman},
  volume={47},
  year={2018},
  publisher={Cambridge university press}
}

@article{duchi2019variance,
  title={Variance-based regularization with convex objectives},
  author={Duchi, John and Namkoong, Hongseok},
  journal={Journal of Machine Learning Research},
  volume={20},
  number={68},
  pages={1--55},
  year={2019}
}

@inproceedings{duan2021risk,
  title={Risk bounds and rademacher complexity in batch reinforcement learning},
  author={Duan, Yaqi and Jin, Chi and Li, Zhiyuan},
  booktitle={International Conference on Machine Learning},
  pages={2892--2902},
  year={2021},
  organization={PMLR}
}


@article{delage2010distributionally,
  title={Distributionally robust optimization under moment uncertainty with application to data-driven problems},
  author={Delage, Erick and Ye, Yinyu},
  journal={Operations research},
  volume={58},
  number={3},
  pages={595--612},
  year={2010},
  publisher={INFORMS}
}

@article{dai2020coindice,
  title={Coindice: Off-policy confidence interval estimation},
  author={Dai, Bo and Nachum, Ofir and Chow, Yinlam and Li, Lihong and Szepesv{\'a}ri, Csaba and Schuurmans, Dale},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={9398--9411},
  year={2020}
}

@article{blanchet2019quantifying,
  title={Quantifying distributional model risk via optimal transport},
  author={Blanchet, Jose and Murthy, Karthyek},
  journal={Mathematics of Operations Research},
  volume={44},
  number={2},
  pages={565--600},
  year={2019},
  publisher={INFORMS}
}

@article{wang2021online,
  title={Online robust reinforcement learning with model uncertainty},
  author={Wang, Yue and Zou, Shaofeng},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={7193--7206},
  year={2021}
}

@article{bertsimas2018data,
  title={Data-driven robust optimization},
  author={Bertsimas, Dimitris and Gupta, Vishal and Kallus, Nathan},
  journal={Mathematical Programming},
  volume={167},
  pages={235--292},
  year={2018},
  publisher={Springer}
}

@article{fournier2015rate,
  title={On the rate of convergence in Wasserstein distance of the empirical measure},
  author={Fournier, Nicolas and Guillin, Arnaud},
  journal={Probability theory and related fields},
  volume={162},
  number={3-4},
  pages={707--738},
  year={2015},
  publisher={Springer}
}


@TechReport{mitchell80,
  author = 	 "T. M. Mitchell",
  title = 	 "The Need for Biases in Learning Generalizations",
  institution =  "Computer Science Department, Rutgers University",
  year = 	 "1980",
  address =	 "New Brunswick, MA",
}

@phdthesis{kearns89,
  author = {M. J. Kearns},
  title =  {Computational Complexity of Machine Learning},
  school = {Department of Computer Science, Harvard University},
  year =   {1989}
}

@Book{MachineLearningI,
  editor = 	 "R. S. Michalski and J. G. Carbonell and T.
		  M. Mitchell",
  title = 	 "Machine Learning: An Artificial Intelligence
		  Approach, Vol. I",
  publisher = 	 "Tioga",
  year = 	 "1983",
  address =	 "Palo Alto, CA"
}

@Book{DudaHart2nd,
  author =       "R. O. Duda and P. E. Hart and D. G. Stork",
  title =        "Pattern Classification",
  publisher =    "John Wiley and Sons",
  edition =      "2nd",
  year =         "2000"
}

@misc{anonymous,
  title= {Suppressed for Anonymity},
  author= {Author, N. N.},
  year= {2021}
}

@InCollection{Newell81,
  author =       "A. Newell and P. S. Rosenbloom",
  title =        "Mechanisms of Skill Acquisition and the Law of
                  Practice", 
  booktitle =    "Cognitive Skills and Their Acquisition",
  pages =        "1--51",
  publisher =    "Lawrence Erlbaum Associates, Inc.",
  year =         "1981",
  editor =       "J. R. Anderson",
  chapter =      "1",
  address =      "Hillsdale, NJ"
}


@Article{Samuel59,
  author = 	 "A. L. Samuel",
  title = 	 "Some Studies in Machine Learning Using the Game of
		  Checkers",
  journal =	 "IBM Journal of Research and Development",
  year =	 "1959",
  volume =	 "3",
  number =	 "3",
  pages =	 "211--229"
}


@article{clavier2023towards,
  title={Towards Minimax Optimality of Model-based Robust Reinforcement Learning},
  author={Clavier, Pierre and Le Pennec, Erwan and Geist, Matthieu},
  journal={arXiv preprint arXiv:2302.05372},
  year={2023}
}

@article{hu2013kullback,
  title={Kullback-Leibler divergence constrained distributionally robust optimization},
  author={Hu, Zhaolin and Hong, L Jeff},
  journal={Available at Optimization Online},
  volume={1},
  number={2},
  pages={9},
  year={2013}
}


@article{liang2023single,
  title={Single-Trajectory Distributionally Robust Reinforcement Learning},
  author={Liang, Zhipeng and Ma, Xiaoteng and Blanchet, Jose and Zhang, Jiheng and Zhou, Zhengyuan},
  journal={arXiv preprint arXiv:2301.11721},
  year={2023}
}

@misc{boucheron2013concentration,
  title={Concentration Inequalities: A Nonasymptotic Theory of Independence. Univ. Press},
  author={Boucheron, S and Lugosi, G and Massart, P},
  year={2013},
  publisher={Oxford}
}
@article{maurer2009empirical,
  title={Empirical bernstein bounds and sample variance penalization},
  author={Maurer, Andreas and Pontil, Massimiliano},
  journal={arXiv preprint arXiv:0907.3740},
  year={2009}
}
@article{iyengar2005robust,
  title={Robust dynamic programming},
  author={Iyengar, Garud N},
  journal={Mathematics of Operations Research},
  volume={30},
  number={2},
  pages={257--280},
  year={2005},
  publisher={INFORMS}
}

@article{chen2020finite,
  title={Finite-sample analysis of contractive stochastic approximation using smooth convex envelopes},
  author={Chen, Zaiwei and Maguluri, Siva Theja and Shakkottai, Sanjay and Shanmugam, Karthikeyan},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={8223--8234},
  year={2020}
}
@article{shi2023curious,
  title={The Curious Price of Distributional Robustness in Reinforcement Learning with a Generative Model},
  author={Shi, Laixi and Li, Gen and Wei, Yuting and Chen, Yuxin and Geist, Matthieu and Chi, Yuejie},
  journal={arXiv preprint arXiv:2305.16589},
  year={2023}
}

@inproceedings{wang2023finite,
  title={A finite sample complexity bound for distributionally robust q-learning},
  author={Wang, Shengbo and Si, Nian and Blanchet, Jose and Zhou, Zhengyuan},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3370--3398},
  year={2023},
  organization={PMLR}
}

@inproceedings{panaganti2022sample,
  title={Sample complexity of robust reinforcement learning with a generative model},
  author={Panaganti, Kishan and Kalathil, Dileep},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={9582--9602},
  year={2022},
  organization={PMLR}
}

@article{yang2022toward,
  title={Toward theoretical understandings of robust Markov decision processes: Sample complexity and asymptotics},
  author={Yang, Wenhao and Zhang, Liangyu and Zhang, Zhihua},
  journal={The Annals of Statistics},
  volume={50},
  number={6},
  pages={3223--3248},
  year={2022},
  publisher={Institute of Mathematical Statistics}
}

@incollection{Bengio+chapter2007,
author = {Bengio, Yoshua and LeCun, Yann},
booktitle = {Large Scale Kernel Machines},
publisher = {MIT Press},
title = {Scaling Learning Algorithms Towards {AI}},
year = {2007}
}

@article{Hinton06,
author = {Hinton, Geoffrey E. and Osindero, Simon and Teh, Yee Whye},
journal = {Neural Computation},
pages = {1527--1554},
title = {A Fast Learning Algorithm for Deep Belief Nets},
volume = {18},
year = {2006}
}

@book{goodfellow2016deep,
title={Deep learning},
author={Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron and Bengio, Yoshua},
volume={1},
year={2016},
publisher={MIT Press}
}

@article{einstein,
author =       "Albert Einstein",
title =        "{Zur Elektrodynamik bewegter Körper}",
journal =      "Annalen der Physik",
volume =       "322",
number =       "10",
pages =        "891--921",
year =         "1905"}

@book{latexcompanion,
author    = "Michel Goossens and Frank Mittelbach and Alexander Samarin",
title     = "The \LaTeX\ Companion",
year      = "1993",
publisher = "Addison-Wesley"}







@article{konda2003onactor,
  title={On actor-critic algorithms},
  author={Konda, Vijay R and Tsitsiklis, John N},
  journal={SIAM Journal on Control and Optimization},
  volume={42},
  number={4},
  pages={1143--1166},
  year={2003},
  publisher={SIAM}
}
@article{sutton1999policy,
  title={Policy gradient methods for reinforcement learning with function approximation},
  author={Sutton, Richard S and McAllester, David and Singh, Satinder and Mansour, Yishay},
  journal={Advances in neural information processing systems},
  volume={12},
  year={1999}
}

@article{bhandari2021finite,
  title={A finite time analysis of temporal difference learning with linear function approximation},
  author={Bhandari, Jalaj and Russo, Daniel and Singal, Raghav},
  journal={Operations Research},
  volume={69},
  number={3},
  pages={950--973},
  year={2021},
  publisher={INFORMS}
}

@book{sutton2018reinforcement,
  title={Reinforcement learning: An introduction},
  author={Sutton, Richard S and Barto, Andrew G},
  year={2018},
  publisher={MIT press}
}





@article{chen2022global,
  title={Global convergence of two-timescale actor-critic for solving linear quadratic regulator},
  author={Chen, Xuyang and Duan, Jingliang and Liang, Yingbin and Zhao, Lin},
  journal={arXiv preprint arXiv:2208.08744},
  year={2022}
}

@article{agarwal2021theory,
  title={On the theory of policy gradient methods: Optimality, approximation, and distribution shift},
  author={Agarwal, Alekh and Kakade, Sham M and Lee, Jason D and Mahajan, Gaurav},
  journal={The Journal of Machine Learning Research},
  volume={22},
  number={1},
  pages={4431--4506},
  year={2021},
  publisher={JMLRORG}
}
@inproceedings{barakat2022analysis,
  title={Analysis of a target-based actor-critic algorithm with linear function approximation},
  author={Barakat, Anas and Bianchi, Pascal and Lehmann, Julien},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={991--1040},
  year={2022},
  organization={PMLR}
}





@article{khodadadian2022finite,
  title={Finite sample analysis of two-time-scale natural actor-critic algorithm},
  author={Khodadadian, Sajad and Doan, Thinh T and Romberg, Justin and Maguluri, Siva Theja},
  journal={IEEE Transactions on Automatic Control},
  year={2022},
  publisher={IEEE}
}

@article{nedic2003least,
  title={Least squares policy evaluation algorithms with linear function approximation},
  author={Nedi{\'c}, A and Bertsekas, Dimitri P},
  journal={Discrete Event Dynamic Systems},
  volume={13},
  number={1-2},
  pages={79--110},
  year={2003},
  publisher={Springer}
}



@inproceedings{khodadadian2021finite,
  title={Finite-sample analysis of off-policy natural actor-critic algorithm},
  author={Khodadadian, Sajad and Chen, Zaiwei and Maguluri, Siva Theja},
  booktitle={International Conference on Machine Learning},
  pages={5420--5431},
  year={2021},
  organization={PMLR}
}

@article{cayci2022finite,
  title={Finite-time analysis of entropy-regularized neural natural actor-critic algorithm},
  author={Cayci, Semih and He, Niao and Srikant, R},
  journal={arXiv preprint arXiv:2206.00833},
  year={2022}
}

@article{olshevsky2022small,
  title={A small gain analysis of single timescale actor critic},
  author={Olshevsky, Alex and Gharesifard, Bahman},
  journal={arXiv preprint arXiv:2203.02591},
  year={2022}
}


@article{zhou2022single,
  title={Single Time-scale Actor-critic Method to Solve the Linear Quadratic Regulator with Convergence Guarantees},
  author={Zhou, Mo and Lu, Jianfeng},
  journal={arXiv preprint arXiv:2202.00048},
  year={2022}
}

@article{yang2019provably,
  title={Provably global convergence of actor-critic: A case for linear quadratic regulator with ergodic cost},
  author={Yang, Zhuoran and Chen, Yongxin and Hong, Mingyi and Wang, Zhaoran},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}
@inproceedings{zhang2020provably,
  title={Provably convergent two-timescale off-policy actor-critic with function approximation},
  author={Zhang, Shangtong and Liu, Bo and Yao, Hengshuai and Whiteson, Shimon},
  booktitle={International Conference on Machine Learning},
  pages={11204--11213},
  year={2020},
  organization={PMLR}
}

@article{qiu2021finite,
  title={On finite-time convergence of actor-critic algorithm},
  author={Qiu, Shuang and Yang, Zhuoran and Ye, Jieping and Wang, Zhaoran},
  journal={IEEE Journal on Selected Areas in Information Theory},
  volume={2},
  number={2},
  pages={652--664},
  year={2021},
  publisher={IEEE}
}



@article{chen2021closing,
  title={Closing the gap: Tighter analysis of alternating stochastic gradient methods for bilevel problems},
  author={Chen, Tianyi and Sun, Yuejiao and Yin, Wotao},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={25294--25307},
  year={2021}
}

@article{wang2021non,
  title={Non-asymptotic analysis for two time-scale TDC with general smooth function approximation},
  author={Wang, Yue and Zou, Shaofeng and Zhou, Yi},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={9747--9758},
  year={2021}
}

@book{rubinstein2016simulation,
  title={Simulation and the Monte Carlo method},
  author={Rubinstein, Reuven Y and Kroese, Dirk P},
  year={2016},
  publisher={John Wiley \& Sons}
}



@inproceedings{sutton1987temporal,
  title={A temporal-difference model of classical conditioning},
  author={Sutton, Richard S and Barto, Andrew G},
  booktitle={Proceedings of the ninth annual conference of the cognitive science society},
  pages={355--378},
  year={1987},
  organization={Seattle, WA}
}

@article{kumar2023sample,
  title={On the sample complexity of actor-critic method for reinforcement learning with function approximation},
  author={Kumar, Harshat and Koppel, Alec and Ribeiro, Alejandro},
  journal={Machine Learning},
  pages={1--35},
  year={2023},
  publisher={Springer}
}

@ARTICLE{Barto1983,
  author={Barto, Andrew G. and Sutton, Richard S. and Anderson, Charles W.},
  journal={IEEE Transactions on Systems, Man, and Cybernetics}, 
  title={Neuronlike adaptive elements that can solve difficult learning control problems}, 
  year={1983},
  volume={SMC-13},
  number={5},
  pages={834-846},
  doi={10.1109/TSMC.1983.6313077}}


@article{chen2022finite,
  title={Finite-sample analysis of off-policy natural actor--critic with linear function approximation},
  author={Chen, Zaiwei and Khodadadian, Sajad and Maguluri, Siva Theja},
  journal={IEEE Control Systems Letters},
  volume={6},
  pages={2611--2616},
  year={2022},
  publisher={IEEE}
}


@article{konda1999actor,
  title={Actor-critic algorithms},
  author={Konda, Vijaymohan},
  journal={PhD thesis, Department of Electrical Engineering and Computer
Science, Massachusetts Institute of Technology},
  year={2002}
}

@inproceedings{du2019gradient,
  title={Gradient descent finds global minima of deep neural networks},
  author={Du, Simon and Lee, Jason and Li, Haochuan and Wang, Liwei and Zhai, Xiyu},
  booktitle={International conference on machine learning},
  pages={1675--1685},
  year={2019},
  organization={PMLR}
}


@inproceedings{miyatospectral,
  title={Spectral Normalization for Generative Adversarial Networks},
  author={Miyato, Takeru and Kataoka, Toshiki and Koyama, Masanori and Yoshida, Yuichi},
  booktitle={International Conference on Learning Representations},
year={2018},
}


@article{neyshabur2017implicit,
  title={Implicit regularization in deep learning},
  author={Neyshabur, Behnam},
  journal={arXiv preprint arXiv:1709.01953},
  year={2017}
}


@article{heidergott2003taylor,
  title={Taylor series expansions for stationary Markov chains},
  author={Heidergott, Bernd and Hordijk, Arie},
  journal={Advances in Applied Probability},
  volume={35},
  number={4},
  pages={1046--1070},
  year={2003},
  publisher={Cambridge University Press}
}


@string{aap = "Adv. Appl. Prob."}
@string{amstat = "Ann. Math. Statist."}
@string{amst="Amer. Math. Soc. Transl."}
@string{allerton="Proc. Annu. Allerton Conf. Communication, Control and Computing"}
@string{arc = "Automat. Remote Contr."}
@string{asilomar = "Proc. Asilomar Conf. Signals, Systems and Computers"}
@string{atttj = "AT\& T Tech. J."}
@string{automatica = "Automatica"}
@string{bstj = "Bell Syst. Tech. J."}
@string{ciss="Proc. Conf. on Information Sciences and Systems (CISS)"}
@string{el = "Electron. Lett."}
@string{ett = "European Trans. on Telecommunications"}
@string{iandc = "Inform. and Comput."}
@string{ibmjrd = "{IBM} J. Res. Dev."}
@string{ieeecommag = "IEEE Commun. Mag."}
@string{ieeejsac = "IEEE J. Sel. Areas Commun."}
@string{ieeepcom = "IEEE Pers. Commun. Mag."}
@string{ieeetac = "IEEE Trans. Automat. Contr. "}
@string{ieeetaes = "IEEE Trans. Aerosp.  Electron. Sys."}
@string{ieeetassp = "IEEE Trans. Acoust., Speech, Signal Proc."}
@string{ieeetcom = "IEEE Trans. Commun."}
@string{ieeetcomt = "IEEE Trans. Commun. Tech."}
@string{ieeetit = "IEEE Trans. Inform. Theory"}
@string{ieeetmag = "IEEE Trans. Magnet."}
@string{ieeetsp = "IEEE Trans. Signal Proc."}
@string{ieeetsmc = "IEEE Trans. Syst., Man, Cybern."}
@string{ieeetvt = "IEEE Trans. Veh. Technol."}
@string{ieeetwireless = "IEEE Trans. Wireless Commun."}
@string{ieeeacmtn = "IEEE/ACM  Trans. Network."}
@string{infocom = "Proc. IEEE Inforcom."}
@string{iretit = "IRE Trans. Inform. Th."}
@string{ijwin = "Internat. J. Wireless Inform. Networks"}
@string{ipsn = "International Workshop on Information Processing in Sensor Networks (IPSN)"}
@string{isit="Proc. IEEE Int. Symp. Information Theory (ISIT)"}
@string{jap = "J. Appl. Phys."}
@string{japrob = "J. Appl. Prob."}
@string{jfa = "J. Functional Anal."}
@string{jfi = "J. Franklin Inst."}
@string{jmaa = "J. Math. Anal. Appl."}
@string{jrss = "J. Roy. Statist. Soc."}
@string{jrsss = "J. Roy. Statist. Soc. Suppl."}
@string{jts = "J. Time Ser. Anal."}
@string{mcss = "Math. Contr. Signals Syst."}
@string{pit = "Probl. Inform. Transm."}
@string{prociee = "Proc. IEE"}
@string{procieee = "Proc. IEEE"}
@string{ptrf = "Probab. Th. Rel. Fields"}
@string{reep = "Radio Eng. Electron. Phys."}
@string{siamjam = "SIAM J. Appl. Math."}
@string{siamjco = "SIAM J. Control Optim."}
@string{sp = "Signal Proc."}
@string{spawc = "IEEE Int. Workshop on Signal Processing Advances for Wireless Communications (SPAWC)"}
@string{tpa = "Theory of Prob. and App."}
@string{wirelesscom= "Proc. WirelessCom, Symp. Inform. Theory"}
@string{wpc = "Wireless Pers. Commun."}
@string{vtc="Vehicular Technology Conference"}

%%%%%%%%%%%%%%%%%%%%%%%%%%%
@string{nips="Proc. Advances in Neural Information Processing Systems (NIPS)"}
@string{nipsnew="Proc. Advances in Neural Information Processing Systems (NeurIPS)"}
@string{uai="Proc. International Conference on Uncertainty in Artificial Intelligence (UAI)"}
@string{icml="Proc. International Conference on Machine Learning (ICML)"}
@string{aaai="Proc. AAAI Conference on Artificial Intelligence (AAAI)"}
@string{aistats="Proc. International Conference on Artifical Intelligence and Statistics (AISTATS)"}
@string{colt="Proc. Annual Conference on Learning Theory (CoLT)"}
@string{iclr="Proc. International Conference on Learning Representations (ICLR)"}
@string{astat = "Ann. Statist."}
@string{jasa = "J. Amer. Stat. Assoc."}
@string{jmlr = "J. Mach. Learn. Res."}

@article{bertsekas2011dynamic,
  title={{Dynamic Programming and Optimal Control 3rd edition, volume II}},
  author={Bertsekas, Dimitri P},
  journal={Belmont, MA: Athena Scientific},
  year={2011}
}


@inproceedings{melo2008analysis,
	title={An analysis of reinforcement learning with function approximation},
	author={Melo, Francisco S and Meyn, Sean P and Ribeiro, M Isabel},
	booktitle=icml,
	pages={664--671},
	year={2008},
	organization={ACM}
}
@book{meyn2012markov,
	title={Markov Chains and Stochastic Stability},
	author={Meyn, Sean P and Tweedie, Richard L},
	year={2012},
	publisher={Springer Science \& Business Media}
}

@inproceedings{bhandari2018finite,
  title={A finite time analysis of temporal difference learning with linear function approximation},
  author={Bhandari, Jalaj and Russo, Daniel and Singal, Raghav},
  booktitle=colt,
  pages={1691--1692},
  year={2018},
 
}
 
@article{kushner2010stochastic,
	title={Stochastic approximation: a survey},
	author={Kushner, Harold},
	journal={Wiley Interdisciplinary Reviews: Computational Statistics},
	volume={2},
	number={1},
	pages={87--96},
	year={2010},
	publisher={Wiley Online Library}
}
@article{lacoste2012simpler,
	title={A simpler approach to obtaining an $O(1/t)$ convergence rate for the projected stochastic subgradient method},
	author={Lacoste-Julien, Simon and Schmidt, Mark and Bach, Francis},
	journal={arXiv preprint arXiv:1212.2002},
	year={2012}
}
@article{bubeck2015convex,
	title={Convex optimization: Algorithms and complexity},
	author={Bubeck, S{\'e}bastien},
	journal={Foundations and Trends{\textregistered} in Machine Learning},
	volume={8},
	number={3-4},
	pages={231--357},
	year={2015},
	publisher={Now Publishers, Inc.}
}
@article{nemirovski2009robust,
	title={Robust stochastic approximation approach to stochastic programming},
	author={Nemirovski, Arkadi and Juditsky, Anatoli and Lan, Guanghui and Shapiro, Alexander},
	journal={SIAM Journal on Optimization},
	volume={19},
	number={4},
	pages={1574--1609},
	year={2009},
	publisher={SIAM}
}
@ARTICLE{Tsitsiklis1997, 
	author={J. N. Tsitsiklis and B.  Roy}, 
	journal={IEEE Transactions on Automatic Control}, 
	title={An analysis of temporal-difference learning with function approximation}, 
	year={1997}, 
	volume={42}, 
	number={5}, 
	pages={674-690},
	month={May},}
@inproceedings{perkins2002existence,
	title={On the existence of fixed points for {Q}-learning and {Sarsa} in partially observable domains},
	author={Perkins, Theodore J and Pendrith, Mark D},
	booktitle=icml,
	pages={490--497},
	year={2002}
}
@article{de2000existence,
	title={On the existence of fixed points for approximate value iteration and temporal-difference learning},
	author={De Farias, Daniela Pucci and Van Roy, Benjamin},
	journal={Journal of Optimization theory and Applications},
	volume={105},
	number={3},
	pages={589--608},
	year={2000},
	publisher={Springer}
}
@inproceedings{perkins2003convergent,
	title={A convergent form of approximate policy iteration},
	author={Perkins, Theodore J and Precup, Doina},
	booktitle=nips,
	pages={1627--1634},
	year={2003}
}
@article{mitrophanov2005sensitivity,
	title={Sensitivity and convergence of uniformly ergodic {M}arkov chains},
	author={Mitrophanov, A. Y.},
	journal={Journal of Applied Probability},
	volume={42},
	number={4},
	pages={1003--1014},
	year={2005},
	publisher={Cambridge University Press}
}
@book{benveniste2012adaptive,
	title={Adaptive Algorithms and Stochastic Approximations},
	author={Benveniste, Albert and M{\'e}tivier, Michel and Priouret, Pierre},
	volume={22},
	year={2012},
	publisher={Springer Science \& Business Media}
}
@article{sutton1988learning,
	title={Learning to predict by the methods of temporal differences},
	author={Sutton, Richard S.},
	journal={Machine learning},
	volume={3},
	number={1},
	pages={9--44},
	year={1988},
	publisher={Springer}
}
@article{doan2019finite,
  title={Finite-time analysis and restarting scheme for linear two-time-scale stochastic approximation},
  author={Doan, Thinh T},
  journal={arXiv preprint arXiv:1912.10583},
  year={2019}
}

@inproceedings{perolat2015approximate,
	title={Approximate dynamic programming for two-player zero-sum {M}arkov games},
	author={Perolat, Julien and Scherrer, Bruno and Piot, Bilal and Pietquin, Olivier},
	booktitle=icml,
	year={2015}
}
@article{singh2000convergence,
	title={Convergence results for single-step on-policy reinforcement-learning algorithms},
	author={Singh, Satinder and Jaakkola, Tommi and Littman, Michael L and Szepesv{\'a}ri, Csaba},
	journal={Machine Learning},
	volume={38},
	number={3},
	pages={287--308},
	year={2000},
	publisher={Springer}
}
@article{gordon1996chattering,
  title={Chattering in {SARSA} ($\lambda$)},
  author={Gordon, Geoffrey J},
  journal={CMU Learning Lab Technical Report},
  year={1996}
}

@InProceedings{srikant2019finite, 
title = {Finite-Time Error Bounds For Linear Stochastic Approximation and {TD} Learning}, 
author = {Srikant, R. and Ying, Lei}, 
booktitle = colt, 
pages = {2803--2830}, 
year = {2019}, 
month = {Jun.},  }



@inproceedings{Asadi2016,
  title={An alternative softmax operator for reinforcement learning},
  author={Asadi, Kavosh and Littman, Michael L},
  booktitle={Proceedings of the 34th International Conference on Machine Learning},
  volume={70},
  pages={243--252},
  year={2017},
  organization={JMLR. org}
}


@inproceedings{gordon2000reinforcement,
	title={Reinforcement Learning with Function Approximation Converges to a Region},
	author={Gordon, Geoffrey J},
	pages={1040--1046},
	year={2001},
booktitle=nips}


@inproceedings{korda2015td,
  title={On {TD}(0) with function approximation: Concentration bounds and a centered variant with exponential convergence},
  author={Korda, Nathaniel and La, Prashanth},
  booktitle=icml,
  pages={626--634},
  year={2015}
}


@inproceedings{gupta2019finite,
  title={Finite-time performance bounds and adaptive learning rate selection for two time-scale reinforcement learning},
  author={Gupta, Harsh and Srikant, R and Ying, Lei},
  booktitle=nipsnew,
  pages={4706--4715},
  year={2019}
}



@article{Antos2008,
author = "A. Antos and C. Szepesvari and R. Munos",
title = "Learning near-optimal policies with {B}ellman-residual minimization based fitted policy iteration and a single sample path",
journal = "Machine Learning",
volume = "71",
number = "1",
year = "2008",
pages = "89-129"
}

@inproceedings{Bowling2001,
	author = "Bowling, M.",
	title = "Rational and convergent learning in stochastic games",
	booktitle = "Proc.  International Joint Conference on Artificial intelligence (IJCAI)",
	year = "2001",
} 

@article{Boyan2002,
author = "Boyan, J. A.",
title = "Technical update: {L}east-squares temporal difference learning",
journal = "Machine Learning",
volume = "49",
year = "2002",
pages = "233-246"
}

@article{Brad1996,
author = "Bradtke, S. J. and Barto, A. G.",
title = "Linear least-squares algorithms for temporal difference learning",
journal = "Machine Learning",
volume = "22",
year = "1996",
pages = "33-57"
}

@article{Coni2007,
author = "Conitzer, V. and Sandholm, T.",
title = "{AWESOME: A} general multiagent learning algorithm that converges in self-play and learns a best response against stationary opponents",
journal = "Machine Learning",
volume = "67",
year = "2007",
pages = "23-43"
}

@article{Devraj2018,
	author = {A. M. Devraj and I. Kontoyiannis and S. P. Meyn},
	title = {Differential Temporal Difference Learning},
	journal = {ArXiv: 1812.11137},
	year = 2018,
	month = "Dec.",
}

@inproceedings{Dalal2018a,
	author = "G. Dalal and B. Szrnyi and G. Thoppe and S. Mannor",
	title = "Finite sample analyses for {TD}(0) with function approximation",
	booktitle = "Proc. AAAI Conference on Artificial Intelligence (AAAI)",
	year = "2018",
	pages= "6144-6160"
} 



@inproceedings{Farah2010,
	author = "Farahmand, A.-M. and Szepesvari, C. and Munos, R.",
	title = "Error propagation for approximate policy and value iteration",
	booktitle = nips,
	year = "2010",
} 

@inproceedings{Ghav2010,
	author = "Mohammad Ghavamzadeh and Alessandro Lazaric and Odalric Maillard and Remi Munos",
	title = "{LSTD} with Random Projections",
	booktitle = nips,
	year = "2010",
} 

@article{Lagou2003,
author = "Lagoudakis, M. G. and Parr, R.",
title = "Least-squares policy iteration",
journal = "Journal of Machine Learning Research",
volume = "4",
year = "2003",
pages = "1107-1149"
}

@inproceedings{Lagou2002,
	author = "Lagoudakis, M. G. and Parr, R.",
	title = "Value function approximation in zero-sum {M}arkov games",
	booktitle = "Proc. Uncertainty in Artificial Intelligence (UAI)",
	year = "2002",
} 

@inproceedings{lakshminarayanan2018linear,
  title={Linear stochastic approximation: {H}ow far does constant step-size and iterate averaging go?},
  author={Lakshminarayanan, Chandrashekar and Szepesvari, Csaba},
  booktitle={Proc. International Conference on Artificial Intelligence and Statistics},
  pages={1347--1355},
  year={2018}
}


@inproceedings{Lazaric2010,
	author = "Alessandro Lazaric and Mohammad Ghavamzadeh and Remi Munos",
	title = "Finite-sample analysis of LSTD",
	booktitle = icml,
	year = "2010",
} 

@article{Lazaric2012,
author = "Lazaric, A. and Ghavamzadeh, M. and Munos, R.",
title = "Finite-sample analysis of least-squares policy iteration",
journal = "Journal of Machine Learning Research",
volume = "13",
year = "2012",
pages = "3041-3074"
}

@article{Lazaric2016,
author = "Lazaric, A. and Ghavamzadeh, M. and Munos, R.",
title = "Analysis of classification-based policy iteration algorithms",
journal = "Journal of Machine Learning Research",
volume = "17",
year = "2016",
pages = "583-612"
}

@inproceedings{Littman1994,
	author = "Littman, M. L.",
	title = "Markov games as a framework for multi-agent reinforcement learning",
	booktitle = icml,
	year = "1994",
} 




@article{Minh2015,
	author = "Mnih, V. and Kavukcuoglu, K. and Silver, D. and Rusu, A. A. and Veness, J. and Bellemare, M. G. and Graves, A. and Riedmiller, M. and Fidjeland, A. K. and Ostrovski, G.",
	title = "Human-level control through deep reinforcement learning",
	journal = "Nature",
	volume = "518",
	year = "2015",
	pages = "529-533"
} 

@inproceedings{Minh2016,
	author = "Mnih, V. and Badia, A. P. and Mirza, M. and Graves, A. and Lillicrap, T. and Harley, T. and Silver, D. and Kavukcuoglu, K.",
	title = "Asynchronous methods for deep reinforcement learning",
	booktitle = icml,
	pages={1928--1937},
	year = "2016",
} 


@article{Munos2008,
author = "R. Munos and C. Szepesvari",
title = "Finite-time bounds for fitted value iteration",
journal = "Journal of Machine Learning Research",
volume = "9",
month = may,
year = "2008",
pages = "815-857"
}

@article{Ormo2002a,
author = "D. Ormoneit and P. Glynn",
title = "Kernel-based reinforcement learning in average-cost problems",
journal = "IEEE Trans. Automatic Control",
volume = "47",
number ="10",
year = "2002",
pages = "1624-1636"
}

@article{Ormo2002b,
author = "Dirk Ormoneit and Saunak Sen",
title = "Kernel-based reinforcement learning",
journal = "Mach. Learning",
volume = "49",
number = "2-3",
year = "2002",
pages = "161-178"
}

@inproceedings{Pires2012,
	author = "Bernardo A. Pires and Csaba Szepesvari",
	title = "Statistical linear estimation with penalized estimators: {A}n application to reinforcement learning",
	booktitle = icml,
	year = "2012",
} 

@inproceedings{Perolat2016a,
	author = "Perolat, J. and Piot, B. and Geist, M. and Scherrer, B. and Pietquin, O. ",
	title = "Softened approximate policy iteration for {M}arkov games",
	booktitle =icml,
	year = "2016",
} 

@inproceedings{Perolat2016b,
	author = "Perolat, J. and Piot and B. and Scherrer B. and Pietquin, O.",
	title = "On the use of non-stationary strategies for solving two-player zero-sum {M}arkov games",
	booktitle = aistats,
	year = "2016",
} 

@inproceedings{Perolat2018,
	author = "Perolat, J. and Piot, B. and Pietquin, O.",
	title = "Actor-critic fictitious play in simultaneous move multistage games",
	booktitle = aistats,
	year = "2018",
} 

@inproceedings{Prasad2015,
	author = "H. L. Prasad and Prashanth L.A. and Shalabh Bhatnagar",
	title = "Two-timescale algorithms for learning {N}ash equilibria in general-sum stochastic games",
	booktitle = "Proc.  International Conference on Autonomous Agents and Multiagent Systems (AAMAS)",
	year = "2015",
} 

@inproceedings{Prash2013,
	author = "La Prashanth and Nathaniel Korda and Remi Munos",
	title = "Fast {LSTD} using stochastic approximation: {F}inite time analysis and application to traffic control",
	booktitle = "Proc. Joint European Conference on Machine Learning and Knowledge Discovery in Databases",
	year = "2013",
} 

@article{Rummery1994,
	author = {G. A. Rummery and M. Niranjan},
	title = {Online {Q}-learning using connectionist systems},
	journal = {Technical Report, {Cambridge University Engineering Department}},
	year = 1994,
	month = sep,
}

@inproceedings{Shah2018,
	author = "Devavrat Shah and Qiaomin Xie",
	title = "{Q}-learning with Nearest Neighbors",
	booktitle = nipsnew,
	year = "2018",
} 

@inproceedings{Srini2018,
	author = "Srinivasan, S. and Lanctot, M. and Zambaldi, V. and Perolat, J. and Tuyls, K. and Munos, R. and Bowling, M.",
	title = "Actor-critic policy optimization in partially observable multiagent environments",
	booktitle = nipsnew,
	year = "2018",
} 

@article{Sutton1988,
author = "Richard S Sutton",
title = "Learning to predict by the methods of temporal differences",
journal = "Machine Learning",
volume = "3",
number = "1",
year = "1988",
pages = "9-44"
}




@inproceedings{sutton2009acov,
  title={A convergent $ {O} (n) $ temporal-difference algorithm for off-policy learning with linear function approximation},
  author={Sutton, Richard S and Maei, Hamid R and Szepesv{\'a}ri, Csaba},
  booktitle=nips,
  pages={1609--1616},
  year={2009}
}


@inproceedings{Tagorti2015,
	author = "Tagorti, M. and Scherrer, B.",
	title = "On the rate of convergence and error bounds for {LSTD} ($\lambda$)",
	booktitle = icml,
	year = "2015",
} 

@inproceedings{Touati2018,
	author = "Ahmed Touati and Pierre-Luc Bacon and Doina Precup and Pascal Vincent",
	title = "Convergent {TREE BACKUP} and {RETRACE} with function approximation",
	booktitle = icml,
	year = "2018",
} 

@inproceedings{Tu2018,
	author = "Stephen Tu and Benjamin Recht",
	title = "Least-squares temporal difference learning for the linear quadratic regulator",
	booktitle = icml,
	year = "2018",
} 



@inproceedings{Wei2017,
	author = "Wei, C.-Y. and Hong and Y.-T. and Lu, C.-J.",
	title = "Online reinforcement learning in stochastic games",
	booktitle = nips,
	year = "2017",
} 

@article{Yang2019,
	author = {Zhuora Yang and Yuchen Xie and Zhaoran Wang},
	title = {A Theoretical Analysis of Deep {Q}-Learning},
	journal = {ArXiv: 1901.00137},
	year = 2019,
	month = jan,
}

@article{Zhang2018,
	author = {Zhang, K. and Yang, Z. and Liu, H. and Zhang, T. and Basar, T.},
	title = {Finite-sample analyses for fully decentralized multi-agent reinforcement learning},
	journal = {arXiv:1812.02783},
	year = 2018,
}

############################################
# References about  cubic regularization
############################################
@article{Nesterov2006,
	author = "Nesterov, Y. and  Polyak, B.",
	journal = "Mathematical Programming",
	title = "Cubic regularization of {N}ewton's method and its global performance",
	year = "2006"
}

@ARTICLE{Carmon2016,
	author = {{Carmon}, Y. and {Duchi}, J.~C.},
	title = "{Gradient descent efficiently finds the cubic-regularized non-convex Newton step}",
	journal = {ArXiv: 1612.00547},
	year = 2016,
	month = "Dec.",
}
@InProceedings{gu2018,
	author = {{Zhou}, D. and {Xu}, P. and {Gu}, Q.},
	title = {Stochastic Variance-Reduced Cubic Regularized Newton Method},
	booktitle = {Proc. International Conference on Machine Learning (ICML)},
	year = 2018,  
}

@book{Lojasiewicz_book,
	title={Ensembles semi-analytiques},
	author={{\L}ojasiewicz, S.},
	publisher={Bures-sur-Yvette: Institut des Hautes Etudes Scientifiques},
	year={1965}
}


@book{Nesterov_2014,
	author = {Nesterov, Y.},
	title = {Introductory lectures on convex optimization: A Basic Course},
	year = {2014},
	publisher = {Springer},
} 

@article{Nemirovski_2009,
author = "Nemirovski, A. S. and Juditsky, A. and Lan, G. and Shapiro, A.",
title = "Robust stochastic approximation approach to stochastic programming",
journal = "SIAM Journal on Optimization",
volume = 19,
year = 2009,
pages = "1574-1609"
}

@inproceedings{allen2019convergence,
  title={A Convergence Theory for Deep Learning via Over-Parameterization},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  booktitle={International Conference on Machine Learning},
  pages={242--252},
  year={2019}
}


@article{dalal2018finite,
  title={Finite Sample Analysis of Two-Timescale Stochastic Approximation with Applications to Reinforcement Learning},
  author={Dalal, Gal and Sz{\"o}r{\'e}nyi, Bal{\'a}zs and Thoppe, Gugan and Mannor, Shie},
  journal={Proceedings of Machine Learning Research},
  volume={75},
  pages={1--35},
  year={2018}
}



@incollection{baird1995residual,
  title={Residual algorithms: Reinforcement learning with function approximation},
  author={Baird, Leemon},
  booktitle={Machine Learning Proceedings},
  pages={30--37},
  year={1995},
  publisher={Elsevier}
}


@article{maei2011gradient,
  title={Gradient temporal-difference learning algorithms},
  author={Maei, Hamid Reza},
  year={2011},
  journal={Thesis, University of Alberta},
}

@inproceedings{sutton2009fast,
  title={Fast gradient-descent methods for temporal-difference learning with linear function approximation},
  author={Sutton, Richard S and Maei, Hamid Reza and Precup, Doina and Bhatnagar, Shalabh and Silver, David and Szepesv{\'a}ri, Csaba and Wiewiora, Eric},
  booktitle=icml,
  pages={993--1000},
  year={2009}
}

@inproceedings{antos2008fitted,
  title={Fitted {Q}-iteration in continuous action-space MDPs},
  author={Antos, Andr{\'a}s and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
  booktitle={Advances in neural information processing systems},
  pages={9--16},
  year={2008}
}

@article{antos2008learning,
  title={Learning near-optimal policies with Bellman-residual minimization based fitted policy iteration and a single sample path},
  author={Antos, Andr{\'a}s and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
  journal={Machine Learning},
  volume={71},
  number={1},
  pages={89--129},
  year={2008},
  publisher={Springer}
}

@incollection{gordon1995stable,
  title={Stable function approximation in dynamic programming},
  author={Gordon, Geoffrey J},
  booktitle={Machine Learning Proceedings 1995},
  pages={261--268},
  year={1995},
  publisher={Elsevier}
}

@article{lagoudakis2003least,
  title={Least-squares policy iteration},
  author={Lagoudakis, Michail G and Parr, Ronald},
  journal={Journal of machine learning research},
  volume={4},
  number={Dec},
  pages={1107--1149},
  year={2003}
}

@inproceedings{szepesvari2004interpolation,
  title={Interpolation-based {Q}-learning},
  author={Szepesv{\'a}ri, Csaba and Smart, William D},
  booktitle=icml,
  pages={100},
  year={2004}
}


@inproceedings{maei2010toward,
  title={Toward off-policy learning control with function approximation},
  author={Maei, Hamid Reza and Szepesv{\'a}ri, Csaba and Bhatnagar, Shalabh and Sutton, Richard S},
  booktitle=icml,
  pages={719--726},
  year={2010}
}



@article{borkar2000ode,
  title={The ODE method for convergence of stochastic approximation and reinforcement learning},
  author={Borkar, Vivek S and Meyn, Sean P},
  journal={SIAM Journal on Control and Optimization},
  volume={38},
  number={2},
  pages={447--469},
  year={2000},
  publisher={SIAM}
}

@article{ghadimi2013stochastic,
  title={Stochastic first- and zeroth-order methods for nonconvex stochastic programming},
  author={Ghadimi, Saeed and Lan, Guanghui},
  journal={SIAM Journal on Optimization},
  volume={23},
  number={4},
  pages={2341--2368},
  year={2013},
  publisher={SIAM}
}





@book{borkar2009stochastic,
  title={Stochastic approximation: a dynamical systems viewpoint},
  author={Borkar, Vivek S},
  volume={48},
  year={2009},
  publisher={Springer}
}


@article{watkins1992q,
  title={{Q}-learning},
  author={Watkins, Christopher JCH and Dayan, Peter},
  journal={Machine learning},
  volume={8},
  number={3-4},
  pages={279--292},
  year={1992},
  publisher={Springer}
}





@inproceedings{borkar2018concentration,
  title={Concentration bounds for two time scale stochastic approximation},
  author={Borkar, Vivek S and Pattathil, Sarath},
  booktitle=allerton,
  pages={504--511},
  year={2018},
  organization={IEEE}
}

@article{kober2013reinforcement,
  title={Reinforcement Learning in Robotics: A Survey},
  author={Kober, Jens and Bagnell, J Andrew and Peters, Jan},
  journal={The International Journal of Robotics Research},
  volume={32},
  number={11},
  pages={1238--1274},
  year={2013},
  publisher={SAGE Publications Sage UK: London, England}
}

@article{yu2017convergence,
  title={On convergence of some gradient-based temporal-differences algorithms for off-policy learning},
  author={Yu, Huizhen},
  journal={arXiv preprint arXiv:1712.09652},
  year={2017}
}
@article{karmakar2018two,
  title={Two time-scale stochastic approximation with controlled {Markov} noise and off-policy temporal-difference learning},
  author={Karmakar, Prasenjit and Bhatnagar, Shalabh},
  journal={Mathematics of Operations Research},
  volume={43},
  number={1},
  pages={130--151},
  year={2018},
  publisher={INFORMS}
}
@inproceedings{wang2017finite,
  title={Finite sample analysis of the GTD policy evaluation algorithms in {M}arkov setting},
  author={Wang, Yue and Chen, Wei and Liu, Yuting and Ma, Zhi-Ming and Liu, Tie-Yan},
  booktitle=nips,
  pages={5504--5513},
  year={2017}
}
@inproceedings{liu2015finite,
  title={Finite-Sample Analysis of Proximal Gradient TD Algorithms.},
  author={Liu, Bo and Liu, Ji and Ghavamzadeh, Mohammad and Mahadevan, Sridhar and Petrik, Marek},
  booktitle=uai,
  pages={504--513},
  year={2015},

}


@article{chen2019performance,
  title={Performance of {Q}-learning with linear function approuimation: Stability and finite-time analysis},
  author={Chen, Zaiwei and Zhang, Sheng and Doan, Thinh T and Maguluri, Siva Theja and Clarke, John-Paul},
  journal={arXiv preprint arXiv:1905.11425},
  year={2019}
}
@inproceedings{fang2018spider,
  title={Spider: Near-optimal non-convex optimization via stochastic path-integrated differential estimator},
  author={Fang, Cong and Li, Chris Junchi and Lin, Zhouchen and Zhang, Tong},
  booktitle=nipsnew,
  pages={689--699},
  year={2018}
}
@article{lan2019lectures,
  title={Lectures on Optimization. Methods for Machine Learning},
  author={Lan, G},
  journal={H. Milton Stewart School of Industrial and Systems Engineering, Georgia Institute of Technology, Atlanta, GA},
  year={2019}
}
@book{nesterov2013introductory,
  title={Introductory Lectures on Convex Optimization: A Basic Course},
  author={Nesterov, Yurii},
  volume={87},
  year={2013},
  publisher={Springer Science \& Business Media}
}

@article{jaganathan2015phase,
  title={Phase retrieval: An overview of recent developments},
  author={Jaganathan, Kishore and Eldar, Yonina C and Hassibi, Babak},
  journal={arXiv preprint arXiv:1510.07713},
  year={2015}
}
@inproceedings{zhang2016reshaped,
  title={Reshaped wirtinger flow for solving quadratic system of equations},
  author={Zhang, Huishuai and Liang, Yingbin},
  booktitle=nips,
  pages={2622--2630},
  year={2016}
}
@article{candes2015phase,
  title={Phase retrieval via Wirtinger flow: Theory and algorithms},
  author={Candes, Emmanuel J and Li, Xiaodong and Soltanolkotabi, Mahdi},
  journal={IEEE Transactions on Information Theory},
  volume={61},
  number={4},
  pages={1985--2007},
  year={2015},
  publisher={IEEE}
}
@article{chen2019gradient,
  title={Gradient descent with random initialization: Fast global convergence for nonconvex phase retrieval},
  author={Chen, Yuxin and Chi, Yuejie and Fan, Jianqing and Ma, Cong},
  journal={Mathematical Programming},
  volume={176},
  number={1-2},
  pages={5--37},
  year={2019},
  publisher={Springer}
}




@article{konda2004convergence,
  title={Convergence rate of linear two-time-scale stochastic approximation},
  author={Konda, Vijay R and Tsitsiklis, John N and others},
  journal={The Annals of Applied Probability},
  volume={14},
  number={2},
  pages={796--819},
  year={2004},
  publisher={Institute of Mathematical Statistics}
}



@article{mokkadem2006convergence,
  title={Convergence rate and averaging of nonlinear two-time-scale stochastic approximation algorithms},
  author={Mokkadem, Abdelkader and Pelletier, Mariane and others},
  journal={The Annals of Applied Probability},
  volume={16},
  number={3},
  pages={1671--1702},
  year={2006},
  publisher={Institute of Mathematical Statistics}
}

@article{doan2020nonlinear,
  title={Nonlinear Two-Time-Scale Stochastic Approximation: Convergence and Finite-Time Performance},
  author={Doan, Thinh T},
  journal={arXiv preprint arXiv:2011.01868},
  year={2020}
}

@article{xu2020sample,
  title={Sample Complexity Bounds for Two Timescale Value-based Reinforcement Learning Algorithms},
  author={Xu, Tengyu and Liang, Yingbin},
  journal={arXiv preprint arXiv:2011.05053},
  year={2020}
}

@article{perkins2002convergent,
  title={A convergent form of approximate policy iteration},
  author={Perkins, Theodore and Precup, Doina},
  journal={Advances in neural information processing systems},
  volume={15},
  pages={1627--1634},
  year={2002}
}

@inproceedings{kaledin2020finite,
  title={Finite time analysis of linear two-timescale stochastic approximation with {M}arkovian noise},
  author={Kaledin, Maxim and Moulines, Eric and Naumov, Alexey and Tadic, Vladislav and Wai, Hoi-To},
  booktitle=colt,
  pages={2144--2203},
  year={2020},
 
}

 

@article{robbins1951stochastic,
  title={A stochastic approximation method},
  author={Robbins, Herbert and Monro, Sutton},
  journal={The annals of mathematical statistics},
  pages={400--407},
  year={1951},
  publisher={JSTOR}
}

@incollection{benaim1999dynamics,
  title={Dynamics of stochastic approximation algorithms},
  author={Bena{\"\i}m, Michel},
  booktitle={Seminaire de probabilites XXXIII},
  pages={1--68},
  year={1999},
  publisher={Springer}
}

@book{kushner2003stochastic,
  title={Stochastic approximation and recursive algorithms and applications},
  author={Kushner, Harold and Yin, G George},
  volume={35},
  year={2003},
  publisher={Springer Science \& Business Media}
}


@article{peters2008natural,
  title={Natural actor-critic},
  author={Peters, Jan and Schaal, Stefan},
  journal={Neurocomputing},
  volume={71},
  number={7-9},
  pages={1180--1190},
  year={2008},
  publisher={Elsevier}
}

@inproceedings{kakade2001natural,
  title={A natural policy gradient},
  author={Kakade, Sham M},
  booktitle=nips,
  volume={14},
  pages={1531--1538},
  year={2001}
}
@article{bhatnagar2009natural,
  title={Natural actor--critic algorithms},
  author={Bhatnagar, Shalabh and Sutton, Richard S and Ghavamzadeh, Mohammad and Lee, Mark},
  journal={Automatica},
  volume={45},
  number={11},
  pages={2471--2482},
  year={2009},
  publisher={Elsevier}
}

@inproceedings{Wang2020Neural,
title={Neural Policy Gradient Methods: Global Optimality and Rates of Convergence},
author={Lingxiao Wang and Qi Cai and Zhuoran Yang and Zhaoran Wang},
booktitle={International Conference on Learning Representations (ICLR)},
year={2020}
}


@article{kumar2019sample,
  title={On the sample complexity of actor-critic method for reinforcement learning with function approximation},
  author={Kumar, Harshat and Koppel, Alec and Ribeiro, Alejandro},
  journal={arXiv preprint arXiv:1910.08412},
  year={2019}
}
@inproceedings{qiu2019finite,
  title={On the finite-time convergence of actor-critic algorithm},
  author={Qiu, Shuang and Yang, Zhuoran and Ye, Jieping and Wang, Zhaoran},
  booktitle={Proc. Optimization Foundations for Reinforcement Learning Workshop at Advances in Neural Information Processing Systems (NeurIPS)},
  year={2019}
}

@article{xu2020non,
  title={Non-asymptotic Convergence Analysis of Two Time-scale (Natural) Actor-Critic Algorithms},
  author={Xu, Tengyu and Wang, Zhe and Liang, Yingbin},
  journal={arXiv preprint arXiv:2005.03557},
  year={2020}
}

@article{wu2020finite,
  title={A finite-time analysis of two time-scale actor-critic methods},
  author={Wu, Yue Frank and Zhang, Weitong and Xu, Pan and Gu, Quanquan},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={17617--17628},
  year={2020}
}

 

@article{brockman2016openai,
  title={{OpenAI Gym}},
  author={Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, John and Tang, Jie and Zaremba, Wojciech},
  journal={arXiv preprint arXiv:1606.01540},
  year={2016}
}

@article{borkar1997stochastic,
  title={Stochastic approximation with two time scales},
  author={Borkar, Vivek S},
  journal={Systems \& Control Letters},
  volume={29},
  number={5},
  pages={291--294},
  year={1997},
  publisher={Elsevier}
}

@inproceedings{tadic2004almost,
  title={Almost sure convergence of two time-scale stochastic approximation algorithms},
  author={Tadic, Vladislav B},
  booktitle={Proceedings of the 2004 American Control Conference},
  volume={4},
  pages={3802--3807},
  year={2004},
  organization={IEEE}
}


@inproceedings{xu2020improving,
  title={Improving sample complexity bounds for (natural) actor-critic algorithms},
  author={Xu, Tengyu and Wang, Zhe and Liang, Yingbin},
  booktitle=nipsnew,
  volume={33},
  year={2020}
}

@inproceedings{dalal2020tale,
  title={A tale of two-timescale reinforcement learning with the tightest finite-time bound},
  author={Dalal, Gal and Szorenyi, Balazs and Thoppe, Gugan},
  booktitle=aaai,
  pages={3701--3708},
  year={2020}
}
@inproceedings{wang2019spiderboost,
  title={SpiderBoost and momentum: Faster variance reduction algorithms},
  author={Wang, Zhe and Ji, Kaiyi and Zhou, Yi and Liang, Yingbin and Tarokh, Vahid},
  booktitle=nipsnew,
  pages={2406--2416},
  year={2019}
}
@inproceedings{bhatnagar2009convergent,
  title={Convergent temporal-difference learning with arbitrary smooth function approximation},
  author={Bhatnagar, Shalabh and Precup, Doina and Silver, David and Sutton, Richard S and Maei, Hamid and Szepesv{\'a}ri, Csaba},
  booktitle=nips,
  volume={22},
  pages={1204--1212},
  year={2009}
}

@inproceedings{xu2020finite,
  title={A finite-time analysis of {Q}-learning with neural network function approximation},
  author={Xu, Pan and Gu, Quanquan},
  booktitle=icml,
  pages={10555--10565},
  year={2020},
 
}
 

@article{cen2020fast,
  title={Fast global convergence of natural policy gradient methods with entropy regularization},
  author={Cen, Shicong and Cheng, Chen and Chen, Yuxin and Wei, Yuting and Chi, Yuejie},
  journal={arXiv preprint arXiv:2007.06558},
  year={2020}
}

@article{bhandari2019global,
  title={Global Optimality Guarantees For Policy Gradient Methods},
  author={Bhandari, Jalaj and Russo, Daniel},
  journal={arXiv e-prints},
  pages={arXiv--1906},
  year={2019}
}

@inproceedings{mei2020global,
  title={On the global convergence rates of softmax policy gradient methods},
  author={Mei, Jincheng and Xiao, Chenjun and Szepesvari, Csaba and Schuurmans, Dale},
  booktitle=icml,
  pages={6820--6829},
  year={2020},
}
 


@inproceedings{ma2020variance,
  title={Variance-Reduced Off-Policy {TDC} Learning: Non-Asymptotic Convergence Analysis},
  author={Ma, Shaocong and Zhou, Yi and Zou, Shaofeng},
  booktitle=nipsnew,
  volume={33},
   pages = {14796--14806},
  year={2020}
}


@article{wai2019variance,
  title={Variance reduced policy evaluation with smooth function approximation},
  author={Wai, Hoi-To and Hong, Mingyi and Yang, Zhuoran and Wang, Zhaoran and Tang, Kexin},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  pages={5784--5795},
  year={2019}
}

@article{sutton2008convergent,
  title={A convergent O (n) algorithm for off-policy temporal-difference learning with linear function approximation},
  author={Sutton, Richard S and Szepesv{\'a}ri, Csaba and Maei, Hamid Reza}
}

@inproceedings{miyato2018spectral,
  title={Spectral Normalization for Generative Adversarial Networks},
  author={Miyato, Takeru and Kataoka, Toshiki and Koyama, Masanori and Yoshida, Yuichi},
  booktitle={Proc. International Conference on Learning Representations (ICLR)},
  year={2018}
}






@article{li2020sample,
  title={Sample complexity of asynchronous {Q}-learning: Sharper analysis and variance reduction},
  author={Li, Gen and Wei, Yuting and Chi, Yuejie and Gu, Yuantao and Chen, Yuxin},
  journal={arXiv preprint arXiv:2006.03041},
  year={2020}
}

@inproceedings{nilim2003robustness,
  title={Robustness in {M}arkov Decision Problems with Uncertain Transition Matrices.},
  author={Nilim, Arnab and El Ghaoui, Laurent},
  booktitle=nips,
  pages={839--846},
  year={2003},
}



@inproceedings{roy2017reinforcement,
  title={Reinforcement learning under model mismatch},
  author={Roy, Aurko and Xu, Huan and Pokutta, Sebastian},
  booktitle=nips,
  pages={3046--3055},
  year={2017}
}



@article{wiesemann2013robust,
  title={Robust {M}arkov decision processes},
  author={Wiesemann, Wolfram and Kuhn, Daniel and Rustem, Ber{\c{c}}},
  journal={Mathematics of Operations Research},
  volume={38},
  number={1},
  pages={153--183},
  year={2013},
  publisher={INFORMS}
}

@article{satia1973markovian,
  title={{M}arkovian decision processes with uncertain transition probabilities},
  author={Satia, Jay K and Lave Jr, Roy E},
  journal={Operations Research},
  volume={21},
  number={3},
  pages={728--740},
  year={1973},
  publisher={INFORMS}
}


@article{vinitsky2020robust,
  title={Robust Reinforcement Learning using Adversarial Populations},
  author={Vinitsky, Eugene and Du, Yuqing and Parvate, Kanaad and Jang, Kathy and Abbeel, Pieter and Bayen, Alexandre},
  journal={arXiv preprint arXiv:2008.01825},
  year={2020}
}

@inproceedings{pinto2017robust,
  title={Robust adversarial reinforcement learning},
  author={Pinto, Lerrel and Davidson, James and Sukthankar, Rahul and Gupta, Abhinav},
  booktitle={International Conference on Machine Learning},
  pages={2817--2826},
  year={2017},
 
}

@article{abdullah2019wasserstein,
  title={Wasserstein robust reinforcement learning},
  author={Abdullah, Mohammed Amin and Ren, Hang and Ammar, Haitham Bou and Milenkovic, Vladimir and Luo, Rui and Zhang, Mingtian and Wang, Jun},
  journal={arXiv preprint arXiv:1907.13196},
  year={2019}
}

@article{hou2020robust,
  title={Robust Reinforcement Learning with {Wasserstein} Constraint},
  author={Hou, Linfang and Pang, Liang and Hong, Xin and Lan, Yanyan and Ma, Zhiming and Yin, Dawei},
  journal={arXiv preprint arXiv:2006.00945},
  year={2020}
}

@article{rajeswaran2016epopt,
  title={Epopt: Learning robust neural network policies using model ensembles},
  author={Rajeswaran, Aravind and Ghotra, Sarvjeet and Ravindran, Balaraman and Levine, Sergey},
  journal={arXiv preprint arXiv:1610.01283},
  year={2016}
}

@inproceedings{atkeson2002nonparametric,
  title={Nonparametric representation of policies and value functions: a trajectory-based approach},
  author={Atkeson, Christopher G and Morimoto, Jun},
  booktitle={Proceedings of the 15th International Conference on Neural Information Processing Systems},
  pages={1643--1650},
  year={2002}
}

@article{morimoto2005robust,
  title={Robust reinforcement learning},
  author={Morimoto, Jun and Doya, Kenji},
  journal={Neural Computation},
  volume={17},
  number={2},
  pages={335--359},
  year={2005},
  publisher={MIT Press}
}

@article{huang2017adversarial,
  title={Adversarial attacks on neural network policies},
  author={Huang, Sandy and Papernot, Nicolas and Goodfellow, Ian and Duan, Yan and Abbeel, Pieter},
  journal={arXiv preprint arXiv:1702.02284},
  year={2017}
}

@article{kos2017delving,
  title={Delving into adversarial attacks on deep policies},
  author={Kos, Jernej and Song, Dawn},
  journal={arXiv preprint arXiv:1705.06452},
  year={2017}
}

@article{lin2017tactics,
  title={Tactics of adversarial attack on deep reinforcement learning agents},
  author={Lin, Yen-Chen and Hong, Zhang-Wei and Liao, Yuan-Hong and Shih, Meng-Li and Liu, Ming-Yu and Sun, Min},
  journal={arXiv preprint arXiv:1703.06748},
  year={2017}
}

@article{pattanaik2017robust,
  title={Robust deep reinforcement learning with adversarial attacks},
  author={Pattanaik, Anay and Tang, Zhenyi and Liu, Shuijing and Bommannan, Gautham and Chowdhary, Girish},
  journal={arXiv preprint arXiv:1712.03632},
  year={2017}
}

@inproceedings{mandlekar2017adversarially,
  title={Adversarially robust policy learning: Active construction of physically-plausible perturbations},
  author={Mandlekar, Ajay and Zhu, Yuke and Garg, Animesh and Fei-Fei, Li and Savarese, Silvio},
  booktitle={Proc. IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},
  pages={3932--3939},
  year={2017},
}

@article{even2003learning,
  title={Learning Rates for {Q}-learning.},
  author={Even-Dar, Eyal and Mansour, Yishay and Bartlett, Peter},
  journal={Journal of machine learning Research},
  volume={5},
  number={1},
  year={2003}
}

@article{beck2012error,
  title={Error bounds for constant step-size {Q}-learning},
  author={Beck, Carolyn L and Srikant, Rayadurgam},
  journal={Systems \& control letters},
  volume={61},
  number={12},
  pages={1203--1208},
  year={2012},
  publisher={Elsevier}
}

@inproceedings{qu2020finite,
  title={Finite-Time Analysis of Asynchronous Stochastic Approximation and $ Q $-Learning},
  author={Qu, Guannan and Wierman, Adam},
  booktitle={Conference on Learning Theory},
  pages={3185--3205},
  year={2020},
 
}

@inproceedings{bertsekas1995neuro,
  title={Neuro-dynamic programming: an overview},
  author={Bertsekas, Dimitri P and Tsitsiklis, John N},
  booktitle={Proceedings of 1995 34th IEEE conference on decision and control},
  volume={1},
  pages={560--564},
  year={1995},
  organization={IEEE}
}

@article{wainwright2019variance,
  title={Variance-reduced $ Q $-learning is minimax optimal},
  author={Wainwright, Martin J},
  journal={arXiv preprint arXiv:1906.04697},
  year={2019}
}

@article{wang2021finite,
  title={Finite-Sample Analysis for Two Time-scale Non-linear {TDC} with General Smooth Function Approximation},
  author={Wang, Yue and Zou, Shaofeng and Zhou, Yi},
  journal={arXiv preprint arXiv:2104.02836},
  year={2021}
}

@article{hub65,
	Author = {P. J. Huber},
	Date-Added = {2020-11-07 09:39:14 -0600},
	Date-Modified = {2020-11-07 10:32:02 -0600},
	Journal = amstat,
	Pages = {1753-1758},
	Title = {A Robust Version of the Probability Ratio Test},
	Volume = 36,
	Year = 1965}

@inproceedings{tamar2014scaling,
  title={Scaling up robust {MDPs} using function approximation},
  author={Tamar, Aviv and Mannor, Shie and Xu, Huan},
  booktitle=icml,
  pages={181--189},
  year={2014}
}
@article{lim2013reinforcement,
  title={Reinforcement learning in robust {M}arkov decision processes},
  author={Lim, Shiau Hong and Xu, Huan and Mannor, Shie},
  journal=nips,
  volume={26},
  pages={701--709},
  year={2013}
}
@inproceedings{nilim2004robustness,
  title={Robustness in {Markov} decision problems with uncertain transition matrices},
  author={Nilim, Arnab and El Ghaoui, Laurent},
  booktitle=nips,
  pages={839--846},
  year={2004}
}
@inproceedings{atkeson2003nonparametric,
  title={Nonparametric representation of policies and value functions: A trajectory-based approach},
  author={Atkeson, Christopher G and Morimoto, Jun},
  booktitle=nips,
  pages={1643--1650},
  year={2003}
}
@article{li2021q,
  title={Is {Q}-Learning Minimax Optimal? A Tight Sample Complexity Analysis},
  author={Li, Gen and Cai, Changxiao and Chen, Yuxin and Gu, Yuantao and Wei, Yuting and Chi, Yuejie},
  journal={arXiv preprint arXiv:2102.06548},
  year={2021}
}
@article{wang2019neural,
  title={Neural policy gradient methods: Global optimality and rates of convergence},
  author={Wang, Lingxiao and Cai, Qi and Yang, Zhuoran and Wang, Zhaoran},
  journal={arXiv preprint arXiv:1909.01150},
  year={2019}
}
@article{zhang2020robust,
  title={Robust Multi-Agent Reinforcement Learning with Model Uncertainty},
  author={Zhang, Kaiqing and Sun, Tao and Tao, Yunzhe and Genc, Sahika and Mallya, Sunil and Basar, Tamer},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}
@article{yin2021near,
  title={Near-optimal offline reinforcement learning via double variance reduction},
  author={Yin, Ming and Bai, Yu and Wang, Yu-Xiang},
  journal={Advances in neural information processing systems},
  volume={34},
  pages={7677--7688},
  year={2021}
}
@article{wang2023sample,
  title={Sample Complexity of Variance-reduced Distributionally Robust Q-learning},
  author={Wang, Shengbo and Si, Nian and Blanchet, Jose and Zhou, Zhengyuan},
  journal={arXiv preprint arXiv:2305.18420},
  year={2023}
}
@article{zhang2020stability,
  title={On the stability and convergence of robust adversarial reinforcement learning: A case study on linear quadratic systems},
  author={Zhang, Kaiqing and Hu, Bin and Basar, Tamer},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}



@misc{rlblogpost,
	title={Deep Reinforcement Learning Doesn't Work Yet},
	author={Irpan, Alex},
	howpublished={\url{https://www.alexirpan.com/2018/02/14/rl-hard.html}},
	year={2018}
}

@misc{li2021safe,
title={Faster Algorithm and Sharper Analysis for Constrained {Markov} Decision Process},
author={Tianjiao Li and Ziwei Guan and Shaofeng Zou and Tengyu Xu and Yingbin Liang and Guanghui Lan},
month={submitted to NeurIPS,},
year=2021,
howpublished={\url{https://www.acsu.buffalo.edu/~szou3/saferl.pdf}}
}
@inproceedings{ding2020natural,
  title={Natural Policy Gradient Primal-Dual Method for Constrained {Markov} Decision Processes},
  author={Ding, Dongsheng and Zhang, Kaiqing and Basar, Tamer and Jovanovic, Mihailo},
  booktitle=nips,
  volume={33},
  year={2020}
}
@inproceedings{paternain2019constrained,
  title={Constrained reinforcement learning has zero duality gap},
  author={Paternain, Santiago and Chamon, Luiz FO and Calvo-Fullana, Miguel and Ribeiro, Alejandro},
  booktitle=nips,
  year={2019},
   volume = {32},
}


@article{kushner2010stochastic,
	title={Stochastic approximation: a survey},
	author={Kushner, Harold},
	journal={Wiley Interdisciplinary Reviews: Computational Statistics},
	volume={2},
	number={1},
	pages={87--96},
	year={2010},
	publisher={Wiley Online Library}
}
@article{lacoste2012simpler,
	title={A simpler approach to obtaining an $O(1/t)$ convergence rate for the projected stochastic subgradient method},
	author={Lacoste-Julien, Simon and Schmidt, Mark and Bach, Francis},
	journal={arXiv preprint arXiv:1212.2002},
	year={2012}
}
@article{bubeck2015convex,
	title={Convex optimization: Algorithms and complexity},
	author={Bubeck, S{\'e}bastien and others},
	journal={Foundations and Trends{\textregistered} in Machine Learning},
	volume={8},
	number={3-4},
	pages={231--357},
	year={2015},
	publisher={Now Publishers, Inc.}
}
@article{nemirovski2009robust,
	title={Robust stochastic approximation approach to stochastic programming},
	author={Nemirovski, Arkadi and Juditsky, Anatoli and Lan, Guanghui and Shapiro, Alexander},
	journal={SIAM Journal on Optimization},
	volume={19},
	number={4},
	pages={1574--1609},
	year={2009},
	publisher={SIAM}
}
@ARTICLE{Tsitsiklis1997, 
	author={J. N. Tsitsiklis and B.  Roy}, 
	journal={IEEE Transactions on Automatic Control}, 
	title={An analysis of temporal-difference learning with function approximation}, 
	year={1997}, 
	volume={42}, 
	number={5}, 
	pages={674-690},
	month={May},}
@inproceedings{perkins2002existence,
	title={On the existence of fixed points for {Q}-learning and {Sarsa} in partially observable domains},
	author={Perkins, Theodore J and Pendrith, Mark D},
	booktitle=icml,
	pages={490--497},
	year={2002}
}
@article{de2000existence,
	title={On the existence of fixed points for approximate value iteration and temporal-difference learning},
	author={De Farias, Daniela Pucci and Van Roy, Benjamin},
	journal={Journal of Optimization theory and Applications},
	volume={105},
	number={3},
	pages={589--608},
	year={2000},
	publisher={Springer}
}

@article{mitrophanov2005sensitivity,
	title={Sensitivity and convergence of uniformly ergodic Markov chains},
	author={Mitrophanov, A. Y.},
	journal={Journal of Applied Probability},
	volume={42},
	number={4},
	pages={1003--1014},
	year={2005},
	publisher={Cambridge University Press}
}
@book{benveniste2012adaptive,
	title={Adaptive Algorithms and Stochastic Approximations},
	author={Benveniste, Albert and M{\'e}tivier, Michel and Priouret, Pierre},
	volume={22},
	year={2012},
	publisher={Springer Science \& Business Media}
}



@inproceedings{perolat2015approximate,
	title={Approximate dynamic programming for two-player zero-sum markov games},
	author={Perolat, Julien and Scherrer, Bruno and Piot, Bilal and Pietquin, Olivier},
	booktitle=icml,
	year={2015}
}
@article{singh2000convergence,
	title={Convergence results for single-step on-policy reinforcement-learning algorithms},
	author={Singh, Satinder and Jaakkola, Tommi and Littman, Michael L and Szepesv{\'a}ri, Csaba},
	journal={Machine Learning},
	volume={38},
	number={3},
	pages={287--308},
	year={2000},
	publisher={Springer}
}







@inproceedings{dalal2017finite,
	title={Finite sample analysis of two-timescale stochastic approximation with applications to reinforcement learning},
	author={Dalal, Gal and Szorenyi, Balazs and Thoppe, Gugan and Mannor, Shie},
	booktitle={Proc. Conference on Learning Theory (COLT)},
	year={2018}
}







@inproceedings{devraj2017zap,
	title={Zap Q-learning},
	author={Devraj, Adithya M and Meyn, Sean},
	booktitle=nips,
	pages={2235--2244},
	year={2017}
}

@inproceedings{maei2009convergent,
	title={Convergent temporal-difference learning with arbitrary smooth function approximation},
	author={Maei, Hamid R and Szepesv{\'a}ri, Csaba and Bhatnagar, Shalabh and Precup, Doina and Silver, David and Sutton, Richard S },
	booktitle=nips,
	pages={1204--1212},
	year={2009}
}
@article{chen2019zap,
	title={Zap {Q-Learning} With Nonlinear Function Approximation},
	author={Chen, Shuhang and Devraj, Adithya M and Bu{\v{s}}i{\'c}, Ana and Meyn, Sean},
	journal={arXiv preprint arXiv:1910.05405},
	year={2019}
}

@inproceedings{dai2018sbeed,
	title={{SBEED}: Convergent Reinforcement Learning with Nonlinear Function Approximation},
	author={Dai, Bo and Shaw, Albert and Li, Lihong and Xiao, Lin and He, Niao and Liu, Zhen and Chen, Jianshu and Song, Le},
	booktitle=icml,
	pages={1133--1142},
	year={2018}
}
@article{nesterov2005smooth,
	title={Smooth minimization of non-smooth functions},
	author={Nesterov, Yu},
	journal={Mathematical programming},
	volume={103},
	number={1},
	pages={127--152},
	year={2005},
	publisher={Springer}
}


@inproceedings{du2018gradient,
	title={Gradient descent provably optimizes over-parameterized neural networks},
	author={Du, Simon S and Zhai, Xiyu and Poczos, Barnabas and Singh, Aarti},
	booktitle=iclr,
	year={2018}
}

@inproceedings{arora2019fine,
	title={Fine-grained analysis of optimization and generalization for overparameterized two-layer neural networks},
	author={Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and Wang, Ruosong},
	booktitle=icml,
	year={2019}
}
@article{fan2019selective,
	title={A selective overview of deep learning},
	author={Fan, Jianqing and Ma, Cong and Zhong, Yiqiao},
	journal={arXiv preprint arXiv:1904.05526},
	year={2019}
}
@inproceedings{sutton2009convergent,
	title={A convergent $o(n) $ temporal-difference algorithm for off-policy learning with linear function approximation},
	author={Sutton, Richard S and Maei, Hamid R and Szepesv{\'a}ri, Csaba},
	booktitle={Advances in neural information processing systems},
	pages={1609--1616},
	year={2009}
}
@article{silver2016mastering,
	title={Mastering the game of {Go} with deep neural networks and tree search},
	author={Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
	journal={Nature},
	volume={529},
	number={7587},
	pages={484},
	year={2016},
	publisher={Nature Publishing Group}
}
@article{arulkumaran2017deep,
	title={Deep reinforcement learning: A brief survey},
	author={Arulkumaran, Kai and Deisenroth, Marc Peter and Brundage, Miles and Bharath, Anil Anthony},
	journal={IEEE Signal Processing Magazine},
	volume={34},
	number={6},
	pages={26--38},
	year={2017},
	publisher={IEEE}
}
@inproceedings{hessel2018rainbow,
	title={Rainbow: Combining improvements in deep reinforcement learning},
	author={Hessel, Matteo and Modayil, Joseph and Van Hasselt, Hado and Schaul, Tom and Ostrovski, Georg and Dabney, Will and Horgan, Dan and Piot, Bilal and Azar, Mohammad and Silver, David},
	booktitle=aaai,
	year={2018}
}

@misc{rlblogpost,
	title={Deep Reinforcement Learning Doesn't Work Yet},
	author={Irpan, Alex},
	howpublished={\url{https://www.alexirpan.com/2018/02/14/rl-hard.html}},
	year={2018}
}
@book{puterman2014markov,
	title={Markov Decision Processes.: Discrete Stochastic Dynamic Programming},
	author={Puterman, Martin L},
	year={2014},
	publisher={John Wiley \& Sons}
}

@article{vinyals2019grandmaster,
	title={Grandmaster level in {StarCraft II} using multi-agent reinforcement learning},
	author={Vinyals, Oriol and Babuschkin, Igor and Czarnecki, Wojciech M and Mathieu, Micha{\"e}l and Dudzik, Andrew and Chung, Junyoung and Choi, David H and Powell, Richard and Ewalds, Timo and Georgiev, Petko and others},
	journal={Nature},
	volume={575},
	number={7782},
	pages={350--354},
	year={2019},
	publisher={Nature Publishing Group}
}
@inproceedings{duan2016benchmarking,
	title={Benchmarking deep reinforcement learning for continuous control},
	author={Duan, Yan and Chen, Xi and Houthooft, Rein and Schulman, John and Abbeel, Pieter},
	booktitle=icml,
	pages={1329--1338},
	year={2016}
}
@inproceedings{wang2018deep,
	title={Deep reinforcement learning for {NLP}},
	author={Wang, William Yang and Li, Jiwei and He, Xiaodong},
	booktitle={Proceedings of ACL 2018, Tutorial Abstracts},
	pages={19--21},
	year={2018}
}
@article{li2017deep,
	title={Deep reinforcement learning: An overview},
	author={Li, Yuxi},
	journal={arXiv preprint arXiv:1701.07274},
	year={2017}
}
@inproceedings{sutton2000policy,
	title={Policy gradient methods for reinforcement learning with function approximation},
	author={Sutton, Richard S and McAllester, David A and Singh, Satinder P and Mansour, Yishay},
	booktitle=nips,
	pages={1057--1063},
	year={2000}
}


@inproceedings{papini2018stochastic,
	title={Stochastic Variance-Reduced Policy Gradient},
	author={Papini, Matteo and Binaghi, Damiano and Canonaco, Giuseppe and Pirotta, Matteo and Restelli, Marcello},
	booktitle=icml,
	pages={4023--4032},
	year={2018}
}

@inproceedings{shen2019hessian,
	title={Hessian Aided Policy Gradient},
	author={Shen, Zebang and Ribeiro, Alejandro and Hassani, Hamed and Qian, Hui and Mi, Chao},
	booktitle=icml,
	pages={5729--5738},
	year={2019}
}

@article{xu2019sample,
	title={Sample Efficient Policy Gradient Methods with Recursive Variance Reduction},
	author={Xu, Pan and Gao, Felicia and Gu, Quanquan},
	journal={arXiv preprint arXiv:1909.08610},
	year={2019}
}
@inproceedings{liang2016deep,
	title={Why deep neural networks for function approximation?},
	author={Liang, Shiyu and Srikant, Rayadurgam},
	booktitle=iclr,
	year={2016}
}
@inproceedings{lee2019target,
	title={Target-Based Temporal Difference Learning},
	author={Lee, Donghwan and He, Niao},
	booktitle=icml,
	year={2019},
	pages={3713--3722},
}


@inproceedings{zhang2018fully,
	title={Fully Decentralized Multi-Agent Reinforcement Learning with Networked Agents},
	author={Zhang, Kaiqing and Yang, Zhuoran and Liu, Han and Zhang, Tong and Basar, Tamer},
	booktitle=icml,
	pages={5867--5876},
	year={2018}
}


@article{xu2019convergence,
	title={On the Convergence of (Stochastic) Gradient Descent with Extrapolation for Non-Convex Optimization},
	author={Xu, Yi and Yuan, Zhuoning and Yang, Sen and Jin, Rong and Yang, Tianbao},
	journal={arXiv preprint arXiv:1901.10682},
	year={2019}
}


@inproceedings{zhang2016understanding,
	title={Understanding deep learning requires rethinking generalization},
	author={Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
	booktitle=iclr,
	year={2016}
}

@inproceedings{lu2017expressive,
	title={The expressive power of neural networks: A view from the width},
	author={Lu, Zhou and Pu, Hongming and Wang, Feicheng and Hu, Zhiqiang and Wang, Liwei},
	booktitle=nips,
	pages={6231--6239},
	year={2017}
}
@inproceedings{li2018learning,
	title={Learning overparameterized neural networks via stochastic gradient descent on structured data},
	author={Li, Yuanzhi and Liang, Yingyu},
	booktitle=nipsnew,
	pages={8157--8166},
	year={2018}
}

@article{mnih2013playing,
	title={Playing {Atari} with deep reinforcement learning},
	author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},
	journal={NIPS Deep Learning Workshop},
	year={2013}
}
@inproceedings{van2016deep,
	title={Deep reinforcement learning with double {Q}-learning},
	author={Van Hasselt, Hado and Guez, Arthur and Silver, David},
	booktitle=aaai,
	year={2016}
}

@inproceedings{mousavi2016deep,
	title={Deep reinforcement learning: an overview},
	author={Mousavi, Seyed Sajad and Schukat, Michael and Howley, Enda},
	booktitle={Proc. SAI Intelligent Systems Conference},
	pages={426--440},
	year={2016},
	organization={Springer}
}

@article{lillicrap2015continuous,
	title={Continuous control with deep reinforcement learning},
	author={Lillicrap, Timothy P and Hunt, Jonathan J and Pritzel, Alexander and Heess, Nicolas and Erez, Tom and Tassa, Yuval and Silver, David and Wierstra, Daan},
	journal={arXiv preprint arXiv:1509.02971},
	year={2015}
}


@article{neyshabur2018towards,
	title={Towards understanding the role of over-parametrization in generalization of neural networks},
	author={Neyshabur, Behnam and Li, Zhiyuan and Bhojanapalli, Srinadh and LeCun, Yann and Srebro, Nathan},
	journal={arXiv preprint arXiv:1805.12076},
	year={2018}
}

@article{karimi2019non,
	title={Non-asymptotic Analysis of Biased Stochastic Approximation Scheme},
	author={Karimi, Belhal and Miasojedow, Blazej and Moulines, {\'E}ric and Wai, Hoi-To},
	journal={arXiv preprint arXiv:1902.00629},
	year={2019}
}




@inproceedings{sutton2009fast,
	title={Fast gradient-descent methods for temporal-difference learning with linear function approximation},
	author={Sutton, Richard S and Maei, Hamid Reza and Precup, Doina and Bhatnagar, Shalabh and Silver, David and Szepesv{\'a}ri, Csaba and Wiewiora, Eric},
	booktitle={Proc. International Conference on Machine Learning (ICML)},
	pages={993--1000},
	year={2009}
}


@article{konda2004convergence,
	title={Convergence rate of linear two-time-scale stochastic approximation},
	author={Konda, Vijay R and Tsitsiklis, John N and others},
	journal={The Annals of Applied Probability},
	volume={14},
	number={2},
	pages={796--819},
	year={2004},
	publisher={Institute of Mathematical Statistics}
}

@article{mokkadem2006convergence,
	title={Convergence rate and averaging of nonlinear two-time-scale stochastic approximation algorithms},
	author={Mokkadem, Abdelkader and Mariane Pelletier},
	journal={The Annals of Applied Probability},
	volume={16},
	number={3},
	pages={1671--1702},
	year={2006},
	publisher={Institute of Mathematical Statistics}
}



@inproceedings{tadic2004almost,
	title={Almost sure convergence of two time-scale stochastic approximation algorithms},
	author={Tadic, Vladislav B},
	booktitle={Proc. American Control Conference},
	volume={4},
	pages={3802--3807},
	year={2004}
}

@article{yu2017convergence,
	title={On Convergence of some Gradient-based Temporal-Differences Algorithms for Off-Policy Learning},
	author={Yu, Huizhen},
	journal={arXiv preprint arXiv:1712.09652},
	year={2017}
}

@article{karmakar2017two,
	title={Two time-scale stochastic approximation with controlled {Markov} noise and off-policy temporal-difference learning},
	author={Karmakar, Prasenjit and Bhatnagar, Shalabh},
	journal={Mathematics of Operations Research},
	volume={43},
	number={1},
	pages={130--151},
	year={2017},
	publisher={INFORMS}
}

@article{ramaswamy2018stability,
	title={Stability of Stochastic Approximations with 'Controlled {Markov}' Noise and Temporal Difference Learning},
	author={Ramaswamy, Arunselvan and Bhatnagar, Shalabh},
	journal={Transactions on Automatic Control},
	pages={2614--2620},
	year={2018},
	publisher={IEEE}
}

@article{karmakar2016dynamics,
	title={Dynamics of stochastic approximation with {Markov} iterate-dependent noise with the stability of the iterates not ensured},
	author={Karmakar, Prasenjit and Bhatnagar, Shalabh},
	journal={arXiv preprint arXiv:1601.02217},
	year={2016}
}

@article{yaji2016stochastic,
	title={Stochastic Recursive Inclusions in two timescales with non-additive iterate dependent {Markov} noise},
	author={Yaji, Vinayaka and Bhatnagar, Shalabh},
	journal={arXiv preprint arXiv:1611.05961},
	year={2016}
}



@article{dann2014policy,
	title={Policy evaluation with temporal differences: A survey and comparison},
	author={Dann, Christoph and Neumann, Gerhard and Peters, Jan},
	journal={The Journal of Machine Learning Research},
	volume={15},
	number={1},
	pages={809--883},
	year={2014},
	publisher={JMLR.org}
}


 




@inproceedings{silver2014deterministic,
	title={Deterministic policy gradient algorithms},
	author={Silver, David and Lever, Guy and Heess, Nicolas and Degris, Thomas and Wierstra, Daan and Riedmiller, Martin},
	booktitle={Proc. International Conference on Machine Learning (ICML)},
	year={2014},
	pages={387--395},
}



@article{yang2018does,
	title={Why Does Stagewise Training Accelerate Convergence of Testing Error Over {SGD}?},
	author={Yang, Tianbao and Yan, Yan and Yuan, Zhuoning and Jin, Rong},
	journal={arXiv preprint arXiv:1812.03934},
	year={2018}
}




@inproceedings{tsitsiklis1996analysis,
	title={Analysis of temporal-diffference learning with function approximation},
	author={Tsitsiklis, John N and Van Roy, Benjamin},
	booktitle={Proc. Advances in Neural Information Processing Systems (NIPS)},
	pages={1075--1081},
	year={1997}
}





@article{borkar2000ode,
	title={The {ODE} method for convergence of stochastic approximation and reinforcement learning},
	author={Borkar, Vivek S and Meyn, Sean P},
	journal={Journal on Control and Optimization},
	volume={38},
	number={2},
	pages={447--469},
	year={2000},
}

@article{kamal2010onthe,
	title={On the convergence, lock-in probability and sample complexity of stochastic approximation},
	author={Sameer Kamal},
	journal={Journal on Control and Optimization},
	volume={48},
	number={8},
	pages={5178--5192},
	year={2010},
	publisher={SIAM}
}

@article{thoppe2015alekseev,
	title={A Concentration Bound for Stochastic Approximation via {Alekseev's} Formula},
	author={Thoppe, Gugan and Borkar, Vivek},
	journal={Stochastic Systems},
	pages={1--26},
	year={2019},
	publisher={INFORMS}
}



@inproceedings{borkar2018concentration,
	title={Concentration bounds for Two Time Scale Stochastic Approximation},
	author={Borkar, Vivek S and Pattathil, Sarath},
	booktitle={Proc. Allerton Conference on Communication, Control, and Computing (Allerton)},
	pages={504--511},
	year={2018},
}

@article{maei2018offpolicyac,
	title={Convergent Actor-Critic Algorithms Under Off-Policy Training and Function Approximation},
	author={Hamid Reza Maei},
	journal={arXiv preprint arXiv:1802.07842},
	year={2018}
}

@inproceedings{maei2010gq,
	title={{GQ} (lambda): A general gradient algorithm for temporal-difference prediction learning with eligibility traces},
	author={Maei, Hamid Reza and Sutton, Richard S},
	booktitle={Proc. Artificial General Intelligence (AGI)},
	year={2010},
	organization={Atlantis Press}
}

@Article{tadi2001td,
	author={Tadi{\'{c}}, Vladislav},
	title={On the Convergence of Temporal-Difference Learning with Linear Function Approximation},
	journal={Machine Learning},
	year={2001},
	month={Mar},
	day={01},
	volume={42},
	number={3},
	pages={241--267}
}


@article{Antos2008,
	author = "A. Antos and C. Szepesvari and R. Munos",
	title = "Learning near-optimal policies with {B}ellman-residual minimization based fitted policy iteration and a single sample path",
	journal = "Machine Learning",
	volume = "71",
	number = "1",
	year = "2008",
	pages = "89-129"
}

@inproceedings{Bowling2001,
	author = "Bowling, M.",
	title = "Rational and convergent learning in stochastic games",
	booktitle = "Proc.  International Joint Conference on Artificial intelligence (IJCAI)",
	pages={1021--1026},
	year = "2001",
} 

@article{Boyan2002,
	author = "Boyan, J. A.",
	title = "Technical update: {L}east-squares temporal difference learning",
	journal = "Machine Learning",
	volume = "49",
	year = "2002",
	pages = "233-246"
}

@article{Brad1996,
	author = "Bradtke, S. J. and Barto, A. G.",
	title = "Linear least-squares algorithms for temporal difference learning",
	journal = "Machine Learning",
	volume = "22",
	year = "1996",
	pages = "33-57"
}

@article{Coni2007,
	author = "Conitzer, V. and Sandholm, T.",
	title = "{AWESOME: A} general multiagent learning algorithm that converges in self-play and learns a best response against stationary opponents",
	journal = "Machine Learning",
	volume = "67",
	year = "2007",
	pages = "23-43"
}

@article{Devraj2018,
	author = {A. M. Devraj and I. Kontoyiannis and S. P. Meyn},
	title = {Differential Temporal Difference Learning},
	journal = {ArXiv: 1812.11137},
	year = 2018,
	month = "Dec.",
}



@inproceedings{Farah2010,
	author = "Farahmand, A.-M. and Szepesvari, C. and Munos, R.",
	title = "Error propagation for approximate policy and value iteration",
	booktitle = nips,
	pages={568--576},
	year = "2010",
} 

@inproceedings{Ghav2010,
	author = "Mohammad Ghavamzadeh and Alessandro Lazaric and Odalric Maillard and Remi Munos",
	title = "{LSTD} with Random Projections",
	booktitle = nips,
	pages={721--729},
	year = "2010",
} 

@article{Lagou2003,
	author = "Lagoudakis, M. G. and Parr, R.",
	title = "Least-squares policy iteration",
	journal = "Journal of Machine Learning Research",
	volume = "4",
	year = "2003",
	pages = "1107-1149"
}

@inproceedings{Lagou2002,
	author = "Lagoudakis, M. G. and Parr, R.",
	title = "Value function approximation in zero-sum {M}arkov games",
	booktitle = "Proc. Uncertainty in Artificial Intelligence (UAI)",
	pages={283--292},
	year = "2002",
} 

@inproceedings{Laksh2018,
	author = "Chandrashekar Lakshminarayanan and Csaba Szepesvari",
	title = "Linear stochastic approximation: {H}ow far does constant step-size and iterate averaging go?",
	booktitle =aistats,
	pages={1347--1355},
	year = "2018",
} 

@inproceedings{Lazaric2010,
	author = "Alessandro Lazaric and Mohammad Ghavamzadeh and Remi Munos",
	title = "Finite-sample analysis of {LSTD}",
	booktitle = icml,
	year = "2010",
	 pages={615--622},
} 

@article{Lazaric2012,
	author = "Lazaric, A. and Ghavamzadeh, M. and Munos, R.",
	title = "Finite-sample analysis of least-squares policy iteration",
	journal = "Journal of Machine Learning Research",
	volume = "13",
	year = "2012",
	pages = "3041-3074"
}

@article{Lazaric2016,
	author = "Lazaric, A. and Ghavamzadeh, M. and Munos, R.",
	title = "Analysis of classification-based policy iteration algorithms",
	journal = "Journal of Machine Learning Research",
	volume = "17",
	year = "2016",
	pages = "583-612"
}

@inproceedings{Littman1994,
	author = "Littman, M. L.",
	title = "Markov games as a framework for multi-agent reinforcement learning",
	booktitle = icml,
	pages={157--163},
	year = "1994",
} 

@inproceedings{Liu2015,
	author = "Bo Liu and Ji Liu and Mohammad Ghavamzadeh and Sridhar Mahadevan and Marek Petrik",
	title = "Finite-sample analysis of proximal gradient {TD} algorithms",
	booktitle = "Proc.  Conference on Uncertainty in Artificial Intelligence (UAI)",
	pages={504--513},
	year = "2015",
} 



@article{Munos2008,
	author = "R. Munos and C. Szepesvari",
	title = "Finite-time bounds for fitted value iteration",
	journal = "Journal of Machine Learning Research",
	volume = "9",
	month = may,
	year = "2008",
	pages = "815-857"
}

@article{Ormo2002a,
	author = "D. Ormoneit and P. Glynn",
	title = "Kernel-based reinforcement learning in average-cost problems",
	journal = "IEEE Trans. Automatic Control",
	volume = "47",
	number ="10",
	year = "2002",
	pages = "1624-1636"
}

@article{Ormo2002b,
	author = "Dirk Ormoneit and Saunak Sen",
	title = "Kernel-based reinforcement learning",
	journal = "Mach. Learning",
	volume = "49",
	number = "2-3",
	year = "2002",
	pages = "161-178"
}

@inproceedings{Pires2012,
	author = "Bernardo A. Pires and Csaba Szepesvari",
	title = "Statistical linear estimation with penalized estimators: {A}n application to reinforcement learning",
	booktitle = icml,
	year = "2012",
	pages={1755--1762},
} 

@inproceedings{Perolat2016a,
	author = "Perolat, J. and Piot, B. and Geist, M. and Scherrer, B. and Pietquin, O. ",
	title = "Softened approximate policy iteration for {M}arkov games",
	booktitle =icml,
	year = "2016",
} 

@inproceedings{Perolat2016b,
	author = "Perolat, J. and Piot and B. and Scherrer B. and Pietquin, O.",
	title = "On the use of non-stationary strategies for solving two-player zero-sum {M}arkov games",
	booktitle = aistats,
	pages={893--901},
	year = "2016",
} 

@inproceedings{Perolat2018,
	author = "Perolat, J. and Piot, B. and Pietquin, O.",
	title = "Actor-critic fictitious play in simultaneous move multistage games",
	booktitle = aistats,
	year = "2018",
} 

@inproceedings{Prasad2015,
	author = "H. L. Prasad and Prashanth L.A. and Shalabh Bhatnagar",
	title = "Two-timescale algorithms for learning {N}ash equilibria in general-sum stochastic games",
	booktitle = "Proc.  International Conference on Autonomous Agents and Multiagent Systems (AAMAS)",
	pages={1371--1379},
	year = "2015",
} 

@inproceedings{Prash2013,
	author = "La Prashanth and Nathaniel Korda and Remi Munos",
	title = "Fast {LSTD} using stochastic approximation: {F}inite time analysis and application to traffic control",
	booktitle = "Proc. Joint European Conference on Machine Learning and Knowledge Discovery in Databases",
	pages={66--81},
	year = "2013",
} 

@article{Rummery1994,
	author = {G. A. Rummery and M. Niranjan},
	title = {Online {Q}-learning using connectionist systems},
	journal = {Technical Report, {Cambridge University Engineering Department}},
	year = 1994,
	month = sep,
}

@inproceedings{Shah2018,
	author = "Devavrat Shah and Qiaomin Xie",
	title = "Q-learning with Nearest Neighbors",
	booktitle = nipsnew,
	pages={3111--3121},
	year = "2018",
} 

@inproceedings{Srini2018,
	author = "Srinivasan, S. and Lanctot, M. and Zambaldi, V. and Perolat, J. and Tuyls, K. and Munos, R. and Bowling, M.",
	title = "Actor-critic policy optimization in partially observable multiagent environments",
	booktitle = nipsnew,
	pages={3422--3435},
	year = "2018",
} 

@article{Sutton1988,
	author = "Richard S Sutton",
	title = "Learning to predict by the methods of temporal differences",
	journal = "Machine Learning",
	volume = "3",
	number = "1",
	year = "1988",
	pages = "9-44"
}

@inproceedings{Sutton2009a,
	author = "R. S. Sutton and H. R. Maei and D. Precup and S. Bhatnagar and D. Silver and C. Szepesvari and E. Wiewiora",
	title = "Fast gradient-descent methods for temporal-difference learning with linear function approximation",
	booktitle = icml,
	pages={993--1000},
	year = "2009",
} 

@inproceedings{Sutton2009b,
	author = "R. S. Sutton and H. R. Maei and C. Szepesvari",
	title = "A convergent o(n) temporal-difference algorithm for off-policy learning with linear function approximation",
	booktitle = nips,
	pages={1609--1616},
	year = "2009",
} 


@inproceedings{Tagorti2015,
	author = "Tagorti, M. and Scherrer, B.",
	title = "On the rate of convergence and error bounds for {LSTD} ($\lambda$)",
	booktitle = icml,
	pages={1521--1529},
	year = "2015",
} 

@inproceedings{Touati2018,
	author = "Ahmed Touati and Pierre-Luc Bacon and Doina Precup and Pascal Vincent",
	title = "Convergent {TREE BACKUP} and {RETRACE} with function approximation",
	booktitle = icml,
	year = "2018",
} 

@inproceedings{Tu2018,
	author = "Stephen Tu and Benjamin Recht",
	title = "Least-squares temporal difference learning for the linear quadratic regulator",
	booktitle = icml,
	year = "2018",
	pages={5012--5021},
} 

@article{Watkins1992,
	author = "C. J. C. H. Watkins and P. Dayan",
	title = "Q-learning",
	journal = "Mach. Learning",
	volume = "8",
	number = "3-4",
	year = "1992",
	pages = "279-292"
}

@inproceedings{Wei2017,
	author = "Wei, C.-Y. and Hong and Y.-T. and Lu, C.-J.",
	title = "Online reinforcement learning in stochastic games",
	booktitle = nips,
	pages={4987--4997},
	year = "2017",
} 

@article{Yang2019,
	author = {Zhuora Yang and Yuchen Xie and Zhaoran Wang},
	title = {A Theoretical Analysis of Deep {Q}-Learning},
	journal = {ArXiv: 1901.00137},
	year = 2019,
}

@article{Zhang2018,
	author = {Zhang, K. and Yang, Z. and Liu, H. and Zhang, T. and Basar, T.},
	title = {Finite-sample analyses for fully decentralized multi-agent reinforcement learning},
	journal = {arXiv:1812.02783},
	year = 2018,
}

############################################
# References about  cubic regularization
############################################
@article{Nesterov2006,
	author = "Nesterov, Y. and  Polyak, B.",
	journal = "Mathematical Programming",
	title = "Cubic regularization of {N}ewton's method and its global performance",
	pages={177--205},
	year = "2006"
}

@ARTICLE{Carmon2016,
	author = {{Carmon}, Y. and {Duchi}, J.~C.},
	title = "{Gradient descent efficiently finds the cubic-regularized non-convex Newton step}",
	journal = {ArXiv: 1612.00547},
	year = 2016,
	month = "Dec.",
}
@InProceedings{gu2018,
	author = {{Zhou}, D. and {Xu}, P. and {Gu}, Q.},
	title = {Stochastic Variance-Reduced Cubic Regularized Newton Method},
	booktitle = {Proc. International Conference on Machine Learning (ICML)},
	year = 2018,  
}

@book{Lojasiewicz_book,
	title={Ensembles semi-analytiques},
	author={{\L}ojasiewicz, S.},
	publisher={Bures-sur-Yvette: Institut des Hautes Etudes Scientifiques},
	year={1965}
}


@book{Nesterov_2014,
	author = {Nesterov, Y.},
	title = {Introductory lectures on convex optimization: A Basic Course},
	year = {2014},
	publisher = {Springer},
} 

@article{Nemirovski_2009,
	author = "Nemirovski, A. S. and Juditsky, A. and Lan, G. and Shapiro, A.",
	title = "Robust stochastic approximation approach to stochastic programming",
	journal = "SIAM Journal on Optimization",
	volume = 19,
	year = 2009,
	pages = "1574-1609"
}
@article{bagnell2001solving,
  title={Solving uncertain {Markov} decision processes},
  author={Bagnell, J Andrew and Ng, Andrew Y and Schneider, Jeff G},
  year={2001},
  publisher={Carnegie Mellon University}
}


@InProceedings{wang2020finite, 
title = {Finite-sample Analysis of {Greedy-GQ} with Linear Function Approximation under {Markovian} Noise}, author = {Wang, Yue and Zou, Shaofeng}, 
pages = {11--20}, 
year = {2020}, 
volume = {124}, 
booktitle= uai
}
@inproceedings{ma2020iclr,
  title={Variance reduced {Greedy-GQ} with linear function approximation and non-asymptotic analysis},
  author={S. Ma and Y. Zhou and S. Zou},
  booktitle={Proc. International Conference on Learning Representations (ICLR)},
  year={2021},
}
@misc{wang2020ijcai,
  title={An improved sample complexity of {Greedy-GQ} with linear function approximation},
  author={Y. Wang and S. Zou and Y. Zhou},
  howpublished={\url{https://www.acsu.buffalo.edu/~szou3/greedygq.pdf}},
  year={2020},
}




@misc{buffaloday,
	author       = {{University at Buffalo}},
	title        = {Buffalo Day for {5G} and Wireless Internet of Things},
	howpublished = {\url{https://buffalowirelessday.wixsite.com/2019}},
}
@misc{buffaloiot,
	title = {Buffalo now has wireless network for the {Internet} of things},
	author = {C. Nealon},
	month= aug,
	year = {2017},
	howpublished = {\url{https://www.buffalo.edu/ubnow/stories/2017/08/jornet-sig-fox.html}}
}

@article{bertsekas2011dynamic,
  title={{Dynamic Programming and Optimal Control 3rd edition, volume II}},
  author={Bertsekas, Dimitri P},
  journal={Belmont, MA: Athena Scientific},
  year={2011}
}


 
@inproceedings{melo2008analysis,
	title={An analysis of reinforcement learning with function approximation},
	author={Melo, Francisco S and Meyn, Sean P and Ribeiro, M Isabel},
	booktitle=icml,
	pages={664--671},
	year={2008},
	organization={ACM}
}
@book{meyn2012markov,
	title={Markov Chains and Stochastic Stability},
	author={Meyn, Sean P and Tweedie, Richard L},
	year={2012},
	publisher={Springer Science \& Business Media}
}

@article{kushner2010stochastic,
	title={Stochastic approximation: a survey},
	author={Kushner, Harold},
	journal={Wiley Interdisciplinary Reviews: Computational Statistics},
	volume={2},
	number={1},
	pages={87--96},
	year={2010},
	publisher={Wiley Online Library}
}
@article{lacoste2012simpler,
	title={A simpler approach to obtaining an $O(1/t)$ convergence rate for the projected stochastic subgradient method},
	author={Lacoste-Julien, Simon and Schmidt, Mark and Bach, Francis},
	journal={arXiv preprint arXiv:1212.2002},
	year={2012}
}
@article{bubeck2015convex,
	title={Convex optimization: Algorithms and complexity},
	author={Bubeck, S{\'e}bastien},
	journal={Foundations and Trends{\textregistered} in Machine Learning},
	volume={8},
	number={3-4},
	pages={231--357},
	year={2015},
	publisher={Now Publishers, Inc.}
}
@inproceedings{schulman2015trust,
  title={Trust region policy optimization},
  author={Schulman, John and Levine, Sergey and Abbeel, Pieter and Jordan, Michael and Moritz, Philipp},
  booktitle=icml,
  pages={1889--1897},
  year={2015},
 
}
@article{nemirovski2009robust,
	title={Robust stochastic approximation approach to stochastic programming},
	author={Nemirovski, Arkadi and Juditsky, Anatoli and Lan, Guanghui and Shapiro, Alexander},
	journal={SIAM Journal on Optimization},
	volume={19},
	number={4},
	pages={1574--1609},
	year={2009},
	publisher={SIAM}
}
@ARTICLE{Tsitsiklis1997, 
	author={J. N. Tsitsiklis and B.  Roy}, 
	journal={IEEE Transactions on Automatic Control}, 
	title={An analysis of temporal-difference learning with function approximation}, 
	year={1997}, 
	volume={42}, 
	number={5}, 
	pages={674-690},
	month={May},}
@inproceedings{perkins2002existence,
	title={On the existence of fixed points for {Q}-learning and {Sarsa} in partially observable domains},
	author={Perkins, Theodore J and Pendrith, Mark D},
	booktitle=icml,
	pages={490--497},
	year={2002}
}
@article{de2000existence,
	title={On the existence of fixed points for approximate value iteration and temporal-difference learning},
	author={De Farias, Daniela Pucci and Van Roy, Benjamin},
	journal={Journal of Optimization theory and Applications},
	volume={105},
	number={3},
	pages={589--608},
	year={2000},
	publisher={Springer}
}
@inproceedings{perkins2003convergent,
	title={A convergent form of approximate policy iteration},
	author={Perkins, Theodore J and Precup, Doina},
	booktitle=nips,
	pages={1627--1634},
	year={2003}
}
@article{mitrophanov2005sensitivity,
	title={Sensitivity and convergence of uniformly ergodic {M}arkov chains},
	author={Mitrophanov, A. Y.},
	journal={Journal of Applied Probability},
	volume={42},
	number={4},
	pages={1003--1014},
	year={2005},
	publisher={Cambridge University Press}
}
@book{benveniste2012adaptive,
	title={Adaptive Algorithms and Stochastic Approximations},
	author={Benveniste, Albert and M{\'e}tivier, Michel and Priouret, Pierre},
	volume={22},
	year={2012},
	publisher={Springer Science \& Business Media}
}

@article{doan2019finite,
  title={Finite-time analysis and restarting scheme for linear two-time-scale stochastic approximation},
  author={Doan, Thinh T},
  journal={arXiv preprint arXiv:1912.10583},
  year={2019}
}

@inproceedings{perolat2015approximate,
	title={Approximate dynamic programming for two-player zero-sum markov games},
	author={Perolat, Julien and Scherrer, Bruno and Piot, Bilal and Pietquin, Olivier},
	booktitle=icml,
	year={2015}
}
@article{singh2000convergence,
	title={Convergence results for single-step on-policy reinforcement-learning algorithms},
	author={Singh, Satinder and Jaakkola, Tommi and Littman, Michael L and Szepesv{\'a}ri, Csaba},
	journal={Machine Learning},
	volume={38},
	number={3},
	pages={287--308},
	year={2000},
	publisher={Springer}
}




@inproceedings{Asadi2016,
  title={An alternative softmax operator for reinforcement learning},
  author={Asadi, Kavosh and Littman, Michael L},
  booktitle={Proceedings of the 34th International Conference on Machine Learning},
  volume={70},
  pages={243--252},
  year={2017},
  organization={JMLR. org}
}


@inproceedings{gordon2000reinforcement,
	title={Reinforcement Learning with Function Approximation Converges to a Region},
	author={Gordon, Geoffrey J},
	pages={1040--1046},
	year={2001},
booktitle=nips}


@inproceedings{korda2015td,
  title={On {TD}(0) with function approximation: Concentration bounds and a centered variant with exponential convergence},
  author={Korda, Nathaniel and La, Prashanth},
  booktitle=icml,
  pages={626--634},
  year={2015}
}





@article{Antos2008,
author = "A. Antos and C. Szepesvari and R. Munos",
title = "Learning near-optimal policies with {B}ellman-residual minimization based fitted policy iteration and a single sample path",
journal = "Machine Learning",
volume = "71",
number = "1",
year = "2008",
pages = "89-129"
}

@inproceedings{Bowling2001,
	author = "Bowling, M.",
	title = "Rational and convergent learning in stochastic games",
	booktitle = "Proc.  International Joint Conference on Artificial intelligence (IJCAI)",
	year = "2001",
} 

@article{Boyan2002,
author = "Boyan, J. A.",
title = "Technical update: {L}east-squares temporal difference learning",
journal = "Machine Learning",
volume = "49",
year = "2002",
pages = "233-246"
}

@article{Brad1996,
author = "Bradtke, S. J. and Barto, A. G.",
title = "Linear least-squares algorithms for temporal difference learning",
journal = "Machine Learning",
volume = "22",
year = "1996",
pages = "33-57"
}

@article{Coni2007,
author = "Conitzer, V. and Sandholm, T.",
title = "{AWESOME: A} general multiagent learning algorithm that converges in self-play and learns a best response against stationary opponents",
journal = "Machine Learning",
volume = "67",
year = "2007",
pages = "23-43"
}

@article{Devraj2018,
	author = {A. M. Devraj and I. Kontoyiannis and S. P. Meyn},
	title = {Differential Temporal Difference Learning},
	journal = {ArXiv: 1812.11137},
	year = 2018,
	month = "Dec.",
}

@inproceedings{Dalal2018a,
	author = "G. Dalal and B. Szrnyi and G. Thoppe and S. Mannor",
	title = "Finite sample analyses for {TD}(0) with function approximation",
	booktitle = "Proc. AAAI Conference on Artificial Intelligence (AAAI)",
	year = "2018",
	pages= "6144-6160"
} 



@inproceedings{Farah2010,
	author = "Farahmand, A.-M. and Szepesvari, C. and Munos, R.",
	title = "Error propagation for approximate policy and value iteration",
	booktitle = nips,
	year = "2010",
} 

@inproceedings{Ghav2010,
	author = "Mohammad Ghavamzadeh and Alessandro Lazaric and Odalric Maillard and Remi Munos",
	title = "{LSTD} with Random Projections",
	booktitle = nips,
	year = "2010",
} 

@article{Lagou2003,
author = "Lagoudakis, M. G. and Parr, R.",
title = "Least-squares policy iteration",
journal = "Journal of Machine Learning Research",
volume = "4",
year = "2003",
pages = "1107-1149"
}

@inproceedings{Lagou2002,
	author = "Lagoudakis, M. G. and Parr, R.",
	title = "Value function approximation in zero-sum {M}arkov games",
	booktitle = "Proc. Uncertainty in Artificial Intelligence (UAI)",
	year = "2002",
} 



@inproceedings{Lazaric2010,
	author = "Alessandro Lazaric and Mohammad Ghavamzadeh and Remi Munos",
	title = "Finite-sample analysis of LSTD",
	booktitle = icml,
	year = "2010",
} 

@article{Lazaric2012,
author = "Lazaric, A. and Ghavamzadeh, M. and Munos, R.",
title = "Finite-sample analysis of least-squares policy iteration",
journal = "Journal of Machine Learning Research",
volume = "13",
year = "2012",
pages = "3041-3074"
}

@article{Lazaric2016,
author = "Lazaric, A. and Ghavamzadeh, M. and Munos, R.",
title = "Analysis of classification-based policy iteration algorithms",
journal = "Journal of Machine Learning Research",
volume = "17",
year = "2016",
pages = "583-612"
}

@inproceedings{Littman1994,
	author = "Littman, M. L.",
	title = "Markov games as a framework for multi-agent reinforcement learning",
	booktitle = icml,
	year = "1994",
} 




@article{Minh2015,
	author = "Mnih, V. and Kavukcuoglu, K. and Silver, D. and Rusu, A. A. and Veness, J. and Bellemare, M. G. and Graves, A. and Riedmiller, M. and Fidjeland, A. K. and Ostrovski, G.",
	title = "Human-level control through deep reinforcement learning",
	journal = "Nature",
	volume = "518",
	year = "2015",
	pages = "529-533"
} 

@inproceedings{Minh2016,
	author = "Mnih, V. and Badia, A. P. and Mirza, M. and Graves, A. and Lillicrap, T. and Harley, T. and Silver, D. and Kavukcuoglu, K.",
	title = "Asynchronous methods for deep reinforcement learning",
	booktitle = icml,
	pages={1928--1937},
	year = "2016",
} 


@article{Munos2008,
author = "R. Munos and C. Szepesvari",
title = "Finite-time bounds for fitted value iteration",
journal = "Journal of Machine Learning Research",
volume = "9",
month = may,
year = "2008",
pages = "815-857"
}

@article{Ormo2002a,
author = "D. Ormoneit and P. Glynn",
title = "Kernel-based reinforcement learning in average-cost problems",
journal = "IEEE Trans. Automatic Control",
volume = "47",
number ="10",
year = "2002",
pages = "1624-1636"
}

@article{Ormo2002b,
author = "Dirk Ormoneit and Saunak Sen",
title = "Kernel-based reinforcement learning",
journal = "Mach. Learning",
volume = "49",
number = "2-3",
year = "2002",
pages = "161-178"
}

@inproceedings{Pires2012,
	author = "Bernardo A. Pires and Csaba Szepesvari",
	title = "Statistical linear estimation with penalized estimators: {A}n application to reinforcement learning",
	booktitle = icml,
	year = "2012",
} 

@inproceedings{Perolat2016a,
	author = "Perolat, J. and Piot, B. and Geist, M. and Scherrer, B. and Pietquin, O. ",
	title = "Softened approximate policy iteration for {M}arkov games",
	booktitle =icml,
	year = "2016",
} 

@inproceedings{Perolat2016b,
	author = "Perolat, J. and Piot and B. and Scherrer B. and Pietquin, O.",
	title = "On the use of non-stationary strategies for solving two-player zero-sum {M}arkov games",
	booktitle = aistats,
	year = "2016",
} 

@inproceedings{Perolat2018,
	author = "Perolat, J. and Piot, B. and Pietquin, O.",
	title = "Actor-critic fictitious play in simultaneous move multistage games",
	booktitle = aistats,
	year = "2018",
} 

@inproceedings{Prasad2015,
	author = "H. L. Prasad and Prashanth L.A. and Shalabh Bhatnagar",
	title = "Two-timescale algorithms for learning {N}ash equilibria in general-sum stochastic games",
	booktitle = "Proc.  International Conference on Autonomous Agents and Multiagent Systems (AAMAS)",
	year = "2015",
} 

@inproceedings{Prash2013,
	author = "La Prashanth and Nathaniel Korda and Remi Munos",
	title = "Fast {LSTD} using stochastic approximation: {F}inite time analysis and application to traffic control",
	booktitle = "Proc. Joint European Conference on Machine Learning and Knowledge Discovery in Databases",
	year = "2013",
} 

@article{Rummery1994,
	author = {G. A. Rummery and M. Niranjan},
	title = {Online {Q}-learning using connectionist systems},
	journal = {Technical Report, {Cambridge University Engineering Department}},
	year = 1994,
	month = sep,
}

@inproceedings{Shah2018,
	author = "Devavrat Shah and Qiaomin Xie",
	title = "Q-learning with Nearest Neighbors",
	booktitle = nipsnew,
	year = "2018",
} 

@inproceedings{Srini2018,
	author = "Srinivasan, S. and Lanctot, M. and Zambaldi, V. and Perolat, J. and Tuyls, K. and Munos, R. and Bowling, M.",
	title = "Actor-critic policy optimization in partially observable multiagent environments",
	booktitle = nipsnew,
	year = "2018",
} 

@article{Sutton1988,
author = "Richard S Sutton",
title = "Learning to predict by the methods of temporal differences",
journal = "Machine Learning",
volume = "3",
number = "1",
year = "1988",
pages = "9-44"
}




@inproceedings{sutton2009b,
  title={A convergent $ {O} (n) $ temporal-difference algorithm for off-policy learning with linear function approximation},
  author={Sutton, Richard S and Maei, Hamid R and Szepesv{\'a}ri, Csaba},
  booktitle=nips,
  pages={1609--1616},
  year={2009}
}


@inproceedings{Tagorti2015,
	author = "Tagorti, M. and Scherrer, B.",
	title = "On the rate of convergence and error bounds for {LSTD} ($\lambda$)",
	booktitle = icml,
	year = "2015",
} 

@inproceedings{Touati2018,
	author = "Ahmed Touati and Pierre-Luc Bacon and Doina Precup and Pascal Vincent",
	title = "Convergent {TREE BACKUP} and {RETRACE} with function approximation",
	booktitle = icml,
	year = "2018",
} 

@inproceedings{Tu2018,
	author = "Stephen Tu and Benjamin Recht",
	title = "Least-squares temporal difference learning for the linear quadratic regulator",
	booktitle = icml,
	year = "2018",
} 



@inproceedings{Wei2017,
	author = "Wei, C.-Y. and Hong and Y.-T. and Lu, C.-J.",
	title = "Online reinforcement learning in stochastic games",
	booktitle = nips,
	year = "2017",
} 

@article{Yang2019,
	author = {Zhuora Yang and Yuchen Xie and Zhaoran Wang},
	title = {A Theoretical Analysis of Deep {Q}-Learning},
	journal = {ArXiv: 1901.00137},
	year = 2019,
	month = jan,
}

@article{Zhang2018,
	author = {Zhang, K. and Yang, Z. and Liu, H. and Zhang, T. and Basar, T.},
	title = {Finite-sample analyses for fully decentralized multi-agent reinforcement learning},
	journal = {arXiv:1812.02783},
	year = 2018,
}

############################################
# References about  cubic regularization
############################################
@article{Nesterov2006,
	author = "Nesterov, Y. and  Polyak, B.",
	journal = "Mathematical Programming",
	title = "Cubic regularization of {N}ewton's method and its global performance",
	year = "2006"
}

@ARTICLE{Carmon2016,
	author = {{Carmon}, Y. and {Duchi}, J.~C.},
	title = "{Gradient descent efficiently finds the cubic-regularized non-convex Newton step}",
	journal = {ArXiv: 1612.00547},
	year = 2016,
	month = "Dec.",
}
@InProceedings{gu2018,
	author = {{Zhou}, D. and {Xu}, P. and {Gu}, Q.},
	title = {Stochastic Variance-Reduced Cubic Regularized Newton Method},
	booktitle = {Proc. International Conference on Machine Learning (ICML)},
	year = 2018,  
}

@book{Lojasiewicz_book,
	title={Ensembles semi-analytiques},
	author={{\L}ojasiewicz, S.},
	publisher={Bures-sur-Yvette: Institut des Hautes Etudes Scientifiques},
	year={1965}
}


@book{Nesterov_2014,
	author = {Nesterov, Y.},
	title = {Introductory lectures on convex optimization: A Basic Course},
	year = {2014},
	publisher = {Springer},
} 

@article{Nemirovski_2009,
author = "Nemirovski, A. S. and Juditsky, A. and Lan, G. and Shapiro, A.",
title = "Robust stochastic approximation approach to stochastic programming",
journal = "SIAM Journal on Optimization",
volume = 19,
year = 2009,
pages = "1574-1609"
}

@inproceedings{allen2019convergence,
  title={A Convergence Theory for Deep Learning via Over-Parameterization},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  booktitle={International Conference on Machine Learning},
  pages={242--252},
  year={2019}
}

















@inproceedings{sutton2009fast,
  title={Fast gradient-descent methods for temporal-difference learning with linear function approximation},
  author={Sutton, Richard S and Maei, Hamid Reza and Precup, Doina and Bhatnagar, Shalabh and Silver, David and Szepesv{\'a}ri, Csaba and Wiewiora, Eric},
  booktitle=icml,
  pages={993--1000},
  year={2009}
}

@inproceedings{antos2008fitted,
  title={Fitted Q-iteration in continuous action-space MDPs},
  author={Antos, Andr{\'a}s and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
  booktitle={Advances in neural information processing systems},
  pages={9--16},
  year={2008}
}

@article{antos2008learning,
  title={Learning near-optimal policies with Bellman-residual minimization based fitted policy iteration and a single sample path},
  author={Antos, Andr{\'a}s and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
  journal={Machine Learning},
  volume={71},
  number={1},
  pages={89--129},
  year={2008},
  publisher={Springer}
}

@incollection{gordon1995stable,
  title={Stable function approximation in dynamic programming},
  author={Gordon, Geoffrey J},
  booktitle={Machine Learning Proceedings 1995},
  pages={261--268},
  year={1995},
  publisher={Elsevier}
}

@article{lagoudakis2003least,
  title={Least-squares policy iteration},
  author={Lagoudakis, Michail G and Parr, Ronald},
  journal={Journal of machine learning research},
  volume={4},
  number={Dec},
  pages={1107--1149},
  year={2003}
}

@inproceedings{szepesvari2004interpolation,
  title={Interpolation-based Q-learning},
  author={Szepesv{\'a}ri, Csaba and Smart, William D},
  booktitle=icml,
  pages={100},
  year={2004}
}


@inproceedings{xu2019two,
  title={Two time-scale off-policy {TD} learning: Non-asymptotic analysis over {Markovian} samples},
  author={Xu, Tengyu and Zou, Shaofeng and Liang, Yingbin},
  booktitle=nipsnew,
  pages={10633--10643},
  year={2019}
}




@inproceedings{maei2010toward,
  title={Toward off-policy learning control with function approximation},
  author={Maei, Hamid Reza and Szepesv{\'a}ri, Csaba and Bhatnagar, Shalabh and Sutton, Richard S},
  booktitle=icml,
  year={2010}
}

@article{borkar2000ode,
  title={The ODE method for convergence of stochastic approximation and reinforcement learning},
  author={Borkar, Vivek S and Meyn, Sean P},
  journal={SIAM Journal on Control and Optimization},
  volume={38},
  number={2},
  pages={447--469},
  year={2000},
  publisher={SIAM}
}

@article{ghadimi2013stochastic,
  title={Stochastic first- and zeroth-order methods for nonconvex stochastic programming},
  author={Ghadimi, Saeed and Lan, Guanghui},
  journal={SIAM Journal on Optimization},
  volume={23},
  number={4},
  pages={2341--2368},
  year={2013},
  publisher={SIAM}
}










@inproceedings{borkar2018concentration,
  title={Concentration bounds for two time scale stochastic approximation},
  author={Borkar, Vivek S and Pattathil, Sarath},
  booktitle=allerton,
  pages={504--511},
  year={2018},
  organization={IEEE}
}
@inproceedings{zou2019finite,
  title={Finite-sample analysis for {SARSA} with linear function approximation},
  author={Zou, Shaofeng and Xu, Tengyu and Liang, Yingbin},
  booktitle=nipsnew,
  pages={8665--8675},
  year={2019}
}

@article{yu2017convergence,
  title={On convergence of some gradient-based temporal-differences algorithms for off-policy learning},
  author={Yu, Huizhen},
  journal={arXiv preprint arXiv:1712.09652},
  year={2017}
}
@article{karmakar2018two,
  title={Two time-scale stochastic approximation with controlled {Markov} noise and off-policy temporal-difference learning},
  author={Karmakar, Prasenjit and Bhatnagar, Shalabh},
  journal={Mathematics of Operations Research},
  volume={43},
  number={1},
  pages={130--151},
  year={2018},
  publisher={INFORMS}
}



@inproceedings{cai2019neural,
  title={Neural temporal-difference learning converges to global optima},
  author={Cai, Qi and Yang, Zhuoran and Lee, Jason D and Wang, Zhaoran},
  booktitle=nipsnew,
  pages={11312--11322},
  year={2019}
}
@article{li2021faster,
  title={Faster algorithm and sharper analysis for constrained markov decision process},
  author={Li, Tianjiao and Guan, Ziwei and Zou, Shaofeng and Xu, Tengyu and Liang, Yingbin and Lan, Guanghui},
  journal={arXiv preprint arXiv:2110.10351},
  year={2021}
}
@article{chen2019performance,
  title={Performance of {Q}-learning with linear function approximation: Stability and finite-time analysis},
  author={Chen, Zaiwei and Zhang, Sheng and Doan, Thinh T and Maguluri, Siva Theja and Clarke, John-Paul},
  journal={arXiv preprint arXiv:1905.11425},
  year={2019}
}
@inproceedings{fang2018spider,
  title={Spider: Near-optimal non-convex optimization via stochastic path-integrated differential estimator},
  author={Fang, Cong and Li, Chris Junchi and Lin, Zhouchen and Zhang, Tong},
  booktitle=nipsnew,
  pages={689--699},
  year={2018}
}
@article{lan2019lectures,
  title={Lectures on Optimization. Methods for Machine Learning},
  author={Lan, G},
  journal={H. Milton Stewart School of Industrial and Systems Engineering, Georgia Institute of Technology, Atlanta, GA},
  year={2019}
}
@book{nesterov2013introductory,
  title={Introductory Lectures on Convex Optimization: A Basic Course},
  author={Nesterov, Yurii},
  volume={87},
  year={2013},
  publisher={Springer Science \& Business Media}
}

@article{jaganathan2015phase,
  title={Phase retrieval: An overview of recent developments},
  author={Jaganathan, Kishore and Eldar, Yonina C and Hassibi, Babak},
  journal={arXiv preprint arXiv:1510.07713},
  year={2015}
}
@inproceedings{zhang2016reshaped,
  title={Reshaped wirtinger flow for solving quadratic system of equations},
  author={Zhang, Huishuai and Liang, Yingbin},
  booktitle=nips,
  pages={2622--2630},
  year={2016}
}
@article{candes2015phase,
  title={Phase retrieval via Wirtinger flow: Theory and algorithms},
  author={Candes, Emmanuel J and Li, Xiaodong and Soltanolkotabi, Mahdi},
  journal={IEEE Transactions on Information Theory},
  volume={61},
  number={4},
  pages={1985--2007},
  year={2015},
  publisher={IEEE}
}
@article{chen2019gradient,
  title={Gradient descent with random initialization: Fast global convergence for nonconvex phase retrieval},
  author={Chen, Yuxin and Chi, Yuejie and Fan, Jianqing and Ma, Cong},
  journal={Mathematical Programming},
  volume={176},
  number={1-2},
  pages={5--37},
  year={2019},
  publisher={Springer}
}





@article{konda2004convergence,
  title={Convergence rate of linear two-time-scale stochastic approximation},
  author={Konda, Vijay R and Tsitsiklis, John N and others},
  journal={The Annals of Applied Probability},
  volume={14},
  number={2},
  pages={796--819},
  year={2004},
  publisher={Institute of Mathematical Statistics}
}



@article{mokkadem2006convergence,
  title={Convergence rate and averaging of nonlinear two-time-scale stochastic approximation algorithms},
  author={Mokkadem, Abdelkader and Pelletier, Mariane and others},
  journal={The Annals of Applied Probability},
  volume={16},
  number={3},
  pages={1671--1702},
  year={2006},
  publisher={Institute of Mathematical Statistics}
}

@article{doan2020nonlinear,
  title={Nonlinear Two-Time-Scale Stochastic Approximation: Convergence and Finite-Time Performance},
  author={Doan, Thinh T},
  journal={arXiv preprint arXiv:2011.01868},
  year={2020}
}

@article{xu2020sample,
  title={Sample Complexity Bounds for Two Timescale Value-based Reinforcement Learning Algorithms},
  author={Xu, Tengyu and Liang, Yingbin},
  journal={arXiv preprint arXiv:2011.05053},
  year={2020}
}

@article{perkins2002convergent,
  title={A convergent form of approximate policy iteration},
  author={Perkins, Theodore and Precup, Doina},
  journal={Advances in neural information processing systems},
  volume={15},
  pages={1627--1634},
  year={2002}
}

@article{robbins1951stochastic,
  title={A stochastic approximation method},
  author={Robbins, Herbert and Monro, Sutton},
  journal={The annals of mathematical statistics},
  pages={400--407},
  year={1951},
  publisher={JSTOR}
}

@incollection{benaim1999dynamics,
  title={Dynamics of stochastic approximation algorithms},
  author={Bena{\"\i}m, Michel},
  booktitle={Seminaire de probabilites XXXIII},
  pages={1--68},
  year={1999},
  publisher={Springer}
}

@book{kushner2003stochastic,
  title={Stochastic approximation and recursive algorithms and applications},
  author={Kushner, Harold and Yin, G George},
  volume={35},
  year={2003},
  publisher={Springer Science \& Business Media}
}



@article{kumar2019sample,
  title={On the sample complexity of actor-critic method for reinforcement learning with function approximation},
  author={Kumar, Harshat and Koppel, Alec and Ribeiro, Alejandro},
  journal={arXiv preprint arXiv:1910.08412},
  year={2019}
}
@inproceedings{qiu2019finite,
  title={On the finite-time convergence of actor-critic algorithm},
  author={Qiu, Shuang and Yang, Zhuoran and Ye, Jieping and Wang, Zhaoran},
  booktitle={Optimization Foundations for Reinforcement Learning Workshop at Advances in Neural Information Processing Systems (NeurIPS)},
  year={2019}
}



@article{archibald1995generation,
  title={{On the generation of Markov decision processes}},
  author={Archibald, TW and McKinnon, KIM and Thomas, LC},
  journal={Journal of the Operational Research Society},
  volume={46},
  number={3},
  pages={354--361},
  year={1995},
  publisher={Taylor \& Francis}
}



@article{borkar1997stochastic,
  title={Stochastic approximation with two time scales},
  author={Borkar, Vivek S},
  journal={Systems \& Control Letters},
  volume={29},
  number={5},
  pages={291--294},
  year={1997},
  publisher={Elsevier}
}

@inproceedings{tadic2004almost,
  title={Almost sure convergence of two time-scale stochastic approximation algorithms},
  author={Tadic, Vladislav B},
  booktitle={Proceedings of the 2004 American Control Conference},
  volume={4},
  pages={3802--3807},
  year={2004},
  organization={IEEE}
}


 

@inproceedings{dalal2020tale,
  title={A tale of two-timescale reinforcement learning with the tightest finite-time bound},
  author={Dalal, Gal and Szorenyi, Balazs and Thoppe, Gugan},
  booktitle=aaai,
  pages={3701--3708},
  year={2020}
}
@inproceedings{wang2019spiderboost,
  title={SpiderBoost and momentum: Faster variance reduction algorithms},
  author={Wang, Zhe and Ji, Kaiyi and Zhou, Yi and Liang, Yingbin and Tarokh, Vahid},
  booktitle=nipsnew,
  pages={2406--2416},
  year={2019}
}
@article{bhatnagar2009convergent,
  title={Convergent temporal-difference learning with arbitrary smooth function approximation},
  author={Bhatnagar, Shalabh and Precup, Doina and Silver, David and Sutton, Richard S and Maei, Hamid and Szepesv{\'a}ri, Csaba},
  journal=nips,
  volume={22},
  pages={1204--1212},
  year={2009}
}

@article{sun2019finite,
  title={Finite-Sample Analysis of Decentralized Temporal-Difference Learning with Linear Function Approximation},
  author={Sun, Jun and Wang, Gang and Giannakis, Georgios B and Yang, Qinmin and Yang, Zaiyue},
  journal={arXiv preprint arXiv:1911.00934},
  year={2019}
}
 



@article{agarwal2018fixed,
	title={Fixed Point Theory in Metric Spaces},
	author={Agarwal, Praveen and Jleli, Mohamed and Samet, Bessem},
	journal={Recent Advances and Applications},
	year={2018},
	publisher={Springer}
}
@book{lan2020first,
	title={First-order and Stochastic Optimization Methods for Machine Learning},
	author={Lan, Guanghui},
	year={2020},
	publisher={Springer}
}

@inproceedings{nachum2017bridging,
	title={Bridging the gap between value and policy based reinforcement learning},
	author={Nachum, Ofir and Norouzi, Mohammad and Xu, Kelvin and Schuurmans, Dale},
	booktitle=nips,
	year={2017}
}



@article{boob2019stochastic,
	title={Stochastic first-order methods for convex and nonconvex functional constrained optimization},
	author={Boob, Digvijay and Deng, Qi and Lan, Guanghui},
	journal={arXiv preprint arXiv:1908.02734},
	year={2019}
}
@inproceedings{lin2020gradient,
	title={On gradient descent ascent for nonconvex-concave minimax problems},
	author={Lin, Tianyi and Jin, Chi and Jordan, Michael},
	booktitle={icml},
	pages={6083--6093},
	year={2020},
	 
}

@article{williams1991function,
	title={Function optimization using connectionist reinforcement learning algorithms},
	author={Williams, Ronald J and Peng, Jing},
	journal={Connection Science},
	volume={3},
	number={3},
	pages={241--268},
	year={1991},
	publisher={Taylor \& Francis}
}
@inproceedings{peters2010relative,
	title={Relative entropy policy search},
	author={Peters, Jan and Mulling, Katharina and Altun, Yasemin},
	booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
	volume={24}, 
	year={2010}
}
@inproceedings{mnih2016asynchronous,
	title={Asynchronous methods for deep reinforcement learning},
	author={Mnih, Volodymyr and Badia, Adria Puigdomenech and Mirza, Mehdi and Graves, Alex and Lillicrap, Timothy and Harley, Tim and Silver, David and Kavukcuoglu, Koray},
	booktitle={International conference on machine learning},
	pages={1928--1937},
	year={2016},
	 
}
@inproceedings{duan2016benchmarking,
	title={Benchmarking deep reinforcement learning for continuous control},
	author={Duan, Yan and Chen, Xi and Houthooft, Rein and Schulman, John and Abbeel, Pieter},
	booktitle={International conference on machine learning},
	pages={1329--1338},
	year={2016},
	 
}
@inproceedings{haarnoja2018soft,
	title={Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor},
	author={Haarnoja, Tuomas and Zhou, Aurick and Abbeel, Pieter and Levine, Sergey},
	booktitle={International Conference on Machine Learning},
	pages={1861--1870},
	year={2018},
	 
}
@inproceedings{hazan2019provably,
	title={Provably efficient maximum entropy exploration},
	author={Hazan, Elad and Kakade, Sham and Singh, Karan and Van Soest, Abby},
	booktitle={International Conference on Machine Learning},
	pages={2681--2691},
	year={2019},
	 
}
@inproceedings{xiao2019,
	author = {Xiao, Chenjun and Huang, Ruitong and Mei, Jincheng and Schuurmans, Dale and M\"{u}ller, Martin},
	title = {Maximum Entropy {Monte-Carlo} Planning},
	booktitle = nips,
	year = {2019}
}

@book{levin2017markov,
	title={Markov Chains and Mixing Times},
	author={Levin, David A and Peres, Yuval},
	volume={107},
	year={2017},
	publisher={American Mathematical Soc.}
}

%================== Safe RL, Constrained MDP






@article{fisac2018general,
	title={A general safety framework for learning-based control in uncertain robotic systems},
	author={Fisac, Jaime F and Akametalu, Anayo K and Zeilinger, Melanie N and Kaynama, Shahab and Gillula, Jeremy and Tomlin, Claire J},
	journal={IEEE Transactions on Automatic Control},
	volume={64},
	number={7},
	pages={2737--2752},
	year={2018},
	publisher={IEEE}
}

@article{ono2015chance,
	title={Chance-constrained dynamic programming with application to risk-aware robotic space exploration},
	author={Ono, Masahiro and Pavone, Marco and Kuwata, Yoshiaki and Balaram, J},
	journal={Autonomous Robots},
	volume={39},
	number={4},
	pages={555--571},
	year={2015},
	publisher={Springer}
}

@book{altman1999constrained,
	title={Constrained {Markov} Decision Processes},
	author={Altman, Eitan},
	volume={7},
	year={1999},
	publisher={CRC Press}
}

@article{chow2017risk,
	title={Risk-constrained reinforcement learning with percentile risk criteria},
	author={Chow, Yinlam and Ghavamzadeh, Mohammad and Janson, Lucas and Pavone, Marco},
	journal={The Journal of Machine Learning Research},
	volume={18},
	number={1},
	pages={6070--6120},
	year={2017},
	publisher={JMLR}
}

@inproceedings{achiam2017constrained,
	title={Constrained policy optimization},
	author={Achiam, Joshua and Held, David and Tamar, Aviv and Abbeel, Pieter},
	booktitle=icml,
	pages={22--31},
	year={2017}
}

@misc{1606.01540,
	Author = {Greg Brockman and Vicki Cheung and Ludwig Pettersson and Jonas Schneider and John Schulman and Jie Tang and Wojciech Zaremba},
	Title = {OpenAI Gym},
	Year = {2016},
	Eprint = {arXiv:1606.01540},
}

@inproceedings{paszke2017automatic,
	title={Automatic differentiation in PyTorch},
	author={Paszke, Adam and Gross, Sam and Chintala, Soumith and Chanan, Gregory and Yang, Edward and DeVito, Zachary and Lin, Zeming and Desmaison, Alban and Antiga, Luca and Lerer, Adam},
	booktitle={NIPS Wokshop},
	year={2017}
}


@inproceedings{chow2018lyapunov,
	title={A {L}yapunov-based approach to safe reinforcement learning},
	author={Chow, Yinlam and Nachum, Ofir and Duenez-Guzman, Edgar and Ghavamzadeh, Mohammad},
	booktitle=nips,
	pages={8092--8101},
	year={2018}
}

@article{garcia2015comprehensive,
	title={A comprehensive survey on safe reinforcement learning},
	author={Garc{\'i}a, Javier and Fern{\'a}ndez, Fernando},
	journal={Journal of Machine Learning Research},
	volume={16},
	number={1},
	pages={1437--1480},
	year={2015}
}

@article{dulac2019challenges,
	title={Challenges of real-world reinforcement learning},
	author={Dulac-Arnold, Gabriel and Mankowitz, Daniel and Hester, Todd},
	journal={arXiv preprint arXiv:1904.12901},
	year={2019}
}

@article{bhandari2020note,
	title={A note on the linear convergence of policy gradient methods},
	author={Bhandari, Jalaj and Russo, Daniel},
	journal={arXiv preprint arXiv:2007.11120},
	year={2020}
}


@article{liang2018accelerated,
	title={Accelerated primal-dual policy optimization for safe reinforcement learning},
	author={Liang, Qingkai and Que, Fanyu and Modiano, Eytan},
	journal={arXiv preprint arXiv:1802.06480},
	year={2018}
}

@inproceedings{tessler2018reward,
	title={Reward constrained policy optimization},
	author={Tessler, Chen and Mankowitz, Daniel J and Mannor, Shie},
	booktitle=iclr,
	year={2018}
}



@inproceedings{yu2019convergent,
	title={Convergent policy optimization for safe reinforcement learning},
	author={Yu, Ming and Yang, Zhuoran and Kolar, Mladen and Wang, Zhaoran},
	booktitle={Proc. Advances in Neural Information Processing Systems (NeurIPS)},
	pages={3127--3139},
	year={2019}
}

@inproceedings{yang2019projection,
	title={Projection-based constrained policy optimization},
	author={Yang, Tsung-Yen and Rosca, Justinian and Narasimhan, Karthik and Ramadge, Peter J},
	booktitle=iclr,
	year={2019}
}

@inproceedings{stooke2020responsive,
	title={Responsive safety in reinforcement learning by {PID} {L}agrangian methods},
	author={Stooke, Adam and Achiam, Joshua and Abbeel, Pieter},
	booktitle=icml,  pages={9133--9143},
	year={2020}
}


@article{chow2019lyapunov,
	title={{Lyapunov}-based safe policy optimization for continuous control},
	author={Chow, Yinlam and Nachum, Ofir and Faust, Aleksandra and Duenez-Guzman, Edgar and Ghavamzadeh, Mohammad},
	journal={arXiv preprint arXiv:1901.10031},
	year={2019}
}


@article{dalal2018safe,
	title={Safe exploration in continuous action spaces},
	author={Dalal, Gal and Dvijotham, Krishnamurthy and Vecerik, Matej and Hester, Todd and Paduraru, Cosmin and Tassa, Yuval},
	journal={arXiv preprint arXiv:1801.08757},
	year={2018}
}

@article{liu2019ipo,
	title={{IPO}: interior-point policy optimization under constraints},
	author={Liu, Yongshuai and Ding, Jiaxin and Liu, Xin},
	journal={arXiv preprint arXiv:1910.09615},
	year={2019}
}

@article{ding2020provably,
	title={Provably efficient safe exploration via primal-dual policy optimization},
	author={Ding, Dongsheng and Wei, Xiaohan and Yang, Zhuoran and Wang, Zhaoran and Jovanovi{\'c}, Mihailo R},
	journal={arXiv preprint arXiv:2003.00534},
	year={2020}
}

@misc{csaba,
	title = {Constrained {MDP}s and the reward hypothesis},
	author = {Csaba Szepesv\'ari},
	howpublished = {\url{http://readingsml.blogspot.com/2020/03/constrained-mdps-and-reward-hypothesis.html}},
	year= {2020},
	note = {Accessed: 2021-01-27}
}

%================== policy gradient

@article{shani2020adaptive,
	title={Adaptive trust region policy optimization: {G}lobal convergence and faster rates for regularized {MDPs}},
	author={Shani, Lior and Efroni, Yonathan and Mannor, Shie},
	journal=aaai,
	year={2020}
}


@article{agarwal2019optimality,
	title={Optimality and approximation with policy gradient methods in {Markov} decision processes},
	author={Agarwal, Alekh and Kakade, Sham M and Lee, Jason D and Mahajan, Gaurav},
	journal={arXiv preprint arXiv:1908.00261},
	year={2019}
}

@article{mei2020on,
	title={On the global convergence rates of softmax policy gradient methods},
	author={Mei, J. and Xiao, C. and Szepesvari, C. and Schuurmans, D.},
	journal={arXiv preprint arXiv:2005.06392},
	year={2020}
}
@article{ouyang2019lower,
	title={Lower complexity bounds of first-order methods for convex-concave bilinear saddle-point problems},
	author={Ouyang, Yuyuan and Xu, Yangyang},
	journal={Mathematical Programming},
	pages={1--35},
	year={2019},
	publisher={Springer}
}
@article{nemirovsky1992information,
	title={Information-based complexity of linear operator equations},
	author={Nemirovsky, Arkadi S},
	journal={Journal of Complexity},
	volume={8},
	number={2},
	pages={153--175},
	year={1992},
	publisher={Academic Press}
}

@inproceedings{liu2019neural,
	title={Neural proximal/trust region policy optimization attains globally optimal policy},
	author={Liu, Boyi and Cai, Qi and Yang, Zhuoran and Wang, Zhaoran},
	booktitle={In Proc. Advances in Neural Information Processing Systems (NeurIPS)},
	volume={32},
	year={2019}
}

@article{cartis2014complexity,
	title={On the complexity of finding first-order critical points in constrained nonlinear optimization},
	author={Cartis, Coralia and Gould, Nicholas IM and Toint, Philippe L},
	journal={Mathematical Programming},
	volume={144},
	number={1},
	pages={93--106},
	year={2014},
	publisher={Springer}
}
@article{facchinei2021ghost,
	title={Ghost penalties in nonconvex constrained optimization: Diminishing stepsizes and iteration complexity},
	author={Facchinei, Francisco and Kungurtsev, Vyacheslav and Lampariello, Lorenzo and Scutari, Gesualdo},
	journal={Mathematics of Operations Research},
	year={2021},
	publisher={INFORMS}
}

@article{wang2017penalty,
	title={Penalty methods with stochastic approximation for stochastic nonlinear programming},
	author={Wang, Xiao and Ma, Shiqian and Yuan, Ya-xiang},
	journal={Mathematics of computation},
	volume={86},
	number={306},
	pages={1793--1820},
	year={2017}
}

@article{LanZhouCSA2020,
	title="Algorithms for stochastic optimization with function or expectation constraints",
	author="G. Lan and Z. Zhou",
	journal="Computational Optimization and Applications",
	volume="76",
	pages="461--498",
	year="2020"
}
@article{LanPMD2021,
	title="Policy Mirror Descent for Reinforcement Learning: Linear Convergence, New Sampling Complexity, and Generalized Problem Classes",
	author="G. Lan",
	journal={arXiv preprint arXiv:2102.00135},
	year={2021}
}

@article{chen2021marl,
	title= "Sample and Communication-Efficient Decentralized Actor-Critic Algorithms with Finite-Time Analysis",
	author="Ziyi Chen and Yi Zhou and Rong-Rong Chen and Shaofeng Zou",
	journal={submitted to NeurIPS},
	year={2021},
}
@article{paternain2019safe,
	title={Safe policies for reinforcement learning via primal-dual methods},
	author={Paternain, Santiago and Calvo-Fullana, Miguel and Chamon, Luiz FO and Ribeiro, Alejandro},
	journal={arXiv preprint arXiv:1911.09101},
	year={2019}
}
@article{efroni2020exploration,
	title={Exploration-exploitation in constrained {MDP}s},
	author={Efroni, Yonathan and Mannor, Shie and Pirotta, Matteo},
	journal={arXiv preprint arXiv:2003.02189},
	year={2020}
}

@inproceedings{xu2021new,
	title={{CRPO}: A New Approach for Safe Reinforcement Learning with Convergence Guarantee},
	author={Tengyu Xu and Yingbin Liang and Guanghui Lan},
	booktitle=icml,
	year={2021},  pages={11480--11491},
}

@article{borkar2005actor,
	title={An actor-critic algorithm for constrained {Markov} decision processes},
	author={Borkar, Vivek S},
	journal={Systems \& control letters},
	volume={54},
	number={3},
	pages={207--213},
	year={2005},
	publisher={Elsevier}
}
@article{bhatnagar2012online,
	title={An online actor--critic algorithm with function approximation for constrained {M}arkov decision processes},
	author={Bhatnagar, Shalabh and Lakshmanan, K},
	journal={Journal of Optimization Theory and Applications},
	volume={153},
	number={3},
	pages={688--708},
	year={2012},
	publisher={Springer}
}
@inproceedings{fazel2018global,
	title={Global convergence of policy gradient methods for the linear quadratic regulator},
	author={Fazel, Maryam and Ge, Rong and Kakade, Sham and Mesbahi, Mehran},
	booktitle=icml,
	pages={1467--1476},
	year={2018},
}
@article{williams1992simple,
	title={Simple statistical gradient-following algorithms for connectionist reinforcement learning},
	author={Williams, Ronald J},
	journal={Machine learning},
	volume={8},
	number={3-4},
	pages={229--256},
	year={1992},
	publisher={Springer}
}
@inproceedings{konda2000actor,
	title={Actor-critic algorithms},
	author={Konda, Vijay R and Tsitsiklis, John N},
	booktitle=nips,
	pages={1008--1014},
	year={2000},

}
@inproceedings{xu2020primal,
	title={A primal approach to constrained policy optimization: Global optimality and finite-time analysis},
	author={Xu, Tengyu and Liang, Yingbin and Lan, Guanghui},
	booktitle=icml,
	year={2021}
}
@article{lan2016algorithms,
	title={Algorithms for stochastic optimization with functional or expectation constraints},
	author={Lan, Guanghui and Zhou, Zhiqiang},
	journal={arXiv preprint arXiv:1604.03887},
	year={2016}
}
@article{schulman2017proximal,
	title={Proximal policy optimization algorithms},
	author={Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
	journal={arXiv preprint arXiv:1707.06347},
	year={2017}
}
@article{kurach2019google,
	title={Google research football: A novel reinforcement learning environment},
	author={Kurach, Karol and Raichuk, Anton and Sta{\'n}czyk, Piotr and Zaj{k{a}}c, Micha{l} and Bachem, Olivier and Espeholt, Lasse and Riquelme, Carlos and Vincent, Damien and Michalski, Marcin and Bousquet, Olivier  },
	journal={arXiv preprint arXiv:1907.11180},
	year={2019}
}
@article{silver2017mastering,
	title={Mastering chess and shogi by self-play with a general reinforcement learning algorithm},
	author={Silver, David and Hubert, Thomas and Schrittwieser, Julian and Antonoglou, Ioannis and Lai, Matthew and Guez, Arthur and Lanctot, Marc and Sifre, Laurent and Kumaran, Dharshan and Graepel, Thore and others},
	journal={arXiv preprint arXiv:1712.01815},
	year={2017}
}
@article{levine2016end,
	title={End-to-end training of deep visuomotor policies},
	author={Levine, Sergey and Finn, Chelsea and Darrell, Trevor and Abbeel, Pieter},
	journal={The Journal of Machine Learning Research},
	volume={17},
	number={1},
	pages={1334--1373},
	year={2016},
}
@article{arel2010reinforcement,
	title={Reinforcement learning-based multi-agent system for network traffic signal control},
	author={Arel, Itamar and Liu, Cong and Urbanik, Tom and Kohls, Airton G},
	journal={IET Intelligent Transport Systems},
	volume={4},
	number={2},
	pages={128--135},
	year={2010},
	publisher={IET}
}
@article{gasparik2018safety,
	title={Safety-first ai for autonomous data centre cooling and industrial control},
	author={Gasparik, A and Gamble, C and Gao, J},
	journal={DeepMind blog},
	year={2018}
}
@article{yu2019reinforcement,
	title={Reinforcement learning in healthcare: A survey},
	author={Yu, Chao and Liu, Jiming and Nemati, Shamim},
	journal={arXiv preprint arXiv:1908.08796},
	year={2019}
}
@article{kiran2020deep,
	title={Deep Reinforcement Learning for Autonomous Driving: A Survey},
	author={Kiran, B Ravi and Sobh, Ibrahim and Talpaert, Victor and Mannion, Patrick and Al Sallab, Ahmad A and Yogamani, Senthil and P{\'e}rez, Patrick},
	journal={arXiv preprint arXiv:2002.00444},
	year={2020}
}
@article{mannor2004geometric,
	title={A geometric approach to multi-criterion reinforcement learning},
	author={Mannor, Shie and Shimkin, Nahum},
	journal={The Journal of Machine Learning Research},
	volume={5},
	pages={325--360},
	year={2004},
	publisher={JMLR. org}
}
@article{van2014multi,
	title={Multi-objective reinforcement learning using sets of {P}areto dominating policies},
	author={Van Moffaert, Kristof and Now{\'e}, Ann},
	journal={The Journal of Machine Learning Research},
	volume={15},
	number={1},
	pages={3483--3512},
	year={2014},
	publisher={JMLR}
}
@article{leike2017ai,
	title={{AI} safety gridworlds},
	author={Leike, Jan and Martic, Miljan and Krakovna, Victoria and Ortega, Pedro A and Everitt, Tom and Lefrancq, Andrew and Orseau, Laurent and Legg, Shane},
	journal={arXiv preprint arXiv:1711.09883},
	year={2017}
}
@article{mania2018simple,
	title={Simple random search provides a competitive approach to reinforcement learning},
	author={Mania, Horia and Guy, Aurelia and Recht, Benjamin},
	journal={arXiv preprint arXiv:1803.07055},
	year={2018}
}

@article{xu2012distributionally,
	title={Distributionally robust {Markov} decision processes},
	author={Xu, Huan and Mannor, Shie},
	journal={Mathematics of Operations Research},
	volume={37},
	number={2},
	pages={288--300},
	year={2012},
	publisher={Informs}
}
@article{yu2015distributionally,
	title={Distributionally robust counterpart in {Markov} decision processes},
	author={Yu, Pengqian and Xu, Huan},
	journal={IEEE Transactions on Automatic Control},
	volume={61},
	number={9},
	pages={2538--2543},
	year={2015},
	publisher={IEEE}
}
@inproceedings{lim2019kernel,
	title={Kernel-based reinforcement learning in robust {Markov} decision processes},
	author={Lim, Shiau Hong and Autef, Arnaud},
	booktitle=icml,
	pages={3973--3981},
	year={2019}
}
@inproceedings{badrinath2021robust,
	title={Robust Reinforcement Learning using Least Squares Policy Iteration with Provable Performance Guarantees},
	author={Badrinath, Kishan Panaganti and Kalathil, Dileep},
	booktitle=icml,
	pages={511--520},
	year={2021},
	 
}
@article{gretton2012kernel,
	title={A kernel two-sample test},
	author={Gretton, Arthur and Borgwardt, Karsten M and Rasch, Malte J and Sch{\"o}lkopf, Bernhard and Smola, Alexander},
	journal={The Journal of Machine Learning Research},
	volume={13},
	number={1},
	pages={723--773},
	year={2012},
	publisher={JMLR}
}
@incollection{kuhn2019wasserstein,
	title={Wasserstein distributionally robust optimization: Theory and applications in machine learning},
	author={Kuhn, Daniel and Esfahani, Peyman Mohajerin and Nguyen, Viet Anh and Shafieezadeh-Abadeh, Soroosh},
	booktitle={Operations Research \& Management Science in the Age of Analytics},
	pages={130--166},
	year={2019},
	publisher={INFORMS}
}

@INPROCEEDINGS{staib2019distribution,
	author = {{Staib}, M. and {Jegelka}, S.},
	title = "{Distributionally Robust Optimization and Generalization in Kernel Methods}",
	booktitle = nipsnew,
	year = 2019,
	pages = {9131-9141}
}
@inproceedings{gao2018wasserstein,
	author = {{Gao}, R. and {Xie}, L. and {Xie}, Y. and {Xu}, H.},
	title = {Robust Hypothesis Testing Using {Wasserstein} Uncertainty Sets},
	booktitle = nipsnew,
	year = {2018},
	pages = {7902-7912}
}
@article{rahimian2019distributionally,
	title={Distributionally robust optimization: A review},
	author={Rahimian, Hamed and Mehrotra, Sanjay},
	journal={arXiv preprint arXiv:1908.05659},
	year={2019}
}

@article{ray2019benchmarking,
	title={Benchmarking safe exploration in deep reinforcement learning},
	author={Ray, Alex and Achiam, Joshua and Amodei, Dario},
	journal={arXiv preprint arXiv:1910.01708},
	volume={7},
	year={2019}
}

@inproceedings{ma2021greedy,
  title={Greedy-GQ with variance reduction: Finite-time analysis and improved complexity},
  author={Ma, Shaocong and Chen, Ziyi and Zhou, Yi and Zou, Shaofeng},
  booktitle=iclr,
  year={2021}
}
@inproceedings{bhandari2021linear,
  title={On the Linear Convergence of Policy Gradient Methods for Finite {MDP}s},
  author={Bhandari, Jalaj and Russo, Daniel},
  booktitle=aistats,
  pages={2386--2394},
  year={2021},
  organization={PMLR}
}

@article{li2021softmax,
  title={Softmax policy gradient methods can take exponential time to converge},
  author={Li, Gen and Wei, Yuting and Chi, Yuejie and Gu, Yuantao and Chen, Yuxin},
  journal={arXiv preprint arXiv:2102.11270},
  year={2021}
}
@inproceedings{laroche2021dr,
  title={Dr Jekyll \& Mr Hyde: the strange case of off-policy policy updates},
  author={Laroche, Romain and des Combes, Remi Tachet},
  booktitle=nipsnew,
  year={2021}
}
@article{zhang2021global,
  title={Global Optimality and Finite Sample Analysis of Softmax Off-Policy Actor Critic under State Distribution Mismatch},
  author={Zhang, Shangtong and Tachet, Remi and Laroche, Romain},
  journal={arXiv preprint arXiv:2111.02997},
  year={2021}
}
@article{cen2021fast,
  title={Fast global convergence of natural policy gradient methods with entropy regularization},
  author={Cen, Shicong and Cheng, Chen and Chen, Yuxin and Wei, Yuting and Chi, Yuejie},
  journal={Operations Research},
  year={2021},
  publisher={INFORMS}
}
@inproceedings{zhang2020variational,
  title={Variational Policy Gradient Method for Reinforcement Learning with General Utilities},
  author={Zhang, Junyu and Koppel, Alec and Bedi, Amrit Singh and Szepesvari, Csaba and Wang, Mengdi},
  booktitle=nipsnew,
  volume={33},
  pages={4572--4583},
  year={2020}
}
@article{xiao2022On,
  title={On the Convergence Rates of Policy Gradient Methods},
  author={Lin, Xiao},
  journal={arXiv preprint arXiv:2201.07443},
  year={2022}
}
@article{tsitsiklis1999average,
  title={Average cost temporal-difference learning},
  author={Tsitsiklis, John N and Van Roy, Benjamin},
  journal={Automatica},
  volume={35},
  number={11},
  pages={1799--1808},
  year={1999},
  publisher={Elsevier}
}

@inproceedings{zhao2020sim,
  title={Sim-to-real transfer in deep reinforcement learning for robotics: a survey},
  author={Zhao, Wenshuai and Queralta, Jorge Pe{\~n}a and Westerlund, Tomi},
  booktitle={2020 IEEE symposium series on computational intelligence (SSCI)},
  pages={737--744},
  year={2020},
  organization={IEEE}
}