@inproceedings{fu2020single,
	title        = {Single-Timescale Actor-Critic Provably Finds Globally Optimal Policy},
	author       = {Zuyue Fu and Zhuoran Yang and Zhaoran Wang},
	year         = 2021,
	booktitle    = {9th International Conference on Learning Representations, {ICLR}}
}

@article{Agarwal2021OntheTheory,
	title        = {On the Theory of Policy Gradient Methods: Optimality, Approximation, and Distribution Shift},
	author       = {Alekh Agarwal and Sham M. Kakade and Jason D. Lee and Gaurav Mahajan},
	year         = 2021,
	journal      = {Journal of Machine Learning Research},
	volume       = 22,
	number       = 98,
	pages        = {1--76},
	url          = {http://jmlr.org/papers/v22/19-736.html}
}

@inproceedings{10.1007/11564096_32,
	title        = {Neural Fitted q Iteration – First Experiences with a Data Efficient Neural Reinforcement Learning Method},
	author       = {Riedmiller, Martin},
	year         = 2005,
	booktitle    = {Proceedings of the 16th European Conference on Machine Learning},
	location     = {Porto, Portugal},
	publisher    = {Springer-Verlag},
	address      = {Berlin, Heidelberg},
	series       = {ECML'05},
	pages        = {317–328},
	doi          = {10.1007/11564096_32},
	isbn         = 3540292438,
	url          = {https://doi.org/10.1007/11564096_32},
	numpages     = 12
}

@article{10.1007/BF00992696,
	title        = {Simple Statistical Gradient-Following Algorithms for Connectionist Reinforcement Learning},
	author       = {Williams, Ronald J.},
	year         = 1992,
	month        = {may},
	journal      = {Mach. Learn.},
	publisher    = {Kluwer Academic Publishers},
	address      = {USA},
	volume       = 8,
	number       = {3–4},
	pages        = {229–256},
	doi          = {10.1007/BF00992696},
	issn         = {0885-6125},
	url          = {https://doi.org/10.1007/BF00992696},
	issue_date   = {May 1992},
	numpages     = 28,
}

@article{10.1007/BF00992699,
	title        = {Self-Improving Reactive Agents Based on Reinforcement Learning, Planning and Teaching},
	author       = {Lin, Long-Ji},
	year         = 1992,
	month        = {may},
	journal      = {Mach. Learn.},
	publisher    = {Kluwer Academic Publishers},
	address      = {USA},
	volume       = 8,
	number       = {3–4},
	pages        = {293–321},
	doi          = {10.1007/BF00992699},
	issn         = {0885-6125},
	url          = {https://doi.org/10.1007/BF00992699},
	issue_date   = {May 1992},
	numpages     = 29,
}

@article{10.1023/A:1022693225949,
	title        = {An Upper Bound on the Loss from Approximate Optimal-Value Functions},
	author       = {Singh, Satinder P. and Yee, Richard C.},
	year         = 1994,
	month        = {sep},
	journal      = {Mach. Learn.},
	publisher    = {Kluwer Academic Publishers},
	address      = {USA},
	volume       = 16,
	number       = 3,
	pages        = {227–233},
	doi          = {10.1023/A:1022693225949},
	issn         = {0885-6125},
	url          = {https://doi.org/10.1023/A:1022693225949},
	issue_date   = {Sept. 1994},
	numpages     = 7,
}

@inproceedings{10.1145/2463372.2463509,
	title        = {Evolving Large-Scale Neural Networks for Vision-Based Reinforcement Learning},
	author       = {Koutn\'{\i}k, Jan and Cuccu, Giuseppe and Schmidhuber, J\"{u}rgen and Gomez, Faustino},
	year         = 2013,
	booktitle    = {Proceedings of the 15th Annual Conference on Genetic and Evolutionary Computation},
	location     = {Amsterdam, The Netherlands},
	publisher    = {Association for Computing Machinery},
	address      = {New York, NY, USA},
	series       = {GECCO '13},
	pages        = {1061–1068},
	doi          = {10.1145/2463372.2463509},
	isbn         = 9781450319638,
	url          = {https://doi.org/10.1145/2463372.2463509},
	numpages     = 8,
}

@inproceedings{10.1145/3387168.3387199,
	title        = {Twin-Delayed DDPG: A Deep Reinforcement Learning Technique to Model a Continuous Movement of an Intelligent Robot Agent},
	author       = {Dankwa, Stephen and Zheng, Wenfeng},
	year         = 2020,
	booktitle    = {Proceedings of the 3rd International Conference on Vision, Image and Signal Processing},
	location     = {Vancouver, BC, Canada},
	publisher    = {Association for Computing Machinery},
	address      = {New York, NY, USA},
	series       = {ICVISP 2019},
	doi          = {10.1145/3387168.3387199},
	isbn         = 9781450376259,
	articleno    = 66,
	numpages     = 5,
}

@book{10.5555/1512940,
	title        = {Stochastic Optimal Control: The Discrete-Time Case},
	author       = {Bertsekas, Dimitri P. and Shreve, Steven E.},
	year         = 2007,
	publisher    = {Athena Scientific},
	isbn         = 1886529035
}

@article{10.5555/1577069.1755882,
	title        = {Efficient Online and Batch Learning Using Forward Backward Splitting},
	author       = {Duchi, John and Singer, Yoram},
	year         = 2009,
	month        = {dec},
	journal      = {J. Mach. Learn. Res.},
	publisher    = {JMLR.org},
	volume       = 10,
	pages        = {2899–2934},
	issn         = {1532-4435},
	issue_date   = {12/1/2009},
	numpages     = 36
}

@book{10.5555/26851,
	title        = {Real and complex analysis, 3rd ed.},
	author       = {Rudin, Walter},
	year         = 1987,
	publisher    = {McGraw-Hill, Inc.},
	address      = {USA},
	isbn         = {0070542341}
}

@inproceedings{10.5555/2976248.2976264,
	title        = {Convex Neural Networks},
	author       = {Bengio, Yoshua and Roux, Nicolas Le and Vincent, Pascal and Delalleau, Olivier and Marcotte, Patrice},
	year         = 2005,
	booktitle    = {Proceedings of the 18th International Conference on Neural Information Processing Systems},
	location     = {Vancouver, British Columbia, Canada},
	publisher    = {MIT Press},
	address      = {Cambridge, MA, USA},
	series       = {NIPS'05},
	pages        = {123–130},
	numpages     = 8
}

@inproceedings{10.5555/2981562.2981564,
	title        = {Fitted Q-Iteration in Continuous Action-Space MDPs},
	author       = {Antos, Andr\'{a}s and Munos, R\'{e}mi and Szepesv\'{a}ri, Csaba},
	year         = 2007,
	booktitle    = {Proceedings of the 20th International Conference on Neural Information Processing Systems},
	location     = {Vancouver, British Columbia, Canada},
	publisher    = {Curran Associates Inc.},
	address      = {Red Hook, NY, USA},
	series       = {NIPS'07},
	pages        = {9–16},
	isbn         = 9781605603520,
	numpages     = 8
}

@inproceedings{10.5555/2986459.2986717,
	title        = {Improved Algorithms for Linear Stochastic Bandits},
	author       = {Abbasi-Yadkori, Yasin and P\'{a}l, D\'{a}vid and Szepesv\'{a}ri, Csaba},
	year         = 2011,
	booktitle    = {Proceedings of the 24th International Conference on Neural Information Processing Systems},
	location     = {Granada, Spain},
	publisher    = {Curran Associates Inc.},
	address      = {Red Hook, NY, USA},
	series       = {NIPS'11},
	pages        = {2312–2320},
	isbn         = 9781618395993,
	numpages     = 9
}

@inproceedings{10.5555/3045390.3045594,
	title        = {Asynchronous Methods for Deep Reinforcement Learning},
	author       = {Mnih, Volodymyr and Badia, Adri\`{a} Puigdom\`{e}nech and Mirza, Mehdi and Graves, Alex and Harley, Tim and Lillicrap, Timothy P. and Silver, David and Kavukcuoglu, Koray},
	year         = 2016,
	booktitle    = {Proceedings of the 33rd International Conference on International Conference on Machine Learning - Volume 48},
	location     = {New York, NY, USA},
	publisher    = {JMLR.org},
	series       = {ICML'16},
	pages        = {1928–1937},
	numpages     = 10
}

@inproceedings{10.5555/3327144.3327299,
	title        = {Lipschitz Regularity of Deep Neural Networks: Analysis and Efficient Estimation},
	author       = {Scaman, Kevin and Virmaux, Aladin},
	year         = 2018,
	booktitle    = {Proceedings of the 32nd International Conference on Neural Information Processing Systems},
	location     = {Montr\'{e}al, Canada},
	publisher    = {Curran Associates Inc.},
	address      = {Red Hook, NY, USA},
	series       = {NIPS'18},
	pages        = {3839–3848},
	numpages     = 10
}

BibTeX
@article{10.5555/944919.944944,
	title        = {Rademacher and Gaussian Complexities: Risk Bounds and Structural Results},
	author       = {Bartlett, Peter L. and Mendelson, Shahar},
	year         = 2003,
	month        = {mar},
	journal      = {J. Mach. Learn. Res.},
	publisher    = {JMLR.org},
	volume       = 3,
	number       = {null},
	pages        = {463–482},
	issn         = {1532-4435},
	issue_date   = {3/1/2003},
	numpages     = 20,
}

BibTeX
@inproceedings{7798980,
	title        = {Learning state representation for deep actor-critic control},
	author       = {Munk, Jelle and Kober, Jens and Babuška, Robert},
	year         = 2016,
	booktitle    = {2016 IEEE 55th Conference on Decision and Control (CDC)},
	volume       = {},
	number       = {},
	pages        = {4667--4673},
	doi          = {10.1109/CDC.2016.7798980}
}

@article{8100645,
	title        = {User Scheduling and Resource Allocation in HetNets With Hybrid Energy Supply: An Actor-Critic Reinforcement Learning Approach},
	author       = {Wei, Yifei and Yu, F. Richard and Song, Mei and Han, Zhu},
	year         = 2018,
	journal      = {IEEE Transactions on Wireless Communications},
	volume       = 17,
	number       = 1,
	pages        = {680--692},
	doi          = {10.1109/TWC.2017.2769644}
}

@article{8513863,
	title        = {Joint Optimization of Caching, Computing, and Radio Resources for Fog-Enabled IoT Using Natural Actor–Critic Deep Reinforcement Learning},
	author       = {Wei, Yifei and Yu, F. Richard and Song, Mei and Han, Zhu},
	year         = 2019,
	journal      = {IEEE Internet of Things Journal},
	volume       = 6,
	number       = 2,
	pages        = {2061--2073},
	doi          = {10.1109/JIOT.2018.2878435}
}

@article{8731635,
	title        = {Deep Deterministic Policy Gradient (DDPG)-Based Energy Harvesting Wireless Communications},
	author       = {Qiu, Chengrun and Hu, Yang and Chen, Yan and Zeng, Bing},
	year         = 2019,
	journal      = {IEEE Internet of Things Journal},
	volume       = 6,
	number       = 5,
	pages        = {8577--8588},
	doi          = {10.1109/JIOT.2019.2921159}
}

@article{8793143,
	title        = {DeepPool: Distributed Model-Free Algorithm for Ride-Sharing Using Deep Reinforcement Learning},
	author       = {Al-Abbasi, Abubakr O. and Ghosh, Arnob and Aggarwal, Vaneet},
	year         = 2019,
	journal      = {IEEE Transactions on Intelligent Transportation Systems},
	volume       = 20,
	number       = 12,
	pages        = {4714--4727},
	doi          = {10.1109/TITS.2019.2931830}
}

@article{8896945,
	title        = {A Deep Actor-Critic Reinforcement Learning Framework for Dynamic Multichannel Access},
	author       = {Zhong, Chen and Lu, Ziyang and Gursoy, M. Cenk and Velipasalar, Senem},
	year         = 2019,
	journal      = {IEEE Transactions on Cognitive Communications and Networking},
	volume       = 5,
	number       = 4,
	pages        = {1125--1139},
	doi          = {10.1109/TCCN.2019.2952909},
}

@inproceedings{8919777,
	title        = {Deep Q-Learning for Chunk-based Caching in Data Processing Networks},
	author       = {Wang, Yimeng and Li, Yongbo and Lan, Tian and Aggarwal, Vaneet},
	year         = 2019,
	booktitle    = {2019 57th Annual Allerton Conference on Communication, Control, and Computing (Allerton)},
	volume       = {},
	number       = {},
	pages        = {910--916},
	doi          = {10.1109/ALLERTON.2019.8919777}
}

@article{9066984,
	title        = {Actor-Critic Deep Reinforcement Learning for Solving Job Shop Scheduling Problems},
	author       = {Liu, Chien-Liang and Chang, Chuan-Chin and Tseng, Chun-Jan},
	year         = 2020,
	journal      = {IEEE Access},
	volume       = 8,
	number       = {},
	pages        = {71752--71762},
	doi          = {10.1109/ACCESS.2020.2987820},
}

@inproceedings{9101333,
	title        = {Application of Deep Q-Network in Portfolio Management},
	author       = {Gao, Ziming and Gao, Yuan and Hu, Yi and Jiang, Zhengyong and Su, Jionglong},
	year         = 2020,
	booktitle    = {2020 5th IEEE International Conference on Big Data Analytics (ICBDA)},
	volume       = {},
	number       = {},
	pages        = {268--275},
	doi          = {10.1109/ICBDA49040.2020.9101333}
}

@article{9351818,
	title        = {Deep Reinforcement Learning for Autonomous Driving: A Survey},
	author       = {Kiran, B Ravi and Sobh, Ibrahim and Talpaert, Victor and Mannion, Patrick and Sallab, Ahmad A. Al and Yogamani, Senthil and Pérez, Patrick},
	year         = 2022,
	journal      = {IEEE Transactions on Intelligent Transportation Systems},
	volume       = 23,
	number       = 6,
	pages        = {4909--4926},
	doi          = {10.1109/TITS.2021.3054625}
}

@article{9576103,
	title        = {Deep Deterministic Policy Gradient With Compatible Critic Network},
	author       = {Wang, Di and Hu, Mengqi},
	year         = 2023,
	journal      = {IEEE Transactions on Neural Networks and Learning Systems},
	volume       = 34,
	number       = 8,
	pages        = {4332--4344},
	doi          = {10.1109/TNNLS.2021.3117790},
}

@inproceedings{9580255,
	title        = {Service Function Chaining in NFV-Enabled Edge Networks with Natural Actor-Critic Deep Reinforcement Learning},
	author       = {Wang, Ruijie and Li, Junhuai and Wang, Kan and Liu, Xuan and Lit, Xuan},
	year         = 2021,
	booktitle    = {2021 IEEE/CIC International Conference on Communications in China (ICCC)},
	volume       = {},
	number       = {},
	pages        = {1095--1100},
	doi          = {10.1109/ICCC52777.2021.9580255}
}

@article{9769873,
	title        = {Finite-Time Analysis of Markov Gradient Descent},
	author       = {Doan, Thinh T.},
	year         = 2022,
	journal      = {IEEE Transactions on Automatic Control},
	publisher    = {IEEE},
	volume       = {},
	number       = {},
	pages        = {1--1},
	doi          = {10.1109/TAC.2022.3172593}
}

@misc{agarwal2020flambe,
	title        = {FLAMBE: Structural Complexity and Representation Learning of Low Rank MDPs},
	author       = {Alekh Agarwal and Sham Kakade and Akshay Krishnamurthy and Wen Sun},
	year         = 2020,
	eprint       = {2006.10814},
	archiveprefix = {arXiv},
	primaryclass = {cs.LG}
}

@inproceedings{agarwal2020optimality,
	title        = {Optimality and approximation with policy gradient methods in markov decision processes},
	author       = {Agarwal, Alekh and Kakade, Sham M and Lee, Jason D and Mahajan, Gaurav},
	year         = 2020,
	journal      = {Journal of Machine Learning Research},
	booktitle    = {Conference on Learning Theory},
	volume       = 22,
	number       = 98,
	pages        = {64--66}
}

@article{agarwal2021concave,
	title        = {Concave Utility Reinforcement Learning with Zero-Constraint Violations},
	author       = {Agarwal, Mridul and Bai, Qinbo and Aggarwal, Vaneet},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2109.05439}
}

@article{agarwal2021online,
	title        = {Online target q-learning with reverse experience replay: Efficiently finding the optimal policy for linear mdps},
	author       = {Agarwal, Naman and Chaudhuri, Syomantak and Jain, Prateek and Nagaraj, Dheeraj and Netrapalli, Praneeth},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2110.08440}
}

@article{agarwal2021theory,
	title        = {On the theory of policy gradient methods: Optimality, approximation, and distribution shift},
	author       = {Agarwal, Alekh and Kakade, Sham M and Lee, Jason D and Mahajan, Gaurav},
	year         = 2021,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLRORG},
	volume       = 22,
	number       = 1,
	pages        = {4431--4506}
}

@article{agarwal2022concave,
	title        = {Concave Utility Reinforcement Learning with Zero-Constraint Violations},
	author       = {Agarwal, Mridul and Bai, Qinbo and Aggarwal, Vaneet},
	year         = 2022,
	journal      = {Transactions on Machine Learning Research}
}

@inproceedings{agarwal2022multi,
	title        = {Multi-objective reinforcement learning with non-linear scalarization},
	author       = {Agarwal, Mridul and Aggarwal, Vaneet and Lan, Tian},
	year         = 2022,
	booktitle    = {Proceedings of the 21st International Conference on Autonomous Agents and Multiagent Systems},
	pages        = {9--17}
}

@article{agarwal2023reinforcement,
	title        = {Reinforcement learning for joint optimization of multiple rewards},
	author       = {Agarwal, Mridul and Aggarwal, Vaneet},
	year         = 2023,
	journal      = {Journal of Machine Learning Research},
	volume       = 24,
	number       = 49,
	pages        = {1--41}
}

@article{agrawal2017optimistic,
	title        = {Optimistic posterior sampling for reinforcement learning: worst-case regret bounds},
	author       = {Agrawal, Shipra and Jia, Randy},
	year         = 2017,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 30
}

@article{al2019deeppool,
	title        = {Deeppool: Distributed model-free algorithm for ride-sharing using deep reinforcement learning},
	author       = {Al-Abbasi, Abubakr O and Ghosh, Arnob and Aggarwal, Vaneet},
	year         = 2019,
	journal      = {IEEE Transactions on Intelligent Transportation Systems},
	publisher    = {IEEE},
	volume       = 20,
	number       = 12,
	pages        = {4714--4727}
}

@article{article,
	title        = {Rademacher complexity for Markov chains: Applications to kernel smoothing and Metropolis–Hastings},
	author       = {Bertail, Patrice and Portier, François},
	year         = 2019,
	month        = 11,
	journal      = {Bernoulli},
	volume       = 25,
	pages        = {3912--3938},
	doi          = {10.3150/19-BEJ1115}
}

@article{auer2008near,
	title        = {Near-optimal regret bounds for reinforcement learning},
	author       = {Auer, Peter and Jaksch, Thomas and Ortner, Ronald},
	year         = 2008,
	journal      = {Advances in neural information processing systems},
	volume       = 21
}

@misc{azar2017minimax,
	title        = {Minimax Regret Bounds for Reinforcement Learning},
	author       = {Mohammad Gheshlaghi Azar and Ian Osband and Rémi Munos},
	year         = 2017,
	booktitle    = {International conference on machine learning},
	pages        = {263--272},
	eprint       = {1703.05449},
	archiveprefix = {arXiv},
	primaryclass = {stat.ML},
	organization = {PMLR}
}

@article{babanezhad2015stopwasting,
	title        = {Stopwasting my gradients: Practical svrg},
	author       = {Babanezhad Harikandeh, Reza and Ahmed, Mohamed Osama and Virani, Alim and Schmidt, Mark and Kone{\v{c}}n{\`y}, Jakub and Sallinen, Scott},
	year         = 2015,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 28
}

@inproceedings{bahdanau2017an,
	title        = {An Actor-Critic Algorithm for Sequence Prediction},
	author       = {Dzmitry Bahdanau and Philemon Brakel and Kelvin Xu and Anirudh Goyal and Ryan Lowe and Joelle Pineau and Aaron Courville and Yoshua Bengio},
	year         = 2017,
	booktitle    = {International Conference on Learning Representations},
	url          = {https://openreview.net/forum?id=SJDaqqveg}
}

@article{Bai_Bedi_Agarwal_Koppel_Aggarwal_2022,
	title        = {Achieving Zero Constraint Violation for Constrained Reinforcement Learning via Primal-Dual Approach},
	author       = {Bai, Qinbo and Bedi, Amrit Singh and Agarwal, Mridul and Koppel, Alec and Aggarwal, Vaneet},
	year         = 2022,
	month        = {Jun.},
	journal      = {Proceedings of the AAAI Conference on Artificial Intelligence},
	volume       = 36,
	pages        = {3682--3689},
	doi          = {10.1609/aaai.v36i4.20281}
}

@inproceedings{bai2023regret,
	title        = {Regret analysis of policy gradient algorithm for infinite horizon average reward Markov decision processes},
	author       = {Bai, Qinbo and Mondal, Washim Uddin and Aggarwal, Vaneet},
	year         = 2024,
	booktitle    = {AAAI Conference on Artificial Intelligence}
}

@inproceedings{balkanski2017sample,
	title        = {The sample complexity of optimizing a convex function},
	author       = {Balkanski, Eric and Singer, Yaron},
	year         = 2017,
	booktitle    = {Conference on Learning Theory},
	pages        = {275--301}
}

@inproceedings{bedi2022hidden,
	title        = {On the hidden biases of policy mirror ascent in continuous action spaces},
	author       = {Bedi, Amrit Singh and Chakraborty, Souradip and Parayil, Anjaly and Sadler, Brian M and Tokekar, Pratap and Koppel, Alec},
	year         = 2022,
	booktitle    = {International Conference on Machine Learning},
	pages        = {1716--1731}
}

@inproceedings{belilovsky2019greedy,
	title        = {Greedy layerwise learning can scale to imagenet},
	author       = {Belilovsky, Eugene and Eickenberg, Michael and Oyallon, Edouard},
	year         = 2019,
	booktitle    = {International conference on machine learning},
	pages        = {583--593},
	organization = {PMLR}
}

@incollection{Bengio+chapter2007,
	title        = {Scaling Learning Algorithms Towards {AI}},
	author       = {Bengio, Yoshua and LeCun, Yann},
	year         = 2007,
	booktitle    = {Large Scale Kernel Machines},
	publisher    = {MIT Press}
}

@incollection{bengio2012practical,
	title        = {Practical recommendations for gradient-based training of deep architectures},
	author       = {Bengio, Yoshua},
	year         = 2012,
	booktitle    = {Neural networks: Tricks of the trade},
	publisher    = {Springer},
	pages        = {437--478}
}

@article{bengio2013representation,
	title        = {Representation Learning: A Review and New Perspectives},
	author       = {Bengio, Yoshua and Courville, Aaron and Vincent, Pascal},
	year         = 2013,
	journal      = {IEEE transactions on pattern analysis and machine intelligence},
	volume       = 35,
	number       = 8,
	pages        = {1798--1828},
	url          = {http://arxiv.org/abs/1206.5538},
	note         = {cite arxiv:1206.5538},
	added-at     = {2017-10-17T10:31:25.000+0200},
	biburl       = {https://www.bibsonomy.org/bibtex/2425e75350a32daac5967f8d51a3b2df1/lenalauber},
	description  = {1206.5538.pdf},
	interhash    = {7e205bfb40d2382b7b67b2292765953a},
	intrahash    = {425e75350a32daac5967f8d51a3b2df1},
	timestamp    = {2017-10-18T08:48:30.000+0200}
}

@article{bertsekas2011approximate,
	title        = {Approximate policy iteration: A survey and some new methods},
	author       = {Bertsekas, Dimitri P},
	year         = 2011,
	journal      = {Journal of Control Theory and Applications},
	publisher    = {Springer},
	volume       = 9,
	number       = 3,
	pages        = {310--335}
}

@inproceedings{beznosikov2023first,
	title        = {First Order Methods with Markovian Noise: from Acceleration to Variational Inequalities},
	author       = {Aleksandr Beznosikov and Sergey Samsonov and Marina Sheshukova and Alexander Gasnikov and Alexey Naumov and Eric Moulines},
	year         = 2023,
	booktitle    = {Thirty-seventh Conference on Neural Information Processing Systems}
}

@article{bhandari2018,
	title        = {A Finite Time Analysis of Temporal Difference Learning With Linear Function Approximation},
	author       = {Jalaj Bhandari and Daniel Russo and Raghav Singal},
	year         = 2018,
	journal      = {CoRR},
	volume       = {abs/1806.02450},
	url          = {http://arxiv.org/abs/1806.02450},
	eprinttype   = {arXiv},
	eprint       = {1806.02450},
	timestamp    = {Mon, 13 Aug 2018 16:47:21 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/abs-1806-02450.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{bhandari2019global,
	title        = {Global optimality guarantees for policy gradient methods},
	author       = {Bhandari, Jalaj and Russo, Daniel},
	year         = 2024,
	journal      = {Operations Research},
	publisher    = {INFORMS}
}

@article{bhatnagar2010actor,
	title        = {An actor--critic algorithm with function approximation for discounted cost constrained Markov decision processes},
	author       = {Bhatnagar, Shalabh},
	year         = 2010,
	journal      = {Systems \& Control Letters},
	publisher    = {Elsevier},
	volume       = 59,
	number       = 12,
	pages        = {760--766}
}

@article{BlumRivest:92,
	title        = {Training a 3-node neural network is {NP}-complete},
	author       = {Blum, A. and Rivest, R.},
	year         = 1992,
	journal      = {Neural Networks},
	volume       = 5,
	pages        = {117--127},
	added-at     = {2008-10-07T16:03:39.000+0200},
	biburl       = {https://www.bibsonomy.org/bibtex/2f3eb9a6bc342d82523dce41c9f8c8c22/brefeld},
	interhash    = {80af957f5aca474dedaece220c70f40f},
	intrahash    = {f3eb9a6bc342d82523dce41c9f8c8c22},
	timestamp    = {2008-10-07T16:03:44.000+0200}
}

@article{bonjour2022decision,
	title        = {Decision making in monopoly using a hybrid deep reinforcement learning approach},
	author       = {Bonjour, Trevor and Haliem, Marina and Alsalem, Aala and Thomas, Shilpa and Li, Hongyu and Aggarwal, Vaneet and Kejriwal, Mayank and Bhargava, Bharat},
	year         = 2022,
	journal      = {IEEE Transactions on Emerging Topics in Computational Intelligence},
	publisher    = {IEEE},
	volume       = 6,
	number       = 6,
	pages        = {1335--1344}
}

@book{books/lib/BertsekasT96,
	title        = {Neuro-dynamic programming.},
	author       = {Bertsekas, Dimitri P. and Tsitsiklis, John N.},
	year         = 1996,
	publisher    = {Athena Scientific},
	series       = {Optimization and neural computation series},
	volume       = 3,
	pages        = {I-XIII, 1--491},
	isbn         = 1886529108,
	added-at     = {2020-07-17T00:00:00.000+0200},
	biburl       = {https://www.bibsonomy.org/bibtex/21d4640073b981299f18f7a6a28da8335/dblp},
	ee           = {https://www.worldcat.org/oclc/35983505},
	interhash    = {2fbe138b7b864bc58d95999e69b5d45b},
	intrahash    = {1d4640073b981299f18f7a6a28da8335},
	timestamp    = {2020-07-24T00:45:17.000+0200}
}

@article{borkar1997actor,
	title        = {The actor-critic algorithm as multi-time-scale stochastic approximation},
	author       = {Borkar, Vivek S and Konda, Vijaymohan R},
	year         = 1997,
	journal      = {Sadhana},
	publisher    = {Springer},
	volume       = 22,
	pages        = {525--543}
}

@article{borkar2000ode,
	title        = {The ODE method for convergence of stochastic approximation and reinforcement learning},
	author       = {Borkar, Vivek S and Meyn, Sean P},
	year         = 2000,
	journal      = {SIAM Journal on Control and Optimization},
	publisher    = {SIAM},
	volume       = 38,
	number       = 2,
	pages        = {447--469}
}

@book{borkar2009stochastic,
	title        = {Stochastic approximation: a dynamical systems viewpoint},
	author       = {Borkar, Vivek S},
	year         = 2009,
	publisher    = {Springer},
	volume       = 48
}

@inproceedings{bresler2020,
	title        = {Least Squares Regression with Markovian Data: Fundamental Limits and Algorithms},
	author       = {Dheeraj Nagaraj and Xian Wu and Guy Bresler and Prateek Jain and Praneeth Netrapalli},
	year         = 2020,
	booktitle    = {Advances in Neural Information Processing Systems},
	timestamp    = {Tue, 19 Jan 2021 15:56:50 +0100},
	biburl       = {https://dblp.org/rec/conf/nips/NagarajWB0N20.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{cai2019neural,
	title        = {Neural temporal-difference learning converges to global optima},
	author       = {Cai, Qi and Yang, Zhuoran and Lee, Jason D and Wang, Zhaoran},
	year         = 2019,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 32
}

@inproceedings{carvalho20nips,
	title        = {{A new convergent variant of $Q$-learning with linear function approximation}},
	author       = {D. Carvalho and Francisco S. Melo and P. Santos},
	year         = 2020,
	booktitle    = {Adv. Neural Information Proc. Systems 33},
	pages        = {19412--19421},
	date-modified = {2022-01-27 21:22:55 +0000},
	url_paper    = {publications/carvalho20nips.pdf}
}

@article{castro2010convergent,
	title        = {A convergent online single time scale actor critic algorithm},
	author       = {Castro, Dotan Di and Meir, Ron},
	year         = 2010,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR. org},
	volume       = 11,
	pages        = {367--410}
}



@article{cayci2022finite,
  title   = {Finite-Time Analysis of Entropy-Regularized Neural Natural Actor-Critic Algorithm},
  author  = {Semih Cayci and Niao He and R. Srikant},
  journal = {Transactions on Machine Learning Research},
  year    = {2024},
  month   = apr,
  url     = {https://openreview.net/forum?id=BkEqk7pS1I}
}


@inproceedings{chen2019information,
	title        = {Information-theoretic considerations in batch reinforcement learning},
	author       = {Chen, Jinglin and Jiang, Nan},
	year         = 2019,
	booktitle    = {International Conference on Machine Learning},
	pages        = {1042--1051}
}

@misc{chen2022finite,
	title        = {Finite-time analysis of single-timescale actor-critic},
	author       = {Chen, Xuyang and Zhao, Lin},
	year         = 2022,
	journal      = {arXiv preprint arXiv:2210.09921},
	publisher    = {arXiv},
	doi          = {10.48550/ARXIV.2210.09921},
	url          = {https://arxiv.org/abs/2210.09921},
	copyright    = {arXiv.org perpetual, non-exclusive license}
}

@article{chen2022scalable,
	title        = {Scalable multi-agent covering option discovery based on kronecker graphs},
	author       = {Chen, Jiayu and Chen, Jingdi and Lan, Tian and Aggarwal, Vaneet},
	year         = 2022,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 35,
	pages        = {30406--30418}
}

@article{chen2024finite,
	title        = {Finite-time analysis of single-timescale actor-critic},
	author       = {Chen, Xuyang and Zhao, Lin},
	year         = 2023,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 36
}

@book{citeulike:163662,
	title        = {Convex Optimization},
	author       = {Boyd, Stephen and Vandenberghe, Lieven},
	year         = 2004,
	month        = {March},
	publisher    = {{Cambridge University Press}},
	isbn         = {0521833787},
	added-at     = {2006-04-12T21:13:14.000+0200},
	biburl       = {https://www.bibsonomy.org/bibtex/2b94b4eabf1023b7e1f52d17b78c36637/darius},
	citeulike-article-id = 163662,
	comment      = {
		Full text available online.

		---

		QA402.5 .B69 2004
	},
	description  = {CiteULike},
	howpublished = {Hardcover},
	interhash    = {9049cf1875745bd4a9dfbfa3f470a681},
	intrahash    = {b94b4eabf1023b7e1f52d17b78c36637},
	priority     = 4,
	timestamp    = {2006-04-12T21:13:14.000+0200}
}

@article{Cuccu2011IntrinsicallyMN,
	title        = {Intrinsically motivated neuroevolution for vision-based reinforcement learning},
	author       = {Giuseppe Cuccu and Matthew D. Luciw and J{\"u}rgen Schmidhuber and Faustino J. Gomez},
	year         = 2011,
	journal      = {2011 IEEE International Conference on Development and Learning (ICDL)},
	volume       = 2,
	pages        = {1--7}
}

@inproceedings{dankwa2019twin,
	title        = {Twin-delayed ddpg: A deep reinforcement learning technique to model a continuous movement of an intelligent robot agent},
	author       = {Dankwa, Stephen and Zheng, Wenfeng},
	year         = 2019,
	booktitle    = {Proceedings of the 3rd international conference on vision, image and signal processing},
	pages        = {1--5}
}

@inproceedings{DBLP:conf/nips/SuttonMSM99,
	title        = {Policy Gradient Methods for Reinforcement Learning with Function Approximation},
	author       = {Richard S. Sutton and David A. McAllester and Satinder Singh and Yishay Mansour},
	year         = 1999,
	booktitle    = {Advances in Neural Information Processing Systems 12, {[NIPS} Conference, Denver, Colorado, USA, November 29 - December 4, 1999]},
	publisher    = {The {MIT} Press},
	pages        = {1057--1063}
}

@article{DBLP:journals/corr/abs-1708-04782,
	title        = {StarCraft {II:} {A} New Challenge for Reinforcement Learning},
	author       = {Oriol Vinyals and Timo Ewalds and Sergey Bartunov and Petko Georgiev and Alexander Sasha Vezhnevets and Michelle Yeo and Alireza Makhzani and Heinrich K{\"{u}}ttler and John P. Agapiou and Julian Schrittwieser and John Quan and Stephen Gaffney and Stig Petersen and Karen Simonyan and Tom Schaul and Hado van Hasselt and David Silver and Timothy P. Lillicrap and Kevin Calderone and Paul Keet and Anthony Brunasso and David Lawrence and Anders Ekermo and Jacob Repp and Rodney Tsing},
	year         = 2017,
	journal      = {CoRR},
	volume       = {abs/1708.04782},
	url          = {http://arxiv.org/abs/1708.04782},
	eprinttype   = {arXiv},
	eprint       = {1708.04782},
	timestamp    = {Sat, 23 Jan 2021 01:20:55 +0100},
	biburl       = {https://dblp.org/rec/journals/corr/abs-1708-04782.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/abs-1708-05866,
	title        = {A Brief Survey of Deep Reinforcement Learning},
	author       = {Kai Arulkumaran and Marc Peter Deisenroth and Miles Brundage and Anil Anthony Bharath},
	year         = 2017,
	journal      = {CoRR},
	volume       = {abs/1708.05866},
	url          = {http://arxiv.org/abs/1708.05866},
	eprinttype   = {arXiv},
	eprint       = {1708.05866},
	timestamp    = {Mon, 13 Aug 2018 16:46:01 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/abs-1708-05866.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/abs-1808-01813,
	title        = {Regret Bounds for Reinforcement Learning via Markov Chain Concentration},
	author       = {Ronald Ortner},
	year         = 2018,
	journal      = {CoRR},
	volume       = {abs/1808.01813},
	url          = {http://arxiv.org/abs/1808.01813},
	eprinttype   = {arXiv},
	eprint       = {1808.01813},
	timestamp    = {Sun, 02 Sep 2018 15:01:57 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/abs-1808-01813.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/abs-1812-02690,
	title        = {Provably Efficient Maximum Entropy Exploration},
	author       = {Elad Hazan and Sham M. Kakade and Karan Singh and Abby Van Soest},
	year         = 2018,
	journal      = {CoRR},
	volume       = {abs/1812.02690},
	url          = {http://arxiv.org/abs/1812.02690},
	eprinttype   = {arXiv},
	eprint       = {1812.02690},
	timestamp    = {Tue, 01 Jan 2019 15:01:25 +0100},
	biburl       = {https://dblp.org/rec/journals/corr/abs-1812-02690.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/abs-2001-03040,
	title        = {Deep Network Approximation for Smooth Functions},
	author       = {Jianfeng Lu and Zuowei Shen and Haizhao Yang and Shijun Zhang},
	year         = 2020,
	journal      = {CoRR},
	volume       = {abs/2001.03040},
	url          = {http://arxiv.org/abs/2001.03040},
	eprinttype   = {arXiv},
	eprint       = {2001.03040},
	timestamp    = {Sat, 23 Jan 2021 01:11:33 +0100},
	biburl       = {https://dblp.org/rec/journals/corr/abs-2001-03040.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/abs-2002-12435,
	title        = {Learning in Markov Decision Processes under Constraints},
	author       = {Rahul Singh and Abhishek Gupta and Ness B. Shroff},
	year         = 2020,
	journal      = {CoRR},
	volume       = {abs/2002.12435},
	url          = {https://arxiv.org/abs/2002.12435},
	eprinttype   = {arXiv},
	eprint       = {2002.12435},
	timestamp    = {Wed, 01 Sep 2021 13:33:01 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/abs-2002-12435.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/abs-2003-02189,
	title        = {Exploration-Exploitation in Constrained MDPs},
	author       = {Yonathan Efroni and Shie Mannor and Matteo Pirotta},
	year         = 2020,
	journal      = {CoRR},
	volume       = {abs/2003.02189},
	url          = {https://arxiv.org/abs/2003.02189},
	eprinttype   = {arXiv},
	eprint       = {2003.02189},
	timestamp    = {Tue, 10 Mar 2020 13:33:48 +0100},
	biburl       = {https://dblp.org/rec/journals/corr/abs-2003-02189.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/abs-2006-05051,
	title        = {Constrained episodic reinforcement learning in concave-convex and knapsack settings},
	author       = {Kiant{\'{e}} Brantley and Miroslav Dud{\'{\i}}k and Thodoris Lykouris and Sobhan Miryoosefi and Max Simchowitz and Aleksandrs Slivkins and Wen Sun},
	year         = 2020,
	journal      = {CoRR},
	volume       = {abs/2006.05051},
	url          = {https://arxiv.org/abs/2006.05051},
	eprinttype   = {arXiv},
	eprint       = {2006.05051},
	timestamp    = {Fri, 12 Jun 2020 14:02:57 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/abs-2006-05051.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

download as .bib file
@article{DBLP:journals/corr/abs-2102-03192,
	title        = {Provably Efficient Algorithms for Multi-Objective Competitive {RL}},
	author       = {Tiancheng Yu and Yi Tian and Jingzhao Zhang and Suvrit Sra},
	year         = 2021,
	journal      = {CoRR},
	volume       = {abs/2102.03192},
	url          = {https://arxiv.org/abs/2102.03192},
	eprinttype   = {arXiv},
	eprint       = {2102.03192},
	timestamp    = {Wed, 10 Feb 2021 15:24:32 +0100},
	biburl       = {https://dblp.org/rec/journals/corr/abs-2102-03192.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

download as .bib file
@article{DBLP:journals/corr/abs-2110-04652,
	title        = {Representation Learning for Online and Offline {RL} in Low-rank MDPs},
	author       = {Masatoshi Uehara and Xuezhou Zhang and Wen Sun},
	year         = 2021,
	journal      = {CoRR},
	volume       = {abs/2110.04652},
	url          = {https://arxiv.org/abs/2110.04652},
	eprinttype   = {arXiv},
	eprint       = {2110.04652},
	timestamp    = {Thu, 21 Oct 2021 16:20:08 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/abs-2110-04652.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/AgrawalJ17,
	title        = {Posterior sampling for reinforcement learning: worst-case regret bounds},
	author       = {Shipra Agrawal and Randy Jia},
	year         = 2017,
	journal      = {CoRR},
	volume       = {abs/1705.07041},
	url          = {http://arxiv.org/abs/1705.07041},
	eprinttype   = {arXiv},
	eprint       = {1705.07041},
	timestamp    = {Mon, 13 Aug 2018 16:46:28 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/AgrawalJ17.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/JiangKALS16,
	title        = {Contextual Decision Processes with Low Bellman Rank are PAC-Learnable},
	author       = {Nan Jiang and Akshay Krishnamurthy and Alekh Agarwal and John Langford and Robert E. Schapire},
	year         = 2016,
	journal      = {CoRR},
	volume       = {abs/1610.09512},
	url          = {http://arxiv.org/abs/1610.09512},
	eprinttype   = {arXiv},
	eprint       = {1610.09512},
	timestamp    = {Tue, 05 Apr 2022 14:09:44 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/JiangKALS16.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/MnihKSGAWR13,
	title        = {Playing Atari with Deep Reinforcement Learning},
	author       = {Volodymyr Mnih and Koray Kavukcuoglu and David Silver and Alex Graves and Ioannis Antonoglou and Daan Wierstra and Martin A. Riedmiller},
	year         = 2013,
	journal      = {CoRR},
	volume       = {abs/1312.5602},
	url          = {http://arxiv.org/abs/1312.5602},
	eprinttype   = {arXiv},
	eprint       = {1312.5602},
	timestamp    = {Mon, 13 Aug 2018 16:47:42 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/MnihKSGAWR13.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/SchulmanLMJA15,
	title        = {Trust Region Policy Optimization},
	author       = {John Schulman and Sergey Levine and Philipp Moritz and Michael I. Jordan and Pieter Abbeel},
	year         = 2015,
	journal      = {CoRR},
	volume       = {abs/1502.05477},
	url          = {http://arxiv.org/abs/1502.05477},
	eprinttype   = {arXiv},
	eprint       = {1502.05477},
	timestamp    = {Mon, 13 Aug 2018 16:48:08 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/SchulmanLMJA15.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{DBLP:journals/corr/SchulmanMLJA15,
	title        = {High-Dimensional Continuous Control Using Generalized Advantage Estimation},
	author       = {John Schulman and Philipp Moritz and Sergey Levine and Michael I. Jordan and Pieter Abbeel},
	year         = 2016,
	booktitle    = {4th International Conference on Learning Representations, {ICLR} 2016, San Juan, Puerto Rico, May 2-4, 2016, Conference Track Proceedings},
	url          = {http://arxiv.org/abs/1506.02438},
	timestamp    = {Thu, 25 Jul 2019 14:25:38 +0200},
	biburl       = {https://dblp.org/rec/journals/corr/SchulmanMLJA15.bib},
	bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{dean2020sample,
	title        = {On the sample complexity of the linear quadratic regulator},
	author       = {Dean, Sarah and Mania, Horia and Matni, Nikolai and Recht, Benjamin and Tu, Stephen},
	year         = 2020,
	journal      = {Foundations of Computational Mathematics},
	publisher    = {Springer},
	volume       = 20,
	number       = 4,
	pages        = {633--679}
}

@article{devlin2018bert,
	title        = {Bert: Pre-training of deep bidirectional transformers for language understanding},
	author       = {Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1810.04805}
}

@article{dewanto2020average,
	title        = {Average-reward model-free reinforcement learning: a systematic review and literature mapping},
	author       = {Dewanto, Vektor and Dunn, George and Eshragh, Ali and Gallagher, Marcus and Roosta, Fred},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2010.08920}
}

@inproceedings{ding2022global,
	title        = {On the global optimum convergence of momentum-based policy gradient},
	author       = {Ding, Yuhao and Zhang, Junzi and Lavaei, Javad},
	year         = 2022,
	booktitle    = {International Conference on Artificial Intelligence and Statistics},
	pages        = {1910--1934}
}

@misc{doan2020convergence,
	title        = {Convergence Rates of Accelerated Markov Gradient Descent with Applications in Reinforcement Learning},
	author       = {Thinh T. Doan and Lam M. Nguyen and Nhan H. Pham and Justin Romberg},
	year         = 2020,
	eprint       = {2002.02873},
	archiveprefix = {arXiv},
	primaryclass = {math.OC}
}

@article{doi:10.1137/080716542,
	title        = {A Fast Iterative Shrinkage-Thresholding Algorithm for Linear Inverse Problems},
	author       = {Beck, Amir and Teboulle, Marc},
	year         = 2009,
	journal      = {SIAM Journal on Imaging Sciences},
	volume       = 2,
	number       = 1,
	pages        = {183--202},
	doi          = {10.1137/080716542},
	url          = {https://doi.org/10.1137/080716542},
	eprint       = {https://doi.org/10.1137/080716542}
}

@inproceedings{dorfman2022,
	title        = {Adapting to Mixing Time in Stochastic Optimization with {M}arkovian Data},
	author       = {Dorfman, Ron and Levy, Kfir Yehuda},
	year         = 2022,
	month        = {Jul},
	booktitle    = {Proceedings of the 39th International Conference on Machine Learning}
}

@book{douc:hal-02022651,
	title        = {{Markov chains}},
	author       = {Douc, Randal and Moulines, Eric and Priouret, Pierre and Soulier, Philippe},
	year         = 2018,
	publisher    = {{Springer}},
	series       = {Operation research and financial engineering},
	pages        = 757,
	url          = {https://hal.science/hal-02022651},
	hal_local_reference = 19568
}

@article{doya2000reinforcement,
	title        = {Reinforcement learning in continuous time and space},
	author       = {Doya, Kenji},
	year         = 2000,
	journal      = {Neural computation},
	publisher    = {MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…},
	volume       = 12,
	number       = 1,
	pages        = {219--245}
}

@inproceedings{du2019gradient,
	title        = {Gradient descent finds global minima of deep neural networks},
	author       = {Du, Simon and Lee, Jason and Li, Haochuan and Wang, Liwei and Zhai, Xiyu},
	year         = 2019,
	booktitle    = {International conference on machine learning},
	pages        = {1675--1685},
	organization = {PMLR}
}

@article{duchi2011,
	title        = {Adaptive Subgradient Methods for Online Learning and Stochastic Optimization},
	author       = {John Duchi and Elad Hazan and Yoram Singer},
	year         = 2011,
	journal      = {Journal of Machine Learning Research},
	volume       = 12,
	number       = 61,
	pages        = {2121--2159},
	url          = {http://jmlr.org/papers/v12/duchi11a.html}
}

@article{duchi2011adaptive,
	title        = {Adaptive subgradient methods for online learning and stochastic optimization.},
	author       = {Duchi, John and Hazan, Elad and Singer, Yoram},
	year         = 2011,
	journal      = {Journal of machine learning research},
	volume       = 12,
	number       = 7
}

@article{duchi2012,
	title        = {Ergodic Mirror Descent},
	author       = {Duchi, John C. and Agarwal, Alekh and Johansson, Mikael and Jordan, Michael I.},
	year         = 2012,
	journal      = {SIAM Journal on Optimization},
	volume       = 22,
	number       = 4,
	pages        = {1549--1578}
}

@article{en15196920,
	title        = {Deep Reinforcement Learning-Based Approach for Autonomous Power Flow Control Using Only Topology Changes},
	author       = {Damjanović, Ivana and Pavić, Ivica and Puljiz, Mate and Brcic, Mario},
	year         = 2022,
	journal      = {Energies},
	volume       = 15,
	number       = 19,
	doi          = {10.3390/en15196920},
	issn         = {1996-1073},
	url          = {https://www.mdpi.com/1996-1073/15/19/6920},
	article-number = 6920
}

@article{even2009online,
	title        = {Online Markov decision processes},
	author       = {Even-Dar, Eyal and Kakade, Sham M and Mansour, Yishay},
	year         = 2009,
	journal      = {Mathematics of Operations Research},
	publisher    = {INFORMS},
	volume       = 34,
	number       = 3,
	pages        = {726--736}
}

@misc{even2023stochastic,
	title        = {Stochastic Gradient Descent under Markovian Sampling Schemes},
	author       = {Mathieu Even},
	year         = 2023,
	eprint       = {2302.14428},
	archiveprefix = {arXiv},
	primaryclass = {math.OC}
}

@article{farahmand2010error,
	title        = {Error propagation for approximate policy and value iteration},
	author       = {Farahmand, Amir-massoud and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
	year         = 2010,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 23
}

@inproceedings{fatkhullin2023stochastic,
	title        = {Stochastic policy gradient methods: Improved sample complexity for fisher-non-degenerate policies},
	author       = {Fatkhullin, Ilyas and Barakat, Anas and Kireeva, Anastasia and He, Niao},
	year         = 2023,
	booktitle    = {International Conference on Machine Learning},
	pages        = {9827--9869},
	organization = {PMLR}
}

@inproceedings{fazel2018global,
	title        = {Global convergence of policy gradient methods for the linear quadratic regulator},
	author       = {Fazel, Maryam and Ge, Rong and Kakade, Sham and Mesbahi, Mehran},
	year         = 2018,
	booktitle    = {International conference on machine learning},
	pages        = {1467--1476},
	organization = {PMLR}
}

@inproceedings{feng2022rank,
	title        = {Rank Diminishing in Deep Neural Networks},
	author       = {Ruili Feng and Kecheng Zheng and Yukun Huang and Deli Zhao and Michael Jordan and Zheng-Jun Zha},
	year         = 2022,
	booktitle    = {Advances in Neural Information Processing Systems},
	url          = {https://openreview.net/forum?id=tIqzLFf3kk},
	editor       = {Alice H. Oh and Alekh Agarwal and Danielle Belgrave and Kyunghyun Cho}
}

@book{folland1999real,
	title        = {Real analysis: modern techniques and their applications},
	author       = {Folland, Gerald B},
	year         = 1999,
	publisher    = {John Wiley \& Sons},
	volume       = 40
}


@inproceedings{fujimoto2019off,
	title        = {Off-policy deep reinforcement learning without exploration},
	author       = {Fujimoto, Scott and Meger, David and Precup, Doina},
	year         = 2019,
	booktitle    = {International conference on machine learning},
	pages        = {2052--2062},
	organization = {PMLR}
}

@inproceedings{ganesh2024orderoptimal,
	title = {A Sharper Global Convergence Analysis for Average Reward Reinforcement Learning via an Actor-Critic Approach},
	author       = {Swetha Ganesh and Washim Uddin Mondal and Vaneet Aggarwal},
	year         = 2025,
	booktitle      = {International Conference on Machine Learning}
}

@misc{ganesh2024variance,
	title        = {Variance-Reduced Policy Gradient Approaches for Infinite Horizon Average Reward Markov Decision Processes},
	author       = {Swetha Ganesh and Washim Uddin Mondal and Vaneet Aggarwal},
	year         = 2024,
	url          = {https://arxiv.org/abs/2404.02108},
	eprint       = {2404.02108},
	archiveprefix = {arXiv},
	primaryclass = {cs.LG}
}

@article{gao2019convergence,
	title        = {Convergence of adversarial training in overparametrized neural networks},
	author       = {Gao, Ruiqi and Cai, Tianle and Li, Haochuan and Hsieh, Cho-Jui and Wang, Liwei and Lee, Jason D},
	year         = 2019,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 32
}

@article{gaur2023global,
	title        = {On the Global Convergence of Natural Actor-Critic with Two-layer Neural Network Parametrization},
	author       = {Gaur, Mudit and Bedi, Amrit Singh and Wang, Di and Aggarwal, Vaneet},
	year         = 2023,
	journal      = {arXiv preprint arXiv:2306.10486},
	eprint       = {2211.07675},
	archiveprefix = {arXiv},
	primaryclass = {cs.LG}
}

@inproceedings{gaur2024closing,
	title        = {Closing the Gap: Achieving Global Convergence (Last Iterate) of Actor-Critic under Markovian Sampling with Neural Network Parametrization},
	author       = {Gaur, Mudit and Aggarwal, Vaneet and Bedi, Amrit Singh and Wang, Di},
	year         = 2024,
	booktitle    = {International Conference on Machine Learning}
}

@inproceedings{geng2020multi,
	title        = {A multi-agent reinforcement learning perspective on distributed traffic engineering},
	author       = {Geng, Nan and Lan, Tian and Aggarwal, Vaneet and Yang, Yuan and Xu, Mingwei},
	year         = 2020,
	booktitle    = {2020 IEEE 28th International Conference on Network Protocols (ICNP)},
	pages        = {1--11},
	organization = {IEEE}
}

@inproceedings{ghavamzadeh2008regularized,
	title        = {Regularized fitted Q-iteration: Application to planning},
	author       = {Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie and others},
	year         = 2008,
	booktitle    = {European Workshop on Reinforcement Learning},
	pages        = {55--68},
	organization = {Springer}
}

@article{giles2015multilevel,
	title        = {Multilevel monte carlo methods},
	author       = {Giles, Michael B},
	year         = 2015,
	journal      = {Acta numerica},
	publisher    = {Cambridge University Press},
	volume       = 24,
	pages        = {259--328}
}

@inproceedings{godfrey2019evaluation,
	title        = {An evaluation of parametric activation functions for deep learning},
	author       = {Godfrey, Luke B},
	year         = 2019,
	booktitle    = {2019 IEEE International Conference on Systems, Man and Cybernetics (SMC)},
	pages        = {3006--3011},
	organization = {IEEE}
}

@inproceedings{gong2020duality,
	title        = {A Duality Approach for Regret Minimization in Average-Award Ergodic Markov Decision Processes},
	author       = {Gong, Hao and Wang, Mengdi},
	year         = 2020,
	booktitle    = {Learning for Dynamics and Control}
}

@article{Gonzalez2023,
	title        = {ASAP: A Semi-Autonomous Precise System for Telesurgery During Communication Delays},
	author       = {Gonzalez, Glebys and Balakuntala, Mythra and Agarwal, Mridul and Low, Tomas and Knoth, Bruce and Kirkpatrick, Andrew W. and McKee, Jessica and Hager, Gregory and Aggarwal, Vaneet and Xue, Yexiang and Voyles, Richard and Wachs, Juan},
	year         = 2023,
	journal      = {IEEE Transactions on Medical Robotics and Bionics},
	volume       = 5,
	number       = 1,
	pages        = {66--78},
	doi          = {10.1109/TMRB.2023.3239674},
}

@book{goodfellow2016deep,
	title        = {Deep learning},
	author       = {Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron and Bengio, Yoshua},
	year         = 2016,
	publisher    = {MIT Press},
	volume       = 1
}

@article{gu2016deep,
	title        = {Deep reinforcement learning for robotic manipulation},
	author       = {Gu, Shixiang and Holly, Ethan and Lillicrap, Timothy and Levine, Sergey},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1610.00633},
	volume       = 1
}

@inproceedings{haarnoja2018soft,
	title        = {Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor},
	author       = {Haarnoja, Tuomas and Zhou, Aurick and Abbeel, Pieter and Levine, Sergey},
	year         = 2018,
	journal      = {arXiv e-prints},
	booktitle    = {International conference on machine learning},
	pages        = {1861--1870}
}

@article{haliem2021distributed,
	title        = {A distributed model-free ride-sharing approach for joint matching, pricing, and dispatching using deep reinforcement learning},
	author       = {Haliem, Marina and Mani, Ganapathy and Aggarwal, Vaneet and Bhargava, Bharat},
	year         = 2021,
	journal      = {IEEE Transactions on Intelligent Transportation Systems},
	publisher    = {IEEE},
	volume       = 22,
	number       = 12,
	pages        = {7931--7942}
}

@article{Haliem2021LearningMG,
	title        = {Decision Making in Monopoly using a Hybrid Deep Reinforcement Learning Approach},
	author       = {Bonjour, Trevor and Haliem, Marina and Alsalem, Aala and Thomas, Shilpa and Li, Hongyu and Aggarwal, Vaneet and Kejriwal, Mayank and Bhargava, Bharat},
	year         = 2022,
	journal      = {IEEE Transactions on Emerging Topics in Computational Intelligence},
	publisher    = {IEEE}
}

@article{hasselt2015doubledqn,
	title        = {Deep Reinforcement Learning with Double Q-learning},
	author       = {van Hasselt, Hado and Guez, Arthur and Silver, David},
	year         = 2015,
	journal      = {CoRR},
	url          = {http://arxiv.org/abs/1509.06461},
	note         = {cite arxiv:1509.06461Comment: AAAI 2016},
	added-at     = {2019-11-18T11:40:13.000+0100},
	biburl       = {https://www.bibsonomy.org/bibtex/2c2bad4b4c5a34cb31a3f569c71e851ab/jan.hofmann1},
	description  = {[1509.06461] Deep Reinforcement Learning with Double Q-learning},
	interhash    = {d3061c37961afb78096e314854dd90bc},
	intrahash    = {c2bad4b4c5a34cb31a3f569c71e851ab},
	timestamp    = {2019-12-09T10:13:49.000+0100}
}

@article{heaton2016deep,
	title        = {Deep portfolio theory},
	author       = {Heaton, JB and Polson, Nicholas G and Witte, JH},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1605.07230}
}

@article{Hinton06,
	title        = {A Fast Learning Algorithm for Deep Belief Nets},
	author       = {Hinton, Geoffrey E. and Osindero, Simon and Teh, Yee Whye},
	year         = 2006,
	journal      = {Neural Computation},
	volume       = 18,
	pages        = {1527--1554}
}

@article{hinton2006fast,
	title        = {A fast learning algorithm for deep belief nets},
	author       = {Hinton, Geoffrey E and Osindero, Simon and Teh, Yee-Whye},
	year         = 2006,
	journal      = {Neural computation},
	publisher    = {MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…},
	volume       = 18,
	number       = 7,
	pages        = {1527--1554}
}

@article{HON2022152,
	title        = {Simultaneous neural network approximation for smooth functions},
	author       = {Sean Hon and Haizhao Yang},
	year         = 2022,
	journal      = {Neural Networks},
	volume       = 154,
	pages        = {152--164},
	doi          = {https://doi.org/10.1016/j.neunet.2022.06.040},
	issn         = {0893-6080},
	url          = {https://www.sciencedirect.com/science/article/pii/S0893608022002556},
}

@article{hsu2015,
	title        = {Mixing time estimation in reversible Markov chains from a single sample path},
	author       = {Hsu, Daniel J and Kontorovich, Aryeh and Szepesv{\'a}ri, Csaba},
	year         = 2015,
	journal      = {Advances in neural information processing systems},
	volume       = 28
}

@misc{https://doi.org/10.48550/arxiv.1805.09545,
	title        = {On the Global Convergence of Gradient Descent for Over-parameterized Models using Optimal Transport},
	author       = {Chizat, Lenaic and Bach, Francis},
	year         = 2018,
	publisher    = {arXiv},
	doi          = {10.48550/ARXIV.1805.09545},
	url          = {https://arxiv.org/abs/1805.09545},
	copyright    = {arXiv.org perpetual, non-exclusive license},
}

@article{jacot2018neural,
	title        = {Neural tangent kernel: Convergence and generalization in neural networks},
	author       = {Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
	year         = 2018,
	journal      = {Advances in neural information processing systems},
	volume       = 31
}

@article{jaksch2010near,
	title        = {Near-optimal Regret Bounds for Reinforcement Learning},
	author       = {Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
	year         = 2010,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR. org},
	volume       = 11,
	pages        = {1563--1600}
}

@article{JMLR:v11:jaksch10a,
	title        = {Near-optimal Regret Bounds for Reinforcement Learning},
	author       = {Thomas Jaksch and Ronald Ortner and Peter Auer},
	year         = 2010,
	journal      = {Journal of Machine Learning Research},
	volume       = 11,
	number       = 51,
	pages        = {1563--1600},
	url          = {http://jmlr.org/papers/v11/jaksch10a.html}
}

@article{JMLR:v17:10-364,
	title        = {Analysis of Classification-based Policy Iteration Algorithms},
	author       = {Alessandro Lazaric and Mohammad Ghavamzadeh and R{\'e}mi Munos},
	year         = 2016,
	journal      = {Journal of Machine Learning Research},
	volume       = 17,
	number       = 19,
	pages        = {1--30},
	url          = {http://jmlr.org/papers/v17/10-364.html}
}

@article{JMLR:v9:munos08a,
	title        = {Finite-Time Bounds for Fitted Value Iteration},
	author       = {R{{\'e}}mi Munos and Csaba Szepesv{{\'a}}ri},
	year         = 2008,
	journal      = {Journal of Machine Learning Research},
	volume       = 9,
	number       = 27,
	pages        = {815--857},
	url          = {http://jmlr.org/papers/v9/munos08a.html}
}

@article{johnson2013accelerating,
	title        = {Accelerating stochastic gradient descent using predictive variance reduction},
	author       = {Johnson, Rie and Zhang, Tong},
	year         = 2013,
	journal      = {Advances in neural information processing systems},
	volume       = 26
}

@article{journals/nature/MnihKSRVBGRFOPB15,
	title        = {Human-level control through deep reinforcement learning.},
	author       = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A. and Veness, Joel and Bellemare, Marc G. and Graves, Alex and Riedmiller, Martin A. and Fidjeland, Andreas and Ostrovski, Georg and Petersen, Stig and Beattie, Charles and Sadik, Amir and Antonoglou, Ioannis and King, Helen and Kumaran, Dharshan and Wierstra, Daan and Legg, Shane and Hassabis, Demis},
	year         = 2015,
	journal      = {Nature},
	volume       = 518,
	number       = 7540,
	pages        = {529--533},
	url          = {http://dblp.uni-trier.de/db/journals/nature/nature518.html#MnihKSRVBGRFOPB15},
	added-at     = {2021-01-06T14:52:16.000+0100},
	biburl       = {https://www.bibsonomy.org/bibtex/27c6f54424f0d4672eae5411091b5bd95/frankyanpan},
	ee           = {https://www.wikidata.org/entity/Q27907579},
	interhash    = {eac59980357d99db87b341b61ef6645f},
	intrahash    = {7c6f54424f0d4672eae5411091b5bd95},
	timestamp    = {2021-01-06T14:52:16.000+0100}
}

@article{kakade2001natural,
	title        = {A natural policy gradient},
	author       = {Kakade, Sham M},
	year         = 2001,
	journal      = {Advances in neural information processing systems},
	volume       = 14
}

@inproceedings{kakade2002approximately,
	title        = {Approximately optimal approximate reinforcement learning},
	author       = {Kakade, Sham and Langford, John},
	year         = 2002,
	booktitle    = {Proceedings of the Nineteenth International Conference on Machine Learning},
	pages        = {267--274}
}

@inproceedings{ke2024an,
	title        = {An Improved Finite-time Analysis of Temporal Difference Learning with Deep Neural Networks},
	author       = {Zhifa Ke and Zaiwen Wen and Junyu Zhang},
	year         = 2024,
	booktitle    = {Forty-first International Conference on Machine Learning},
	url          = {https://openreview.net/forum?id=dqdctVbSfs}
}

@inproceedings{khodadadian2021finite,
	title        = {Finite-sample analysis of off-policy natural actor-critic algorithm},
	author       = {Khodadadian, Sajad and Chen, Zaiwei and Maguluri, Siva Theja},
	year         = 2021,
	booktitle    = {International Conference on Machine Learning},
	pages        = {5420--5431},
	organization = {PMLR}
}

@phdthesis{konda-thesis,
	title        = {Actor-critic algorithms},
	author       = {Konda, Vijay},
	year         = 2000,
	advisor      = {Tsitsiklis, John N.},
	school       = {University of Alberta}
}

@article{konda1999actor,
	title        = {Actor-critic algorithms},
	author       = {Konda, Vijay and Tsitsiklis, John},
	year         = 1999,
	journal      = {Advances in neural information processing systems},
	volume       = 12
}

@inproceedings{Konda2000,
	title        = {Actor-Critic Algorithms},
	author       = {Konda, Vijay and Tsitsiklis, John},
	year         = 1999,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {MIT Press},
	volume       = 12,
	pages        = {},
	editor       = {S. Solla and T. Leen and K. M\"{u}ller}
}

@inproceedings{kozakowski2022q,
	title        = {Q-value weighted regression: Reinforcement learning with limited data},
	author       = {Kozakowski, Piotr and Kaiser, Lukasz and Michalewski, Henryk and Mohiuddin, Afroz and Ka{\'n}ska, Katarzyna},
	year         = 2022,
	booktitle    = {2022 International Joint Conference on Neural Networks (IJCNN)},
	pages        = {1--8},
	organization = {IEEE}
}

@article{krizhevsky2017imagenet,
	title        = {Imagenet classification with deep convolutional neural networks},
	author       = {Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
	year         = 2017,
	journal      = {Communications of the ACM},
	publisher    = {AcM New York, NY, USA},
	volume       = 60,
	number       = 6,
	pages        = {84--90}
}

@article{kumar2019sample,
	title        = {On the sample complexity of actor-critic method for reinforcement learning with function approximation},
	author       = {Kumar, Harshat and Koppel, Alec and Ribeiro, Alejandro},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1910.08412}
}

@article{kumar2023sample,
	title        = {On the sample complexity of actor-critic method for reinforcement learning with function approximation},
	author       = {Kumar, Harshat and Koppel, Alec and Ribeiro, Alejandro},
	year         = 2023,
	journal      = {Machine Learning},
	publisher    = {Springer},
	pages        = {1--35}
}

@misc{kumar2024global,
	title        = {On the Global Convergence of Policy Gradient in Average Reward Markov Decision Processes},
	author       = {Navdeep Kumar and Yashaswini Murthy and Itai Shufaro and Kfir Y. Levy and R. Srikant and Shie Mannor},
	year         = 2024,
	eprint       = {2403.06806},
	archiveprefix = {arXiv},
	primaryclass = {cs.LG}
}

@article{Kurt1991251,
	title        = {{Approximation capabilities of multilayer feedforward networks}},
	author       = {Kurt and Hornik},
	year         = 1991,
	journal      = {Neural Networks},
	volume       = 4,
	number       = 2,
	pages        = {251--257},
	doi          = {10.1016/0893-6080(91)90009-T},
	url          = {http://www.sciencedirect.com/science/article/pii/089360809190009T},
	added-at     = {2012-03-02T03:39:18.000+0100},
	biburl       = {https://www.bibsonomy.org/bibtex/2d845936959b8df79fdcb9faf0aefddaf/baby9992006},
	citeulike-article-id = 10393099,
	citeulike-linkout-0 = {http://dx.doi.org/10.1016/0893-6080(91)90009-T},
	citeulike-linkout-1 = {http://www.sciencedirect.com/science/article/pii/089360809190009T},
	interhash    = {e49bb72b924b28634ae50e9fd10b67d9},
	intrahash    = {d845936959b8df79fdcb9faf0aefddaf},
	posted-at    = {2012-02-28 13:19:09},
	priority     = 2,
	timestamp    = {2012-03-02T03:39:20.000+0100}
}

@article{lan2023policy,
	title        = {Policy mirror descent for reinforcement learning: Linear convergence, new sampling complexity, and generalized problem classes},
	author       = {Lan, Guanghui},
	year         = 2023,
	journal      = {Mathematical programming},
	publisher    = {Springer},
	volume       = 198,
	number       = 1,
	pages        = {1059--1106}
}

@inproceedings{leahy2022convergence,
	title        = {Convergence of policy gradient for entropy regularized MDPs with neural network approximation in the mean-field regime},
	author       = {Leahy, James-Michael and Kerimkulov, Bekzhan and Siska, David and Szpruch, Lukasz},
	year         = 2022,
	booktitle    = {International Conference on Machine Learning},
	pages        = {12222--12252}
}

@article{lee2020stochastic,
	title        = {Stochastic latent actor-critic: Deep reinforcement learning with a latent variable model},
	author       = {Lee, Alex X and Nagabandi, Anusha and Abbeel, Pieter and Levine, Sergey},
	year         = 2020,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 33,
	pages        = {741--752}
}

@inproceedings{leuenberger2018actor,
	title        = {Actor-critic reinforcement learning with neural networks in continuous games},
	author       = {Leuenberger, Gabriel and Wiering, Marco A},
	year         = 2018,
	booktitle    = {ICAART 2018-Proceedings of the 10th International Conference on Agents and Artificial Intelligence},
	organization = {SciTePress}
}

@book{levin2017markov,
	title        = {Markov chains and mixing times},
	author       = {Levin, David A and Peres, Yuval},
	year         = 2017,
	publisher    = {American Mathematical Soc.},
	volume       = 107
}

@article{levy2017online,
	title        = {Online to offline conversions, universality and adaptive minibatch sizes},
	author       = {Levy, Kfir},
	year         = 2017,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 30
}

@inproceedings{li2019convergence,
	title        = {On the convergence of stochastic gradient descent with adaptive stepsizes},
	author       = {Li, Xiaoyu and Orabona, Francesco},
	year         = 2019,
	booktitle    = {The 22nd international conference on artificial intelligence and statistics},
	pages        = {983--992}
}

@inproceedings{li2019efficient,
	title        = {Efficient ridesharing order dispatching with mean field multi-agent reinforcement learning},
	author       = {Li, Minne and Qin, Zhiwei and Jiao, Yan and Yang, Yaodong and Wang, Jun and Wang, Chenxi and Wu, Guobin and Ye, Jieping},
	year         = 2019,
	booktitle    = {The world wide web conference},
	pages        = {983--994}
}

@article{li2019reinforcement,
	title        = {Reinforcement learning applications},
	author       = {Li, Yuxi},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1908.06973}
}

@inproceedings{li2020video,
	title        = {Video recommendation with multi-gate mixture of experts soft actor critic},
	author       = {Li, Dingcheng and Li, Xu and Wang, Jun and Li, Ping},
	year         = 2020,
	booktitle    = {Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval},
	pages        = {1553--1556}
}

@article{lillicrap2015continuous,
	title        = {Continuous control with deep reinforcement learning},
	author       = {Lillicrap, Timothy P and Hunt, Jonathan J and Pritzel, Alexander and Heess, Nicolas and Erez, Tom and Tassa, Yuval and Silver, David and Wierstra, Daan},
	year         = 2015,
	journal      = {arXiv preprint arXiv:1509.02971}
}

@article{Lin1992SelfimprovingRA,
	title        = {Self-improving reactive agents based on reinforcement learning, planning and teaching},
	author       = {Longxin Lin},
	year         = 1992,
	journal      = {Machine Learning},
	volume       = 8,
	pages        = {293--321}
}

@article{ling2023cooperating,
	title        = {Cooperating Graph Neural Networks with Deep Reinforcement Learning for Vaccine Prioritization},
	author       = {Lu Ling and Washim Uddin Mondal and Satish V. Ukkusuri},
	year         = 2023,
	journal      = {arXiv preprint arXiv:2305.05163}
}

@article{liu2020improved,
	title        = {An improved analysis of (variance-reduced) policy gradient and natural policy gradient methods},
	author       = {Liu, Yanli and Zhang, Kaiqing and Basar, Tamer and Yin, Wotao},
	year         = 2020,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 33,
	pages        = {7624--7636}
}

@article{liu2020linearity,
	title        = {On the linearity of large non-linear models: when and why the tangent kernel is constant},
	author       = {Liu, Chaoyue and Zhu, Libin and Belkin, Misha},
	year         = 2020,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 33,
	pages        = {15954--15964}
}

@article{liu2021federated,
	title        = {Federated learning meets natural language processing: A survey},
	author       = {Liu, Ming and Ho, Stella and Wang, Mengqi and Gao, Longxiang and Jin, Yuan and Zhang, He},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2107.12603}
}

@inproceedings{liu2021finrl,
	title        = {FinRL: Deep reinforcement learning framework to automate trading in quantitative finance},
	author       = {Liu, Xiao-Yang and Yang, Hongyang and Gao, Jiechao and Wang, Christina Dan},
	year         = 2021,
	booktitle    = {Proceedings of the Second ACM International Conference on AI in Finance},
	pages        = {1--9}
}

@phdthesis{maei-thesis,
	title        = {Gradient temporal-difference learning algorithms},
	author       = {Maei, Hamid Reza},
	year         = 2011,
	publisher    = {University of Alberta},
	address      = {CAN},
	isbn         = 9780494894552,
	note         = {AAINR89455},
	advisor      = {Sutton, Richard S.}
}

@article{maei2018convergent,
	title        = {Convergent actor-critic algorithms under off-policy training and function approximation},
	author       = {Maei, Hamid Reza},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1802.07842}
}

@misc{Maes03reinforcementlearning,
	title        = {Reinforcement Learning in Large State Spaces: Simulated Robotic Soccer as a testbed},
	author       = {Sam Maes and Karl Tuyls and Bernard Manderick},
	year         = 2003
}

@inproceedings{masiha2022stochastic,
	title        = {Stochastic Second-Order Methods Improve Best-Known Sample Complexity of {SGD} for Gradient-Dominated Functions},
	author       = {Saeed Masiha and Saber Salehkaleybar and Niao He and Negar Kiyavash and Patrick Thiran},
	year         = 2022,
	booktitle    = {Advances in Neural Information Processing Systems},
	editor       = {Alice H. Oh and Alekh Agarwal and Danielle Belgrave and Kyunghyun Cho}
}

@article{mcafee2012big,
	title        = {Big data: the management revolution},
	author       = {McAfee, Andrew and Brynjolfsson, Erik and Davenport, Thomas H and Patil, DJ and Barton, Dominic},
	year         = 2012,
	journal      = {Harvard business review},
	publisher    = {Cambridge},
	volume       = 90,
	number       = 10,
	pages        = {60--68}
}

@inproceedings{mei2020global,
	title        = {On the global convergence rates of softmax policy gradient methods},
	author       = {Mei, Jincheng and Xiao, Chenjun and Szepesvari, Csaba and Schuurmans, Dale},
	year         = 2020,
	booktitle    = {International Conference on Machine Learning},
	pages        = {6820--6829}
}

@inproceedings{melo2008analysis,
	title        = {An analysis of reinforcement learning with function approximation},
	author       = {Melo, Francisco S and Meyn, Sean P and Ribeiro, M Isabel},
	year         = 2008,
	booktitle    = {Proceedings of the 25th international conference on Machine learning},
	pages        = {664--671}
}

@inproceedings{mertikopoulos2018cycles,
	title        = {Cycles in adversarial regularized learning},
	author       = {Mertikopoulos, Panayotis and Papadimitriou, Christos and Piliouras, Georgios},
	year         = 2018,
	booktitle    = {Proceedings of the twenty-ninth annual ACM-SIAM symposium on discrete algorithms},
	pages        = {2703--2717},
	organization = {SIAM}
}

@inproceedings{mertikopoulos2018optimistic,
	title        = {Optimistic mirror descent in saddle-point problems: Going the extra(-gradient) mile},
	author       = {Panayotis Mertikopoulos and Bruno Lecouat and Houssam Zenati and Chuan-Sheng Foo and Vijay Chandrasekhar and Georgios Piliouras},
	year         = 2019,
	booktitle    = {International Conference on Learning Representations},
	url          = {https://openreview.net/forum?id=Bkg8jjC9KQ}
}

@article{Mey2020ANO,
	title        = {A Note on High-Probability versus In-Expectation Guarantees of Generalization Bounds in Machine Learning},
	author       = {Alexander Mey},
	year         = 2020,
	journal      = {ArXiv},
	volume       = {abs/2010.02576}
}

@article{mitrophanov2005sensitivity,
	title        = {Sensitivity and convergence of uniformly ergodic Markov chains},
	author       = {Mitrophanov, A Yu},
	year         = 2005,
	journal      = {Journal of Applied Probability},
	publisher    = {Cambridge University Press},
	volume       = 42,
	number       = 4,
	pages        = {1003--1014}
}

@article{mnih2013playing,
	title        = {Playing atari with deep reinforcement learning},
	author       = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},
	year         = 2013,
	journal      = {arXiv preprint arXiv:1312.5602}
}

@inproceedings{mnih2016asynchronous,
	title        = {Asynchronous methods for deep reinforcement learning},
	author       = {Mnih, Volodymyr and Badia, Adria Puigdomenech and Mirza, Mehdi and Graves, Alex and Lillicrap, Timothy and Harley, Tim and Silver, David and Kavukcuoglu, Koray},
	year         = 2016,
	booktitle    = {International conference on machine learning},
	pages        = {1928--1937}
}


@inproceedings{mondal2024improved,
	title        = {Improved sample complexity analysis of natural policy gradient algorithm with general parameterization for infinite horizon discounted reward markov decision processes},
	author       = {Mondal, Washim U and Aggarwal, Vaneet},
	year         = 2024,
	booktitle    = {International Conference on Artificial Intelligence and Statistics},
	pages        = {3097--3105},
	organization = {PMLR}
}

@inproceedings{morgan2021model,
	title        = {Model predictive actor-critic: Accelerating robot skill acquisition with deep reinforcement learning},
	author       = {Morgan, Andrew S and Nandha, Daljeet and Chalvatzaki, Georgia and D’Eramo, Carlo and Dollar, Aaron M and Peters, Jan},
	year         = 2021,
	booktitle    = {2021 IEEE International Conference on Robotics and Automation (ICRA)},
	pages        = {6672--6678},
	organization = {IEEE}
}

@article{munos:inria-00124685,
	title        = {{Performance Bounds in $L_p$ norm for Approximate Value Iteration}},
	author       = {Munos, R{\'e}mi},
	year         = 2007,
	journal      = {{SIAM Journal on Control and Optimization}},
	publisher    = {{Society for Industrial and Applied Mathematics}},
	volume       = 46,
	number       = 2,
	pages        = {541--561},
	doi          = {10.1137/040614384},
	url          = {https://hal.inria.fr/inria-00124685},
	pdf          = {https://hal.inria.fr/inria-00124685/file/avi_siam_final.pdf},
	hal_id       = {inria-00124685},
	hal_version  = {v1}
}

@inproceedings{munos2003error,
	title        = {Error bounds for approximate policy iteration},
	author       = {Munos, R{\'e}mi},
	year         = 2003,
	booktitle    = {ICML},
	volume       = 3,
	pages        = {560--567}
}

@inproceedings{NEURIPS2018_210f760a,
	title        = {Fighting Boredom in Recommender Systems with Linear Reinforcement Learning},
	author       = {WARLOP, Romain and Lazaric, Alessandro and Mary, J\'{e}r\'{e}mie},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 31,
	pages        = {}
}

@inproceedings{NEURIPS2018_d3b1fb02,
	title        = {Is Q-Learning Provably Efficient?},
	author       = {Jin, Chi and Allen-Zhu, Zeyuan and Bubeck, Sebastien and Jordan, Michael I},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 31,
	pages        = {}
}

@inproceedings{NEURIPS2019_a02ffd91,
	title        = {Regret Minimization for Reinforcement Learning with Vectorial Feedback and Complex Objectives},
	author       = {Cheung, Wang Chi},
	year         = 2019,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 32,
	pages        = {}
}

@inproceedings{NEURIPS2020_4eab60e5,
	title        = {Sample Complexity of Asynchronous Q-Learning: Sharper Analysis and Variance Reduction},
	author       = {Li, Gen and Wei, Yuting and Chi, Yuejie and Gu, Yuantao and Chen, Yuxin},
	year         = 2020,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 33,
	pages        = {7031--7043}
}

@inproceedings{NEURIPS2020_56577889,
	title        = {An Improved Analysis of  (Variance-Reduced) Policy Gradient and Natural Policy Gradient Methods},
	author       = {Liu, Yanli and Zhang, Kaiqing and Basar, Tamer and Yin, Wotao},
	year         = 2020,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 33,
	pages        = {7624--7636}
}

@inproceedings{NEURIPS2021_096ffc29,
	title        = {Finite Sample Analysis of Average-Reward TD Learning and Q-Learning},
	author       = {Zhang, Sheng and Zhang, Zhe and Maguluri, Siva Theja},
	year         = 2021,
	booktitle    = {Advances in Neural Information Processing Systems},
	volume       = 34,
	pages        = {1230--1242}
}

@inproceedings{NEURIPS2021_70a32110,
	title        = {Provably Efficient Reinforcement Learning with Linear Function Approximation under Adaptivity Constraints},
	author       = {Wang, Tianhao and Zhou, Dongruo and Gu, Quanquan},
	year         = 2021,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 34,
	pages        = {13524--13536}
}

@inproceedings{NIPS1994_ef50c335,
	title        = {Generalization in Reinforcement Learning: Safely Approximating the Value Function},
	author       = {Boyan, Justin and Moore, Andrew},
	year         = 1994,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {MIT Press},
	volume       = 7,
	pages        = {}
}

@inproceedings{NIPS2001_4b86abe4,
	title        = {A Natural Policy Gradient},
	author       = {Kakade, Sham M},
	year         = 2001,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {MIT Press},
	volume       = 14,
	pages        = {}
}

@inproceedings{NIPS2007_da0d1111,
	title        = {Fitted Q-iteration in continuous action-space MDPs},
	author       = {Antos, Andr\'{a}s and Szepesv\'{a}ri, Csaba and Munos, R\'{e}mi},
	year         = 2007,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 20,
	pages        = {}
}

@inproceedings{NIPS2013_6a5889bb,
	title        = {(More) Efficient Reinforcement Learning via Posterior Sampling},
	author       = {Osband, Ian and Russo, Daniel and Van Roy, Benjamin},
	year         = 2013,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 26,
	pages        = {}
}

@inproceedings{NIPS2017_453fadbd,
	title        = {Hindsight Experience Replay},
	author       = {Andrychowicz, Marcin and Wolski, Filip and Ray, Alex and Schneider, Jonas and Fong, Rachel and Welinder, Peter and McGrew, Bob and Tobin, Josh and Pieter Abbeel, OpenAI and Zaremba, Wojciech},
	year         = 2017,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 30,
	pages        = {}
}

@misc{osband2013more,
	title        = {(More) Efficient Reinforcement Learning via Posterior Sampling},
	author       = {Ian Osband and Daniel Russo and Benjamin Van Roy},
	year         = 2013,
	eprint       = {1306.0940},
	archiveprefix = {arXiv},
	primaryclass = {stat.ML}
}

@inproceedings{pane2016actor,
	title        = {Actor-critic reinforcement learning for tracking control in robotics},
	author       = {Pane, Yudha P and Nageshrao, Subramanya P and Babu{\v{s}}ka, Robert},
	year         = 2016,
	booktitle    = {2016 IEEE 55th conference on decision and control (CDC)},
	pages        = {5819--5826},
	organization = {IEEE}
}

@article{papini2017adaptive,
	title        = {Adaptive batch size for safe policy gradients},
	author       = {Papini, Matteo and Pirotta, Matteo and Restelli, Marcello},
	year         = 2017,
	journal      = {Advances in neural information processing systems},
	volume       = 30
}

@inproceedings{papini2018stochastic,
	title        = {Stochastic variance-reduced policy gradient},
	author       = {Papini, Matteo and Binaghi, Damiano and Canonaco, Giuseppe and Pirotta, Matteo and Restelli, Marcello},
	year         = 2018,
	booktitle    = {International conference on machine learning},
	pages        = {4026--4035},
	organization = {PMLR}
}

@article{patel2023ada,
	title        = {Ada-NAV: Adaptive Trajectory-Based Sample Efficient Policy Learning for Robotic Navigation},
	author       = {Patel, Bhrij and Weerakoon, Kasun and Suttle, Wesley A and Koppel, Alec and Sadler, Brian M and Bedi, Amrit Singh and Manocha, Dinesh},
	year         = 2023,
	journal      = {arXiv preprint arXiv:2306.06192}
}

@inproceedings{patel2024global,
	title        = {Global Optimality without Mixing Time Oracles in Average-reward RL via Multi-level Actor-Critic},
	author       = {Patel, Bhrij and Suttle, Wesley A and Koppel, Alec and Aggarwal, Vaneet and Sadler, Brian M and Bedi, Amrit Singh and Manocha, Dinesh},
	year         = 2024,
	booktitle    = {International Conference on Machine Learning}
}

@inproceedings{pesquerel2022imed,
	title        = {IMED-RL: Regret optimal learning of ergodic Markov decision processes},
	author       = {Pesquerel, Fabien and Maillard, Odalric-Ambrym},
	year         = 2022,
	booktitle    = {NeurIPS 2022-Thirty-sixth Conference on Neural Information Processing Systems}
}

@inproceedings{peters2005natural,
	title        = {Natural actor-critic},
	author       = {Peters, Jan and Vijayakumar, Sethu and Schaal, Stefan},
	year         = 2005,
	booktitle    = {Machine Learning: ECML 2005: 16th European Conference on Machine Learning, Porto, Portugal, October 3-7, 2005. Proceedings 16},
	pages        = {280--291},
	organization = {Springer}
}

@article{peters2008natural,
	title        = {Natural actor-critic},
	author       = {Peters, Jan and Schaal, Stefan},
	year         = 2008,
	journal      = {Neurocomputing},
	publisher    = {Elsevier},
	volume       = 71,
	number       = {7-9},
	pages        = {1180--1190}
}

@article{pinto2017asymmetric,
	title        = {Asymmetric actor critic for image-based robot learning},
	author       = {Pinto, Lerrel and Andrychowicz, Marcin and Welinder, Peter and Zaremba, Wojciech and Abbeel, Pieter},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1710.06542}
}

@article{pirotta2013adaptive,
	title        = {Adaptive step-size for policy gradient methods},
	author       = {Pirotta, Matteo and Restelli, Marcello and Bascetta, Luca},
	year         = 2013,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 26
}

@article{pirotta2015policy,
	title        = {Policy gradient in lipschitz markov decision processes},
	author       = {Pirotta, Matteo and Restelli, Marcello and Bascetta, Luca},
	year         = 2015,
	journal      = {Machine Learning},
	publisher    = {Springer},
	volume       = 100,
	pages        = {255--283}
}

@inproceedings{pmlr-v119-pilanci20a,
	title        = {Neural Networks are Convex Regularizers: Exact Polynomial-time Convex Optimization Formulations for Two-layer Networks},
	author       = {Pilanci, Mert and Ergen, Tolga},
	year         = 2020,
	month        = {13--18 Jul},
	booktitle    = {Proceedings of the 37th International Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 119,
	pages        = {7695--7705},
	url          = {https://proceedings.mlr.press/v119/pilanci20a.html},
	pdf          = {http://proceedings.mlr.press/v119/pilanci20a/pilanci20a.pdf}
}

@inproceedings{pmlr-v120-yang20a,
	title        = {A Theoretical Analysis of Deep Q-Learning},
	author       = {Fan, Jianqing and Wang, Zhaoran and Xie, Yuchen and Yang, Zhuoran},
	year         = 2020,
	month        = {10--11 Jun},
	booktitle    = {Proceedings of the 2nd Conference on Learning for Dynamics and Control},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 120,
	pages        = {486--489},
	url          = {https://proceedings.mlr.press/v120/yang20a.html},
	pdf          = {http://proceedings.mlr.press/v120/yang20a/yang20a.pdf}
}

@inproceedings{pmlr-v162-bartan22a,
	title        = {Neural {F}isher Discriminant Analysis: Optimal Neural Network Embeddings in Polynomial Time},
	author       = {Bartan, Burak and Pilanci, Mert},
	year         = 2022,
	month        = {17--23 Jul},
	booktitle    = {Proceedings of the 39th International Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 162,
	pages        = {1647--1663},
	url          = {https://proceedings.mlr.press/v162/bartan22a.html},
	pdf          = {https://proceedings.mlr.press/v162/bartan22a/bartan22a.pdf}
}

@inproceedings{pmlr-v162-chen22f,
	title        = {On the Sample Complexity of Learning Infinite-horizon Discounted Linear Kernel {MDP}s},
	author       = {Chen, Yuanzhou and He, Jiafan and Gu, Quanquan},
	year         = 2022,
	month        = {17--23 Jul},
	booktitle    = {Proceedings of the 39th International Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 162,
	pages        = {3149--3183},
	url          = {https://proceedings.mlr.press/v162/chen22f.html},
	pdf          = {https://proceedings.mlr.press/v162/chen22f/chen22f.pdf}
}

@inproceedings{pmlr-v162-mishkin22a,
	title        = {Fast Convex Optimization for Two-Layer {R}e{LU} Networks: Equivalent Model Classes and Cone Decompositions},
	author       = {Mishkin, Aaron and Sahiner, Arda and Pilanci, Mert},
	year         = 2022,
	month        = {17--23 Jul},
	booktitle    = {Proceedings of the 39th International Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 162,
	pages        = {15770--15816},
	url          = {https://proceedings.mlr.press/v162/mishkin22a.html},
	pdf          = {https://proceedings.mlr.press/v162/mishkin22a/mishkin22a.pdf}
}

@inproceedings{pmlr-v162-sahiner22a,
	title        = {Unraveling Attention via Convex Duality: Analysis and Interpretations of Vision Transformers},
	author       = {Sahiner, Arda and Ergen, Tolga and Ozturkler, Batu and Pauly, John and Mardani, Morteza and Pilanci, Mert},
	year         = 2022,
	month        = {17--23 Jul},
	booktitle    = {Proceedings of the 39th International Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 162,
	pages        = {19050--19088},
	url          = {https://proceedings.mlr.press/v162/sahiner22a.html},
	pdf          = {https://proceedings.mlr.press/v162/sahiner22a/sahiner22a.pdf}
}

@inproceedings{pmlr-v189-chen23b,
	title        = {On the Convergence of Decentralized Adaptive Gradient Methods},
	author       = {Chen, Xiangyi and Karimi, Belhal and Zhao, Weijie and Li, Ping},
	year         = 2023,
	month        = {12--14 Dec},
	booktitle    = {Proceedings of The 14th Asian Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 189,
	pages        = {217--232},
	editor       = {Khan, Emtiyaz and Gonen, Mehmet},
	pdf          = {https://proceedings.mlr.press/v189/chen23b/chen23b.pdf}
}

@inproceedings{pmlr-v195-yue23a,
	title        = {On the Lower Bound of Minimizing Polyak-Łojasiewicz functions},
	author       = {Yue, Pengyun and Fang, Cong and Lin, Zhouchen},
	year         = 2023,
	month        = {12--15 Jul},
	booktitle    = {Proceedings of Thirty Sixth Conference on Learning Theory},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 195,
	pages        = {2948--2968},
	editor       = {Neu, Gergely and Rosasco, Lorenzo},
	pdf          = {https://proceedings.mlr.press/v195/yue23a/yue23a.pdf}
}

@inproceedings{pmlr-v202-gaur23a,
	title        = {On the Global Convergence of Fitted Q-Iteration with Two-layer Neural Network Parametrization},
	author       = {Gaur, Mudit and Aggarwal, Vaneet and Agarwal, Mridul},
	year         = 2023,
	month        = {23--29 Jul},
	booktitle    = {Proceedings of the 40th International Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 202,
	pages        = {11013--11049},
	editor       = {Krause, Andreas and Brunskill, Emma and Cho, Kyunghyun and Engelhardt, Barbara and Sabato, Sivan and Scarlett, Jonathan},
	pdf          = {https://proceedings.mlr.press/v202/gaur23a/gaur23a.pdf}
}

@inproceedings{pmlr-v70-bellemare17a,
	title        = {A Distributional Perspective on Reinforcement Learning},
	author       = {Marc G. Bellemare and Will Dabney and R{\'e}mi Munos},
	year         = 2017,
	month        = {06--11 Aug},
	booktitle    = {Proceedings of the 34th International Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 70,
	pages        = {449--458},
	url          = {https://proceedings.mlr.press/v70/bellemare17a.html},
	pdf          = {http://proceedings.mlr.press/v70/bellemare17a/bellemare17a.pdf}
}

@inproceedings{pmlr-v70-yang17f,
	title        = {A Richer Theory of Convex Constrained Optimization with Reduced Projections and Improved Rates},
	author       = {Tianbao Yang and Qihang Lin and Lijun Zhang},
	year         = 2017,
	month        = {06--11 Aug},
	booktitle    = {Proceedings of the 34th International Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 70,
	pages        = {3901--3910},
	url          = {https://proceedings.mlr.press/v70/yang17f.html},
	pdf          = {http://proceedings.mlr.press/v70/yang17f/yang17f.pdf}
}

@inproceedings{pmlr-v80-papini18a,
	title        = {Stochastic Variance-Reduced Policy Gradient},
	author       = {Papini, Matteo and Binaghi, Damiano and Canonaco, Giuseppe and Pirotta, Matteo and Restelli, Marcello},
	year         = 2018,
	month        = {10--15 Jul},
	booktitle    = {Proceedings of the 35th International Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 80,
	pages        = {4026--4035},
	url          = {https://proceedings.mlr.press/v80/papini18a.html},
	pdf          = {http://proceedings.mlr.press/v80/papini18a/papini18a.pdf}
}

@inproceedings{pmlr-v97-allen-zhu19a,
	title        = {A Convergence Theory for Deep Learning via Over-Parameterization},
	author       = {Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
	year         = 2019,
	month        = {09--15 Jun},
	booktitle    = {Proceedings of the 36th International Conference on Machine Learning},
	publisher    = {PMLR},
	series       = {Proceedings of Machine Learning Research},
	volume       = 97,
	pages        = {242--252},
	url          = {https://proceedings.mlr.press/v97/allen-zhu19a.html},
	editor       = {Chaudhuri, Kamalika and Salakhutdinov, Ruslan},
	pdf          = {http://proceedings.mlr.press/v97/allen-zhu19a/allen-zhu19a.pdf}
}

@article{Polyak1963GradientMF,
	title        = {Gradient methods for the minimisation of functionals},
	author       = {Boris Polyak},
	year         = 1963,
	journal      = {Ussr Computational Mathematics and Mathematical Physics},
	volume       = 3,
	pages        = {864--878}
}

@book{puterman2014markov,
	title        = {Markov decision processes: discrete stochastic dynamic programming},
	author       = {Puterman, Martin L},
	year         = 2014,
	publisher    = {John Wiley \& Sons},
	added-at     = {2017-04-07T12:13:11.000+0200},
	biburl       = {https://www.bibsonomy.org/bibtex/22e7ac99cd30c4892171e5a7cef1bc7a7/becker},
	interhash    = {6cec8f775a265d8741171d17e4a4e7d0},
	intrahash    = {2e7ac99cd30c4892171e5a7cef1bc7a7},
	timestamp    = {2017-04-07T12:13:11.000+0200}
}

@article{qiu2021,
	title        = {On Finite-Time Convergence of Actor-Critic Algorithm},
	author       = {Qiu, Shuang and Yang, Zhuoran and Ye, Jieping and Wang, Zhaoran},
	year         = 2021,
	journal      = {IEEE Journal on Selected Areas in Information Theory},
	volume       = 2,
	number       = 2,
	pages        = {652--664},
	doi          = {10.1109/JSAIT.2021.3078754}
}

@article{qiu2021finite,
	title        = {On finite-time convergence of actor-critic algorithm},
	author       = {Qiu, Shuang and Yang, Zhuoran and Ye, Jieping and Wang, Zhaoran},
	year         = 2021,
	journal      = {IEEE Journal on Selected Areas in Information Theory},
	publisher    = {IEEE},
	volume       = 2,
	number       = 2,
	pages        = {652--664}
}

@article{rattray1998natural,
	title        = {Natural gradient descent for on-line learning},
	author       = {Rattray, Magnus and Saad, David and Amari, Shun-ichi},
	year         = 1998,
	journal      = {Physical review letters},
	publisher    = {APS},
	volume       = 81,
	number       = 24,
	pages        = 5461
}

@article{reddi2019convergence,
	title        = {On the convergence of adam and beyond},
	author       = {Reddi, Sashank J and Kale, Satyen and Kumar, Sanjiv},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1904.09237}
}

@article{riemer2021continual,
	title        = {Continual Learning In Environments With Polynomial Mixing Times},
	author       = {Riemer, Matthew and Raparthy, Sharath Chandra and Cases, Ignacio and Subbaraj, Gopeshh and Touzel, Maximilian Puelma and Rish, Irina},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2112.07066}
}

@article{roy1997,
	title        = {An analysis of temporal-difference learning with function approximation},
	author       = {Tsitsiklis, J.N. and Van Roy, B.},
	year         = 1997,
	journal      = {IEEE Transactions on Automatic Control},
	volume       = 42,
	number       = 5,
	pages        = {674--690},
	doi          = {10.1109/9.580874}
}

@article{roy1999,
	title        = {Average cost temporal-difference learning},
	author       = {Tsitsiklis, J.N. and Van Roy, B.},
	year         = 1999,
	journal      = {Automatica},
	volume       = 35,
	number       = 11,
	pages        = {1799--1808},
	issn         = {0005-1098}
}

@article{sahiner2020convex,
	title        = {Convex regularization behind neural reconstruction},
	author       = {Sahiner, Arda and Mardani, Morteza and Ozturkler, Batu and Pilanci, Mert and Pauly, John},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2012.05169}
}

@article{sahiner2020vector,
	title        = {Vector-output relu neural network problems are copositive programs: Convex analysis of two layer networks and polynomial-time algorithms},
	author       = {Sahiner, Arda and Ergen, Tolga and Pauly, John and Pilanci, Mert},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2012.13329}
}

@article{SchmidtHieber2020NonparametricRU,
	title        = {Nonparametric regression using deep neural networks with ReLU activation function},
	author       = {Johannes Schmidt-Hieber},
	year         = 2020,
	journal      = {ArXiv},
	volume       = {abs/1708.06633}
}

@inproceedings{shani2020adaptive,
	title        = {Adaptive trust region policy optimization: Global convergence and faster rates for regularized mdps},
	author       = {Shani, Lior and Efroni, Yonathan and Mannor, Shie},
	year         = 2020,
	booktitle    = {Proceedings of the AAAI Conference on Artificial Intelligence},
	volume       = 34,
	pages        = {5668--5675}
}

@article{silver2016mastering,
	title        = {Mastering the game of Go with deep neural networks and tree search},
	author       = {Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
	year         = 2016,
	journal      = {nature},
	publisher    = {Nature Publishing Group},
	volume       = 529,
	number       = 7587,
	pages        = {484--489}
}

@article{silver2017mastering,
	title        = {Mastering the game of Go without human knowledge},
	author       = {Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and Chen, Yutian and Lillicrap, Timothy and Hui, Fan and Sifre, Laurent and van den Driessche, George and Graepel, Thore and Hassabis, Demis},
	year         = 2017,
	month        = oct,
	journal      = {Nature},
	publisher    = {Macmillan Publishers Limited, part of Springer Nature. All rights reserved.},
	volume       = 550,
	pages        = {354--},
	url          = {http://dx.doi.org/10.1038/nature24270},
	added-at     = {2017-12-15T02:14:58.000+0100},
	biburl       = {https://www.bibsonomy.org/bibtex/2ecdfbfcceb55ee5f14c1c375ad71f2cb/achakraborty},
	description  = {Mastering the game of Go without human knowledge | Nature},
	interhash    = {c45d318e105d0f2d62ccc28c2699d9d4},
	intrahash    = {ecdfbfcceb55ee5f14c1c375ad71f2cb},
	timestamp    = {2017-12-15T02:14:58.000+0100}
}

@article{sun2018markov,
	title        = {On markov chain gradient descent},
	author       = {Sun, Tao and Sun, Yuejiao and Yin, Wotao},
	year         = 2018,
	journal      = {Advances in neural information processing systems},
	volume       = 31
}

@inproceedings{suttle2023beyond,
	title        = {Beyond Exponentially Fast Mixing in Average-Reward Reinforcement Learning via Multi-Level Monte Carlo Actor-Critic},
	author       = {Suttle, Wesley A and Bedi, Amrit and Patel, Bhrij and Sadler, Brian M and Koppel, Alec and Manocha, Dinesh},
	year         = 2023,
	booktitle    = {International Conference on Machine Learning},
	pages        = {33240--33267}
}

@article{sutton1988,
	title        = {Learning to Predict by the Method of Temporal Differences},
	author       = {Sutton, Richard},
	year         = 1988,
	month        = {08},
	journal      = {Machine Learning},
	volume       = 3,
	pages        = {9--44},
	doi          = {10.1007/BF00115009}
}

@article{sutton1988learning,
	title        = {Learning to predict by the methods of temporal differences},
	author       = {Sutton, Richard S},
	year         = 1988,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 3,
	pages        = {9--44}
}

@book{Sutton1998,
	title        = {Reinforcement Learning: An Introduction},
	author       = {Sutton, Richard S. and Barto, Andrew G.},
	year         = 2018,
	publisher    = {The MIT Press},
	url          = {http://incompleteideas.net/book/the-book-2nd.html},
	added-at     = {2019-07-13T10:11:53.000+0200},
	biburl       = {https://www.bibsonomy.org/bibtex/2f46601cf8b13d39d1378af0d79438b12/lanteunis},
	edition      = {Second},
	interhash    = {ac6b144aaec1819919a2fba9f705c852},
	intrahash    = {f46601cf8b13d39d1378af0d79438b12},
	timestamp    = {2019-07-13T10:11:53.000+0200}
}

@article{sutton1999policy,
	title        = {Policy gradient methods for reinforcement learning with function approximation},
	author       = {Sutton, Richard S and McAllester, David and Singh, Satinder and Mansour, Yishay},
	year         = 1999,
	journal      = {Advances in neural information processing systems},
	volume       = 12
}

@book{sutton2018reinforcement,
	title        = {Reinforcement learning: An introduction},
	author       = {Sutton, Richard S and Barto, Andrew G},
	year         = 2018,
	publisher    = {MIT press}
}

@article{swirszcz2016local,
	title        = {Local minima in training of neural networks},
	author       = {Swirszcz, Grzegorz and Czarnecki, Wojciech Marian and Pascanu, Razvan},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1611.06310}
}

@article{tadic2001convergence,
	title        = {On the convergence of temporal-difference learning with linear function approximation},
	author       = {Tadi{\'c}, Vladislav},
	year         = 2001,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 42,
	pages        = {241--267}
}

@article{tamboli2024reinforced,
	title        = {Reinforced Sequential Decision-Making for Sepsis Treatment: The POSNEGDM Framework with Mortality Classifier and Transformer},
	author       = {Tamboli, Dipesh and Chen, Jiayu and Jotheeswaran, Kiran Pranesh and Yu, Denny and Aggarwal, Vaneet},
	year         = 2024,
	journal      = {IEEE Journal of Biomedical and Health Informatics},
	publisher    = {IEEE}
}

@article{tang2022highway,
	title        = {Highway decision-making and motion planning for autonomous driving via soft actor-critic},
	author       = {Tang, Xiaolin and Huang, Bing and Liu, Teng and Lin, Xianke},
	year         = 2022,
	journal      = {IEEE Transactions on Vehicular Technology},
	publisher    = {IEEE},
	volume       = 71,
	number       = 5,
	pages        = {4706--4717}
}

@article{tesauro1995temporal,
	title        = {Temporal difference learning and TD-Gammon},
	author       = {Tesauro, Gerald and others},
	year         = 1995,
	journal      = {Communications of the ACM},
	volume       = 38,
	number       = 3,
	pages        = {58--68}
}

@inproceedings{tian2023convergence,
	title        = {Convergence of Actor-Critic with Multi-Layer Neural Networks},
	author       = {Haoxing Tian and Alex Olshevsky and Ioannis Paschalidis},
	year         = 2023,
	booktitle    = {Thirty-seventh Conference on Neural Information Processing Systems}
}

@inproceedings{van2016deep,
	title        = {Deep reinforcement learning with double q-learning},
	author       = {Van Hasselt, Hado and Guez, Arthur and Silver, David},
	year         = 2016,
	booktitle    = {Proceedings of the AAAI conference on artificial intelligence},
	volume       = 30
}

@article{wang2016sample,
	title        = {Sample efficient actor-critic with experience replay},
	author       = {Wang, Ziyu and Bapst, Victor and Heess, Nicolas and Mnih, Volodymyr and Munos, Remi and Kavukcuoglu, Koray and de Freitas, Nando},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1611.01224}
}

@inproceedings{wang2019neural,
	title        = {Neural Policy Gradient Methods: Global Optimality and Rates of Convergence},
	author       = {Wang, Lingxiao and Cai, Qi and Yang, Zhuoran and Wang, Zhaoran},
	year         = 2019,
	journal      = {International Conference on Learning Representations},
	booktitle    = {International Conference on Learning Representations}
}

@inproceedings{wang2020off,
	title        = {Off-policy recommendation system without exploration},
	author       = {Wang, Chengwei and Zhou, Tengfei and Chen, Chen and Hu, Tianlei and Chen, Gang},
	year         = 2020,
	booktitle    = {Pacific-Asia Conference on Knowledge Discovery and Data Mining},
	pages        = {16--27},
	organization = {Springer}
}

@inproceedings{wang2021hidden,
	title        = {The Hidden Convex Optimization Landscape of Regularized Two-Layer ReLU Networks: an Exact Characterization of Optimal Solutions},
	author       = {Wang, Yifei and Lacotte, Jonathan and Pilanci, Mert},
	year         = 2021,
	booktitle    = {International Conference on Learning Representations}
}

@inproceedings{wang2024nonasymptotic,
	title        = {Non-Asymptotic Analysis for Single-Loop (Natural) Actor-Critic with Compatible Function Approximation},
	author       = {Yudan Wang and Yue Wang and Yi Zhou and Shaofeng Zou},
	year         = 2024,
	booktitle    = {International Conference on Machine Learning}
}

@article{Watkins1992,
	title        = {Q-learning},
	author       = {Watkins, Christopher J. C. H. and Dayan, Peter},
	year         = 1992,
	month        = may,
	day          = 01,
	journal      = {Machine Learning},
	volume       = 8,
	number       = 3,
	pages        = {279--292},
	doi          = {10.1007/BF00992698},
	issn         = {1573-0565},
	url          = {https://doi.org/10.1007/BF00992698},
	added-at     = {2020-01-01T20:16:30.000+0100},
	biburl       = {https://www.bibsonomy.org/bibtex/2416ac9f845c6ccea5a7eacee4dedead8/lanteunis},
	interhash    = {a4436f9e14335d677f156049cb798253},
	intrahash    = {416ac9f845c6ccea5a7eacee4dedead8},
	timestamp    = {2020-01-01T20:16:30.000+0100}
}

@article{wei1810margin,
	title        = {On the margin theory of feedforward neural networks, 2018. preprint},
	author       = {Wei, C and Lee, JD and Liu, Q and Ma, T},
	journal      = {arXiv preprint arXiv:1810.05369}
}

@inproceedings{wei2020model,
	title        = {Model-free reinforcement learning in infinite-horizon average-reward markov decision processes},
	author       = {Wei, Chen-Yu and Jahromi, Mehdi Jafarnia and Luo, Haipeng and Sharma, Hiteshi and Jain, Rahul},
	year         = 2020,
	booktitle    = {International conference on machine learning},
	pages        = {10170--10180},
	organization = {PMLR}
}

@inproceedings{wei2021learning,
	title        = {Learning infinite-horizon average-reward mdps with linear function approximation},
	author       = {Wei, Chen-Yu and Jahromi, Mehdi Jafarnia and Luo, Haipeng and Jain, Rahul},
	year         = 2021,
	booktitle    = {International Conference on Artificial Intelligence and Statistics},
	pages        = {3007--3015},
	organization = {PMLR}
}

@inproceedings{Weissman2003InequalitiesFT,
	title        = {Inequalities for the L1 Deviation of the Empirical Distribution},
	author       = {Tsachy Weissman and Erik Ordentlich and Gadiel Seroussi and Sergio Verd{\'u} and Marcelo J. Weinberger},
	year         = 2003
}

@article{whiteson2020provably,
	title        = {Provably Convergent Two- Timescale Off- Policy Actor- Critic with Function Approximation},
	author       = {Whiteson, Shimon},
	year         = 2020
}

@inproceedings{williams1990mathematical,
	title        = {A mathematical analysis of actor-critic architectures for learning optimal controls through incremental dynamic programming},
	author       = {Williams, Ronald J and Baird, LC},
	year         = 1990,
	booktitle    = {Proceedings of the Sixth Yale Workshop on Adaptive and Learning Systems},
	pages        = {96--101},
	organization = {Citeseer}
}

@article{williams1992simple,
	title        = {Simple statistical gradient-following algorithms for connectionist reinforcement learning},
	author       = {Williams, Ronald J},
	year         = 1992,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 8,
	number       = 3,
	pages        = {229--256}
}

@inproceedings{wolfermixing2020,
	title        = {Mixing Time Estimation in Ergodic Markov Chains from a Single Trajectory with Contraction Methods},
	author       = {Wolfer, Geoffrey},
	year         = 2020,
	month        = {Feb},
	booktitle    = {Proceedings of the 31st International Conference  on Algorithmic Learning Theory}
}

@inproceedings{wu2017training,
	title        = {Training Agent for First-Person Shooter Game with Actor-Critic Curriculum Learning},
	author       = {Yuxin Wu and Yuandong Tian},
	year         = 2017,
	booktitle    = {International Conference on Learning Representations},
	url          = {https://openreview.net/forum?id=Hk3mPK5gg}
}

@article{wu2020finite,
	title        = {A finite-time analysis of two time-scale actor-critic methods},
	author       = {Wu, Yue Frank and Zhang, Weitong and Xu, Pan and Gu, Quanquan},
	year         = 2020,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 33,
	pages        = {17617--17628}
}

@article{DBLP:journals/corr/SchulmanWDRK17,
  author       = {John Schulman and
                  Filip Wolski and
                  Prafulla Dhariwal and
                  Alec Radford and
                  Oleg Klimov},
  title        = {Proximal Policy Optimization Algorithms},
  journal      = {CoRR},
  volume       = {abs/1707.06347},
  year         = {2017}
}

@article{xiao2014proximal,
	title        = {A proximal stochastic gradient method with progressive variance reduction},
	author       = {Xiao, Lin and Zhang, Tong},
	year         = 2014,
	journal      = {SIAM Journal on Optimization},
	publisher    = {SIAM},
	volume       = 24,
	number       = 4,
	pages        = {2057--2075}
}

@inproceedings{xu2019sample,
	title        = {Sample Efficient Policy Gradient Methods with Recursive Variance Reduction},
	author       = {Xu, Pan and Gao, Felicia and Gu, Quanquan},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1909.08610},
	booktitle    = {International Conference on Learning Representations}
}

@inproceedings{xu2020finite,
	title        = {A finite-time analysis of Q-learning with neural network function approximation},
	author       = {Xu, Pan and Gu, Quanquan},
	year         = 2020,
	booktitle    = {International Conference on Machine Learning},
	pages        = {10555--10565},
	organization = {PMLR}
}

@inproceedings{xu2020improved,
	title        = {An improved convergence analysis of stochastic variance-reduced policy gradient},
	author       = {Xu, Pan and Gao, Felicia and Gu, Quanquan},
	year         = 2020,
	booktitle    = {Uncertainty in Artificial Intelligence},
	pages        = {541--551}
}

@article{xu2020improving,
	title        = {Improving sample complexity bounds for (natural) actor-critic algorithms},
	author       = {Xu, Tengyu and Wang, Zhe and Liang, Yingbin},
	year         = 2020,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 33,
	pages        = {4358--4369}
}

@article{xu2020non,
	title        = {Non-asymptotic convergence analysis of two time-scale (natural) actor-critic algorithms},
	author       = {Xu, Tengyu and Wang, Zhe and Liang, Yingbin},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2005.03557}
}



@inproceedings{xu2021doubly,
	title        = {Doubly robust off-policy actor-critic: Convergence and optimality},
	author       = {Xu, Tengyu and Yang, Zhuoran and Wang, Zhaoran and Liang, Yingbin},
	year         = 2021,
	booktitle    = {International Conference on Machine Learning},
	pages        = {11581--11591},
	organization = {PMLR}
}

@article{Yang2020ReinforcementLI,
	title        = {Reinforcement learning in sustainable energy and electric systems: a survey},
	author       = {Ting Yang and Liyuan Zhao and Wei Li and Albert Y. Zomaya},
	year         = 2020,
	journal      = {Annu. Rev. Control.},
	volume       = 49,
	pages        = {145--163}
}

@article{yarotsky2017error,
	title        = {Error bounds for approximations with deep ReLU networks},
	author       = {Yarotsky, Dmitry},
	year         = 2017,
	journal      = {Neural Networks},
	publisher    = {Elsevier},
	volume       = 94,
	pages        = {103--114}
}

@inproceedings{yuan2022general,
	title        = {A general sample complexity analysis of vanilla policy gradient},
	author       = {Yuan, Rui and Gower, Robert M and Lazaric, Alessandro},
	year         = 2022,
	booktitle    = {International Conference on Artificial Intelligence and Statistics},
	pages        = {3332--3380},
	organization = {PMLR}
}

@article{yuan2022linear,
	title        = {Linear convergence of natural policy gradient methods with log-linear policies},
	author       = {Yuan, Rui and Du, Simon S and Gower, Robert M and Lazaric, Alessandro and Xiao, Lin},
	year         = 2022,
	journal      = {arXiv preprint arXiv:2210.01400}
}

@inproceedings{yuan2023linear,
	title        = {Linear Convergence of Natural Policy Gradient Methods with Log-Linear Policies},
	author       = {Rui Yuan and Simon Shaolei Du and Robert M. Gower and Alessandro Lazaric and Lin Xiao},
	year         = 2023,
	booktitle    = {The Eleventh International Conference on Learning Representations},
	url          = {https://openreview.net/forum?id=-z9hdsyUwVQ}
}

@inproceedings{zhang2019convergence,
	title        = {Convergence and iteration complexity of policy gradient method for infinite-horizon reinforcement learning},
	author       = {Zhang, Kaiqing and Koppel, Alec and Zhu, Hao and Ba{\c{s}}ar, Tamer},
	year         = 2019,
	booktitle    = {2019 IEEE 58th Conference on Decision and Control (CDC)},
	pages        = {7415--7422},
	organization = {IEEE}
}

@article{zhang2020almost,
	title        = {Almost optimal model-free reinforcement learningvia reference-advantage decomposition},
	author       = {Zhang, Zihan and Zhou, Yuan and Ji, Xiangyang},
	year         = 2020,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 33,
	pages        = {15198--15207}
}

@article{zhang2020global,
	title        = {Global convergence of policy gradient methods to (almost) locally optimal policies},
	author       = {Zhang, Kaiqing and Koppel, Alec and Zhu, Hao and Basar, Tamer},
	year         = 2020,
	journal      = {SIAM Journal on Control and Optimization},
	publisher    = {SIAM},
	volume       = 58,
	number       = 6,
	pages        = {3586--3612}
}

@inproceedings{zhang2020provably,
	title        = {Provably convergent two-timescale off-policy actor-critic with function approximation},
	author       = {Zhang, Shangtong and Liu, Bo and Yao, Hengshuai and Whiteson, Shimon},
	year         = 2020,
	booktitle    = {International Conference on Machine Learning},
	pages        = {11204--11213},
	organization = {PMLR}
}

@inproceedings{zhang2021,
	title        = {On-Policy Deep Reinforcement Learning for the Average-Reward Criterion},
	author       = {Yiming Zhang and Keith W. Ross},
	year         = 2021,
	booktitle    = {{ICML}},
	publisher    = {{PMLR}},
	series       = {Proceedings of Machine Learning Research},
	volume       = 139,
	pages        = {12535--12545}
}

@inproceedings{zhang2021on,
	title        = {On the Convergence and Sample Efficiency of Variance-Reduced Policy Gradient Method},
	author       = {Junyu Zhang and Chengzhuo Ni and Zheng Yu and Csaba Szepesvari and Mengdi Wang},
	year         = 2021,
	booktitle    = {Advances in Neural Information Processing Systems},
	editor       = {A. Beygelzimer and Y. Dauphin and P. Liang and J. Wortman Vaughan}
}

@inproceedings{zhang2022convergence,
	title        = {Convergence and Optimality of Policy Gradient Methods in Weakly Smooth Settings},
	author       = {Zhang, Matthew S and Erdogdu, Murat A and Garg, Animesh},
	year         = 2022,
	booktitle    = {Proceedings of the AAAI Conference on Artificial Intelligence},
	volume       = 36,
	pages        = {9066--9073}
}

@article{zhang2022duasvs,
	title        = {Duasvs: A mobile data saving strategy in short-form video streaming},
	author       = {Zhang, Guanghui and Zhang, Jie and Liu, Ke and Guo, Jing and Lee, Jack YB and Hu, Haibo and Aggarwal, Vaneet},
	year         = 2022,
	journal      = {IEEE Transactions on Services Computing},
	publisher    = {IEEE},
	volume       = 16,
	number       = 2,
	pages        = {1066--1078}
}

@article{zheng2022stochastic,
	title        = {Stochastic Integrated Actor--Critic for Deep Reinforcement Learning},
	author       = {Zheng, Jiaohao and Kurt, Mehmet Necip and Wang, Xiaodong},
	year         = 2022,
	journal      = {IEEE Transactions on Neural Networks and Learning Systems},
	publisher    = {IEEE}
}

@article{zhong2019deep,
	title        = {A deep actor-critic reinforcement learning framework for dynamic multichannel access},
	author       = {Zhong, Chen and Lu, Ziyang and Gursoy, M Cenk and Velipasalar, Senem},
	year         = 2019,
	journal      = {IEEE Transactions on Cognitive Communications and Networking},
	publisher    = {IEEE},
	volume       = 5,
	number       = 4,
	pages        = {1125--1139}
}

@inproceedings{zhou2021provably,
	title        = {Provably efficient reinforcement learning for discounted mdps with feature mapping},
	author       = {Zhou, Dongruo and He, Jiafan and Gu, Quanquan},
	year         = 2021,
	booktitle    = {International Conference on Machine Learning},
	pages        = {12793--12802},
	organization = {PMLR}
}

@article{zhou2022pac,
	title        = {Pac: Assisted value factorization with counterfactual predictions in multi-agent reinforcement learning},
	author       = {Zhou, Hanhan and Lan, Tian and Aggarwal, Vaneet},
	year         = 2022,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 35,
	pages        = {15757--15769}
}

@inproceedings{zhu2021one,
	title        = {One-pass Stochastic Gradient Descent in overparametrized two-layer neural networks},
	author       = {Zhu, Hanjing and Xu, Jiaming},
	year         = 2021,
	booktitle    = {International Conference on Artificial Intelligence and Statistics},
	pages        = {3673--3681},
	organization = {PMLR}
}

@article{zou2019finite,
	title        = {Finite-sample analysis for sarsa with linear function approximation},
	author       = {Zou, Shaofeng and Xu, Tengyu and Liang, Yingbin},
	year         = 2019,
	journal      = {Advances in neural information processing systems},
	volume       = 32
}

@inproceedings{nagaraj2020neurips,
 author = {Nagaraj, Dheeraj and Wu, Xian and Bresler, Guy and Jain, Prateek and Netrapalli, Praneeth},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {16666--16676},
 title = {Least Squares Regression with Markovian Data: Fundamental Limits and Algorithms},
 volume = {33},
 year = {2020}
}

@inproceedings{NTK,
 author = {Jacot, Arthur and Gabriel, Franck and Hongler, Clement},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {},
 title = {Neural Tangent Kernel: Convergence and Generalization in Neural Networks},
 volume = {31},
 year = {2018}
}


@InProceedings{pmlr-v119-xu20c,
  title = 	 {A Finite-Time Analysis of Q-Learning with Neural Network Function Approximation},
  author =       {Xu, Pan and Gu, Quanquan},
  booktitle = 	 {Proceedings of the 37th International Conference on Machine Learning},
  pages = 	 {10555--10565},
  year = 	 {2020},
  volume = 	 {119},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {13--18 Jul},
  publisher =    {PMLR}
}

@misc{nussbaum2003notessecondeigenvaluegoogle,
      title={Notes on the Second Eigenvalue of the Google Matrix}, 
      author={Roger Nussbaum},
      year={2003},
      eprint={math/0307056},
      archivePrefix={arXiv},
      primaryClass={math.FA},
      url={https://arxiv.org/abs/math/0307056}, 
}

@book{LevinPeresWilmer2006,
  added-at = {2010-01-19T17:51:27.000+0100},
  author = {Levin, David A. and Peres, Yuval and Wilmer, Elizabeth L.},
  publisher = {American Mathematical Society},
  title = {{Markov chains and mixing times}},
  year = 2006
}

@InProceedings{Liu2022AnIA,
  title={An Improved Analysis of (Variance-Reduced) Policy Gradient and Natural Policy Gradient Methods},
  author={Yanli Liu and K. Zhang and Tamer Başar and Wotao Yin},
  booktitle={Conference on Neural Information Processing Systems (NeurIPS)},
  year={2020},
  series = 	 {Proceedings of Machine Learning Research},
  publisher =    {PMLR}
}