%PRELIMINARIES
@article{li2017deep,
  title={Deep reinforcement learning: An overview},
  author={Li, Yuxi},
  journal={arXiv preprint arXiv:1701.07274},
  year={2017}
}
@article{even2009online,
  title={Online Markov decision processes},
  author={Even-Dar, Eyal and Kakade, Sham M and Mansour, Yishay},
  journal={Mathematics of Operations Research},
  volume={34},
  number={3},
  pages={726--736},
  year={2009},
  publisher={INFORMS}
}
@article{ratliff2021introduction,
  title={An Introduction to Learning in Games},
  author={Ratliff, Lillian},
  year={2021}
}
@inproceedings{flet2021learning,
  title={Learning value functions in deep policy gradients using residual variance},
  author={Flet-Berliac, Yannis and Ouhamma, Reda and Maillard, Odalric-Ambrym and Preux, Philippe},
  booktitle={ICLR 2021-International Conference on Learning Representations},
  year={2021}
}
@article{johnson2013accelerating,
  title={Accelerating stochastic gradient descent using predictive variance reduction},
  author={Johnson, Rie and Zhang, Tong},
  journal={Advances in neural information processing systems},
  volume={26},
  pages={315--323},
  year={2013},
  publisher={Citeseer}
}
@inproceedings{papini2018stochastic,
  title={Stochastic variance-reduced policy gradient},
  author={Papini, Matteo and Binaghi, Damiano and Canonaco, Giuseppe and Pirotta, Matteo and Restelli, Marcello},
  booktitle={International Conference on Machine Learning},
  pages={4026--4035},
  year={2018},
  organization={PMLR}
}
@inproceedings{zhao2016regularized,
  title={Regularized policy gradients: direct variance reduction in policy gradient estimation},
  author={Zhao, Tingting and Niu, Gang and Xie, Ning and Yang, Jucheng and Sugiyama, Masashi},
  booktitle={Asian Conference on Machine Learning},
  pages={333--348},
  year={2016},
  organization={PMLR}
}
@article{lin2019adaptive,
  title={Adaptive auxiliary task weighting for reinforcement learning},
  author={Lin, Xingyu and Baweja, Harjatin Singh and Kantor, George and Held, David},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}
@article{jaderberg2016reinforcement,
  title={Reinforcement learning with unsupervised auxiliary tasks},
  author={Jaderberg, Max and Mnih, Volodymyr and Czarnecki, Wojciech Marian and Schaul, Tom and Leibo, Joel Z and Silver, David and Kavukcuoglu, Koray},
  journal={arXiv preprint arXiv:1611.05397},
  year={2016}
}
@inproceedings{kartal2019terminal,
  title={Terminal prediction as an auxiliary task for deep reinforcement learning},
  author={Kartal, Bilal and Hernandez-Leal, Pablo and Taylor, Matthew E},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence and Interactive Digital Entertainment},
  volume={15},
  number={1},
  pages={38--44},
  year={2019}
}
@inproceedings{namkoong2017variance,
  title={Variance-based regularization with convex objectives},
  author={Namkoong, Hongseok and Duchi, John C},
  booktitle={Proceedings of the 31st International Conference on Neural Information Processing Systems},
  pages={2975--2984},
  year={2017}
}
@article{shapley1964some,
  title={Some topics in two-person games},
  author={Shapley, Lloyd},
  journal={Advances in game theory},
  volume={52},
  pages={1--29},
  year={1964}
}
@article{xie2020provable,
  title={Provable Fictitious Play for General Mean-Field Games},
  author={Xie, Qiaomin and Yang, Zhuoran and Wang, Zhaoran and Minca, Andreea},
  journal={arXiv preprint arXiv:2010.04211},
  year={2020}
}
@article{lambert2005fictitious,
  title={A fictitious play approach to large-scale optimization},
  author={Lambert Iii, Theodore J and Epelman, Marina A and Smith, Robert L},
  journal={Operations Research},
  volume={53},
  number={3},
  pages={477--489},
  year={2005},
  publisher={INFORMS}
}
@article{robinson1951iterative,
  title={An iterative method of solving a game},
  author={Robinson, Julia},
  journal={Annals of mathematics},
  pages={296--301},
  year={1951},
  publisher={JSTOR}
}
@article{hart2003uncoupled,
  title={Uncoupled dynamics do not lead to Nash equilibrium},
  author={Hart, Sergiu and Mas-Colell, Andreu},
  journal={American Economic Review},
  volume={93},
  number={5},
  pages={1830--1836},
  year={2003}
}
@phdthesis{mertikopoulos2019online,
  title={Online optimization and learning in games: Theory and applications},
  author={Mertikopoulos, Panayotis},
  year={2019},
  school={Grenoble 1 UGA-Universit{\'e} Grenoble Alpes}
}
@book{fudenberg1998theory,
  title={The theory of learning in games},
  author={Fudenberg, Drew and Drew, Fudenberg and Levine, David K and Levine, David K},
  volume={2},
  year={1998},
  publisher={MIT press}
}
@article{morimoto2005robust,
  title={Robust reinforcement learning},
  author={Morimoto, Jun and Doya, Kenji},
  journal={Neural computation},
  volume={17},
  number={2},
  pages={335--359},
  year={2005},
  publisher={MIT Press}
}
@inproceedings{gruver2020multi,
  title={Multi-agent adversarial inverse reinforcement learning with latent variables},
  author={Gruver, Nate and Song, Jiaming and Kochenderfer, Mykel J and Ermon, Stefano},
  booktitle={Proceedings of the 19th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={1855--1857},
  year={2020}
}
@inproceedings{yu2019multi,
  title={Multi-agent adversarial inverse reinforcement learning},
  author={Yu, Lantao and Song, Jiaming and Ermon, Stefano},
  booktitle={International Conference on Machine Learning},
  pages={7194--7201},
  year={2019},
  organization={PMLR}
}
@inproceedings{wang2018competitive,
  title={Competitive multi-agent inverse reinforcement learning with sub-optimal demonstrations},
  author={Wang, Xingyu and Klabjan, Diego},
  booktitle={International Conference on Machine Learning},
  pages={5143--5151},
  year={2018},
  organization={PMLR}
}
@inproceedings{gergely2010online,
  title={Online Markov decision processes under bandit feedback},
  author={Gergely Neu, Andr{\'a}s Gy{\"o}rgy and Szepesv{\'a}ri, Csaba and Antos, Andr{\'a}s},
  booktitle={Proceedings of the Twenty-Fourth Annual Conference on Neural Information Processing Systems},
  year={2010}
}
@article{lykouris2019corruption,
  title={Corruption robust exploration in episodic reinforcement learning},
  author={Lykouris, Thodoris and Simchowitz, Max and Slivkins, Aleksandrs and Sun, Wen},
  journal={arXiv preprint arXiv:1911.08689},
  year={2019}
}
@inproceedings{yu2009arbitrarily,
  title={Arbitrarily modulated Markov decision processes},
  author={Yu, Jia Yuan and Mannor, Shie},
  booktitle={Proceedings of the 48h IEEE Conference on Decision and Control (CDC) held jointly with 2009 28th Chinese Control Conference},
  pages={2946--2953},
  year={2009},
  organization={IEEE}
}
@inproceedings{jin2020learning,
  title={Learning adversarial Markov decision processes with bandit feedback and unknown transition},
  author={Jin, Chi and Jin, Tiancheng and Luo, Haipeng and Sra, Suvrit and Yu, Tiancheng},
  booktitle={International Conference on Machine Learning},
  pages={4860--4869},
  year={2020},
  organization={PMLR}
}
@inproceedings{rosenberg2019online,
  title={Online convex optimization in adversarial markov decision processes},
  author={Rosenberg, Aviv and Mansour, Yishay},
  booktitle={International Conference on Machine Learning},
  pages={5478--5486},
  year={2019},
  organization={PMLR}
}
@inproceedings{zimin2013online,
  title={Online learning in episodic Markovian decision processes by relative entropy policy search},
  author={Zimin, Alexander and Neu, Gergely},
  booktitle={Neural Information Processing Systems 26},
  year={2013}
}
@inproceedings{neu2012adversarial,
  title={The adversarial stochastic shortest path problem with unknown transition probabilities},
  author={Neu, Gergely and Gyorgy, Andras and Szepesv{\'a}ri, Csaba},
  booktitle={Artificial Intelligence and Statistics},
  pages={805--813},
  year={2012},
  organization={PMLR}
}
@inproceedings{dick2014online,
  title={Online learning in markov decision processes with changing cost sequences},
  author={Dick, Travis and Gyorgy, Andras and Szepesvari, Csaba},
  booktitle={International Conference on Machine Learning},
  pages={512--520},
  year={2014},
  organization={PMLR}
}
@article{skoulakis2020evolutionary,
  title={Evolutionary Game Theory Squared: Evolving Agents in Endogenously Evolving Zero-Sum Games},
  author={Skoulakis, Stratis and Fiez, Tanner and Sim, Ryann and Piliouras, Georgios and Ratliff, Lillian},
  booktitle={Proceedings of the 19th International Conference on Autonomous Agents and MultiAgent Systems},
  year={2021}
}
@inproceedings{mazumdar2020policy,
  title={Policy-Gradient Algorithms Have No Guarantees of Convergence in Linear Quadratic Games},
  author={Mazumdar, Eric and Ratliff, Lillian J and Jordan, Michael I and Sastry, S Shankar},
  booktitle={Proceedings of the 19th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={860--868},
  year={2020}
}
@article{brafman2002r,
  title={R-max-a general polynomial time algorithm for near-optimal reinforcement learning},
  author={Brafman, Ronen I and Tennenholtz, Moshe},
  journal={Journal of Machine Learning Research},
  volume={3},
  number={Oct},
  pages={213--231},
  year={2002}
}
@inproceedings{lin2020gradient,
  title={On gradient descent ascent for nonconvex-concave minimax problems},
  author={Lin, Tianyi and Jin, Chi and Jordan, Michael},
  booktitle={International Conference on Machine Learning},
  pages={6083--6093},
  year={2020},
  organization={PMLR}
}
@article{whittle1981risk,
  title={Risk-sensitive linear/quadratic/Gaussian control},
  author={Whittle, Peter},
  journal={Advances in Applied Probability},
  pages={764--777},
  year={1981},
  publisher={JSTOR}
}
@article{jacobson1973optimal,
  title={Optimal stochastic linear systems with exponential performance criteria and their relation to deterministic differential games},
  author={Jacobson, David},
  journal={IEEE Transactions on Automatic control},
  volume={18},
  number={2},
  pages={124--131},
  year={1973},
  publisher={IEEE}
}
@inproceedings{zhang2020policy,
  title={Policy Optimization for $\mathcal{H}_2$ Linear Control with $\mathcal{H}_2$ $\infty$ Robustness Guarantee: Implicit Regularization and Global Convergence},
  author={Zhang, Kaiqing and Hu, Bin and Basar, Tamer},
  booktitle={Learning for Dynamics and Control},
  pages={179--190},
  year={2020},
  organization={PMLR}
}
@book{bacsar2008h,
  title={H-infinity optimal control and related minimax design problems: a dynamic game approach},
  author={Ba{\c{s}}ar, Tamer and Bernhard, Pierre},
  year={2008},
  publisher={Springer Science \& Business Media}
}
@inproceedings{jin2020local,
  title={What is local optimality in nonconvex-nonconcave minimax optimization?},
  author={Jin, Chi and Netrapalli, Praneeth and Jordan, Michael},
  booktitle={International Conference on Machine Learning},
  pages={4880--4889},
  year={2020},
  organization={PMLR}
}
@article{silver2017mastering,
  title={Mastering the game of go without human knowledge},
  author={Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and others},
  journal={nature},
  volume={550},
  number={7676},
  pages={354--359},
  year={2017},
  publisher={Nature Publishing Group}
}
@inproceedings{ramponi2020truly,
  title={Truly Batch Model-Free Inverse Reinforcement Learning about Multiple Intentions},
  author={Ramponi, Giorgia and Likmeta, Amarildo and Metelli, Alberto Maria and Tirinzoni, Andrea and Restelli, Marcello},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2359--2369},
  year={2020},
  organization={PMLR}
}
@misc{mangoubi2021greedy,
      title={Greedy Adversarial Equilibrium: An Efficient Alternative to Nonconvex-Nonconcave Min-Max Optimization}, 
      author={Oren Mangoubi and Nisheeth K. Vishnoi},
      year={2021},
      eprint={2006.12363},
      archivePrefix={arXiv},
      primaryClass={cs.DS}
}
@article{celli2020no,
  title={No-regret learning dynamics for extensive-form correlated equilibrium},
  author={Celli, Andrea and Marchesi, Alberto and Farina, Gabriele and Gatti, Nicola},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}
@article {shayegan2019,
	Title = {α-Rank: Multi-Agent Evaluation by Evolution},
	Author = {Omidshafiei, Shayegan and Papadimitriou, Christos and Piliouras, Georgios and Tuyls, Karl and Rowland, Mark and Lespiau, Jean-Baptiste and Czarnecki, Wojciech M and Lanctot, Marc and Perolat, Julien and Munos, Remi},
	DOI = {10.1038/s41598-019-45619-9},
	Number = {1},
	Volume = {9},
	Month = {July},
	Year = {2019},
	Journal = {Scientific reports},
	ISSN = {2045-2322},
	Pages = {9937}
}


@article{papadimitriou2019game,
  title={Game dynamics as the meaning of a game},
  author={Papadimitriou, Christos and Piliouras, Georgios},
  journal={ACM SIGecom Exchanges},
  volume={16},
  number={2},
  pages={53--63},
  year={2019},
  publisher={ACM New York, NY, USA}
}
@inproceedings{parameswaran2020,
 author = {Kamalaruban, Parameswaran and Huang, Yu-Ting and Hsieh, Ya-Ping and Rolland, Paul and Shi, Cheng and Cevher, Volkan},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 pages = {8127--8138},
 publisher = {Curran Associates, Inc.},
 title = {Robust Reinforcement Learning via Adversarial training with Langevin Dynamics},
 url = {https://proceedings.neurips.cc/paper/2020/file/5cb0e249689cd6d8369c4885435a56c2-Paper.pdf},
 volume = {33},
 year = {2020}
}

@inproceedings{hsieh2019finding,
  title={Finding mixed nash equilibria of generative adversarial networks},
  author={Hsieh, Ya-Ping and Liu, Chen and Cevher, Volkan},
  booktitle={International Conference on Machine Learning},
  pages={2810--2819},
  year={2019},
  organization={PMLR}
}
@article{hsieh2021adaptive,
  title={Adaptive Learning in Continuous Games: Optimal Regret Bounds and Convergence to Nash Equilibrium},
  author={Hsieh, Yu-Guan and Antonakopoulos, Kimon and Mertikopoulos, Panayotis},
  journal={arXiv preprint arXiv:2104.12761},
  year={2021}
}
@article{arulkumaran2017brief,
  title={A brief survey of deep reinforcement learning},
  author={Arulkumaran, Kai and Deisenroth, Marc Peter and Brundage, Miles and Bharath, Anil Anthony},
  journal={arXiv preprint arXiv:1708.05866},
  year={2017}
}
@article{bai2021sample,
  title={Sample-Efficient Learning of Stackelberg Equilibria in General-Sum Games},
  author={Bai, Yu and Jin, Chi and Wang, Huan and Xiong, Caiming},
  journal={arXiv preprint arXiv:2102.11494},
  year={2021}
}
@inproceedings{agrawal2017optimistic,
  title={Optimistic posterior sampling for reinforcement learning: worst-case regret bounds},
  author={Agrawal, Shipra and Jia, Randy},
  booktitle={Proceedings of the 31st International Conference on Neural Information Processing Systems},
  pages={1184--1194},
  year={2017}
}
@inproceedings{osband2017posterior,
  title={Why is posterior sampling better than optimism for reinforcement learning?},
  author={Osband, Ian and Van Roy, Benjamin},
  booktitle={International Conference on Machine Learning},
  pages={2701--2710},
  year={2017},
  organization={PMLR}
}
@article{osband2013more,
  title={(More) efficient reinforcement learning via posterior sampling},
  author={Osband, Ian and Van Roy, Benjamin and Russo, Daniel},
  journal={Advances in Neural Information Processing Systems},
  year={2013}
}
@inproceedings{fruit2018near,
  title={Near Optimal Exploration-Exploitation in Non-Communicating Markov Decision Processes},
  author={Fruit, Ronan and Pirotta, Matteo and Lazaric, Alessandro},
  booktitle={32nd Conference on Neural Information Processing Systems},
  year={2018}
}
@inproceedings{bartlett2009regal,
  title={REGAL: a regularization based algorithm for reinforcement learning in weakly communicating MDPs},
  author={Bartlett, Peter and Tewari, Ambuj},
  booktitle={Uncertainty in Artificial Intelligence: Proceedings of the 25th Conference},
  pages={35--42},
  year={2009},
  organization={AUAI Press}
}
@article{rajasekaran2017inverse,
  title={Inverse reinforce learning with nonparametric behavior clustering},
  author={Rajasekaran, Siddharthan and Zhang, Jinwei and Fu, Jie},
  journal={arXiv preprint arXiv:1712.05514},
  year={2017}
}
@article{franccois2018introduction,
  title={An Introduction to Deep Reinforcement Learning},
  author={Fran{\c{c}}ois-Lavet, Vincent and Henderson, Peter and Islam, Riashat and Bellemare, Marc G and Pineau, Joelle},
  journal={Foundations and Trends in Machine Learning},
  volume={11},
  number={3-4},
  pages={219--354},
  year={2018},
  publisher={Now Publishers Boston—Delft}
}

@inproceedings{konda2000actor,
  title={Actor-critic algorithms},
  author={Konda, Vijay R and Tsitsiklis, John N},
  booktitle={Advances in neural information processing systems},
  pages={1008--1014},
  year={2000},
  organization={Citeseer}
}

@article{peters2008natural,
  title={Natural actor-critic},
  author={Peters, Jan and Schaal, Stefan},
  journal={Neurocomputing},
  volume={71},
  number={7-9},
  pages={1180--1190},
  year={2008},
  publisher={Elsevier}
}
@inproceedings{schulman2015trust,
  title={Trust region policy optimization},
  author={Schulman, John and Levine, Sergey and Abbeel, Pieter and Jordan, Michael and Moritz, Philipp},
  booktitle={International conference on machine learning},
  pages={1889--1897},
  year={2015},
  organization={PMLR}
}
@article{schulman2017proximal,
  title={Proximal policy optimization algorithms},
  author={Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
  journal={arXiv preprint arXiv:1707.06347},
  year={2017}
}
@article{chung2020distributed,
  title={Distributed deep reinforcement learning for intelligent load scheduling in residential smart grid},
  author={Chung, Hwei-Ming and Maharjan, Sabita and Zhang, Yan and Eliassen, Frank},
  journal={IEEE Transactions on Industrial Informatics},
  year={2020},
  publisher={IEEE}
}
@article{taslaman2014principal,
  title={The principal angles and the gap},
  author={Taslaman, Leo},
  year={2014},
  publisher={Manchester Institute for Mathematical Sciences, University of Manchester}
}
@article{knyazev2010angles,
  title={Angles between infinite dimensional subspaces with applications to the Rayleigh--Ritz and alternating projectors methods},
  author={Knyazev, Andrew and Jujunashvili, Abram and Argentati, Merico},
  journal={Journal of Functional Analysis},
  volume={259},
  number={6},
  pages={1323--1345},
  year={2010},
  publisher={Elsevier}
}
@book{morgenstern1953theory,
  title={Theory of games and economic behavior},
  author={Morgenstern, Oskar and Von Neumann, John},
  year={1953},
  publisher={Princeton university press}
}

@inproceedings{YuWL18,
  title={Deep-Reinforcement Learning Multiple Access for Heterogeneous Wireless Networks},
  author={Yu, Yiding and Wang, Taotao and Liew, Soung Chang},
  booktitle={2018 IEEE International Conference on Communications (ICC)},
  pages={1--7},
  year={2018},
  organization={IEEE}
}
@inproceedings{abbasi2013online,
  title={Online learning in Markov decision processes with adversarially chosen transition probability distributions},
  author={Abbasi-Yadkori, Yasin and Bartlett, Peter L and Kanade, Varun and Seldin, Yevgeny and Szepesv{\'a}ri, Csaba},
  booktitle={Proceedings of the 26th International Conference on Neural Information Processing Systems-Volume 2},
  pages={2508--2516},
  year={2013}
}

@inproceedings{balakrishna2020policy,
  title={On-policy robot imitation learning from a converging supervisor},
  author={Balakrishna, Ashwin and Thananjeyan, Brijen and Lee, Jonathan and Li, Felix and Zahed, Arsh and Gonzalez, Joseph E and Goldberg, Ken},
  booktitle={Conference on Robot Learning},
  pages={24--41},
  year={2020},
  organization={PMLR}
}
@inproceedings{tateo2017gradient,
  title={Gradient-based minimization for multi-expert inverse reinforcement learning},
  author={Tateo, Davide and Pirotta, Matteo and Restelli, Marcello and Bonarini, Andrea},
  booktitle={2017 IEEE Symposium Series on Computational Intelligence (SSCI)},
  pages={1--8},
  year={2017},
  organization={IEEE}
}
@article{shalev2016safe,
  title={Safe, multi-agent, reinforcement learning for autonomous driving},
  author={Shalev-Shwartz, Shai and Shammah, Shaked and Shashua, Amnon},
  journal={arXiv preprint arXiv:1610.03295},
  year={2016}
}
@article{lillicrap2015continuous,
  title={Continuous control with deep reinforcement learning},
  author={Lillicrap, Timothy P and Hunt, Jonathan J and Pritzel, Alexander and Heess, Nicolas and Erez, Tom and Tassa, Yuval and Silver, David and Wierstra, Daan},
  journal={arXiv preprint arXiv:1509.02971},
  year={2015}
}

@article{mnih2015human,
  title={Human-level control through deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and others},
  journal={nature},
  volume={518},
  number={7540},
  pages={529--533},
  year={2015},
  publisher={Nature Publishing Group}
}
@article{jaksch2010near,
  title={Near-optimal Regret Bounds for Reinforcement Learning.},
  author={Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
  journal={Journal of Machine Learning Research},
  volume={11},
  number={4},
  year={2010}
}
@misc{domingues2020episodic,
      title={Episodic Reinforcement Learning in Finite MDPs: Minimax Lower Bounds Revisited}, 
      author={Omar Darwiche Domingues and Pierre Ménard and Emilie Kaufmann and Michal Valko},
      year={2020},
      eprint={2010.03531},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}
@article{feller1957introduction,
  title={An introduction to probability theory and its applications},
  author={Feller, William},
  year={1957}
}
@inproceedings{azar2017minimax,
  title={Minimax regret bounds for reinforcement learning},
  author={Azar, Mohammad Gheshlaghi and Osband, Ian and Munos, R{\'e}mi},
  booktitle={International Conference on Machine Learning},
  pages={263--272},
  year={2017},
  organization={PMLR}
}
@book{cesa2006prediction,
  title={Prediction, learning, and games},
  author={Cesa-Bianchi, Nicolo and Lugosi, G{\'a}bor},
  year={2006},
  publisher={Cambridge university press}
}
@inproceedings{silver2014deterministic,
  title={Deterministic policy gradient algorithms},
  author={Silver, David and Lever, Guy and Heess, Nicolas and Degris, Thomas and Wierstra, Daan and Riedmiller, Martin},
  booktitle={International conference on machine learning},
  pages={387--395},
  year={2014},
  organization={PMLR}
}
@inproceedings{haarnoja2018soft,
  title={Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor},
  author={Haarnoja, Tuomas and Zhou, Aurick and Abbeel, Pieter and Levine, Sergey},
  booktitle={International Conference on Machine Learning},
  pages={1861--1870},
  year={2018},
  organization={PMLR}
}
@book{rummery1994line,
  title={On-line Q-learning using connectionist systems},
  author={Rummery, Gavin A and Niranjan, Mahesan},
  volume={37},
  year={1994},
  publisher={University of Cambridge, Department of Engineering Cambridge, UK}
}

@book{sutton1998introduction,
  title={Introduction to reinforcement learning},
  author={Sutton, Richard S and Barto, Andrew G and others},
  volume={135},
  year={1998},
  publisher={MIT press Cambridge}
}
@inproceedings{perkins2003convergent,
  title={A convergent form of approximate policy iteration},
  author={Perkins, Theodore J and Precup, Doina},
  booktitle={Advances in neural information processing systems},
  pages={1627--1634},
  year={2003},
  organization={Citeseer}
}
@inproceedings{melo2008analysis,
  title={An analysis of reinforcement learning with function approximation},
  author={Melo, Francisco S and Meyn, Sean P and Ribeiro, M Isabel},
  booktitle={Proceedings of the 25th international conference on Machine learning},
  pages={664--671},
  year={2008}
}
@book{bertsekas1995dynamic,
  title={Dynamic programming and optimal control},
  author={Bertsekas, Dimitri P and Bertsekas, Dimitri P and Bertsekas, Dimitri P and Bertsekas, Dimitri P},
  volume={1},
  number={2},
  year={1995},
  publisher={Athena scientific Belmont, MA}
}
@book{powell2007approximate,
  title={Approximate Dynamic Programming: Solving the curses of dimensionality},
  author={Powell, Warren B},
  volume={703},
  year={2007},
  publisher={John Wiley \& Sons}
}
@book{nocedal2006numerical,
  title={Numerical optimization},
  author={Nocedal, Jorge and Wright, Stephen},
  year={2006},
  publisher={Springer Science \& Business Media}
}
@article{spokoiny2012parametric,
  title={Parametric estimation. Finite sample theory},
  author={Spokoiny, Vladimir and others},
  journal={The Annals of Statistics},
  volume={40},
  number={6},
  pages={2877--2909},
  year={2012},
  publisher={Institute of Mathematical Statistics}
}


@article{bellman1957markovian,
  title={A Markovian decision process},
  author={Bellman, Richard},
  journal={Journal of mathematics and mechanics},
  pages={679--684},
  year={1957},
  publisher={JSTOR}
}
@inproceedings{peters2010relative,
  title={Relative entropy policy search},
  author={Peters, Jan and Mulling, Katharina and Altun, Yasemin},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={24},
  number={1},
  year={2010}
}
@book{puterman2014markov,
  title={Markov decision processes: discrete stochastic dynamic programming},
  author={Puterman, Martin L},
  year={2014},
  publisher={John Wiley \& Sons}
}
@article{singh2000convergence,
  title={Convergence results for single-step on-policy reinforcement-learning algorithms},
  author={Singh, Satinder and Jaakkola, Tommi and Littman, Michael L and Szepesv{\'a}ri, Csaba},
  journal={Machine learning},
  volume={38},
  number={3},
  pages={287--308},
  year={2000},
  publisher={Springer}
}
@article{szepesvari2010algorithms,
  title={Algorithms for reinforcement learning},
  author={Szepesv{\'a}ri, Csaba},
  journal={Synthesis lectures on artificial intelligence and machine learning},
  volume={4},
  number={1},
  pages={1--103},
  year={2010},
  publisher={Morgan \& Claypool Publishers}
}

@article{watkins1992q,
  title={Q-learning},
  author={Watkins, Christopher JCH and Dayan, Peter},
  journal={Machine learning},
  volume={8},
  number={3-4},
  pages={279--292},
  year={1992},
  publisher={Springer}
}


@article{kearns1999finite,
  title={Finite-sample convergence rates for Q-learning and indirect algorithms},
  author={Kearns, Michael and Singh, Satinder},
  journal={Advances in neural information processing systems},
  pages={996--1002},
  year={1999},
  publisher={MIT; 1998}
}

@article{tsitsiklis1994asynchronous,
  title={Asynchronous stochastic approximation and Q-learning},
  author={Tsitsiklis, John N},
  journal={Machine learning},
  volume={16},
  number={3},
  pages={185--202},
  year={1994},
  publisher={Springer}
}

@inproceedings{szepesvari1997asymptotic,
  title={The asymptotic convergence-rate of Q-learning},
  author={Szepesv{\'a}ri, Csaba and others},
  booktitle={NIPS},
  volume={10},
  pages={1064--1070},
  year={1997},
  organization={Citeseer}
}

@article{jin2018q,
  title={Is Q-learning provably efficient?},
  author={Jin, Chi and Allen-Zhu, Zeyuan and Bubeck, Sebastien and Jordan, Michael I},
  journal={arXiv preprint arXiv:1807.03765},
  year={2018}
}

@article{williams1992simple,
  title={Simple statistical gradient-following algorithms for connectionist reinforcement learning},
  author={Williams, Ronald J},
  journal={Machine learning},
  volume={8},
  number={3-4},
  pages={229--256},
  year={1992},
  publisher={Springer}
}
@article{kakade2001natural,
  title={A natural policy gradient},
  author={Kakade, Sham M},
  journal={Advances in neural information processing systems},
  volume={14},
  year={2001}
}
@article{peters2008reinforcement,
  title={Reinforcement learning of motor skills with policy gradients},
  author={Peters, Jan and Schaal, Stefan},
  journal={Neural networks},
  volume={21},
  number={4},
  pages={682--697},
  year={2008},
  publisher={Elsevier}
}
@article{deisenroth2013survey,
  title={A survey on policy search for robotics},
  author={Deisenroth, Marc Peter and Neumann, Gerhard and Peters, Jan and others},
  journal={Foundations and trends in Robotics},
  volume={2},
  number={1-2},
  pages={388--403},
  year={2013},
  publisher={now publishers}
}
@techreport{baxter1999direct,
  title={Direct gradient-based reinforcement learning: I. gradient estimation algorithms},
  author={Baxter, Jonathan and Bartlett, Peter L},
  year={1999},
  institution={Citeseer}
}
@article{baxter2001infinite,
  title={Infinite-horizon policy-gradient estimation},
  author={Baxter, Jonathan and Bartlett, Peter L},
  journal={Journal of Artificial Intelligence Research},
  volume={15},
  pages={319--350},
  year={2001}
}
@article{williams1992simple,
  title={Simple statistical gradient-following algorithms for connectionist reinforcement learning},
  author={Williams, Ronald J},
  journal={Machine learning},
  volume={8},
  number={3-4},
  pages={229--256},
  year={1992},
  publisher={Springer}
}
@inproceedings{sutton1999policy,
  title={Policy gradient methods for reinforcement learning with function approximation.},
  author={Sutton, Richard S and McAllester, David A and Singh, Satinder P and Mansour, Yishay and others},
  booktitle={NIPs},
  volume={99},
  pages={1057--1063},
  year={1999},
  organization={Citeseer}
}
%IRL
@inproceedings{bain1995framework,
  title={A Framework for Behavioural Cloning.},
  author={Bain, Michael and Sammut, Claude},
  booktitle={Machine Intelligence 15},
  pages={103--129},
  year={1995}
}
@inproceedings{pirotta2016inverse,
  title={Inverse reinforcement learning through policy gradient minimization},
  author={Pirotta, Matteo and Restelli, Marcello},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={30},
  number={1},
  year={2016}
}
@article{ramponi2020inverse,
   author = {Ramponi, Giorgia and Drappo, Gianluca and Restelli, Marcello},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 pages = {2458--2468},
 publisher = {Curran Associates, Inc.},
 title = {Inverse Reinforcement Learning from a Gradient-based Learner},
 url = {https://proceedings.neurips.cc/paper/2020/file/19aa6c6fb4ba9fcf39e893ff1fd5b5bd-Paper.pdf},
 volume = {33},
 year = {2020}
}
@inproceedings{russell1998learning,
  title={Learning agents for uncertain environments},
  author={Russell, Stuart},
  booktitle={Proceedings of the eleventh annual conference on Computational learning theory},
  pages={101--103},
  year={1998}
}
@article{osa2018,
  author    = {Takayuki Osa and
               Joni Pajarinen and
               Gerhard Neumann and
               J. Andrew Bagnell and
               Pieter Abbeel and
               Jan Peters},
  title     = {An Algorithmic Perspective on Imitation Learning},
  journal   = {CoRR},
  volume    = {abs/1811.06711},
  year      = {2018},
  _url       = {http://arxiv.org/abs/1811.06711},
  archivePrefix = {arXiv},
  eprint    = {1811.06711},
  timestamp = {Sun, 25 Nov 2018 18:57:12 +0100},
  bib_url    = {https://dblp.org/rec/journals/corr/abs-1811-06711.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{jaynes1957information,
  title={Information theory and statistical mechanics},
  author={Jaynes, Edwin T},
  journal={Physical review},
  volume={106},
  number={4},
  pages={620},
  year={1957},
  publisher={APS}
}
@inproceedings{ziebart2008maximum,
  title={Maximum entropy inverse reinforcement learning.},
  author={Ziebart, Brian D and Maas, Andrew L and Bagnell, J Andrew and Dey, Anind K},
  booktitle={Aaai},
  volume={8},
  pages={1433--1438},
  year={2008},
  organization={Chicago, IL, USA}
}

@inproceedings{ziebart2010modeling,
  title={Modeling interaction via the principle of maximum causal entropy},
  author={Ziebart, Brian D and Bagnell, J Andrew and Dey, Anind K},
  booktitle={ICML},
  year={2010}
}
@inproceedings{ramachandran2007bayesian,
  title={Bayesian Inverse Reinforcement Learning.},
  author={Ramachandran, Deepak and Amir, Eyal},
  booktitle={IJCAI},
  volume={7},
  pages={2586--2591},
  year={2007}
}

@article{levine2011nonlinear,
  title={Nonlinear inverse reinforcement learning with gaussian processes},
  author={Levine, Sergey and Popovic, Zoran and Koltun, Vladlen},
  journal={Advances in neural information processing systems},
  volume={24},
  pages={19--27},
  year={2011}
}
@article{ratliff2007boosting,
  title={Boosting structured prediction for imitation learning},
  author={Ratliff, Nathan and Bradley, David and Bagnell, J Andrew and Chestnutt, Joel},
  year={2007},
  publisher={Carnegie Mellon University}
}
@inproceedings{ng2000algorithms,
  title={Algorithms for inverse reinforcement learning.},
  author={Ng, Andrew Y and Russell, Stuart J and others},
  booktitle={Icml},
  volume={1},
  pages={2},
  year={2000}
}
@inproceedings{babes2011apprenticeship,
  title={Apprenticeship learning about multiple intentions},
  author={Babes, Monica and Marivate, Vukosi N and Subramanian, Kaushik and Littman, Michael L},
  booktitle={ICML},
  year={2011}
}
@inproceedings{abbeel2004apprenticeship,
  title={Apprenticeship learning via inverse reinforcement learning},
  author={Abbeel, Pieter and Ng, Andrew Y},
  booktitle={Proceedings of the twenty-first international conference on Machine learning},
  pages={1},
  year={2004}
}
@article{likmeta2021dealing,
  title={Dealing with multiple experts and non-stationarity in inverse reinforcement learning: an application to real-life problems},
  author={Likmeta, Amarildo and Metelli, Alberto Maria and Ramponi, Giorgia and Tirinzoni, Andrea and Giuliani, Matteo and Restelli, Marcello},
  journal={Machine Learning},
  pages={1--36},
  year={2021},
  publisher={Springer}
}
@article{ramponi2021online,
  title={Online Learning in Non-Cooperative Configurable Markov Decision Process},
  author={Ramponi, Giorgia and Metelli, Alberto Maria and Concetti, Alessandro and Restelli, Marcello},
   journal={AAAI Workshop on Reinforcement Learning in Games},
  year={2021}
}
@article{ramponi2020newton,
  title={Newton Optimization on Helmholtz Decomposition for Continuous Games},
  author={Ramponi, Giorgia and Restelli, Marcello},
 journal={Thirty-Fifth AAAI conference on artificial intelligence},
  year={2021}
}
@inproceedings{ramponi2020truly,
  title={Truly Batch Model-Free Inverse Reinforcement Learning about Multiple Intentions},
  author={Ramponi, Giorgia and Likmeta, Amarildo and Metelli, Alberto Maria and Tirinzoni, Andrea and Restelli, Marcello},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2359--2369},
  year={2020},
  organization={PMLR}
}

@article{silver2010learning,
  title={Learning from demonstration for autonomous navigation in complex unstructured terrain},
  author={Silver, David and Bagnell, J Andrew and Stentz, Anthony},
  journal={The International Journal of Robotics Research},
  volume={29},
  number={12},
  pages={1565--1592},
  year={2010},
  publisher={SAGE Publications Sage UK: London, England}
}


@InProceedings{boularias2011relative, title = {Relative Entropy Inverse Reinforcement Learning}, author = {Abdeslam Boularias and Jens Kober and Jan Peters}, booktitle = {Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics}, pages = {182--189}, year = {2011}, editor = {Geoffrey Gordon and David Dunson and Miroslav Dudík}, volume = {15}, series = {Proceedings of Machine Learning Research}, address = {Fort Lauderdale, FL, USA}, month = {11--13 Apr}, publisher = {JMLR Workshop and Conference Proceedings}, pdf = {http://proceedings.mlr.press/v15/boularias11a/boularias11a.pdf}, _url = {http://proceedings.mlr.press/v15/boularias11a.html}}

@inproceedings{ho2016generative,
  title={Generative adversarial imitation learning},
  author={Ho, Jonathan and Ermon, Stefano},
  booktitle={Proceedings of the 30th International Conference on Neural Information Processing Systems},
  pages={4572--4580},
  year={2016}
}
@inproceedings{abbeel2004apprenticeship,
  title={Apprenticeship learning via inverse reinforcement learning},
  author={Abbeel, Pieter and Ng, Andrew Y},
  booktitle={Proceedings of the twenty-first international conference on Machine learning},
  pages={1},
  year={2004}
}

@inproceedings{ratliff2006maximum,
  title={Maximum margin planning},
  author={Ratliff, Nathan D and Bagnell, J Andrew and Zinkevich, Martin A},
  booktitle={Proceedings of the 23rd international conference on Machine learning},
  pages={729--736},
  year={2006}
}

@inproceedings{chen2013noisy,
  title={Noisy and missing data regression: Distribution-oblivious support recovery},
  author={Chen, Yudong and Caramanis, Constantine},
  booktitle={International Conference on Machine Learning},
  pages={383--391},
  year={2013}
}
%MARL

%survey
@article{zhang2019multi,
  title={Multi-agent reinforcement learning: A selective overview of theories and algorithms},
  author={Zhang, Kaiqing and Yang, Zhuoran and Ba{\c{s}}ar, Tamer},
  journal={arXiv preprint arXiv:1911.10635},
  year={2019}
}

@article{bucsoniu2010multi,
  title={Multi-agent reinforcement learning: An overview},
  author={Bu{\c{s}}oniu, Lucian and Babu{\v{s}}ka, Robert and De Schutter, Bart},
  journal={Innovations in multi-agent systems and applications-1},
  pages={183--221},
  year={2010},
  publisher={Springer}
}
@article{nguyen2020deep,
  title={Deep reinforcement learning for multiagent systems: A review of challenges, solutions, and applications},
  author={Nguyen, Thanh Thi and Nguyen, Ngoc Duy and Nahavandi, Saeid},
  journal={IEEE transactions on cybernetics},
  volume={50},
  number={9},
  pages={3826--3839},
  year={2020},
  publisher={IEEE}
}
@article{oroojlooyjadid2019review,
  title={A review of cooperative multi-agent deep reinforcement learning},
  author={OroojlooyJadid, Afshin and Hajinezhad, Davood},
  journal={arXiv preprint arXiv:1908.03963},
  year={2019}
}
@article{da2019survey,
  title={A survey on transfer learning for multiagent reinforcement learning systems},
  author={Da Silva, Felipe Leno and Costa, Anna Helena Reali},
  journal={Journal of Artificial Intelligence Research},
  volume={64},
  pages={645--703},
  year={2019}
}
@article{hernandez2019survey,
  title={A survey and critique of multiagent deep reinforcement learning},
  author={Hernandez-Leal, Pablo and Kartal, Bilal and Taylor, Matthew E},
  journal={Autonomous Agents and Multi-Agent Systems},
  volume={33},
  number={6},
  pages={750--797},
  year={2019},
  publisher={Springer}
}
@article{papoudakis2019dealing,
  title={Dealing with non-stationarity in multi-agent deep reinforcement learning},
  author={Papoudakis, Georgios and Christianos, Filippos and Rahman, Arrasy and Albrecht, Stefano V},
  journal={arXiv preprint arXiv:1906.04737},
  year={2019}
}

@book{myerson2013game,
  title={Game theory},
  author={Myerson, Roger B},
  year={2013},
  publisher={Harvard university press}
}
@article{nash1950equilibrium,
  title={Equilibrium points in n-person games},
  author={Nash, John F and others},
  journal={Proceedings of the national academy of sciences},
  volume={36},
  number={1},
  pages={48--49},
  year={1950},
  publisher={USA}
}
@incollection{papadimitriou1992inefficient,
  title={On inefficient proofs of existence and complexity classes},
  author={Papadimitriou, Christos H},
  booktitle={Annals of Discrete Mathematics},
  volume={51},
  pages={245--250},
  year={1992},
  publisher={Elsevier}
}
@article{daskalakis2009complexity,
  title={The complexity of computing a Nash equilibrium},
  author={Daskalakis, Constantinos and Goldberg, Paul W and Papadimitriou, Christos H},
  journal={SIAM Journal on Computing},
  volume={39},
  number={1},
  pages={195--259},
  year={2009},
  publisher={SIAM}
}
@book{von2010market,
  title={Market structure and equilibrium},
  author={Von Stackelberg, Heinrich},
  year={2010},
  publisher={Springer Science and Business Media}
}

@article{aumann1987correlated,
  title={Correlated equilibrium as an expression of Bayesian rationality},
  author={Aumann, Robert J},
  journal={Econometrica: Journal of the Econometric Society},
  pages={1--18},
  year={1987},
  publisher={JSTOR}
}
@book{von2007theory,
  title={Theory of games and economic behavior (commemorative edition)},
  author={Von Neumann, John and Morgenstern, Oskar},
  year={2007},
  publisher={Princeton university press}
}
@article{fudenberg1991perfect,
  title={Perfect Bayesian equilibrium and sequential equilibrium},
  author={Fudenberg, Drew and Tirole, Jean},
  journal={journal of Economic Theory},
  volume={53},
  number={2},
  pages={236--260},
  year={1991},
  publisher={Elsevier}
}
@article{hoffman1966nonterminating,
  title={On nonterminating stochastic games},
  author={Hoffman, Alan J and Karp, Richard M},
  journal={Management Science},
  volume={12},
  number={5},
  pages={359--370},
  year={1966},
  publisher={INFORMS}
}
@article{rao1973algorithms,
  title={Algorithms for discounted stochastic games},
  author={Rao, Singiresu S and Chandrasekaran, R and Nair, KPK},
  journal={Journal of Optimization Theory and Applications},
  volume={11},
  number={6},
  pages={627--637},
  year={1973},
  publisher={Springer}
}
@article{szepesvari1999unified,
  title={A unified analysis of value-function-based reinforcement-learning algorithms},
  author={Szepesv{\'a}ri, Csaba and Littman, Michael L},
  journal={Neural computation},
  volume={11},
  number={8},
  pages={2017--2060},
  year={1999},
  publisher={MIT Press}
}
@article{van1978discounted,
  title={Discounted Markov games: Generalized policy iteration method},
  author={Van Der Wal, J},
  journal={Journal of Optimization Theory and Applications},
  volume={25},
  number={1},
  pages={125--138},
  year={1978},
  publisher={Springer}
}
@book{vanderbei2015linear,
  title={Linear programming},
  author={Vanderbei, Robert J and others},
  volume={3},
  year={2015},
  publisher={Springer}
}
@article{shapley1953stochastic,
  title={Stochastic games},
  author={Shapley, Lloyd S},
  journal={Proceedings of the national academy of sciences},
  volume={39},
  number={10},
  pages={1095--1100},
  year={1953},
  publisher={National Acad Sciences}
}
@inproceedings{boutilier1996planning,
  title={Planning, learning and coordination in multiagent decision processes},
  author={Boutilier, Craig},
  booktitle={TARK},
  volume={96},
  pages={195--210},
  year={1996},
  organization={Citeseer}
}
@inproceedings{lauer2000algorithm,
  title={An algorithm for distributed reinforcement learning in cooperative multi-agent systems},
  author={Lauer, Martin and Riedmiller, Martin},
  booktitle={In Proceedings of the Seventeenth International Conference on Machine Learning},
  year={2000},
  organization={Citeseer}
}
@article{yoshikawa1978decomposition,
  title={Decomposition of dynamic team decision problems},
  author={Yoshikawa, T},
  journal={IEEE Transactions on Automatic Control},
  volume={23},
  number={4},
  pages={627--632},
  year={1978},
  publisher={IEEE}
}
@inproceedings{perolat2018actor,
  title={Actor-critic fictitious play in simultaneous move multistage games},
  author={Perolat, Julien and Piot, Bilal and Pietquin, Olivier},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={919--928},
  year={2018},
  organization={PMLR}
}
@inproceedings{rajeswaran2017towards,
  title={Towards generalization and simplicity in continuous control},
  author={Rajeswaran, Aravind and Lowrey, Kendall and Todorov, Emanuel and Kakade, Sham},
  booktitle={Proceedings of the 31st International Conference on Neural Information Processing Systems},
  pages={6553--6564},
  year={2017}
}
@article{zhang2018decoupling,
  title={Decoupling dynamics and reward for transfer learning},
  author={Zhang, Amy and Satija, Harsh and Pineau, Joelle},
  journal={arXiv preprint arXiv:1804.10689},
  year={2018}
}
@inproceedings{ota2020can,
  title={Can Increasing Input Dimensionality Improve Deep Reinforcement Learning?},
  author={Ota, Kei and Oiki, Tomoaki and Jha, Devesh and Mariyama, Toshisada and Nikovski, Daniel},
  booktitle={International Conference on Machine Learning},
  pages={7424--7433},
  year={2020},
  organization={PMLR}
}
@inproceedings{munk2016learning,
  title={Learning state representation for deep actor-critic control},
  author={Munk, Jelle and Kober, Jens and Babu{\v{s}}ka, Robert},
  booktitle={2016 IEEE 55th Conference on Decision and Control (CDC)},
  pages={4667--4673},
  year={2016},
  organization={IEEE}
}
@phdthesis{vroman2014maximum,
  title={Maximum likelihood inverse reinforcement learning},
  author={Vroman, Monica C},
  year={2014},
  school={Rutgers University-Graduate School-New Brunswick}
}
@article{lagoudakis2012value,
  title={Value function approximation in zero-sum markov games},
  author={Lagoudakis, Michail G and Parr, Ronald},
  booktitle={Proceedings of the Eighteenth conference on Uncertainty in artificial intelligence},
  pages={283--292},
  year={2002}
}
@article{wang2002reinforcement,
  title={Reinforcement learning to play an optimal Nash equilibrium in team Markov games},
  author={Wang, Xiaofeng and Sandholm, Tuomas},
  journal={Advances in neural information processing systems},
  volume={15},
  pages={1603--1610},
  year={2002}
}
@book{gonzalez2013discrete,
  title={Discrete--time stochastic control and dynamic potential games: the Euler--Equation approach},
  author={Gonz{\'a}lez-S{\'a}nchez, David and Hern{\'a}ndez-Lerma, On{\'e}simo},
  year={2013},
  publisher={Springer Science \& Business Media}
}
@article{la2016potential,
  title={Potential Game Theory},
  author={L{\~a}, Quang Duy and Chew, Yong Huat and Soong, Boon-Hee},
  journal={pringer International Publishing},
  year={2016},
  publisher={Springer}
}
@article{littman2001value,
  title={Value-function reinforcement learning in Markov games},
  author={Littman, Michael L},
  journal={Cognitive systems research},
  volume={2},
  number={1},
  pages={55--66},
  year={2001},
  publisher={Elsevier}
}
@incollection{littman1994markov,
  title={Markov games as a framework for multi-agent reinforcement learning},
  author={Littman, Michael L},
  booktitle={Machine learning proceedings 1994},
  pages={157--163},
  year={1994},
  publisher={Elsevier}
}
@article{jacobson1973optimal,
  title={Optimal stochastic linear systems with exponential performance criteria and their relation to deterministic differential games},
  author={Jacobson, David},
  journal={IEEE Transactions on Automatic control},
  volume={18},
  number={2},
  pages={124--131},
  year={1973},
  publisher={IEEE}
}
@book{bacsar2008h,
  title={H-infinity optimal control and related minimax design problems: a dynamic game approach},
  author={Ba{\c{s}}ar, Tamer and Bernhard, Pierre},
  year={2008},
  publisher={Springer Science \& Business Media}
}
@inproceedings{perolat2017learning,
  title={Learning Nash equilibrium for general-sum Markov games from batch data},
  author={P{\'e}rolat, Julien and Strub, Florian and Piot, Bilal and Pietquin, Olivier},
  booktitle={Artificial Intelligence and Statistics},
  pages={232--241},
  year={2017},
  organization={PMLR}
}
@article{hu2003nash,
  title={Nash Q-learning for general-sum stochastic games},
  author={Hu, Junling and Wellman, Michael P},
  journal={Journal of machine learning research},
  volume={4},
  number={Nov},
  pages={1039--1069},
  year={2003}
}
@article{kearns2013fast,
  title={Fast planning in stochastic games},
  author={Kearns, Michael and Mansour, Yishay and Singh, Satinder},
  journal={arXiv preprint arXiv:1301.3867},
  year={2013}
}
@article{zinkevich2006cyclic,
  title={Cyclic equilibria in Markov games},
  author={Zinkevich, Martin and Greenwald, Amy and Littman, Michael},
  journal={Advances in Neural Information Processing Systems},
  volume={18},
  pages={1641},
  year={2006},
  publisher={MIT; 1998}
}
@article{chen2009settling,
  title={Settling the complexity of computing two-player Nash equilibria},
  author={Chen, Xi and Deng, Xiaotie and Teng, Shang-Hua},
  journal={Journal of the ACM (JACM)},
  volume={56},
  number={3},
  pages={1--57},
  year={2009},
  publisher={ACM New York, NY, USA}
}
@inproceedings{chasnov2020convergence,
  title={Convergence analysis of gradient-based learning in continuous games},
  author={Chasnov, Benjamin and Ratliff, Lillian and Mazumdar, Eric and Burden, Samuel},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={935--944},
  year={2020},
  organization={PMLR}
}
@article{hsiehoptimization,
  title={Optimization in Open Networks via Dual Averaging},
  author={Hsieh, Yu-Guan and Iutzeler, Franck and Malick, J{\'e}r{\^o}me and Mertikopoulos, Panayotis}
}
@article{wan2021online,
  title={Online Strongly Convex Optimization with Unknown Delays},
  author={Wan, Yuanyu and Tu, Wei-Wei and Zhang, Lijun},
  journal={arXiv preprint arXiv:2103.11354},
  year={2021}
}
@inproceedings{heliou2020gradient,
  title={Gradient-free online learning in continuous games with delayed rewards},
  author={H{\'e}liou, Am{\'e}lie and Mertikopoulos, Panayotis and Zhou, Zhengyuan},
  booktitle={International Conference on Machine Learning},
  pages={4172--4181},
  year={2020},
  organization={PMLR}
}

@book{hespanha2018linear,
  title={Linear systems theory},
  author={Hespanha, Joao P},
  year={2018},
  publisher={Princeton university press}
}
@inproceedings{zhou2018learning,
  title={Learning in games with lossy feedback},
  author={Zhou, Zhengyuan and Mertikopoulos, Panayotis and Athey, Susan and Bambos, Nicholas and Glynn, Peter and Ye, Yinyu},
  booktitle={NIPS 2018-Thirty-second Conference on Neural Information Processing Systems},
  pages={1--11},
  year={2018}
}
@inproceedings{bravo2018bandit,
  title={Bandit learning in concave N-person games},
  author={Bravo, Mario and Leslie, David S and Mertikopoulos, Panayotis},
  booktitle={NIPS 2018-Thirty-second Conference on Neural Information Processing Systems},
  pages={1--24},
  year={2018}
}
@inproceedings{chasnov2020stability,
  title={Stability of Gradient Learning Dynamics in Continuous Games: Scalar Action Spaces},
  author={Chasnov, Benjamin J and Calderone, Daniel and A{\c{c}}{\i}kme{\c{s}}e, Beh{\c{c}}et and Burden, Samuel A and Ratliff, Lillian J},
  booktitle={2020 59th IEEE Conference on Decision and Control (CDC)},
  pages={3543--3548},
  year={2020},
  organization={IEEE}
}
@book{la1976stability,
  title={The stability of dynamical systems},
  author={La Salle, Joseph P},
  year={1976},
  publisher={SIAM}
}

@inproceedings{schafer2019competitive,
  title={Competitive gradient descent},
  author={Sch{\"a}fer, Florian and Anandkumar, Anima},
  booktitle={Advances in Neural Information Processing Systems},
  pages={7623--7633},
  year={2019}
}
@inproceedings{song2019convergence,
  title={Convergence of multi-agent learning with a finite step size in general-sum games},
  author={Song, Xinliang and Wang, Tonghan and Zhang, Chongjie},
  booktitle={Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={935--943},
  year={2019},
  organization={International Foundation for Autonomous Agents and Multiagent Systems}
}
@inproceedings{mescheder2017numerics,
  title={The numerics of gans},
  author={Mescheder, Lars and Nowozin, Sebastian and Geiger, Andreas},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1825--1835},
  year={2017}
}
@article{conitzer2007awesome,
  title={AWESOME: A general multiagent learning algorithm that converges in self-play and learns a best response against stationary opponents},
  author={Conitzer, Vincent and Sandholm, Tuomas},
  journal={Machine Learning},
  volume={67},
  number={1-2},
  pages={23--43},
  year={2007},
  publisher={Springer}
}
@article{bowling2002multiagent,
  title={Multiagent learning using a variable learning rate},
  author={Bowling, Michael and Veloso, Manuela},
  journal={Artificial Intelligence},
  volume={136},
  number={2},
  pages={215--250},
  year={2002},
  publisher={Elsevier}
}
@InProceedings{fiez2019convergence, title = {Implicit Learning Dynamics in Stackelberg Games: Equilibria Characterization, Convergence Analysis, and Empirical Study}, author = {Fiez, Tanner and Chasnov, Benjamin and Ratliff, Lillian}, booktitle = {Proceedings of the 37th International Conference on Machine Learning}, pages = {3133--3144}, year = {2020}, editor = {Hal Daumé III and Aarti Singh}, volume = {119}, series = {Proceedings of Machine Learning Research}, month = {13--18 Jul}, publisher = {PMLR}, pdf = {http://proceedings.mlr.press/v119/fiez20a/fiez20a.pdf} }
@inproceedings{zhang2010multi,
  title={Multi-agent learning with policy prediction},
  author={Zhang, Chongjie and Lesser, Victor},
  booktitle={Twenty-fourth AAAI conference on artificial intelligence},
  year={2010}
}
@inproceedings{foerster2018learning,
  title={Learning with opponent-learning awareness},
  author={Foerster, Jakob and Chen, Richard Y and Al-Shedivat, Maruan and Whiteson, Shimon and Abbeel, Pieter and Mordatch, Igor},
  booktitle={Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems},
  pages={122--130},
  year={2018},
  organization={International Foundation for Autonomous Agents and Multiagent Systems}
}
@article{mazumdar2019finding,
  title={On finding local nash equilibria (and only local nash equilibria) in zero-sum games},
  author={Mazumdar, Eric V and Jordan, Michael I and Sastry, S Shankar},
  journal={arXiv preprint arXiv:1901.00838},
  year={2019}
}
@article{letcher2019differentiable,
  title={Differentiable Game Mechanics.},
  author={Letcher, Alistair and Balduzzi, David and Racaniere, S{\'e}bastien and Martens, James and Foerster, Jakob N and Tuyls, Karl and Graepel, Thore},
  journal={Journal of Machine Learning Research},
  volume={20},
  number={84},
  pages={1--40},
  year={2019}
}
@inproceedings{balduzzi2018mechanics,
  title={The mechanics of n-player differentiable games},
  author={Balduzzi, David and Racaniere, Sebastien and Martens, James and Foerster, Jakob and Tuyls, Karl and Graepel, Thore},
  booktitle={International Conference on Machine Learning},
  pages={354--363},
  year={2018},
  organization={PMLR}
}
@article{lee2017first,
  title={First-order methods almost always avoid saddle points},
  author={Lee, Jason D and Panageas, Ioannis and Piliouras, Georgios and Simchowitz, Max and Jordan, Michael I and Recht, Benjamin},
  journal={arXiv preprint arXiv:1710.07406},
  year={2017}
}

@inproceedings{adolphs2019local,
  title={Local saddle point optimization: A curvature exploitation approach},
  author={Adolphs, Leonard and Daneshmand, Hadi and Lucchi, Aurelien and Hofmann, Thomas},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={486--495},
  year={2019},
  organization={PMLR}
}
@article{mertikopoulos2018optimistic,
  title={Optimistic mirror descent in saddle-point problems: Going the extra (gradient) mile},
  author={Mertikopoulos, Panayotis and Lecouat, Bruno and Zenati, Houssam and Foo, Chuan-Sheng and Chandrasekhar, Vijay and Piliouras, Georgios},
  journal={arXiv preprint arXiv:1807.02629},
  year={2018}
}
@inproceedings{mertikopoulos2018cycles,
  title={Cycles in adversarial regularized learning},
  author={Mertikopoulos, Panayotis and Papadimitriou, Christos and Piliouras, Georgios},
  booktitle={Proceedings of the Twenty-Ninth Annual ACM-SIAM Symposium on Discrete Algorithms},
  pages={2703--2717},
  year={2018},
  organization={SIAM}
}

%%MIRL papers
@article{manton2003geometry,
  title={The geometry of weighted low-rank approximations},
  author={Manton, Jonathan H and Mahony, Robert and Hua, Yingbo},
  journal={IEEE Transactions on Signal Processing},
  volume={51},
  number={2},
  pages={500--514},
  year={2003},
  publisher={IEEE}
}

@book{casella2002statistical,
  title={Statistical inference},
  author={Casella, George and Berger, Roger L},
  volume={2},
  year={2002},
  publisher={Duxbury Pacific Grove, CA}
}
@book{gupta2018matrix,
  title={Matrix variate distributions},
  author={Gupta, Arjun K and Nagar, Daya K},
  year={2018},
  publisher={Chapman and Hall/CRC}
}
@article{ledoit2004well,
  title={A well-conditioned estimator for large-dimensional covariance matrices},
  author={Ledoit, Olivier and Wolf, Michael},
  journal={Journal of multivariate analysis},
  volume={88},
  number={2},
  pages={365--411},
  year={2004},
  publisher={Elsevier}
}
@article{dempster1977maximum,
  title={Maximum likelihood from incomplete data via the EM algorithm},
  author={Dempster, Arthur P and Laird, Nan M and Rubin, Donald B},
  journal={Journal of the Royal Statistical Society: Series B (Methodological)},
  volume={39},
  number={1},
  pages={1--22},
  year={1977},
  publisher={Wiley Online Library}
}
@article{bilmes1998gentle,
  title={A gentle tutorial of the EM algorithm and its application to parameter estimation for Gaussian mixture and hidden Markov models},
  author={Bilmes, Jeff A and others},
  journal={International Computer Science Institute},
  volume={4},
  number={510},
  pages={126},
  year={1998}
}
@article{choi2012nonparametric,
  title={Nonparametric Bayesian inverse reinforcement learning for multiple reward functions},
  author={Choi, Jaedeug and Kim, Kee-Eung},
  journal={Advances in Neural Information Processing Systems},
  volume={25},
  pages={305--313},
  year={2012},
  publisher={Citeseer}
}
@inproceedings{almingol2015learning,
  title={Learning multiple behaviours using hierarchical clustering of rewards},
  author={Almingol, Javier and Montesano, Luis},
  booktitle={2015 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},
  pages={4608--4613},
  year={2015},
  organization={IEEE}
}
@article{rajasekaran2017inverse,
  title={Inverse reinforce learning with nonparametric behavior clustering},
  author={Rajasekaran, Siddharthan and Zhang, Jinwei and Fu, Jie},
  journal={arXiv preprint arXiv:1712.05514},
  year={2017}
}
@book{dorato2000linear,
  title={Linear quadratic control: an introduction},
  author={Dorato, Peter and Cerone, Vito and Abdallah, Chaouki},
  year={2000},
  publisher={Krieger Publishing Co., Inc.}
}
@inproceedings{klein2013cascaded,
  title={A cascaded supervised learning approach to inverse reinforcement learning},
  author={Klein, Edouard and Piot, Bilal and Geist, Matthieu and Pietquin, Olivier},
  booktitle={Joint European conference on machine learning and knowledge discovery in databases},
  pages={1--16},
  year={2013},
  organization={Springer}
}

@article{argall2009survey,
  title={A survey of robot learning from demonstration},
  author={Argall, Brenna D and Chernova, Sonia and Veloso, Manuela and Browning, Brett},
  journal={Robotics and autonomous systems},
  volume={57},
  number={5},
  pages={469--483},
  year={2009},
  publisher={Elsevier}
}
@article{hussein2017imitation,
  title={Imitation learning: A survey of learning methods},
  author={Hussein, Ahmed and Gaber, Mohamed Medhat and Elyan, Eyad and Jayne, Chrisina},
  journal={ACM Computing Surveys (CSUR)},
  volume={50},
  number={2},
  pages={21},
  year={2017},
  publisher={ACM}
}
@article{pirotta2016reinforcement,
  title={Reinforcement learning: from theory to algorithms},
  author={Pirotta, Matteo},
  year={2016},
  publisher={Italy}
}
@book{sutton2018reinforcement,
  title={Reinforcement learning: An introduction},
  author={Sutton, Richard S and Barto, Andrew G},
  year={2018},
  publisher={MIT press}
}
@inproceedings{sutton2000policy,
  title={Policy gradient methods for reinforcement learning with function approximation},
  author={Sutton, Richard S and McAllester, David A and Singh, Satinder P and Mansour, Yishay},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1057--1063},
  year={2000}
}
@article{jain2019model,
  title={Model-Free IRL using Maximum Likelihood Estimation},
  author={Jain, Vinamra and Doshi, Prashant and Banerjee, Bikramjit},
  year={2019}
}
@inproceedings{klein2012inverse,
  title={Inverse reinforcement learning through structured classification},
  author={Klein, Edouard and Geist, Matthieu and Piot, Bilal and Pietquin, Olivier},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1007--1015},
  year={2012}
}
@inproceedings{sehnke2008policy,
  title={Policy gradients with parameter-based exploration for control},
  author={Sehnke, Frank and Osendorfer, Christian and R{\"u}ckstie{\ss}, Thomas and Graves, Alex and Peters, Jan and Schmidhuber, J{\"u}rgen},
  booktitle={International Conference on Artificial Neural Networks},
  pages={387--396},
  year={2008},
  organization={Springer}
}

%% Learning from a Learner
@inproceedings{lflpaper,
  title={Learning from a learner},
  author={Jacq, Alexis and Geist, Matthieu and Paiva, Ana and Pietquin, Olivier},
  booktitle={International Conference on Machine Learning},
  pages={2990--2999},
  year={2019},
  organization={PMLR}
}
@article{shteingart2014reinforcement,
  title={Reinforcement learning and human behavior},
  author={Shteingart, Hanan and Loewenstein, Yonatan},
  journal={Current Opinion in Neurobiology},
  volume={25},
  pages={93--98},
  year={2014},
  publisher={Elsevier}
}
@inproceedings{brown2019extrapolating,
  title={Extrapolating Beyond Suboptimal Demonstrations via Inverse Reinforcement Learning from Observations},
  author={Brown, Daniel and Goo, Wonjoon and Nagarajan, Prabhat and Niekum, Scott},
  booktitle={International Conference on Machine Learning},
  pages={783--792},
  year={2019}
}
@inproceedings{pomerleau1989alvinn,
  title={Alvinn: An autonomous land vehicle in a neural network},
  author={Pomerleau, Dean A},
  booktitle={Advances in neural information processing systems},
  pages={305--313},
  year={1989}
}
@article{goyal2019first,
  title={A First-Order Approach To Accelerated Value Iteration},
  author={Goyal, Vineet and Grand-Clement, Julien},
  journal={arXiv preprint arXiv:1905.09963},
  year={2019}
}
@article{schulman2017equivalence,
  title={Equivalence between policy gradients and soft q-learning},
  author={Schulman, John and Chen, Xi and Abbeel, Pieter},
  journal={arXiv preprint arXiv:1704.06440},
  year={2017}
}
@article{tseng2001convergence,
  title={Convergence of a block coordinate descent method for nondifferentiable minimization},
  author={Tseng, Paul},
  journal={Journal of optimization theory and applications},
  volume={109},
  number={3},
  pages={475--494},
  year={2001},
  publisher={Springer}
}
@inproceedings{pirotta2013adaptive,
  title={Adaptive step-size for policy gradient methods},
  author={Pirotta, Matteo and Restelli, Marcello and Bascetta, Luca},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1394--1402},
  year={2013}
}
@inproceedings{mcwilliams2014fast,
  title={Fast and robust least squares estimation in corrupted linear models},
  author={McWilliams, Brian and Krummenacher, Gabriel and Lucic, Mario and Buhmann, Joachim M},
  booktitle={Advances in Neural Information Processing Systems},
  pages={415--423},
  year={2014}
}

@article{wedin1973perturbation,
  title={Perturbation theory for pseudo-inverses},
  author={Wedin, Perke},
  journal={BIT Numerical Mathematics},
  volume={13},
  number={2},
  pages={217--232},
  year={1973},
  publisher={Springer}
}

@article{fixeddesign,
author    = {Philippe Rigollet},
title     = { High-Dimensional Statistics. Spring 2015},
journal = {Massachusetts Institute of Technology: MIT OpenCourseWare},
year = {2015}
}
@inproceedings{haarnoja2018soft,
  title={Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor},
  author={Haarnoja, Tuomas and Zhou, Aurick and Abbeel, Pieter and Levine, Sergey},
  booktitle={International Conference on Machine Learning},
  pages={1861--1870},
  year={2018},
  organization={PMLR}
}
@article{openai,
  author    = {Greg Brockman and
               Vicki Cheung and
               Ludwig Pettersson and
               Jonas Schneider and
               John Schulman and
               Jie Tang and
               Wojciech Zaremba},
  title     = {OpenAI Gym},
  journal   = {CoRR},
  volume    = {abs/1606.01540},
  year      = {2016}
}

@inproceedings{ibarz2018reward,
  title={Reward learning from human preferences and demonstrations in Atari},
  author={Ibarz, Borja and Leike, Jan and Pohlen, Tobias and Irving, Geoffrey and Legg, Shane and Amodei, Dario},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8011--8023},
  year={2018}
}
@inproceedings{christiano2017deep,
  title={Deep reinforcement learning from human preferences},
  author={Christiano, Paul F and Leike, Jan and Brown, Tom and Martic, Miljan and Legg, Shane and Amodei, Dario},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4299--4307},
  year={2017}
}
@article{castro2019inverse,
  title={Inverse Reinforcement Learning with Multiple Ranked Experts},
  author={Castro, Pablo Samuel and Li, Shijian and Zhang, Daqing},
  journal={arXiv preprint arXiv:1907.13411},
  year={2019}
}
@inproceedings{rabinowitz2018machine,
  title={Machine Theory of Mind},
  author={Rabinowitz, Neil and Perbet, Frank and Song, Francis and Zhang, Chiyuan and Eslami, SM Ali and Botvinick, Matthew},
  booktitle={International Conference on Machine Learning},
  pages={4218--4227},
  year={2018}
}
@inproceedings{shum2019theory,
  title={Theory of minds: Understanding behavior in groups through inverse planning},
  author={Shum, Michael and Kleiman-Weiner, Max and Littman, Michael L and Tenenbaum, Joshua B},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={33},
  pages={6163--6170},
  year={2019}
}
@inproceedings{natarajan2010multi,
  title={Multi-agent inverse reinforcement learning},
  author={Natarajan, Sriraam and Kunapuli, Gautam and Judah, Kshitij and Tadepalli, Prasad and Kersting, Kristian and Shavlik, Jude},
  booktitle={2010 Ninth International Conference on Machine Learning and Applications},
  pages={395--400},
  year={2010},
  organization={IEEE}
}
@inproceedings{hadfield2016cooperative,
  title={Cooperative inverse reinforcement learning},
  author={Hadfield-Menell, Dylan and Dragan, Anca and Abbeel, Pieter and Russell, Stuart},
  booktitle={Proceedings of the 30th International Conference on Neural Information Processing Systems},
  pages={3916--3924},
  year={2016}
}
@article{lin2019multi,
  title={Multi-agent inverse reinforcement learning for certain general-sum stochastic games},
  author={Lin, Xiaomin and Adams, Stephen C and Beling, Peter A},
  journal={Journal of Artificial Intelligence Research},
  volume={66},
  pages={473--502},
  year={2019}
}
@inproceedings{haarnoja2018soft,
  title={Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor},
  author={Haarnoja, Tuomas and Zhou, Aurick and Abbeel, Pieter and Levine, Sergey},
  booktitle={International Conference on Machine Learning},
  pages={1861--1870},
  year={2018},
  organization={PMLR}
}

%%NOHD
@article{paternain2019newton,
  title={A Newton-based method for nonconvex optimization with fast evasion of saddle points},
  author={Paternain, Santiago and Mokhtari, Aryan and Ribeiro, Alejandro},
  journal={SIAM Journal on Optimization},
  volume={29},
  number={1},
  pages={343--368},
  year={2019},
  publisher={SIAM}
}
@article{goodfellow2014generative,
  title={Generative Adversarial Nets},
  author={Goodfellow, Ian and Pouget-Abadie, Jean and Mirza, Mehdi and Xu, Bing and Warde-Farley, David and Ozair, Sherjil and Courville, Aaron and Bengio, Yoshua},
  journal={Advances in Neural Information Processing Systems},
  volume={27},
  pages={2672--2680},
  year={2014}
}
@inproceedings{letcher2018stable,
  title={Stable Opponent Shaping in Differentiable Games},
  author={Letcher, Alistair and Foerster, Jakob and Balduzzi, David and Rockt{\"a}schel, Tim and Whiteson, Shimon},
  booktitle={International Conference on Learning Representations},
  year={2018}
}

@inproceedings{schafer2019competitive,
  title={Competitive gradient descent},
  author={Sch{\"a}fer, Florian and Anandkumar, Anima},
  booktitle={Advances in Neural Information Processing Systems},
  pages={7623--7633},
  year={2019}
}

%% continuous games
@inproceedings{skoulakis2020evolutionary,
  title={Evolutionary Game Theory Squared: Evolving Agents in Endogenously Evolving Zero-Sum Games},
  author={Skoulakis, Stratis and Fiez, Tanner and Sim, Ryann and Piliouras, Georgios and Ratliff, Lillian},
  journal={To appear at proceedings of the AAAI Conference on Artificial Intelligence},
  year={2021}
}
@incollection{ko1995complexity,
  title={On the complexity of min-max optimization problems and their approximation},
  author={Ko, Ker-I and Lin, Chih-Long},
  booktitle={Minimax and Applications},
  pages={219--239},
  year={1995},
  publisher={Springer}
}
@article{daskalakis2020complexity,
  title={The complexity of constrained min-max optimization},
  author={Daskalakis, Constantinos and Skoulakis, Stratis and Zampetakis, Manolis},
  journal={arXiv preprint arXiv:2009.09623},
  year={2020}
}
@article{sorin2020replicator,
  title={Replicator dynamics: Old and new},
  author={Sorin, Sylvain},
  journal={Journal of Dynamics \& Games},
  volume={7},
  number={4},
  pages={365},
  year={2020},
  publisher={American Institute of Mathematical Sciences}
}
@article{borgers1997learning,
  title={Learning through reinforcement and replicator dynamics},
  author={B{\"o}rgers, Tilman and Sarin, Rajiv},
  journal={Journal of economic theory},
  volume={77},
  number={1},
  pages={1--14},
  year={1997},
  publisher={Elsevier}
}
@book{hofbauer1998evolutionary,
  title={Evolutionary games and population dynamics},
  author={Hofbauer, Josef and Sigmund, Karl and others},
  year={1998},
  publisher={Cambridge university press}
}
@inproceedings{tessler2019action,
  title={Action robust reinforcement learning and applications in continuous control},
  author={Tessler, Chen and Efroni, Yonathan and Mannor, Shie},
  booktitle={International Conference on Machine Learning},
  pages={6215--6224},
  year={2019},
  organization={PMLR}
}
@article{mannor2016robust,
  title={Robust MDPs with k-rectangular uncertainty},
  author={Mannor, Shie and Mebel, Ofir and Xu, Huan},
  journal={Mathematics of Operations Research},
  volume={41},
  number={4},
  pages={1484--1509},
  year={2016},
  publisher={INFORMS}
}
@article{lim2013reinforcement,
  title={Reinforcement learning in robust markov decision processes},
  author={Lim, Shiau Hong and Xu, Huan and Mannor, Shie},
  journal={Advances in Neural Information Processing Systems},
  volume={26},
  pages={701--709},
  year={2013}
}
@article{nilim2005robust,
  title={Robust control of Markov decision processes with uncertain transition matrices},
  author={Nilim, Arnab and El Ghaoui, Laurent},
  journal={Operations Research},
  volume={53},
  number={5},
  pages={780--798},
  year={2005},
  publisher={INFORMS}
}
@inproceedings{mannor2012lightning,
  title={Lightning does not strike twice: robust MDPs with coupled uncertainty},
  author={Mannor, Shie and Mebel, Ofir and Xu, Huan},
  booktitle={Proceedings of the 29th International Coference on International Conference on Machine Learning},
  pages={451--458},
  year={2012}
}
@inproceedings{cardoso2019competing,
  title={Competing against Nash equilibria in adversarially changing zero-sum games},
  author={Cardoso, Adrian Rivera and Abernethy, Jacob and Wang, He and Xu, Huan},
  booktitle={International Conference on Machine Learning},
  pages={921--930},
  year={2019},
  organization={PMLR}
}
@article{duvocelle2018learning,
  title={Learning in time-varying games},
  author={Duvocelle, Benoit and Mertikopoulos, Panayotis and Staudigl, Mathias and Vermeulen, Dries},
  journal={arXiv preprint arXiv:1809.03066},
  year={2018}
}
@inproceedings{ratliff2013characterization,
  title={Characterization and computation of local Nash equilibria in continuous games},
  author={Ratliff, Lillian J and Burden, Samuel A and Sastry, S Shankar},
  booktitle={2013 51st Annual Allerton Conference on Communication, Control, and Computing (Allerton)},
  pages={917--924},
  year={2013},
  organization={IEEE}
}
@inproceedings{fiez2020implicit,
  title={Implicit learning dynamics in stackelberg games: Equilibria characterization, convergence analysis, and empirical study},
  author={Fiez, Tanner and Chasnov, Benjamin and Ratliff, Lillian},
  booktitle={International Conference on Machine Learning},
  pages={3133--3144},
  year={2020},
  organization={PMLR}
}
@article{fiez2020gradient,
  title={Gradient descent-ascent provably converges to strict local minmax equilibria with a finite timescale separation},
  author={Fiez, Tanner and Ratliff, Lillian},
  journal={arXiv preprint arXiv:2009.14820},
  year={2020}
}
@inproceedings{chasnov2020convergence,
  title={Convergence analysis of gradient-based learning in continuous games},
  author={Chasnov, Benjamin and Ratliff, Lillian and Mazumdar, Eric and Burden, Samuel},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={935--944},
  year={2020},
  organization={PMLR}
}
@article{zheng2021stackelberg,
  title={Stackelberg Actor-Critic: A Game-Theoretic Perspective},
  author={Zheng, Liyuan and Fiez, Tanner and Alumbaugh, Zane and Chasnov, Benjamin and Ratliff, Lillian J},
  year={2021}
}
@article{mazumdar2020gradient,
  title={On gradient-based learning in continuous games},
  author={Mazumdar, Eric and Ratliff, Lillian J and Sastry, S Shankar},
  journal={SIAM Journal on Mathematics of Data Science},
  volume={2},
  number={1},
  pages={103--131},
  year={2020},
  publisher={SIAM}
}


@article{chasnov2019convergence,
  title={Convergence analysis of gradient-based learning with non-uniform learning rates in non-cooperative multi-agent settings},
  author={Chasnov, Benjamin and Ratliff, Lillian J and Mazumdar, Eric and Burden, Samuel A},
  journal={arXiv preprint arXiv:1906.00731},
  year={2019}
}
@inproceedings{isola2017image,
  title={Image-to-image translation with conditional adversarial networks},
  author={Isola, Phillip and Zhu, Jun-Yan and Zhou, Tinghui and Efros, Alexei A},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={1125--1134},
  year={2017}
}
@inproceedings{ledig2017photo,
  title={Photo-realistic single image super-resolution using a generative adversarial network},
  author={Ledig, Christian and Theis, Lucas and Husz{\'a}r, Ferenc and Caballero, Jose and Cunningham, Andrew and Acosta, Alejandro and Aitken, Andrew and Tejani, Alykhan and Totz, Johannes and Wang, Zehan and others},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={4681--4690},
  year={2017}
}
@article{nie2018relgan,
  title={Relgan: Relational generative adversarial networks for text generation},
  author={Nie, Weili and Narodytska, Nina and Patel, Ankit},
  year={2018}
}
@inproceedings{yu2017seqgan,
  title={Seqgan: Sequence generative adversarial nets with policy gradient},
  author={Yu, Lantao and Zhang, Weinan and Wang, Jun and Yu, Yong},
  booktitle={Thirty-First AAAI Conference on Artificial Intelligence},
  year={2017}
}
@inproceedings{papadimitriou2016nash,
  title={From Nash equilibria to chain recurrent sets: Solution concepts and topology},
  author={Papadimitriou, Christos and Piliouras, Georgios},
  booktitle={Proceedings of the 2016 ACM Conference on Innovations in Theoretical Computer Science},
  pages={227--235},
  year={2016}
}
@inproceedings{daskalakis2017training,
  title={Training GANs with Optimism},
  author={Daskalakis, Constantinos and Ilyas, Andrew and Syrgkanis, Vasilis and Zeng, Haoyang},
  booktitle={International Conference on Learning Representations},
  year={2018}
}
@article{perolat2020poincar,
  title={From Poincar\'e Recurrence to Convergence in Imperfect Information Games: Finding Equilibrium via Regularization},
  author={Perolat, Julien and Munos, Remi and Lespiau, Jean-Baptiste and Omidshafiei, Shayegan and Rowland, Mark and Ortega, Pedro and Burch, Neil and Anthony, Thomas and Balduzzi, David and De Vylder, Bart and others},
  journal={arXiv preprint arXiv:2002.08456},
  year={2020}
}
@inproceedings{heusel2017gans,
  title={Gans trained by a two time-scale update rule converge to a local nash equilibrium},
  author={Heusel, Martin and Ramsauer, Hubert and Unterthiner, Thomas and Nessler, Bernhard and Hochreiter, Sepp},
  booktitle={Advances in neural information processing systems},
  pages={6626--6637},
  year={2017}
}
@article{ratliff2016characterization,
  title={On the characterization of local Nash equilibria in continuous games},
  author={Ratliff, Lillian J and Burden, Samuel A and Sastry, S Shankar},
  journal={IEEE Transactions on Automatic Control},
  volume={61},
  number={8},
  pages={2301--2307},
  year={2016},
  publisher={IEEE}
}
@book{borkar2009stochastic,
  title={Stochastic approximation: a dynamical systems viewpoint},
  author={Borkar, Vivek S},
  volume={48},
  year={2009},
  publisher={Springer}
}
@article{bu2019global,
  title={Global convergence of policy gradient for sequential zero-sum linear quadratic dynamic games},
  author={Bu, Jingjing and Ratliff, Lillian J and Mesbahi, Mehran},
  journal={arXiv preprint arXiv:1911.04672},
  year={2019}
}

@article{yang2020global,
  title={Global convergence and variance-reduced optimization for a class of nonconvex-nonconcave minimax problems},
  author={Yang, Junchi and Kiyavash, Negar and He, Niao},
  journal={arXiv preprint arXiv:2002.09621},
  year={2020}
}
@article{letcher2020impossibility,
  title={On the impossibility of global convergence in multi-loss optimization},
  author={Letcher, Alistair},
  journal={arXiv preprint arXiv:2005.12649},
  year={2020}
}
@inproceedings{diakonikolas2020efficient,
  title={Efficient methods for structured nonconvex-nonconcave min-max optimization},
  author={Diakonikolas, Jelena and Daskalakis, Constantinos and Jordan, Michael},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2746--2754},
  year={2021},
  organization={PMLR}
}
@article{monderer1996potential,
  title={Potential games},
  author={Monderer, Dov and Shapley, Lloyd S},
  journal={Games and economic behavior},
  volume={14},
  number={1},
  pages={124--143},
  year={1996},
  publisher={Elsevier}
}
@article{rosenthal1973class,
  title={A class of games possessing pure-strategy Nash equilibria},
  author={Rosenthal, Robert W},
  journal={International Journal of Game Theory},
  volume={2},
  number={1},
  pages={65--67},
  year={1973}
}
@article{lee2016gradient,
  title={Gradient Descent Converges to Minimizers},
  author={Lee, Jason D and Simchowitz, Max and Jordan, Michael I and Recht, Benjamin},
  journal={University of California, Berkeley},
  volume={1050},
  pages={4},
  year={2016}
}
@article{rosen1965existence,
  title={Existence and uniqueness of equilibrium points for concave n-person games},
  author={Rosen, J Ben},
  journal={Econometrica: Journal of the Econometric Society},
  pages={520--534},
  year={1965},
  publisher={JSTOR}
}
@article{facchinei2007generalized,
  title={Generalized Nash equilibrium problems},
  author={Facchinei, Francisco and Kanzow, Christian},
  journal={4or},
  volume={5},
  number={3},
  pages={173--210},
  year={2007},
  publisher={Springer}
}
@inproceedings{singh2000nash,
  title={Nash Convergence of Gradient Dynamics in General-Sum Games.},
  author={Singh, Satinder P and Kearns, Michael J and Mansour, Yishay},
  booktitle={UAI},
  pages={541--548},
  year={2000}
}
@article{balduzzi2020smooth,
  title={Smooth markets: A basic mechanism for organizing gradient-based learners},
  author={Balduzzi, David and Czarnecki, Wojciech M and Anthony, Thomas W and Gemp, Ian M and Hughes, Edward and Leibo, Joel Z and Piliouras, Georgios and Graepel, Thore},
  journal={arXiv preprint arXiv:2001.04678},
  year={2020}
}

@article{bu2019global,
  title={Global convergence of policy gradient for sequential zero-sum linear quadratic dynamic games},
  author={Bu, Jingjing and Ratliff, Lillian J and Mesbahi, Mehran},
  journal={arXiv preprint arXiv:1911.04672},
  year={2019}
}
@article{zhang2021derivative,
  title={Derivative-Free Policy Optimization for Risk-Sensitive and Robust Control Design: Implicit Regularization and Sample Complexity},
  author={Zhang, Kaiqing and Zhang, Xiangyuan and Hu, Bin and Ba{\c{s}}ar, Tamer},
  journal={arXiv preprint arXiv:2101.01041}
}
@article{gravell2020policy,
  title={Policy iteration for linear quadratic games with stochastic parameters},
  author={Gravell, Benjamin and Ganapathy, Karthik and Summers, Tyler},
  journal={IEEE Control Systems Letters},
  volume={5},
  number={1},
  pages={307--312},
  year={2020},
  publisher={IEEE}
}
@inproceedings{zhang2019policy,
  title={Policy optimization provably converges to Nash equilibria in zero-sum linear quadratic games},
  author={Zhang, Kaiqing and Yang, Zhuoran and Basar, Tamer},
  booktitle={Advances in Neural Information Processing Systems},
  pages={11602--11614},
  year={2019}
}
@article{gemp2018global,
  title={Global convergence to the equilibrium of gans using variational inequalities},
  author={Gemp, Ian and Mahadevan, Sridhar},
  journal={arXiv preprint arXiv:1808.01531},
  year={2018}
}
@inproceedings{nagarajan2017gradient,
  title={Gradient descent GAN optimization is locally stable},
  author={Nagarajan, Vaishnavh and Kolter, J Zico},
  booktitle={Advances in neural information processing systems},
  pages={5585--5595},
  year={2017}
}
@inproceedings{adolphs2019local,
  title={Local saddle point optimization: A curvature exploitation approach},
  author={Adolphs, Leonard and Daneshmand, Hadi and Lucchi, Aurelien and Hofmann, Thomas},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={486--495},
  year={2019},
  organization={PMLR}
}
@inproceedings{shen2019hessian,
  title={Hessian aided policy gradient},
  author={Shen, Zebang and Ribeiro, Alejandro and Hassani, Hamed and Qian, Hui and Mi, Chao},
  booktitle={International Conference on Machine Learning},
  pages={5729--5738},
  year={2019},
  organization={PMLR}
}
@techreport{wills1958vector,
  title={Vector analysis with and introduction to tensor analysis},
  author={Wills, Albert Potter},
  year={1958}
}
@article{nedic2009distributed,
  title={Distributed subgradient methods for multi-agent optimization},
  author={Nedic, Angelia and Ozdaglar, Asuman},
  journal={IEEE Transactions on Automatic Control},
  volume={54},
  number={1},
  pages={48--61},
  year={2009},
  publisher={IEEE}
}
@article{li1987distributed,
  title={Distributed algorithms for the computation of noncooperative equilibria},
  author={Li, Shu and Ba{\c{s}}ar, Tamer},
  journal={Automatica},
  volume={23},
  number={4},
  pages={523--533},
  year={1987},
  publisher={Elsevier}
}
@book{facchinei2007finite,
  title={Finite-dimensional variational inequalities and complementarity problems},
  author={Facchinei, Francisco and Pang, Jong-Shi},
  year={2007},
  publisher={Springer Science \& Business Media}
}
@article{chasnovgradient,
  title={Gradient Conjectures for Strategic Multi-Agent Learning},
  year={2019},
  author={Chasnov, Benjamin and Fiez, Tanner and Ratliff, Lillian J}
}
%%To be cited
@article{candogan2011flows,
  title={Flows and decompositions of games: Harmonic and potential games},
  author={Candogan, Ozan and Menache, Ishai and Ozdaglar, Asuman and Parrilo, Pablo A},
  journal={Mathematics of Operations Research},
  volume={36},
  number={3},
  pages={474--503},
  year={2011},
  publisher={INFORMS}
}
@book{chong2004introduction,
  title={An introduction to optimization},
  author={Chong, Edwin KP and Zak, Stanislaw H},
  year={2004},
  publisher={John Wiley \& Sons}
}
@article{kober2013reinforcement,
  title={Reinforcement learning in robotics: A survey},
  author={Kober, Jens and Bagnell, J Andrew and Peters, Jan},
  journal={The International Journal of Robotics Research},
  volume={32},
  number={11},
  pages={1238--1274},
  year={2013},
  publisher={SAGE Publications Sage UK: London, England}
}
@article{pham2018cooperative,
  title={Cooperative and distributed reinforcement learning of drones for field coverage},
  author={Pham, Huy Xuan and La, Hung Manh and Feil-Seifer, David and Nefian, Aria},
  journal={arXiv preprint arXiv:1803.07250},
  year={2018}
}
@article{cortes2004coverage,
  title={Coverage control for mobile sensing networks},
  author={Cortes, Jorge and Martinez, Sonia and Karatas, Timur and Bullo, Francesco},
  journal={IEEE Transactions on robotics and Automation},
  volume={20},
  number={2},
  pages={243--255},
  year={2004},
  publisher={IEEE}
}
@article{dall2013distributed,
  title={Distributed optimal power flow for smart microgrids},
  author={Dall'Anese, Emiliano and Zhu, Hao and Giannakis, Georgios B},
  journal={IEEE Transactions on Smart Grid},
  volume={4},
  number={3},
  pages={1464--1475},
  year={2013},
  publisher={IEEE}
}
@inproceedings{corke2005networked,
  title={Networked robots: Flying robot navigation using a sensor net},
  author={Corke, Peter and Peterson, Ron and Rus, Daniela},
  booktitle={Robotics research. The eleventh international symposium},
  pages={234--243},
  year={2005},
  organization={Springer}
}
@article{nash1951non,
  title={Non-cooperative games},
  author={Nash, John},
  journal={Annals of mathematics},
  pages={286--295},
  year={1951},
  publisher={JSTOR}
}
@inproceedings{raghunathan2019game,
  title={Game theoretic optimization via gradient-based nikaido-isoda function},
  author={Raghunathan, Arvind and Cherian, Anoop and Jha, Devesh},
  booktitle={International Conference on Machine Learning},
  pages={5291--5300},
  year={2019},
  organization={PMLR}
}

@article{ravi2016optimization,
  title={Optimization as a model for few-shot learning},
  author={Ravi, Sachin and Larochelle, Hugo},
  year={2016}
}
@inproceedings{maclaurin2015gradient,
  title={Gradient-based hyperparameter optimization through reversible learning},
  author={Maclaurin, Dougal and Duvenaud, David and Adams, Ryan},
  booktitle={International conference on machine learning},
  pages={2113--2122},
  year={2015},
  organization={PMLR}
}
@inproceedings{tramer2017ensemble,
  title={Ensemble adversarial training: Attacks and defenses},
  author={Tram{\`e}r, F and Boneh, D and Kurakin, A and Goodfellow, I and Papernot, N and McDaniel, P},
  booktitle={6th International Conference on Learning Representations, ICLR 2018-Conference Track Proceedings},
  year={2018}
}
@inproceedings{madry2017towards,
  title={Towards Deep Learning Models Resistant to Adversarial Attacks},
  author={Madry, Aleksander and Makelov, Aleksandar and Schmidt, Ludwig and Tsipras, Dimitris and Vladu, Adrian},
  booktitle={International Conference on Learning Representations},
  year={2018}
}
@Article{brown2019superhuman,
  title={Superhuman AI for multiplayer poker},
  author={Brown, Noam and Sandholm, Tuomas},
  journal={Science},
  volume={365},
  number={6456},
  pages={885--890},
  year={2019},
  publisher={American Association for the Advancement of Science}
}

%%NCONF
@inproceedings{metelli2018configurable,
  title={Configurable Markov decision processes},
  author={Metelli, Alberto Maria and Mutti, Mirco and Restelli, Marcello},
  booktitle={International Conference on Machine Learning},
  pages={3491--3500},
  year={2018},
  organization={PMLR}
}
@inproceedings{metelli2019reinforcement,
  title={Reinforcement learning in configurable continuous environments},
  author={Metelli, Alberto Maria and Ghelfi, Emanuele and Restelli, Marcello},
  booktitle={International Conference on Machine Learning},
  pages={4546--4555},
  year={2019}
}
@article{breton1988sequential,
  title={Sequential Stackelberg equilibria in two-person games},
  author={Breton, Michele and Alj, Abderrahmane and Haurie, Alain},
  journal={Journal of Optimization Theory and Applications},
  volume={59},
  number={1},
  pages={71--97},
  year={1988},
  publisher={Springer}
}
@inproceedings{balcan2015commitment,
  title={Commitment without regrets: Online learning in stackelberg security games},
  author={Balcan, Maria-Florina and Blum, Avrim and Haghtalab, Nika and Procaccia, Ariel D},
  booktitle={Proceedings of the sixteenth ACM conference on economics and computation},
  pages={61--78},
  year={2015}
}
@inproceedings{peng2019learning,
  title={Learning optimal strategies to commit to},
  author={Peng, Binghui and Shen, Weiran and Tang, Pingzhong and Zuo, Song},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={33},
  number={01},
  pages={2149--2156},
  year={2019}
}
@article{sessa2020learning,
  title={Learning to Play Sequential Games versus Unknown Opponents},
  author={Sessa, Pier Giuseppe and Bogunovic, Ilija and Kamgarpour, Maryam and Krause, Andreas},
  journal={Advances in Neural Information Processing Systems 33},
  year={2020},
  publisher={Curran}
}
@incollection{NIPS2019_8800,
title = {Almost Horizon-Free Structure-Aware Best Policy Identification with a Generative Model},
author = {Zanette, Andrea and Kochenderfer, Mykel J and Brunskill, Emma},
booktitle = {Advances in Neural Information Processing Systems 32},
editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d Alch\'{e}-Buc and E. Fox and R. Garnett},
pages = {5625--5634},
year = {2019},
publisher = {Curran Associates, Inc.},
}
@article{azar2013minimax,
  title={Minimax PAC bounds on the sample complexity of reinforcement learning with a generative model},
  author={Azar, Mohammad Gheshlaghi and Munos, R{\'e}mi and Kappen, Hilbert J},
  journal={Machine learning},
  volume={91},
  number={3},
  pages={325--349},
  year={2013},
  publisher={Springer}
}
@inproceedings{tirinzoni2020sequential,
  title={Sequential transfer in reinforcement learning with a generative model},
  author={Tirinzoni, Andrea and Poiani, Riccardo and Restelli, Marcello},
  booktitle={International Conference on Machine Learning},
  pages={9481--9492},
  year={2020},
  organization={PMLR}
}
@book{lattimore2020bandit,
  title={Bandit algorithms},
  author={Lattimore, Tor and Szepesv{\'a}ri, Csaba},
  year={2020},
  publisher={Cambridge University Press}
}
@article{auer2002finite,
  title={Finite-time analysis of the multiarmed bandit problem},
  author={Auer, Peter and Cesa-Bianchi, Nicolo and Fischer, Paul},
  journal={Machine learning},
  volume={47},
  number={2-3},
  pages={235--256},
  year={2002},
  publisher={Springer}
}
@article{metelli2019policy,
  title={Policy space identification in configurable environments},
  author={Metelli, Alberto Maria and Manneschi, Guglielmo and Restelli, Marcello},
  journal={arXiv preprint arXiv:1909.03984},
  year={2019}
}
@inproceedings{gallego2019reinforcement,
  title={Reinforcement learning under threats},
  author={Gallego, Victor and Naveiro, Roi and Insua, David Rios},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={33},
  number={01},
  pages={9939--9940},
  year={2019}
}
@article{zhang2009general,
  title={A general approach to environment design with one agent},
  author={Zhang, Haoqi and Chen, Yiling and Parkes, David C},
  year={2009},
  publisher={Morgan Kaufmann Publishers Inc.}
}
@inproceedings{nilim2003robustness,
  author    = {Arnab Nilim and
               Laurent El Ghaoui},
  editor    = {Sebastian Thrun and
               Lawrence K. Saul and
               Bernhard Sch{\"{o}}lkopf},
  title     = {Robustness in Markov Decision Problems with Uncertain Transition Matrices},
  booktitle = {Advances in Neural Information Processing Systems 16 [Neural Information
               Processing Systems, {NIPS} 2003, December 8-13, 2003, Vancouver and
               Whistler, British Columbia, Canada]},
  pages     = {839--846},
  publisher = {{MIT} Press},
  year      = {2003}
}
@article{iyengar2005robust,
  title={Robust dynamic programming},
  author={Iyengar, Garud N},
  journal={Mathematics of Operations Research},
  volume={30},
  number={2},
  pages={257--280},
  year={2005},
  publisher={INFORMS}
}
@article{lai1985asymptotically,
  title={Asymptotically efficient adaptive allocation rules},
  author={Lai, Tze Leung and Robbins, Herbert},
  journal={Advances in applied mathematics},
  volume={6},
  number={1},
  pages={4--22},
  year={1985},
  publisher={Academic Press}
}
@inproceedings{garivier2011kl,
  title={The KL-UCB algorithm for bounded stochastic bandits and beyond},
  author={Garivier, Aur{\'e}lien and Capp{\'e}, Olivier},
  booktitle={Proceedings of the 24th annual conference on learning theory},
  pages={359--376},
  year={2011}
}
@inproceedings{auer2009near,
  title={Near-optimal regret bounds for reinforcement learning},
  author={Auer, Peter and Jaksch, Thomas and Ortner, Ronald},
  booktitle={Advances in neural information processing systems},
  pages={89--96},
  year={2009}
}
@inproceedings{bartlett2012regal,
  title={REGAL: a regularization based algorithm for reinforcement learning in weakly communicating MDPs},
  author={Bartlett, Peter and Tewari, Ambuj},
  booktitle={Uncertainty in Artificial Intelligence: Proceedings of the 25th Conference},
  pages={35--42},
  year={2009},
  organization={AUAI Press}
}

@article{bai2020near,
  title={Near-Optimal Reinforcement Learning with Self-Play},
  author={Bai, Yu and Jin, Chi and Yu, Tiancheng},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}

@inproceedings{bai2020provable,
  title={Provable self-play algorithms for competitive reinforcement learning},
  author={Bai, Yu and Jin, Chi},
  booktitle={International Conference on Machine Learning},
  pages={551--560},
  year={2020},
  organization={PMLR}
}

@article{zhang2020model,
  title={Model-Based Multi-Agent RL in Zero-Sum Markov Games with Near-Optimal Sample Complexity},
  author={Zhang, Kaiqing and Kakade, Sham and Basar, Tamer and Yang, Lin},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}
@inproceedings{bubeck2013bounded,
  title={Bounded regret in stochastic multi-armed bandits},
  author={Bubeck, S{\'e}bastien and Perchet, Vianney and Rigollet, Philippe},
  booktitle={Conference on Learning Theory},
  pages={122--134},
  year={2013}
}

@inproceedings{lattimore2014bounded,
  title={Bounded regret for finite-armed structured bandits},
  author={Lattimore, Tor and Munos, R{\'e}mi},
  booktitle={Advances in Neural Information Processing Systems},
  pages={550--558},
  year={2014}
}
@inproceedings{wei2017online,
  title={Online reinforcement learning in stochastic games},
  author={Wei, Chen-Yu and Hong, Yi-Te and Lu, Chi-Jen},
  booktitle={Proceedings of the 31st International Conference on Neural Information Processing Systems},
  pages={4994--5004},
  year={2017}
}
@article{nguyen2019deep,
  title={Deep reinforcement learning for cyber security},
  author={Nguyen, Thanh Thi and Reddi, Vijay Janapa},
  journal={arXiv preprint arXiv:1906.05799},
  year={2019}
}
@inproceedings{bansal2018emergent,
  title={Emergent Complexity via Multi-Agent Competition},
  author={Bansal, Trapit and Pachocki, Jakub and Sidor, Szymon and Sutskever, Ilya and Mordatch, Igor},
  booktitle={International Conference on Learning Representations},
  year={2018}
}
@article{lowe2017multi,
  title={Multi-Agent Actor-Critic for Mixed Cooperative-Competitive Environments},
  author={Lowe, Ryan and WU, YI and Tamar, Aviv and Harb, Jean and Pieter Abbeel, OpenAI and Mordatch, Igor},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  pages={6379--6390},
  year={2017}
}
@article{foerster2016learning,
  title={Learning to communicate with deep multi-agent reinforcement learning},
  author={Foerster, Jakob N and Assael, Yannis M and De Freitas, Nando and Whiteson, Shimon},
  journal={arXiv preprint arXiv:1605.06676},
  year={2016}
}
@article{brown2018superhuman,
  title={Superhuman AI for heads-up no-limit poker: Libratus beats top professionals},
  author={Brown, Noam and Sandholm, Tuomas},
  journal={Science},
  volume={359},
  number={6374},
  pages={418--424},
  year={2018},
  publisher={American Association for the Advancement of Science}
}
@article{moravvcik2017deepstack,
  title={Deepstack: Expert-level artificial intelligence in heads-up no-limit poker},
  author={Morav{\v{c}}{\'\i}k, Matej and Schmid, Martin and Burch, Neil and Lis{\`y}, Viliam and Morrill, Dustin and Bard, Nolan and Davis, Trevor and Waugh, Kevin and Johanson, Michael and Bowling, Michael},
  journal={Science},
  volume={356},
  number={6337},
  pages={508--513},
  year={2017},
  publisher={American Association for the Advancement of Science}
}

@article{tian2020provably,
  title={Provably Efficient Online Agnostic Learning in Markov Games},
  author={Tian, Yi and Wang, Yuanhao and Yu, Tiancheng and Sra, Suvrit},
  journal={arXiv preprint arXiv:2010.15020},
  year={2020}
}
@inproceedings{xie2020learning,
  title={Learning zero-sum simultaneous-move markov games using function approximation and correlated equilibrium},
  author={Xie, Qiaomin and Chen, Yudong and Wang, Zhaoran and Yang, Zhuoran},
  booktitle={Conference on Learning Theory},
  pages={3674--3682},
  year={2020},
  organization={PMLR}
}
@article{zhang2020robust,
  title={Robust Multi-Agent Reinforcement Learning with Model Uncertainty},
  author={Zhang, Kaiqing and Sun, Tao and Tao, Yunzhe and Genc, Sahika and Mallya, Sunil and Basar, Tamer},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}
@inproceedings{sidford2020solving,
  title={Solving discounted stochastic two-player games with near-optimal time and sample complexity},
  author={Sidford, Aaron and Wang, Mengdi and Yang, Lin and Ye, Yinyu},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2992--3002},
  year={2020},
  organization={PMLR}
}

@article{liu2020sharp,
  title={A Sharp Analysis of Model-based Reinforcement Learning with Self-Play},
  author={Liu, Qinghua and Yu, Tiancheng and Bai, Yu and Jin, Chi},
  journal={arXiv preprint arXiv:2010.01604},
  year={2020}
}
@inproceedings{li2020exploration,
  title={Exploration Analysis in Finite-Horizon Turn-based Stochastic Games},
  author={Li, Jialian and Zhou, Yichi and Ren, Tongzheng and Zhu, Jun},
  booktitle={Conference on Uncertainty in Artificial Intelligence},
  pages={201--210},
  year={2020},
  organization={PMLR}
}
@book{hestenes1952methods,
  title={Methods of conjugate gradients for solving linear systems},
  author={Hestenes, Magnus Rudolph and Stiefel, Eduard and others},
  volume={49},
  number={1},
  year={1952},
  publisher={NBS Washington, DC}
}
@InProceedings{perolat15, title = {Approximate Dynamic Programming for Two-Player Zero-Sum Markov Games}, author = {Perolat, Julien and Scherrer, Bruno and Piot, Bilal and Pietquin, Olivier}, booktitle = {Proceedings of the 32nd International Conference on Machine Learning}, pages = {1321--1329}, year = {2015}, editor = {Bach, Francis and Blei, David}, volume = {37}, series = {Proceedings of Machine Learning Research}, address = {Lille, France}, month = {07--09 Jul}, publisher = {PMLR}, pdf = {http://proceedings.mlr.press/v37/perolat15.pdf}}
@inproceedings{szepesvari1996generalized,
  title={Generalized markov decision processes: Dynamic-programming and reinforcement-learning algorithms},
  author={Szepesv{\'a}ri, Csaba and Littman, Michael L},
  booktitle={Proceedings of International Conference of Machine Learning},
  volume={96},
  year={1996}
}
@article{coniglio2020computing,
  title={Computing a pessimistic stackelberg equilibrium with multiple followers: The mixed-pure case},
  author={Coniglio, Stefano and Gatti, Nicola and Marchesi, Alberto},
  journal={Algorithmica},
  volume={82},
  number={5},
  pages={1189--1238},
  year={2020},
  publisher={Springer}
}
@article{simchowitz2019non,
  title={Non-asymptotic gap-dependent regret bounds for tabular MDPs},
  author={Simchowitz, Max and Jamieson, Kevin},
  journal={arXiv preprint arXiv:1905.03814},
  year={2019}
}