\begin{thebibliography}{48}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
  \providecommand{\doi}[1]{doi: #1}\else
  \providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi

\bibitem[Abbasi-Yadkori et~al.(2013)Abbasi-Yadkori, Bartlett, Kanade, Seldin,
  and Szepesv{\'a}ri]{abbasi2013online}
Yasin Abbasi-Yadkori, Peter~L Bartlett, Varun Kanade, Yevgeny Seldin, and Csaba
  Szepesv{\'a}ri.
\newblock Online learning in markov decision processes with adversarially
  chosen transition probability distributions.
\newblock In \emph{Proceedings of the 26th International Conference on Neural
  Information Processing Systems-Volume 2}, pages 2508--2516, 2013.

\bibitem[Auer et~al.(2002)Auer, Cesa-Bianchi, and Fischer]{auer2002finite}
Peter Auer, Nicolo Cesa-Bianchi, and Paul Fischer.
\newblock Finite-time analysis of the multiarmed bandit problem.
\newblock \emph{Machine learning}, 47\penalty0 (2-3):\penalty0 235--256, 2002.

\bibitem[Azar et~al.(2013)Azar, Munos, and Kappen]{azar2013minimax}
Mohammad~Gheshlaghi Azar, R{\'e}mi Munos, and Hilbert~J Kappen.
\newblock Minimax pac bounds on the sample complexity of reinforcement learning
  with a generative model.
\newblock \emph{Machine learning}, 91\penalty0 (3):\penalty0 325--349, 2013.

\bibitem[Bai and Jin(2020)]{bai2020provable}
Yu~Bai and Chi Jin.
\newblock Provable self-play algorithms for competitive reinforcement learning.
\newblock In \emph{International Conference on Machine Learning}, pages
  551--560. PMLR, 2020.

\bibitem[Bai et~al.(2020)Bai, Jin, and Yu]{bai2020near}
Yu~Bai, Chi Jin, and Tiancheng Yu.
\newblock Near-optimal reinforcement learning with self-play.
\newblock \emph{Advances in Neural Information Processing Systems}, 33, 2020.

\bibitem[Bai et~al.(2021)Bai, Jin, Wang, and Xiong]{bai2021sample}
Yu~Bai, Chi Jin, Huan Wang, and Caiming Xiong.
\newblock Sample-efficient learning of stackelberg equilibria in general-sum
  games.
\newblock \emph{arXiv preprint arXiv:2102.11494}, 2021.

\bibitem[Balcan et~al.(2015)Balcan, Blum, Haghtalab, and
  Procaccia]{balcan2015commitment}
Maria-Florina Balcan, Avrim Blum, Nika Haghtalab, and Ariel~D Procaccia.
\newblock Commitment without regrets: Online learning in stackelberg security
  games.
\newblock In \emph{Proceedings of the sixteenth ACM conference on economics and
  computation}, pages 61--78, 2015.

\bibitem[Bansal et~al.(2018)Bansal, Pachocki, Sidor, Sutskever, and
  Mordatch]{bansal2018emergent}
Trapit Bansal, Jakub Pachocki, Szymon Sidor, Ilya Sutskever, and Igor Mordatch.
\newblock Emergent complexity via multi-agent competition.
\newblock In \emph{International Conference on Learning Representations}, 2018.

\bibitem[Brafman and Tennenholtz(2002)]{brafman2002r}
Ronen~I Brafman and Moshe Tennenholtz.
\newblock R-max-a general polynomial time algorithm for near-optimal
  reinforcement learning.
\newblock \emph{Journal of Machine Learning Research}, 3\penalty0
  (Oct):\penalty0 213--231, 2002.

\bibitem[Brockman et~al.(2016)Brockman, Cheung, Pettersson, Schneider,
  Schulman, Tang, and Zaremba]{openai}
Greg Brockman, Vicki Cheung, Ludwig Pettersson, Jonas Schneider, John Schulman,
  Jie Tang, and Wojciech Zaremba.
\newblock Openai gym.
\newblock \emph{CoRR}, abs/1606.01540, 2016.

\bibitem[Brown and Sandholm(2018)]{brown2018superhuman}
Noam Brown and Tuomas Sandholm.
\newblock Superhuman ai for heads-up no-limit poker: Libratus beats top
  professionals.
\newblock \emph{Science}, 359\penalty0 (6374):\penalty0 418--424, 2018.

\bibitem[Chung et~al.(2020)Chung, Maharjan, Zhang, and
  Eliassen]{chung2020distributed}
Hwei-Ming Chung, Sabita Maharjan, Yan Zhang, and Frank Eliassen.
\newblock Distributed deep reinforcement learning for intelligent load
  scheduling in residential smart grid.
\newblock \emph{IEEE Transactions on Industrial Informatics}, 2020.

\bibitem[Da~Silva and Costa(2019)]{da2019survey}
Felipe~Leno Da~Silva and Anna Helena~Reali Costa.
\newblock A survey on transfer learning for multiagent reinforcement learning
  systems.
\newblock \emph{Journal of Artificial Intelligence Research}, 64:\penalty0
  645--703, 2019.

\bibitem[Dick et~al.(2014)Dick, Gyorgy, and Szepesvari]{dick2014online}
Travis Dick, Andras Gyorgy, and Csaba Szepesvari.
\newblock Online learning in markov decision processes with changing cost
  sequences.
\newblock In \emph{International Conference on Machine Learning}, pages
  512--520. PMLR, 2014.

\bibitem[Domingues et~al.(2020)Domingues, Ménard, Kaufmann, and
  Valko]{domingues2020episodic}
Omar~Darwiche Domingues, Pierre Ménard, Emilie Kaufmann, and Michal Valko.
\newblock Episodic reinforcement learning in finite mdps: Minimax lower bounds
  revisited, 2020.

\bibitem[Even-Dar et~al.(2009)Even-Dar, Kakade, and Mansour]{even2009online}
Eyal Even-Dar, Sham~M Kakade, and Yishay Mansour.
\newblock Online markov decision processes.
\newblock \emph{Mathematics of Operations Research}, 34\penalty0 (3):\penalty0
  726--736, 2009.

\bibitem[Foerster et~al.(2016)Foerster, Assael, De~Freitas, and
  Whiteson]{foerster2016learning}
Jakob~N Foerster, Yannis~M Assael, Nando De~Freitas, and Shimon Whiteson.
\newblock Learning to communicate with deep multi-agent reinforcement learning.
\newblock \emph{arXiv preprint arXiv:1605.06676}, 2016.

\bibitem[Gergely~Neu et~al.(2010)Gergely~Neu, Szepesv{\'a}ri, and
  Antos]{gergely2010online}
Andr{\'a}s~Gy{\"o}rgy Gergely~Neu, Csaba Szepesv{\'a}ri, and Andr{\'a}s Antos.
\newblock Online markov decision processes under bandit feedback.
\newblock In \emph{Proceedings of the Twenty-Fourth Annual Conference on Neural
  Information Processing Systems}, 2010.

\bibitem[Hernandez-Leal et~al.(2019)Hernandez-Leal, Kartal, and
  Taylor]{hernandez2019survey}
Pablo Hernandez-Leal, Bilal Kartal, and Matthew~E Taylor.
\newblock A survey and critique of multiagent deep reinforcement learning.
\newblock \emph{Autonomous Agents and Multi-Agent Systems}, 33\penalty0
  (6):\penalty0 750--797, 2019.

\bibitem[Jaksch et~al.(2010)Jaksch, Ortner, and Auer]{jaksch2010near}
Thomas Jaksch, Ronald Ortner, and Peter Auer.
\newblock Near-optimal regret bounds for reinforcement learning.
\newblock \emph{Journal of Machine Learning Research}, 11\penalty0 (4), 2010.

\bibitem[Jin et~al.(2020)Jin, Jin, Luo, Sra, and Yu]{jin2020learning}
Chi Jin, Tiancheng Jin, Haipeng Luo, Suvrit Sra, and Tiancheng Yu.
\newblock Learning adversarial markov decision processes with bandit feedback
  and unknown transition.
\newblock In \emph{International Conference on Machine Learning}, pages
  4860--4869. PMLR, 2020.

\bibitem[Lattimore and Szepesv{\'a}ri(2020)]{lattimore2020bandit}
Tor Lattimore and Csaba Szepesv{\'a}ri.
\newblock \emph{Bandit algorithms}.
\newblock Cambridge University Press, 2020.

\bibitem[Li et~al.(2020)Li, Zhou, Ren, and Zhu]{li2020exploration}
Jialian Li, Yichi Zhou, Tongzheng Ren, and Jun Zhu.
\newblock Exploration analysis in finite-horizon turn-based stochastic games.
\newblock In \emph{Conference on Uncertainty in Artificial Intelligence}, pages
  201--210. PMLR, 2020.

\bibitem[Lillicrap et~al.(2015)Lillicrap, Hunt, Pritzel, Heess, Erez, Tassa,
  Silver, and Wierstra]{lillicrap2015continuous}
Timothy~P Lillicrap, Jonathan~J Hunt, Alexander Pritzel, Nicolas Heess, Tom
  Erez, Yuval Tassa, David Silver, and Daan Wierstra.
\newblock Continuous control with deep reinforcement learning.
\newblock \emph{arXiv preprint arXiv:1509.02971}, 2015.

\bibitem[Liu et~al.(2020)Liu, Yu, Bai, and Jin]{liu2020sharp}
Qinghua Liu, Tiancheng Yu, Yu~Bai, and Chi Jin.
\newblock A sharp analysis of model-based reinforcement learning with
  self-play.
\newblock \emph{arXiv preprint arXiv:2010.01604}, 2020.

\bibitem[Lowe et~al.(2017)Lowe, WU, Tamar, Harb, Pieter~Abbeel, and
  Mordatch]{lowe2017multi}
Ryan Lowe, YI~WU, Aviv Tamar, Jean Harb, OpenAI Pieter~Abbeel, and Igor
  Mordatch.
\newblock Multi-agent actor-critic for mixed cooperative-competitive
  environments.
\newblock \emph{Advances in Neural Information Processing Systems},
  30:\penalty0 6379--6390, 2017.

\bibitem[Lykouris et~al.(2019)Lykouris, Simchowitz, Slivkins, and
  Sun]{lykouris2019corruption}
Thodoris Lykouris, Max Simchowitz, Aleksandrs Slivkins, and Wen Sun.
\newblock Corruption robust exploration in episodic reinforcement learning.
\newblock \emph{arXiv preprint arXiv:1911.08689}, 2019.

\bibitem[Morav{\v{c}}{\'\i}k et~al.(2017)Morav{\v{c}}{\'\i}k, Schmid, Burch,
  Lis{\`y}, Morrill, Bard, Davis, Waugh, Johanson, and
  Bowling]{moravvcik2017deepstack}
Matej Morav{\v{c}}{\'\i}k, Martin Schmid, Neil Burch, Viliam Lis{\`y}, Dustin
  Morrill, Nolan Bard, Trevor Davis, Kevin Waugh, Michael Johanson, and Michael
  Bowling.
\newblock Deepstack: Expert-level artificial intelligence in heads-up no-limit
  poker.
\newblock \emph{Science}, 356\penalty0 (6337):\penalty0 508--513, 2017.

\bibitem[Neu et~al.(2012)Neu, Gyorgy, and Szepesv{\'a}ri]{neu2012adversarial}
Gergely Neu, Andras Gyorgy, and Csaba Szepesv{\'a}ri.
\newblock The adversarial stochastic shortest path problem with unknown
  transition probabilities.
\newblock In \emph{Artificial Intelligence and Statistics}, pages 805--813.
  PMLR, 2012.

\bibitem[Papoudakis et~al.(2019)Papoudakis, Christianos, Rahman, and
  Albrecht]{papoudakis2019dealing}
Georgios Papoudakis, Filippos Christianos, Arrasy Rahman, and Stefano~V
  Albrecht.
\newblock Dealing with non-stationarity in multi-agent deep reinforcement
  learning.
\newblock \emph{arXiv preprint arXiv:1906.04737}, 2019.

\bibitem[Peng et~al.(2019)Peng, Shen, Tang, and Zuo]{peng2019learning}
Binghui Peng, Weiran Shen, Pingzhong Tang, and Song Zuo.
\newblock Learning optimal strategies to commit to.
\newblock In \emph{Proceedings of the AAAI Conference on Artificial
  Intelligence}, volume~33, pages 2149--2156, 2019.

\bibitem[Rosenberg and Mansour(2019)]{rosenberg2019online}
Aviv Rosenberg and Yishay Mansour.
\newblock Online convex optimization in adversarial markov decision processes.
\newblock In \emph{International Conference on Machine Learning}, pages
  5478--5486. PMLR, 2019.

\bibitem[Sessa et~al.(2020)Sessa, Bogunovic, Kamgarpour, and
  Krause]{sessa2020learning}
Pier~Giuseppe Sessa, Ilija Bogunovic, Maryam Kamgarpour, and Andreas Krause.
\newblock Learning to play sequential games versus unknown opponents.
\newblock \emph{Advances in Neural Information Processing Systems 33}, 2020.

\bibitem[Shalev-Shwartz et~al.(2016)Shalev-Shwartz, Shammah, and
  Shashua]{shalev2016safe}
Shai Shalev-Shwartz, Shaked Shammah, and Amnon Shashua.
\newblock Safe, multi-agent, reinforcement learning for autonomous driving.
\newblock \emph{arXiv preprint arXiv:1610.03295}, 2016.

\bibitem[Shapley(1953)]{shapley1953stochastic}
Lloyd~S Shapley.
\newblock Stochastic games.
\newblock \emph{Proceedings of the national academy of sciences}, 39\penalty0
  (10):\penalty0 1095--1100, 1953.

\bibitem[Sidford et~al.(2020)Sidford, Wang, Yang, and Ye]{sidford2020solving}
Aaron Sidford, Mengdi Wang, Lin Yang, and Yinyu Ye.
\newblock Solving discounted stochastic two-player games with near-optimal time
  and sample complexity.
\newblock In \emph{International Conference on Artificial Intelligence and
  Statistics}, pages 2992--3002. PMLR, 2020.

\bibitem[Silver et~al.(2017)Silver, Schrittwieser, Simonyan, Antonoglou, Huang,
  Guez, Hubert, Baker, Lai, Bolton, et~al.]{silver2017mastering}
David Silver, Julian Schrittwieser, Karen Simonyan, Ioannis Antonoglou, Aja
  Huang, Arthur Guez, Thomas Hubert, Lucas Baker, Matthew Lai, Adrian Bolton,
  et~al.
\newblock Mastering the game of go without human knowledge.
\newblock \emph{nature}, 550\penalty0 (7676):\penalty0 354--359, 2017.

\bibitem[Simchowitz and Jamieson(2019)]{simchowitz2019non}
Max Simchowitz and Kevin Jamieson.
\newblock Non-asymptotic gap-dependent regret bounds for tabular mdps.
\newblock \emph{arXiv preprint arXiv:1905.03814}, 2019.

\bibitem[Sutton and Barto(2018)]{sutton2018reinforcement}
Richard~S Sutton and Andrew~G Barto.
\newblock \emph{Reinforcement learning: An introduction}.
\newblock MIT press, 2018.

\bibitem[Sutton et~al.(1998)Sutton, Barto, et~al.]{sutton1998introduction}
Richard~S Sutton, Andrew~G Barto, et~al.
\newblock \emph{Introduction to reinforcement learning}, volume 135.
\newblock MIT press Cambridge, 1998.

\bibitem[Tian et~al.(2020)Tian, Wang, Yu, and Sra]{tian2020provably}
Yi~Tian, Yuanhao Wang, Tiancheng Yu, and Suvrit Sra.
\newblock Provably efficient online agnostic learning in markov games.
\newblock \emph{arXiv preprint arXiv:2010.15020}, 2020.

\bibitem[Wei et~al.(2017)Wei, Hong, and Lu]{wei2017online}
Chen-Yu Wei, Yi-Te Hong, and Chi-Jen Lu.
\newblock Online reinforcement learning in stochastic games.
\newblock In \emph{Proceedings of the 31st International Conference on Neural
  Information Processing Systems}, pages 4994--5004, 2017.

\bibitem[Xie et~al.(2020)Xie, Chen, Wang, and Yang]{xie2020learning}
Qiaomin Xie, Yudong Chen, Zhaoran Wang, and Zhuoran Yang.
\newblock Learning zero-sum simultaneous-move markov games using function
  approximation and correlated equilibrium.
\newblock In \emph{Conference on Learning Theory}, pages 3674--3682. PMLR,
  2020.

\bibitem[Yu and Mannor(2009)]{yu2009arbitrarily}
Jia~Yuan Yu and Shie Mannor.
\newblock Arbitrarily modulated markov decision processes.
\newblock In \emph{Proceedings of the 48h IEEE Conference on Decision and
  Control (CDC) held jointly with 2009 28th Chinese Control Conference}, pages
  2946--2953. IEEE, 2009.

\bibitem[Zhang et~al.(2019)Zhang, Yang, and Ba{\c{s}}ar]{zhang2019multi}
Kaiqing Zhang, Zhuoran Yang, and Tamer Ba{\c{s}}ar.
\newblock Multi-agent reinforcement learning: A selective overview of theories
  and algorithms.
\newblock \emph{arXiv preprint arXiv:1911.10635}, 2019.

\bibitem[Zhang et~al.(2020{\natexlab{a}})Zhang, Kakade, Basar, and
  Yang]{zhang2020model}
Kaiqing Zhang, Sham Kakade, Tamer Basar, and Lin Yang.
\newblock Model-based multi-agent rl in zero-sum markov games with near-optimal
  sample complexity.
\newblock \emph{Advances in Neural Information Processing Systems}, 33,
  2020{\natexlab{a}}.

\bibitem[Zhang et~al.(2020{\natexlab{b}})Zhang, Sun, Tao, Genc, Mallya, and
  Basar]{zhang2020robust}
Kaiqing Zhang, Tao Sun, Yunzhe Tao, Sahika Genc, Sunil Mallya, and Tamer Basar.
\newblock Robust multi-agent reinforcement learning with model uncertainty.
\newblock \emph{Advances in Neural Information Processing Systems}, 33,
  2020{\natexlab{b}}.

\bibitem[Zimin and Neu(2013)]{zimin2013online}
Alexander Zimin and Gergely Neu.
\newblock Online learning in episodic markovian decision processes by relative
  entropy policy search.
\newblock In \emph{Neural Information Processing Systems 26}, 2013.

\end{thebibliography}
