\relax 
\providecommand\hyper@newdestlabel[2]{}
\providecommand\HyperFirstAtBeginDocument{\AtBeginDocument}
\HyperFirstAtBeginDocument{\ifx\hyper@anchor\@undefined
\global\let\oldcontentsline\contentsline
\gdef\contentsline#1#2#3#4{\oldcontentsline{#1}{#2}{#3}}
\global\let\oldnewlabel\newlabel
\gdef\newlabel#1#2{\newlabelxx{#1}#2}
\gdef\newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}}
\AtEndDocument{\ifx\hyper@anchor\@undefined
\let\contentsline\oldcontentsline
\let\newlabel\oldnewlabel
\fi}
\fi}
\global\let\hyper@last\relax 
\gdef\HyperFirstAtBeginDocument#1{#1}
\providecommand\HyField@AuxAddToFields[1]{}
\providecommand\HyField@AuxAddToCoFields[2]{}
\providecommand\babel@aux[2]{}
\@nameuse{bbl@beforestart}
\bibstyle{plainnat}
\citation{sutton2018reinforcement}
\citation{silver2017mastering}
\citation{openai}
\citation{moravvcik2017deepstack,brown2018superhuman}
\citation{shalev2016safe}
\citation{foerster2016learning,lowe2017multi,bansal2018emergent}
\citation{lillicrap2015continuous}
\citation{chung2020distributed}
\citation{shapley1953stochastic}
\citation{sutton1998introduction}
\citation{xie2020learning,wei2017online,tian2020provably}
\providecommand \oddpage@label [2]{}
\babel@aux{american}{}
\@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{1}{section.1}\protected@file@percent }
\citation{xie2020learning}
\citation{xie2020learning,tian2020provably}
\citation{xie2020learning}
\citation{sessa2020learning}
\citation{shapley1953stochastic}
\citation{zhang2019multi,da2019survey,hernandez2019survey,papoudakis2019dealing}
\citation{bai2020near,zhang2020robust}
\citation{bai2020provable,sidford2020solving,li2020exploration,liu2020sharp,zhang2020model}
\citation{sidford2020solving,zhang2020model}
\citation{liu2020sharp}
\citation{li2020exploration}
\citation{bai2021sample}
\citation{brafman2002r}
\citation{wei2017online}
\citation{xie2020learning}
\citation{tian2020provably}
\citation{bai2020near}
\citation{xie2020learning}
\@writefile{toc}{\contentsline {section}{\numberline {2}Related works}{2}{section.2}\protected@file@percent }
\newlabel{sec:relatedworks}{{2}{2}{Related works}{section.2}{}}
\citation{even2009online,gergely2010online,zimin2013online,rosenberg2019online,jin2020learning,dick2014online}
\citation{yu2009arbitrarily,neu2012adversarial,lykouris2019corruption}
\citation{shapley1953stochastic,xie2020learning,bai2020near}
\citation{xie2020learning,wei2017online}
\@writefile{toc}{\contentsline {section}{\numberline {3}Preliminaries}{3}{section.3}\protected@file@percent }
\newlabel{sec:preliminaries}{{3}{3}{Preliminaries}{section.3}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.1}Turn-based Markov Games: background and notation}{3}{subsection.3.1}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Value Functions}{3}{paragraph*.1}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {4}Problem statement}{3}{section.4}\protected@file@percent }
\newlabel{sec:problem}{{4}{3}{Problem statement}{section.4}{}}
\citation{tian2020provably}
\citation{balcan2015commitment,peng2019learning,sessa2020learning}
\citation{balcan2015commitment,peng2019learning,sessa2020learning}
\citation{lattimore2020bandit}
\citation{auer2002finite}
\newlabel{eq:regredef}{{4}{4}{Problem statement}{equation.4.4}{}}
\@writefile{toc}{\contentsline {paragraph}{Bandit vs Turn-based MG}{4}{paragraph*.2}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {5}Hardness of Learning in Turn-based two-player Markov Games}{4}{section.5}\protected@file@percent }
\newlabel{sec:lowerbound}{{5}{4}{Hardness of Learning in Turn-based two-player Markov Games}{section.5}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces The composite Turn-based Markov Game was constructed for the lower bound. The states belonging to $\mathcal  {S}_2$ are in orange, the ones belonging to $\mathcal  {S}_1$ in blue. The dashed lines corresponds to the transition probabilities taking action $a^\star $, the others taking any other action $a \in \mathcal  {A}$ with $a \ne  a^\star $. The dots indicate the chain composed of $N$ states. We omit the self-loop in $s_f$, which corresponds to the fact that $s_f$ is a sink state.\relax }}{5}{figure.caption.3}\protected@file@percent }
\providecommand*\caption@xref[2]{\@setref\relax\@undefined{#1}}
\newlabel{fig:mdplower}{{1}{5}{The composite Turn-based Markov Game was constructed for the lower bound. The states belonging to $\state _2$ are in orange, the ones belonging to $\state _1$ in blue. The dashed lines corresponds to the transition probabilities taking action $a^\star $, the others taking any other action $a \in \actions $ with $a \ne a^\star $. The dots indicate the chain composed of $N$ states. We omit the self-loop in $s_f$, which corresponds to the fact that $s_f$ is a sink state.\relax }{figure.caption.3}{}}
\newlabel{thmt@@prop@data}{{\def \theequation {\@arabic {\c@equation }}\def \theHequation {(restate \theHthmt@dummyctr )5.\@arabic {\c@equation }}\setcounter {equation}{4}}{5}{Hardness of Learning in Turn-based two-player Markov Games}{section.5}{}}
\@writefile{loe}{\contentsline {prp}{\ifthmt@listswap Proposition~5.1\else \numberline {5.1}Proposition\fi }{5}{prp.5.1}\protected@file@percent }
\newlabel{thmt@@prop}{{5.1}{5}{}{prp.5.1}{}}
\@writefile{toc}{\contentsline {paragraph}{Intuition on lower bound}{5}{paragraph*.4}\protected@file@percent }
\newlabel{thmt@@lb@data}{{\def \theequation {\@arabic {\c@equation }}\def \theHequation {(restate \theHthmt@dummyctr )5.\@arabic {\c@equation }}\setcounter {equation}{4}}{5}{Intuition on lower bound}{paragraph*.4}{}}
\@writefile{loe}{\contentsline {thm}{\ifthmt@listswap Theorem~5.1\else \numberline {5.1}Theorem\fi \thmtformatoptarg {Lower bound for online Turn-based Stochastic Game}}{5}{thm.5.1}\protected@file@percent }
\newlabel{thmt@@lb}{{5.1}{5}{Lower bound for online Turn-based Stochastic Game}{thm.5.1}{}}
\newlabel{thm:lb}{{5.1}{5}{Lower bound for online Turn-based Stochastic Game}{thm.5.1}{}}
\citation{jaksch2010near,domingues2020episodic}
\citation{simchowitz2019non}
\citation{bai2020near}
\citation{tian2020provably}
\citation{abbasi2013online}
\citation{azar2013minimax}
\newlabel{eq:regret1}{{5}{6}{Lower bound for online Turn-based Stochastic Game}{equation.5.5}{}}
\newlabel{eq:regret2}{{6}{6}{Lower bound for online Turn-based Stochastic Game}{equation.5.6}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Proof sketch of Theorem \ref  {thm:lb}}{6}{subsection.5.1}\protected@file@percent }
\newlabel{thmt@@lbinformation@data}{{\def \theequation {\@arabic {\c@equation }}\def \theHequation {(restate \theHthmt@dummyctr )5.\@arabic {\c@equation }}\setcounter {equation}{6}}{6}{Proof sketch of Theorem \ref {thm:lb}}{subsection.5.1}{}}
\newlabel{thmt@@lbinformation}{{5.1}{6}{Lower bound for online Turn-based Stochastic Game}{lemma.5.1}{}}
\newlabel{lemma:lbinformation}{{5.1}{6}{Lower bound for online Turn-based Stochastic Game}{lemma.5.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Discussion on Theorem \ref  {thm:lb}}{6}{subsection.5.2}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {6}TMG Optimistic Policies Value Iteration}{6}{section.6}\protected@file@percent }
\newlabel{sec:algorithm}{{6}{6}{TMG Optimistic Policies Value Iteration}{section.6}{}}
\newlabel{sec:algotsg}{{6}{6}{TMG Optimistic Policies Value Iteration}{section.6}{}}
\@writefile{toc}{\contentsline {paragraph}{TMG-OPVI algorithm}{6}{paragraph*.5}\protected@file@percent }
\citation{azar2013minimax}
\@writefile{toc}{\contentsline {paragraph}{Two levels of optimism}{7}{paragraph*.6}\protected@file@percent }
\@writefile{loa}{\contentsline {algorithm}{\numberline {1}{\ignorespaces TMG-OPVI\relax }}{7}{algorithm.1}\protected@file@percent }
\newlabel{alg:tbs}{{1}{7}{TMG-OPVI\relax }{algorithm.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {6.1}Regret Guarantees}{7}{subsection.6.1}\protected@file@percent }
\@writefile{toc}{\contentsline {paragraph}{Estimation of the transition model}{7}{paragraph*.7}\protected@file@percent }
\newlabel{thmt@@opvi@data}{{\def \theequation {\@arabic {\c@equation }}\def \theHequation {(restate \theHthmt@dummyctr )6.\@arabic {\c@equation }}\setcounter {equation}{6}}{7}{Estimation of the transition model}{paragraph*.7}{}}
\@writefile{loe}{\contentsline {thm}{\ifthmt@listswap Theorem~6.1\else \numberline {6.1}Theorem\fi \thmtformatoptarg {Regret of TMG-OPVI}}{7}{thm.6.1}\protected@file@percent }
\newlabel{thmt@@opvi}{{6.1}{7}{Regret of TMG-OPVI}{thm.6.1}{}}
\newlabel{thm:tsgreg}{{6.1}{7}{Regret of TMG-OPVI}{thm.6.1}{}}
\citation{azar2013minimax}
\citation{tian2020provably}
\citation{abbasi2013online}
\citation{tian2020provably}
\citation{balcan2015commitment}
\citation{sessa2020learning}
\@writefile{toc}{\contentsline {subsection}{\numberline {6.2}Proof sketch of Theorem \ref  {thm:tsgreg}}{8}{subsection.6.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {6.3}Discussion on Theorem \ref  {thm:tsgreg}}{8}{subsection.6.3}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {7}Conclusions}{8}{section.7}\protected@file@percent }
\bibdata{uai2022-template.bib}
\bibcite{abbasi2013online}{{1}{2013}{{Abbasi-Yadkori et~al.}}{{Abbasi-Yadkori, Bartlett, Kanade, Seldin, and Szepesv{\'a}ri}}}
\bibcite{auer2002finite}{{2}{2002}{{Auer et~al.}}{{Auer, Cesa-Bianchi, and Fischer}}}
\bibcite{azar2013minimax}{{3}{2013}{{Azar et~al.}}{{Azar, Munos, and Kappen}}}
\bibcite{bai2020provable}{{4}{2020}{{Bai and Jin}}{{}}}
\bibcite{bai2020near}{{5}{2020}{{Bai et~al.}}{{Bai, Jin, and Yu}}}
\bibcite{bai2021sample}{{6}{2021}{{Bai et~al.}}{{Bai, Jin, Wang, and Xiong}}}
\bibcite{balcan2015commitment}{{7}{2015}{{Balcan et~al.}}{{Balcan, Blum, Haghtalab, and Procaccia}}}
\bibcite{bansal2018emergent}{{8}{2018}{{Bansal et~al.}}{{Bansal, Pachocki, Sidor, Sutskever, and Mordatch}}}
\bibcite{brafman2002r}{{9}{2002}{{Brafman and Tennenholtz}}{{}}}
\bibcite{openai}{{10}{2016}{{Brockman et~al.}}{{Brockman, Cheung, Pettersson, Schneider, Schulman, Tang, and Zaremba}}}
\bibcite{brown2018superhuman}{{11}{2018}{{Brown and Sandholm}}{{}}}
\bibcite{chung2020distributed}{{12}{2020}{{Chung et~al.}}{{Chung, Maharjan, Zhang, and Eliassen}}}
\bibcite{da2019survey}{{13}{2019}{{Da~Silva and Costa}}{{}}}
\bibcite{dick2014online}{{14}{2014}{{Dick et~al.}}{{Dick, Gyorgy, and Szepesvari}}}
\bibcite{domingues2020episodic}{{15}{2020}{{Domingues et~al.}}{{Domingues, Ménard, Kaufmann, and Valko}}}
\bibcite{even2009online}{{16}{2009}{{Even-Dar et~al.}}{{Even-Dar, Kakade, and Mansour}}}
\bibcite{feller1957introduction}{{17}{1957}{{Feller}}{{}}}
\bibcite{foerster2016learning}{{18}{2016}{{Foerster et~al.}}{{Foerster, Assael, De~Freitas, and Whiteson}}}
\bibcite{gergely2010online}{{19}{2010}{{Gergely~Neu et~al.}}{{Gergely~Neu, Szepesv{\'a}ri, and Antos}}}
\bibcite{hernandez2019survey}{{20}{2019}{{Hernandez-Leal et~al.}}{{Hernandez-Leal, Kartal, and Taylor}}}
\bibcite{jaksch2010near}{{21}{2010}{{Jaksch et~al.}}{{Jaksch, Ortner, and Auer}}}
\bibcite{jin2020learning}{{22}{2020}{{Jin et~al.}}{{Jin, Jin, Luo, Sra, and Yu}}}
\bibcite{lattimore2020bandit}{{23}{2020}{{Lattimore and Szepesv{\'a}ri}}{{}}}
\bibcite{li2020exploration}{{24}{2020}{{Li et~al.}}{{Li, Zhou, Ren, and Zhu}}}
\bibcite{lillicrap2015continuous}{{25}{2015}{{Lillicrap et~al.}}{{Lillicrap, Hunt, Pritzel, Heess, Erez, Tassa, Silver, and Wierstra}}}
\bibcite{liu2020sharp}{{26}{2020}{{Liu et~al.}}{{Liu, Yu, Bai, and Jin}}}
\bibcite{lowe2017multi}{{27}{2017}{{Lowe et~al.}}{{Lowe, WU, Tamar, Harb, Pieter~Abbeel, and Mordatch}}}
\bibcite{lykouris2019corruption}{{28}{2019}{{Lykouris et~al.}}{{Lykouris, Simchowitz, Slivkins, and Sun}}}
\bibcite{moravvcik2017deepstack}{{29}{2017}{{Morav{\v {c}}{\'\i }k et~al.}}{{Morav{\v {c}}{\'\i }k, Schmid, Burch, Lis{\`y}, Morrill, Bard, Davis, Waugh, Johanson, and Bowling}}}
\bibcite{neu2012adversarial}{{30}{2012}{{Neu et~al.}}{{Neu, Gyorgy, and Szepesv{\'a}ri}}}
\bibcite{papoudakis2019dealing}{{31}{2019}{{Papoudakis et~al.}}{{Papoudakis, Christianos, Rahman, and Albrecht}}}
\bibcite{peng2019learning}{{32}{2019}{{Peng et~al.}}{{Peng, Shen, Tang, and Zuo}}}
\bibcite{rosenberg2019online}{{33}{2019}{{Rosenberg and Mansour}}{{}}}
\bibcite{sessa2020learning}{{34}{2020}{{Sessa et~al.}}{{Sessa, Bogunovic, Kamgarpour, and Krause}}}
\bibcite{shalev2016safe}{{35}{2016}{{Shalev-Shwartz et~al.}}{{Shalev-Shwartz, Shammah, and Shashua}}}
\bibcite{shapley1953stochastic}{{36}{1953}{{Shapley}}{{}}}
\bibcite{sidford2020solving}{{37}{2020}{{Sidford et~al.}}{{Sidford, Wang, Yang, and Ye}}}
\bibcite{silver2017mastering}{{38}{2017}{{Silver et~al.}}{{Silver, Schrittwieser, Simonyan, Antonoglou, Huang, Guez, Hubert, Baker, Lai, Bolton, et~al.}}}
\bibcite{simchowitz2019non}{{39}{2019}{{Simchowitz and Jamieson}}{{}}}
\bibcite{sutton2018reinforcement}{{40}{2018}{{Sutton and Barto}}{{}}}
\bibcite{sutton1998introduction}{{41}{1998}{{Sutton et~al.}}{{Sutton, Barto, et~al.}}}
\bibcite{tian2020provably}{{42}{2020}{{Tian et~al.}}{{Tian, Wang, Yu, and Sra}}}
\bibcite{wei2017online}{{43}{2017}{{Wei et~al.}}{{Wei, Hong, and Lu}}}
\bibcite{xie2020learning}{{44}{2020}{{Xie et~al.}}{{Xie, Chen, Wang, and Yang}}}
\bibcite{yu2009arbitrarily}{{45}{2009}{{Yu and Mannor}}{{}}}
\bibcite{zhang2019multi}{{46}{2019}{{Zhang et~al.}}{{Zhang, Yang, and Ba{\c {s}}ar}}}
\bibcite{zhang2020model}{{47}{2020{a}}{{Zhang et~al.}}{{Zhang, Kakade, Basar, and Yang}}}
\bibcite{zhang2020robust}{{48}{2020{b}}{{Zhang et~al.}}{{Zhang, Sun, Tao, Genc, Mallya, and Basar}}}
\bibcite{zimin2013online}{{49}{2013}{{Zimin and Neu}}{{}}}
\@writefile{toc}{\contentsline {section}{\numberline {A}Lower bound}{13}{appendix.A}\protected@file@percent }
\newlabel{sec:lbproof}{{A}{13}{Lower bound}{appendix.A}{}}
\@writefile{loe}{\contentsline {prp}{\ifthmt@listswap Proposition~5.1\else \numberline {5.1}Proposition\fi }{13}{prp.dummy.9}\protected@file@percent }
\newlabel{eq:v1}{{7}{13}{Lower bound}{equation.A.7}{}}
\newlabel{eq:v2}{{8}{13}{Lower bound}{equation.A.8}{}}
\newlabel{eq:v3}{{9}{13}{Lower bound}{equation.A.9}{}}
\citation{jaksch2010near,domingues2020episodic}
\citation{simchowitz2019non}
\citation{simchowitz2019non}
\@writefile{loe}{\contentsline {thm}{\ifthmt@listswap Theorem~5.1\else \numberline {5.1}Theorem\fi \thmtformatoptarg {Lower bound for online Turn-based Stochastic Game}}{14}{thm.dummy.11}\protected@file@percent }
\citation{feller1957introduction}
\citation{domingues2020episodic}
\newlabel{p:006}{{11}{15}{Lower bound}{equation.A.11}{}}
\newlabel{p:0009}{{12}{15}{Lower bound}{equation.A.12}{}}
\newlabel{p:0002}{{13}{15}{Lower bound}{equation.A.13}{}}
\newlabel{p:0003}{{14}{15}{Lower bound}{equation.A.14}{}}
\@writefile{toc}{\contentsline {section}{\numberline {B}Proof of Theorem \ref  {thm:tsgreg}}{17}{appendix.B}\protected@file@percent }
\newlabel{apx:proofalg}{{B}{17}{Proof of Theorem \ref {thm:tsgreg}}{appendix.B}{}}
\@writefile{loe}{\contentsline {thm}{\ifthmt@listswap Theorem~6.1\else \numberline {6.1}Theorem\fi \thmtformatoptarg {Regret of TMG-OPVI}}{17}{thm.dummy.14}\protected@file@percent }
\newlabel{eq:aterm}{{15}{17}{Proof of Theorem \ref {thm:tsgreg}}{equation.B.15}{}}
\newlabel{p:00010}{{16}{18}{Proof of Theorem \ref {thm:tsgreg}}{equation.B.16}{}}
\newlabel{p:201}{{23}{19}{Proof of Theorem \ref {thm:tsgreg}}{equation.B.23}{}}
\newlabel{p:202}{{24}{19}{Proof of Theorem \ref {thm:tsgreg}}{equation.B.24}{}}
\newlabel{p:203}{{25}{19}{Proof of Theorem \ref {thm:tsgreg}}{equation.B.25}{}}
\@writefile{toc}{\contentsline {section}{\numberline {C}Auxiliar lemmas}{20}{appendix.C}\protected@file@percent }
\newlabel{lemma:aux1}{{C.1}{20}{}{lemma.C.1}{}}
\newlabel{lemma:visitationonline}{{C.2}{20}{}{lemma.C.2}{}}
\newlabel{lemma:confint}{{C.3}{20}{}{lemma.C.3}{}}
\newlabel{LastPage}{{}{20}{}{page.20}{}}
\xdef\lastpage@lastpage{20}
\xdef\lastpage@lastpageHy{20}
\gdef \@abspage@last{20}
