@article{MT05,
    title={On the Empirical State-Action Frequencies in {Markov} Decision Processes Under General Policies},
    author={Shie Mannor and John N. Tsitsiklis},
    year={2005},
    journal={Mathematics of Operations Research},
    volume={30},
    pages={545-561}
}

@book{MS06,
    title={Repeated Games and Reputations: Long-Run Relationships},
    author={George Mailath and Larry Samuelson},
    year={2006},
    publisher={Oxford University Press}
}

@article{Knight2018,
  doi = {10.21105/joss.00904},
  url = {https://doi.org/10.21105/joss.00904},
  year = {2018},
  publisher = {The Open Journal},
  volume = {3},
  number = {30},
  pages = {904},
  author = {Vincent Knight and James Campbell},
  title = {Nashpy: A {Python} library for the computation of {Nash} equilibria},
  journal = {Journal of Open Source Software}
}

@misc{CR21,
    title={Towards cooperation in learning games},
    author={Jesse Clifton and Maxime Riché},
    year={2021}
}

@article{GS15, author = {Ganzfried, Sam and Sandholm, Tuomas}, title = {Safe Opponent Exploitation}, year = {2015}, issue_date = {April 2015}, publisher = {Association for Computing Machinery}, address = {New York, NY, USA}, volume = {3}, number = {2}, issn = {2167-8375}, url = {https://doi.org/10.1145/2716322}, doi = {10.1145/2716322}, abstract = {We consider the problem of playing a repeated two-player zero-sum game safety: that is, guaranteeing at least the value of the game per period in expectation regardless of the strategy used by the opponent. Playing a stage-game equilibrium strategy at each time step clearly guarantees safety, and prior work has (incorrectly) stated that it is impossible to simultaneously deviate from a stage-game equilibrium (in hope of exploiting a suboptimal opponent) and to guarantee safety. We show that such profitable deviations are indeed possible specifically in games where certain types of “gift” strategies exist, which we define formally. We show that the set of strategies constituting such gifts can be strictly larger than the set of iteratively weakly-dominated strategies; this disproves another recent assertion which states that all noniteratively weakly dominated strategies are best responses to each equilibrium strategy of the other player. We present a full characterization of safe strategies, and develop efficient algorithms for exploiting suboptimal opponents while guaranteeing safety. We also provide analogous results for extensive-form games of perfect and imperfect information, and present safe exploitation algorithms and full characterizations of safe strategies for those settings as well. We present experimental results in Kuhn poker, a canonical test problem for game-theoretic algorithms. Our experiments show that (1) aggressive safe exploitation strategies significantly outperform adjusting the exploitation within stage-game equilibrium strategies only and (2) all the safe exploitation strategies significantly outperform a (nonsafe) best response strategy against strong dynamic opponents.}, journal = {ACM Trans. Econ. Comput.}, month = {apr}, articleno = {8}, numpages = {28}, keywords = {opponent exploitation, multiagent learning, Game theory} }

@misc{SLRC21,
      title={Normative Disagreement as a Challenge for Cooperative {AI}}, 
      author={Julian Stastny and Maxime Riché and Alexander Lyzhov and Johannes Treutlein and Allan Dafoe and Jesse Clifton},
      year={2021},
      eprint={2111.13872},
      archivePrefix={arXiv},
      primaryClass={cs.MA}
}

@article{JOA0,
    title={Near-optimal Regret Bounds for Reinforcement Learning},
    author={Thomas Jaksch and Ronald Ortner and Peter Auer},
    year={2010},
    journal={Journal of Machine Learning Research},
    volume={11},
    pages={1563-1600}
}

@article{LM12,
    title={Computing Optimal Strategies to Commit to in Stochastic Games},
    author={Joshua Letchford and Liam MacDermed and Vincent Conitzer and Ronald Parr and Charles L. Isbell},
    year={2012},
    journal={Proceedings of the Twenty-Sixth AAAI Conference on Artificial Intelligence}
}

@article{HK17,
    title={Learning against sequential opponents in repeated stochastic games},
    author={Pablo Hernandez-Leal and Michael Kaisers},
    year={2017},
    journal={The 3rd Multidisciplinary Conference on Reinforcement Learning and Decision Making}}
    
@article{LS01,
    title={Implicit Negotiation in Repeated Games},
    author={Michael Littman and Peter Stone},
    year={2001},
    journal={Revised Papers from the 8th International Workshop on Intelligent Agents VIII}
}

@article{CS06,
    title={{AWESOME}: A general multiagent learning algorithm that converges in self-play and learns a best response against stationary opponents},
    author={Vincent Conitzer and Tuomas Sandholm},
    year={2006},
    journal={Machine Learning}
}

@article{F17,
  author    = {Jakob N. Foerster and
               Richard Y. Chen and
               Maruan Al{-}Shedivat and
               Shimon Whiteson and
               Pieter Abbeel and
               Igor Mordatch},
  title     = {Learning with Opponent-Learning Awareness},
  journal   = {Proc. of the 17th International Conference on Autonomous Agents and Multiagent Systems},
  year      = {2017}
}

@article{H14,
    title={Using a Priori Information for Fast Learning Against Non-stationary Opponents},
    author={Pablo Hernandez-Leal and Enrique Munoz de Cote and L. Enrique Sucar},
    year={2014},
    journal={Ibero-American Conference on Artificial Intelligence}
}

@article{H16a,
    title={Efficiently detecting switches against non-stationary opponents},
    author={Pablo Hernandez-Leal and Yusen Zhan and Matthew E. Taylor and L. Enrique Sucar and Enrique Munoz de Cote},
    year={2016},
    journal={Autonomous Agents and Multi-Agent Systems}
}

@article{H16b,
    title={Identifying and Tracking Switching, Non-Stationary Opponents: A {Bayesian} Approach},
    author={Pablo Hernandez-Leal and Matthew E. Taylor and Benjamin Rosman and L. Enrique Sucar and Enrique Munoz de Cote},
    year={2016},
    journal={The Workshops of the Thirtieth AAAI Conference on Artificial Intelligence Multiagent Interaction without Prior Coordination}
}

@article{C14,
    title={Towards Minimizing Disappointment in Repeated Games},
    author={Jacob W. Crandall},
    year={2014},
    journal={Journal of Artificial Intelligence Research},
    volume={49},
    pages={111-142}
}

@article{A20,
      title={Model-Based Reinforcement Learning with Value-Targeted Regression}, 
      author={Alex Ayoub and Zeyu Jia and Csaba Szepesvari and Mengdi Wang and Lin F. Yang},
      year={2020},
      journal={Proceedings of the 2nd Conference on Learning for Dynamics and Control}
}

@article{RD19,
      title={Learning to Collaborate in {Markov} Decision Processes}, 
      author={Goran Radanovic and Rati Devidze and David C. Parkes and Adish Singla},
      year={2019},
      journal={Proceedings of the 36th International Conference on Machine Learning}
}

@book{A84,
    title={The Evolution of Cooperation},
    author={Robert Axelrod},
    year={1984},
    publisher={Basic Books}
}

@article{YM09,
    title={Online Learning in {Markov} Decision Processes with Arbitrarily Changing Rewards and Transitions},
    author={Jia Yuan Yu and Shie Mannor},
    year={2009},
    journal={International Conference on Game Theory for Networks}
}

@article{T18,
      title={Scalar Posterior Sampling with Applications}, 
      author={Georgios Theocharous and Zheng Wen and Yasin Abbasi-Yadkori and Nikos Vlassis},
      year={2018},
      journal={32nd Conference on Neural Information Processing Systems}
}

@article{O17,
      title={Learning Unknown {Markov} Decision Processes: A {Thompson} Sampling Approach}, 
      author={Yi Ouyang and Mukul Gagrani and Ashutosh Nayyar and Rahul Jain},
      year={2017},
      journal={31st Conference on Neural Information Processing Systems}
}

@article{ER18,
    title={Learning Against Non-Stationary Agents with Opponent Modelling and Deep Reinforcement Learning},
    author={Richard Everett and Stephen Roberts},
    year={2018},
    journal={Association for the Advancement of Artificial Intelligence}
}

@book{puterman,
    title={Markov Decision Processes: Discrete Stochastic Dynamic Programming},
    author={Martin L. Puterman},
    year={1994},
    publisher={John Wiley \& Sons, Inc.},
    address={New York, NY, USA}
}

@article{T33,
    title={On the likelihood that one unknown probability exceeds another in view of the evidence of two samples},
    author={William R. Thompson},
    journal={Biometrika},
    year={1933},
    volume={24},
    edition={3-4},
    pages={285--294}
}

@article{E14,
    title={Fast Adaptive Learning in Repeated Stochastic Games by Game Abstraction},
    author={Mohamed Elidrisi and Nicholas Johnson and Maria Gini and Jacob Crandall},
    journal={Proceedings of the 13th International Conference on Autonomous Agents and Multiagent Systems},
    year={2014}
}

@article{P18,
      title={Concentration inequalities for {Markov} chains by {Marton} couplings and spectral methods}, 
      author={Daniel Paulin},
      year={2015},
      journal={Electronic Journal of Probability},
      volume={20},
      pages={1-32}
}

@article{W17,
      title={Online Reinforcement Learning in Stochastic Games}, 
      author={Chen-Yu Wei and Yi-Te Hong and Chi-Jen Lu},
      year={2017},
      journal={31st Conference on Neural Information Processing Systems}
}

@InProceedings{F20, title = {{A Theoretical Analysis of Deep Q-Learning}}, author = {J. Fan and Z. Wang and Y. Xie and Z. Yang}, pages = {486--489}, year = {2020}, editor = {Alexandre M. Bayen and Ali Jadbabaie and George Pappas and Pablo A. Parrilo and Benjamin Recht and Claire Tomlin and Melanie Zeilinger}, volume = {120}, series = {Proceedings of Machine Learning Research}, address = {The Cloud}, publisher = {PMLR}}

@article{B20,
      title={Provable Self-Play Algorithms for Competitive Reinforcement Learning}, 
      author={Y. Bai and C. Jin},
      year={2020},
      journal={Proceedings of the 37th International Conference on Machine Learning}
}

@article{MA20,
      title={Regret Bounds for Decentralized Learning in Cooperative Multi-Agent Dynamical Systems}, 
      author={S. M. Asghari and Y. Ouyang and A. Nayyar},
      year={2020},
      journal={Proceedings of the 36th Conference on Uncertainty in Artificial Intelligence}
}

@article{HB16,
      title={Opponent Modeling in Deep Reinforcement Learning}, 
      author={He He and Jordan Boyd-Graber and Kevin Kwok and Hal Daumé III},
      year={2016},
      journal={Proceedings of The 33rd International Conference on Machine Learning}
}

@article{L19,
      title={Stable Opponent Shaping in Differentiable Games}, 
      author={Alistair Letcher and Jakob Foerster and David Balduzzi and Tim Rocktäschel and Shimon Whiteson},
      year={2019},
      journal={},
      eprint={1811.08469},
      archivePrefix={arXiv},
      primaryClass={cs.MA}
}

@article{T11,
      title={Freedman's inequality for matrix martingales}, 
      author={Joel A. Tropp},
      year={2011},
      journal={Electronic Communications in Probability}
}

@article{T20,
      title={Towards Learning Multi-agent Negotiations via Self-Play}, 
      author={Y. C. Tang},
      year={2019},
      journal={ICCV Workshop}
}

@article{TD20,
    title={A Novel Individually Rational Objective In Multi-Agent Multi-Armed Bandits: Algorithms and Regret Bounds},
    author={Aristide C.Y. Tossou and Christos Dimitrakakis and Jaroslaw Rzepecki and Katja Hofmann},
    year={2020},
    journal={Proc. of the 19th International Conference on Autonomous Agents and Multiagent Systems}
}

@article{PS04,
    title={New Criteria and a New Algorithm for Learning in Multi-Agent Systems},
    author={Rob Powers and Yoav Shoham},
    year={2004},
    journal={Neural Information Processing Systems}
}

@article{DM17,
    title={Safely using predictions in general-sum normal form games},
    author={Steven Damer and Maria Gini},
    year={2017},
    journal={Proc. of the 16th International Conference on Autonomous Agents and Multiagent Systems}
}

@article{A19,
      title={An Empirical Study on the Practical Impact of Prior Beliefs over Policy Types}, 
      author={Stefano V. Albrecht and Jacob W. Crandall and Subramanian Ramamoorthy},
      year={2015},
      journal={Proceedings of the Twenty-Ninth AAAI Conference on Artificial Intelligence}
}

@book{NS03,
    editor={Abraham Neyman and S. Sorin},
    title={Stochastic Games and Applications},
    year={2003},
    publisher={Springer Netherlands}
}

@article{C20,
    title={When autonomous agents model other agents: An appeal for altered judgment coupled with mouths, ears, and a little more tape},
    author={Jacob W. Crandall},
    year={2020},
    journal={Artificial Intelligence}
}

@article{A01,
    title={Sharp inequalities for the beta function},
    author={Horst Alzer},
    year={2001},
    journal={Indagationes Mathematicae}
}

@article{M17,
   title={On the sub-Gaussianity of the {Beta} and {Dirichlet} distributions},
   volume={22},
   ISSN={1083-589X},
   url={http://dx.doi.org/10.1214/17-ECP92},
   DOI={10.1214/17-ecp92},
   number={0},
   journal={Electronic Communications in Probability},
   publisher={Institute of Mathematical Statistics},
   author={Marchal, Olivier and Arbel, Julyan},
   year={2017}
}

@article {Press10409,
	author = {Press, William H. and Dyson, Freeman J.},
	title = {Iterated Prisoner{\textquoteright}s Dilemma contains strategies that dominate any evolutionary opponent},
	volume = {109},
	number = {26},
	pages = {10409--10413},
	year = {2012},
	doi = {10.1073/pnas.1206569109},
	publisher = {National Academy of Sciences},
	abstract = {The two-player Iterated Prisoner{\textquoteright}s Dilemma game is a model for both sentient and evolutionary behaviors, especially including the emergence of cooperation. It is generally assumed that there exists no simple ultimatum strategy whereby one player can enforce a unilateral claim to an unfair share of rewards. Here, we show that such strategies unexpectedly do exist. In particular, a player X who is witting of these strategies can (i) deterministically set her opponent Y{\textquoteright}s score, independently of his strategy or response, or (ii) enforce an extortionate linear relation between her and his scores. Against such a player, an evolutionary player{\textquoteright}s best response is to accede to the extortion. Only a player with a theory of mind about his opponent can do better, in which case Iterated Prisoner{\textquoteright}s Dilemma is an Ultimatum Game.},
	issn = {0027-8424},
	URL = {https://www.pnas.org/content/109/26/10409},
	eprint = {https://www.pnas.org/content/109/26/10409.full.pdf},
	journal = {Proceedings of the National Academy of Sciences}
}

@article{SPtournament,
    title={Extortion and cooperation in the Prisoner’s Dilemma},
    author={Alexander J. Stewart and Joshua B. Plotkin},
    year={2012},
    journal={Proceedings of the National Academy of Sciences},
    volume={109},
    page={10134-10135}
}

@article{CO18,
    title={Cooperating with machines},
    author={Jacob W. Crandall and Mayada Oudah and Tennom and Fatimah Ishowo-Oloko and Sherief Abdallah and Jean-François Bonnefon and Manuel Cebrian and Azim Shariff and Michael A. Goodrich and Iyad Rahwan},
    year={2018},
    journal={Nature Communications},
    volume={9},
    pages={233}
}

@article{IG20,
    title={Towards Cooperating in Repeated Interactions Without Repeating Structure},
    author={Huy Pham},
    year={2020},
    journal={Theses and Dissertations}
}

@ARTICLE{NBS,
title = {The Bargaining Problem},
author = {Nash, John},
year = {1950},
journal = {Econometrica},
volume = {18},
number = {2},
pages = {155-162},
url = {https://EconPapers.repec.org/RePEc:ecm:emetrp:v:18:y:1950:i:2:p:155-162}
}

@Inbook{insistent,
author="Roger B. Myerson",
title="Bargaining and Cooperation in
Two-Person Games",
bookTitle="Game Theory: Analysis of Conflict",
year="1991",
publisher="Harvard University Press",
pages="370--416"
}

@article{NCIRL19,
 author = {Zhang, Xiangyuan and Zhang, Kaiqing and Miehling, Erik and Basar, Tamer},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Non-Cooperative Inverse Reinforcement Learning},
 url = {https://proceedings.neurips.cc/paper/2019/file/56bd37d3a2fda0f2f41925019c81011d-Paper.pdf},
 volume = {32},
 year = {2019}
}

@article{BT04,
    title={Efficient learning equilibrium},
    author={Ronen I. Brafman and Moshe Tennenholtz},
    year={2004},
    journal={Artificial Intelligence},
    volume={159},
    pages={27-47}
}

@incollection{fictplay,
    author={Brown, George W.},
    year={1951},
    title={Iterative Solutions of Games by Fictitious Play},
    booktitle={Activity Analysis of Production and Allocation},
    editor={T. C. Koopmans},
    city={New York},
    publisher={Wiley}
}

@InProceedings{fcl,
  title = 	 {Foolproof Cooperative Learning},
  author =       {Jacq, Alexis and Perolat, Julien and Geist, Matthieu and Pietquin, Olivier},
  booktitle = 	 {Proceedings of The 12th Asian Conference on Machine Learning},
  pages = 	 {401--416},
  year = 	 {2020},
  editor = 	 {Sinno Jialin Pan and Masashi Sugiyama},
  volume = 	 {129},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Bangkok, Thailand},
  month = 	 {18--20 Nov},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v129/jacq20a/jacq20a.pdf},
  url = 	 {http://proceedings.mlr.press/v129/jacq20a.html},
  abstract = 	 {This paper extends the notion of learning algorithms and learning equilibriums from repeated games theory to stochastic games. We introduce Foolproof Cooperative Learning (FCL), an algorithm that converges to an equilibrium strategy that allows cooperative strategies in self-play setting while being not exploitable by selfish learners. By construction, FCL is a learning equilibrium for repeated symmetric games. We illustrate the behavior of FCL on symmetric matrix and grid games, and its robustness to selfish learners. }
}

@InProceedings{optQ, title = {Model-free Reinforcement Learning in Infinite-horizon Average-reward {Markov} Decision Processes}, author = {Chen-Yu Wei and Mehdi Jafarnia-Jahromi and Haipeng Luo and Hiteshi Sharma and Rahul Jain}, year = {2020}, volume = {119}, series = {Proceedings of the 37th International Conference on Machine Learning}}

@misc{topology,
    title={Navigating the topology of 2x2 games: an introductory note on payoff families, normalization, and natural order},
    author={Bryan Bruns},
    year={2010},
    eprint={1010.4727},
    archivePrefix={arXiv},
    url={https://arxiv.org/abs/1010.4727}
}

@article{WD92,
    title={Q-learning},
    author={Chris J. Watkins and Peter Dayan},
    journal={Machine Learning},
    volume={8},
    pages={279-292},
    year={1992}
}

@article{EEE04,
    title={Exploration-Exploitation Tradeoffs for Experts Algorithms in Reactive Environments},
    author={D. de Farias and N. Megiddo},
    year={2004},
    journal={Advances in Neural Information Processing Systems 17}
}

@article{LS05,
    title={A Polynomial-time {Nash} Equilibrium Algorithm for Repeated Games},
    author={Michael L. Littman and Peter Stone},
    year={2005},
    journal={Decision Support Systems},
    volume={39},
    pages={55-66}
}

@article{Rao19,
    title={{A Hoeffding inequality for Markov chains}},
    author={Shravas Rao},
    year={2019},
    journal={Electronic Communications in Probability},
    volume={24},
    pages={1-11}
}

@article{CG10,
    title={Learning to compete, coordinate, and cooperate in repeated games using reinforcement learning},
    author={Jacob W. Crandall and Michael A. Goodrich},
    year={2010},
    journal={Machine Learning},
    volume={82},
    pages={281-314}
}

@article{PS05, author = {Powers, Rob and Shoham, Yoav},
title = {Learning against Opponents with Bounded Memory},
year = {2005},  journal = {Proceedings of the 19th International Joint Conference on Artificial Intelligence}}

@article{ICML10-chakraborty,
	author    = "Doran Chakraborty and Peter Stone",
	title     = {Convergence, Targeted Optimality and Safety in Multiagent Learning},
	journal = "Proceedings of the Twenty-seventh International Conference on Machine Learning (ICML)",
	location  = "Haifa, Israel",
	year      = "2010",
	abstract  = {
		This paper introduces a novel multiagent learning algorithm which
		achieves convergence, targeted optimality against memory-bounded
		adversaries, and safety, in arbitrary repeated games.  Called CMLeS, its
		most novel aspect is the manner in which it guarantees (in a PAC sense)
		targeted optimality against memory-bounded adversaries, via efficient
		exploration and exploitation. CMLeS is fully implemented and we present
		empirical results demonstrating its effectiveness.
	},
}


@article{C15,
  author    = {Jacob W. Crandall},
  title     = {Robust Learning for Repeated Stochastic Games via Meta-Gaming},
  journal   = {Proceedings of the Twenty-Fourth International Joint Conference on Artificial Intelligence},
  year      = {2015}
}

@article{O20,
    author={Ronald Ortner},
    title={Regret Bounds for Reinforcement Learning via {Markov} Chain Concentration},
    journal={Journal of Artificial Intelligence Research},
    year={2020}
}