% ------new citations--------

@inproceedings{liu2021cooperative,
  title={Cooperative exploration for multi-agent deep reinforcement learning},
  author={Liu, Iou-Jen and Jain, Unnat and Yeh, Raymond A and Schwing, Alexander},
  booktitle={International Conference on Machine Learning},
  pages={6826--6836},
  year={2021},
  organization={PMLR}
}

@article{cao2012overview,
  title={An overview of recent progress in the study of distributed multi-agent coordination},
  author={Cao, Yongcan and Yu, Wenwu and Ren, Wei and Chen, Guanrong},
  journal={IEEE Transactions on Industrial informatics},
  volume={9},
  number={1},
  pages={427--438},
  year={2012},
  publisher={IEEE}
}

@article{huttenrauch2017guided,
  title={Guided deep reinforcement learning for swarm systems},
  author={H{\"u}ttenrauch, Maximilian and {\v{S}}o{\v{s}}i{\'c}, Adrian and Neumann, Gerhard},
  journal={arXiv preprint arXiv:1709.06011},
  year={2017}
}

@article{taiga2019benchmarking,
  title={Benchmarking bonus-based exploration methods on the arcade learning environment},
  author={Ta{\"\i}ga, Adrien Ali and Fedus, William and Machado, Marlos C and Courville, Aaron and Bellemare, Marc G},
  journal={arXiv preprint arXiv:1908.02388},
  year={2019}
}

@article{ecoffet2021first,
  title={First return, then explore},
  author={Ecoffet, Adrien and Huizinga, Joost and Lehman, Joel and Stanley, Kenneth O and Clune, Jeff},
  journal={Nature},
  volume={590},
  number={7847},
  pages={580--586},
  year={2021},
  publisher={Nature Publishing Group}
}

@article{burda2018exploration,
  title={Exploration by random network distillation},
  author={Burda, Yuri and Edwards, Harrison and Storkey, Amos and Klimov, Oleg},
  journal={arXiv preprint arXiv:1810.12894},
  year={2018}
}

@inproceedings{pathak2019self,
  title={Self-supervised exploration via disagreement},
  author={Pathak, Deepak and Gandhi, Dhiraj and Gupta, Abhinav},
  booktitle={International conference on machine learning},
  pages={5062--5071},
  year={2019},
  organization={PMLR}
}

@inproceedings{pathak2017curiosity,
  title={Curiosity-driven exploration by self-supervised prediction},
  author={Pathak, Deepak and Agrawal, Pulkit and Efros, Alexei A and Darrell, Trevor},
  booktitle={International conference on machine learning},
  pages={2778--2787},
  year={2017},
  organization={PMLR}
}

@article{vitter1985random,
  title={Random sampling with a reservoir},
  author={Vitter, Jeffrey S},
  journal={ACM Transactions on Mathematical Software (TOMS)},
  volume={11},
  number={1},
  pages={37--57},
  year={1985},
  publisher={ACM New York, NY, USA}
}

@inproceedings{badanidiyuru2014streaming,
  title={Streaming submodular maximization: Massive data summarization on the fly},
  author={Badanidiyuru, Ashwinkumar and Mirzasoleiman, Baharan and Karbasi, Amin and Krause, Andreas},
  booktitle={Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining},
  pages={671--680},
  year={2014}
}

@inproceedings{schlegel2017adapting,
  title={Adapting kernel representations online using submodular maximization},
  author={Schlegel, Matthew and Pan, Yangchen and Chen, Jiecao and White, Martha},
  booktitle={International Conference on Machine Learning},
  pages={3037--3046},
  year={2017},
  organization={PMLR}
}

@article{pan2018organizing,
  title={Organizing experience: a deeper look at replay mechanisms for sample-based planning in continuous state domains},
  author={Pan, Yangchen and Zaheer, Muhammad and White, Adam and Patterson, Andrew and White, Martha},
  journal={International Joint Conference on Artificial Intelligence},
  year={2018}
}

% ^^^^^new citations^^^^^^

@inproceedings{kocsis2006bandit,
  title={Bandit based monte-carlo planning},
  author={Kocsis, Levente and Szepesv{\'a}ri, Csaba},
  booktitle={Machine Learning: ECML 2006: 17th European Conference on Machine Learning Berlin, Germany, September 18-22, 2006 Proceedings 17},
  pages={282--293},
  year={2006},
  organization={Springer}
}

@article{seow2009collaborative,
  title={A collaborative multiagent taxi-dispatch system},
  author={Seow, Kiam Tian and Dang, Nam Hai and Lee, Der-Horng},
  journal={IEEE Transactions on Automation science and engineering},
  volume={7},
  number={3},
  pages={607--616},
  year={2009},
  publisher={IEEE}
}

@article{perrusquia2021multi,
  title={Multi-agent reinforcement learning for redundant robot control in task-space},
  author={Perrusqu{\'\i}a, Adolfo and Yu, Wen and Li, Xiaoou},
  journal={International Journal of Machine Learning and Cybernetics},
  volume={12},
  pages={231--241},
  year={2021},
  publisher={Springer}
}

@article{shalev2016safe,
  title={Safe, multi-agent, reinforcement learning for autonomous driving},
  author={Shalev-Shwartz, Shai and Shammah, Shaked and Shashua, Amnon},
  journal={arXiv preprint arXiv:1610.03295},
  year={2016}
}

@article{lowe2017multi,
  title={Multi-agent actor-critic for mixed cooperative-competitive environments},
  author={Lowe, Ryan and Wu, Yi I and Tamar, Aviv and Harb, Jean and Pieter Abbeel, OpenAI and Mordatch, Igor},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@article{yu2021surprising,
  title={The surprising effectiveness of ppo in cooperative, multi-agent games},
  author={Yu, Chao and Velu, Akash and Vinitsky, Eugene and Wang, Yu and Bayen, Alexandre and Wu, Yi},
  journal={arXiv preprint arXiv:2103.01955},
  year={2021}
}

@book{oliehoek2016concise,
  title={A concise introduction to decentralized POMDPs},
  author={Oliehoek, Frans A and Amato, Christopher},
  year={2016},
  publisher={Springer}
}

@article{bernstein2002complexity,
  title={The complexity of decentralized control of Markov decision processes},
  author={Bernstein, Daniel S and Givan, Robert and Immerman, Neil and Zilberstein, Shlomo},
  journal={Mathematics of operations research},
  volume={27},
  number={4},
  pages={819--840},
  year={2002},
  publisher={INFORMS}
}

@article{oliehoek2008optimal,
  title={Optimal and approximate Q-value functions for decentralized POMDPs},
  author={Oliehoek, Frans A and Spaan, Matthijs TJ and Vlassis, Nikos},
  journal={Journal of Artificial Intelligence Research},
  volume={32},
  pages={289--353},
  year={2008}
}

@inproceedings{son2019qtran,
  title={Qtran: Learning to factorize with transformation for cooperative multi-agent reinforcement learning},
  author={Son, Kyunghwan and Kim, Daewoo and Kang, Wan Ju and Hostallero, David Earl and Yi, Yung},
  booktitle={International conference on machine learning},
  pages={5887--5896},
  year={2019},
  organization={PMLR}
}

@article{sunehag2017value,
  title={Value-decomposition networks for cooperative multi-agent learning},
  author={Sunehag, Peter and Lever, Guy and Gruslys, Audrunas and Czarnecki, Wojciech Marian and Zambaldi, Vinicius and Jaderberg, Max and Lanctot, Marc and Sonnerat, Nicolas and Leibo, Joel Z and Tuyls, Karl and others},
  journal={arXiv preprint arXiv:1706.05296},
  year={2017}
}

@inproceedings{rashid2018qmix,
  title={Qmix: Monotonic value function factorisation for deep multi-agent reinforcement learning},
  author={Rashid, Tabish and Samvelyan, Mikayel and Schroeder, Christian and Farquhar, Gregory and Foerster, Jakob and Whiteson, Shimon},
  booktitle={International conference on machine learning},
  pages={4295--4304},
  year={2018},
  organization={PMLR}
}

@article{wang2020qplex,
  title={Qplex: Duplex dueling multi-agent q-learning},
  author={Wang, Jianhao and Ren, Zhizhou and Liu, Terry and Yu, Yang and Zhang, Chongjie},
  journal={arXiv preprint arXiv:2008.01062},
  year={2020}
}

@article{yang2020qatten,
  title={Qatten: A general framework for cooperative multiagent reinforcement learning},
  author={Yang, Yaodong and Hao, Jianye and Liao, Ben and Shao, Kun and Chen, Guangyong and Liu, Wulong and Tang, Hongyao},
  journal={arXiv preprint arXiv:2002.03939},
  year={2020}
}

@inproceedings{jaques2019social,
  title={Social influence as intrinsic motivation for multi-agent deep reinforcement learning},
  author={Jaques, Natasha and Lazaridou, Angeliki and Hughes, Edward and Gulcehre, Caglar and Ortega, Pedro and Strouse, DJ and Leibo, Joel Z and De Freitas, Nando},
  booktitle={International conference on machine learning},
  pages={3040--3049},
  year={2019},
  organization={PMLR}
}

@article{mahajan2019maven,
  title={Maven: Multi-agent variational exploration},
  author={Mahajan, Anuj and Rashid, Tabish and Samvelyan, Mikayel and Whiteson, Shimon},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@article{wang2019influence,
  title={Influence-based multi-agent exploration},
  author={Wang, Tonghan and Wang, Jianhao and Wu, Yi and Zhang, Chongjie},
  journal={arXiv preprint arXiv:1910.05512},
  year={2019}
}

@article{kim2020maximum,
  title={A maximum mutual information framework for multi-agent reinforcement learning},
  author={Kim, Woojun and Jung, Whiyoung and Cho, Myungsik and Sung, Youngchul},
  journal={arXiv preprint arXiv:2006.02732},
  year={2020}
}

@article{li2021celebrating,
  title={Celebrating diversity in shared multi-agent reinforcement learning},
  author={Li, Chenghao and Wang, Tonghan and Wu, Chengjie and Zhao, Qianchuan and Yang, Jun and Zhang, Chongjie},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={3991--4002},
  year={2021}
}

@article{li2022pmic,
  title={PMIC: Improving Multi-Agent Reinforcement Learning with Progressive Mutual Information Collaboration},
  author={Li, Pengyi and Tang, Hongyao and Yang, Tianpei and Hao, Xiaotian and Sang, Tong and Zheng, Yan and Hao, Jianye and Taylor, Matthew E and Wang, Zhen},
  journal={arXiv preprint arXiv:2203.08553},
  year={2022}
}

@inproceedings{mordatch2018emergence,
  title={Emergence of grounded compositional language in multi-agent populations},
  author={Mordatch, Igor and Abbeel, Pieter},
  booktitle={Proceedings of the AAAI conference on artificial intelligence},
  volume={32},
  number={1},
  year={2018}
}

@article{albrecht2015game,
  title={A game-theoretic model and best-response learning method for ad hoc coordination in multiagent systems},
  author={Albrecht, Stefano V and Ramamoorthy, Subramanian},
  journal={arXiv preprint arXiv:1506.01170},
  year={2015}
}

@article{albrecht2019reasoning,
  title={Reasoning about hypothetical agent behaviours and their parameters},
  author={Albrecht, Stefano V and Stone, Peter},
  journal={arXiv preprint arXiv:1906.11064},
  year={2019}
}

@article{christianos2020shared,
  title={Shared experience actor-critic for multi-agent reinforcement learning},
  author={Christianos, Filippos and Sch{\"a}fer, Lukas and Albrecht, Stefano},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={10707--10717},
  year={2020}
}

@article{samvelyan2019starcraft,
  title={The starcraft multi-agent challenge},
  author={Samvelyan, Mikayel and Rashid, Tabish and De Witt, Christian Schroeder and Farquhar, Gregory and Nardelli, Nantas and Rudner, Tim GJ and Hung, Chia-Man and Torr, Philip HS and Foerster, Jakob and Whiteson, Shimon},
  journal={arXiv preprint arXiv:1902.04043},
  year={2019}
}

@article{amin2021survey,
  title={A survey of exploration methods in reinforcement learning},
  author={Amin, Susan and Gomrokchi, Maziar and Satija, Harsh and van Hoof, Herke and Precup, Doina},
  journal={arXiv preprint arXiv:2109.00157},
  year={2021}
}

@article{bellemare2016unifying,
  title={Unifying count-based exploration and intrinsic motivation},
  author={Bellemare, Marc and Srinivasan, Sriram and Ostrovski, Georg and Schaul, Tom and Saxton, David and Munos, Remi},
  journal={Advances in neural information processing systems},
  volume={29},
  year={2016}
}

@inproceedings{ostrovski2017count,
  title={Count-based exploration with neural density models},
  author={Ostrovski, Georg and Bellemare, Marc G and Oord, A{\"a}ron and Munos, R{\'e}mi},
  booktitle={International conference on machine learning},
  pages={2721--2730},
  year={2017},
  organization={PMLR}
}

@article{tang2017exploration,
  title={\# exploration: A study of count-based exploration for deep reinforcement learning},
  author={Tang, Haoran and Houthooft, Rein and Foote, Davis and Stooke, Adam and Xi Chen, OpenAI and Duan, Yan and Schulman, John and DeTurck, Filip and Abbeel, Pieter},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@article{houthooft2016vime,
  title={Vime: Variational information maximizing exploration},
  author={Houthooft, Rein and Chen, Xi and Duan, Yan and Schulman, John and De Turck, Filip and Abbeel, Pieter},
  journal={Advances in neural information processing systems},
  volume={29},
  year={2016}
}

@article{osband2016deep,
  title={Deep exploration via bootstrapped DQN},
  author={Osband, Ian and Blundell, Charles and Pritzel, Alexander and Van Roy, Benjamin},
  journal={Advances in neural information processing systems},
  volume={29},
  year={2016}
}

@article{chen2021variational,
  title={Variational automatic curriculum learning for sparse-reward cooperative multi-agent problems},
  author={Chen, Jiayu and Zhang, Yuanxin and Xu, Yuanfan and Ma, Huimin and Yang, Huazhong and Song, Jiaming and Wang, Yu and Wu, Yi},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={9681--9693},
  year={2021}
}

@article{zheng2021episodic,
  title={Episodic multi-agent reinforcement learning with curiosity-driven exploration},
  author={Zheng, Lulu and Chen, Jiarui and Wang, Jianhao and He, Jiamin and Hu, Yujing and Chen, Yingfeng and Fan, Changjie and Gao, Yang and Zhang, Chongjie},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={3757--3769},
  year={2021}
}

@article{kuba2021trust,
  title={Trust region policy optimisation in multi-agent reinforcement learning},
  author={Kuba, Jakub Grudzien and Chen, Ruiqing and Wen, Muning and Wen, Ying and Sun, Fanglei and Wang, Jun and Yang, Yaodong},
  journal={arXiv preprint arXiv:2109.11251},
  year={2021}
}

@article{li2022ace,
  title={ACE: Cooperative Multi-agent Q-learning with Bidirectional Action-Dependency},
  author={Li, Chuming and Liu, Jie and Zhang, Yinmin and Wei, Yuhong and Niu, Yazhe and Yang, Yaodong and Liu, Yu and Ouyang, Wanli},
  journal={arXiv preprint arXiv:2211.16068},
  year={2022}
}

@article{wen2022multi,
  title={Multi-agent reinforcement learning is a sequence modeling problem},
  author={Wen, Muning and Kuba, Jakub Grudzien and Lin, Runji and Zhang, Weinan and Wen, Ying and Wang, Jun and Yang, Yaodong},
  journal={arXiv preprint arXiv:2205.14953},
  year={2022}
}

@article{wang2022more,
  title={More Centralized Training, Still Decentralized Execution: Multi-Agent Conditional Policy Factorization},
  author={Wang, Jiangxing and Ye, Deheng and Lu, Zongqing},
  journal={arXiv preprint arXiv:2209.12681},
  year={2022}
}

@book{ziebart2010modeling,
  title={Modeling purposeful adaptive behavior with the principle of maximum causal entropy},
  author={Ziebart, Brian D},
  year={2010},
  publisher={Carnegie Mellon University}
}

@inproceedings{tan1993multi,
  title={Multi-agent reinforcement learning: Independent vs. cooperative agents},
  author={Tan, Ming},
  booktitle={Proceedings of the tenth international conference on machine learning},
  pages={330--337},
  year={1993}
}

@article{auer2002finite,
  title={Finite-time analysis of the multiarmed bandit problem},
  author={Auer, Peter and Cesa-Bianchi, Nicolo and Fischer, Paul},
  journal={Machine learning},
  volume={47},
  pages={235--256},
  year={2002},
  publisher={Springer}
}

@article{papoudakis2020benchmarking,
  title={Benchmarking multi-agent deep reinforcement learning algorithms in cooperative tasks},
  author={Papoudakis, Georgios and Christianos, Filippos and Sch{\"a}fer, Lukas and Albrecht, Stefano V},
  journal={arXiv preprint arXiv:2006.07869},
  year={2020}
}

@article{snedecor1980statistical,
  title={Statistical methods. IOWA},
  author={Snedecor, George W and Cochran, William G},
  journal={Iowa State University Press. Starkstein, SE, \& Robinson, RG (1989). Affective disorders and cerebral vascular disease. The British Journal of Psychiatry},
  volume={154},
  pages={170--182},
  year={1980}
}

@article{jin2018q,
  title={Is Q-learning provably efficient?},
  author={Jin, Chi and Allen-Zhu, Zeyuan and Bubeck, Sebastien and Jordan, Michael I},
  journal={Advances in neural information processing systems},
  volume={31},
  year={2018}
}

@inproceedings{jin2020provably,
  title={Provably efficient reinforcement learning with linear function approximation},
  author={Jin, Chi and Yang, Zhuoran and Wang, Zhaoran and Jordan, Michael I},
  booktitle={Conference on Learning Theory},
  pages={2137--2143},
  year={2020},
  organization={PMLR}
}

@article{yang2020function,
  title={On function approximation in reinforcement learning: Optimism in the face of large state spaces},
  author={Yang, Zhuoran and Jin, Chi and Wang, Zhaoran and Wang, Mengdi and Jordan, Michael I},
  journal={arXiv preprint arXiv:2011.04622},
  year={2020}
}

@article{rashid2020optimistic,
  title={Optimistic exploration even with a pessimistic initialisation},
  author={Rashid, Tabish and Peng, Bei and Boehmer, Wendelin and Whiteson, Shimon},
  journal={arXiv preprint arXiv:2002.12174},
  year={2020}
}