@comment{article{aimag,
author = {Eck, Adam and Soh, Leen-Kiat and Doshi, Prashant},
title = {Decision making in open agent systems},
journal = {AI Magazine},
volume = {44},
number = {4},
pages = {508-523},
doi = {https://doi.org/10.1002/aaai.12131},
url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/aaai.12131},
eprint = {https://onlinelibrary.wiley.com/doi/pdf/10.1002/aaai.12131},
abstract = {Abstract In many real-world applications of AI, the set of actors and tasks are not constant, but instead change over time. Robots tasked with suppressing wildfires eventually run out of limited suppressant resources and need to temporarily disengage from the collaborative work in order to recharge, or they might become damaged and leave the environment permanently. In a large business organization, objectives and goals change with the market, requiring workers to adapt to perform different sets of tasks across time. We call these multiagent systems (MAS) open agent systems (OASYS), and the openness of the sets of agents and tasks necessitates new capabilities and modeling for decision making compared to planning and learning in closed environments. In this article, we discuss three notions of openness: agent openness, task openness, and type openness. We also review the past and current research on addressing the novel challenges brought about by openness in OASYS. We share lessons learned from these efforts and suggest directions for promising future work in this area. We also encourage the community to engage and participate in this area of MAS research to address critical real-world problems in the application of AI to enhance our daily lives.},
year = {2023}
}}

@comment{Sycara, K. (1998). , AI Magazine, 19(2), pp. 79-92.}

@manual{USFARiskManagement,
  title        = {Risk Management Practices},
  author       = {{U.S. Fire Admin}},
  year= {2018},
  url = {https://www.usfa.fema.gov/downloads/pdf/publications/risk_management_practices.pdf},
  organization = {Federal Emergency Management Agency},
  note = {Accessed: 1/15/25}
}

@inproceedings{10.5555/1029693.1029710,
 author = {Boutilier, Craig},
 booktitle = {TARK '96},
 pages = {195–210},
 title = {Planning, learning and coordination in multiagent decision processes},
 year = {1996}
}

@inproceedings{Abebe:2022,
 author = {Abebe, R. and Eck, A. and Ikeokwu, C. and Taggart, S.},
 booktitle = {AAAI’2022},
 pages = {4744-4751},
 title = {An Algorithmic Introduction to Savings Circles},
 year = {2022}
}
@misc{bergami2016joininggraphs,
      title={On Joining Graphs}, 
      author={Giacomo Bergami and Matteo Magnani and Danilo Montesi},
      year={2016},
      eprint={1608.05594},
      archivePrefix={arXiv},
      primaryClass={cs.DB},
      url={https://arxiv.org/abs/1608.05594}, 
}

@inproceedings{Abel18:Policy,
 author = {Abel, David and Jinnai, Yuu and Guo, Sophie Yue and Konidaris, George and Littman, Michael},
 booktitle = {ICML},
 pages = {20--29},
 publisher = {PMLR},
 title = {Policy and Value Transfer in Lifelong Reinforcement Learning},
 year = {2018}
}

@inproceedings{Abrahametal2021,
 address = {Los Alamitos, CA, USA},
 author = {S. Abraham and Z. Carmichael and S. Banerjee and R. VidalMata and A. Agrawal and M. Al Islam and W. Scheirer and J. Cleland-Huang},
 booktitle = {WAIN},
 pages = {113-120},
 publisher = {IEEE Computer Society},
 title = {Adaptive Autonomy in Human-on-the-Loop Vision-Based Robotics Systems},
 year = {2021}
}

@inproceedings{Adhikari21:PointBasedCIPOMDP,
 author = {Adhikari, Sarit
and Gmytrasiewicz, Piotr},
 booktitle = {Multi-Agent Systems},
 editor = {Rosenfeld, Ariel
and Talmon, Nimrod},
 pages = {245--263},
 publisher = {Springer International Publishing},
 title = {Point Based Solution Method for Communicative IPOMDPs},
 year = {2021}
}

@inproceedings{Ahn07,
 author = {J. Ahn and D. DeAngelis and S. Barber},
 booktitle = {IAT’07},
 pages = {229--235},
 title = {Attitude Driven Team Formation Using Multi-Dimensional Trust},
 year = {2007}
}

@article{aimag,
 author = {Eck, Adam and Soh, L.-K. and Doshi, P.},
 journal = {AI Magazine},
 number = {4},
 pages = {508-523},
 title = {Decision Making in Open Agent Systems},
 volume = {44},
 year = {2023}
}

@misc{airportq,
 author = {Uber},
 journal = {Uber.com},
 note = {Accessed: 2025-01-11},
 title = {Airport {Queue} {Access}},
 url = {https://help.uber.com/en/driving-and-delivering/article/queue-access?nodeId=1275528f-2ba7-4ca4-982f-6aca4e3fad99},
 year = {2025}
}

@inproceedings{Albrecht13:LBF,
 booktitle = {AAMAS '13},
 numpages = {2},
 pages = {1155–1156},
 title = {A Game-Theoretic Model and Best-Response Learning Method for Ad Hoc Coordination in Multiagent Systems},
 year = {2013}
}

@article{ALEXANDRE2023119500,
 author = {Claudio Reginaldo Alexandre and João Balsa},
 journal = {Expert Systems with Applications},
 pages = {119500},
 title = {Incorporating machine learning and a risk-based strategy in an anti-money laundering multiagent system},
 volume = {217},
 year = {2023}
}

@article{AlexandreBalsa2023,
 author = {C. R. Alexandre and J. Balsa},
 journal = {Expert Systems with Applications},
 pages = {119500},
 title = {Incorporating Machine Learning and a Risk-Based Strategy in an Anti-Money Laundering Multiagent System},
 volume = {217},
 year = {2023}
}

@inproceedings{Amani23:Lifelong,
 author = {S. Kim and L. F. Yang and C.-A. Cheng},
 booktitle = {ICLR},
 title = {Provably Efficient Lifelong Reinforcement Learning with Linear Representation},
 year = {2023}
}

@inproceedings{Amato:Scalable,
 author = {Amato, Christopher and Oliehoek, Frans A},
 booktitle = {Twenty-Ninth AAAI Conference on Artificial Intelligence},
 title = {Scalable planning and learning for multiagent {POMDP}s},
 year = {2015}
}

@inproceedings{Ammar14:Online,
 author = {Ammar, Haitham Bou and Eaton, Eric and Ruvolo, Paul and Taylor, Matthew},
 booktitle = {ICML},
 pages = {1206--1214},
 publisher = {PMLR},
 title = {Online Multi-Task Learning for Policy Gradient Methods},
 volume = {32},
 year = {2014}
}

@inproceedings{Andreas17:Modular,
author = {Jacob Andreas and Dan Klein and Sergey Levine},
 booktitle = {ICML},
 numpages = {10},
 pages = {166–175},
 title = {Modular Multitask Reinforcement Learning with Policy Sketches},
 year = {2017}
}

@phdthesis{anil2024,
 address = {Athens, GA},
 author = {Gayathri, A.},
 month = {July},
 school = {University of Georgia},
 title = {Multi-Agent Reinforcement Learning for Task Open Systems},
 type = {Master's Thesis},
 year = {2024}
}

@book{Baker85:Management,
 author = {Kenneth R. Baker and Dean H. Kropp},
 publisher = {Wiley},
 title = {Management Science: An Introduction to the Use of Decision Models},
 year = {1985}
}

@article{Barambonesetal2022,
 author = {J. Barambones and J. Cano-Benito and I. Sanchez-Rivero and R. Imbert and F. Richoux},
 journal = {IEEE Transactions on Games},
 pages = {1-15},
 title = {Multi-Agent Systems on Virtual Games: A Systematic Mapping Study},
 year = {2022}
}

@incollection{Bates92,
 author = {J. Bates and A. B. Loyall and W. S. Reilly},
 booktitle = {MAAMAW'1992},
 pages = {55--68},
 publisher = {Springer-Verlag
Berlin, Heidelberg},
 title = {An Architecture for Action, Emotion, and Social Behavior},
 year = {1992}
}

@article{Battaglia18:Relational,
 author = {Peter W. Battaglia and
Jessica B. Hamrick and
Victor Bapst and
Alvaro Sanchez{-}Gonzalez and
Vin{\'{\i}}cius Flores Zambaldi and
Mateusz Malinowski and
Andrea Tacchetti and
David Raposo and
Adam Santoro and
Ryan Faulkner and
{\c{C}}aglar G{\"{u}}l{\c{c}}ehre and
H. Francis Song and
Andrew J. Ballard and
Justin Gilmer and
George E. Dahl and
Ashish Vaswani and
Kelsey R. Allen and
Charles Nash and
Victoria Langston and
Chris Dyer and
Nicolas Heess and
Daan Wierstra and
Pushmeet Kohli and
Matthew M. Botvinick and
Oriol Vinyals and
Yujia Li and
Razvan Pascanu},
 journal = {CoRR},
 title = {Relational inductive biases, deep learning, and graph networks},
 url = {http://arxiv.org/abs/1806.01261},
 volume = {abs/1806.01261},
 year = {2018}
}

@article{BellmanMDP,
 author = {Richard Bellman},
 coden = {IUMJAB},
 issue = {4},
 journal = {Indiana Univ. Math. J.},
 mrclass = {},
 pages = {679--684},
 title = {A Markovian Decision Process},
 volume = {6},
 year = {1957}
}

@article{Benjaafar21:Autonomous,
 author = {Saif Benjaafar and Zicheng Wang and Xiaotang Yang},
 journal = {SSRN},
 month = {09},
 title = {Autonomous Vehicles for Ride-Hailing},
 url = {https://ssrn.com/abstract=3919411},
 year = {2021}
}

@article{Bernstein02:Complexity,
 author = {Daniel S. Bernstein and Robert Givan and Neil Immerman and Shlomo Zilberstein},
 journal = {Mathematics of Operations Research},
 number = {4},
 pages = {819--840},
 title = {The Complexity of Decentralized Control of {M}arkov Decision Processes},
 volume = {27},
 year = {2002}
}

@article{Best:Dec-MCTS,
 author = {Graeme Best and Oliver M Cliff and Timothy Patten and Ramgopal R Mettu and Robert Fitch},
 journal = {The International Journal of Robotics Research},
 number = {2-3},
 pages = {316-337},
 title = {Dec-MCTS: Decentralized planning for multi-robot active perception},
 volume = {38},
 year = {2019}
}

@book{Bonabeau:Swarm,
 address = {USA},
 author = {Bonabeau, Eric and Dorigo, Marco and Theraulaz, Guy},
 isbn = {0195131584},
 publisher = {Oxford University Press, Inc.},
 title = {From Natural to Artificial Swarm Intelligence},
 year = {1999}
}

@article{Borrego13:Team,
 author = {Borrego, Maura and Karlin, Jennifer and McNair, Lisa and Beddoes, Kacey},
 journal = {Journal of Engineering Education},
 title = {Team Effectiveness Theory from Industrial and Organizational Psychology Applied to Engineering Student Project Teams: A Research Review},
 volume = {102},
 year = {2013}
}

@article{Boychuk09:Fire,
 author = {D. Boychuk and W.J. Braun and R.J. Kulperger and Z.L. Krougly and D.A. Stanford},
 journal = {Environmental and Ecological Statistics},
 number = {2},
 pages = {133--151},
 title = {A Stochastic Forest Fire Growth Model},
 volume = {16},
 year = {2009}
}

@article{Bradshawetal2012,
 author = {Bradshaw, J. M. and Dignum, V. and Jonker, C. and Sierhuis, M.},
 issue_date = {March 2012},
 journal = {IEEE Intelligent Systems},
 number = {2},
 pages = {8-13},
 title = {Human–Agent-Robot Teamwork},
 volume = {27},
 year = {2012}
}

@article{Brafman03:RMax,
 author = {Brafman, Ronen I. and Tennenholtz, Moshe},
 issue_date = {3/1/2003},
 journal = {J. Mach. Learn. Res.},
 numpages = {19},
 pages = {213–231},
 publisher = {JMLR.org},
 title = {R-Max - a General Polynomial Time Algorithm for near-Optimal Reinforcement Learning},
 url = {https://doi.org/10.1162/153244303765208377},
 volume = {3},
 year = {2003}
}

@misc{Brockman16:Gym,
 author = {Greg Brockman and Vicki Cheung and Ludwig Pettersson and Jonas Schneider and John Schulman and Jie Tang and Wojciech Zaremba},
 eprint = {arXiv:1606.01540},
 title = {OpenAI Gym},
 year = {2016}
}

@inproceedings{Calandriello14:Sparse,
 author = {Calandriello, Daniele and Lazaric, Alessandro and Restelli, Marcello},
 booktitle = {NeurIPS},
 editor = {Z. Ghahramani and M. Welling and C. Cortes and N. Lawrence and K.Q. Weinberger},
 title = {Sparse Multi-Task Reinforcement Learning},
 volume = {27},
 year = {2014}
}

@inproceedings{Calmet04:Liberal,
 author = {J. Calmet and A. Daemi and R. Endsuleit and T. Mie},
 booktitle = {Engineering Societies in the Agents World IV, LNCS},
 pages = {81--92},
 title = {A Liberal Approach to Openness in Societies of Agents},
 volume = {3071},
 year = {2004}
}

@book{cartpole,
 address = {USA},
 author = {Busoniu, Lucian and Babuska, Robert and Schutter, Bart De and Ernst, Damien},
 edition = {1st},
 publisher = {CRC Press, Inc.},
 title = {Reinforcement Learning and Dynamic Programming Using Function Approximators},
 year = {2010}
}

@inbook{Chakraborty:Swarm,
 author = {Chakraborty, Amrita and Kar, Arpan Kumar},
 booktitle = {Nature-Inspired Computing and Optimization: Theory and Applications},
 editor = {Patnaik, Srikanta and Yang, Xin-She and Nakamatsu, Kazumi},
 isbn = {978-3-319-50920-4},
 pages = {475--494},
 publisher = {Springer International Publishing},
 title = {Swarm Intelligence: A Review of Algorithms},
 year = {2017}
}

@article{Chandak19:Lifelong,
 author = {Yash Chandak and 
Georgios Theocharous and 
Chris Nota and 
Philip S. Thomas},
 journal = {CoRR},
 title = {Lifelong Learning with a Changing Action Set},
 volume = {abs/1906.01770},
 year = {2019}
}

@inproceedings{Chandrasekaran:Open,
 author = {Chandrasekaran, Muthukumaran and Eck, Adam and Doshi, Prashant and Soh, Leenkiat},
 booktitle = {UAI'16},
 numpages = {10},
 pages = {82--91},
 publisher = {AUAI Press},
 title = {Individual Planning in Open and Typed Agent Systems},
 year = {2016}
}

@inproceedings{Chen15:Considering,
 author = {B. Chen and X. Chen and A. Timsina and L.-K. Soh},
 booktitle = {AAMAS},
 pages = {1861--1862},
 title = {Considering Agent and Task Openness in Ad Hoc Team Formation (Extended Abstract)},
 year = {2015}
}

@inproceedings{Chen16:Collaborative,
 author = {B. Chen and A. Eck and L.-K. Soh},
 booktitle = {AAMAS},
 pages = {1441-1442},
 title = {Collaborative Human Task Assignment for Open Systems (Extended Abstract)},
 year = {2016}
}

@inbook{Chen2018:Lifelong,
 author = {Chen, Zhiyuan
and Liu, Bing},
 booktitle = {Lifelong Machine Learning},
 pages = {139--152},
 publisher = {Springer International Publishing},
 title = {Lifelong Reinforcement Learning},
 year = {2018}
}

@article{ChenBarnes2014,
 author = {Chen, Jessie Y. C. and Barnes, Michael J.},
 journal = {IEEE Transactions on Human-Machine Systems},
 number = {1},
 pages = {13-29},
 title = {Human–Agent Teaming for Multirobot Control: A Review of Human Factors Issues},
 volume = {44},
 year = {2014}
}

@article{Chiriacescu:2013,
 author = {V. Chiriacescu and L.-K. Soh and D.F. Shell},
 journal = {International Journal of Cognitive Informatics and Cognitive Computing},
 number = {4},
 pages = {1-25},
 title = {Understanding Human Learning Using a Multi-Agent Simulation of the Unified Learning Model},
 volume = {7},
 year = {2013}
}

@inproceedings{Claes:Warehouse,
 author = {Claes, Daniel and Oliehoek, Frans and Baier, Hendrik and Tuyls, Karl},
 booktitle = {AAMAS '17},
 location = {S\&\#227;o Paulo, Brazil},
 pages = {492--500},
 title = {Decentralised Online Planning for Multi-Robot Warehouse Commissioning},
 year = {2017}
}

@inproceedings{Cohen:OpenDecPOMDPs,
 author = {Cohen, Jonathan and Dibangoye, Jilles-Steeve  and Mouaddib, Abdel-Illah },
 booktitle = {IEEE 29th International Conference on Tools with Artificial Intelligence (ICTAI)},
 pages = {977-984},
 title = {Open Decentralized POMDPs},
 year = {2017}
}

@inproceedings{Doshi08,
 author = {P. Doshi and D. Perez},
 booktitle = {AAAI'2008},
 pages = {63-68},
 title = {Generalized Point Based Value Iteration for Interactive POMDPs},
 year = {2008}
}

@article{Doshi12:Decision,
 author = {Prashant Doshi},
 journal = {AI Magazine},
 number = {4},
 pages = {82--95},
 title = {Decision Making in Complex Multiagent Settings: A Tale of Two Frameworks},
 volume = {33},
 year = {2012}
}

@article{Doshi:IPF,
 author = {Doshi, Prashant and Gmytrasiewicz, Piotr J.},
 journal = {JAIR},
 number = {1},
 numpages = {41},
 pages = {297--337},
 title = {Monte Carlo Sampling Methods for Approximating Interactive POMDPs},
 volume = {34},
 year = {2009}
}

@article{Drew2021,
 author = {Daniel Drew},
 journal = {Current Robotics Reports},
 pages = {189-200},
 title = {Multi-Agent Systems for Search and Rescue Applications},
 volume = {2},
 year = {2021}
}

@article{Eck:2013a,
 author = {A. Eck and L.-K. Soh},
 journal = {Multiagent and Grid Systems},
 pages = {197-226},
 title = {MineralMiner: An Active Sensing Simulation Environment},
 volume = {9},
 year = {2013}
}

@article{Eck:2013b,
 author = {A. Eck and L.-K. Soh},
 journal = {Autonomous Agents and Multiagent Systems},
 number = {2},
 pages = {202-244},
 title = {Observer Effect from Stateful Resources in Agent Sensing},
 volume = {26},
 year = {2013}
}

@inproceedings{Eck:2014,
 author = {A. Eck and L.-K. Soh},
 booktitle = {AAMAS’2014},
 pages = {741-748},
 title = {Online Heuristic Planning for Highly Uncertain Domains},
 year = {2014}
}

@inproceedings{Eck:2015a,
 author = {A. Eck and L.-K. Soh},
 booktitle = {AAMAS’2015},
 pages = {367-376},
 title = {To Ask, Sense, or Share: Ad Hoc Information Gathering},
 year = {2015}
}

@article{Eck:2015b,
 author = {A. Eck and L.-K. Soh and K. Olson and A.L. McCutcheon and J. Smyth},
 journal = {IEEE Computer Magazine},
 month = {November},
 pages = {112-116},
 title = {Understanding the Human Condition through Survey Informatics},
 year = {2015}
}

@article{Eck:2016,
 author = {A. Eck and L.-K. Soh and S. Devlin and D. Kudenko},
 journal = {Autonomous Agents and Multiagent Systems},
 number = {3},
 pages = {403-445},
 title = {Potential-Based Reward Shaping for Finite Horizon Online POMDP Planning},
 volume = {30},
 year = {2016}
}

@inproceedings{Eck:AAAI2020,
 author = {Eck, Adam and Shah, Maulik and Doshi, Prashant and Soh, Leenkiat},
 booktitle = {AAAI'20},
 location = {New York City, New York, USA},
 title = {Scalable Decision-Theoretic Planning in Open and Typed Multiagent Systems},
 year = {2020}
}

@inproceedings{ella,
 abstract = {The problem of learning multiple consecutive tasks, known as lifelong learning, is of great importance to the creation of intelligent, general-purpose, and flexible machines.  In this paper, we develop a method for online multi-task learning in the lifelong learning setting.  The proposed Efficient Lifelong Learning Algorithm (ELLA) maintains a sparsely shared basis for all task models, transfers knowledge from the basis to learn each new task, and refines the basis over time to maximize performance across all tasks. We show that ELLA has strong connections to both online dictionary learning for sparse coding and state-of-the-art batch multi-task learning methods, and provide robust theoretical performance guarantees.  We show empirically that ELLA yields nearly identical performance to batch multi-task learning while learning tasks sequentially in three orders of magnitude (over 1,000x) less time.},
 address = {Atlanta, Georgia, USA},
 author = {Ruvolo, Paul and Eaton, Eric},
 booktitle = {Proceedings of Machine Learning Research},
 editor = {Dasgupta, Sanjoy and McAllester, David},
 month = {17--19 Jun},
 pages = {507--515},
 pdf = {http://proceedings.mlr.press/v28/ruvolo13.pdf},
 publisher = {PMLR},
 title = {{ELLA}: An Efficient Lifelong Learning Algorithm},
 url = {https://proceedings.mlr.press/v28/ruvolo13.html},
 volume = {28},
 year = {2013}
}

@book{Epstein96,
 address = {Westport, CT, USA},
 author = {R. Epstein},
 publisher = {Praeger},
 title = {Cognition, Creativity, and Behavior: Selected Essays},
 year = {1996}
}

@inproceedings{Farinellietal2003,
 author = {A. Farinelli and G. Grisetti and L. Iocchi and S. Lo Cascio and D. Nardi },
 booktitle = {Proceedings 2003 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2003)},
 page = {3138-3143},
 title = {Design and Evaluation of Multi Agent Systems for Rescue Operations},
 year = {2003}
}

@article{Ferns11:Bisimulation,
 author = {N. Ferns and P. Panangaden and D. Precup},
 journal = {SIAM Journal on Computing},
 number = {6},
 title = {Bisimulation Metrics for Continuous Markov Decision Processes},
 volume = {40},
 year = {2011}
}

@techreport{Finney04:FARSITE,
 author = {Mark A. Finney},
 institution = {Rocky Mountain Research Station, Forest Service, USDA},
 month = {February},
 number = {RMRS-RP-4 Revised},
 title = {FARSITE: Fire Area Simulator—Model Development and Evaluation},
 type = {Research paper},
 year = {2004}
}

@inproceedings{Foerster18:LOLA,
 abstract = {Multi-agent settings are quickly gathering importance in machine learning. This includes a plethora of recent work on deep multi-agent reinforcement learning, but also can be extended to hierarchical reinforcement learning, generative adversarial networks and decentralised optimisation. In all these settings the presence of multiple learning agents renders the training problem non-stationary and often leads to unstable training or undesired final results. We present Learning with Opponent-Learning Awareness (LOLA), a method in which each agent shapes the anticipated learning of the other agents in the environment. The LOLA learning rule includes an additional term that accounts for the impact of one agent's policy on the anticipated parameter update of the other agents. Preliminary results show that the encounter of two LOLA agents leads to the emergence of tit-for-tat and therefore cooperation in the iterated prisoners' dilemma (IPD), while independent learning does not. In this domain, LOLA also receives higher payouts compared to a naive learner, and is robust against exploitation by higher order gradient-based methods. Applied to infinitely repeated matching pennies, LOLA agents converge to the Nash equilibrium. In a round robin tournament we show that LOLA agents can successfully shape the learning of a range of multi-agent learning algorithms from literature, resulting in the highest average returns on the IPD. We also show that the LOLA update rule can be efficiently calculated using an extension of the likelihood ratio policy gradient estimator, making the method suitable for model-free reinforcement learning. This method thus scales to large parameter and input spaces and nonlinear function approximators. We also apply LOLA to a grid world task with an embedded social dilemma using deep recurrent policies and opponent modelling. Again, by explicitly considering the learning of the other agent, LOLA agents learn to cooperate out of self-interest.},
 address = {Richland, SC},
 author = {Foerster, Jakob and Chen, Richard Y. and Al-Shedivat, Maruan and Whiteson, Shimon and Abbeel, Pieter and Mordatch, Igor},
 booktitle = {AAMAS '18},
 keywords = {general-sum, deep reinforcement learning, iterated prisoners dilemma, multi-agent learning},
 location = {Stockholm, Sweden},
 numpages = {9},
 pages = {122–130},
 publisher = {International Foundation for Autonomous Agents and Multiagent Systems},
 title = {Learning with Opponent-Learning Awareness},
 year = {2018}
}

@article{Foruzan:2018,
 author = {Foruzan, E. and Soh, L.-K. and Asgarpoor, S.},
 journal = {IEEE Transactions on Power Systems},
 number = {5},
 pages = {5749-5758},
 title = {Reinforcement Learning Approach for Optimal Distributed Energy Management in a Microgrid},
 volume = {33},
 year = {2018}
}

@article{Frankel:History,
 author = {Martin R. Frankel and Lester R. Frankel},
 journal = {Public Opinion Quarterly},
 number = {4},
 pages = {S127--S138},
 title = {Fifty Years of Survey Sampling in the United States},
 volume = {51},
 year = {1987}
}

@inproceedings{Gaoetal2006,
 author = {S. Gao and D. Xu and H. Wang and Y. Wang},
 booktitle = {Proceedings of the 2006 International Conference on Service Operations and Logistics, and Informatics},
 doi = {https://doi.org/10.1109/SOLI.2006.328967},
 pages = {851-856},
 title = {Intelligent Anti-Money Laundering System},
 year = {2006}
}

@article{Gaoetal2009,
 author = {S. Gao and D. Xu and H. Wang and P. Green},
 issue = {2},
 journal = {Journal of Knowledge Management},
 pages = {63-75},
 title = {Knowledge-based Anti-Money Laundering: A Software Agent Bank Application},
 volume = {13},
 year = {2009}
}

@inproceedings{Garg:DESPOTalpha,
 address = {FreiburgimBreisgau, Germany},
 author = {Neha Priyadarshini Garg AND David Hsu AND Wee Sun Lee},
 booktitle = {Proceedings of Robotics: Science and Systems},
 doi = {10.15607/RSS.2019.XV.006},
 month = {June},
 title = {DESPOT-Alpha: Online POMDP Planning with Large State and Observation Spaces},
 year = {2019}
}

@misc{GAT,
 archiveprefix = {arXiv},
 author = {Petar Veličković and Guillem Cucurull and Arantxa Casanova and Adriana Romero and Pietro Liò and Yoshua Bengio},
 eprint = {1710.10903},
 primaryclass = {stat.ML},
 title = {Graph Attention Networks},
 url = {https://arxiv.org/abs/1710.10903},
 year = {2018}
}

@article{Gmytrasiewicz05:Framework:JAIR,
 author = {Piotr J. Gmytrasiewicz and Prashant Doshi},
 journal = {JAIR},
 pages = {49--79},
 title = {A Framework for Sequential Planning in Multiagent Settings},
 volume = {24},
 year = {2005}
}
@inbook{stern-roni,
author = {Stern, Roni},
title = {Multi-Agent Path Finding – An Overview},
year = {2022},
isbn = {978-3-030-33273-0},
publisher = {Springer-Verlag},
address = {Berlin, Heidelberg},
url = {https://doi.org/10.1007/978-3-030-33274-7_6},
abstract = {Multi-Agent Pathfinding (MAPF) is the problem of finding paths for multiple agents such that every agent reaches its goal and the agents do not collide. In recent years, there has been a growing interest in MAPF in the Artificial Intelligence (AI) research community. This interest is partially because real-world MAPF applications, such as warehouse management, multi-robot teams, and aircraft management, are becoming more prevalent. In this overview, we discuss several possible definitions of the MAPF problem. Then, we survey MAPF algorithms, starting with fast but incomplete algorithms, then fast, complete but not optimal algorithms, and finally optimal algorithms. Then, we describe approximately optimal algorithms and conclude with non-classical MAPF and pointers for future reading and future work.},
booktitle = {Artificial Intelligence},
pages = {96–115},
numpages = {20}
}

@inproceedings{Gmytrasiewicz:AAMAS19,
 abstract = {We build on the Interactive POMDP (IPOMDP) framework, which extends POMDPs to multi-agent settings, and include communication which can take place between the agents. While IPOMDPs endow the agents with models of their environments and models of other agents, we supplement IPOMDPs with communicative acts available to the agents to formulate Communicative IPOMDPs (CIPOMDPs). We treat communication as a type of action; hence decisions regarding communicative acts should be based on decision-theoretic planning using Bellman optimality principle, just as they are for all other actions. As in any form of planning, the results of actions need to be precisely specified. We use Bayes update to derive how agents update their beliefs in CIPOMDPs; updates are due to their actions, observations, messages they send to other agents, and messages they receive from others. Without communication, CIPOMDPs reduce to IPOMDPs. Without other agents they all become classical POMDPs.},
 address = {Richland, SC},
 author = {Gmytrasiewicz, Piotr and Adhikari, Sarit},
 booktitle = {AAMAS '19},
 isbn = {9781450363099},
 keywords = {speech act theory, single and multi-agent planning and scheduling},
 location = {Montreal QC, Canada},
 numpages = {3},
 pages = {1985–1987},
 publisher = {International Foundation for Autonomous Agents and Multiagent Systems},
 title = {Optimal Sequential Planning for Communicative Actions: A Bayesian Approach},
 year = {2019}
}

@article{Gmytrasiewicz:JAIR20,
 author = {Piotr Gmytrasiewicz},
 journal = {JAIR},
 pages = {753-776},
 title = {How to Do Things with Words: A Bayesian Approach},
 volume = {68},
 year = {2020}
}

@inproceedings{Guestrin02:Context,
 author = {Guestrin, C. and Venkataraman, S. and Koller, D.},
 booktitle = {Eighteenth National Conference on Artificial Intelligence},
 pages = {253–259},
 title = {Context-specific multiagent coordination and planning with fac-
tored {MDP}s},
 year = {2002}
}

@article{guyot2006,
 author = {Guyot, Paul and Honiden, Shinichi},
 issn = {1460-7425},
 journal = {Journal of Artificial Societies and Social Simulation},
 keywords = {Agent-Based Participatory Simulations, Multi-Agent Systems, Role-Playing 

Games, Validation, Negotiation Support Tool},
 number = {4},
 pages = {8},
 title = {Agent-Based Participatory Simulations: Merging Multi-Agent Systems and Role-Playing Games},
 url = {https://www.jasss.org/9/4/8.html},
 volume = {9},
 year = {2006}
}

@inproceedings{Haider23:OOD,
 abstract = {Reliability of reinforcement learning (RL) agents is a largely unsolved problem. Especially in situations that substantially differ from their training environment, RL agents often exhibit unpredictable behavior, potentially leading to performance loss, safety violations or catastrophic failure. Reliable decision making agents should therefore be able to cast an alert whenever they encounter situations they have never seen before and do not know how to handle. While the problem, also known as out-of-distribution (OOD) detection, has received considerable attention in other domains such as image classification or sensory data analysis, it is less frequently studied in the context of RL. In fact, there is not even a common understanding of what OOD actually means in RL. In this work, we want to bridge this gap and approach the topic of OOD in RL from a general perspective. For this, we formulate OOD in RL as severe perturbations of the Markov decision process (MDP). To detect such perturbations, we introduce a predictive algorithm utilizing probabilistic dynamics models and bootstrapped ensembles. Since existing benchmarks are sparse and limited in their complexity, we also propose a set of evaluation scenarios with OOD occurrences. A detailed analysis of our approach shows superior detection performance compared to existing baselines from related fields.},
 address = {Richland, SC},
 author = {Haider, Tom and Roscher, Karsten and Schmoeller da Roza, Felippe and G\"{u}nnemann, Stephan},
 booktitle = {AAMAS '23},
 isbn = {9781450394321},
 keywords = {OOD detection, sequential decision making, AI safety, reinforcement learning, anomaly detection},
 location = {London, United Kingdom},
 numpages = {9},
 pages = {851–859},
 title = {Out-of-Distribution Detection for Reinforcement Learning Agents with Probabilistic Dynamics Models},
 year = {2023}
}

@inproceedings{Hamilton17:Inductive,
 author = {Hamilton, William L. and Ying, Rex and Leskovec, Jure},
 booktitle = {NIPS},
 location = {Long Beach, California, USA},
 numpages = {11},
 pages = {1025–1035},
 title = {Inductive Representation Learning on Large Graphs},
 year = {2017}
}

@inproceedings{He22:Reinforcement,
 author = {He, Keyang and Doshi, Prashant and Banerjee, Bikramjit},
 booktitle = {Proceedings of Machine Learning Research},
 month = {01--05 Aug},
 pages = {780--789},
 publisher = {PMLR},
 title = {Reinforcement learning in many-agent settings under partial observability},
 year = {2022}
}

@incollection{Herd18,
 author = {S. Herd and S. J. Read and R. O’Reilly and and D. J. Jilk},
 booktitle = {Artificial Intelligence Safety and Security)},
 editor = {R. V. Yampolskiy},
 publisher = {Chapman and Hall/CRC},
 title = {Goal Changes in Intelligent Agents},
 year = {2018}
}

@inproceedings{Hoang13:Interactive,
 author = {Trong Nghia Hoang and Kian Hsiang Low},
 booktitle = {23rd International Joint Conference on AI (IJCAI)},
 title = {Interactive {POMDP} Lite: Towards Practical Planning to Predict and Exploit Intentions for Interacting with Self-Interested Agents},
 year = {2013}
}

@article{Hula:MCTS,
 author = {A. Hula and P.R. Montague and P. Dayan},
 journal = {PLOS Computational Biology},
 number = {6},
 title = {{M}onte {C}arlo Planning Method Estimates Planning Horizons during Interactive Social Exchange},
 volume = {11},
 year = {2015}
}

@article{Huynh06:Integrated,
 author = {T. D. Huynh and N. R. Jennings and N. R. Shadbolt},
 journal = {Autonomous Agents and Multi-Agent Systems},
 number = {2},
 pages = {119--154},
 title = {An Integrated Trust and Reputation Model for Open Multi-Agent Systems},
 volume = {13},
 year = {2006}
}

@inproceedings{Jamroga13:Modularity,
 author = {W. Jamroga and A. Meski and M. Szreter},
 booktitle = {GandALF},
 pages = {224--239},
 title = {Modularity and Openness in Modeling Multi-Agent Systems},
 year = {2013}
}

@article{Jennings93,
 author = {N.R. Jennings},
 journal = {The Knowledge Engineering Review},
 number = {3},
 pages = {223-250},
 title = {Commitments and Conventions: The Foundation of Coordination in Multi-Agent Systems},
 volume = {8},
 year = {1993}
}

@inproceedings{Jiang2020Graph,
 author = {Jiechuan Jiang and Chen Dun and Tiejun Huang and Zongqing Lu},
 booktitle = {ICLR},
 title = {Graph Convolutional Reinforcement Learning},
 year = {2020}
}

@article{Johnson12,
 author = {M. Johnson and J. M. Bradshaw and P. Feltovich and C. Jonker and B. van Riemsdijk and M. Sierhui},
 journal = {IEEE Intelligent Systems},
 number = {2},
 pages = {43-51},
 title = {Autonomy and Interdependence in Human-Agent-Robot Teams},
 volume = {27},
 year = {2012}
}

@article{Jumadinov14:Strategic,
 author = {J. Jumadinova and P. Dasgupta and and L.-K. Soh},
 journal = {IEEE Trans. Systems, Man, \& Cybernetics-Part A},
 number = {8},
 pages = {1003--1014},
 title = {Strategic Capability-Learning for Improved Multi-agent Collaboration in Ad-hoc Environments},
 volume = {44},
 year = {2014}
}

@inproceedings{Jumadinova:2012,
 author = {J Jumadinova and P. Dasgupta and L.-K. Soh},
 booktitle = {IAT’2012},
 pages = {287-292},
 title = {Strategic Capability-Learning for Improved Multi-Agent Collaboration in Ad-Hoc Environments},
 year = {2012}
}

@article{Kahrobaee:2014,
 author = {Kahrobaee, S. and Rajabzadeh, R. and Soh, L.-K. and Asgarpoor, S.},
 journal = {Electric Power Systems Research},
 pages = {123-132},
 title = {Multiagent Study of Smart Grid Customers with Neighborhood Electricity Trading},
 volume = {111},
 year = {2014}
}

@InProceedings{pmlr-v48-mniha16,
  title = 	 {Asynchronous Methods for Deep Reinforcement Learning},
  author = 	 {Mnih, Volodymyr and Badia, Adria Puigdomenech and Mirza, Mehdi and Graves, Alex and Lillicrap, Timothy and Harley, Tim and Silver, David and Kavukcuoglu, Koray},
  booktitle = 	 {Proceedings of The 33rd International Conference on Machine Learning},
  pages = 	 {1928--1937},
  year = 	 {2016},
  editor = 	 {Balcan, Maria Florina and Weinberger, Kilian Q.},
  volume = 	 {48},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {New York, New York, USA},
  month = 	 {20--22 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v48/mniha16.pdf},
  url = 	 {https://proceedings.mlr.press/v48/mniha16.html},
  abstract = 	 {We propose a conceptually simple and lightweight framework for deep reinforcement learning that uses asynchronous gradient descent for optimization of deep neural network controllers. We present asynchronous variants of four standard reinforcement learning algorithms and show that parallel actor-learners have a stabilizing effect on training allowing all four methods to successfully train neural network controllers. The best performing method, an asynchronous variant of actor-critic, surpasses the current state-of-the-art on the Atari domain while training for half the time on a single multi-core CPU instead of a GPU. Furthermore, we show that asynchronous actor-critic succeeds on a wide variety of continuous motor control problems as well as on a new task of navigating random 3D mazes using a visual input.}
}


@inproceedings{Kakarlapudi22:Decision,
 author = {Kakarlapudi, Anirudh and Anil, Gayathri and Eck, Adam and Doshi, Prashant and Soh, Leen-Kiat},
 booktitle = {UAI},
 pages = {938--948},
 title = {Decision-theoretic planning with communication in open multiagent systems},
 year = {2022}
}

@article{Khandaker:2010,
 author = {N. Khandaker and L.-K. Soh},
 journal = {IEEE Transactions on Learning Technologies},
 number = {3},
 pages = {190-202},
 title = {ClassroomWiki: A Collaborative Wiki for Instructional Use with Multiagent Group Formation},
 volume = {3},
 year = {2010}
}

@article{Khandaker:2011a,
 author = {N. Khandaker and L.-K. Soh},
 journal = {International Journal of Artificial Intelligence in Education},
 number = {3},
 pages = {231-268},
 title = {Improving Group Selection and Assessment in an Asynchronous Collaborative Learning Application},
 volume = {20},
 year = {2010}
}

@article{Khandaker:2011b,
 author = {N. Khandaker and L.-K. Soh},
 journal = {IEEE Transactions on Systems, Man, and Cybernetics: Part C: Applications and Reviews},
 number = {4},
 pages = {533-543},
 title = {SimCoL: A Simulation Tool for Computer-Supported Collaborative Learning},
 volume = {41},
 year = {2011}
}

@inproceedings{Kim06:Exploiting,
 author = {Yoonheui Kim and Ranjit Nair and Pradeep Varakantham and Milind Tambe and Makoto Yokoo},
 booktitle = {AAAI Spring Symposium on Distributed Plan and Schedule Management},
 title = {Exploiting locality of interaction in networked distributed {POMDP}s},
 year = {2006}
}

@inproceedings{Kim21:MetaMAPG,
 author = {Kim, Dong Ki and Liu, Miao and Riemer, Matthew D and Sun, Chuangchuang and Abdulhai, Marwa and Habibi, Golnaz and Lopez-Cot, Sebastian and Tesauro, Gerald and How, Jonathan},
 booktitle = {Proceedings of Machine Learning Research},
 editor = {Meila, Marina and Zhang, Tong},
 month = {18--24 Jul},
 pages = {5541--5550},
 pdf = {http://proceedings.mlr.press/v139/kim21g/kim21g.pdf},
 publisher = {PMLR},
 title = {A Policy Gradient Algorithm for Learning to Learn in Multiagent Reinforcement Learning},
 url = {https://proceedings.mlr.press/v139/kim21g.html},
 volume = {139},
 year = {2021}
}

@misc{kingma2017adammethodstochasticoptimization,
 archiveprefix = {arXiv},
 author = {Diederik P. Kingma and Jimmy Ba},
 eprint = {1412.6980},
 primaryclass = {cs.LG},
 title = {Adam: A Method for Stochastic Optimization},
 url = {https://arxiv.org/abs/1412.6980},
 year = {2017}
}

@inproceedings{Kitanoetal1999,
 author = {H. Kitano and S. Tadokoro and I. Noda, H. Matsubara and T. Takahashi and A. Shinjou and S. Shimada},
 booktitle = {Proceedings of the IEEE International Conference on Systems, Man, and Cybernetics (SMC'99)},
 page = {VI-739-VI-743},
 title = {RoboCup Rescue: Search and Rescue in Large-Scale Disasters as a Domain for Autonomous Agents Research},
 year = {1999}
}

@inproceedings{Kocsis06:UCT,
 acmid = {2091633},
 address = {Berlin, Heidelberg},
 author = {Kocsis, Levente and Szepesv\'{a}ri, Csaba},
 booktitle = {ECML'06},
 doi = {10.1007/11871842_29},
 isbn = {3-540-45375-X, 978-3-540-45375-8},
 location = {Berlin, Germany},
 numpages = {12},
 pages = {282--293},
 publisher = {Springer-Verlag},
 title = {Bandit Based Monte-carlo Planning},
 url = {http://dx.doi.org/10.1007/11871842_29},
 year = {2006}
}

@article{Kolouri2023:Lifelong,
 author = {S. Kolouri and Ali Abbasi and Soroush Abbasi Koohpayegani and Parsa Nooralinejad and Hamed Pirsiavash},
 journal = {IEEE Signal Processing Letters},
 pages = {1812-1816},
 title = {Multia-Agent Lifelong Implicit Neural Learning},
 volume = {30},
 year = {2023}
}

@article{Kongetal2023,
 author = {W.-R. Kong and D.-Y. Zhou and Y.-J. Du and Y. Zhou and Y.-y Zhao},
 journal = {IET Control Theory and Applications},
 pages = {1840-1862},
 title = {Hierarchical multi-agent reinforcement learning for multi-aircraft close -range air combat},
 volume = {17},
 year = {2023}
}

@inproceedings{Kumaretal2021,
 author = {A. Kumar and S. Das and V. Tyagi and R. N. Shaw and A. Ghosh},
 booktitle = {Studies in Computational Intelligence},
 editor = {J. C. Bansal and M. Paprzycki and M. Bianchini and S. Das},
 pages = {851-856},
 publisher = {Springer, Singapore},
 title = {Analysis of Classifier Algorithms to Detect Anti-Money Laundering},
 volume = {950},
 year = {2006}
}

@article{Lasry:MeanFieldGames,
 author = {J.-M. Lasry and P.-L. Lions},
 doi = {10.1007/s11537-007-0657-8},
 journal = {Japanese Journal of Mathematics},
 pages = {229-260},
 title = {Mean field games},
 volume = {2},
 year = {2007}
}

@article{Lerman:DynamicMRTA,
 author = {Lerman, Kristina and Jones, Chris and Galstyan, Aram and Matar\'{\i}c, Maja J},
 issue_date = {March     2006},
 journal = {Int. J. Rob. Res.},
 number = {3},
 pages = {225--241},
 title = {Analysis of Dynamic Task Allocation in Multi-Robot Systems},
 volume = {25},
 year = {2006}
}

@article{Lesser95,
 author = {V. Lesser},
 journal = {ACM Computing Surveys},
 number = {3},
 pages = {340-342},
 title = {Multiagent Systems: An Emerging Subdiscipline of AI},
 volume = {27},
 year = {1995}
}

@inproceedings{LiSoh:2004,
 author = {X. Li and L.-K. Soh},
 booktitle = {IAT’2004},
 pages = {133-139},
 title = {Learning How to Plan and Instantiate a Plan in Multi-Agent Coalition Formation},
 year = {2004}
}

@incollection{Littman:MarkovGames,
 author = {Michael L. Littman},
 booktitle = {ICML},
 publisher = {PMLR},
 pages = {157-163},
 title = {Markov games as a framework for multi-agent reinforcement learning},
 year = {1994}
}

@inproceedings{Littman:QMDP,
 author = {Michael L Littman and Anthony R Cassandra and Leslie Pack Kaelbling},
 booktitle = {ICML'95},
 pages = {362-370},
 title = {Learning policies for partially observable environments: Scaling up},
 year = {1995}
}

@article{Liu:TVMDP,
 author = {Liu, Lantao and Sukhatme, Gaurav S.},
 doi = {10.1109/LRA.2018.2801479},
 journal = {IEEE Robotics and Automation Letters},
 keywords = {Space exploration;Markov processes;Hidden Markov models;Oceans;Vehicle dynamics;Standards;AI-based methods;motion and path planning;marine robotics},
 number = {3},
 pages = {1631-1638},
 title = {A Solution to Time-Varying Markov Decision Processes},
 volume = {3},
 year = {2018}
}

@book{Lohr:Sampling,
 address = {Boston, MA},
 author = {Sharon L. Lohr},
 edition = {2nd},
 publisher = {Brooks/Cole},
 title = {Sampling: Design and Analysis},
 year = {2010}
}

@inproceedings{Lowe2017:MAPE,
 author = {Lowe, Ryan and Wu, Yi and Tamar, Aviv and Harb, Jean and Abbeel, Pieter and Mordatch, Igor},
 booktitle = {NIPS},
 title = {Multi-Agent Actor-Critic for Mixed Cooperative-Competitive Environments},
 year = {2017}
}

@article{Macworth09,
 author = {A. K. Mackworth},
 journal = {AI Magazine},
 number = {1},
 pages = {7-28},
 title = {Agents, Bodies, Constraints, Dynamics, and Evolution},
 volume = {30},
 year = {2009}
}

@inproceedings{MalaschukDyumin2020,
 author = {Malaschuk, O. and Dyumin, A.},
 booktitle = {Mechanisms and Machine Science},
 doi = {https://doi.org/10.1007/978-3-030-33491-8_10},
 editor = {Misyurin, S. and Arakelian, V. and Avetisyan, A.},
 publisher = {Springer, Cham},
 title = {Intelligent Multi-agent System for Rescue Missions},
 volume = {80},
 year = {2020}
}

@inproceedings{Mendez22:Modular,
 author = {J.A. Mendez and H. van Seijen and E. Eaton},
 booktitle = {ICLR'2022},
 title = {Modular Lifelong Reinforcement Learning via Neural Composition},
 year = {2022}
}

@inproceedings{Mguni:MeanField,
 author = {D. Mguni and J. Jennings and E.M. de Cote},
 booktitle = {AAMAS},
 pages = {4686-4693},
 title = {Decentralized Learning in Systems with Many, Many Strategic Agents},
 year = {2018}
}

@inproceedings{Miller13:Meta,
 author = {Lee Dee Miller and
Leen{-}Kiat Soh},
 booktitle = {Proceedings of the 6th International Conference on Educational Data
Mining, Memphis, Tennessee, USA, July 6-9, 2013},
 editor = {Sidney K. D'Mello and
Rafael A. Calvo and
Andrew Olney},
 pages = {129--136},
 publisher = {International Educational Data Mining Society},
 title = {Meta-Reasoning Algorithm for Improving Analysis of Student Interactions
with Learning Objects using Supervised Learning},
 year = {2013}
}

@article{Mirsky:AdHocSurvey,
 author = {Mirsky, Reuth and Carlucho, Ignacio and Rahman, Arrasy and Fosong, Elliot and Macke, William and Sridharan, Mohan and Stone, Peter and Albrecht, Stefano V.},
 copyright = {arXiv.org perpetual, non-exclusive license},
 doi = {10.48550/ARXIV.2202.10450},
 journal = {arXiv},
 keywords = {Multiagent Systems (cs.MA), Artificial Intelligence (cs.AI), FOS: Computer and information sciences, FOS: Computer and information sciences},
 publisher = {arXiv},
 title = {A Survey of Ad Hoc Teamwork: Definitions, Methods, and Open Problems},
 url = {https://arxiv.org/abs/2202.10450},
 year = {2022}
}

@article{Mostafa19,
 author = {S. A. Mostafa and M. S. Ahamad and A. Mustapha},
 journal = {Artificial Intelligence Review},
 pages = {149-186},
 title = {Adjustable Autonomy: A Systematic Literature Review},
 volume = {51},
 year = {2019}
}

@inproceedings{MullerFischer2014,
 author = {Müller, J.P. and Fischer, K.},
 booktitle = {Agent-Oriented Software Engineering},
 doi = {https://doi.org/10.1007/978-3-642-54432-3_3},
 editor = {Shehory, O. and Sturm, A.},
 publisher = {Springer, Berlin, Heidelberg},
 title = {Application Impact of Multi-agent Systems and Technologies: A Survey},
 year = {2014}
}

@article{nac,
 abstract = {In this paper, we suggest a novel reinforcement learning architecture, the Natural Actor-Critic. The actor updates are achieved using stochastic policy gradients employing Amari's natural gradient approach, while the critic obtains both the natural policy gradient and additional parameters of a value function simultaneously by linear regression. We show that actor improvements with natural policy gradients are particularly appealing as these are independent of coordinate frame of the chosen policy representation, and can be estimated more efficiently than regular policy gradients. The critic makes use of a special basis function parameterization motivated by the policy-gradient compatible function approximation. We show that several well-known reinforcement learning methods such as the original Actor-Critic and Bradtke's Linear Quadratic Q-Learning are in fact Natural Actor-Critic algorithms. Empirical evaluations illustrate the effectiveness of our techniques in comparison to previous methods, and also demonstrate their applicability for learning control on an anthropomorphic robot arm.},
 author = {Jan Peters and Stefan Schaal},
 doi = {https://doi.org/10.1016/j.neucom.2007.11.026},
 issn = {0925-2312},
 journal = {Neurocomputing},
 keywords = {Policy-gradient methods, Compatible function approximation, Natural gradients, Actor-Critic methods, Reinforcement learning, Robot learning},
 note = {Progress in Modeling, Theory, and Application of Computational Intelligenc},
 number = {7},
 pages = {1180-1190},
 title = {Natural Actor-Critic},
 url = {https://www.sciencedirect.com/science/article/pii/S0925231208000532},
 volume = {71},
 year = {2008}
}

@inproceedings{Nair05:Networked,
 author = {Ranjit Nair and Pradeep Varakantham and Milind Tambe and Makoto Yokoo},
 booktitle = {Twentieth AAAI Conference on Artificial Intelligence},
 pages = {133--139},
 title = {Networked Distributed {POMDP}s: A Synthesis of Distributed Constraint Optimization and {POMDP}s},
 year = {2005}
}

@article{Neyman:Sampling,
 author = {J. Neyman},
 journal = {Journal of the Royal Statistical Society},
 number = {4},
 pages = {558--625},
 title = {On the Two Different Aspects of the Representative Method: The Method of Stratified Sampling and the Method of Purposive Selection},
 volume = {97},
 year = {1934}
}

@inproceedings{Ng12:Bayes,
 author = {Brenda Ng and Kofi Boakye and Carol Meyers and Andrew Wang},
 booktitle = {AAAI},
 pages = {1408 -- 1414},
 title = {Bayes-Adaptive Interactive {POMDP}s},
 year = {2012}
}

@article{Ng_Meyers_Boakye_Nitao_2010,
 abstractnote = {&lt;p&gt;We examine the suitability of using decision processes to model real-world systems of intelligent adversaries. Decision processes have long been used to study cooperative multiagent interactions, but their practical applicability to adversarial problems has received minimal study. We address the pros and cons of applying sequential decision-making in this area, using the crime of money laundering as a specific example. Motivated by case studies, we abstract out a model of the money laundering process, using the framework of interactive partially observable Markov decision processes (I-POMDPs). We address why this framework is well suited for modeling adversarial interactions. Particle filtering and value iteration are used to solve the model, with the application of different pruning and look-ahead strategies to assess the tradeoffs between solution quality and algorithmic run time. Our results show that there is a large gap in the level of realism that can currently be achieved by such decision models, largely due to computational demands that limit the size of problems that can be solved. While these results represent solutions to a simplified model of money laundering, they illustrate nonetheless the kinds of agent interactions that cannot be captured by standard approaches such as anomaly detection. This implies that I-POMDP methods may be valuable in the future, when algorithmic capabilities have further evolved.&lt;/p&gt;},
 author = {Ng, Brenda and Meyers, Carol and Boakye, Kofi and Nitao, John},
 doi = {10.1609/aaai.v24i2.18818},
 journal = {AAAI Conference on Artificial Intelligence},
 month = {Jul.},
 number = {2},
 pages = {1814-1820},
 title = {Towards Applying Interactive POMDPs to Real-World Adversary Modeling},
 url = {https://ojs.aaai.org/index.php/AAAI/article/view/18818},
 volume = {24},
 year = {2010}
}

@inproceedings{Nguyen:CDecPOMDP,
 author = {Nguyen, Duc Thien  and Kumar, Akshat and Lau, Hoong Chuin},
 booktitle = {AAAI},
 pages = {3036-3043},
 title = {Collective Multiagent Sequential Decision Making Under Uncertainty},
 year = {2017}
}

@article{Nunes:TaskAlloc,
 
 author = {Ernesto Nunes and Marie Manner and Hakim Mitiche and Maria Gini},
 doi = {https://doi.org/10.1016/j.robot.2016.10.008},
 issn = {0921-8890},
 journal = {Robotics and Autonomous Systems},
 keywords = {Task allocation, Taxonomy, Multi-robot coordination, Temporal constraints, Time-extended assignments},
 note = {Special Issue on New Research Frontiers for Intelligent Autonomous Systems},
 pages = {55-70},
 title = {A taxonomy for task allocation problems with temporal and ordering constraints},
 url = {https://www.sciencedirect.com/science/article/pii/S0921889016306157},
 volume = {90},
 year = {2017}
}

@inproceedings{Oliehoek08:Exploiting,
 author = {Frans Oliehoek and Matthijs Spaan and Shimon Whiteson and Nikos Vlassis},
 booktitle = {AAMAS},
 pages = {517--524},
 title = {Exploiting Locality of Interaction in Factored Dec-{POMDP}s},
 year = {2008}
}

@inproceedings{Oliehoek:APPXSOL,
 author = {Oliehoek, Frans A and Whiteson, Shimon and Spaan, Matthijs TJ},
 booktitle = {International Joint Conference on Autonomous Agents and Multi-Agent Systems (AAMAS)},
 pages = {563--570},
 title = {Approximate solutions for factored {Dec-POMDPs} with many agents},
 year = {2013}
}

@book{Oliehoek:DecPOMDP,
 author = {Oliehoek, Frans A. and Amato, Christopher},
 edition = {1st},
 publisher = {Springer Publishing Company, Incorporated},
 title = {A Concise Introduction to Decentralized POMDPs},
 year = {2016}
}

@inproceedings{Oliehoek:ExpLocality,
 author = {Oliehoek, Frans A and Spaan, Matthijs TJ and Whiteson, Shimon and Vlassis, Nikos},
 booktitle = {7th International Joint conference on Autonomous agents and Multiagent Systems (AAMAS)},
 pages = {517--524},
 title = {Exploiting locality of interaction in factored {Dec-POMDP}s},
 year = {2008}
}

@inproceedings{Omidshafiei2017:Multitask,
 author = {S. Omidshafiei and Jason Pazis and Christopher Amato and Jonathan P. How and John Vian},
 booktitle = {ICML},
 title = {Deep Decentralized Multi-Task Multi-Agent Reinforcement Learning under Partial Observability},
 year = {2017}
}

@article{Ornik2021-ts,
 abstract = {This paper proposes a formal approach to online learning and
planning for agents operating in a priori unknown, time-varying
environments. The proposed method computes the maximally likely
model of the environment, given the observations about the
environment made by an agent earlier in the system run and
assuming knowledge of a bound on the maximal rate of change of
system dynamics. Such an approach generalizes the estimation
method commonly used in learning algorithms for unknown Markov
decision processes with time-invariant transition probabilities,
but is also able to quickly and correctly identify the system
dynamics following a change. Based on the proposed method, we
generalize the exploration bonuses used in learning for
time-invariant Markov decision processes by introducing a notion
of uncertainty in a learned time-varying model, and develop a
control policy for time-varying Markov decision processes based
on the exploitation and exploration trade-off. We demonstrate the
proposed methods on four numerical examples: a patrolling task
with a change in system dynamics, a two-state MDP with
periodically changing outcomes of actions, a wind flow estimation
task, and a multi-armed bandit problem with periodically changing
probabilities of different rewards.},
 address = {United States},
 author = {Ornik, Melkior and Topcu, Ufuk},
 journal = {J Mach Learn Res},
 keywords = {Markov decision processes; changing environment; maximum
likelihood estimation; online learning; uncertainty
quantification},
 language = {en},
 month = {February},
 pages = {1--40},
 title = {Learning and Planning for {Time-Varying} {MDPs} Using Maximum
Likelihood Estimation},
 volume = {22},
 year = {2021}
}

@inproceedings{Papoudakis21:Benchmarks,
 author = {Georgios Papoudakis and Filippos Christianos and Lukas Sch{\"a}fer and Stefano V Albrecht},
 booktitle = {NeurIPS},
 title = {Benchmarking Multi-Agent Deep Reinforcement Learning Algorithms in Cooperative Tasks},
 url = {https://openreview.net/forum?id=cIrPX-Sn5n},
 year = {2021}
}

@article{PechoucekMarik2008,
 author = {M. Pěchouček and V. Mařík},
 doi = {https://doi.org/10.1007/s10458-008-9050-0},
 journal = {Autonomous Agents and Multi-Agent Systems},
 pages = {397–431},
 title = {Industrial deployment of multi-agent technologies: review and selected case studies},
 volume = {17},
 year = {2008}
}

@article{Peteranetz17:CC,
 author = {M. Peteranetz and A. Flanigan and D. F. Shell and L.-K. Soh},
 journal = {IEEE Transactions on Education},
 number = {4},
 pages = {305-313},
 title = {Computational Creativity: An Avenue for Promoting Learning in Computer Science},
 volume = {60},
 year = {2017}
}

@article{Peteranetz18:CC,
 author = {M. Peteranetz and A. E. Flanigan and D. F. Shell and L.-K. Soh},
 journal = {IEEE Transactions on Education},
 number = {3},
 pages = {195-203},
 title = {Helping Engineering Students Learn in Introductory Computer Science (CS1) Using Computational Creativity Exercises (CCEs)},
 volume = {61},
 year = {2018}
}

@inproceedings{Peteranetz19:CC,
 author = {M. Peteranetz and L.-K. Soh and E. Ingraham},
 booktitle = {SIGCSE’2020},
 month = {25 Feb--1 Mar},
 pages = {442-448},
 title = {Building Computational Creativity in an Online Course for Non-Majors},
 year = {2019}
}

@inproceedings{Peteranetz20:CCTest,
 author = {M. Peteranetz and P. M. Morrow and L.-K. Soh },
 booktitle = {SIGCSE’2020},
 month = {11--14 Mar},
 pages = {926-932},
 title = {Development and Validation of the Computational Thinking Concepts and Skills Test},
 year = {2020}
}

@article{Pineau06,
 author = {J. Pineau G. Gordon and S. Thrun},
 journal = {JAIR},
 pages = {335-380},
 title = {Anytime Point-Based Approximations for Large POMDPs},
 volume = {27},
 year = {2006}
}

@article{Pinyol11:Computational,
 author = {I. Pinyol and J. Sabater-Mir},
 journal = {Artificial Intelligence Review},
 pages = {1--25},
 title = {Computational Trust and Reputation models for Open Multi-Agent Systems: A Review},
 volume = {40},
 year = {2011}
}

@inproceedings{pmlr-v119-boehmer20a,
 author = {Boehmer, Wendelin and Kurin, Vitaly and Whiteson, Shimon},
 booktitle = {Machine Learning Research},
 editor = {III, Hal Daumé and Singh, Aarti},
 pages = {980--991},
 publisher = {PMLR},
 title = {Deep Coordination Graphs},
 volume = {119},
 year = {2020}
}

@inproceedings{pmlr-v70-gilmer17a,
 author = {Justin Gilmer and Samuel S. Schoenholz and Patrick F. Riley and Oriol Vinyals and George E. Dahl},
 booktitle = {Machine Learning Research},
 editor = {Precup, Doina and Teh, Yee Whye},
 pages = {1263--1272},
 publisher = {PMLR},
 title = {Neural Message Passing for Quantum Chemistry},
 volume = {70},
 year = {2017}
}

@inproceedings{polgrad,
 author = {Sutton, Richard S and McAllester, David and Singh, Satinder and Mansour, Yishay},
 booktitle = {NeurIPS},
 editor = {S. Solla and T. Leen and K. M\"{u}ller},
 pages = {},
 publisher = {MIT Press},
 title = {Policy Gradient Methods for Reinforcement Learning with Function Approximation},
 url = {https://proceedings.neurips.cc/paper_files/paper/1999/file/464d828b85b0bed98e80ade0a5c43b0f-Paper.pdf},
 volume = {12},
 year = {1999}
}

@article{Pynadath02:Communicative,
 author = {David Pynadath and Milind Tambe},
 journal = {JAIR},
 pages = {389--426},
 title = {The Communicative Multiagent Team Decision Problem: Analyzing Teamwork theories and Models},
 volume = {16},
 year = {2002}
}

@incollection{Qu:IEM,
 author = {Qu, Xia and Doshi, Prashant},
 booktitle = {NeurIPS},
 editor = {C. Cortes and N. D. Lawrence and D. D. Lee and M. Sugiyama and R. Garnett},
 pages = {478--486},
 publisher = {Curran Associates, Inc.},
 title = {Individual Planning in Infinite-Horizon Multiagent Settings: Inference, Structure and Scalability},
 url = {http://papers.nips.cc/paper/5734-individual-planning-in-infinite-horizon-multiagent-settings-inference-structure-and-scalability.pdf},
 year = {2015}
}

@phdthesis{quadrotor,
 author = {Bouabdallah, Samir},
 doi = {https://doi.org/10.5075/epfl-thesis-3727},
 institution = {I2S},
 pages = {155},
 publisher = {EPFL},
 school = {École Polytechnique Fédérale De Lausanne},
 title = {Design and control of quadrotors with application to  autonomous flying},
 year = {2007}
}

@inproceedings{Rahman:OpenAdHoc,
 author = {Rahman, Muhammad A and Hopner, Niklas and Christianos, Filippos and Albrecht, Stefano V},
 booktitle = {Machine Learning Research},
 editor = {Meila, Marina and Zhang, Tong},
 pages = {8776--8786},
 publisher = {PMLR},
 title = {Towards Open Ad Hoc Teamwork Using Graph-based Policy Learning},
 volume = {139},
 year = {2021}
}

@inproceedings{Rajeswaran17:EPOpt,
 author = {Aravind Rajeswaran and
Sarvjeet Ghotra and
Balaraman Ravindran and
Sergey Levine},
booktitle = {5th International Conference on Learning Representations {ICLR}},
 title = {EPOpt: Learning Robust Neural Network Policies Using Model Ensembles},
 year = {2017}
}

@article{Ralha19,
 author = {C. G. Ralha and A. H. D. Mendes and L. A. Laranjeira and A. P. F. Araujo and A. C. M. A. Melo},
 journal = {Future Generation Computer Systems},
 pages = {80-96},
 title = { Multiagent System for Dynamic Resource Provisioning in Cloud Computing Platforms},
 volume = {94},
 year = {2019}
}

@article{Ramchurn05,
 author = {S. D. Ramchurn and D. Huynh and N. R. Jennings},
 journal = {The Knowledge Engineering Review},
 number = {1},
 pages = {1-25},
 title = {Trust in Multi-Agent Systems},
 volume = {19},
 year = {2005}
}

@article{reinforce,
 added-at = {2008-03-11T14:52:34.000+0100},
 author = {Williams, R. J.},
 biburl = {https://www.bibsonomy.org/bibtex/294224c3e53bfe80ade7218b3a0283465/idsia},
 citeulike-article-id = {2374762},
 interhash = {b90d65a735ae02a940f5075b0fd7ebe7},
 intrahash = {94224c3e53bfe80ade7218b3a0283465},
 journal = {Machine Learning},
 keywords = {daanbib},
 pages = {229--256},
 priority = {2},
 timestamp = {2008-03-11T15:05:47.000+0100},
 title = {Simple statistical gradient-following algorithms for connectionist reinforcement learning},
 volume = {8},
 year = {1992}
}

@inproceedings{Riedletal2003,
 author = {M. Riedl and C. J. Saretto and R. M. Young},
 booktitle = {Proceedings of the 2nd International Joint Conference on Autonomous Agents and Multiagent Systems (AAMAS)},
 pages = {741-748},
 title = {Managing Interaction between Users and Agents in a Multi-Agent Storytelling Environment},
 year = {2003}
}

@inproceedings{Ross07:Bayes,
 author = {Stephane Ross and Brahim Chaib-draa and Joelle Pineau},
 booktitle = {NIPS},
 title = {Bayes-Adaptive {POMDP}s},
 year = {2007}
}

@article{Ross08:Online,
 author = {Ross, St{\'e}phane and Pineau, Joelle and Paquet, S{\'e}bastien and Chaib-draa, Brahim},
 issn = {1076-9757},
 issue_date = {May 2008},
 journal = {J. Artif. Int. Res.},
 month = {July},
 number = {1},
 numpages = {42},
 pages = {663--704},
 title = {Online Planning Algorithms for {POMDP}s},
 volume = {32},
 year = {2008}
}

@inbook{Roughgarden16:BSD,
 author = {Roughgarden, Tim},
 booktitle = {Twenty Lectures on Algorithmic Game Theory},
 doi = {10.1017/CBO9781316779309.017},
 pages = {216–229},
 place = {Cambridge},
 publisher = {Cambridge University Press},
 title = {Best-Response Dynamics},
 year = {2016}
}

@article{Roughgarden:ActAnon,
 author = {Tim Roughgarden and Eva Tardos},
 journal = {Journal of ACM},
 number = {2},
 pages = {236-259},
 title = {How Bad is Selfish Routing},
 volume = {49},
 year = {2002}
}

@book{Roughgarden:AGT,
 author = {Roughgarden, Tim},
 doi = {10.1017/CBO9781316779309},
 place = {Cambridge},
 publisher = {Cambridge University Press},
 title = {Twenty Lectures on Algorithmic Game Theory},
 year = {2016}
}

@inproceedings{Ruan15:Bismulation,
 author = {S. S. Ruan and G. Comanici and P. Panangaden and D. Precup},
 booktitle = {29th AAAI Conference on Artificial Intelligence},
 pages = {2578-3584},
 title = {Representation Discovery for MDPs Using Bisimulation Metrics},
 year = {2015}
}

@inproceedings{Samvelyan19:Starcraft,
 abstract = {In the last few years, deep multi-agent reinforcement learning (RL) has become a highly active area of research. A particularly challenging class of problems in this area is partially observable, cooperative, multi-agent learning, in which teams of agents must learn to coordinate their behaviour while conditioning only on their private observations. This is an attractive research area since such problems are relevant to a large number of real-world systems and are also more amenable to evaluation than general-sum problems. Standardised environments such as the ALE and MuJoCo have allowed single-agent RL to move beyond toy domains, such as grid worlds. However, there is no comparable benchmark for cooperative multi-agent RL. As a result, most papers in this field use one-off toy problems, making it difficult to measure real progress. In this paper, we propose the StarCraft Multi-Agent Challenge (SMAC) as a benchmark problem to fill this gap. SMAC is based on the popular real-time strategy game StarCraft II and focuses on micromanagement challenges where each unit is controlled by an independent agent that must act based on local observations. We offer a diverse set of challenge maps and recommendations for best practices in benchmarking and evaluations. We also open-source a deep multi-agent RL learning framework including state-of-the-art algorithms. We believe that SMAC can provide a standard benchmark environment for years to come. Videos of our best agents for several SMAC scenarios are available at: https://youtu.be/VZ7zmQ_obZ0.},
 address = {Richland, SC},
 author = {Samvelyan, Mikayel and Rashid, Tabish and Schroeder de Witt, Christian and Farquhar, Gregory and Nardelli, Nantas and Rudner, Tim G. J. and Hung, Chia-Man and Torr, Philip H. S. and Foerster, Jakob and Whiteson, Shimon},
 booktitle = {AAMAS},
 isbn = {9781450363099},
 keywords = {reinforcement learning, multi-agent learning, starcraft},
 location = {Montreal QC, Canada},
 numpages = {3},
 pages = {2186–2188},
 publisher = {International Foundation for Autonomous Agents and Multiagent Systems},
 title = {The StarCraft Multi-Agent Challenge},
 year = {2019}
}

@unpublished{Sanner10:RDDL,
 author = {Scott Sanner},
 note = {Specification in IPPC 2011},
 title = {Relational Dynamic Influence Diagram Language (RDDL): Language Description},
 year = {2010}
}

@inproceedings{Sedlmeier20:OOD,
 author = {Andreas Sedlmeier and Thomas Gabor and Thomy Phan and Lenz Belzner and Claudia Linnhoff-Popien},
 booktitle = {ICAART'2020},
 doi = {10.5220/0008949905220529},
 title = {Uncertainty-based Out-of-Distribution Classification in Deep Reinforcement Learning},
 url = {https://doi.org/10.5220%2F0008949905220529},
 year = {2020}
}

@article{Shapley:StochasticGames,
 author = {L. S. Shapley },
 doi = {10.1073/pnas.39.10.1095},
 eprint = {https://www.pnas.org/doi/pdf/10.1073/pnas.39.10.1095},
 journal = {Proceedings of the National Academy of Sciences},
 number = {10},
 pages = {1095-1100},
 title = {Stochastic Games*},
 url = {https://www.pnas.org/doi/abs/10.1073/pnas.39.10.1095},
 volume = {39},
 year = {1953}
}

@incollection{Shehory00:Software,
 author = {O. Shehory},
 booktitle = {1st International Workshop on Agent-Oriented Software Engineering, Revised papers},
 pages = {77--89},
 publisher = {ACM},
 title = {Software Architecture Attributes of Multi-agent Systems},
 year = {2000}
}

@inproceedings{Shell14:Improving,
 author = {Duane F. Shell and
Melissa Patterson Hazley and
Leen{-}Kiat Soh and
Lee Dee Miller and
Vlad Chiriacescu and
Elizabeth Ingraham},
 booktitle = {{IEEE} Frontiers in Education Conference, {FIE} 2014, Proceedings,
Madrid, Spain, October 22-25, 2014},
 pages = {1--7},
 publisher = {{IEEE} Computer Society},
 title = {Improving learning of computational thinking using computational creativity
exercises in a college {CSI} computer science course for engineers},
 year = {2014}
}

@article{Shell:2015,
 author = {Shell, D. and Soh, L.-K. and Chriacescu, V.},
 journal = {International Journal of Cognitive Informatics and Natural Intelligence},
 number = {3},
 pages = {1-24},
 title = {Modeling Self-Efficacy as a Dynamic Cognitive Process with the Computational-Unified Learning Model (C-ULM): Implications for Cognitive Informatics and Cognitive Computing},
 volume = {9},
 year = {2015}
}

@article{Sheridan2011,
 author = {Sheridan, Thomas},
 issue_date = {July 2011},
 journal = {IEEE Transactions on Systems, Man, and Cybernetics--Part A: Systems and Humans},
 number = {4},
 pages = {662-667},
 title = {Adaptive Automation, Level of Automation, Allocation Authority, Supervisory Control, and Adaptive Control: Distinctions and Modes of Adaptation},
 volume = {41},
 year = {2011}
}

@article{Shrestha19:Organizational,
 author = {Yash Raj Shrestha and Shiko M. Ben-Menahem and Georg von Krogh},
 journal = {California Management Review},
 number = {4},
 pages = {66-83},
 title = {Organizational Decision-Making Structures in the Age of Artificial Intelligence},
 volume = {61},
 year = {2019}
}

@inproceedings{Silver10:POMCP,
 author = {Silver, David and Veness, Joel},
 booktitle = {NIPS'10},
 pages = {2164--2172},
 title = {Monte-Carlo Planning in Large POMDPs},
 year = {2010}
}

@article{Silver16:Go,
 author = {David Silver and Aja Huang and Christopher J. Maddison and Arthur Guez and Laurent Sifre and George van den Driessche and Julian Schrittwieser and Ioannis Antonoglou and Veda Panneershelvam and Marc Lanctot and Sander Dieleman and Dominik Grewe and John Nham and Nal Kalchbrenner and Ilya Sutskever and Timothy Lillicrap and Madeleine Leach and Koray Kavukcuoglu and Thore Graepel and Demis Hassabis},
 journal = {Nature},
 pages = {484--503},
 title = {Mastering the game of Go with deep neural networks and tree search},
 volume = {529},
 year = {2016}
}

@article{Silver17:Go,
 author = {Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and Chen, Yutian and Lillicrap, Timothy and Hui, Fan and Sifre, Laurent and van den Driessche, George and Graepel, Thore and Hassabis, Demis},
 doi = {10.1038/nature24270},
 issn = {0028-0836},
 journal = {Nature},
 month = {October},
 number = {7676},
 pages = {354--359},
 publisher = {Nature Research},
 title = {{Mastering the game of {Go} without human knowledge}},
 volume = {550},
 year = {2017}
}

@inproceedings{Skrynnik24:Lifelong,
 author = {A. Skrynnik and A. Andreychuk and M. Nesterova and K. Yakovlev and A. Panov},
 booktitle = {AAAI-24},
 title = {Learn to Follow: Decentralized Lifelong Multi-Agent Pathfinding via Planning and Learning},
 year = {2024}
}

@inproceedings{Sodhani21:Multitask,
 author = {Sodhani, Shagun and Zhang, Amy and Pineau, Joelle},
 booktitle = {Proceedings of Machine Learning Research},
 pages = {9767--9779},
 publisher = {PMLR},
 title = {Multi-Task Reinforcement Learning with Context-based Representations},
 volume = {139},
 year = {2021}
}

@article{Soh15:CC,
 author = {L.-K. Soh and D. F. Shell and E. Ingraham and S. Ramsay and Brian Moore},
 journal = {Communications of the ACM},
 number = {8},
 pages = {33-35},
 title = {Viewpoint: Improving Learning and Achievement in Introductory Computer Science through Computational Creativity},
 volume = {58},
 year = {2015}
}

@inproceedings{Soh:2001,
 author = {L.-K. Soh and C. Tsatsoulis},
 booktitle = {IJCAI’01},
 pages = {1121-1127},
 title = {Reflective Negotiating Agents for Real-Time Multisensor Target Tracking},
 year = {2001}
}

@inproceedings{Soh:2003,
 author = {L.-K. Soh and C. Tsatsoulis},
 booktitle = {Proceedings of the IEEE International Conference on Systems, Man, and Cybernetics},
 pages = {1481-1486},
 title = {Utility-Based Multiagent Coalition Formation with Incomplete Information and Time Constraints},
 year = {2003}
}

@article{Soh:2005,
 author = {L.-K. Soh and C. Tsatsoulis},
 journal = {Autonomous Agents and Multi-Agent Systems},
 pages = {215-271},
 title = {A Real-Time Negotiation Model and A Multi-Agent Sensor Network Implementation},
 volume = {11},
 year = {2005}
}

@inproceedings{SohChen:2005,
 author = {L.-K. Soh and C. Chen},
 booktitle = {AAMAS},
 pages = {745-752},
 title = {Balancing Ontological and Operational Factors in Refining Multiagent Neighborhoods},
 year = {2005}
}

@inproceedings{SohLi:2003,
 author = {L.-K. Soh and X. Li},
 booktitle = {IJCAI’2003},
 pages = {619-624},
 title = {An Integrated Multilevel Learning Approach to Multiagent Coalition Formation},
 year = {2003}
}

@inproceedings{SohLi:2004,
 author = {L.-K. Soh and X. Li},
 booktitle = {AAMAS’2004},
 pages = {1048-1055},
 title = {Adaptive, Confidence-Based Multiagent Negotiation Strategy},
 year = {2004}
}

@inproceedings{Somani13:DESPOT,
 acmid = {2999810},
 address = {USA},
 author = {Somani, Adhiraj and Ye, Nan and Hsu, David and Lee, Wee Sun},
 booktitle = {NIPS'13},
 location = {Lake Tahoe, Nevada},
 numpages = {9},
 pages = {1772--1780},
 publisher = {Curran Associates Inc.},
 title = {DESPOT: Online POMDP Planning with Regularization},
 url = {http://dl.acm.org/citation.cfm?id=2999792.2999810},
 year = {2013}
}

@inproceedings{Sonu15:Anon,
 author = {E. Sonu and Y. Chen and P. Doshi},
 booktitle = {ICAPS},
 pages = {202--210},
 title = {Individual Planning in Agent Populations: Exploiting Anonymity and Frame-Action Hypergraphs},
 year = {2015}
}

@article{Sonu17,
 author = {E. Sonu and Y. Chen and P. Doshi},
 journal = {JAIR},
 pages = {725--770},
 title = {ecision-Theoretic Planning under Anonymity in Agent Populations},
 volume = {59},
 year = {2017}
}

@article{Sonu:2017,
 acmid = {3176805},
 address = {USA},
 author = {Sonu, Ekhlas and Chen, Yingke and Doshi, Prashant},
 issn = {1076-9757},
 issue_date = {May 2017},
 journal = {J. Artif. Int. Res.},
 month = {May},
 number = {1},
 numpages = {46},
 pages = {725--770},
 publisher = {AI Access Foundation},
 title = {Decision-theoretic Planning Under Anonymity in Agent Populations},
 url = {http://dl.acm.org/citation.cfm?id=3176788.3176805},
 volume = {59},
 year = {2017}
}

@inproceedings{Stone10:adhoc,
 author = {Peter Stone and Gal A. Kaminka and Sarit Kraus and Jeffrey S. Rosenschein},
 booktitle = {Twenty-Fourth AAAI Conference on Artificial Intelligence},
 title = {Ad Hoc Autonomous Agent Teams:  Collaboration without Pre-Coordination},
 year = {2010}
}

@inproceedings{Stone:AdHoc,
 author = {Peter Stone and Gal A. Kaminka and Sarit Kraus and Jeffrey S. Rosenschein},
 booktitle = {AAAI'10},
 pages = {1504-1509},
 title = {Ad Hoc Autonomous Agent Teams: Collaboration without Pre-Coordination},
 year = {2010}
}

@inproceedings{Subramanian:POMeanField,
 author = {S.G. Subramanian and M.E. Taylor and M. Crowley and P. Poupart},
 booktitle = {AAMAS 2021},
 pages = {537-545},
 title = {Partially Observable Mean Field Reinforcement Learning},
 year = {2021}
}

@inproceedings{Subramanian:TypedMeanField,
 author = {S.G. Subramanian and P. Poupart and M.E. Taylor and N. Hegde},
 booktitle = {AAMAS 2020},
 pages = {411-419},
 title = {Multi Type Mean Field Reinforcement Learning},
 year = {2020}
}

@article{Sunberg:PFT-DPW,
 author = {Sunberg, Zachary and Kochenderfer, Mykel},
 copyright = {arXiv.org perpetual, non-exclusive license},
 doi = {10.48550/ARXIV.1709.06196},
 journal = {arXiv},
 keywords = {Artificial Intelligence (cs.AI), Robotics (cs.RO), Systems and Control (eess.SY), FOS: Computer and information sciences, FOS: Computer and information sciences, FOS: Electrical engineering, electronic engineering, information engineering, FOS: Electrical engineering, electronic engineering, information engineering},
 publisher = {arXiv},
 title = {Online algorithms for POMDPs with continuous state, action, and observation spaces},
 url = {https://arxiv.org/abs/1709.06196},
 year = {2017}
}

@article{supersuit,
 author = {Terry, J. K and Black, Benjamin and Hari, Ananth},
 journal = {arXiv preprint arXiv:2008.08932},
 title = {SuperSuit: Simple Microwrappers for Reinforcement Learning Environments},
 year = {2020}
}

@book{Sutton:RL,
  author = {Sutton, Richard S. and Barto, Andrew G.},
 publisher = {A Bradford Book},
 title = {Reinforcement Learning: An Introduction},
 year = {2018}
}

@article{Sycara98,
 author = {K. Sycara},
 journal = {AI Magazine},
 number = {2},
 pages = {79-92},
 title = {Multiagent Systems},
 volume = {19},
 year = {1998}
}

@inproceedings{Tanaka03:Multitask,
 author = {Tanaka, F. and Yamamura, M.},
 booktitle = {IEEE International Symposium on Computational Intelligence in Robotics and Automation},
 pages = {1108-1113 vol.3},
 title = {Multitask reinforcement learning on the distribution of MDPs},
 volume = {3},
 year = {2003}
}

@Article{electronics9091363,
AUTHOR = {Vithayathil Varghese, Nelson and Mahmoud, Qusay H.},
TITLE = {A Survey of Multi-Task Deep Reinforcement Learning},
JOURNAL = {Electronics},
VOLUME = {9},
YEAR = {2020},
NUMBER = {9},
ARTICLE-NUMBER = {1363},
URL = {https://www.mdpi.com/2079-9292/9/9/1363},
ISSN = {2079-9292},
ABSTRACT = {Driven by the recent technological advancements within the field of artificial intelligence research, deep learning has emerged as a promising representation learning technique across all of the machine learning classes, especially within the reinforcement learning arena. This new direction has given rise to the evolution of a new technological domain named deep reinforcement learning, which combines the representational learning power of deep learning with existing reinforcement learning methods. Undoubtedly, the inception of deep reinforcement learning has played a vital role in optimizing the performance of reinforcement learning-based intelligent agents with model-free based approaches. Although these methods could improve the performance of agents to a greater extent, they were mainly limited to systems that adopted reinforcement learning algorithms focused on learning a single task. At the same moment, the aforementioned approach was found to be relatively data-inefficient, particularly when reinforcement learning agents needed to interact with more complex and rich data environments. This is primarily due to the limited applicability of deep reinforcement learning algorithms to many scenarios across related tasks from the same environment. The objective of this paper is to survey the research challenges associated with multi-tasking within the deep reinforcement arena and present the state-of-the-art approaches by comparing and contrasting recent solutions, namely DISTRAL (DIStill & TRAnsfer Learning), IMPALA(Importance Weighted Actor-Learner Architecture) and PopArt that aim to address core challenges such as scalability, distraction dilemma, partial observability, catastrophic forgetting and negative knowledge transfer.},
DOI = {10.3390/electronics9091363}
}
@inproceedings{okumura2023lacam,
  title={LaCAM: Search-Based Algorithm for Quick Multi-Agent Pathfinding},
  booktitle={Proceedings of AAAI Conference on Artificial Intelligence (AAAI)},
  year={2023},
  author={Okumura, Keisuke}
}


@article{Taylor09:Transfer,
 author = {Taylor, Matthew E. and Stone, Peter},
 journal = {J. Mach. Learn. Res.},
 month = {dec},
 numpages = {53},
 pages = {1633–1685},
 title = {Transfer Learning for Reinforcement Learning Domains: A Survey},
 volume = {10},
 year = {2009}
}

@article{terry2021pettingzoo,
 author = {Terry, J and Black, Benjamin and Grammel, Nathaniel and Jayakumar, Mario and Hari, Ananth and Sullivan, Ryan and Santos, Luis S and Dieffendahl, Clemens and Horsch, Caroline and Perez-Vicente, Rodrigo and others},
 journal = {NeurIPS},
 pages = {15032--15043},
 title = {Pettingzoo: Gym for multi-agent reinforcement learning},
 volume = {34},
 year = {2021}
}

@inproceedings{Thomas:rhoPOMCP,
 author = {Vincent Thomas and Geremy Hutin and Olivier Buffet},
 booktitle = {ECAI'20},
 title = {Monte Information-Oriented Planning},
 year = {2020}
}

@article{Thrun1995:Lifelong,
 author = {Thrun, S. and Mitchell, T.M.},
 issue_date = {July 1995},
 journal = {Robotics and Autonomous Systems},
 number = {1-2},
 pages = {25-46},
 title = {Lifelong Robot Learning},
 volume = {15},
 year = {1995}
}

@misc{uber,
 author = {Uber},
 howpublished = {\url{https://www.uber.com/blog/california/keeping-you-in-the-drivers-seat-1}},
 note = {Accessed: 2024-08-15},
 title = {{Uber} Airline Pickup},
 year = {2019}
}

@inproceedings{Ure15:Fire,
 author = {N.K. Ure and S. Omidshafiei and B.T. Lopez and A.-A. Agha-Mohammadi and J.P. How and J. Vian},
 booktitle = {IROS},
 pages = {5181--5188},
 title = {Online Heterogeneous Multiagent Learning Under Limited Communication with Applications to Forest Fire Management},
 year = {2015}
}

@article{varghese2021:Multitask,
 author = {Varghese, Nelson Vithayathil and Mahmoud, Qusay H.},
 doi = {10.1109/ACCESS.2021.3065710},
 journal = {IEEE Access},
 keywords = {Reinforcement learning;Task analysis;Intelligent agents;Transfer learning;Optimization;Mathematical model;Games;Machine learning;deep reinforcement learning;neural networks;transfer learning;actor-critic;multi-task worker},
 number = {},
 pages = {44681-44703},
 title = {A Hybrid Multi-Task Learning Approach for Optimizing Deep Reinforcement Learning Agents},
 volume = {9},
 year = {2021}
}

@incollection{Vassileva99:Openness,
 author = {J. I. Vassileva and J. E. Greer and G. I. McCalla},
 booktitle = {Workshop on Open, Interactive, and Other Overt Approaches to Learner Modeling},
 publisher = {AIED Society},
 title = {Openness and Disclosure in Multi-Agent Learner Models},
 year = {1999}
}

@inproceedings{Velagapudi:DistModel,
 author = {Velagapudi, Prasanna and Varakantham, Pradeep and Sycara, Katia and Scerri, Paul},
 booktitle = {10th International Conference on Autonomous Agents and Multiagent Systems-Volume 3},
 pages = {955--962},
 title = {Distributed model shaping for scaling to decentralized {POMDP}s with hundreds of agents},
 year = {2011}
}

@inproceedings{Volodymyr16:Asynchronous,
 author = {Mnih, Volodymyr and Badia, Adri\`{a} Puigdom\`{e}nech and Mirza, Mehdi and Graves, Alex and Harley, Tim and Lillicrap, Timothy P. and Silver, David and Kavukcuoglu, Koray},
 booktitle = {Proceedings of the 33rd International Conference on International Conference on Machine Learning - Volume 48},
 numpages = {10},
 pages = {1928–1937},
 title = {Asynchronous Methods for Deep Reinforcement Learning},
 year = {2016}
}

@inproceedings{Wang16:Structural,
 author = {Wang, Daixin and Cui, Peng and Zhu, Wenwu},
 booktitle = {KDD '16},
 numpages = {10},
 pages = {1225–1234},
 title = {Structural Deep Network Embedding},
 year = {2016}
}

@article{Wei16:Dynamic,
 author = {Wei,Changyun and Hindriks,Koen V. and Jonker,Catholijn M.},
 journal = {Applied Intelligence},
 month = {09},
 number = {2},
 pages = {383-401},
 title = {Dynamic task allocation for multi-robot search and retrieval tasks},
 volume = {45},
 year = {2016}
}

@inproceedings{Weyns04,
 author = {D. Weyns and H. V. D. Parunak and F. Michel and T. Holvoet and J. Ferber},
 booktitle = {E4MAS’2004},
 pages = {1-47},
 title = {Environments for Multiagent Systems State-of-the-Art and Research Challenges},
 year = {2004}
}

@inproceedings{Yang:MeanField,
 author = {Y. Yang and R. Luo and M. Li and M. Zhou and W. Zhang and J. Wang},
 booktitle = {ICML 2018},
 pages = {4686-4693},
 title = {Mean Field Multi-agent Reinforcement Learning},
 year = {2018}
}

@inproceedings{Yuetal2023:Multitask,
 author = {Y. Yu and Q. Yin and J. Zhang and K. Huang},
 booktitle = {AAMAS},
 pages = {1615-1623},
 title = {Prioritized tasks mining for multi-task cooperative multi-agent reinforcement learning},
 year = {2023}
}

@inproceedings{Zhang23:PolicyChange,
 author = {Ziqian Zhang and Lei Yuan and Lihe Li and Ke Xue and Chengxing Jia and Cong Guan and Chao Qian and Yang Yu},
 booktitle = {Proceedings of the 39th Conference on Uncertainty in Artificial Intelligence},
 title = {Fast Teammate Adaptation in the Presence of Sudden Policy Change},
 url = {https://openreview.net/pdf?id=akeVfjnPfTS},
 year = {2023}
}

@inproceedings{Zhangetal2023:Multitask,
 author = {Fuxiang Zhang and Chengxing Jia and Yi-Chen Li and Lei Yuan and Yang Yu and Zhongzhang Zhang},
 booktitle = {ICLR},
 title = {Discovering generalizable multi-agent coordination skills from multi-task offline data},
 year = {2023}
}

@article{Zhou:OpenEnvironments,
 abstract = {{Conventional machine learning studies generally assume close-environment scenarios where important factors of the learning process hold invariant. With the great success of machine learning, nowadays, more and more practical tasks, particularly those involving open-environment scenarios where important factors are subject to change, called open-environment machine learning in this article, are present to the community. Evidently, it is a grand challenge for machine learning turning from close environment to open environment. It becomes even more challenging since, in various big data tasks, data are usually accumulated with time, like streams, while it is hard to train the machine learning model after collecting all data as in conventional studies. This article briefly introduces some advances in this line of research, focusing on techniques concerning emerging new classes, decremental/incremental features, changing data distributions and varied learning objectives, and discusses some theoretical issues.}},
 author = {Zhou, Zhi-Hua},
 doi = {10.1093/nsr/nwac123},
 eprint = {https://academic.oup.com/nsr/article-pdf/9/8/nwac123/45957092/nwac123.pdf},
 issn = {2095-5138},
 journal = {National Science Review},
 month = {07},
 note = {nwac123},
 number = {8},
 title = {{Open-environment machine learning}},
 url = {https://doi.org/10.1093/nsr/nwac123},
 volume = {9},
 year = {2022}
}
