@inproceedings{ferns2014bisimulation,
  title={Bisimulation Metrics are Optimal Value Functions.},
  author={Ferns, Norman and Precup, Doina},
  booktitle={UAI},
  pages={210--219},
  year={2014}
}

@inproceedings{larsen1989bisimulation,
  title={Bisimulation through probabilistic testing (preliminary report)},
  author={Larsen, Kim G and Skou, Arne},
  booktitle={Proceedings of the 16th ACM SIGPLAN-SIGACT symposium on Principles of programming languages},
  pages={344--352},
  year={1989}
}

@inproceedings{zhang2020learning,
  title={Learning Invariant Representations for Reinforcement Learning without Reconstruction},
  author={Zhang, Amy and McAllister, Rowan Thomas and Calandra, Roberto and Gal, Yarin and Levine, Sergey},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@article{agarwal2021deep,
  title={Deep reinforcement learning at the edge of the statistical precipice},
  author={Agarwal, Rishabh and Schwarzer, Max and Castro, Pablo Samuel and Courville, Aaron C and Bellemare, Marc},
  journal={Advances in neural information processing systems},
  volume={34},
  pages={29304--29320},
  year={2021}
}

@inproceedings{balaji2020factoredrl,
  title={Factoredrl: Leveraging factored graphs for deep reinforcement learning},
  author={Balaji, Bharathan and Christodoulou, Petros and Lu, Xiaoyu and Jeon, Byungsoo and Bell-Masterson, Jordan},
  booktitle={NeurIPS 2023 Workshop on Deep Reinforcement Learning},
  year={2020}
}

@inproceedings{geisser2020trial,
  title={Trial-Based Heuristic Tree Search for MDPs with Factored Action Spaces},
  author={Gei{\ss}er, Florian and Speck, David and Keller, Thomas},
  booktitle={Proceedings of the International Symposium on Combinatorial Search},
  volume={11},
  pages={38--47},
  year={2020}
}

@article{acid2003searching,
  title={Searching for Bayesian network structures in the space of restricted acyclic partially directed graphs},
  author={Acid, Silvia and de Campos, Luis M},
  journal={Journal of Artificial Intelligence Research},
  volume={18},
  pages={445--490},
  year={2003}
}

@inproceedings{ramsey2006adjacency,
  title={Adjacency-faithfulness and conservative causal inference},
  author={Ramsey, Joseph and Spirtes, Peter and Zhang, Jiji},
  booktitle={Proceedings of the Twenty-Second Conference on Uncertainty in Artificial Intelligence},
  pages={401--408},
  year={2006}
}

@inproceedings{ozair2021vector,
  title={Vector quantized models for planning},
  author={Ozair, Sherjil and Li, Yazhe and Razavi, Ali and Antonoglou, Ioannis and Van Den Oord, Aaron and Vinyals, Oriol},
  booktitle={international conference on machine learning},
  pages={8302--8313},
  year={2021},
  organization={PMLR}
}

@inproceedings{grill2020monte,
  title={Monte-Carlo tree search as regularized policy optimization},
  author={Grill, Jean-Bastien and Altch{\'e}, Florent and Tang, Yunhao and Hubert, Thomas and Valko, Michal and Antonoglou, Ioannis and Munos, R{\'e}mi},
  booktitle={International Conference on Machine Learning},
  pages={3769--3778},
  year={2020},
  organization={PMLR}
}

@inproceedings{couetoux2011continuous,
  title={Continuous upper confidence trees},
  author={Cou{\"e}toux, Adrien and Hoock, Jean-Baptiste and Sokolovska, Nataliya and Teytaud, Olivier and Bonnard, Nicolas},
  booktitle={Learning and Intelligent Optimization: 5th International Conference, LION 5, Rome, Italy, January 17-21, 2011. Selected Papers 5},
  pages={433--445},
  year={2011},
  organization={Springer}
}

@inproceedings{hostetler2014state,
  title={State aggregation in Monte Carlo tree search},
  author={Hostetler, Jesse and Fern, Alan and Dietterich, Tom},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={28},
  year={2014}
}

@inproceedings{ye2021mastering,
	author = {Ye, Weirui and Liu, Shaohuai and Kurutach, Thanard and Abbeel, Pieter and Gao, Yang},
	booktitle = {Advances in Neural Information Processing Systems},
	pages = {25476--25488},
	publisher = {Curran Associates, Inc.},
	title = {Mastering Atari Games with Limited Data},
	volume = {34},
	year = {2021}
}



@inproceedings{brouillard2020differentiable,
 author = {Brouillard, Philippe and Lachapelle, S\'{e}bastien and Lacoste, Alexandre and Lacoste-Julien, Simon and Drouin, Alexandre},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {21865--21877},
 title = {Differentiable Causal Discovery from Interventional Data},
 volume = {33},
 year = {2020}
}

@article{Selvaraju2016GradCAMVE,
  title={Grad-CAM: Visual Explanations from Deep Networks via Gradient-Based Localization},
  author={Ramprasaath R. Selvaraju and Abhishek Das and Ramakrishna Vedantam and Michael Cogswell and Devi Parikh and Dhruv Batra},
  journal={International Journal of Computer Vision},
  year={2016},
  volume={128},
  pages={336-359}
}

@article{rosin2011multi,
  title={Multi-armed bandits with episode context},
  author={Rosin, Christopher D},
  journal={Annals of Mathematics and Artificial Intelligence},
  volume={61},
  number={3},
  pages={203--230},
  year={2011},
  publisher={Springer}
}

@inproceedings{
sokota2021monte,
title={Monte Carlo Tree Search With Iteratively Refining State Abstractions},
author={Samuel Sokota and Caleb Ho and Zaheen Farraz Ahmad and J Zico Kolter},
booktitle={Advances in Neural Information Processing Systems},
year={2021}
}

@article{silver2017mastering,
  title={Mastering the game of go without human knowledge},
  author={Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and others},
  journal={Nature},
  volume={550},
  number={7676},
  pages={354--359},
  year={2017},
  publisher={Nature Publishing Group}
}

@article{boutilier2013contextspecific,
  author    = {Craig Boutilier and
               Nir Friedman and
               Mois{\'{e}}s Goldszmidt and
               Daphne Koller},
  title     = {Context-Specific Independence in Bayesian Networks},
  journal   = {CoRR},
  volume    = {abs/1302.3562},
  year      = {2013},
  eprinttype = {arXiv},
  eprint    = {1302.3562},
  timestamp = {Mon, 13 Aug 2018 16:46:49 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1302-3562.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{schrittwieser2020mastering,
  title={Mastering atari, go, chess and shogi by planning with a learned model},
  author       = {Julian Schrittwieser and
                  Ioannis Antonoglou and
                  Thomas Hubert and
                  Karen Simonyan and
                  Laurent Sifre and
                  Simon Schmitt and
                  Arthur Guez and
                  Edward Lockhart and
                  Demis Hassabis and
                  Thore Graepel and
                  Timothy P. Lillicrap and
                  David Silver},
  journal={Nature},
  volume={588},
  number={7839},
  pages={604--609},
  year={2020},
  publisher={Nature Publishing Group UK London}
}

@inproceedings{He2016IdentityMI,
  title={Identity Mappings in Deep Residual Networks},
  author={Kaiming He and X. Zhang and Shaoqing Ren and Jian Sun},
  booktitle={European Conference on Computer Vision},
  year={2016},
}

@article{Ba2016LayerN,
  title={Layer normalization},
  author={Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E},
  journal={arXiv preprint arXiv:1607.06450},
  year={2016}
}

@article{silver2016mastering,
  author       = {David Silver and
                  Aja Huang and
                  Chris J. Maddison and
                  Arthur Guez and
                  Laurent Sifre and
                  George van den Driessche and
                  Julian Schrittwieser and
                  Ioannis Antonoglou and
                  Vedavyas Panneershelvam and
                  Marc Lanctot and
                  Sander Dieleman and
                  Dominik Grewe and
                  John Nham and
                  Nal Kalchbrenner and
                  Ilya Sutskever and
                  Timothy P. Lillicrap and
                  Madeleine Leach and
                  Koray Kavukcuoglu and
                  Thore Graepel and
                  Demis Hassabis},
  title        = {Mastering the game of Go with deep neural networks and tree search},
  journal      = {Nature},
  volume       = {529},
  number       = {7587},
  pages        = {484--489},
  year         = {2016}
}



@inproceedings{
hwang2023on,
title={On Discovery of Local Independence over Continuous Variables via Neural Contextual Decomposition},
author={Inwoo Hwang and Yunhyeok Kwak and Yeon-Ji Song and Byoung-Tak Zhang and Sanghack Lee},
booktitle={2nd Conference on Causal Learning and Reasoning},
year={2023}
}

@misc{gym_minigrid,
  author = {Chevalier-Boisvert, Maxime and Willems, Lucas and Pal, Suman},
  title = {Minimalistic Gridworld Environment for OpenAI Gym},
  year = {2018},
  publisher = {GitHub},
  journal = {GitHub repository},
  howpublished = {\url{https://github.com/maximecb/gym-minigrid}},
}

@misc{SchraderSokoban2018,
  author = {Schrader, Max-Philipp B.},
  title = {gym-sokoban},
  year = {2018},
  publisher = {GitHub},
  journal = {GitHub repository},
  howpublished = {\url{https://github.com/mpSchrader/gym-sokoban}},
  commit = {#CommitId}
}

@article{jang2016categorical,
  title={Categorical reparameterization with gumbel-softmax},
  author={Jang, Eric and Gu, Shixiang and Poole, Ben},
  journal={arXiv preprint arXiv:1611.01144},
  year={2016}
}

@article{maddison2016concrete,
  title={The concrete distribution: A continuous relaxation of discrete random variables},
  author={Maddison, Chris J and Mnih, Andriy and Teh, Yee Whye},
  journal={arXiv preprint arXiv:1611.00712},
  year={2016}
}

@inproceedings{kingma2015adam,
  author       = {Diederik P. Kingma and
                  Jimmy Ba},
  title        = {Adam: {A} Method for Stochastic Optimization},
  booktitle    = {3rd International Conference on Learning Representations, {ICLR} 2015,
                  San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings},
  year         = {2015}
}

@inproceedings{
loshchilov2018decoupled,
title={Decoupled Weight Decay Regularization},
author={Ilya Loshchilov and Frank Hutter},
booktitle={International Conference on Learning Representations},
year={2019}
}


@inproceedings{monte_sokota_2021,
 author = {Sokota, Samuel and Ho, Caleb Y and Ahmad, Zaheen and Kolter, J. Zico},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {18698--18709},
 publisher = {Curran Associates, Inc.},
 title = {Monte Carlo Tree Search With Iteratively Refining State Abstractions},
 volume = {34},
 year = {2021}
}

@inproceedings{chitnis2021camps,
  title={CAMPS: Learning context-specific abstractions for efficient planning in factored MDPs},
  author={Chitnis, Rohan and Silver, Tom and Kim, Beomjoon and Kaelbling, Leslie and Lozano-Perez, Tomas},
  booktitle={Conference on Robot Learning},
  pages={64--79},
  year={2021},
  organization={PMLR}
}

@inproceedings{tang2022leveraging,
 author = {Tang, Shengpu and Makar, Maggie and Sjoding, Michael and Doshi-Velez, Finale and Wiens, Jenna},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {34272--34286},
 title = {Leveraging Factored Action Spaces for Efficient Offline Reinforcement Learning in Healthcare},
 volume = {35},
 year = {2022}
}

@misc{rebello2023leveraging,
      title={Leveraging Factored Action Spaces for Off-Policy Evaluation}, 
      author={Aaman Rebello and Shengpu Tang and Jenna Wiens and Sonali Parbhoo},
      year={2023},
      eprint={2307.07014},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@inproceedings{Tkachuk2023EfficientPI,
  title={Efficient Planning in Combinatorial Action Spaces with Applications to Cooperative Multi-Agent Reinforcement Learning},
  author={Volodymyr Tkachuk and Seyed Alireza Bakhtiari and Johannes Kirschner and Matej Jusup and Ilija Bogunovic and Csaba Szepesvari},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  year={2023}
}

@article{Mahajan2021ReinforcementLI,
  title={Reinforcement Learning in Factored Action Spaces using Tensor Decompositions},
  author={Anuj Mahajan and Mikayel Samvelyan and Lei Mao and Viktor Makoviychuk and Animesh Garg and Jean Kossaifi and Shimon Whiteson and Yuke Zhu and Anima Anandkumar},
  journal={ArXiv},
  year={2021},
  volume={abs/2110.14538}
}

@inproceedings{
schrittwieser2021online,
title={Online and Offline Reinforcement Learning by Planning with a Learned Model},
author={Julian Schrittwieser and Thomas K Hubert and Amol Mandhane and Mohammadamin Barekatain and Ioannis Antonoglou and David Silver},
booktitle={Advances in Neural Information Processing Systems},
year={2021}
}

% Exported from Litmaps (https://www.litmaps.com)

@article{betazero_moss_2023,
    title = {BetaZero: Belief-State Planning for Long-Horizon POMDPs using Learned Approximations},
    doi = {10.48550/ARXIV.2306.00249},
    author = {Moss, Robert J. and Corso, Anthony and Caers, J. and Kochenderfer, Mykel J.},
    journal = {arXiv.org},
    year = {2023},
    litmapsId = {262123338}
}


@article{adaptive_hoerger_2023,
    title = {Adaptive Discretization using Voronoi Trees for Continuous POMDPs},
    doi = {10.48550/ARXIV.2302.10439},
    author = {Hoerger, Marcus and Kurniawati, H. and Kroese, Dirk P. and Ye, N.},
    journal = {ArXiv},
    year = {2023},
    litmapsId = {254273937}
}



@article{elastic_xu_2022,
    title = {Elastic Monte Carlo Tree Search with State Abstraction for Strategy Game Playing},
    doi = {10.1109/COG51982.2022.9893587},
    author = {Xu, Linjie and Grueso, Jorge Hurtado and Jeurissen, Dominik and Liebana, Diego Perez and Dockhorn, Alexander},
    journal = {2022 IEEE Conference on Games (CoG)},
    year = {2022},
    litmapsId = {237757625}
}



@article{voronoi_lim_2021,
    title = {Voronoi Progressive Widening: Efficient Online Solvers for Continuous State, Action, and Observation POMDPs},
    doi = {10.1109/CDC45484.2021.9683490},
    author = {Lim, M. H. and Tomlin, C. and Sunberg, Zachary},
    journal = {IEEE Conference on Decision and Control},
    year = {2021},
    litmapsId = {31597323}
}




@article{game_dockhorn_2021,
    title = {Game State and Action Abstracting Monte Carlo Tree Search for General Strategy Game-Playing},
    doi = {10.1109/COG52621.2021.9619029},
    author = {Dockhorn, Alexander and Grueso, Jorge Hurtado and Jeurissen, Dominik and Xu, Linjie and Liebana, Diego Perez},
    journal = {2021 IEEE Conference on Games (CoG)},
    year = {2021},
    litmapsId = {254513056}
}


@article{montecarlo_lee_2020,
    title = {Monte-Carlo Tree Search in Continuous Action Spaces with Value Gradients},
    doi = {10.1609/AAAI.V34I04.5885},
    author = {Lee, Jongmin and Jeon, Wonseok and Kim, Geon-hyeong and Kim, Kee-Eung},
    journal = {AAAI Conference on Artificial Intelligence},
    year = {2020},
    litmapsId = {35222053}
}

@inproceedings{monte_kim_2020,
  title={Monte carlo tree search in continuous spaces using voronoi optimistic optimization with regret bounds},
  author={Kim, Beomjoon and Lee, Kyungjae and Lim, Sungbin and Kaelbling, Leslie and Lozano-P{\'e}rez, Tom{\'a}s},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  pages={9916--9924},
  year={2020}
}


@article{memoryaugmented_xiao_2018,
    title = {Memory-Augmented Monte Carlo Tree Search.},
    author = {Xiao, Chenjun and Mei, Jincheng and Müller, Martin},
    journal = {national conference on artificial intelligence},
    year = {2018},
    litmapsId = {249547845}
}


@article{samplebased_hostetler_2017,
    title = {Sample-Based Tree Search with Fixed and Adaptive State Abstractions},
    doi = {10.1613/JAIR.5483},
    author = {Hostetler, Jesse and Fern, Alan and Dietterich, Thomas G.},
    journal = {J. Artif. Intell. Res.},
    year = {2017},
    litmapsId = {129976835}
}


@article{curling_ohto_2017,
    title = {A Curling Agent Based on the Monte-Carlo Tree Search Considering the Similarity of the Best Action Among Similar States},
    doi = {10.1007/978-3-319-71649-7_13},
    author = {Ohto, Katsuki and Tanaka, Tetsuro},
    journal = {Lecture Notes in Computer Science},
    year = {2017},
    litmapsId = {134178261}
}


@article{monte_yee_2016,
    title = {Monte Carlo tree search in continuous action spaces with execution uncertainty},
    author = {Yee, Timothy and Lisy, Viliam and Bowling, Michael},
    year = {2016},
    litmapsId = {262401124}
}


@article{markovian_bai_2016,
    title = {Markovian state and action abstractions for MDPS via hierarchical MCTS},
    author = {Bai, Aijun and Srivastava, Siddharth and Russell, Stuart},
    year = {2016},
    litmapsId = {126779923}
}


@article{action_ahmad_2016,
    title = {Action selection for hammer shots in curling},
    author = {Ahmad, Zaheen Farraz and Holte, Robert C. and Bowling, Michael},
    year = {2016},
    litmapsId = {218576472}
}


@article{ogauct_anand_2016,
    title = {OGA-UCT: on-the-go abstractions in UCT},
    author = {Anand, Ankit and Noothigattu, Ritesh and Singla, Parag},
    year = {2016},
    litmapsId = {90267217}
}


@article{asapuct_anand_2015,
    title = {ASAP-UCT: abstraction of state-action pairs in UCT},
    author = {Anand, Ankit and Grover, Aditya and Mausam, Mausam and Singla, Parag},
    year = {2015},
    litmapsId = {126768827}
}


@article{progressive_hostetler_2015,
    title = {Progressive abstraction refinement for sparse sampling},
    author = {Hostetler, Jesse and Fern, Alan and Dietterich, Thomas G.},
    year = {2015},
    litmapsId = {155384540}
}


@article{progressive_hostetler_2015,
    title = {Progressive abstraction refinement for sparse sampling},
    author = {Hostetler, Jesse and Fern, Alan and Dietterich, Thomas},
    journal = {uncertainty in artificial intelligence},
    year = {2015},
    litmapsId = {249676244}
}


@article{state_hostetler_2014,
    title = {State aggregation in Monte Carlo tree search},
    author = {Hostetler, Jesse and Fern, Alan and Dietterich, Thomas G.},
    year = {2014},
    litmapsId = {185653089}
}


@article{improving_jiang_2014,
    title = {Improving UCT planning via approximate homomorphisms},
    doi = {10.5555/2615731.2617453},
    author = {Jiang, Nan and Singh, Satinder and Lewis, Richard L.},
    year = {2014},
    litmapsId = {78087659}
}


@article{continuous_coutoux_2011,
    title = {Continuous upper confidence trees},
    doi = {10.1007/978-3-642-25566-3_32},
    author = {Couëtoux, Adrien and Hoock, Jean-Baptiste and Sokolovska, Nataliya and Teytaud, Olivier and Bonnard, Nicolas},
    journal = {learning and intelligent optimization},
    year = {2011},
    litmapsId = {93330834}
}

@article{browne2012survey,
  title={A survey of monte carlo tree search methods},
  author={Browne, Cameron B and Powley, Edward and Whitehouse, Daniel and Lucas, Simon M and Cowling, Peter I and Rohlfshagen, Philipp and Tavener, Stephen and Perez, Diego and Samothrakis, Spyridon and Colton, Simon},
  journal={IEEE Transactions on Computational Intelligence and AI in games},
  volume={4},
  number={1},
  pages={1--43},
  year={2012},
  publisher={IEEE}
}

@article{guestrin2003efficient,
  title={Efficient solution algorithms for factored MDPs},
  author={Guestrin, Carlos and Koller, Daphne and Parr, Ronald and Venkataraman, Shobha},
  journal={Journal of Artificial Intelligence Research},
  volume={19},
  pages={399--468},
  year={2003}
}

@book{puterman2014markov,
  title={Markov decision processes: discrete stochastic dynamic programming},
  author={Puterman, Martin L},
  year={2014},
  publisher={John Wiley \& Sons}
}

@article{bellman1957markovian,
  title={A Markovian decision process},
  author={Bellman, Richard},
  journal={Journal of mathematics and mechanics},
  pages={679--684},
  year={1957},
  publisher={JSTOR}
}

@inproceedings{coulom2006efficient,
  title={Efficient selectivity and backup operators in Monte-Carlo tree search},
  author={Coulom, R{\'e}mi},
  booktitle={International conference on computers and games},
  pages={72--83},
  year={2006},
  organization={Springer}
}

@inproceedings{Qin2014ContextualCB,
  title={Contextual Combinatorial Bandit and its Application on Diversified Online Recommendation},
  author={Lijing Qin and Shouyuan Chen and Xiaoyan Zhu},
  booktitle={SDM},
  year={2014}
}

@inproceedings{Chen2018ContextualCM,
  title={Contextual Combinatorial Multi-armed Bandits with Volatile Arms and Submodular Reward},
  author={Lixing Chen and J. Xu and Zhuo Lu},
  booktitle={Neural Information Processing Systems},
  year={2018}
}

@article{Chen2014CombinatorialMB,
  author  = {Wei Chen and Yajun Wang and Yang Yuan and Qinshi Wang},
  title   = {Combinatorial Multi-Armed Bandit and Its Extension to Probabilistically Triggered Arms},
  journal = {Journal of Machine Learning Research},
  year    = {2016},
  volume  = {17},
  number  = {50},
  pages   = {1-33}
}

@article{pinto2017,
  author  = {Jervis Pinto and Alan Fern},
  title   = {Learning Partial Policies to Speedup MDP Tree Search via Reduction to I.I.D. Learning},
  journal = {Journal of Machine Learning Research},
  year    = {2017},
  volume  = {18},
  number  = {65},
  pages   = {1--35}
}

@article{veeriah2022grasp,
  title   = {GrASP: Gradient-Based Affordance Selection for Planning},
  author  = {Vivek Veeriah and Zeyu Zheng and Richard Lewis and Satinder Singh},
  year    = {2022},
  journal = {arXiv preprint arXiv: 2202.04772}
}
