@inproceedings{Haarnoja2018,
  doi = {10.1109/icra.2018.8460756},
  url = {https://doi.org/10.1109/icra.2018.8460756},
  year = {2018},
  month = may,
  publisher = {{IEEE}},
  author = {Tuomas Haarnoja and Vitchyr Pong and Aurick Zhou and Murtaza Dalal and Pieter Abbeel and Sergey Levine},
  title = {Composable Deep Reinforcement Learning for Robotic Manipulation},
  booktitle = {2018 {IEEE} International Conference on Robotics and Automation ({ICRA})}
}

@misc{haarnoja2019_learn,
      title={Soft Actor-Critic Algorithms and Applications}, 
      author={Tuomas Haarnoja and Aurick Zhou and Kristian Hartikainen and George Tucker and Sehoon Ha and Jie Tan and Vikash Kumar and Henry Zhu and Abhishek Gupta and Pieter Abbeel and Sergey Levine},
      year={2019},
      eprint={1812.05905},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{paszke2019pytorch,
  title={Pytorch: An imperative style, high-performance deep learning library},
  author={Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and others},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@article{LevineTutorial,
	author = {Levine, Sergey},
	title = {{Reinforcement Learning and Control as Probabilistic Inference: Tutorial and Review}},
	journal = {arXiv},
	year = {2018},
	month = {May},
	eprint = {1805.00909},
	url = {https://arxiv.org/abs/1805.00909v3}
}

@article{boolean,
  title   = {A Boolean task algebra for reinforcement learning},
  author  = {Geraud Nangue Tasse and Steven James and Benjamin Rosman},
  journal = {Advances in Neural Information Processing Systems},
  volume  = {33},
  pages   = {9497--9507},
  year    = {2020}
}


@InProceedings{vanNiekerk,
  title = 	 {Composing Value Functions in Reinforcement Learning},
  author =       {Van Niekerk, Benjamin and James, Steven and Earle, Adam and Rosman, Benjamin},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {6401--6409},
  year = 	 {2019},
  editor = 	 {Chaudhuri, Kamalika and Salakhutdinov, Ruslan},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {06},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/van-niekerk19a/van-niekerk19a.pdf},
  url = 	 {https://proceedings.mlr.press/v97/van-niekerk19a.html},
  abstract = 	 {An important property for lifelong-learning agents is the ability to combine existing skills to solve new unseen tasks. In general, however, it is unclear how to compose existing skills in a principled manner. Under the assumption of deterministic dynamics, we prove that optimal value function composition can be achieved in entropy-regularised reinforcement learning (RL), and extend this result to the standard RL setting. Composition is demonstrated in a high-dimensional video game, where an agent with an existing library of skills is immediately able to solve new tasks without the need for further learning.}
}

@book{suttonBook,
  title={Reinforcement learning: An introduction},
  author={Sutton, Richard S and Barto, Andrew G},
  year={2018},
  publisher={MIT press}
}

@inproceedings{Todorov,
 author = {Todorov, Emanuel},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {Y. Bengio and D. Schuurmans and J. Lafferty and C. Williams and A. Culotta},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Compositionality of optimal control laws},
 url = {https://proceedings.neurips.cc/paper/2009/file/3eb71f6293a2a31f3569e10af6552658-Paper.pdf},
 volume = {22},
 year = {2009}
}

@inproceedings{eysenbach,
title={Maximum Entropy {RL} (Provably) Solves Some Robust {RL} Problems},
author={Benjamin Eysenbach and Sergey Levine},
booktitle={International Conference on Learning Representations},
year={2022},
url={https://openreview.net/forum?id=PtSAD3caaA2}
}

@inproceedings{Cheng_pess,
 author = {Cheng, Ching-An and Kolobov, Andrey and Swaminathan, Adith},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {M. Ranzato and A. Beygelzimer and Y. Dauphin and P.S. Liang and J. Wortman Vaughan},
 pages = {13550--13563},
 publisher = {Curran Associates, Inc.},
 title = {Heuristic-Guided Reinforcement Learning},
 url = {https://proceedings.neurips.cc/paper/2021/file/70d31b87bd021441e5e6bf23eb84a306-Paper.pdf},
 volume = {34},
 year = {2021}
}

@article{openAI,
  title={Openai gym},
  author={Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, John and Tang, Jie and Zaremba, Wojciech},
  journal={arXiv preprint arXiv:1606.01540},
  year={2016}
}

@article{ZiebartThesis,
author = "Brian D. Ziebart",
title = "{Modeling Purposeful Adaptive Behavior with the Principle of Maximum Causal Entropy}",
year = "2010", month = "12",
url = "https://kilthub.cmu.edu/articles/thesis/Modeling_Purposeful_Adaptive_Behavior_with_the_Principle_of_Maximum_Causal_Entropy/6720692",
doi = "10.1184/R1/6720692.v1",
journal={Carnegie Mellon University} 
}

@article{BandT,
author = {Bertsekas, Dimitri P. and Tsitsiklis, John N.},
title = {An Analysis of Stochastic Shortest Path Problems},
year = {1991},
issue_date = {August 1991},
publisher = {INFORMS},
address = {Linthicum, MD, USA},
volume = {16},
number = {3},
issn = {0364-765X},
abstract = {We consider a stochastic version of the classical shortest path problem whereby for each node of a graph, we must choose a probability distribution over the set of successor nodes so as to reach a certain destination node with minimum expected cost. The costs of transition between successive nodes can be positive as well as negative. We prove natural generalizations of the standard results for the deterministic shortest path problem, and we extend the corresponding theory for undiscounted finite state Markovian decision problems by removing the usual restriction that costs are either all nonnegative or all nonpositive.},
journal = {Math. Oper. Res.},
month = {aug},
pages = {580–595},
numpages = {16},
keywords = {shortest path, policy iteration, first passage, dynamic programming, Markovian decision problems}
}


@InProceedings{TL_bound,
  title = 	 {Directed Exploration in Reinforcement Learning with Transferred Knowledge},
  author = 	 {Mann, Timothy A. and Choe, Yoonsuck},
  booktitle = 	 {Proceedings of the Tenth European Workshop on Reinforcement Learning},
  pages = 	 {59--76},
  year = 	 {2013},
  editor = 	 {Deisenroth, Marc Peter and Szepesvári, Csaba and Peters, Jan},
  volume = 	 {24},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Edinburgh, Scotland},
  month = 	 {30 Jun--01 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v24/mann12a/mann12a.pdf},
  url = 	 {https://proceedings.mlr.press/v24/mann12a.html},
  abstract = 	 {Experimental results suggest that transfer learning (TL), compared to learning from scratch, can decrease exploration by reinforcement learning (RL) algorithms. Most existing TL algorithms for RL are heuristic and may result in worse performance than learning from scratch (i.e., negative transfer). We introduce a theoretically grounded and flexible approach that transfers action-values via an intertask mapping and, based on those, explores the target task systematically. We characterize positive transfer as (1) decreasing sample complexity in the target task compared to the sample complexity of the base RL algorithm (without transferred action-values) and (2) guaranteeing that the algorithm converges to a near-optimal policy (i.e., negligible optimality loss). The sample complexity of our approach is no worse than the base algorithm's, and our analysis reveals that positive transfer can occur even with highly inaccurate and partial intertask mappings. Finally, we empirically test directed exploration with transfer in a multijoint reaching task, which highlights the value of our analysis and the robustness of our approach under imperfect conditions.}
}

@article{taylor_survey,
  author  = {Matthew E. Taylor and Peter Stone},
  title   = {Transfer Learning for Reinforcement Learning Domains: A Survey},
  journal = {Journal of Machine Learning Research},
  year    = {2009},
  volume  = {10},
  number  = {56},
  pages   = {1633-1685},
  url     = {http://jmlr.org/papers/v10/taylor09a.html}
}

@article{sutton_4room,
title = {Between MDPs and semi-MDPs: A framework for temporal abstraction in reinforcement learning},
journal = {Artificial Intelligence},
volume = {112},
number = {1},
pages = {181-211},
year = {1999},
issn = {0004-3702},
doi = {https://doi.org/10.1016/S0004-3702(99)00052-1},
url = {https://www.sciencedirect.com/science/article/pii/S0004370299000521},
author = {Richard S. Sutton and Doina Precup and Satinder Singh},
keywords = {Temporal abstraction, Reinforcement learning, Markov decision processes, Options, Macros, Macroactions, Subgoals, Intra-option learning, Hierarchical planning, Semi-Markov decision processes},
}


@book{hardy1952inequalities,
  title={Inequalities},
  author={Hardy, Godfrey Harold and Littlewood, John Edensor and P{\'o}lya, George and P{\'o}lya, Gy{\"o}rgy and others},
  year={1952},
  publisher={Cambridge university press}
}

@article{dayan1993improving,
  title={Improving generalization for temporal difference learning: The successor representation},
  author={Dayan, Peter},
  journal={Neural computation},
  volume={5},
  number={4},
  pages={613--624},
  year={1993},
  publisher={MIT Press}
}

@article{barreto2019option,
  title={The option keyboard: Combining skills in reinforcement learning},
  author={Barreto, Andr{\'e} and Borsa, Diana and Hou, Shaobo and Comanici, Gheorghe and Ayg{\"u}n, Eser and Hamel, Philippe and Toyama, Daniel and Mourad, Shibl and Silver, David and Precup, Doina and others},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@article{hierarchy,
  title={Hierarchical reinforcement learning: A comprehensive survey},
  author={Pateria, Shubham and Subagdja, Budhitama and Tan, Ah-hwee and Quek, Chai},
  journal={ACM Computing Surveys (CSUR)},
  volume={54},
  number={5},
  pages={1--35},
  year={2021},
  publisher={ACM New York, NY, USA}
}

@article{barreto_sf,
  title={Successor features for transfer in reinforcement learning},
  author={Barreto, Andr{\'e} and Dabney, Will and Munos, R{\'e}mi and Hunt, Jonathan J and Schaul, Tom and van Hasselt, Hado P and Silver, David},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  year={2017}
}
@incollection{peng_MCP,
	title = {MCP: Learning Composable Hierarchical Control with Multiplicative Compositional Policies},
	author = {Peng, Xue Bin and Chang, Michael and Zhang, Grace and Abbeel, Pieter and Levine, Sergey},
	booktitle = {Advances in Neural Information Processing Systems 32},
	editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
	pages = {3681--3692},
	year = {2019},
	publisher = {Curran Associates, Inc.},
	url = {https://proceedings.neurips.cc/paper/2019/file/95192c98732387165bf8e396c0f2dad2-Paper.pdf}
}

@inproceedings{lee2021sunrise,
  title        = {Sunrise: A simple unified framework for ensemble learning in deep reinforcement learning},
  author       = {Lee, Kimin and Laskin, Michael and Srinivas, Aravind and Abbeel, Pieter},
  booktitle    = {International Conference on Machine Learning},
  pages        = {6131--6141},
  year         = {2021},
  organization = {PMLR}
}



@InProceedings{hunt_diverg,
  title = 	 {Composing Entropic Policies using Divergence Correction},
  author =       {Hunt, Jonathan and Barreto, Andre and Lillicrap, Timothy and Heess, Nicolas},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {2911--2920},
  year = 	 {2019},
  editor = 	 {Chaudhuri, Kamalika and Salakhutdinov, Ruslan},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--15 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/hunt19a/hunt19a.pdf},
  url = 	 {https://proceedings.mlr.press/v97/hunt19a.html},
  abstract = 	 {Composing skills mastered in one task to solve novel tasks promises dramatic improvements in the data efficiency of reinforcement learning. Here, we analyze two recent works composing behaviors represented in the form of action-value functions and show that they perform poorly in some situations. As part of this analysis, we extend an important generalization of policy improvement to the maximum entropy framework and introduce an algorithm for the practical implementation of successor features in continuous action spaces. Then we propose a novel approach which addresses the failure cases of prior work and, in principle, recovers the optimal policy during transfer. This method works by explicitly learning the (discounted, future) divergence between base policies. We study this approach in the tabular case and on non-trivial continuous control problems with compositional structure and show that it outperforms or matches existing methods across all tasks considered.}
}

@inproceedings{littman,
  title={A generalized reinforcement-learning model: Convergence and applications},
  author={Littman, Michael L and Szepesv{\'a}ri, Csaba},
  booktitle={ICML},
  volume={96},
  pages={310--318},
  year={1996},
  organization={Citeseer}
}

@InProceedings{haarnoja_SAC,
  title = 	 {Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor},
  author =       {Haarnoja, Tuomas and Zhou, Aurick and Abbeel, Pieter and Levine, Sergey},
  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
  pages = 	 {1861--1870},
  year = 	 {2018},
  editor = 	 {Dy, Jennifer and Krause, Andreas},
  volume = 	 {80},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {10--15 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v80/haarnoja18b/haarnoja18b.pdf},
  url = 	 {https://proceedings.mlr.press/v80/haarnoja18b.html},
  abstract = 	 {Model-free deep reinforcement learning (RL) algorithms have been demonstrated on a range of challenging decision making and control tasks. However, these methods typically suffer from two major challenges: very high sample complexity and brittle convergence properties, which necessitate meticulous hyperparameter tuning. Both of these challenges severely limit the applicability of such methods to complex, real-world domains. In this paper, we propose soft actor-critic, an off-policy actor-critic deep RL algorithm based on the maximum entropy reinforcement learning framework. In this framework, the actor aims to maximize expected reward while also maximizing entropy. That is, to succeed at the task while acting as randomly as possible. Prior deep RL methods based on this framework have been formulated as Q-learning methods. By combining off-policy updates with a stable stochastic actor-critic formulation, our method achieves state-of-the-art performance on a range of continuous control benchmark tasks, outperforming prior on-policy and off-policy methods. Furthermore, we demonstrate that, in contrast to other off-policy algorithms, our approach is very stable, achieving very similar performance across different random seeds.}
}


@InProceedings{saxe_hier,
  title = 	 {Hierarchy Through Composition with Multitask {LMDP}s},
  author =       {Andrew M. Saxe and Adam C. Earle and Benjamin Rosman},
  booktitle = 	 {Proceedings of the 34th International Conference on Machine Learning},
  pages = 	 {3017--3026},
  year = 	 {2017},
  editor = 	 {Precup, Doina and Teh, Yee Whye},
  volume = 	 {70},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {06--11 Aug},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v70/saxe17a/saxe17a.pdf},
  url = 	 {https://proceedings.mlr.press/v70/saxe17a.html},

}
@article{hong2022bilinear,
  title={Bilinear value networks},
  author={Hong, Zhang-Wei and Yang, Ge and Agrawal, Pulkit},
  journal={arXiv preprint arXiv:2204.13695},
  year={2022}
}

@inproceedings{kimconstrained,
  title={Constrained GPI for Zero-Shot Transfer in Reinforcement Learning},
  author={Kim, Jaekyeom and Park, Seohong and Kim, Gunhee},
  booktitle={Advances in Neural Information Processing Systems},
  url={https://openreview.net/forum?id=sWNT5lT7l9G},
  year={2022}
}

@inproceedings{boolean_stoch,
title={Generalisation in Lifelong Reinforcement Learning through Logical Composition},
author={Geraud Nangue Tasse and Steven James and Benjamin Rosman},
booktitle={Deep RL Workshop NeurIPS 2021},
year={2021},
url={https://openreview.net/forum?id=kO-Cgmasm7}
}

@article{alver,
  title={Constructing a good behavior basis for transfer using generalized policy updates},
  author={Alver, Safa and Precup, Doina},
  journal={arXiv preprint arXiv:2112.15025},
  year={2021}
}

@article{tokamak,
  title={Magnetic control of tokamak plasmas through deep reinforcement learning},
  author={Degrave, Jonas and Felici, Federico and Buchli, Jonas and Neunert, Michael and Tracey, Brendan and Carpanese, Francesco and Ewalds, Timo and Hafner, Roland and Abdolmaleki, Abbas and de Las Casas, Diego and others},
  journal={Nature},
  volume={602},
  number={7897},
  pages={414--419},
  year={2022},
  publisher={Nature Publishing Group}
}
@article{gochess,
  title={Mastering atari, go, chess and shogi by planning with a learned model},
  author={Schrittwieser, Julian and Antonoglou, Ioannis and Hubert, Thomas and Simonyan, Karen and Sifre, Laurent and Schmitt, Simon and Guez, Arthur and Lockhart, Edward and Hassabis, Demis and Graepel, Thore and others},
  journal={Nature},
  volume={588},
  number={7839},
  pages={604--609},
  year={2020},
  publisher={Nature Publishing Group}
}
@ARTICLE{lungcancer,
  title    = "Reinforcement learning strategies for clinical trials in nonsmall
              cell lung cancer",
  author   = "Zhao, Yufan and Zeng, Donglin and Socinski, Mark A and Kosorok,
              Michael R",
  abstract = "Typical regimens for advanced metastatic stage IIIB/IV nonsmall
              cell lung cancer (NSCLC) consist of multiple lines of treatment.
              We present an adaptive reinforcement learning approach to
              discover optimal individualized treatment regimens from a
              specially designed clinical trial (a ``clinical reinforcement
              trial'') of an experimental treatment for patients with advanced
              NSCLC who have not been treated previously with systemic therapy.
              In addition to the complexity of the problem of selecting optimal
              compounds for first- and second-line treatments based on
              prognostic factors, another primary goal is to determine the
              optimal time to initiate second-line therapy, either immediately
              or delayed after induction therapy, yielding the longest overall
              survival time. A reinforcement learning method called Q-learning
              is utilized, which involves learning an optimal regimen from
              patient data generated from the clinical reinforcement trial.
              Approximating the Q-function with time-indexed parameters can be
              achieved by using a modification of support vector regression
              that can utilize censored data. Within this framework, a
              simulation study shows that the procedure can extract optimal
              regimens for two lines of treatment directly from clinical data
              without prior knowledge of the treatment effect mechanism. In
              addition, we demonstrate that the design reliably selects the
              best initial time for second-line therapy while taking into
              account the heterogeneity of NSCLC across patients.",
  journal  = "Biometrics",
  volume   =  67,
  number   =  4,
  pages    = "1422--1433",
  month    =  mar,
  year     =  2011,
  language = "en"
}

@inproceedings{other_4room,
 author = {Bellemare, Marc and Dabney, Will and Dadashi, Robert and Ali Taiga, Adrien and Castro, Pablo Samuel and Le Roux, Nicolas and Schuurmans, Dale and Lattimore, Tor and Lyle, Clare},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {A Geometric Perspective on Optimal Representations for Reinforcement Learning},
 url = {https://proceedings.neurips.cc/paper/2019/file/3cf2559725a9fdfa602ec8c887440f32-Paper.pdf},
 volume = {32},
 year = {2019}
}
@article{dsilver1,
  title={Grandmaster level in StarCraft II using multi-agent reinforcement learning},
  author={Vinyals, Oriol and Babuschkin, Igor and Czarnecki, Wojciech M and Mathieu, Micha{\"e}l and Dudzik, Andrew and Chung, Junyoung and Choi, David H and Powell, Richard and Ewalds, Timo and Georgiev, Petko and others},
  journal={Nature},
  volume={575},
  number={7782},
  pages={350--354},
  year={2019},
  publisher={Nature Publishing Group}
}



@article{dsilver2,
author = {David Silver  and Thomas Hubert  and Julian Schrittwieser  and Ioannis Antonoglou  and Matthew Lai  and Arthur Guez  and Marc Lanctot  and Laurent Sifre  and Dharshan Kumaran  and Thore Graepel  and Timothy Lillicrap  and Karen Simonyan  and Demis Hassabis },
title = {A general reinforcement learning algorithm that masters chess, shogi, and Go through self-play},
journal = {Science},
volume = {362},
number = {6419},
pages = {1140-1144},
year = {2018},
doi = {10.1126/science.aar6404},
URL = {https://www.science.org/doi/abs/10.1126/science.aar6404},
eprint = {https://www.science.org/doi/pdf/10.1126/science.aar6404},
abstract = {Computers can beat humans at increasingly complex games, including chess and Go. However, these programs are typically constructed for a particular game, exploiting its properties, such as the symmetries of the board on which it is played. Silver et al. developed a program called AlphaZero, which taught itself to play Go, chess, and shogi (a Japanese version of chess) (see the Editorial, and the Perspective by Campbell). AlphaZero managed to beat state-of-the-art programs specializing in these three games. The ability of AlphaZero to adapt to various game rules is a notable step toward achieving a general game-playing system. Science, this issue p. 1140; see also pp. 1087 and 1118 AlphaZero teaches itself to play three different board games and beats state-of-the-art programs in each. The game of chess is the longest-studied domain in the history of artificial intelligence. The strongest programs are based on a combination of sophisticated search techniques, domain-specific adaptations, and handcrafted evaluation functions that have been refined by human experts over several decades. By contrast, the AlphaGo Zero program recently achieved superhuman performance in the game of Go by reinforcement learning from self-play. In this paper, we generalize this approach into a single AlphaZero algorithm that can achieve superhuman performance in many challenging games. Starting from random play and given no domain knowledge except the game rules, AlphaZero convincingly defeated a world champion program in the games of chess and shogi (Japanese chess), as well as Go.}}


@InProceedings{pmlr-v80-haarnoja18a,
  title = 	 {Latent Space Policies for Hierarchical Reinforcement Learning},
  author =       {Haarnoja, Tuomas and Hartikainen, Kristian and Abbeel, Pieter and Levine, Sergey},
  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
  pages = 	 {1851--1860},
  year = 	 {2018},
  editor = 	 {Dy, Jennifer and Krause, Andreas},
  volume = 	 {80},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {10--15 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v80/haarnoja18a/haarnoja18a.pdf},
  url = 	 {https://proceedings.mlr.press/v80/haarnoja18a.html},
  abstract = 	 {We address the problem of learning hierarchical deep neural network policies for reinforcement learning. In contrast to methods that explicitly restrict or cripple lower layers of a hierarchy to force them to use higher-level modulating signals, each layer in our framework is trained to directly solve the task, but acquires a range of diverse strategies via a maximum entropy reinforcement learning objective. Each layer is also augmented with latent random variables, which are sampled from a prior distribution during the training of that layer. The maximum entropy objective causes these latent variables to be incorporated into the layer’s policy, and the higher level layer can directly control the behavior of the lower layer through this latent space. Furthermore, by constraining the mapping from latent variables to actions to be invertible, higher layers retain full expressivity: neither the higher layers nor the lower layers are constrained in their behavior. Our experimental evaluation demonstrates that we can improve on the performance of single-layer policies on standard benchmark tasks simply by adding additional layers, and that our method can solve more complex sparse-reward tasks by learning higher-level policies on top of high-entropy skills optimized for simple low-level objectives.}
}

@inproceedings{nemecek,
  title={Policy caches with successor features},
  author={Nemecek, Mark and Parr, Ronald},
  booktitle={International Conference on Machine Learning},
  pages={8025--8033},
  year={2021},
  organization={PMLR}
}

@article{ourAAAI,
  title={Utilizing Prior Solutions for Reward Shaping and Composition in Entropy-Regularized Reinforcement Learning},
  author={Adamczyk, Jacob and Arriojas, Argenis and Tiomkin, Stas and Kulkarni, Rahul V},
  journal={arXiv preprint arXiv:2212.01174},
  year={2022}
}


@article{stable-baselines3,
  author  = {Antonin Raffin and Ashley Hill and Adam Gleave and Anssi Kanervisto and Maximilian Ernestus and Noah Dormann},
  title   = {Stable-Baselines3: Reliable Reinforcement Learning Implementations},
  journal = {Journal of Machine Learning Research},
  year    = {2021},
  volume  = {22},
  number  = {268},
  pages   = {1-8},
  url     = {http://jmlr.org/papers/v22/20-1364.html}
}