Automatically generated by Mendeley Desktop 1.19.4
Any changes to this file will be lost if it is regenerated by Mendeley.

BibTeX export options can be customized via Preferences -> BibTeX in Mendeley Desktop

@article{bachman2019learning,
  title={Learning representations by maximizing mutual information across views},
  author={Bachman, Philip and Hjelm, R Devon and Buchwalter, William},
  journal={NeurIPS'2019},
}

@article{kearns2002near,
  title={Near-optimal reinforcement learning in polynomial time},
  author={Kearns, Michael and Singh, Satinder},
  journal={Machine learning},
  volume={49},
  number={2-3},
  pages={209--232},
  year={2002},
  publisher={Springer}
}

@inproceedings{stanchenfeld14,
 author = {Stachenfeld, Kimberly L and Botvinick, Matthew and Gershman, Samuel J},
 booktitle = {NeurIPS},
 title = {Design Principles of the Hippocampal Cognitive Map},
 year = {2014}
}



@article{hinton2012neural,
  title={Neural networks for machine learning lecture 6D - A separate, adaptive learning rate for each connection},
  author={Hinton, Geoffrey and Srivastava, Nitish and Swersky, Kevin},
  year={2012}
}

@article{kingma2014adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik P and Ba, Jimmy},
  journal={arXiv:1412.6980},
  year={2014}
}

@article{oudeyer2007intrinsic,
  title={Intrinsic motivation systems for autonomous mental development},
  author={Oudeyer, Pierre-Yves and Kaplan, Frdric and Hafner, Verena V},
  journal={IEEE transactions on evolutionary computation},
  year={2007},
}

@article{oudeyer2009intrinsic,
  title={What is intrinsic motivation? A typology of computational approaches},
  author={Oudeyer, Pierre-Yves and Kaplan, Frederic},
  journal={Frontiers in neurorobotics},
  year={2009},
}

@article{bougie2020skill,
  title={Skill-based curiosity for intrinsically motivated reinforcement learning},
  author={Bougie, Nicolas and Ichise, Ryutaro},
  journal={Machine Learning},
  year={2020},
}

@inproceedings{bromley1994signature,
 author = {Bromley, Jane and Guyon, Isabelle and LeCun, Yann and S\"{a}ckinger, Eduard and Shah, Roopak},
 booktitle = {NeurIPS},
 title = {Signature Verification using a "Siamese" Time Delay Neural Network},
 year = {1993}
}



@inproceedings{li2021learning,
title={Learning Subgoal Representations with Slow Dynamics},
author={Siyuan Li and Lulu Zheng and Jianhao Wang and Chongjie Zhang},
booktitle={ICLR},
year={2021}
}

@article{brafman2002r,
  title={R-max-a general polynomial time algorithm for near-optimal reinforcement learning},
  author={Brafman, Ronen I and Tennenholtz, Moshe},
  journal={Journal of Machine Learning Research},
  volume={3},
  number={Oct},
  pages={213--231},
  year={2002}
}

@article{jaksch2010near,
  title={Near-optimal regret bounds for reinforcement learning},
  author={Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
  journal={Journal of Machine Learning Research},
  volume={11},
  number={Apr},
  pages={1563--1600},
  year={2010}
}

@inproceedings{Badia2020Never,
title={Never Give Up: Learning Directed Exploration Strategies},
author={Adrià Puigdomènech Badia and Pablo Sprechmann and Alex Vitvitskyi and Daniel Guo and Bilal Piot and Steven Kapturowski and Olivier Tieleman and Martin Arjovsky and Alexander Pritzel and Andrew Bolt and Charles Blundell},
booktitle={ICLR},
year={2020},
}

@inproceedings{peng2019mcp,
  title={Mcp: Learning composable hierarchical control with multiplicative compositional policies},
  author={Peng, Xue Bin and Chang, Michael and Zhang, Grace and Abbeel, Pieter and Levine, Sergey},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3681--3692},
  year={2019}
}

@inproceedings{
burda2018exploration,
title={Exploration by random network distillation},
author={Yuri Burda and Harrison Edwards and Amos Storkey and Oleg Klimov},
booktitle={ICLR},
year={2019},
}

@inproceedings{bacon2017option,
  title={The option-critic architecture},
  author={Bacon, Pierre-Luc and Harb, Jean and Precup, Doina},
  booktitle={Thirty-First AAAI Conference on Artificial Intelligence},
  year={2017}
}

@inproceedings{todorov2012mujoco,
  title={Mujoco: A physics engine for model-based control},
  author={Todorov, Emanuel and Erez, Tom and Tassa, Yuval},
  booktitle={2012 IEEE/RSJ International Conference on Intelligent Robots and Systems},
  year={2012},
}

@article{wiskott2002slow,
  title={Slow feature analysis: Unsupervised learning of invariances},
  author={Wiskott, Laurenz and Sejnowski, Terrence J},
  journal={Neural computation},
  volume={14},
  number={4},
  pages={715--770},
  year={2002},
  publisher={MIT Press}
}

@inproceedings{he2020momentum,
  title={Momentum contrast for unsupervised visual representation learning},
  author={He, Kaiming and Fan, Haoqi and Wu, Yuxin and Xie, Saining and Girshick, Ross},
  booktitle={CVPR},
  year={2020}
}
@inproceedings{chen2020simple,
  title={A simple framework for contrastive learning of visual representations},
  author={Chen, Ting and Kornblith, Simon and Norouzi, Mohammad and Hinton, Geoffrey},
  booktitle={ICML},
  year={2020},
}

@inproceedings{zhang2021made,
title={{MADE}: Exploration via Maximizing Deviation from Explored Regions},
author={Tianjun Zhang and Paria Rashidinejad and Jiantao Jiao and Yuandong Tian and Joseph E. Gonzalez and Stuart Russell},
booktitle={NeurIPS},
year={2021}
}

@inproceedings{mazoure2020,
 author = {Mazoure, Bogdan and Tachet des Combes, Remi and Doan, Thang Long and Bachman, Philip and Hjelm, R Devon},
 booktitle = {NeurIPS},
 title = {Deep Reinforcement and InfoMax Learning},
 year = {2020}
}

@inproceedings{
stooke2021decoupling,
title={Decoupling Representation Learning from Reinforcement Learning},
author={Adam Stooke and Kimin Lee and Pieter Abbeel and Michael Laskin},
booktitle={ICML},
year={2021}
}

@inproceedings{yarats2021reinforcement,
      title={Reinforcement Learning with Prototypical Representations}, 
      author={Denis Yarats and Rob Fergus and Alessandro Lazaric and Lerrel Pinto},
      year={2021},
      booktitle={ICML},
}

@inproceedings{schwarzer2021dataefficient,
title={Data-Efficient Reinforcement Learning with Self-Predictive Representations},
author={Max Schwarzer and Ankesh Anand and Rishab Goel and R Devon Hjelm and Aaron Courville and Philip Bachman},
booktitle={ICLR},
year={2021}
}

@inproceedings{grill2020bootstrap,
 author = {Grill, Jean-Bastien and Strub, Florian and Altch\'{e}, Florent and Tallec, Corentin and Richemond, Pierre and Buchatskaya, Elena and Doersch, Carl and Avila Pires, Bernardo and Guo, Zhaohan and Gheshlaghi Azar, Mohammad and Piot, Bilal and kavukcuoglu, koray and Munos, Remi and Valko, Michal},
 booktitle = {NeurIPS},
 title = {Bootstrap Your Own Latent - A New Approach to Self-Supervised Learning},
 year = {2020}
}

@inproceedings{ostrovski2017count,
  title={Count-based exploration with neural density models},
  author={Ostrovski, Georg and Bellemare, Marc G and van den Oord, A{\"a}ron and Munos, R{\'e}mi},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={2721--2730},
  year={2017},
  organization={JMLR. org}
}

@phdthesis{machado2019thesis,
  title={Efficient Exploration in Reinforcement Learning through Time-Based Representations},
  author={Machado, M Cholodovskis},
  year={2019},
  school={University of Alberta}
}

@inproceedings{bellemare2016unifying,
  title={Unifying count-based exploration and intrinsic motivation},
  author={Bellemare, Marc and Srinivasan, Sriram and Ostrovski, Georg and Schaul, Tom and Saxton, David and Munos, Remi},
  booktitle={Advances in neural information processing systems},
  pages={1471--1479},
  year={2016}
}

@article{machado2018count,
  title={Count-based exploration with the successor representation},
  author={Machado, Marlos C and Bellemare, Marc G and Bowling, Michael},
  journal={arXiv preprint arXiv:1807.11622},
  year={2018}
}

@incollection{tang2017,
title = {\#Exploration: A Study of Count-Based Exploration for Deep Reinforcement Learning},
author = {Tang, Haoran and Houthooft, Rein and Foote, Davis and Stooke, Adam and Xi Chen, OpenAI and Duan, Yan and Schulman, John and DeTurck, Filip and Abbeel, Pieter},
booktitle = {Advances in Neural Information Processing Systems 30},
editor = {I. Guyon and U. V. Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
pages = {2753--2762},
year = {2017},
publisher = {Curran Associates, Inc.},
url = {http://papers.nips.cc/paper/6868-exploration-a-study-of-count-based-exploration-for-deep-reinforcement-learning.pdf},
}

@article{nachum2019does,
  title={Why does hierarchy (sometimes) work so well in reinforcement learning?},
  author={Nachum, Ofir and Tang, Haoran and Lu, Xingyu and Gu, Shixiang and Lee, Honglak and Levine, Sergey},
  journal={arXiv preprint arXiv:1909.10618},
  year={2019}
}

@inproceedings{Whitney2020Dynamics,
title={Dynamics-Aware Embeddings},
author={William Whitney and Rajat Agarwal and Kyunghyun Cho and Abhinav Gupta},
booktitle={ICLR},
year={2020}
}

@inproceedings{haarnoja2018soft,
  title={Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor},
  author={Haarnoja, Tuomas and Zhou, Aurick and Abbeel, Pieter and Levine, Sergey},
  booktitle={ICML},
  year={2018}
}

@article{haarnoja2018soft_learntemp,
  title={Soft actor-critic algorithms and applications},
  author={Haarnoja, Tuomas and Zhou, Aurick and Hartikainen, Kristian and Tucker, George and Ha, Sehoon and Tan, Jie and Kumar, Vikash and Zhu, Henry and Gupta, Abhishek and Abbeel, Pieter and others},
  journal={arXiv preprint arXiv:1812.05905},
  year={2018}
}

@article{lillicrap2015continuous,
  title={Continuous control with deep reinforcement learning},
  author={Lillicrap, Timothy P and Hunt, Jonathan J and Pritzel, Alexander and Heess, Nicolas and Erez, Tom and Tassa, Yuval and Silver, David and Wierstra, Daan},
  journal={arXiv:1509.02971},
  year={2015}
}

@inproceedings{schmid1991,
author = {Schmidhuber, J\"{u}rgen},
title = {A Possibility for Implementing Curiosity and Boredom in Model-Building Neural Controllers},
year = {1991},
booktitle = {Proceedings of the First International Conference on Simulation of Adaptive Behavior on From Animals to Animats},
}

@inproceedings{nasiriany2019planning,
  title={Planning with Goal-Conditioned Policies},
  author={Nasiriany, Soroush and Pong, Vitchyr and Lin, Steven and Levine, Sergey},
  booktitle={NeurIPS},
  year={2019}
}

@inproceedings{banijamali2018robust,
  title={Robust Locally-Linear Controllable Embedding},
  author={Banijamali, Ershad and Shu, Rui and Bui, Hung and Ghodsi, Ali and others},
  booktitle={AISTATS},
  year={2018}
}

@inproceedings{watter2015embed,
  title={Embed to control: A locally linear latent dynamics model for control from raw images},
  author={Watter, Manuel and Springenberg, Jost and Boedecker, Joschka and Riedmiller, Martin},
  booktitle={NeurIPS},
  year={2015}
}

@article{schrittwieser2020mastering,
  title={Mastering atari, go, chess and shogi by planning with a learned model},
  author={Schrittwieser, Julian and Antonoglou, Ioannis and Hubert, Thomas and Simonyan, Karen and Sifre, Laurent and Schmitt, Simon and Guez, Arthur and Lockhart, Edward and Hassabis, Demis and Graepel, Thore and others},
  journal={Nature},
  year={2020},
}

@article{bengio2017indep,
  title={Independently controllable features},
  author={Bengio, Emmanuel and Thomas, Valentin and Pineau, Joelle and Precup, Doina and Bengio, Yoshua},
  journal={arXiv:1703.07718},
  year={2017}
}

@InProceedings{misra2020kinematic,
author = {Misra, Dipendra and Henaff, Mikael and Krishnamurthy, Akshay and Langford, John},
title = {Kinematic State Abstraction and Provably Efficient Rich-Observation Reinforcement Learning},
booktitle = {ICML},
year = {2020},
}

@inproceedings{caron2020,
 author = {Caron, Mathilde and Misra, Ishan and Mairal, Julien and Goyal, Priya and Bojanowski, Piotr and Joulin, Armand},
 booktitle = {NeurIPS},
 title = {Unsupervised Learning of Visual Features by Contrasting Cluster Assignments},
 year = {2020}
}

@article{laskin2020curl,
  title={CURL: Contrastive Unsupervised Representations for Reinforcement Learning},
  author={Laskin, Michael and Srinivas, Aravind and Abbeel, Pieter},
  journal={ICML},
  year={2020}
}

@inproceedings{hafner2019learning,
  title={Learning Latent Dynamics for Planning from Pixels},
  author={Hafner, Danijar and Lillicrap, Timothy and Fischer, Ian and Villegas, Ruben and Ha, David and Lee, Honglak and Davidson, James},
  booktitle={ICML},
  year={2019}
}
@inproceedings{hafner2020dream,
title={Dream to Control: Learning Behaviors by Latent Imagination},
author={Danijar Hafner and Timothy Lillicrap and Jimmy Ba and Mohammad Norouzi},
booktitle={ICLR},
year={2020},
}

@inproceedings{vezhnevets2017feudal,
  title={Feudal networks for hierarchical reinforcement learning},
  author={Vezhnevets, Alexander Sasha and Osindero, Simon and Schaul, Tom and Heess, Nicolas and Jaderberg, Max and Silver, David and Kavukcuoglu, Koray},
  booktitle={ICML},
  year={2017},
}

@article{bengio2013,
author = {Bengio, Yoshua and Courville, Aaron and Vincent, Pascal},
title = {Representation Learning: A Review and New Perspectives},
year = {2013},
journal = {IEEE Trans. Pattern Anal. Mach. Intell.},
keywords = {Feature extraction, unsupervised learning, neural nets, Boltzmann machine, Machine learning, feature learning, Learning systems, Speech recognition, Abstracts, Deep learning, Manifolds, autoencoder, representation learning, Neural networks}
}

@article{Sutton1999,
abstract = {Learning, planning, and representing knowledge at multiple levels of temporal abstraction are key, longstanding challenges in AI. The ways how to address these challenges within the mathematical framework of reinforcement learning and Markov decision processes (MDP) are discussed. A set options defined over an MDP constitutes a semi-MDP (SMDP), and the theory of SMDPs provides the foundation for the theory of options.},
author = {Sutton, Richard S. and Precup, Doina and Singh, Satinder},
journal = {Artificial Intelligence},
title = {{Between MDPs and semi-MDPs: A framework for temporal abstraction in reinforcement learning}},
year = {1999}
}
@inproceedings{Frans2018,
abstract = {We develop a metalearning approach for learning hierarchically structured policies, improving sample efficiency on unseen tasks through the use of shared primitives—policies that are executed for large numbers of timesteps. Specifically, a set of primitives are shared within a distribution of tasks, and are switched between by task-specific policies. We provide a concrete metric for measuring the strength of such hierarchies, leading to an optimization problem for quickly reaching high reward on unseen tasks. We then present an algorithm to solve this problem end-to-end through the use of any off-the-shelf reinforcement learning method, by repeatedly sampling new tasks and resetting task-specific policies. We successfully discover1 meaningful motor primitives for the directional movement of four-legged robots, solely by interacting with distributions of mazes. We also demonstrate the transferability of primitives to solve long-timescale sparse-reward obstacle courses, and we enable 3D humanoid robots to robustly walk and crawl with the same policy.},
author = {Frans, Kevin and Ho, Jonathan and Chen, Xi and Abbeel, Pieter and Schulman, John},
booktitle = {ICLR 2018},
title = {{Meta learning shared hierarchies}},
year = {2018}
}
@article{Osentoski2010,
abstract = {Much past work on solving Markov decision processes (MDPs) using reinforcement learning (RL) has relied on combining parameter estimation methods with hand-designed function approximation architectures for representing value functions. Recently, there has been growing interest in a broader framework that combines representation discovery and control learning, where value functions are approximated using a linear combination of task-dependent basis functions learned during the course of solving a particular MDP. This paper introduces an approach to automatic basis function construction for hierarchical reinforcement learning (HRL). Our approach generalizes past work on basis construction to multi-level action hierarchies by forming a compressed representation of a semi-Markov decision process (SMDP) at multiple levels of temporal abstraction. The specific approach is based on hierarchical spectral analysis of graphs induced on an SMDP's state space from sample trajectories. We present experimental results on benchmark SMDPs, showing significant speedups when compared to hand-designed approximation architectures. Copyright {\textcopyright} 2010, International Foundation for Autonomous Agents and Multiagent Systems (www.ifaamas.org). All rights reserved.},
author = {Osentoski, Sarah and Mahadevan, Sridhar},
journal = {Proceedings of the International Joint Conference on Autonomous Agents and Multiagent Systems, AAMAS},
keywords = {Hierarchical Reinforcement Learning,Representation Discovery,Semi-Markov Decision Processes},
title = {{Basis function construction for hierarchical reinforcement learning}},
year = {2010}
}
@article{Richthofer2016,
abstract = {Every organism in an environment, whether biological, robotic or virtual, must be able to predict certain aspects of its environment in order to survive or perform whatever task is intended. It needs a model that is capable of estimating the consequences of possible actions, so that planning, control, and decision-making become feasible. For scientific purposes, such models are usually created in a problem specific manner using differential equations and other techniques from control-and system-theory. In contrast to that, we aim for an unsupervised approach that builds up the desired model in a self-organized fashion. Inspired by Slow Feature Analysis (SFA), our approach is to extract subsignals from the input, that behave as predictable as possible. These "predictable features" are highly relevant for modeling, because predictability is a desired property of the needed consequence-estimating model by definition. In our approach, we measure predictability with respect to a certain prediction model. We focus here on the solution of the arising optimization problem and present a tractable algorithm based on algebraic methods which we call Predictable Feature Analysis (PFA). We prove that the algorithm finds the globally optimal signal if this signal can be predicted with low error. To deal with cases where the optimal signal has a significant prediction error, we provide a robust, heuristically motivated variant of the algorithm and verify it empirically. Additionally, we give formal criteria a prediction model must meet to be suitable for measuring predictability in the PFA setting and also provide a suitable default model along with a formal proof that it meets these criteria.},
archivePrefix = {arXiv},
arxivId = {1311.2503},
author = {Richthofer, Stefan and Wiskott, Laurenz},
doi = {10.1109/ICMLA.2015.158},
eprint = {1311.2503},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Predictable Feature Analysis.pdf:pdf},
isbn = {9781509002870},
journal = {Proceedings - 2015 IEEE 14th International Conference on Machine Learning and Applications, ICMLA 2015},
keywords = {Data analysis,Dimensionality reduction,Feature extraction,Model building,Predictability,Self organization,Signal analysis,Unsupervised learning},
pages = {190--196},
title = {{Predictable feature analysis}},
year = {2016}
}
@article{Luciw2012,
abstract = {The Slow Feature Analysis (SFA) unsupervised learning framework extracts features representing the underlying causes of the changes within a tem- porally coherent high-dimensional raw sensory in- put signal. We develop the first online version of SFA, via a combination of incremental Princi- pal Components Analysis and Minor Components Analysis. Unlike standard batch-based SFA, on- line SFA adapts along with non-stationary environ- ments, which makes it a generally useful unsuper- vised preprocessor for autonomous learning agents. We compare online SFA to batch SFA in several experiments and show that it indeed learns without a teacher to encode the input stream by informa- tive slow features representing meaningful abstract environmental properties. We extend online SFA to deep networks in hierarchical fashion, and use them to successfully extract abstract object position information from high-dimensional video.},
author = {Luciw, M},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Hierarchical Incremental Slow Feature Analysis.pdf:pdf},
isbn = {978-1-57735-516-8},
journal = {Workshop on Deep {\ldots}},
number = {7},
pages = {2007},
title = {{Hierarchical incremental slow feature analysis}},
url = {http://www.neuraldynamics.eu/documents/publications/posters{\_}talks/LuciwKompellaSchmidhuber2012{\_}workshop{\_}abstract.pdf},
volume = {20},
year = {2012}
}
@article{Oh2015,
abstract = {Motivated by vision-based reinforcement learning (RL) problems, in particular Atari games from the recent benchmark Aracade Learning Environment (ALE), we consider spatio-temporal prediction problems where future image-frames depend on control variables or actions as well as previous frames. While not composed of natural scenes, frames in Atari games are high-dimensional in size, can involve tens of objects with one or more objects being controlled by the actions directly and many other objects being influenced indirectly, can involve entry and departure of objects, and can involve deep partial observability. We propose and evaluate two deep neural network architectures that consist of encoding, actionconditional transformation, and decoding layers based on convolutional neural networks and recurrent neural networks. Experimental results show that the proposed architectures are able to generate visually-realistic frames that are also useful for control over approximately 100-step action-conditional futures in some games. To the best of our knowledge, this paper is the first to make and evaluate long-term predictions on high-dimensional video conditioned by control inputs.},
archivePrefix = {arXiv},
arxivId = {1507.08750},
author = {Oh, Junhyuk and Guo, Xiaoxiao and Lee, Honglak and Lewis, Richard and Singh, Satinder},
eprint = {1507.08750},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Action-Conditional Video Prediction using Deep Networks in Atari Games.pdf:pdf},
issn = {10495258},
journal = {Advances in Neural Information Processing Systems},
pages = {2863--2871},
title = {{Action-conditional video prediction using deep networks in Atari games}},
volume = {2015-Janua},
year = {2015}
}
@article{Eysenbach19,
abstract = {Intelligent creatures can explore their environments and learn useful skills without supervision. In this paper, we propose “Diversity is All You Need”(DIAYN), a method for learning useful skills without a reward function. Our proposed method learns skills by maximizing an information theoretic objective using a maximum entropy policy. On a variety of simulated robotic tasks, we show that this simple objective results in the unsupervised emergence of diverse skills, such as walking and jumping. In a number of reinforcement learning benchmark environments, our method is able to learn a skill that solves the benchmark task despite never receiving the true task reward. We show how pretrained skills can provide a good parameter initialization for downstream tasks, and can be composed hierarchically to solve complex, sparse reward tasks. Our results suggest that unsupervised discovery of skills can serve as an effective pretraining mechanism for overcoming challenges of exploration and data efficiency in reinforcement learning.},
author = {Eysenbach, Benjamin and Ibarz, Julian and Gupta, Abhishek and Levine, Sergey},
journal = {ICLR},
title = {{Diversity is all you need: Learning skills without a reward function}},
year = {2019}
}

@article{Pan2017,
abstract = {Model-free Reinforcement Learning (RL) offers an attractive approach to learn control policies for high-dimensional systems, but its relatively poor sample complexity often forces training in simulated environments. Even in simulation, goal-directed tasks whose natural reward function is sparse remain intractable for state-of-the-art model-free algorithms for continuous control. The bottleneck in these tasks is the prohibitive amount of exploration required to obtain a learning signal from the initial state of the system. In this work, we leverage physical priors in the form of an approximate system dynamics model to design a curriculum scheme for a model-free policy optimization algorithm. Our Backward Reachability Curriculum (BaRC) begins policy training from states that require a small number of actions to accomplish the task, and expands the initial state distribution backwards in a dynamically-consistent manner once the policy optimization algorithm demonstrates sufficient performance. BaRC is general, in that it can accelerate training of any model-free RL algorithm on a broad class of goal-directed continuous control MDPs. Its curriculum strategy is physically intuitive, easy-to-tune, and allows incorporating physical priors to accelerate training without hindering the performance, flexibility, and applicability of the model-free RL algorithm. We evaluate our approach on two representative dynamic robotic learning problems and find substantial performance improvement relative to previous curriculum generation techniques and naive exploration strategies.},
archivePrefix = {arXiv},
arxivId = {1806.06161},
author = {Pan, Xinlei and You, Yurong and Wang, Ziyu Ziyan and Lu, Cewu and Taylor, Matthew E and Stone, Peter and Schaul, Tom and Horgan, Daniel and Gregor, Karol and Silver, David and Hilleli, Bar and El-yaniv, Ran and Bansal, Somil and Calandra, Roberto and Chua, Kurtland and Levine, Sergey and Tomlin, Claire and Mahadevan, Sridhar and Tumer, Kagan and Knudson, Matt and Tai, Lei and Paolo, Giuseppe and Liu, Ming and Chaplot, Devendra Singh and Lample, Guillaume and Sathyendra, Kanthashree Mysore and Salakhutdinov, Ruslan and Hosoda, K and Asada, M and Noda, S and Tawaratsumida, S and Kulkarni, Tejas D. and Narasimhan, Karthik R. and Saeedi, Ardavan and Tenenbaum, Joshua B. and Ivanovic, Boris and Harrison, James and Sharma, Apoorva and Chen, Mo and Pavone, Marco and Frans, Kevin and Ho, Jonathan and Chen, Xi and Abbeel, Pieter and Schulman, John and Florensa, Carlos and Held, David and Wulfmeier, Markus and Zhang, Marvin Michael and Abbeel, Pieter and Heess, Nicolas and TB, Dhruva and Sriram, Srinivasan and Lemmon, Jay and Merel, Josh and Wayne, Greg and Tassa, Yuval and Erez, Tom and Wang, Ziyu Ziyan and Eslami, S. M. Ali and Riedmiller, Martin and Silver, David and Oh, Junhyuk and Singh, Satinder and Lee, Honglak and Kohli, Pushmeet and Chebotar, Yevgen and Hausman, Karol and Zhang, Marvin Michael and Sukhatme, Gaurav and Schaal, Stefan and Levine, Sergey and Li, Yuxi},
doi = {10.1109/IROS.2017.8202134},
eprint = {1806.06161},
isbn = {9781510810587},
issn = {1938-7228},
journal = {Journal of Machine Learning Research},
keywords = {1,1998,Human AI Collaboration Track,actions with goal,automatic curricu-,behavior acquisition,control,example,exploration,leaning agents take sequential,manifold learning,markov decision processes,maximizing a reward,multi task learning,problems,q-learning,reinforcement learning,rl,robotic manipulation,signal,soccer robot,spectral graph theory,state-action,sutton barto,transfer learning,transfer learning objectives,value function approximation,vision-based reinforcement learning,which may time delayed},
number = {Nips},
pages = {1633--1685},
pmid = {24239943},
title = {{BaRC: Backward Reachability Curriculum for Robotic Reinforcement Learning}},
url = {http://arxiv.org/abs/1701.07274{\%}0Ahttp://arxiv.org/abs/1703.03078{\%}0Ahttp://arxiv.org/abs/1706.05064{\%}0Ahttp://arxiv.org/abs/1707.02286{\%}0Ahttp://arxiv.org/abs/1707.05300{\%}0Ahttp://arxiv.org/abs/1710.09767{\%}0Ahttp://arxiv.org/abs/1806.06161{\%}0Ahttp://arxiv.org/},
volume = {23},
year = {2017}
}

@InProceedings{mnih2016AC,
  title = 	 {Asynchronous Methods for Deep Reinforcement Learning},
  author = 	 {Mnih, Volodymyr and Badia, Adria Puigdomenech and Mirza, Mehdi and Graves, Alex and Lillicrap, Timothy and Harley, Tim and Silver, David and Kavukcuoglu, Koray},
  year = 	 {2016},
  series = 	 {Proceedings of Machine Learning Research},
}

@article{Ghosh2019,
abstract = {Representation learning is a central challenge across a range of machine learning areas. In reinforcement learning, effective and functional representations have the potential to tremendously accelerate learning progress and solve more challenging problems. Most prior work on representation learning has focused on generative approaches, learning representations that capture all the underlying factors of variation in the observation space in a more disentangled or well-ordered manner. In this paper, we instead aim to learn functionally salient representations: representations that are not necessarily complete in terms of capturing all factors of variation in the observation space, but rather aim to capture those factors of variation that are important for decision making - that are “actionable.” These representations are aware of the dynamics of the environment, and capture only the elements of the observation that are necessary for decision making rather than all factors of variation, eliminating the need for explicit reconstruction. We show how these learned representations can be useful to improve exploration for sparse reward problems, to enable long horizon hierarchical reinforcement learning, and as a state representation for learning policies for downstream tasks. We evaluate our method on a number of simulated environments, and compare it to prior methods for representation learning, exploration, and hierarchical reinforcement learning.},
author = {Ghosh, Dibya and Gupta, Abhishek and Levine, Sergey},
journal = {ICLR},
title = {{Learning actionable representations with goal-conditioned policies}},
year = {2019}
}
@article{Nachum2019c,
abstract = {Hierarchical reinforcement learning has demonstrated significant success at solving difficult reinforcement learning (RL) tasks. Previous works have motivated the use of hierarchy by appealing to a number of intuitive benefits, including learning over temporally extended transitions, exploring over temporally extended periods, and training and exploring in a more semantically meaningful action space, among others. However, in fully observed, Markovian settings, it is not immediately clear why hierarchical RL should provide benefits over standard "shallow" RL architectures. In this work, we isolate and evaluate the claimed benefits of hierarchical RL on a suite of tasks encompassing locomotion, navigation, and manipulation. Surprisingly, we find that most of the observed benefits of hierarchy can be attributed to improved exploration, as opposed to easier policy learning or imposed hierarchical structures. Given this insight, we present exploration techniques inspired by hierarchy that achieve performance competitive with hierarchical RL while at the same time being much simpler to use and implement.},
author = {Nachum, Ofir and Tang, Haoran and Lu, Xingyu and Gu, Shixiang and Lee, Honglak and Levine, Sergey},
title = {{Why Does Hierarchy (Sometimes) Work So Well in Reinforcement Learning?}},
year = {2019}
}
@article{Hengst2002,
abstract = {An open problem in reinforcement learning is discovering hierarchical structure. HEXQ, an algorithm which automatically attempts to decompose and solve a model-free fac- tored MDP hierarchically is described. By searching for aliased Markov sub-space re- gions based on the state variables the algo- rithm uses temporal and state abstraction to construct a hierarchy of interlinked smaller MDPs.},
author = {Hengst, Bernhard},
doi = {10.1.1.9.5839},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Discovering Hierarchy in Reinforcement Learning with HEXQ.pdf:pdf},
isbn = {1-55860-873-7},
journal = {Icml},
number = {1},
pages = {243--250},
title = {{Discovering hierarchy in reinforcement learning with HEXQ}},
url = {http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.9.5839},
year = {2002}
}
@article{Bonarini2006,
abstract = {We present SMILe (Self-Motivated Incremental Learning), a new learning$\backslash$nframework where an agent learns a set of abilities needed to face$\backslash$nseveral tasks in its environment, by following a biologically inspired,$\backslash$nself--motivated approach that loops over three main phases. In the$\backslash$nbabbling phase, the agent randomly explores the environment, in a$\backslash$nway similar to what animal puppies do. This provides information$\backslash$nabout the effects of action on the environment. In the motivating$\backslash$nphase, the agent identifies what is interesting in the environment$\backslash$nand develops an intrinsic motivation in achieving situations with$\backslash$nhighest interest. In the skill acquisition phase, the agent learns$\backslash$nthe skills needed to reach the most interesting state, guided by$\backslash$na self-generated reinforcement function. Once a new skill is available$\backslash$nthe babbling phase can start again with the enlarged set of abilities,$\backslash$nand learning continues all the life long. We present results on a$\backslash$ngridworld abstraction of a robotic environment to show how SMILe$\backslash$nmakes it possible to learn skills that enable the agent to perform$\backslash$nwell and robustly in many different tasks.},
author = {Bonarini, A and Lazaric, A and Restelli, M},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Self-development{\_}framework{\_}for{\_}reinforcement{\_}learn.pdf:pdf},
journal = {Proceedings of the Fifth International Conference on Development and Learning (ICDL)},
number = {June 2014},
title = {{Self-Development Framework for Reinforcement Learning Agents}},
year = {2006}
}
@article{Savinov2019,
abstract = {Rewards are sparse in the real world and most of today's reinforcement learning algorithms struggle with such sparsity. One solution to this problem is to allow the agent to create rewards for itself - thus making rewards dense and more suitable for learning. In particular, inspired by curious behaviour in animals, observing something novel could be rewarded with a bonus. Such bonus is summed up with the real task reward - making it possible for RL algorithms to learn from the combined reward. We propose a new curiosity method which uses episodic memory to form the novelty bonus. To determine the bonus, the current observation is compared with the observations in memory. Crucially, the comparison is done based on how many environment steps it takes to reach the current observation from those in memory - which incorporates rich information about environment dynamics. This allows us to overcome the known “couch-potato” issues of prior work - when the agent finds a way to instantly gratify itself by exploiting actions which lead to hardly predictable consequences. We test our approach in visually rich 3D environments in VizDoom, DMLab and MuJoCo. In navigational tasks from VizDoom and DMLab, our agent outperforms the state-of-the-art curiosity method ICM. In MuJoCo, an ant equipped with our curiosity module learns locomotion out of the first-person-view curiosity only. The code is available at https://github.com/google-research/episodic-curiosity.},
author = {Savinov, Nikolay and Raichuk, Anton and Marinier, Rapha{\"{e}}l and Vincent, Damien and Pollefeys, Marc and Lillicrap, Timothy and Gelly, Sylvain},
journal = {ICLR},
title = {{Episodic curiosity through reachability}},
year = {2019}
}

@inproceedings{pathakICMl17curiosity,
    Author = {Pathak, Deepak and
    Agrawal, Pulkit and
    Efros, Alexei A. and
    Darrell, Trevor},
    Title = {Curiosity-driven Exploration
    by Self-supervised Prediction},
    Booktitle = {ICML},
    Year = {2017}
}

@inproceedings{chopra2005learning,
  title={Learning a similarity metric discriminatively, with application to face verification},
  author={Chopra, Sumit and Hadsell, Raia and LeCun, Yann},
  booktitle={CVPR},
  year={2005},
}

@article{Luciw2012a,
abstract = {We show that Incremental Slow Feature Analysis (IncSFA) provides a low complexity method for learning Proto-Value Functions (PVFs). It has been shown that a small number of PVFs provide a good basis set for linear approximation of value functions in reinforcement environments. Our method learns PVFs from a high-dimensional sensory input stream, as the agent explores its world, without building a transition model, adjacency matrix, or covariance matrix. A temporal-difference based reinforcement learner improves a value function approximation upon the features, and the agent uses the value function to achieve rewards successfully. The algorithm is local in space and time, furthering the biological plausibility and applicability of PVFs. {\textcopyright} 2012 Springer-Verlag.},
author = {Luciw, Matthew and Schmidhuber, Juergen},
doi = {10.1007/978-3-642-33266-1_35},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Low Complexity Proto-Value Function Learning from Sensory Observations with Incremental Slow Feature Analysis.pdf:pdf},
isbn = {9783642332654},
issn = {03029743},
journal = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
keywords = {Biologically Inspired Reinforcement Learning,Incremental Slow Feature Analysis,Proto-Value Functions},
number = {PART 2},
pages = {279--287},
title = {{Low complexity proto-value function learning from sensory observations with incremental slow feature analysis}},
volume = {7553 LNCS},
year = {2012}
}
@article{Goerg1962,
author = {Goerg, Georg M.},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Forecastable Component Analysis.pdf:pdf},
issn = {00349453},
journal = {Revista espa??ola de oto-neuro-oftalmolog??a y neurocirug??a},
keywords = {HEMIPLEGIA/therapy,MULTIPLE SCLEROSIS/therapy,PYRAMIDAL TRACTS/diseases,SPINAL CORD/diseases},
pages = {233--237},
title = {{Forecastable Component Analysis}},
volume = {21},
year = {1962}
}
@article{Paine2019,
abstract = {This paper introduces R2D3, an agent that makes efficient use of demonstrations to solve hard exploration problems in partially observable environments with highly variable initial conditions. We also introduce a suite of eight tasks that combine these three properties, and show that R2D3 can solve several of the tasks where other state of the art methods (both with and without demonstrations) fail to see even a single successful trajectory after tens of billions of steps of exploration.},
archivePrefix = {arXiv},
arxivId = {1909.01387},
author = {Paine, Tom Le and Gulcehre, Caglar and Shahriari, Bobak and Denil, Misha and Hoffman, Matt and Soyer, Hubert and Tanburn, Richard and Kapturowski, Steven and Rabinowitz, Neil and Williams, Duncan and Barth-Maron, Gabriel and Wang, Ziyu and de Freitas, Nando and Team, Worlds},
eprint = {1909.01387},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}2{\_}2019/Making Efficient Use of Demonstrations to Solve Hard Exploration Problems.pdf:pdf},
pages = {1--19},
title = {{Making Efficient Use of Demonstrations to Solve Hard Exploration Problems}},
url = {http://arxiv.org/abs/1909.01387},
year = {2019}
}
@article{Nickel2017,
abstract = {Representation learning has become an invaluable approach for learning from symbolic data such as text and graphs. However, state-of-the-art embedding methods typically do not account for latent hierarchical structures which are characteristic for many complex symbolic datasets. In this work, we introduce a new approach for learning hierarchical representations of symbolic data by embedding them into hyperbolic space - or more precisely into an n-dimensional Poincar{\'{e}} ball. Due to the underlying hyperbolic geometry, this allows us to learn parsimonious representations of symbolic data by simultaneously capturing hierarchy and similarity. We present an efficient algorithm to learn the embeddings based on Riemannian optimization and show experimentally that Poincar{\'{e}} embeddings can outperform Euclidean embeddings significantly on data with latent hierarchies, both in terms of representation capacity and in terms of generalization ability.},
archivePrefix = {arXiv},
arxivId = {1705.08039},
author = {Nickel, Maximilian and Kiela, Douwe},
eprint = {1705.08039},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Poincar{\'{e}} Embeddings for Learning Hierarchical Representations.pdf:pdf},
issn = {10495258},
journal = {Advances in Neural Information Processing Systems},
pages = {6339--6348},
title = {{Poincar{\'{e}} embeddings for learning hierarchical representations}},
volume = {2017-Decem},
year = {2017}
}
@inproceedings{Sharma2019,
abstract = {The use of imitation learning to learn a single policy for a complex task that has multiple modes or hierarchical structure can be challenging. In fact, previous work has shown that when the modes are known, learning separate policies for each mode or sub-task can greatly improve the performance of imitation learning. In this work, we discover the interaction between sub-tasks from their resulting state-action trajectory sequences using a directed graphical model. We propose a new algorithm based on the generative adversarial imitation learning framework which automatically learns sub-task policies from unsegmented demonstrations. Our approach maximizes the directed information flow in the graphical model between sub-task latent variables and their generated trajectories. We also show how our approach connects with the existing Options framework, which is commonly used to learn hierarchical policies.},
author = {Sharma, Mohit and Sharma, Arjun and Rhinehart, Nick and Kitani, Kris M},
booktitle = {ICLR},
title = {{Directed-info gail: Learning hierarchical policies from unsegmented demonstrations using directed information}},
year = {2019}
}
@article{Machado2017,
abstract = {Representation learning and option discovery are two of the biggest challenges in reinforcement learning (RL). Proto-value functions (PVFs) are a well-known approach for representation learning in MDPs. In this paper we address the option discovery problem by showing how PVFs implicitly define options. We do it by introducing eigenpurposes, intrinsic reward functions derived from the learned representations. The options discovered from eigenpurposes traverse the principal directions of the state space. They are useful for multiple tasks because they are discovered without taking the environment's rewards into consideration. Moreover, different options act at different time scales, making them helpful for exploration. We demonstrate features of eigenpurposes in traditional tabular domains as well as in Atari 2600 games.},
author = {Machado, Marlos C. and Bellemare, Marc G. and Bowling, Michael},
journal = {ICML},
title = {{A laplacian framework for option discovery in reinforcement learning}},
year = {2017}
}
@article{Creutzig2008,
abstract = {Understanding the guiding principles of sensory coding strategies is a main goal in computational neuroscience. Among others, the principles of predictive coding and slowness appear to capture aspects of sensory processing. Predictive coding postulates that sensory systems are adapted to the structure of their input signals such that information about future inputs is encoded. Slow feature analysis (SFA) is a method for extracting slowly varying components from quickly varying input signals, thereby learning temporally invariant features. Here, we use the information bottleneck method to state an information-theoretic objective function for temporally local predictive coding. We then show that the linear case of SFA can be interpreted as a variant of predictive coding that maximizes the mutual information between the current output of the system and the input signal in the next time step. This demonstrates that the slowness principle and predictive coding are intimately related. {\textcopyright} 2007 Massachusetts Institute of Technology.},
author = {Creutzig, Felix and Sprekeler, Henning},
doi = {10.1162/neco.2008.01-07-455},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Predictive Coding and the Slowness Principle An Information-Theoretic Approach.pdf:pdf},
issn = {08997667},
journal = {Neural Computation},
number = {4},
pages = {1026--1041},
pmid = {18085988},
title = {{Predictive coding and the slowness principle: An information-theoretic approach}},
volume = {20},
year = {2008}
}
@article{Kompella2012,
abstract = {We introduce here an incremental version of slow feature analysis (IncSFA), combining candid covariance-free incremental principal components analysis (CCIPCA) and covariance-free incremental minor components analysis (CIMCA). IncSFA's feature updating complexity is linear with respect to the input dimensionality,while batch SFA's (BSFA) updating complexity is cubic. IncSFAdoes not need to store, or even compute, any covariance matrices. The drawback to IncSFA is data efficiency: it does not use each data point as effectively as BSFA. But IncSFA allows SFA to be tractably applied, with just a few parameters, directly on highdimensional input streams (e.g., visual input of an autonomous agent), while BSFA has to resort to hierarchical receptive-field-based architectures when the input dimension is too high. Further, IncSFA's updates have simple Hebbian and anti-Hebbian forms, extending the biologicalplausibility of SFA. Experimental results show IncSFA learns the same set of features as BSFA and can handle a few cases where BSFA fails. {\textcopyright} 2012 Massachusetts Institute of Technology.},
author = {Kompella, Varun Raj and Luciw, Matthew and Schmidhuber, J{\"{u}}rgen},
doi = {10.1162/NECO_a_00344},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Incremental Slow Feature Analysis Adaptive Low-Complexity Slow Feature Updating from High-Dimensional Input Streams.pdf:pdf},
issn = {1530888X},
journal = {Neural Computation},
keywords = {component analysis,incremental learning,mca,minor component analysis,pca,principal,sfa,slow feature analysis},
number = {11},
pages = {2994--3024},
title = {{Incremental slow feature analysis: Adaptive low-complexity slow feature updating from high-dimensional input streams}},
volume = {24},
year = {2012}
}
@article{Choi2019,
abstract = {This paper investigates whether learning contingency-awareness and controllable aspects of an environment can lead to better exploration in reinforcement learning. To investigate this question, we consider an instantiation of this hypothesis evaluated on the Arcade Learning Element (ALE). In this study, we develop an attentive dynamics model (ADM) that discovers controllable elements of the observations, which are often associated with the location of the character in Atari games. The ADM is trained in a self-supervised fashion to predict the actions taken by the agent. The learned contingency information is used as a part of the state representation for exploration purposes. We demonstrate that combining actor-critic algorithm with count-based exploration using our representation achieves impressive results on a set of notoriously challenging Atari games due to sparse rewards.1 For example, we report a state-of-the-art score of {\textgreater}11,000 points on MONTEZUMA's REVENGE without using expert demonstrations, explicit high-level information (e.g., RAM states), or supervisory data. Our experiments confirm that contingency-awareness is indeed an extremely powerful concept for tackling exploration problems in reinforcement learning and opens up interesting research questions for further investigations.},
archivePrefix = {arXiv},
arxivId = {1811.01483},
author = {Choi, Jongwook and Guo, Yijie and Moczulski, Marcin and Oh, Junhyuk and Wu, Neal and Norouzi, Mohammad and Lee, Honglak},
journal = {ICLR},
title = {{Contingency-aware exploration in reinforcement learning}},
year = 2019
}

@article{Bellemare2012,
abstract = {Contingency awareness is the recognition that some aspects of a future observation are under an agent's control while others are solely determined by the environment. This paper explores the idea of contingency awareness in reinforcement learning using the platform of Atari 2600 games. We introduce a technique for accurately identifying contingent regions and describe how to exploit this knowledge to generate improved features for value function approximation. We evaluate the performance of our techniques empirically, using 46 unseen, diverse, and challenging games for the Atari 2600 console. Our results suggest that contingency awareness is a generally useful concept for model-free reinforcement learning agents. Copyright {\textcopyright} 2012, Association for the Advancement of Artificial Intelligence. All rights reserved.},
author = {Bellemare, Marc G. and Veness, Joel and Bowling, Michael},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Investigating Contingency Awareness Using Atari 2600 Games.pdf:pdf},
isbn = {9781577355687},
journal = {Proceedings of the National Conference on Artificial Intelligence},
keywords = {Machine Learning (Main Track)},
pages = {864--871},
title = {{Investigating contingency awareness using Atari 2600 games}},
volume = {2},
year = {2012}
}
@article{Achiam2018,
abstract = {We explore methods for option discovery based on variational inference and make two algorithmic contributions. First: we highlight a tight connection between variational option discovery methods and variational autoencoders, and introduce Variational Autoencoding Learning of Options by Reinforcement (VALOR), a new method derived from the connection. In VALOR, the policy encodes contexts from a noise distribution into trajectories, and the decoder recovers the contexts from the complete trajectories. Second: we propose a curriculum learning approach where the number of contexts seen by the agent increases whenever the agent's performance is strong enough (as measured by the decoder) on the current set of contexts. We show that this simple trick stabilizes training for VALOR and prior variational option discovery methods, allowing a single agent to learn many more modes of behavior than it could with a fixed context distribution. Finally, we investigate other topics related to variational option discovery, including fundamental limitations of the general approach and the applicability of learned options to downstream tasks.},
archivePrefix = {arXiv},
arxivId = {1807.10299},
author = {Achiam, Joshua and Edwards, Harrison and Amodei, Dario and Abbeel, Pieter},
eprint = {1807.10299},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/Variational Option Discovery Algorithms.pdf:pdf},
pages = {1--29},
title = {{Variational Option Discovery Algorithms}},
url = {http://arxiv.org/abs/1807.10299},
year = {2018}
}
@article{Nachum2018,
abstract = {Hierarchical reinforcement learning (HRL) is a promising approach to extend traditional reinforcement learning (RL) methods to solve more complex tasks. Yet, the majority of current HRL methods require careful task-specific design and on-policy training, making them difficult to apply in real-world scenarios. In this paper, we study how we can develop HRL algorithms that are general, in that they do not make onerous additional assumptions beyond standard RL algorithms, and efficient, in the sense that they can be used with modest numbers of interaction samples, making them suitable for real-world problems such as robotic control. For generality, we develop a scheme where lower-level controllers are supervised with goals that are learned and proposed automatically by the higher-level controllers. To address efficiency, we propose to use off-policy experience for both higher- and lower-level training. This poses a considerable challenge, since changes to the lower-level behaviors change the action space for the higher-level policy, and we introduce an off-policy correction to remedy this challenge. This allows us to take advantage of recent advances in off-policy model-free RL to learn both higher- and lower-level policies using substantially fewer environment interactions than on-policy algorithms. We term the resulting HRL agent HIRO and find that it is generally applicable and highly sample-efficient. Our experiments show that HIRO can be used to learn highly complex behaviors for simulated robots, such as pushing objects and utilizing them to reach target locations, 1 learning from only a few million samples, equivalent to a few days of real-time interaction. In comparisons with a number of prior HRL methods, we find that our approach substantially outperforms previous state-of-the-art techniques 2},
archivePrefix = {arXiv},
arxivId = {1805.08296},
author = {Nachum, Ofir and Lee, Honglak and Gu, Shixiang and Levine, Sergey},
eprint = {1805.08296},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Data efficient HRL.pdf:pdf},
issn = {10495258},
journal = {Advances in Neural Information Processing Systems},
number = {Nips},
pages = {3303--3313},
title = {{Data-efficient hierarchical reinforcement learning}},
volume = {2018-Decem},
year = {2018}
}
@article{Jonschkowski2015,
abstract = {Robot learning is critically enabled by the availability of appropriate state representations. We propose a robotics-specific approach to learning such state representations. As robots accomplish tasks by interacting with the physical world, we can facilitate representation learning by considering the structure imposed by physics; this structure is reflected in the changes that occur in the world and in the way a robot can effect them. By exploiting this structure in learning, robots can obtain state representations consistent with the aspects of physics relevant to the learning task. We name this prior knowledge about the structure of interactions with the physical world robotic priors. We identify five robotic priors and explain how they can be used to learn pertinent state representations. We demonstrate the effectiveness of this approach in simulated and real robotic experiments with distracting moving objects. We show that our method extracts task-relevant state representations from high-dimensional observations, even in the presence of task-irrelevant distractions. We also show that the state representations learned by our method greatly improve generalization in reinforcement learning.},
author = {Jonschkowski, Rico and Brock, Oliver},
doi = {10.1007/s10514-015-9459-7},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Learning State Representations with Robotic Priors.pdf:pdf},
issn = {15737527},
journal = {Autonomous Robots},
keywords = {Prior knowledge,Reinforcement learning,Representation learning,Robot learning},
number = {3},
pages = {407--428},
title = {{Learning state representations with robotic priors}},
volume = {39},
year = {2015}
}

@misc{machado2021temporal,
      title={Temporal Abstraction in Reinforcement Learning with the Successor Representation}, 
      author={Marlos C. Machado and Andre Barreto and Doina Precup},
      year={2021},
      note={arXiv 2110.05740},
}

@inproceedings{Hansen2020Fast,
title={Fast Task Inference with Variational Intrinsic Successor Features},
author={Steven Hansen and Will Dabney and Andre Barreto and David Warde-Farley and Tom Van de Wiele and Volodymyr Mnih},
booktitle={ICLR},
year={2020}
}

@article{Machado2018,
abstract = {Options in reinforcement learning allow agents to hierarchically decompose a task into subtasks, having the potential to speed up learning and planning. However, autonomously learning effective sets of options is still a major challenge in the field. In this paper we focus on the recently introduced idea of using representation learning methods to guide the option discovery process. Specifically, we look at eigenoptions, options obtained from representations that encode diffusive information flow in the environment. We extend the existing algorithms for eigenoption discovery to settings with stochastic transitions and in which handcrafted features are not available. We propose an algorithm that discovers eigenoptions while learning non-linear state representations from raw pixels. It exploits recent successes in the deep reinforcement learning literature and the equivalence between proto-value functions and the successor representation. We use traditional tabular domains to provide intuition about our approach and Atari 2600 games to demonstrate its potential.},
author = {Machado, Marlos C. and Rosenbaum, Clemens and Guo, Xiaoxiao and Liu, Miao and Tesauro, Gerald and Campbell, Murray},
journal = {ICLR},
title = {{Eigenoption discovery through the deep successor representation}},
year = {2018}
}

@article{Harutyunyan2019,
abstract = {In this work, we consider the problem of autonomously discovering behavioral abstractions, or options, for reinforcement learning agents. We propose an algorithm that focuses on the termination condition, as opposed to -- as is common -- the policy. The termination condition is usually trained to optimize a control objective: an option ought to terminate if another has better value. We offer a different, information-theoretic perspective, and propose that terminations should focus instead on the compressibility of the option's encoding -- arguably a key reason for using abstractions. To achieve this algorithmically, we leverage the classical options framework, and learn the option transition model as a "critic" for the termination condition. Using this model, we derive gradients that optimize the desired criteria. We show that the resulting options are non-trivial, intuitively meaningful, and useful for learning and planning.},
archivePrefix = {arXiv},
arxivId = {1902.09996},
author = {Harutyunyan, Anna and Dabney, Will and Borsa, Diana and Heess, Nicolas and Munos, Remi and Precup, Doina},
eprint = {1902.09996},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/The Termination Critic.pdf:pdf},
title = {{The Termination Critic}},
url = {http://arxiv.org/abs/1902.09996},
volume = {89},
year = {2019}
}

@article{Gregor2019,
abstract = {We introduce a new unsupervised reinforcement learning method for discovering the set of intrinsic options available to an agent. This set is learned by maximizing the number of different states an agent can reliably reach, as measured by the mutual information between the set of options and option termination states. To this end, we instantiate two policy gradient based algorithms, one that creates an explicit embedding space of options and one that represents options implicitly. Both algorithms also yield a tractable and explicit empowerment measure, which is useful for empowerment maximizing agents. Furthermore, they scale well with function approximation and we demonstrate their applicability on a range of tasks.},
author = {Gregor, Karol and Rezende, Danilo Jimenez and Wierstra, Daan},
journal = {ICLR},
title = {{Variational intrinsic control}},
year = {2017}
}
@article{Kulkarni,
abstract = {Learning goal-directed behavior in environments with sparse feedback is a major challenge for reinforcement learning algorithms. One of the key difficulties is insufficient exploration, resulting in an agent being unable to learn robust policies. Intrinsically motivated agents can explore new behavior for their own sake rather than to directly solve external goals. Such intrinsic behaviors could eventually help the agent solve tasks posed by the environment. We present hierarchical-DQN (h-DQN), a framework to integrate hierarchical action-value functions, operating at different temporal scales, with goal-driven intrinsically motivated deep reinforcement learning. A top-level q-value function learns a policy over intrinsic goals, while a lower-level function learns a policy over atomic actions to satisfy the given goals. h-DQN allows for flexible goal specifications, such as functions over entities and relations. This provides an efficient space for exploration in complicated environments. We demonstrate the strength of our approach on two problems with very sparse and delayed feedback: (1) a complex discrete stochastic decision process with stochastic transitions, and (2) the classic ATARI game - 'Montezuma's Revenge'.},
archivePrefix = {arXiv},
arxivId = {1604.06057},
author = {Kulkarni, Tejas D. and Narasimhan, Karthik R. and Saeedi, Ardavan and Tenenbaum, Joshua B.},
eprint = {1604.06057},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}2{\_}2019/Hierarchical Deep Reinforcement Learning- Integrating Temporal Abstraction and Intrinsic Motivation.pdf:pdf},
issn = {10495258},
journal = {Advances in Neural Information Processing Systems},
pages = {3682--3690},
title = {{Hierarchical deep reinforcement learning: Integrating temporal abstraction and intrinsic motivation}},
year = {2016}
}

@article{mahadevan07aJMLR,
  author  = {Sridhar Mahadevan and Mauro Maggioni},
  title   = {Proto-value Functions: A Laplacian Framework for Learning Representation and Control in Markov Decision Processes},
  journal = {Journal of Machine Learning Research},
  year    = {2007}
}

@article{Mahadevan2005a,
abstract = {We investigate the problem of automatically constructing efficient representations or basis functions for approximating value functions based on analyzing the structure and topology of the state space. In particular, two novel approaches to value function approximation are explored based on automatically constructing basis functions on state spaces that can be represented as graphs or manifolds: one approach uses the eigenfunctions of the Laplacian, in effect performing a global Fourier analysis on the graph; the second approach is based on diffusion wavelets, which generalize classical wavelets to graphs using multiscale dilations induced by powers of a diffusion operator or random walk on the graph. Together, these approaches form the foundation of a new generation of methods for solving large Markov decision processes, in which the underlying representation and policies are simultaneously learned.},
author = {Mahadevan, Sridhar and Maggioni, Mauro},
journal = {NeurIPS},
title = {{Value function approximation with diffusion wavelets and Laplacian eigenfunctions}},
year = {2005}
}
@article{Richthofer2017,
abstract = {Predictable Feature Analysis (PFA) (Richthofer, Wiskott, ICMLA 2015) is an algorithm that performs dimensionality reduction on high dimensional input signal. It extracts those subsignals that are most predictable according to a certain prediction model. We refer to these extracted signals as predictable features. In this work we extend the notion of PFA to take supplementary information into account for improving its predictions. Such information can be a multidimensional signal like the main input to PFA, but is regarded external. That means it won't participate in the feature extraction - no features get extracted or composed of it. Features will be exclusively extracted from the main input such that they are most predictable based on themselves and the supplementary information. We refer to this enhanced PFA as PFAx (PFA extended). Even more important than improving prediction quality is to observe the effect of supplementary information on feature selection. PFAx transparently provides insight how the supplementary information adds to prediction quality and whether it is valuable at all. Finally we show how to invert that relation and can generate the supplementary information such that it would yield a certain desired outcome of the main signal. We apply this to a setting inspired by reinforcement learning and let the algorithm learn how to control an agent in an environment. With this method it is feasible to locally optimize the agent's state, i.e. reach a certain goal that is near enough. We are preparing a follow-up paper that extends this method such that also global optimization is feasible.},
archivePrefix = {arXiv},
arxivId = {1712.00634},
author = {Richthofer, Stefan and Wiskott, Laurenz},
eprint = {1712.00634},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/PFAx Predictable Feature Analysis to Perform Control.pdf:pdf},
pages = {1--24},
title = {{PFAx: Predictable Feature Analysis to Perform Control}},
url = {http://arxiv.org/abs/1712.00634},
year = {2017}
}
@article{Bellemare2019,
abstract = {We propose a new perspective on representation learning in reinforcement learning based on geometric properties of the space of value functions. We leverage this perspective to provide formal evidence regarding the usefulness of value functions as auxiliary tasks. Our formulation considers adapting the representation to minimize the (linear) approximation of the value function of all stationary policies for a given environment. We show that this optimization reduces to making accurate predictions regarding a special class of value functions which we call adversarial value functions (AVFs). We demonstrate that using value functions as auxiliary tasks corresponds to an expected-error relaxation of our formulation, with AVFs a natural candidate, and identify a close relationship with proto-value functions (Mahadevan, 2005). We highlight characteristics of AVFs and their usefulness as auxiliary tasks in a series of experiments on the four-room domain.},
archivePrefix = {arXiv},
arxivId = {1901.11530},
author = {Bellemare, Marc G. and Dabney, Will and Dadashi, Robert and Taiga, Adrien Ali and Castro, Pablo Samuel and Roux, Nicolas Le and Schuurmans, Dale and Lattimore, Tor and Lyle, Clare},
eprint = {1901.11530},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/A Geometric Perspective on Optimal Representations for Reinforcement Learning.pdf:pdf},
title = {{A Geometric Perspective on Optimal Representations for Reinforcement Learning}},
url = {http://arxiv.org/abs/1901.11530},
year = {2019}
}
@article{Tiwari2018,
abstract = {Hierarchical reinforcement learning deals with the problem of breaking down large tasks into meaningful sub-tasks. Autonomous discovery of these sub-tasks has remained a challenging problem. We propose a novel method of learning sub-tasks by combining paradigms of routing in computer networks and graph based skill discovery within the options framework to define meaningful sub-goals. We apply the recent advancements of learning embeddings using Riemannian optimisation in the hyperbolic space to embed the state set into the hyperbolic space and create a model of the environment. In doing so we enforce a global topology on the states and are able to exploit this topology to learn meaningful sub-tasks. We demonstrate empirically, both in discrete and continuous domains, how these embeddings can improve the learning of meaningful sub-tasks.},
archivePrefix = {arXiv},
arxivId = {1812.01487},
author = {Tiwari, Saket and Prannoy, M.},
eprint = {1812.01487},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Hyperbolic Embeddings for Learning Options in Hierarchical Reinforcement Learning.pdf:pdf},
title = {{Hyperbolic Embeddings for Learning Options in Hierarchical Reinforcement Learning}},
url = {http://arxiv.org/abs/1812.01487},
volume = {89},
year = {2018}
}
@article{Petrik2007,
abstract = {Recently, a method based on Laplacian eigenfunctions was proposed to automatically construct a basis for value function approximation in MDPs. We show that its success may be explained by drawing a connection between the spectrum of the Laplacian and the value function of the MDP. This explanation helps us to identify more precisely the conditions that this method requires to achieve good performance. Based on this, we propose a modification of the Laplacian method for which we derive an analytical bound on the approximation error. Further, we show that the method is related the augmented Krylov methods, commonly used to solve sparse linear systems. Finally, we empirically demonstrate that in basis construction the augmented Krylov methods may significantly outperform the Laplacian methods in terms of both speed and quality.},
author = {Petrik, Marek},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/An Analysis of Laplacian Methods for Value Function Approximation in MDPs IJCAI07-414.pdf:pdf},
issn = {10450823},
journal = {IJCAI International Joint Conference on Artificial Intelligence},
keywords = {Markov-decision processes,reinforcement learning},
pages = {2574--2579},
title = {{An analysis of Laplacian methods for value function approximation in MDPs}},
year = {2007}
}
@article{Mahadevan2005,
abstract = {This paper presents a novel framework called proto-reinforcement learning (PEL), based on a mathematical model of a proto-value function: these are task-independent basis functions that form the building blocks of all value functions on a given state space manifold. Proto-value functions are learned not from rewards, but instead from analyzing the topology of the state space. Formally, proto-value functions are Fourier eigenfunctions of the Laplace-Beltrami diffusion operator on the state space manifold. Proto-value functions facilitate structural decomposition of large state spaces, and form geodesically smooth orthonormal basis functions for approximating any value function. The theoretical basis for proto-value functions combines insights from spectral graph theory, harmonic analysis, and Riemannian manifolds. Protovalue functions enable a novel generation of algorithms called representation policy iteration, unifying the learning of representation and behavior.},
author = {Mahadevan, Sridhar},
journal = {ICML},
title = {{Proto-value functions: Developmental reinforcement learning}},
year = {2005}
}
@article{Oord2018,
abstract = {While supervised learning has enabled great progress in many applications, unsupervised learning has not seen such widespread adoption, and remains an important and challenging endeavor for artificial intelligence. In this work, we propose a universal unsupervised learning approach to extract useful representations from high-dimensional data, which we call Contrastive Predictive Coding. The key insight of our model is to learn such representations by predicting the future in latent space by using powerful autoregressive models. We use a probabilistic contrastive loss which induces the latent space to capture information that is maximally useful to predict future samples. It also makes the model tractable by using negative sampling. While most prior work has focused on evaluating representations for a particular modality, we demonstrate that our approach is able to learn useful representations achieving strong performance on four distinct domains: speech, images, text and reinforcement learning in 3D environments.},
archivePrefix = {arXiv},
arxivId = {1807.03748},
author = {van den Oord, Aaron and Li, Yazhe and Vinyals, Oriol},
eprint = {1807.03748},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Representation Learning with Contrastive Predictive Coding.pdf:pdf},
title = {{Representation Learning with Contrastive Predictive Coding}},
url = {http://arxiv.org/abs/1807.03748},
year = {2018}
}
@article{Thomas2017,
abstract = {It has been postulated that a good representation is one that disentangles the underlying explanatory factors of variation. However, it remains an open question what kind of training framework could potentially achieve that. Whereas most previous work focuses on the static setting (e.g., with images), we postulate that some of the causal factors could be discovered if the learner is allowed to interact with its environment. The agent can experiment with different actions and observe their effects. More specifically, we hypothesize that some of these factors correspond to aspects of the environment which are independently controllable, i.e., that there exists a policy and a learnable feature for each such aspect of the environment, such that this policy can yield changes in that feature with minimal changes to other features that explain the statistical variations in the observed data. We propose a specific objective function to find such factors and verify experimentally that it can indeed disentangle independently controllable aspects of the environment without any extrinsic reward signal.},
archivePrefix = {arXiv},
arxivId = {1708.01289},
author = {Thomas, Valentin and Pondard, Jules and Bengio, Emmanuel and Sarfati, Marc and Beaudoin, Philippe and Meurs, Marie-Jean and Pineau, Joelle and Precup, Doina and Bengio, Yoshua},
eprint = {1708.01289},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Independently Controllable Factors.pdf:pdf},
pages = {1--13},
title = {{Independently Controllable Factors}},
url = {http://arxiv.org/abs/1708.01289},
year = {2017}
}
@article{Hazan2018,
abstract = {Suppose an agent is in a (possibly unknown) Markov Decision Process in the absence of a reward signal, what might we hope that an agent can efficiently learn to do? This work studies a broad class of objectives that are defined solely as functions of the state-visitation frequencies that are induced by how the agent behaves. For example, one natural, intrinsically defined, objective problem is for the agent to learn a policy which induces a distribution over state space that is as uniform as possible, which can be measured in an entropic sense. We provide an efficient algorithm to optimize such such intrinsically defined objectives, when given access to a black box planning oracle (which is robust to function approximation). Furthermore, when restricted to the tabular setting where we have sample based access to the MDP, our proposed algorithm is provably efficient, both in terms of its sample and computational complexities. Key to our algorithmic methodology is utilizing the conditional gradient method (a.k.a. the Frank-Wolfe algorithm) which utilizes an approximate MDP solver.},
archivePrefix = {arXiv},
arxivId = {1812.02690},
author = {Hazan, Elad and Kakade, Sham M. and Singh, Karan and {Van Soest}, Abby},
eprint = {1812.02690},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Provably Efficient Maximum Entropy Exploration.pdf:pdf},
pages = {1--16},
title = {{Provably Efficient Maximum Entropy Exploration}},
url = {http://arxiv.org/abs/1812.02690},
year = {2018}
}
@article{Nickel2018,
abstract = {We are concerned with the discovery of hierarchical relationships from large-scale unstructured similarity scores. For this purpose, we study different models of hyperbolic space and find that learning embeddings in the Lorentz model is substantially more efficient than in the Poincar{\'{e}}-ball model. We show that the proposed approach allows us to learn high-quality embeddings of large taxonomies which yield improvements over Poincar{\'{e}} embeddings, especially in low dimensions. Lastly, we apply our model to discover hierarchies in two real-world datasets: we show that an embedding in hyperbolic space can reveal important aspects of a company's organizational structure as well as reveal historical relationships between language families.},
archivePrefix = {arXiv},
arxivId = {1806.03417},
author = {Nickel, Maximilian and Kiela, Douwe},
eprint = {1806.03417},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Learning Continuous Hierarchies in the Lorentz Model of Hyperbolic Geometry.pdf:pdf},
isbn = {9781510867963},
journal = {35th International Conference on Machine Learning, ICML 2018},
pages = {6071--6080},
title = {{Learning continuous hierarchies in the Lorentz model of hyperbolic geometry}},
volume = {9},
year = {2018}
}
@article{Haque2019,
abstract = {Learning disentangled representation from any unlabelled data is a non-trivial problem. In this paper we propose Information Maximising Autoencoder (InfoAE) where the encoder learns powerful disentangled representation through maximizing the mutual information between the representation and given information in an unsupervised fashion. We have evaluated our model on MNIST dataset and achieved 98.9 ({\$}\backslashpm .1{\$}) {\$}\backslash{\%}{\$} test accuracy while using complete unsupervised training.},
archivePrefix = {arXiv},
arxivId = {1904.08613},
author = {Haque, Kazi Nazmul and Latif, Siddique and Rana, Rajib},
eprint = {1904.08613},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/DISENTANGLED REPRESENTATION LEARNING WITH INFORMATION MAXIMIZING AUTOENCODER.pdf:pdf},
pages = {1--5},
title = {{Disentangled Representation Learning with Information Maximizing Autoencoder}},
url = {http://arxiv.org/abs/1904.08613},
year = {2019}
}
@inproceedings{Wu2019,
abstract = {The smallest eigenvectors of the graph Laplacian are well-known to provide a succinct representation of the geometry of a weighted graph. In reinforcement learning (RL), where the weighted graph may be interpreted as the state transition process induced by a behavior policy acting on the environment, approximating the eigenvectors of the Laplacian provides a promising approach to state representation learning. However, existing methods for performing this approximation are ill-suited in general RL settings for two main reasons: First, they are computationally expensive, often requiring operations on large matrices. Second, these methods lack adequate justification beyond simple, tabular, finite-state settings. In this paper, we present a fully general and scalable method for approximating the eigenvectors of the Laplacian in a model-free RL context. We systematically evaluate our approach and empirically show that it generalizes beyond the tabular, finite-state setting. Even in tabular, finite-state settings, its ability to approximate the eigenvectors outperforms previous proposals. Finally, we show the potential benefits of using a Laplacian representation learned using our method in goal-achieving RL tasks, providing evidence that our technique can be used to significantly improve the performance of an RL agent.},
author = {Wu, Yifan and Tucker, George and Nachum, Ofir},
booktitle = {ICLR},
title = {{The Laplacian in RL: Learning representations with efficient approximations}},
year = {2019}
}

@article{pong2019skew,
  title={Skew-fit: State-covering self-supervised reinforcement learning},
  author={Pong, Vitchyr H and Dalal, Murtaza and Lin, Steven and Nair, Ashvin and Bahl, Shikhar and Levine, Sergey},
  journal={ICML},
  year={2020}
}

@inproceedings{
kamienny2021direct,
title={Direct then Diffuse: Incremental Unsupervised Skill Discovery for State Covering and Goal Reaching},
author={Pierre-Alexandre Kamienny and Jean Tarbouriech and Alessandro Lazaric and Ludovic Denoyer},
booktitle={ICML, Workshop on Unsupervised Reinforcement Learning},
year={2021}
}

@article{ecoffet2021nature,
  title={First return then explore},
  author={Adrien Ecoffet and Joost Huizinga and Joel Lehman and Kenneth O. Stanley and Jeff Clune},
  journal={Nature},
  year={2021}
}

@article{ecoffet2019go,
  title={Go-explore: a new approach for hard-exploration problems},
  author={Ecoffet, Adrien and Huizinga, Joost and Lehman, Joel and Stanley, Kenneth O and Clune, Jeff},
  journal={arXiv:1901.10995},
  year={2019}
}

@article{Osentoski2007,
abstract = {This paper introduces a new approach to action-value function approximation by learning basis functions from a spectral decomposition of the state-action manifold. This paper extends previous work on using Laplacian bases for value function approximation by using the actions of the agent as part of the representation when creating basis functions. The approach results in a nonlinear learned representation particularly suited to approximating action-value functions, without incurring the wasteful duplication of state bases in previous work. We discuss two techniques to create state-action graphs: off-policy and on-policy. We show that these graphs have a greater expressive power and have better performance over state-based Laplacian basis functions in domains modeled as Semi-Markov Decision Processes (SMDPs). We present a simple graph partitioning method to scale the approach to large discrete MDPs.},
author = {Osentoski, Sarah and Mahadevan, Sridhar},
doi = {10.1145/1273496.1273585},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Learning State-Action Basis Functions for Hierarchical MDPs (icml07).pdf:pdf},
journal = {ACM International Conference Proceeding Series},
pages = {705--712},
title = {{Learning state-action basis functions for hierarchical MDPs}},
volume = {227},
year = {2007}
}

@book{trefethen1997numerical,
  added-at = {2020-08-19T03:33:06.000+0200},
  address = {Philadelphia},
  author = {Trefethen, Lloyd Nicholas and Bau, David},
  description = {Numerical linear algebra (Book, 1997) [WorldCat.org]},
  keywords = {65-01-numerical-analysis-instructional-exposition 65f15-numerical-eigenvalues-eigenvectors},
  publisher = {SIAM},
  refid = {933331628},
  timestamp = {2021-02-25T06:15:39.000+0100},
  title = {Numerical Linear Algebra},
  year = 1997
}


@article{Dilokthanakul2019,
abstract = {One of the main concerns of deep reinforcement learning (DRL) is the data inefficiency problem, which stems both from an inability to fully utilize data acquired and from naive exploration strategies. In order to alleviate these problems, we propose a DRL algorithm that aims to improve data efficiency via both the utilization of unrewarded experiences and the exploration strategy by combining ideas from unsupervised auxiliary tasks, intrinsic motivation, and hierarchical reinforcement learning (HRL). Our method is based on a simple HRL architecture with a metacontroller and a subcontroller. The subcontroller is intrinsically motivated by the metacontroller to learn to control aspects of the environment, with the intention of giving the agent: 1) a neural representation that is generically useful for tasks that involve manipulation of the environment and 2) the ability to explore the environment in a temporally extended manner through the control of the metacontroller. In this way, we reinterpret the notion of pixel- and feature-control auxiliary tasks as reusable skills that can be learned via an intrinsic reward. We evaluate our method on a number of Atari 2600 games. We found that it outperforms the baseline in several environments and significantly improves performance in one of the hardest games - Montezuma's revenge - for which the ability to utilize sparse data is key. We found that the inclusion of intrinsic reward is crucial for the improvement in the performance and that most of the benefit seems to be derived from the representations learned during training.},
archivePrefix = {arXiv},
arxivId = {1705.06769},
author = {Dilokthanakul, Nat and Kaplanis, Christos and Pawlowski, Nick and Shanahan, Murray},
doi = {10.1109/TNNLS.2019.2891792},
eprint = {1705.06769},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Feature Control as Intrinsic Motivation for Hierarchical Reinforcement Learning.pdf:pdf},
issn = {21622388},
journal = {IEEE Transactions on Neural Networks and Learning Systems},
keywords = {Auxiliary task,deep reinforcement learning (DRL),hierarchical reinforcement learning (HRL),intrinsic motivation},
number = {11},
pages = {3409--3418},
title = {{Feature Control as Intrinsic Motivation for Hierarchical Reinforcement Learning}},
volume = {30},
year = {2019}
}

@inproceedings{
Jinnai2020Exploration,
title={Exploration in Reinforcement Learning with Deep Covering Options},
author={Yuu Jinnai and Jee Won Park and Marlos C. Machado and George Konidaris},
booktitle={ICLR},
year={2020}
}

@article{von2007tutorial,
  title={A tutorial on spectral clustering},
  author={Von Luxburg, Ulrike},
  journal={Statistics and computing},
  volume={17},
  number={4},
  pages={395--416},
  year={2007},
  publisher={Springer}
}

@inproceedings{Sharma2020Dynamics,
title={Dynamics-Aware Unsupervised Discovery of Skills},
author={Archit Sharma and Shixiang Gu and Sergey Levine and Vikash Kumar and Karol Hausman},
booktitle={ICLR},
year={2020}
}

@article{Ferrante2008,
abstract = {Reinforcement Learning research is traditionally devoted to solve single-task problems. Therefore, anytime a new task is faced, learning must be restarted from scratch. Recently, several studies have addressed the issue of reusing the knowledge acquired in solving previous related tasks by transfer- ring information about policies and value functions. In this paper, we analyze the use of proto-value functions under the transfer learning perspective. Proto-value functions are effective basis functions for the approximation of value functions defined over the graph obtained by a random walk on the environment. The definition of this graph is a key aspect in transfer transfer problems in which both the reward function and the dynamics change. Therefore, we introduce policy-based proto-value functions, which can be obtained by considering the graph generated by a random walk guided by the optimal policy of one of the tasks at hand. We compare the effectiveness of policy-based and standard proto-value functions, on different transfer problems defined on a simple grid-world environment. Copyright {\textcopyright} 2008, International Foundation for Autonomous Agents and Multiagent Systems (www.ifaamas.org). All rights reserved.},
author = {Ferrante, Eliseo and Lazaric, Alessandro and Restelli, Marcello},
file = {:Users/akram/Research/literature/RL/unsup skill discovery/lit{\_}1{\_}2019/Transfer of Task Representation in Reinforcement Learning using Policy-based Proto-value Functions.pdf:pdf},
isbn = {9781605604701},
issn = {15582914},
journal = {Proceedings of the International Joint Conference on Autonomous Agents and Multiagent Systems, AAMAS},
keywords = {Proto-value functions,Reinforcement Learning,Transfer Learning},
pages = {1301--1304},
title = {{Transfer of task representation in Reinforcement Learning using policy-based proto-value functions}},
volume = {3},
year = {2008}
}
@article{Nachum2019b,
abstract = {We study the problem of representation learning in goal-conditioned hierarchical reinforcement learning. In such hierarchical structures, a higher-level controller solves tasks by iteratively communicating goals which a lower-level policy is trained to reach. Accordingly, the choice of representation - the mapping of observation space to goal space - is crucial. To study this problem, we develop a notion of sub-optimality of a representation, defined in terms of expected reward of the optimal hierarchical policy using this representation. We derive expressions which bound the sub-optimality and show how these expressions can be translated to representation learning objectives which may be optimized in practice. Results on a number of difficult continuous-control tasks show that our approach to representation learning yields qualitatively better representations as well as quantitatively better hierarchical policies, compared to existing methods.},
author = {Nachum, Ofir and Gu, Shixiang and Lee, Honglak and Levine, Sergey},
journal = {ICLR},
title = {{Near-optimal representation learning for hierarchical reinforcement learning}},
year = {2019}
}
@article{Jaderberg2019,
abstract = {Deep reinforcement learning agents have achieved state-of-the-art results by directly maximising cumulative reward. However, environments contain a much wider variety of possible training signals. In this paper, we introduce an agent that also learns separate policies for maximising many other pseudo-reward functions simultaneously by reinforcement learning. All of these tasks share a common representation that, like unsupervised learning, continues to develop in the absence of extrinsic rewards. We also introduce a novel mechanism for focusing this representation upon extrinsic rewards, so that learning can rapidly adapt to the most relevant aspects of the actual task. Our agent significantly outperforms the previous state-of-the-art on Atari, averaging 880{\%} expert human performance, and a challenging suite of first-person, three-dimensional Labyrinth tasks leading to a mean speedup in learning of 10× and averaging 87{\%} expert human performance on Labyrinth.},
archivePrefix = {arXiv},
arxivId = {1611.05397},
author = {Jaderberg, Max and Mnih, Volodymyr and Czarnecki, Wojciech Marian and Schaul, Tom and Leibo, Joel Z. and Silver, David and Kavukcuoglu, Koray},
eprint = {1611.05397},
journal = {ICLR},
pages = {1--14},
title = {{Reinforcement learning with unsupervised auxiliary tasks}},
year = {2017}
}


@article{Mnih2015,
  author    = {Volodymyr Mnih and
               Koray Kavukcuoglu and
               David Silver and
               Andrei A. Rusu and
               Joel Veness and
               Marc G. Bellemare and
               Alex Graves and
               Martin A. Riedmiller and
               Andreas Fidjeland and
               Georg Ostrovski and
               Stig Petersen and
               Charles Beattie and
               Amir Sadik and
               Ioannis Antonoglou and
               Helen King and
               Dharshan Kumaran and
               Daan Wierstra and
               Shane Legg and
               Demis Hassabis},
  title     = {Human-level control through deep reinforcement learning},
  journal   = {Nature},
  year      = {2015}
}

@article{Bellemare2020,
  author    = {Marc G. Bellemare and
               Salvatore Candido and
               Pablo Samuel Castro and
               Jun Gong and
               Marlos C. Machado and
               Subhodeep Moitra and
               Sameera S. Ponda and
               Ziyu Wang},
  title     = {Autonomous navigation of stratospheric balloons using reinforcement learning},
  journal   = {Nature},
  year      = {2020},
}

@inproceedings{Dayan92,
  author    = {Peter Dayan and Geoffrey E. Hinton},
  title     = {Feudal Reinforcement Learning},
  booktitle = {NIPS},
  year      = {1992}
}

@article{Bellemare2013,
  author    = {Marc G. Bellemare and
               Yavar Naddaf and
               Joel Veness and
               Michael Bowling},
  title     = {The Arcade Learning Environment: An Evaluation Platform for General Agents},
  journal   = {Journal of Artificial Intelligence Research},
  volume    = {47},
  pages     = {253--279},
  year      = {2013}
}