@article{abel2021expressivity,
  title={On the Expressivity of Markov Reward},
  author={Abel, David and Dabney, Will and Harutyunyan, Anna and Ho, Mark K and Littman, Michael and Precup, Doina and Singh, Satinder},
  journal={NIPS},
  year={2021}
}

@article{janner2021offline,
  title={Offline Reinforcement Learning as One Big Sequence Modeling Problem},
  author={Janner, Michael and Li, Qiyang and Levine, Sergey},
  journal={NeurIPS},
  year={2021}
}

@article{chen2021decision,
  title={Decision transformer: Reinforcement learning via sequence modeling},
  author={Chen, Lili and Lu, Kevin and Rajeswaran, Aravind and Lee, Kimin and Grover, Aditya and Laskin, Misha and Abbeel, Pieter and Srinivas, Aravind and Mordatch, Igor},
  journal={NeurIPS},
  volume={34},
  year={2021}
}

@article{furuta2021generalized,
  title={Generalized Decision Transformer for Offline Hindsight Information Matching},
  author={Furuta, Hiroki and Matsuo, Yutaka and Gu, Shixiang Shane},
  journal={ arXiv:2111.10364},
  year={2021}
}

@inproceedings{choromanski2020rethinking,
  title={Rethinking Attention with Performers},
  author={Choromanski, Krzysztof Marcin and Likhosherstov, Valerii and Dohan, David and Song, Xingyou and Gane, Andreea and Sarlos, Tamas and Hawkins, Peter and Davis, Jared Quincy and Mohiuddin, Afroz and Kaiser, Lukasz and others},
  booktitle={ICLR},
  year={2020}
}

@article{ma2018noise,
  title={Noise contrastive estimation and negative sampling for conditional models: Consistency and statistical efficiency},
  author={Ma, Zhuang and Collins, Michael},
  journal={ arXiv:1809.01812},
  year={2018}
}

@article{ross2008online,
  title={Online planning algorithms for POMDPs},
  author={Ross, St{\'e}phane and Pineau, Joelle and Paquet, S{\'e}bastien and Chaib-Draa, Brahim},
  journal={Journal of Artificial Intelligence Research},
  volume={32},
  pages={663--704},
  year={2008}
}

@book{krishnamurthy2016partially,
title={Partially Observed Markov Decision Processes},
author={Krishnamurthy, Vikram},
year={2016},
publisher={Cambridge University Press}
}


@book{puterman2014markov,
  title={Markov decision processes: discrete stochastic dynamic programming},
  author={Puterman, Martin L},
  year={2014},
  publisher={John Wiley \& Sons}
}

@article{banino2021coberl,
  title={CoBERL: Contrastive BERT for Reinforcement Learning},
  author={Banino, Andrea and Puidomenech Badia, Adri{\`a} and Walker, Jacob and Scholtes, Tim and Mitrovic, Jovana and Blundell, Charles},
  journal={arXiv e-prints},
  pages={arXiv--2107},
  year={2021}
}

@article{zheng2022online,
  title={Online Decision Transformer},
  author={Zheng, Qinqing and Zhang, Amy and Grover, Aditya},
  journal={ arXiv:2202.05607},
  year={2022}
}


@inproceedings{igl2018deep,
  title={Deep variational reinforcement learning for POMDPs},
  author={Igl, Maximilian and Zintgraf, Luisa and Le, Tuan Anh and Wood, Frank and Whiteson, Shimon},
  booktitle={ICML},
  year={2018}
}

@inproceedings{singh2004predictive,
  title={Predictive state representations: a new theory for modeling dynamical systems},
  author={Singh, Satinder and James, Michael R and Rudary, Matthew R},
  booktitle={UAI},
  year={2004}
}

@article{littman2001predictive,
  title={Predictive representations of state},
  author={Littman, Michael and Sutton, Richard S},
  journal={NeurIPS},
  volume={14},
  year={2001}
}

@inproceedings{azizzadenesheli2016reinforcement,
  title={Reinforcement learning of POMDPs using spectral methods},
  author={Azizzadenesheli, Kamyar and Lazaric, Alessandro and Anandkumar, Animashree},
  booktitle={COLT},
  year={2016},
}

@article{aastrom1965optimal,
  title={Optimal control of Markov processes with incomplete state information},
  author={{\AA}str{\"o}m, Karl Johan},
  journal={Journal of mathematical analysis and applications},
  volume={10},
  number={1},
  pages={174--205},
  year={1965},
  publisher={Academic Press}
}

@article{kaelbling1998planning,
  title={Planning and acting in partially observable stochastic domains},
  author={Kaelbling, Leslie Pack and Littman, Michael L and Cassandra, Anthony R},
  journal={Artificial intelligence},
  volume={101},
  number={1-2},
  pages={99--134},
  year={1998},
  publisher={Elsevier}
}


@inproceedings{guo2016pac,
  title={A PAC RL algorithm for episodic pomdps},
  author={Guo, Zhaohan and Doroudi, Shayan and Brunskill, Emma},
  booktitle={AISTATS},
  year={2016}
}

@article{jin2020sample,
  title={Sample-efficient reinforcement learning of undercomplete POMDPs},
  author={Jin, Chi and Kakade, Sham and Krishnamurthy, Akshay and Liu, Qinghua},
  journal={NeurIPS},
  year={2020}
}

@article{papadimitriou1987complexity,
  title={The complexity of Markov decision processes},
  author={Papadimitriou, Christos H and Tsitsiklis, John N},
  journal={Mathematics of operations research},
  volume={12},
  number={3},
  pages={441--450},
  year={1987},
  publisher={INFORMS}
}

@inproceedings{madani1998computability,
  title={On the computability of infinite-horizon partially observable Markov decision processes},
  author={Madani, Omid and Hanks, S and Condon, A},
  booktitle={AAAI98 Fall Symposium on Planning with POMDPs, Orlando, FL},
  year={1998},
  organization={Citeseer}
}

@article{vlassis2012computational,
  title={On the computational complexity of stochastic controller optimization in POMDPs},
  author={Vlassis, Nikos and Littman, Michael L and Barber, David},
  journal={ACM Transactions on Computation Theory (TOCT)},
  volume={4},
  number={4},
  pages={1--8},
  year={2012},
  publisher={ACM New York, NY, USA}
}

@inproceedings{mossel2005learning,
  title={Learning nonsingular phylogenies and hidden Markov models},
  author={Mossel, Elchanan and Roch, S{\'e}bastien},
  booktitle={Proceedings of the thirty-seventh annual ACM symposium on Theory of computing},
  pages={366--375},
  year={2005}
}

@article{cai2022sample,
  title={Sample-Efficient Reinforcement Learning for POMDPs with Linear Function Approximations},
  author={Cai, Qi and Yang, Zhuoran and Wang, Zhaoran},
  journal={ arXiv:2204.09787},
  year={2022}
}

@article{efroni2022provable,
  title={Provable Reinforcement Learning with a Short-Term Memory},
  author={Efroni, Yonathan and Jin, Chi and Krishnamurthy, Akshay and Miryoosefi, Sobhan},
  journal={ arXiv:2202.03983},
  year={2022}
}

@article{krishnamurthy2016pac,
  title={PAC reinforcement learning with rich observations},
  author={Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John},
  journal={NeurIPS},
  year={2016}
}

@inproceedings{jiang2017contextual,
  title={Contextual decision processes with low bellman rank are pac-learnable},
  author={Jiang, Nan and Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John and Schapire, Robert E},
  booktitle={ICML},
  pages={1704--1713},
  year={2017},
  organization={PMLR}
}

@inproceedings{du2019provably,
  title={Provably efficient RL with rich observations via latent state decoding},
  author={Du, Simon and Krishnamurthy, Akshay and Jiang, Nan and Agarwal, Alekh and Dudik, Miroslav and Langford, John},
  booktitle={ICML},
  year={2019},
}

@inproceedings{misra2020kinematic,
  title={Kinematic state abstraction and provably efficient rich-observation reinforcement learning},
  author={Misra, Dipendra and Henaff, Mikael and Krishnamurthy, Akshay and Langford, John},
  booktitle={ICML},
  pages={6961--6971},
  year={2020},
  organization={PMLR}
}

@article{jaeger2000observable,
  title={Observable operator models for discrete stochastic time series},
  author={Jaeger, Herbert},
  journal={Neural computation},
  volume={12},
  number={6},
  pages={1371--1398},
  year={2000},
  publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…}
}

@article{boots2011closing,
  title={Closing the learning-planning loop with predictive state representations},
  author={Boots, Byron and Siddiqi, Sajid M and Gordon, Geoffrey J},
  journal={The International Journal of Robotics Research},
  volume={30},
  number={7},
  pages={954--966},
  year={2011},
  publisher={SAGE Publications Sage UK: London, England}
}

@article{boots2013hilbert,
  title={Hilbert space embeddings of predictive state representations},
  author={Boots, Byron and Gordon, Geoffrey and Gretton, Arthur},
  journal={UAI},
  year={2013}
}

@inproceedings{kulesza2015spectral,
  title={Spectral learning of predictive state representations with insufficient statistics},
  author={Kulesza, Alex and Jiang, Nan and Singh, Satinder},
  booktitle={Twenty-Ninth AAAI Conference on Artificial Intelligence},
  year={2015}
}

@article{downey2017predictive,
  title={Predictive state recurrent neural networks},
  author={Downey, Carlton and Hefny, Ahmed and Boots, Byron and Gordon, Geoffrey J and Li, Boyue},
  journal={NeurIPS},
  volume={30},
  year={2017}
}

@article{guo2018neural,
  title={Neural predictive belief representations},
  author={Guo, Zhaohan Daniel and Azar, Mohammad Gheshlaghi and Piot, Bilal and Pires, Bernardo A and Munos, R{\'e}mi},
  journal={arXiv:1811.06407},
  year={2018}
}

@inproceedings{hefny2018recurrent,
  title={Recurrent predictive state policy networks},
  author={Hefny, Ahmed and Marinho, Zita and Sun, Wen and Srinivasa, Siddhartha and Gordon, Geoffrey},
  booktitle={ICML},
  year={2018}
}

@inproceedings{hamilton2013modelling,
  title={Modelling sparse dynamical systems with compressed predictive state representations},
  author={Hamilton, William L and Fard, Mahdi Milani and Pineau, Joelle},
  booktitle={ICML},
  pages={178--186},
  year={2013},
  organization={PMLR}
}

@article{hamilton2014efficient,
  title={Efficient learning and planning with compressed predictive states},
  author={Hamilton, William and Fard, Mahdi Milani and Pineau, Joelle},
  journal={The JMLR},
  volume={15},
  number={1},
  pages={3395--3439},
  year={2014},
  publisher={JMLR. org}
}

@inproceedings{parisotto2020stabilizing,
  title={Stabilizing transformers for reinforcement learning},
  author={Parisotto, Emilio and Song, Francis and Rae, Jack and Pascanu, Razvan and Gulcehre, Caglar and Jayakumar, Siddhant and Jaderberg, Max and Kaufman, Raphael Lopez and Clark, Aidan and Noury, Seb and others},
  booktitle={ICML},
  pages={7487--7498},
  year={2020},
  organization={PMLR}
}

@inproceedings{jin2020provably,
  title={Provably efficient reinforcement learning with linear function approximation},
  author={Jin, Chi and Yang, Zhuoran and Wang, Zhaoran and Jordan, Michael I},
  booktitle={COLT},
  year={2020}
}

@article{uehara2021representation,
  title={Representation Learning for Online and Offline RL in Low-rank MDPs},
  author={Uehara, Masatoshi and Zhang, Xuezhou and Sun, Wen},
  journal={ arXiv:2110.04652},
  year={2021}
}

@inproceedings{james2004planning,
  title={Planning with predictive state representations},
  author={James, Michael R and Singh, Satinder and Littman, Michael L},
  booktitle={2004 ICML and Applications, 2004. Proceedings.},
  pages={304--311},
  year={2004},
  organization={IEEE}
}

@article{levine2016end,
  title={End-to-end training of deep visuomotor policies},
  author={Levine, Sergey and Finn, Chelsea and Darrell, Trevor and Abbeel, Pieter},
  journal={The JMLR},
  volume={17},
  number={1},
  pages={1334--1373},
  year={2016},
  publisher={JMLR. org}
}

@article{li2016deep,
  title={Deep reinforcement learning for dialogue generation},
  author={Li, Jiwei and Monroe, Will and Ritter, Alan and Galley, Michel and Gao, Jianfeng and Jurafsky, Dan},
  journal={arXiv:1606.01541},
  year={2016}
}

@inproceedings{jiang2021towards,
  title={Towards Automatic Evaluation of Dialog Systems: A Model-Free Off-Policy Evaluation Approach},
  author={Jiang, Haoming and Dai, Bo and Yang, Mengjiao and Zhao, Tuo and Wei, Wei},
  booktitle={EMNLP},
  year={2021}
}

@inproceedings{izadi2008point,
  title={Point-based planning for predictive state representations},
  author={Izadi, Masoumeh T and Precup, Doina},
  booktitle={Conference of the Canadian Society for Computational Studies of Intelligence},
  pages={126--137},
  year={2008},
  organization={Springer}
}

@inproceedings{shani2005model,
  title={Model-based online learning of POMDPs},
  author={Shani, Guy and Brafman, Ronen I and Shimony, Solomon E},
  booktitle={European Conference on Machine Learning},
  pages={353--364},
  year={2005},
  organization={Springer}
}

@article{spaan2005perseus,
  title={Perseus: Randomized point-based value iteration for POMDPs},
  author={Spaan, Matthijs TJ and Vlassis, Nikos},
  journal={Journal of artificial intelligence research},
  volume={24},
  pages={195--220},
  year={2005}
}

@article{deisenroth2012solving,
  title={Solving nonlinear continuous state-action-observation POMDPs for mechanical systems with Gaussian noise},
  author={Deisenroth, Marc Peter and Peters, Jan},
  year={2012}
}

@article{lee2020stochastic,
  title={Stochastic latent actor-critic: Deep reinforcement learning with a latent variable model},
  author={Lee, Alex X and Nagabandi, Anusha and Abbeel, Pieter and Levine, Sergey},
  journal={NeurIPS},
  year={2020}
}

@article{gregor2019shaping,
  title={Shaping belief states with generative environment models for rl},
  author={Gregor, Karol and Jimenez Rezende, Danilo and Besse, Frederic and Wu, Yan and Merzic, Hamza and van den Oord, Aaron},
  journal={NeurIPS},
  year={2019}
}

@inproceedings{zhang2019solar,
  title={Solar: Deep structured representations for model-based reinforcement learning},
  author={Zhang, Marvin and Vikram, Sharad and Smith, Laura and Abbeel, Pieter and Johnson, Matthew and Levine, Sergey},
  booktitle={ICML},
  year={2019},
}

@article{golowich2022planning,
  title={Planning in observable POMDPs in quasipolynomial time},
  author={Golowich, Noah and Moitra, Ankur and Rohatgi, Dhruv},
  journal={arXiv:2201.04735},
  year={2022}
}

@article{golowich2022learning,
  title={Learning in Observable POMDPs, without Computationally Intractable Oracles},
  author={Golowich, Noah and Moitra, Ankur and Rohatgi, Dhruv},
  journal={arXiv:2206.03446},
  year={2022}
}

@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={NeurIPS},
  volume={30},
  year={2017}
}

@article{devlin2018bert,
  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={ arXiv:1810.04805},
  year={2018}
}

@article{abbasi2011improved,
  title={Improved algorithms for linear stochastic bandits},
  author={Abbasi-Yadkori, Yasin and P{\'a}l, D{\'a}vid and Szepesv{\'a}ri, Csaba},
  journal={NeurIPS},
  year={2011}
}

@article{wang2022embed,
  title={Embed to Control Partially Observed Systems: Representation Learning with Provable Sample Efficiency},
  author={Wang, Lingxiao and Cai, Qi and Yang, Zhuoran and Wang, Zhaoran},
  journal={ arXiv:2205.13476},
  year={2022}
}

@inproceedings{anandkumar2012method,
  title={A method of moments for mixture models and hidden Markov models},
  author={Anandkumar, Animashree and Hsu, Daniel and Kakade, Sham M},
  booktitle={Conference on Learning Theory},
  pages={33--1},
  year={2012},
  organization={JMLR Workshop and Conference Proceedings}
}

@article{anandkumar2014tensor,
  title={Tensor decompositions for learning latent variable models},
  author={Anandkumar, Animashree and Ge, Rong and Hsu, Daniel and Kakade, Sham M and Telgarsky, Matus},
  journal={JMLR},
  volume={15},
  pages={2773--2832},
  year={2014},
  publisher={JMLR}
}

@article{venkatraman2017predictive,
  title={Predictive-state decoders: Encoding the future into recurrent networks},
  author={Venkatraman, Arun and Rhinehart, Nicholas and Sun, Wen and Pinto, Lerrel and Hebert, Martial and Boots, Byron and Kitani, Kris and Bagnell, J},
  journal={NeurIPS},
  volume={30},
  year={2017}
}

@article{agarwal2020flambe,
  title={Flambe: Structural complexity and representation learning of low rank mdps},
  author={Agarwal, Alekh and Kakade, Sham and Krishnamurthy, Akshay and Sun, Wen},
  journal={NeurIPS},
  year={2020}
}

@inproceedings{ren2022free,
  title={A free lunch from the noise: Provable and practical exploration for representation learning},
  author={Ren, Tongzheng and Zhang, Tianjun and Szepesv{\'a}ri, Csaba and Dai, Bo},
  booktitle={UAI},
  year={2022}
}

@inproceedings{zhang2022making,
  title={Making Linear MDPs Practical via Contrastive Representation Learning},
  author={Zhang, Tianjun and Ren, Tongzheng and Yang, Mengjiao and Gonzalez, Joseph and Schuurmans, Dale and Dai, Bo},
  booktitle={ICML},
  year={2022}
}



@article{uehara2022provably,
  title={Provably efficient reinforcement learning in partially observable dynamical systems},
  author={Uehara, Masatoshi and Sekhari, Ayush and Lee, Jason D and Kallus, Nathan and Sun, Wen},
  journal={ arXiv:2206.12020},
  year={2022}
}



@article{mnih2013playing,
  title={Playing atari with deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},
  journal={ arXiv:1312.5602},
  year={2013}
}

@article{berner2019dota,
  title={Dota 2 with large scale deep reinforcement learning},
  author={Berner, Christopher and Brockman, Greg and Chan, Brooke and Cheung, Vicki and Debiak, Przemyslaw and Dennison, Christy and Farhi, David and Fischer, Quirin and Hashme, Shariq and Hesse, Chris and others},
  journal={arXiv:1912.06680},
  year={2019}
}

@inproceedings{wierstra2007solving,
  title={Solving deep memory POMDPs with recurrent policy gradients},
  author={Wierstra, Daan and Foerster, Alexander and Peters, Jan and Schmidhuber, Juergen},
  booktitle={ICANN},
  year={2007}
}

@inproceedings{hausknecht2015deep,
  title={Deep recurrent q-learning for partially observable mdps},
  author={Hausknecht, Matthew and Stone, Peter},
  booktitle={AAAI fall symposium series},
  year={2015}
}

@article{zhan2022pac,
  title={PAC Reinforcement Learning for Predictive State Representations},
  author={Zhan, Wenhao and Uehara, Masatoshi and Sun, Wen and Lee, Jason D},
  journal={ arXiv:2207.05738},
  year={2022}
}

@inproceedings{even2007value,
  title={The Value of Observation for Monitoring Dynamic Systems.},
  author={Even-Dar, Eyal and Kakade, Sham M and Mansour, Yishay},
  booktitle={IJCAI},
  year={2007}
}

@article{ren2023spectral,
  title={Spectral Decomposition Representation for Reinforcement Learning},
  author={Ren, Tongzheng and Zhang, Tianjun and Lee, Lisa and Gonzalez, Joseph E and Schuurmans, Dale and Dai, Bo},
  journal={ICLR},
  year={2023}
}

@inproceedings{qiu2022contrastive,
  title={Contrastive UCB: Provably Efficient Contrastive Self-Supervised Learning in Online Reinforcement Learning},
  author={Qiu, Shuang and Wang, Lingxiao and Bai, Chenjia and Yang, Zhuoran and Wang, Zhaoran},
  booktitle={ICML},
  year={2022},
}

@article{rahimi2007random,
  title={Random features for large-scale kernel machines},
  author={Rahimi, Ali and Recht, Benjamin},
  journal={NeurIPS},
  volume={20},
  year={2007}
}

@article{arbel2020generalized,
  title={Generalized energy based models},
  author={Arbel, Michael and Zhou, Liang and Gretton, Arthur},
  journal={ICLR},
  year={2021}
}

@inproceedings{haarnoja2018soft,
  title={Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor},
  author={Haarnoja, Tuomas and Zhou, Aurick and Abbeel, Pieter and Levine, Sergey},
  booktitle={ICML},
  year={2018}
}

@article{zhu2017improving,
  title={On improving deep reinforcement learning for pomdps},
  author={Zhu, Pengfei and Li, Xin and Poupart, Pascal and Miao, Guanghui},
  journal={ arXiv:1704.07978},
  year={2017}
}


@article{liu2022partially,
  title={When Is Partially Observable Reinforcement Learning Not Scary?},
  author={Liu, Qinghua and Chung, Alan and Szepesv{\'a}ri, Csaba and Jin, Chi},
  journal={ arXiv:2204.08967},
  year={2022}
}


@article{karl2016deep,
  title={Deep variational bayes filters: Unsupervised learning of state space models from raw data},
  author={Karl, Maximilian and Soelch, Maximilian and Bayer, Justin and Van der Smagt, Patrick},
  journal={arXiv:1605.06432},
  year={2016}
}

@article{watter2015embed,
  title={Embed to control: A locally linear latent dynamics model for control from raw images},
  author={Watter, Manuel and Springenberg, Jost and Boedecker, Joschka and Riedmiller, Martin},
  journal={NeurIPS},
  year={2015}
}

@inproceedings{hafner2019learning,
  title={Learning latent dynamics for planning from pixels},
  author={Hafner, Danijar and Lillicrap, Timothy and Fischer, Ian and Villegas, Ruben and Ha, David and Lee, Honglak and Davidson, James},
  booktitle={ICML},
  year={2019}
}

@article{hafner2019dream,
  title={Dream to control: Learning behaviors by latent imagination},
  author={Hafner, Danijar and Lillicrap, Timothy and Ba, Jimmy and Norouzi, Mohammad},
  journal={arXiv:1912.01603},
  year={2019}
}

@article{hafner2020mastering,
  title={Mastering atari with discrete world models},
  author={Hafner, Danijar and Lillicrap, Timothy and Norouzi, Mohammad and Ba, Jimmy},
  journal={arXiv:2010.02193},
  year={2020}
}

@article{schmidhuber1990reinforcement,
  title={Reinforcement learning in Markovian and non-Markovian environments},
  author={Schmidhuber, J{\"u}rgen},
  journal={NeurIPS},
  volume={3},
  year={1990}
}

@article{bakker2001reinforcement,
  title={Reinforcement learning with long short-term memory},
  author={Bakker, Bram},
  journal={NeurIPS},
  year={2001}
}

@article{han2019variational,
  title={Variational recurrent models for solving partially observable control tasks},
  author={Han, Dongqi and Doya, Kenji and Tani, Jun},
  journal={ arXiv:1912.10703},
  year={2019}
}

@article{heess2015memory,
  title={Memory-based control with recurrent neural networks},
  author={Heess, Nicolas and Hunt, Jonathan J and Lillicrap, Timothy P and Silver, David},
  journal={arXiv:1512.04455},
  year={2015}
}

@article{ni2021recurrent,
  title={Recurrent model-free rl is a strong baseline for many pomdps},
  author={Ni, Tianwei and Eysenbach, Benjamin and Salakhutdinov, Ruslan},
  journal={ arXiv:2110.05038},
  year={2021}
}

@inproceedings{meng2021memory,
  title={Memory-based deep reinforcement learning for pomdps},
  author={Meng, Lingheng and Gorbet, Rob and Kuli{\'c}, Dana},
  booktitle={IROS},
  year={2021}
}

@inproceedings{laskin2020curl,
  title={Curl: Contrastive unsupervised representations for reinforcement learning},
  author={Laskin, Michael and Srinivas, Aravind and Abbeel, Pieter},
  booktitle={ICML},
  pages={5639--5650},
  year={2020},
  organization={PMLR}
}

@article{laskin2020reinforcement,
  title={Reinforcement learning with augmented data},
  author={Laskin, Misha and Lee, Kimin and Stooke, Adam and Pinto, Lerrel and Abbeel, Pieter and Srinivas, Aravind},
  journal={NeurIPS},
  volume={33},
  pages={19884--19895},
  year={2020}
}

@article{kostrikov2020image,
  title={Image augmentation is all you need: Regularizing deep reinforcement learning from pixels},
  author={Kostrikov, Ilya and Yarats, Denis and Fergus, Rob},
  journal={ICLR},
  year={2021}
}

@inproceedings{du2019provably,
  title={Provably efficient RL with rich observations via latent state decoding},
  author={Du, Simon and Krishnamurthy, Akshay and Jiang, Nan and Agarwal, Alekh and Dudik, Miroslav and Langford, John},
  booktitle={ICML},
  pages={1665--1674},
  year={2019},
  organization={PMLR}
}

@inproceedings{ferns2004metrics,
  title={Metrics for Finite Markov Decision Processes.},
  author={Ferns, Norm and Panangaden, Prakash and Precup, Doina},
  booktitle={UAI},
  year={2004}
}

@inproceedings{gelada2019deepmdp,
  title={Deepmdp: Learning continuous latent space models for representation learning},
  author={Gelada, Carles and Kumar, Saurabh and Buckman, Jacob and Nachum, Ofir and Bellemare, Marc G},
  booktitle={ICML},
  year={2019}
}

@article{zhang2020learning,
  title={Learning invariant representations for reinforcement learning without reconstruction},
  author={Zhang, Amy and McAllister, Rowan and Calandra, Roberto and Gal, Yarin and Levine, Sergey},
  journal={ arXiv:2006.10742},
  year={2020}
}

@article{dayan1993improving,
  title={Improving generalization for temporal difference learning: The successor representation},
  author={Dayan, Peter},
  journal={Neural computation},
  volume={5},
  number={4},
  pages={613--624},
  year={1993},
  publisher={MIT Press}
}

@article{barreto2017successor,
  title={Successor features for transfer in reinforcement learning},
  author={Barreto, Andr{\'e} and Dabney, Will and Munos, R{\'e}mi and Hunt, Jonathan J and Schaul, Tom and van Hasselt, Hado P and Silver, David},
  journal={NeurIPS},
  year={2017}
}

@article{kulkarni2016deep,
  title={Deep successor reinforcement learning},
  author={Kulkarni, Tejas D and Saeedi, Ardavan and Gautam, Simanta and Gershman, Samuel J},
  journal={arXiv:1606.02396},
  year={2016}
}

@article{mahadevan2007proto,
  title={Proto-value Functions: A Laplacian Framework for Learning Representation and Control in Markov Decision Processes.},
  author={Mahadevan, Sridhar and Maggioni, Mauro},
  journal={JMLR},
  volume={8},
  number={10},
  year={2007}
}

@article{wu2018laplacian,
  title={The laplacian in rl: Learning representations with efficient approximations},
  author={Wu, Yifan and Tucker, George and Nachum, Ofir},
  journal={arXiv:1810.04586},
  year={2018}
}

@article{duan2019state,
  title={State aggregation learning from markov transition data},
  author={Duan, Yaqi and Ke, Tracy and Wang, Mengdi},
  journal={NeurIPS},
  year={2019}
}

@article{oord2018representation,
  title={Representation learning with contrastive predictive coding},
  author={Oord, Aaron van den and Li, Yazhe and Vinyals, Oriol},
  journal={ arXiv:1807.03748},
  year={2018}
}

@article{nachum2021provable,
  title={Provable representation learning for imitation with contrastive fourier features},
  author={Nachum, Ofir and Yang, Mengjiao},
  journal={NeurIPS},
  year={2021}
}


@article{yang2021trail,
  title={Trail: Near-optimal imitation learning with suboptimal data},
  author={Yang, Mengjiao and Levine, Sergey and Nachum, Ofir},
  journal={ arXiv:2110.14770},
  year={2021}
}

@inproceedings{stooke2021decoupling,
  title={Decoupling representation learning from reinforcement learning},
  author={Stooke, Adam and Lee, Kimin and Abbeel, Pieter and Laskin, Michael},
  booktitle={ICML},
  pages={9870--9879},
  year={2021},
  organization={PMLR}
}

@article{schwarzer2020data,
  title={Data-efficient reinforcement learning with self-predictive representations},
  author={Schwarzer, Max and Anand, Ankesh and Goel, Rishab and Hjelm, R Devon and Courville, Aaron and Bachman, Philip},
  journal={ arXiv:2007.05929},
  year={2020}
}

@article{mazoure2020deep,
  title={Deep reinforcement and infomax learning},
  author={Mazoure, Bogdan and Tachet des Combes, Remi and Doan, Thang Long and Bachman, Philip and Hjelm, R Devon},
  journal={NeurIPS},
  year={2020}
}

@article{brockman2016openai,
  title={Openai gym},
  author={Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, John and Tang, Jie and Zaremba, Wojciech},
  journal={arXiv:1606.01540},
  year={2016}
}

@article{tassa2018deepmind,
  title={Deepmind control suite},
  author={Tassa, Yuval and Doron, Yotam and Muldal, Alistair and Erez, Tom and Li, Yazhe and Casas, Diego de Las and Budden, David and Abdolmaleki, Abbas and Merel, Josh and Lefrancq, Andrew and others},
  journal={arXiv:1801.00690},
  year={2018}
}

@article{chua2018deep,
  title={Deep reinforcement learning in a handful of trials using probabilistic dynamics models},
  author={Chua, Kurtland and Calandra, Roberto and McAllister, Rowan and Levine, Sergey},
  journal={NeurIPS},
  volume={31},
  year={2018}
}

@article{kurutach2018model,
  title={Model-ensemble trust-region policy optimization},
  author={Kurutach, Thanard and Clavera, Ignasi and Duan, Yan and Tamar, Aviv and Abbeel, Pieter},
  journal={arXiv:1802.10592},
  year={2018}
}

@inproceedings{schulman2015trust,
  title={Trust region policy optimization},
  author={Schulman, John and Levine, Sergey and Abbeel, Pieter and Jordan, Michael and Moritz, Philipp},
  booktitle={ICML},
  year={2015}
}

@article{schulman2017proximal,
  title={Proximal policy optimization algorithms},
  author={Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
  journal={ arXiv:1707.06347},
  year={2017}
}

@article{wang2019benchmarking,
  title={Benchmarking model-based reinforcement learning},
  author={Wang, Tingwu and Bao, Xuchan and Clavera, Ignasi and Hoang, Jerrick and Wen, Yeming and Langlois, Eric and Zhang, Shunshi and Zhang, Guodong and Abbeel, Pieter and Ba, Jimmy},
  journal={ arXiv:1907.02057},
  year={2019}
}

@inproceedings{li2004iterative,
  title={Iterative linear quadratic regulator design for nonlinear biological movement systems.},
  author={Li, Weiwei and Todorov, Emanuel},
  booktitle={ICINCO (1)},
  pages={222--229},
  year={2004},
  organization={Citeseer}
}

@inproceedings{yu2020meta,
  title={Meta-world: A benchmark and evaluation for multi-task and meta reinforcement learning},
  author={Yu, Tianhe and Quillen, Deirdre and He, Zhanpeng and Julian, Ryan and Hausman, Karol and Finn, Chelsea and Levine, Sergey},
  booktitle={CORL},
  year={2020}
}

@inproceedings{yarats2021improving,
  title={Improving sample efficiency in model-free reinforcement learning from images},
  author={Yarats, Denis and Zhang, Amy and Kostrikov, Ilya and Amos, Brandon and Pineau, Joelle and Fergus, Rob},
  booktitle={AAAI},
  year={2021}
}

@article{song2021train,
  title={How to train your energy-based models},
  author={Song, Yang and Kingma, Diederik P},
  journal={ arXiv:2101.03288},
  year={2021}
}



@inproceedings{gangwani2020learning,
  title={Learning belief representations for imitation learning in pomdps},
  author={Gangwani, Tanmay and Lehman, Joel and Liu, Qiang and Peng, Jian},
  booktitle={UAI},
  year={2020},
}

@article{weigand2021reinforcement,
  title={Reinforcement Learning using Guided Observability},
  author={Weigand, Stephan and Klink, Pascal and Peters, Jan and Pajarinen, Joni},
  journal={arXiv:2104.10986},
  year={2021}
}