@book{Sutton2018,
	author = {Sutton, Richard S. and Barto, Andrew G.},
	title = {Reinforcement Learning: An Introduction},
	year = {2018},
	isbn = {978-0-26203924-6},
	publisher = {The MIT Press},
	url={https://mitpress.mit.edu/books/reinforcement-learning-second-edition},
}

@inproceedings{
Zhu2020The,
title={The Ingredients of Real World Robotic Reinforcement Learning},
author={Henry Zhu and Justin Yu and Abhishek Gupta and Dhruv Shah and Kristian Hartikainen and Avi Singh and Vikash Kumar and Sergey Levine},
booktitle={International Conference on Learning Representations},
year={2020},
url={https://openreview.net/forum?id=rJe2syrtvS}
}

@article{Levine2018May,
	author = {Levine, Sergey},
	title = {{Reinforcement Learning and Control as Probabilistic Inference: Tutorial and Review}},
	journal = {arXiv},
	year = {2018},
	month = {May},
	eprint = {1805.00909},
	url = {https://arxiv.org/abs/1805.00909v3}
}


@article{Silver2018Dec,
	author = {Silver, David and Hubert, Thomas and Schrittwieser, Julian and Antonoglou, Ioannis and Lai, Matthew and Guez, Arthur and Lanctot, Marc and Sifre, Laurent and Kumaran, Dharshan and Graepel, Thore and Lillicrap, Timothy and Simonyan, Karen and Hassabis, Demis},
	title = {{A general reinforcement learning algorithm that masters chess, shogi, and Go through self-play}},
	journal = {Science},
	volume = {362},
	number = {6419},
	pages = {1140--1144},
	year = {2018},
	month = {Dec},
	issn = {0036-8075},
	publisher = {American Association for the Advancement of Science},
	doi = {10.1126/science.aar6404}
}

@article{Schrittwieser2020Dec,
	author = {Schrittwieser, Julian and Antonoglou, Ioannis and Hubert, Thomas and Simonyan, Karen and Sifre, Laurent and Schmitt, Simon and Guez, Arthur and Lockhart, Edward and Hassabis, Demis and Graepel, Thore and Lillicrap, Timothy and Silver, David},
	title = {{Mastering Atari, Go, chess and shogi by planning with a learned model}},
	journal = {Nature},
	volume = {588},
	number = {7839},
	pages = {604--609},
	year = {2020},
	month = {Dec},
	issn = {1476-4687},
	publisher = {Nature Publishing Group},
	doi = {10.1038/s41586-020-03051-4}
}	
	
@incollection{Haarnoja2017Jul,
author = {Haarnoja, Tuomas and Tang, Haoran and Abbeel, Pieter and Levine, Sergey},
	title = {{Reinforcement Learning with Deep Energy-Based Policies}},
	booktitle = {{International Conference on Machine Learning}},
	journal = {PMLR},
	pages = {1352--1361},
	year = {2017},
	month = {Jul},
	issn = {2640-3498},
	publisher = {PMLR},
	url = {http://proceedings.mlr.press/v70/haarnoja17a.html}
}


@book{kardar_2007, place={Cambridge}, title={Statistical Physics of Particles}, DOI={10.1017/CBO9780511815898}, publisher={Cambridge University Press}, author={Kardar, Mehran}, year={2007}}

%%refs for relevant works
@article{friston2009free,
  title={The free-energy principle: a rough guide to the brain?},
  author={Friston, Karl},
  journal={Trends in cognitive sciences},
  volume={13},
  number={7},
  pages={293--301},
  year={2009},
  publisher={Elsevier}
}

@article{friston2006free,
  title={A free energy principle for the brain},
  author={Friston, Karl and Kilner, James and Harrison, Lee},
  journal={Journal of physiology-Paris},
  volume={100},
  number={1-3},
  pages={70--87},
  year={2006},
  publisher={Elsevier}
}

@incollection{tishby2011information,
  title={Information theory of decisions and actions},
  author={Tishby, Naftali and Polani, Daniel},
  booktitle={Perception-action cycle},
  pages={601--636},
  year={2011},
  publisher={Springer}
}

@article{tiomkin2017unified,
  title={A unified bellman equation for causal information and value in markov decision processes},
  author={Tiomkin, Stas and Tishby, Naftali},
  journal={arXiv preprint arXiv:1703.01585},
  year={2017}
}

%%%%% max rl
@article{haarnoja2018soft,
  title={Soft actor-critic algorithms and applications},
  author={Haarnoja, Tuomas and Zhou, Aurick and Hartikainen, Kristian and Tucker, George and Ha, Sehoon and Tan, Jie and Kumar, Vikash and Zhu, Henry and Gupta, Abhishek and Abbeel, Pieter and others},
  journal={arXiv preprint arXiv:1812.05905},
  year={2018}
}
@article{nachum2017bridging,
  title={Bridging the gap between value and policy based reinforcement learning},
  author={Nachum, Ofir and Norouzi, Mohammad and Xu, Kelvin and Schuurmans, Dale},
  journal={arXiv preprint arXiv:1702.08892},
  year={2017}
}

@InProceedings{pmlr-v80-haarnoja18b,
  title = 	 {Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor},
  author =       {Haarnoja, Tuomas and Zhou, Aurick and Abbeel, Pieter and Levine, Sergey},
  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
  pages = 	 {1861--1870},
  year = 	 {2018},
  editor = 	 {Dy, Jennifer and Krause, Andreas},
  volume = 	 {80},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {10--15 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v80/haarnoja18b/haarnoja18b.pdf},
  url = 	 {https://proceedings.mlr.press/v80/haarnoja18b.html},
  abstract = 	 {Model-free deep reinforcement learning (RL) algorithms have been demonstrated on a range of challenging decision making and control tasks. However, these methods typically suffer from two major challenges: very high sample complexity and brittle convergence properties, which necessitate meticulous hyperparameter tuning. Both of these challenges severely limit the applicability of such methods to complex, real-world domains. In this paper, we propose soft actor-critic, an off-policy actor-critic deep RL algorithm based on the maximum entropy reinforcement learning framework. In this framework, the actor aims to maximize expected reward while also maximizing entropy. That is, to succeed at the task while acting as randomly as possible. Prior deep RL methods based on this framework have been formulated as Q-learning methods. By combining off-policy updates with a stable stochastic actor-critic formulation, our method achieves state-of-the-art performance on a range of continuous control benchmark tasks, outperforming prior on-policy and off-policy methods. Furthermore, we demonstrate that, in contrast to other off-policy algorithms, our approach is very stable, achieving very similar performance across different random seeds.}
}


% duality between optimal control and estimation
@article{mitter2000duality,
  title={The duality between estimation and control},
  author={Mitter, Sanjoy K and Newton, NJ},
  journal={Published in Festschrift for A. Bennoussan},
  year={2000}
}

@inproceedings{todorov2008general,
  title={General duality between optimal control and estimation},
  author={Todorov, Emanuel},
  booktitle={2008 47th IEEE Conference on Decision and Control},
  pages={4286--4292},
  year={2008},
  organization={IEEE}
}


% robotics 
@inproceedings{toussaint2009robot,
  title={Robot trajectory optimization using approximate inference},
  author={Toussaint, Marc},
  booktitle={Proceedings of the 26th annual international conference on machine learning},
  pages={1049--1056},
  year={2009}
}

%%% https://dl.acm.org/doi/10.1145/3292500.3330975


@article{eysenbach2021maximum,
	author = {Eysenbach, Benjamin and Levine, Sergey},
	title = {{Maximum Entropy RL (Provably) Solves Some Robust RL Problems}},
	journal = {arXiv},
	year = {2021},
	month = {Mar},
	eprint = {2103.06257},
	url = {https://arxiv.org/abs/2103.06257v1}
}

@article{rose2021reinforcement,
  title={A reinforcement learning approach to rare trajectory sampling},
  author={Rose, Dominic C and Mair, Jamie F and Garrahan, Juan P},
  journal={New Journal of Physics},
  volume={23},
  number={1},
  pages={013013},
  year={2021},
  publisher={IOP Publishing}
}

@article{das2021reinforcement,
  title={Reinforcement learning of rare diffusive dynamics},
  author={Das, Avishek and Rose, Dominic C and Garrahan, Juan P and Limmer, David T},
  journal={arXiv preprint arXiv:2105.04321},
  year={2021}
}

@article{arriojas2021closed,
  title = {Entropy regularized reinforcement learning using large deviation theory},
  author = {Arriojas, Argenis and Adamczyk, Jacob and Tiomkin, Stas and Kulkarni, Rahul V.},
  journal = {Phys. Rev. Res.},
  volume = {5},
  issue = {2},
  pages = {023085},
  numpages = {11},
  year = {2023},
  month = {May},
  publisher = {American Physical Society},
  doi = {10.1103/PhysRevResearch.5.023085},
  url = {https://link.aps.org/doi/10.1103/PhysRevResearch.5.023085}
}


@article{rawlik2012stochastic,
  title={On stochastic optimal control and reinforcement learning by approximate inference},
  author={Rawlik, Konrad and Toussaint, Marc and Vijayakumar, Sethu},
  journal={Proceedings of Robotics: Science and Systems VIII},
  year={2012}
}

@article{Kappen2012May,
	author = {Kappen, Hilbert J. and G{\ifmmode\acute{o}\else\'{o}\fi}mez, Vicen{\ifmmode\mbox{\c{c}}\else\c{c}\fi} and Opper, Manfred},
	title = {{Optimal control as a graphical model inference problem}},
	journal = {Mach. Learn.},
	volume = {87},
	number = {2},
	pages = {159--182},
	year = {2012},
	month = {May},
	issn = {1573-0565},
	publisher = {Springer US},
	doi = {10.1007/s10994-012-5278-7}
}


@inproceedings{theodorou2012relative,
  title={Relative entropy and free energy dualities: Connections to path integral and kl control},
  author={Theodorou, Evangelos A and Todorov, Emanuel},
  booktitle={2012 ieee 51st ieee conference on decision and control (cdc)},
  pages={1466--1473},
  year={2012},
  organization={IEEE}
}

@book{puterman2014markov,
  title={Markov decision processes: discrete stochastic dynamic programming},
  author={Puterman, Martin L},
  year={2014},
  publisher={John Wiley \& Sons}
}

@article{zhu2020ingredients,
  title={The ingredients of real-world robotic reinforcement learning},
  author={Zhu, Henry and Yu, Justin and Gupta, Abhishek and Shah, Dhruv and Hartikainen, Kristian and Singh, Avi and Kumar, Vikash and Levine, Sergey},
  journal={arXiv preprint arXiv:2004.12570},
  year={2020}
}

@ARTICLE{9397429,  author={Cao, Zhong and Xu, Shaobing and Peng, Huei and Yang, Diange and Zidek, Robert},  journal={IEEE Transactions on Intelligent Transportation Systems},   title={Confidence-Aware Reinforcement Learning for Self-Driving Cars},   year={2021},  volume={},  number={},  pages={1-12},  doi={10.1109/TITS.2021.3069497}}

@article{yu2019reinforcement,
  title={Reinforcement learning in healthcare: A survey},
  author={Yu, Chao and Liu, Jiming and Nemati, Shamim},
  journal={arXiv preprint arXiv:1908.08796},
  year={2019}
}

@article{charpentier2021reinforcement,
  title={Reinforcement learning in economics and finance},
  author={Charpentier, Arthur and Elie, Romuald and Remlinger, Carl},
  journal={Computational Economics},
  pages={1--38},
  year={2021},
  publisher={Springer}
}

@article{levine2020offline,
  title={Offline reinforcement learning: Tutorial, review, and perspectives on open problems},
  author={Levine, Sergey and Kumar, Aviral and Tucker, George and Fu, Justin},
  journal={arXiv preprint arXiv:2005.01643},
  year={2020}
}

@article{kumar2020conservative,
  title={Conservative q-learning for offline reinforcement learning},
  author={Kumar, Aviral and Zhou, Aurick and Tucker, George and Levine, Sergey},
  journal={arXiv preprint arXiv:2006.04779},
  year={2020}
}

@article{todorov2009efficient,
  title={Efficient computation of optimal actions},
  author={Todorov, Emanuel},
  journal={Proceedings of the national academy of sciences},
  volume={106},
  number={28},
  pages={11478--11483},
  year={2009},
  publisher={National Acad Sciences}
}

@inproceedings{ng1999policy,
  title={Policy invariance under reward transformations: Theory and application to reward shaping},
  author={Ng, Andrew Y and Harada, Daishi and Russell, Stuart},
  booktitle={Icml},
  volume={99},
  pages={278--287},
  year={1999}
}

@inproceedings{NIPS1995_8f1d4362,
 author = {Sutton, Richard S},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {D. Touretzky and M. C. Mozer and M. Hasselmo},
 pages = {},
 publisher = {MIT Press},
 title = {Generalization in Reinforcement Learning: Successful Examples Using Sparse Coarse Coding},
 url = {https://proceedings.neurips.cc/paper/1995/file/8f1d43620bc6bb580df6e80b0dc05c48-Paper.pdf},
 volume = {8},
 year = {1996}
}

@article{JMLR:v16:geramifard15a,
  author  = {Alborz Geramifard and Christoph Dann and Robert H. Klein and William Dabney and Jonathan P. How},
  title   = {RLPy: A Value-Function-Based Reinforcement Learning Framework for Education and Research},
  journal = {Journal of Machine Learning Research},
  year    = {2015},
  volume  = {16},
  number  = {46},
  pages   = {1573-1578},
  url     = {http://jmlr.org/papers/v16/geramifard15a.html}
}

@article{brockman2016openai,
  title={Openai gym},
  author={Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, John and Tang, Jie and Zaremba, Wojciech},
  journal={arXiv preprint arXiv:1606.01540},
  year={2016}
}

@ARTICLE{6313077,
  author={Barto, Andrew G. and Sutton, Richard S. and Anderson, Charles W.},
  journal={IEEE Transactions on Systems, Man, and Cybernetics}, 
  title={Neuronlike adaptive elements that can solve difficult learning control problems}, 
  year={1983},
  volume={SMC-13},
  number={5},
  pages={834-846},
  doi={10.1109/TSMC.1983.6313077}}

@phdthesis{MooreAW1990,
author = {Moore, Andrew W},
title = {Efficient Memory-Based Learning for Robot Control},
publisher = {University of Cambridge Computer Laboratory},
school = {University of Cambridge},
year = {1990}
}


@inproceedings{NIPS2004_02f657d5,
 author = {Abbeel, Pieter and Ng, Andrew},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {L. Saul and Y. Weiss and L. Bottou},
 pages = {},
 publisher = {MIT Press},
 title = {Learning first-order Markov models for control},
 url = {https://proceedings.neurips.cc/paper/2004/file/02f657d55eaf1c4840ce8d66fcdaf90c-Paper.pdf},
 volume = {17},
 year = {2005}
}

@misc{asadi2018simple,
      title={Towards a Simple Approach to Multi-step Model-based Reinforcement Learning}, 
      author={Kavosh Asadi and Evan Cater and Dipendra Misra and Michael L. Littman},
      year={2018},
      eprint={1811.00128},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@INPROCEEDINGS{606886,  author={Atkeson, C.G. and Santamaria, J.C.},  booktitle={Proceedings of International Conference on Robotics and Automation},   title={A comparison of direct and model-based reinforcement learning},   year={1997},  volume={4},  number={},  pages={3557-3564 vol.4},  doi={10.1109/ROBOT.1997.606886}}

@inproceedings{10.5555/3294771.3294858,
author = {Berkenkamp, Felix and Turchetta, Matteo and Schoellig, Angela P. and Krause, Andreas},
title = {Safe Model-Based Reinforcement Learning with Stability Guarantees},
year = {2017},
isbn = {9781510860964},
publisher = {Curran Associates Inc.},
address = {Red Hook, NY, USA},
abstract = {Reinforcement learning is a powerful paradigm for learning optimal policies from experimental data. However, to find optimal policies, most reinforcement learning algorithms explore all possible actions, which may be harmful for real-world systems. As a consequence, learning algorithms are rarely applied on safety-critical systems in the real world. In this paper, we present a learning algorithm that explicitly considers safety, defined in terms of stability guarantees. Specifically, we extend control-theoretic results on Lyapunov stability verification and show how to use statistical models of the dynamics to obtain high-performance control policies with provable stability certificates. Moreover, under additional regularity assumptions in terms of a Gaussian process prior, we prove that one can effectively and safely collect data in order to learn about the dynamics and thus both improve control performance and expand the safe region of the state space. In our experiments, we show how the resulting algorithm can safely optimize a neural network policy on a simulated inverted pendulum, without the pendulum ever falling down.},
booktitle = {Proceedings of the 31st International Conference on Neural Information Processing Systems},
pages = {908–919},
numpages = {12},
location = {Long Beach, California, USA},
series = {NIPS'17}
}

@INPROCEEDINGS{620043,  author={Boone, G.},  booktitle={Proceedings of International Conference on Robotics and Automation},   title={Efficient reinforcement learning: model-based Acrobot control},   year={1997},  volume={1},  number={},  pages={229-234 vol.1},  doi={10.1109/ROBOT.1997.620043}}

@InProceedings{pmlr-v80-corneil18a,
  title = 	 {Efficient Model-Based Deep Reinforcement Learning with Variational State Tabulation},
  author =       {Corneil, Dane and Gerstner, Wulfram and Brea, Johanni},
  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
  pages = 	 {1049--1058},
  year = 	 {2018},
  editor = 	 {Dy, Jennifer and Krause, Andreas},
  volume = 	 {80},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {10--15 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v80/corneil18a/corneil18a.pdf},
  url = 	 {https://proceedings.mlr.press/v80/corneil18a.html},
  abstract = 	 {Modern reinforcement learning algorithms reach super-human performance on many board and video games, but they are sample inefficient, i.e. they typically require significantly more playing experience than humans to reach an equal performance level. To improve sample efficiency, an agent may build a model of the environment and use planning methods to update its policy. In this article we introduce Variational State Tabulation (VaST), which maps an environment with a high-dimensional state space (e.g. the space of visual inputs) to an abstract tabular model. Prioritized sweeping with small backups, a highly efficient planning method, can then be used to update state-action values. We show how VaST can rapidly learn to maximize reward in tasks like 3D navigation and efficiently adapt to sudden changes in rewards or transition probabilities.}
}

@inproceedings{
lowrey2018plan,
title={Plan Online, Learn Offline: Efficient Learning and Exploration via Model-Based Control},
author={Kendall Lowrey and Aravind Rajeswaran and Sham Kakade and Emanuel Todorov and Igor Mordatch},
booktitle={International Conference on Learning Representations},
year={2019},
url={https://openreview.net/forum?id=Byey7n05FQ},
}

@article{Mnih2015Feb,
	author = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A. and Veness, Joel and Bellemare, Marc G. and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K. and Ostrovski, Georg and Petersen, Stig and Beattie, Charles and Sadik, Amir and Antonoglou, Ioannis and King, Helen and Kumaran, Dharshan and Wierstra, Daan and Legg, Shane and Hassabis, Demis},
	title = {{Human-level control through deep reinforcement learning}},
	journal = {Nature},
	volume = {518},
	number = {7540},
	pages = {529--533},
	year = {2015},
	month = {Feb},
	issn = {1476-4687},
	publisher = {Nature Publishing Group},
	doi = {10.1038/nature14236}
}
@article{Watkins1992May,
	author = {Watkins, Christopher J. C. H. and Dayan, Peter},
	title = {{Q-learning}},
	journal = {Mach. Learn.},
	volume = {8},
	number = {3},
	pages = {279--292},
	year = {1992},
	month = {May},
	issn = {1573-0565},
	publisher = {Kluwer Academic Publishers},
	doi = {10.1007/BF00992698}
}

@inproceedings{NIPS2010_091d584f,
 author = {Hasselt, Hado},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {J. Lafferty and C. Williams and J. Shawe-Taylor and R. Zemel and A. Culotta},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Double Q-learning},
 url = {https://proceedings.neurips.cc/paper/2010/file/091d584fced301b442654dd8c23b3fc9-Paper.pdf},
 volume = {23},
 year = {2010}
}

@ARTICLE{8169685,  author={Kiumarsi, Bahare and Vamvoudakis, Kyriakos G. and Modares, Hamidreza and Lewis, Frank L.},  journal={IEEE Transactions on Neural Networks and Learning Systems},   title={Optimal and Autonomous Control Using Reinforcement Learning: A Survey},   year={2018},  volume={29},  number={6},  pages={2042-2062},  doi={10.1109/TNNLS.2017.2773458}}

%% the following two are the ones suggested by reviewer 3 in AAAI

@InProceedings{pmlr-v97-hazan19a,
  title = 	 {Provably Efficient Maximum Entropy Exploration},
  author =       {Hazan, Elad and Kakade, Sham and Singh, Karan and Van Soest, Abby},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {2681--2691},
  year = 	 {2019},
  editor = 	 {Chaudhuri, Kamalika and Salakhutdinov, Ruslan},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {09--15 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/hazan19a/hazan19a.pdf},
  url = 	 {https://proceedings.mlr.press/v97/hazan19a.html},
  abstract = 	 {Suppose an agent is in a (possibly unknown) Markov Decision Process in the absence of a reward signal, what might we hope that an agent can efficiently learn to do? This work studies a broad class of objectives that are defined solely as functions of the state-visitation frequencies that are induced by how the agent behaves. For example, one natural, intrinsically defined, objective problem is for the agent to learn a policy which induces a distribution over state space that is as uniform as possible, which can be measured in an entropic sense. We provide an efficient algorithm to optimize such such intrinsically defined objectives, when given access to a black box planning oracle (which is robust to function approximation). Furthermore, when restricted to the tabular setting where we have sample based access to the MDP, our proposed algorithm is provably efficient, both in terms of its sample and computational complexities. Key to our algorithmic methodology is utilizing the conditional gradient method (a.k.a. the Frank-Wolfe algorithm) which utilizes an approximate MDP solver.}
}

@inproceedings{NEURIPS2020_30ee748d,
 author = {Zhang, Junyu and Koppel, Alec and Bedi, Amrit Singh and Szepesvari, Csaba and Wang, Mengdi},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 pages = {4572--4583},
 publisher = {Curran Associates, Inc.},
 title = {Variational Policy Gradient Method for Reinforcement Learning with General Utilities},
 url = {https://proceedings.neurips.cc/paper/2020/file/30ee748d38e21392de740e2f9dc686b6-Paper.pdf},
 volume = {33},
 year = {2020}
}

@book{koller2009probabilistic,
  title={Probabilistic graphical models: principles and techniques},
  author={Koller, Daphne and Friedman, Nir},
  year={2009},
  publisher={MIT press}
}


@InProceedings{levine2013Jun,
  title = 	 {Guided Policy Search},
  author = 	 {Levine, Sergey and Koltun, Vladlen},
  booktitle = 	 {Proceedings of the 30th International Conference on Machine Learning},
  pages = 	 {1--9},
  year = 	 {2013},
  editor = 	 {Dasgupta, Sanjoy and McAllester, David},
  volume = 	 {28},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Atlanta, Georgia, USA},
  month = 	 {17--19 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v28/levine13.pdf},
  url = 	 {https://proceedings.mlr.press/v28/levine13.html}
}


@inproceedings{levine2013Dec,
 author = {Levine, Sergey and Koltun, Vladlen},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {C.J. Burges and L. Bottou and M. Welling and Z. Ghahramani and K.Q. Weinberger},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Variational Policy Search via Trajectory Optimization},
 url = {https://proceedings.neurips.cc/paper/2013/file/38af86134b65d0f10fe33d30dd76442e-Paper.pdf},
 volume = {26},
 year = {2013}
}


@inproceedings{Ziebart2010Jun,
  author={Brian D. Ziebart and J. Andrew Bagnell and Anind K. Dey},
  title={Modeling Interaction via the Principle of Maximum Causal Entropy},
  year={2010},
  cdate={1262304000000},
  pages={1255-1262},
  url={https://icml.cc/Conferences/2010/papers/28.pdf},
  booktitle={ICML}
}

@article{watson2021stochastic,
  author    = {Joe Watson and
               Hany Abdulsamad and
               Rolf Findeisen and
               Jan Peters},
  title     = {Stochastic Control through Approximate Bayesian Input Inference},
  journal   = {CoRR},
  volume    = {abs/2105.07693},
  year      = {2021},
  url       = {https://arxiv.org/abs/2105.07693},
  eprinttype = {arXiv},
  eprint    = {2105.07693},
  timestamp = {Tue, 18 May 2021 18:46:40 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/abs-2105-07693.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{peters2010Jul,
author = {Peters, Jan and Mülling, Katharina and Altün, Yasemin},
title = {Relative Entropy Policy Search},
year = {2010},
publisher = {AAAI Press},
booktitle = {Proceedings of the Twenty-Fourth AAAI Conference on Artificial Intelligence},
pages = {1607–1612},
numpages = {6},
location = {Atlanta, Georgia},
series = {AAAI'10}
}

@article{kalman1960Mar,
    author = {Kalman, R. E.},
    title = "{A New Approach to Linear Filtering and Prediction Problems}",
    journal = {Journal of Basic Engineering},
    volume = {82},
    number = {1},
    pages = {35-45},
    year = {1960},
    month = {03},
    abstract = "{The classical filtering and prediction problem is re-examined using the Bode-Shannon representation of random processes and the “state-transition” method of analysis of dynamic systems. New results are: (1) The formulation and methods of solution of the problem apply without modification to stationary and nonstationary statistics and to growing-memory and infinite-memory filters. (2) A nonlinear difference (or differential) equation is derived for the covariance matrix of the optimal estimation error. From the solution of this equation the co-efficients of the difference (or differential) equation of the optimal linear filter are obtained without further calculations. (3) The filtering problem is shown to be the dual of the noise-free regulator problem. The new method developed here is applied to two well-known problems, confirming and extending earlier results. The discussion is largely self-contained and proceeds from first principles; basic concepts of the theory of random processes are reviewed in the Appendix.}",
    issn = {0021-9223},
    doi = {10.1115/1.3662552},
    url = {https://doi.org/10.1115/1.3662552},
    eprint = {https://asmedigitalcollection.asme.org/fluidsengineering/article-pdf/82/1/35/5518977/35\_1.pdf},
}

@article{kalman1960contributions,
    author = {Kalman, R. E.},
    title = "{Contributions to the Theory of Optimal Control}",
    journal = {Boletin de la Sociedad Matematica Mexicana},
    volume = {5},
    pages = {102-119},
    year = {1960},
    url = {https://www.ee.iitb.ac.in/~belur/ee640/optimal-classic-paper.pdf}
}


@InProceedings{Attias2003Jan,
  title = 	 {Planning by Probabilistic Inference},
  author =       {Attias, Hagai},
  booktitle = 	 {Proceedings of the Ninth International Workshop on Artificial Intelligence and Statistics},
  pages = 	 {9--16},
  year = 	 {2003},
  editor = 	 {Bishop, Christopher M. and Frey, Brendan J.},
  volume = 	 {R4},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {03--06 Jan},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/r4/attias03a/attias03a.pdf},
  url = 	 {https://proceedings.mlr.press/r4/attias03a.html},
  note =         {Reissued by PMLR on 01 April 2021.}
}


@inproceedings{Toussaint2006Jun,
author = {Toussaint, Marc and Storkey, Amos},
title = {Probabilistic Inference for Solving Discrete and Continuous State Markov Decision Processes},
year = {2006},
isbn = {1595933832},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/1143844.1143963},
doi = {10.1145/1143844.1143963},
booktitle = {Proceedings of the 23rd International Conference on Machine Learning},
pages = {945–952},
numpages = {8},
location = {Pittsburgh, Pennsylvania, USA},
series = {ICML '06}
}

@inproceedings{Todorov2006,
 author = {Todorov, Emanuel},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {B. Sch\"{o}lkopf and J. Platt and T. Hoffman},
 pages = {},
 publisher = {MIT Press},
 title = {Linearly-solvable Markov decision problems},
 url = {https://proceedings.neurips.cc/paper/2006/file/d806ca13ca3449af72a1ea5aedbed26a-Paper.pdf},
 volume = {19},
 year = {2006}
}


@InProceedings{Furmston2010May,
  title = 	 {Variational methods for Reinforcement Learning},
  author = 	 {Furmston, Thomas and Barber, David},
  booktitle = 	 {Proceedings of the Thirteenth International Conference on Artificial Intelligence and Statistics},
  pages = 	 {241--248},
  year = 	 {2010},
  editor = 	 {Teh, Yee Whye and Titterington, Mike},
  volume = 	 {9},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Chia Laguna Resort, Sardinia, Italy},
  month = 	 {13--15 May},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v9/furmston10a/furmston10a.pdf},
  url = 	 {https://proceedings.mlr.press/v9/furmston10a.html}
}

@article{Furmston2012,
  author    = {Thomas Furmston and
               David Barber},
  title     = {Efficient Inference in Markov Control Problems},
  journal   = {CoRR},
  volume    = {abs/1202.3720},
  year      = {2012},
  url       = {http://arxiv.org/abs/1202.3720},
  eprinttype = {arXiv},
  eprint    = {1202.3720},
  timestamp = {Mon, 13 Aug 2018 16:48:16 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1202-3720.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@INPROCEEDINGS{Lioutikov2014Sep,
  author={Lioutikov, Rudolf and Paraschos, Alexandros and Peters, Jan and Neumann, Gerhard},
  booktitle={2014 IEEE International Conference on Robotics and Automation (ICRA)}, 
  title={Sample-based informationl-theoretic stochastic optimal control}, 
  year={2014},
  month={sep},
  volume={},
  number={},
  pages={3896-3902},
  doi={10.1109/ICRA.2014.6907424}
}

@article{Fox2015Dec,
  author    = {Roy Fox and
               Ari Pakman and
               Naftali Tishby},
  title     = {G-Learning: Taming the Noise in Reinforcement Learning via Soft Updates},
  journal   = {CoRR},
  volume    = {abs/1512.08562},
  year      = {2015},
  url       = {http://arxiv.org/abs/1512.08562},
  eprinttype = {arXiv},
  eprint    = {1512.08562},
  timestamp = {Mon, 13 Aug 2018 16:49:08 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/FoxPT15.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@InProceedings{Akrour2016Jun,
  title = 	 {Model-Free Trajectory Optimization for Reinforcement Learning},
  author = 	 {Akrour, Riad and Neumann, Gerhard and Abdulsamad, Hany and Abdolmaleki, Abbas},
  booktitle = 	 {Proceedings of The 33rd International Conference on Machine Learning},
  pages = 	 {2961--2970},
  year = 	 {2016},
  editor = 	 {Balcan, Maria Florina and Weinberger, Kilian Q.},
  volume = 	 {48},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {New York, New York, USA},
  month = 	 {20--22 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v48/akrour16.pdf},
  url = 	 {https://proceedings.mlr.press/v48/akrour16.html}
}

@inproceedings{
piche2018probabilistic,
title={Probabilistic Planning with Sequential Monte Carlo methods},
author={Alexandre Pich\'{e} and Valentin Thomas and Cyril Ibrahim and Yoshua Bengio and Chris Pal},
booktitle={International Conference on Learning Representations},
year={2019},
url={https://openreview.net/forum?id=ByetGn0cYX},
}

@inproceedings{
  watson2021advancing,
  author={Watson, Joe and Peters, Jan},
  booktitle={2021 American Control Conference (ACC)},
  title={Advancing Trajectory Optimization with Approximate Inference: Exploration, Covariance Control and Adaptive Risk},
  year={2021},
  volume={},
  number={},
  pages={1231-1236},
  doi={10.23919/ACC50511.2021.9482657}
}


@InProceedings{finn2017model,
  title = 	 {Model-Agnostic Meta-Learning for Fast Adaptation of Deep Networks},
  author =       {Chelsea Finn and Pieter Abbeel and Sergey Levine},
  booktitle = 	 {Proceedings of the 34th International Conference on Machine Learning},
  pages = 	 {1126--1135},
  year = 	 {2017},
  editor = 	 {Precup, Doina and Teh, Yee Whye},
  volume = 	 {70},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {06--11 Aug},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v70/finn17a/finn17a.pdf},
  url = 	 {https://proceedings.mlr.press/v70/finn17a.html},
  abstract = 	 {We propose an algorithm for meta-learning that is model-agnostic, in the sense that it is compatible with any model trained with gradient descent and applicable to a variety of different learning problems, including classification, regression, and reinforcement learning. The goal of meta-learning is to train a model on a variety of learning tasks, such that it can solve new learning tasks using only a small number of training samples. In our approach, the parameters of the model are explicitly trained such that a small number of gradient steps with a small amount of training data from a new task will produce good generalization performance on that task. In effect, our method trains the model to be easy to fine-tune. We demonstrate that this approach leads to state-of-the-art performance on two few-shot image classification benchmarks, produces good results on few-shot regression, and accelerates fine-tuning for policy gradient reinforcement learning with neural network policies.}
}


@InProceedings{mei2020ontheglobal,
  title = 	 {On the Global Convergence Rates of Softmax Policy Gradient Methods},
  author =       {Mei, Jincheng and Xiao, Chenjun and Szepesvari, Csaba and Schuurmans, Dale},
  booktitle = 	 {Proceedings of the 37th International Conference on Machine Learning},
  pages = 	 {6820--6829},
  year = 	 {2020},
  editor = 	 {III, Hal Daumé and Singh, Aarti},
  volume = 	 {119},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {13--18 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v119/mei20b/mei20b.pdf},
  url = 	 {https://proceedings.mlr.press/v119/mei20b.html},
  abstract = 	 {We make three contributions toward better understanding policy gradient methods in the tabular setting. First, we show that with the true gradient, policy gradient with a softmax parametrization converges at a $O(1/t)$ rate, with constants depending on the problem and initialization. This result significantly expands the recent asymptotic convergence results. The analysis relies on two findings: that the softmax policy gradient satisfies a Ł{}ojasiewicz inequality, and the minimum probability of an optimal action during optimization can be bounded in terms of its initial value. Second, we analyze entropy regularized policy gradient and show that it enjoys a significantly faster linear convergence rate $O(e^{-t})$ toward softmax optimal policy. This result resolves an open question in the recent literature. Finally, combining the above two results and additional new $\Omega(1/t)$ lower bound results, we explain how entropy regularization improves policy optimization, even with the true gradient, from the perspective of convergence rate. The separation of rates is further explained using the notion of non-uniform Ł{}ojasiewicz degree. These results provide a theoretical understanding of the impact of entropy and corroborate existing empirical studies.}
}

@article{cen2022fastglobal,
    author = {Cen, Shicong and Cheng, Chen and Chen, Yuxin and Wei, Yuting and Chi, Yuejie},
    title = {Fast Global Convergence of Natural Policy Gradient Methods with Entropy Regularization},
    journal = {Operations Research},
    volume = {70},
    number = {4},
    pages = {2563-2578},
    year = {2022},
    doi = {10.1287/opre.2021.2151},
    URL = { https://doi.org/10.1287/opre.2021.2151 },
    eprint = { https://doi.org/10.1287/opre.2021.2151 } ,
    abstract = { Preconditioning and Regularization Enable Faster Reinforcement LearningNatural policy gradient (NPG) methods, in conjunction with entropy regularization to encourage exploration, are among the most popular policy optimization algorithms in contemporary reinforcement learning. Despite the empirical success, the theoretical underpinnings for NPG methods remain severely limited. In “Fast Global Convergence of Natural Policy Gradient Methods with Entropy Regularization”, Cen, Cheng, Chen, Wei, and Chi develop nonasymptotic convergence guarantees for entropy-regularized NPG methods under softmax parameterization, focusing on tabular discounted Markov decision processes. Assuming access to exact policy evaluation, the authors demonstrate that the algorithm converges linearly at an astonishing rate that is independent of the dimension of the state-action space. Moreover, the algorithm is provably stable vis-à-vis inexactness of policy evaluation. Accommodating a wide range of learning rates, this convergence result highlights the role of preconditioning and regularization in enabling fast convergence. }
}

@misc{githubRepo,
  author = {Arriojas, Argenis and Adamczyk, Jacob and Tiomkin, Stas and Kulkarni, Rahul V.},
  title = {Code for reproducibility},
  year = 2023,
  month = jun,
  howpublished = {\url{https://github.com/argearriojas/UAI23-Arriojas_611}}
}