@article{derman2021twice,
  title={Twice regularized MDPs and the equivalence between robustness and regularization},
  author={Derman, Esther and Geist, Matthieu and Mannor, Shie},
  journal={Advances in Neural Information Processing Systems},
  year={2021}
}
@article{wiesemann2013robust,
  title={Robust Markov decision processes},
  author={Wiesemann, Wolfram and Kuhn, Daniel and Rustem, Ber{\c{c}}},
  journal={Mathematics of Operations Research},
  volume={38},
  number={1},
  pages={153--183},
  year={2013},
  publisher={INFORMS}
}
@article{yang2021towards,
  title={Towards Theoretical Understandings of Robust Markov Decision Processes: Sample Complexity and Asymptotics},
  author={Yang, Wenhao and Zhang, Liangyu and Zhang, Zhihua},
  journal={arXiv preprint arXiv:2105.03863},
  year={2021}
}
@inproceedings{panaganti2022sample,
  title={Sample Complexity of Robust Reinforcement Learning with a Generative Model},
  author={Panaganti, Kishan and Kalathil, Dileep},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  year={2022}
}
@inproceedings{mei2020global,
  title={On the global convergence rates of softmax policy gradient methods},
  author={Mei, Jincheng and Xiao, Chenjun and Szepesvari, Csaba and Schuurmans, Dale},
  booktitle={International Conference on Machine Learning},
  year={2020}
}

@article{wang2022policy,
  title={Policy Gradient Method For Robust Reinforcement Learning},
  author={Wang, Yue and Zou, Shaofeng},
  journal={International Conference on Machine Learning},
  year={2022}
}
@article{kallus2022doubly,
  title={Doubly Robust Distributionally Robust Off-Policy Evaluation and Learning},
  author={Kallus, Nathan and Mao, Xiaojie and Wang, Kaiwen and Zhou, Zhengyuan},
  journal={International Conference on Machine Learning},
  year={2022}
}
@inproceedings{zhou2021finite,
  title={Finite-sample regret bound for distributionally robust offline tabular reinforcement learning},
  author={Zhou, Zhengqing and Zhou, Zhengyuan and Bai, Qinxun and Qiu, Linhai and Blanchet, Jose and Glynn, Peter},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  year={2021}
}
@article{qi2020robust,
  title={Robust Batch Policy Learning in Markov Decision Processes},
  author={Qi, Zhengling and Liao, Peng},
  journal={arXiv preprint arXiv:2011.04185},
  year={2020}
}
@inproceedings{badrinath2021robust,
  title={Robust reinforcement learning using least squares policy iteration with provable performance guarantees},
  author={Badrinath, Kishan Panaganti and Kalathil, Dileep},
  booktitle={International Conference on Machine Learning},
  year={2021}
}
@article{iyengar2005robust,
  title={Robust dynamic programming},
  author={Iyengar, Garud N},
  journal={Mathematics of Operations Research},
  volume={30},
  number={2},
  pages={257--280},
  year={2005},
  publisher={INFORMS}
}
@article{nilim2005robust,
  title={Robust control of Markov decision processes with uncertain transition matrices},
  author={Nilim, Arnab and El Ghaoui, Laurent},
  journal={Operations Research},
  volume={53},
  number={5},
  pages={780--798},
  year={2005},
  publisher={INFORMS}
}
@article{schulman2017proximal,
  title={Proximal policy optimization algorithms},
  author={Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
  journal={arXiv preprint arXiv:1707.06347},
  year={2017}
}
@article{jin2019learning,
  title={Learning adversarial mdps with bandit feedback and unknown transition},
  author={Jin, Chi and Jin, Tiancheng and Luo, Haipeng and Sra, Suvrit and Yu, Tiancheng},
  journal={arXiv preprint arXiv:1912.01192},
  year={2019}
}
@inproceedings{schulman2015trust,
  title={Trust region policy optimization},
  author={Schulman, John and Levine, Sergey and Abbeel, Pieter and Jordan, Michael and Moritz, Philipp},
  booktitle={International conference on machine learning},
  year={2015}
}
@article{li2022policy,
  title={Policy Learning for Robust Markov Decision Process with a Mismatched Generative Model},
  author={Li, Jialian and Ren, Tongzheng and Yan, Dong and Su, Hang and Zhu, Jun},
  year={2022}
}
@book{puterman2014markov,
  title={Markov decision processes: discrete stochastic dynamic programming},
  author={Puterman, Martin L},
  year={2014},
  publisher={John Wiley \& Sons}
}
@inproceedings{wu2022nearly,
  title={Nearly optimal policy optimization with stable at any time guarantee},
  author={Wu, Tianhao and Yang, Yunchang and Zhong, Han and Wang, Liwei and Du, Simon and Jiao, Jiantao},
  booktitle={International Conference on Machine Learning},
  year={2022}
}

@InProceedings{pmlr-v139-badrinath21a,
  title = 	 {Robust Reinforcement Learning using Least Squares Policy Iteration with Provable Performance Guarantees},
  author =       {Badrinath, Kishan Panaganti and Kalathil, Dileep},
  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
  year = 	 {2021}
}

@inproceedings{mannor2012lightning,
  title={Lightning does not strike twice: robust MDPs with coupled uncertainty},
  author={Mannor, Shie and Mebel, Ofir and Xu, Huan},
  booktitle={Proceedings of the 29th International Coference on International Conference on Machine Learning},
  year={2012}
}
@article{mohajerin2018data,
  title={Data-driven distributionally robust optimization using the Wasserstein metric: Performance guarantees and tractable reformulations},
  author={Mohajerin Esfahani, Peyman and Kuhn, Daniel},
  journal={Mathematical Programming},
  volume={171},
  number={1},
  pages={115--166},
  year={2018},
  publisher={Springer}
}

@article{chen2018robust,
  title={A robust learning approach for regression models based on distributionally robust optimization},
  author={Chen, Ruidi and Paschalidis, Ioannis C},
  journal={Journal of Machine Learning Research},
  volume={19},
  number={13},
  year={2018}
}
@article{orabona2019modern,
  title={A modern introduction to online learning},
  author={Orabona, Francesco},
  journal={arXiv preprint arXiv:1912.13213},
  year={2019}
}
@article{shi2022distributionally,
  title={Distributionally Robust Model-Based Offline Reinforcement Learning with Near-Optimal Sample Complexity},
  author={Shi, Laixi and Chi, Yuejie},
  journal={arXiv preprint arXiv:2208.05767},
  year={2022}
}
@article{auer2002nonstochastic,
  title={The nonstochastic multiarmed bandit problem},
  author={Auer, Peter and Cesa-Bianchi, Nicolo and Freund, Yoav and Schapire, Robert E},
  journal={SIAM journal on computing},
  volume={32},
  number={1},
  pages={48--77},
  year={2002},
  publisher={SIAM}
}
@inproceedings{cai2020provably,
  title={Provably efficient exploration in policy optimization},
  author={Cai, Qi and Yang, Zhuoran and Jin, Chi and Wang, Zhaoran},
  booktitle={International Conference on Machine Learning},
  year={2020}
}
@inproceedings{shani2020optimistic,
  title={Optimistic policy optimization with bandit feedback},
  author={Shani, Lior and Efroni, Yonathan and Rosenberg, Aviv and Mannor, Shie},
  booktitle={International Conference on Machine Learning},
  year={2020}
}
@inproceedings{lancewicki2022learning,
  title={Learning adversarial markov decision processes with delayed feedback},
  author={Lancewicki, Tal and Rosenberg, Aviv and Mansour, Yishay},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  year={2022}
}
@article{fei2020dynamic,
  title={Dynamic regret of policy optimization in non-stationary environments},
  author={Fei, Yingjie and Yang, Zhuoran and Wang, Zhaoran and Xie, Qiaomin},
  journal={Advances in Neural Information Processing Systems},
  year={2020}
}
@article{he2021nearly,
  title={Nearly optimal regret for learning adversarial mdps with linear function approximation},
  author={He, Jiafan and Zhou, Dongruo and Gu, Quanquan},
  journal={arXiv preprint arXiv:2102.08940},
  year={2021}
}
@article{zhong2021optimistic,
  title={Optimistic Policy Optimization is Provably Efficient in Non-stationary MDPs},
  author={Zhong, Han and Yang, Zhuoran and Szepesv{\'a}ri, Zhaoran Wang Csaba},
  journal={arXiv preprint arXiv:2110.08984},
  year={2021}
}
@inproceedings{cobbe2019quantifying,
  title={Quantifying generalization in reinforcement learning},
  author={Cobbe, Karl and Klimov, Oleg and Hesse, Chris and Kim, Taehoon and Schulman, John},
  booktitle={International Conference on Machine Learning},
  year={2019}
}
@article{luo2021policy,
  title={Policy optimization in adversarial mdps: Improved exploration via dilated bonuses},
  author={Luo, Haipeng and Wei, Chen-Yu and Lee, Chung-Wei},
  journal={Advances in Neural Information Processing Systems},
  year={2021}
}
@inproceedings{zanette2021cautiously,
  title={Cautiously optimistic policy optimization and exploration with linear function approximation},
  author={Zanette, Andrea and Cheng, Ching-An and Agarwal, Alekh},
  booktitle={Conference on Learning Theory},
  year={2021}
}

@article{agarwal2019reinforcement,
  title={Reinforcement learning: Theory and algorithms},
  author={Agarwal, Alekh and Jiang, Nan and Kakade, Sham M and Sun, Wen},
  journal={CS Dept., UW Seattle, Seattle, WA, USA, Tech. Rep},
  pages={10--4},
  year={2019}
}
@article{even2004experts,
  title={Experts in a Markov decision process},
  author={Even-Dar, Eyal and Kakade, Sham M and Mansour, Yishay},
  journal={Advances in neural information processing systems},
  year={2004}
}
@inproceedings{zhang2022corruption,
  title={Corruption-robust offline reinforcement learning},
  author={Zhang, Xuezhou and Chen, Yiding and Zhu, Xiaojin and Sun, Wen},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  year={2022}
}
@inproceedings{lykouris2021corruption,
  title={Corruption-robust exploration in episodic reinforcement learning},
  author={Lykouris, Thodoris and Simchowitz, Max and Slivkins, Alex and Sun, Wen},
  booktitle={Conference on Learning Theory},
  year={2021}
}

@inproceedings{chen2021improved,
  title={Improved corruption robust algorithms for episodic reinforcement learning},
  author={Chen, Yifang and Du, Simon and Jamieson, Kevin},
  booktitle={International Conference on Machine Learning},
  year={2021}
}
@InProceedings{pmlr-v119-jin20c,
  title = 	 {Learning Adversarial {M}arkov Decision Processes with Bandit Feedback and Unknown Transition},
  year = {2020},
  author =       {Jin, Chi and Jin, Tiancheng and Luo, Haipeng and Sra, Suvrit and Yu, Tiancheng},
  booktitle = 	 {International Conference on Machine Learning}
}

@article{jin2020simultaneously,
  title={Simultaneously learning stochastic and adversarial episodic mdps with known transition},
  author={Jin, Tiancheng and Luo, Haipeng},
  journal={Advances in neural information processing systems},
  year={2020}
}
@inproceedings{rosenberg2019online,
  title={Online convex optimization in adversarial {M}arkov decision processes},
  author={Rosenberg, Aviv and Mansour, Yishay},
  booktitle={International Conference on Machine Learning},
  year={2019}
}

@misc{bartlett2013theoretical,
  title={Theoretical Statistics. {L}ecture 12},
  author={Bartlett, Peter},
  year={2013}
}
@inproceedings{raileanu2021decoupling,
  title={Decoupling value and policy for generalization in reinforcement learning},
  author={Raileanu, Roberta and Fergus, Rob},
  booktitle={International Conference on Machine Learning},
  year={2021}
}
@inproceedings{song2019observational,
  title={Observational Overfitting in Reinforcement Learning},
  author={Song, Xingyou and Jiang, Yiding and Tu, Stephen and Du, Yilun and Neyshabur, Behnam},
  booktitle={International Conference on Learning Representations},
  year={2019}
}
@article{behzadian2021fast,
  title={Fast Algorithms for $ L\_$\backslash$infty $-constrained S-rectangular Robust MDPs},
  author={Behzadian, Bahram and Petrik, Marek and Ho, Chin Pang},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={25982--25992},
  year={2021}
}
@article{neu2010online,
  title={Online Markov decision processes under bandit feedback},
  author={Neu, Gergely and Antos, Andras and Gy{\"o}rgy, Andr{\'a}s and Szepesv{\'a}ri, Csaba},
  journal={Advances in Neural Information Processing Systems},
  year={2010}
}
@inproceedings{chen2021minimax,
  title={Minimax regret for stochastic shortest path with adversarial costs and known transition},
  author={Chen, Liyu and Luo, Haipeng and Wei, Chen-Yu},
  booktitle={Conference on Learning Theory},
  year={2021}
}
@article{farebrother2018generalization,
  title={Generalization and regularization in DQN},
  author={Farebrother, Jesse and Machado, Marlos C and Bowling, Michael},
  journal={arXiv preprint arXiv:1810.00123},
  year={2018}
}
@article{packer2018assessing,
  title={Assessing generalization in deep reinforcement learning},
  author={Packer, Charles and Gao, Katelyn and Kos, Jernej and Kr{\"a}henb{\"u}hl, Philipp and Koltun, Vladlen and Song, Dawn},
  journal={arXiv preprint arXiv:1810.12282},
  year={2018}
}
@inproceedings{kaiser2019model,
  title={Model Based Reinforcement Learning for Atari},
  author={Kaiser, {\L}ukasz and Babaeizadeh, Mohammad and Mi{\l}os, Piotr and Osi{\'n}ski, B{\l}a{\.z}ej and Campbell, Roy H and Czechowski, Konrad and Erhan, Dumitru and Finn, Chelsea and Kozakowski, Piotr and Levine, Sergey and others},
  booktitle={International Conference on Learning Representations},
  year={2019}
}
@article{wang2021online,
  title={Online robust reinforcement learning with model uncertainty},
  author={Wang, Yue and Zou, Shaofeng},
  journal={Advances in Neural Information Processing Systems},
  year={2021}
}

@article{ma22distribution,
  title = {Distributionally Robust Offline Reinforcement Learning with Linear Function Approximation},
  author = {Ma, Xiaoteng and Liang, Zhipeng and Xia, Li and Zhang, Jiheng and Blanchet, Jose and Liu, Mingwen and Zhao, Qianchuan and Zhou, Zhengyuan},
  journal={arXiv preprint arXiv:2209.06620},
  year={2022}
}

@article{efroni2019tight,
  title={Tight regret bounds for model-based reinforcement learning with greedy policies},
  author={Efroni, Yonathan and Merlis, Nadav and Ghavamzadeh, Mohammad and Mannor, Shie},
  journal={Advances in Neural Information Processing Systems},
  year={2019}
}
@misc{rlberry,
    author = {Domingues, Omar Darwiche and Flet-Berliac, Yannis and Leurent, Edouard and M{\'e}nard, Pierre and Shang, Xuedong and Valko, Michal},
    doi = {10.5281/zenodo.5544540},
    month = {10},
    title = {{rlberry - A Reinforcement Learning Library for Research and Education}},
    url = {https://github.com/rlberry-py/rlberry},
    year = {2021}
}
@book{sutton2018reinforcement,
  title={Reinforcement learning: An introduction},
  author={Sutton, Richard S and Barto, Andrew G},
  year={2018},
  publisher={MIT press}
}
@article{dann2017unifying,
  title={Unifying PAC and regret: Uniform PAC bounds for episodic reinforcement learning},
  author={Dann, Christoph and Lattimore, Tor and Brunskill, Emma},
  journal={Advances in Neural Information Processing Systems},
  year={2017}
}
@article{satia1973markovian,
  title={Markovian decision processes with uncertain transition probabilities},
  author={Satia, Jay K and Lave Jr, Roy E},
  journal={Operations Research},
  volume={21},
  number={3},
  pages={728--740},
  year={1973},
  publisher={INFORMS}
}
@article{white1994markov,
  title={Markov decision processes with imprecise transition probabilities},
  author={White III, Chelsea C and Eldeib, Hany K},
  journal={Operations Research},
  volume={42},
  number={4},
  pages={739--749},
  year={1994},
  publisher={INFORMS}
}
@inproceedings{liu2022distributionally,
  title={Distributionally Robust $ Q $-Learning},
  author={Liu, Zijian and Bai, Qinxun and Blanchet, Jose and Dong, Perry and Xu, Wei and Zhou, Zhengqing and Zhou, Zhengyuan},
  booktitle={International Conference on Machine Learning},
  pages={13623--13643},
  year={2022},
  organization={PMLR}
}