@misc{levine2020offline,
      title={Offline Reinforcement Learning: Tutorial, Review, and Perspectives on Open Problems}, 
      author={Sergey Levine and Aviral Kumar and George Tucker and Justin Fu},
      year={2020},
      eprint={2005.01643},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@InProceedings{haarnoja18bSA,
title = {Soft {A}ctor-{C}ritic: {O}ff-{P}olicy {M}aximum {E}ntropy {D}eep {R}einforcement {L}earning with a {S}tochastic {A}ctor},
author = {Haarnoja, T. and Zhou, A. and Abbeel, P. and Levine, S.},
booktitle = {Proceedings of the 35th International Conference on Machine Learning},
pages = {1861--1870},
year = {2018}
}

@INPROCEEDINGS{Konda2000Actor-Critic,
    author = {Vijay Konda and John Tsitsiklis},
    title = {Actor-Critic Algorithms},
    booktitle = {SIAM Journal on Control and Optimization},
    year = {2000},
    pages = {1008--1014},
    publisher = {MIT Press}
}

@article{Steimle2018MM,
  title={Multi-model {Markov} Decision Processes},
  author={Lauren Steimle and David Kaufman and Brian Denton},
  journal = {Optimization Online},
  year={2018}
}

@article{bucholz2019CMDP,
author = {Buchholz, Peter and Scheftelowitsch, Dimitri},
year = {2019},
title = {Computation of weighted sums of rewards for concurrent {MDPs}},
volume = {89},
journal = {Mathematical Methods of Operations Research},
}

@article{Hoffman2013SVI,
author = {Hoffman, Matthew D. and Blei, David M. and Wang, Chong and Paisley, John},
title = {Stochastic Variational Inference},
year = {2013},
publisher = {JMLR.org},
volume = {14},
number = {1},
journal = {Journal of Machine Learning Research},
pages = {1303–1347},
}
@article{Plummer2003JAGS,
author = {Plummer, Martyn},
year = {2003},
title = {JAGS: A Program for Analysis of Bayesian Graphical Models using Gibbs Sampling},
volume = {124},
journal = {3rd International Workshop on Distributed Statistical Computing (DSC 2003); Vienna, Austria}
}

@article{hoffman2011nouturn,
  author  = {Matthew D. Hoffman and Andrew Gelman},
  title   = {The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo},
  journal = {Journal of Machine Learning Research},
  year    = {2014},
  volume  = {15},
  number  = {47},
  pages   = {1593-1623},
  url     = {http://jmlr.org/papers/v15/hoffman14a.html}
}

@article{hester2017DQ, title={Deep Q-learning From Demonstrations}, volume={32}, url={https://ojs.aaai.org/index.php/AAAI/article/view/11757}, abstractNote={ &lt;p&gt; Deep reinforcement learning (RL) has achieved several high profile successes in difficult decision-making problems. However, these algorithms typically require a huge amount of data before they reach reasonable performance. In fact, their performance during learning can be extremely poor. This may be acceptable for a simulator, but it severely limits the applicability of deep RL to many real-world tasks, where the agent must learn in the real environment. In this paper we study a setting where the agent may access data from previous control of the system. We present an algorithm, Deep Q-learning from Demonstrations (DQfD), that leverages small sets of demonstration data to massively accelerate the learning process even from relatively small amounts of demonstration data and is able to automatically assess the necessary ratio of demonstration data while learning thanks to a prioritized replay mechanism. DQfD works by combining temporal difference updates with supervised classification of the demonstrator’s actions. We show that DQfD has better initial performance than Prioritized Dueling Double Deep Q-Networks (PDD DQN) as it starts with better scores on the first million steps on 41 of 42 games and on average it takes PDD DQN 83 million steps to catch up to DQfD’s performance. DQfD learns to out-perform the best demonstration given in 14 of 42 games. In addition, DQfD leverages human demonstrations to achieve state-of-the-art results for 11 games. Finally, we show that DQfD performs better than three related algorithms for incorporating demonstration data into DQN. &lt;/p&gt; }, number={1}, journal={Proceedings of the AAAI Conference on Artificial Intelligence}, author={Hester, Todd and Vecerik, Matej and Pietquin, Olivier and Lanctot, Marc and Schaul, Tom and Piot, Bilal and Horgan, Dan and Quan, John and Sendonaris, Andrew and Osband, Ian and Dulac-Arnold, Gabriel and Agapiou, John and Leibo, Joel and Gruslys, Audrunas}, year={2018} }

@inproceedings{lillicrap2019DDPG,
  author    = {Timothy P. Lillicrap and
               Jonathan J. Hunt and
               Alexander Pritzel and
               Nicolas Heess and
               Tom Erez and
               Yuval Tassa and
               David Silver and
               Daan Wierstra},
  editor    = {Yoshua Bengio and
               Yann LeCun},
  title     = {Continuous control with deep reinforcement learning},
  booktitle = {4th International Conference on Learning Representations, {ICLR} 2016,
               San Juan, Puerto Rico, May 2-4, 2016, Conference Track Proceedings},
  year      = {2016},
  url       = {http://arxiv.org/abs/1509.02971},
  timestamp = {Thu, 25 Jul 2019 14:25:37 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/LillicrapHPHETS15.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{Box1973BayesianLR,
  title={Bayesian inference in statistical analysis},
  author={G. P. Box and G. C. Tiao},
  journal={International Statistical Review},
  year={1973},
  volume={43},
  pages={242}
}

@inbook{bucholz2020CMFMP,
author = {Buchholz, Peter and Scheftelowitsch, Dimitri},
year = {2020},
pages = {37-53},
title = {Concurrent MDPs with Finite Markovian Policies},
publisher = {Springer, Cham},
}

@article{merakli2019RiskA,
author = {Meraklı, Merve and Küçükyavuz, Simge},
year = {2019},
pages = {1-52},
title = {Risk Aversion to Parameter Uncertainty in {Markov} Decision Processes with an Application to Slow-Onset Disaster Relief},
journal = {IISE Transactions},
}
@article{Krokhmal2003PO,
author = {Krokhmal, Pavlo and Palmquist, Jonas and Uryasev, Stan},
year = {2003},
title = {Portfolio Optimization With Conditional Value-at-Risk Objective and Constraints},
volume = {4},
journal = {Journal of Risk},
}

@inproceedings{Mannor2012LightningDN,
    title = {Lightning Does Not Strike Twice: {{Robust MDPs}} with Coupled Uncertainty},
    booktitle = {International {{Conference}} on {{Machine Learning}} ({{ICML}})},
    author = {Mannor, Shie and Mebel, O and Xu, H},
    year = {2012},
    file = {/home/marek/Documents/Zotero/storage/RZHH7UJY/Mannor, Mebel, Xu - 2012 - Lightning does not strike twice Robust MDPs with coupled uncertainty.pdf}
}

@InProceedings{tang2019worst,
  title = 	 {Worst Cases Policy Gradients},
  author =       {Tang, Yichuan Charlie and Zhang, Jian and Salakhutdinov, Ruslan},
  booktitle = 	 {Proceedings of the Conference on Robot Learning},
  pages = 	 {1078--1093},
  year = 	 {2020},
  editor = 	 {Kaelbling, Leslie Pack and Kragic, Danica and Sugiura, Komei},
  volume = 	 {100},
  series = 	 {Proceedings of Machine Learning Research},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v100/tang20a/tang20a.pdf},
  url = 	 {https://proceedings.mlr.press/v100/tang20a.html},
  abstract = 	 {Recent advances in deep reinforcement learning have demonstrated the capability of learning complex control policies from many types of environment. When learning policies for safety critical applications, it is important to be sensitive to risks and avoid catastrophic events. Towards this goal, we propose an actor-critic framework which models the uncertainty of the future and simultaneously learns a policy based on that uncertainty model. Specifically, given a distribution of the future return for any state and action, we optimize policies for varying levels of conditional Value-at-Risk. The learned policy can map the same state to different actions depending on the propensity for risk. We demonstrate the effectiveness of our approach in the domain of driving simulations, where we learn maneuvers in two scenarios. Our learned controller can dynamically select actions along a continuous axis, where safe and conservative behaviors are found at one end while riskier behaviors are found at the other. Finally, when testing with very different simulation parameters, our risk-averse policies generalize significantly better compared to other reinforcement learning approaches.}
}

@article{epstein2003RecursiveMP,
title = {Recursive multiple-priors},
journal = {Journal of Economic Theory},
volume = {113},
number = {1},
pages = {1-31},
year = {2003},
}

@inproceedings{russel2019BeyondCR,
  author    = {Reazul Hasan Russel and Marek Petrik},
  title     = {Beyond Confidence Regions: Tight {Bayesian} Ambiguity Sets for Robust
               {MDPs}},
  booktitle = {International Conference on Neural Information Processing Systems},
  year = {2019}
}

@inproceedings{Derman2019BayesianRobust,
  author    = {Esther Derman and
               Daniel J. Mankowitz and
               Timothy A. Mann and
               Shie Mannor},
  title     = {A {Bayesian} Approach to Robust Reinforcement Learning},
  booktitle = {Conference on Uncertainty in Artificial Intelligence},
  year      = {2019},
  eprint    = {1905.08188}
}

@INPROCEEDINGS{Wang2007DualR,
  author={T. {Wang} and M. {Bowling} and D. {Schuurmans}},
  booktitle={IEEE International Symposium on Approximate Dynamic Programming and Reinforcement Learning},
  title={Dual Representations for Dynamic Programming and Reinforcement Learning},
  year={2007},
  pages={44-51},
}
@article{gupta2019NearOpt,
author = {Gupta, Vishal},
year = {2019},
title = {Near-Optimal {Bayesian} Ambiguity Sets for Distributionally Robust Optimization},
journal = {Management Science},
}
@inproceedings{Petrik2016SafePI,
 author = {Petrik, Marek and Ghavamzadeh, Mohammad and Chow, Yinlam},
 title = {Safe Policy Improvement by Minimizing Robust Baseline Regret},
 booktitle = {International Conference on Neural Information Processing Systems},
 year = {2016},
 pages = {2306--2314},
}

@book{bersekas2007DP,
author = {Bertsekas, Dimitri P.},
title = {Dynamic Programming and Optimal Control, Vol. II},
year = {2007},
publisher = {Athena Scientific},
edition = {3rd},
}

@article{Ahmed2017SamplingA,
	author = {Ahmed, Asrar and Jaillet, Patrick},
	journal = {Journal of Artificial Intelligence Research},
	pages = {229--264},
	title = {{Sampling Based Approaches for Minimizing Regret in Uncertain {Markov} Decision Processes}},
	volume = {59},
	year = {2017}
}

@article{Delage2009Percentile,
	author = {Delage, E. and Mannor, S.},
	journal = {Operations Research},
	number = {1},
	pages = {203--213},
	title = {{Percentile Optimization for {Markov} Decision Processes with Parameter Uncertainty}},
	volume = {58},
	year = {2010}
}



@article{Roorda2004CoherentAM,
  title={Coherent Acceptability Measures in Multiperiod Models},
  author={Berend Roorda and Johannes Schumacher and Jacob Engwerda},
  year={2005},
  journal = {Mathematical Finance},
  volume = {15},
  issue = {4},
  pages = {589--612}
}

@INPROCEEDINGS{Prashant2014PG,
    author = {L. A. Prashanth},
    title = {Policy Gradients for {CVaR}-Constrained {MDP}s},
    booktitle = {In Proceedings of the 25th International Conference on Algorithmic Learning Theory},
    pages = {155-169},
    year = {2014}
}

@article{iyengar2005RMDP,
 author = {Garud N. Iyengar},
 journal = {Mathematics of Operations Research},
 number = {2},
 pages = {257--280},
 publisher = {INFORMS},
 title = {Robust Dynamic Programming},
 volume = {30},
 year = {2005}
}

@article{Wiesemann2013RMDP,
 author = {Wiesemann, Wolfram and Kuhn, Daniel and Rustem, Ber\c{c}},
 title = {Robust {Markov} Decision Processes},
 journal = {Mathematics of Operations Research},
 volume = {38},
 number = {1},
 year = {2013},
 pages = {153--183},
 publisher = {INFORMS},
}

@INPROCEEDINGS{Thrun93issuesin,
    author = {Sebastian Thrun and Anton Schwartz},
    title = {Issues in Using Function Approximation for Reinforcement Learning},
    booktitle = {In Proceedings of the Fourth Connectionist Models Summer School},
    year = {1993},
    publisher = {Erlbaum}
}

@book{putterman1994DP,
author = {Puterman, Martin L.},
title = {Markov Decision Processes: Discrete Stochastic Dynamic Programming},
year = {2005},
publisher = {John Wiley and Sons, Inc.},
edition = {2nd},
}

@InProceedings{rakhsha2020policy,
  title = 	 {Policy Teaching via Environment Poisoning: Training-time Adversarial Attacks against Reinforcement Learning},
  author =       {Rakhsha, Amin and Radanovic, Goran and Devidze, Rati and Zhu, Xiaojin and Singla, Adish},
  booktitle = 	 {International Conference on Machine Learning},
  pages = 	 {7974--7984},
  year = 	 {2020},
  abstract = 	 {We study a security threat to reinforcement learning where an attacker poisons the learning environment to force the agent into executing a target policy chosen by the attacker. As a victim, we consider RL agents whose objective is to find a policy that maximizes average reward in undiscounted infinite-horizon problem settings. The attacker can manipulate the rewards or the transition dynamics in the learning environment at training-time and is interested in doing so in a stealthy manner. We propose an optimization framework for finding an \emph{optimal stealthy attack} for different measures of attack cost. We provide sufficient technical conditions under which the attack is feasible and provide lower/upper bounds on the attack cost. We instantiate our attacks in two settings: (i) an \emph{offline} setting where the agent is doing planning in the poisoned environment, and (ii) an \emph{online} setting where the agent is learning a policy using a regret-minimization framework with poisoned feedback. Our results show that the attacker can easily succeed in teaching any target policy to the victim under mild conditions and highlight a significant security threat to reinforcement learning agents in practice.}
}

@article{pearmutter1994fast,
author = {Pearlmutter, Barak A.},
title = {Fast Exact Multiplication by the {H}essian},
year = {1994},
publisher = {MIT Press},
address = {Cambridge, MA, USA},
volume = {6},
number = {1},
issn = {0899-7667},
journal = {Neural Computing}
}
@misc{diakonikolas2019recent,
      title={Recent Advances in Algorithmic High-Dimensional Robust Statistics}, 
      author={Ilias Diakonikolas and Daniel M. Kane},
      year={2019},
      eprint={1911.05911},
      archivePrefix={arXiv},
      primaryClass={cs.DS}
}
@misc{broderick2021automatic,
      title={An Automatic Finite-Sample Robustness Metric: Can Dropping a Little Data Change Conclusions?}, 
      author={Tamara Broderick and Ryan Giordano and Rachael Meager},
      year={2021},
      eprint={2011.14999},
      archivePrefix={arXiv},
      primaryClass={stat.ME}
}
@InProceedings{riedmiller2005fqe,
author="Riedmiller, Martin",
editor="Gama, Jo{\~a}o
and Camacho, Rui
and Brazdil, Pavel B.
and Jorge, Al{\'i}pio M{\'a}rio
and Torgo, Lu{\'i}s",
title="Neural Fitted Q Iteration -- First Experiences with a Data Efficient Neural Reinforcement Learning Method",
booktitle="Machine Learning: ECML 2005",
year="2005",
publisher="Springer Berlin Heidelberg"}


@inproceedings{Farajtabar2018MoreRD,
  title={More Robust Doubly Robust Off-policy Evaluation},
  author={Mehrdad Farajtabar and Yinlam Chow and Mohammad Ghavamzadeh},
  booktitle={Thirty-fifth International Conference on Machine Learning (ICML) },
  year={2018}
}

@article{Chen2019adversarial,
author = {Chen, Tong and Liu, Jiqiang and Xiang, Yingxiao and Niu, Wenjia and Tong, Endong and Han, Zhen},
year = {2019},
pages = {},
title = {Adversarial attack and defense in reinforcement learning-from AI security view},
volume = {2},
journal = {Cybersecurity},
}
@inproceedings{Ma2019PolicyPI,
  title={Policy Poisoning in Batch Reinforcement Learning and Control},
  author={Yuzhe Ma and Xuezhou Zhang and Wen Sun and Xiaojin Zhu},
  booktitle={Thirty-third Conference on Neural Information Processing Systems},
  year={2019}
}

@INPROCEEDINGS{hiv,
  author={Ernst, Damien and Stan, Guy-Bart and Goncalves, Jorge and Wehenkel, Louis},
  booktitle={IEEE Conference on Decision and Control}, 
  title={Clinical data based optimal STI strategies for HIV: a reinforcement learning approach}, 
  year={2006},
  volume={},
  number={},
  pages={667-672}}



@conference{Wu2017RobustOPT,
	title = {Robust Optimization for Tree-Structured Stochastic Network Design},
	author = {Xiaojian Wu and Akshat Kumar and Daniel Sheldon and Shlomo Zilberstein},
	year = {2017},
	booktitle = {Thirty First AAAI Conference on Artificial Intelligence},
	pages = {4545--4551},
}

@conference{Saisubramanian2018SR,
	title = {Safe Reduced Models for Probabilistic Planning},
	author = {Sandhya Saisubramanian and Shlomo Zilberstein},
	year = {2018},
	booktitle = {ICML/IJCAI/AAMAS Workshop on Planning and Learning},
}

@inproceedings{Petrik2014RAAM,
	author = {Petrik, Marek and Subramanian, Dharmashankar},
	booktitle = { International Conference on Neural Information Processing Systems
},
	title = {{RAAM : The benefits of robustness in approximating aggregated MDPs in reinforcement learning}},
	year = {2014}
}


@conference{Wu2017Multi-Agent,
	title = {Multi-Agent Planning with Baseline Regret Minimization},
	author = {Feng Wu and Shlomo Zilberstein and Xiaoping Chen},
	year = {2017},
	booktitle = {Twenty Sixth International Joint Conference on Artificial Intelligence},
	pages = {444--450},
}
@article{Dhaene2002Comonotonicity,
author = {Dhaene, Jan and Goovaerts, Marc and Denuit, Michel and Kaas, Rob and Vyncke, David},
year = {2002},
pages = {3-33},
title = {The Concept of Comonotonicity in Actuarial Science And Finance: Theory},
volume = {31},
journal = {Insurance: Mathematics and Economics},
}

@book{vaart1998Asymptotic, place={Cambridge}, series={Cambridge Series in Statistical and Probabilistic Mathematics}, title={Asymptotic Statistics}, publisher={Cambridge University Press}, author={Vaart, A. W. van der}, year={1998}}

@article{chen1985AsymNormality,
author = {Chen, Chan-Fu},
title = {On Asymptotic Normality of Limiting Density Functions with {Bayesian} Implications},
journal = {Journal of the Royal Statistical Society: Series B (Methodological)},
volume = {47},
number = {3},
pages = {540-546},
year = {1985}
}


@article{Teh2006HDP,
author = {Yee Whye Teh and Michael I Jordan and Matthew J Beal and David M Blei},
title = {Hierarchical {Dirichlet} {Processes}},
journal = {Journal of the American Statistical Association},
volume = {101},
number = {476},
pages = {1566-1581},
year  = {2006},
publisher = {Taylor \& Francis},

}
@InProceedings{Pang2018FastBellman,
  title = 	 {Fast {B}ellman Updates for Robust {MDP}s},
  author = 	 {Ho, Chin Pang and Petrik, Marek and Wiesemann, Wolfram},
  booktitle = 	 {International Conference on Machine Learning},
  pages = 	 {1979--1988},
  year = 	 {2018},
  editor = 	 {Dy, Jennifer and Krause, Andreas},
  series = 	 {Proceedings of Machine Learning Research},
  publisher = 	 {PMLR}
}
@phdthesis{letallec2007Robust,
  title={Robust, risk-sensitive, and data-driven control of {Markov} decision processes},
  author={Yann Le Tallec},
  year={2007},
  school = {MIT}
}

@inproceedings{Ho2018FastBellman,
	author = {Ho, Chin Pang and Petrik, Marek and Wiesemann, Wolfram},
	booktitle = {Proceedings of the 35th International Conference on Machine Learning Conference},
	pages = {1979--1988},
	title = {Fast {Bellman} Updates for Robust {MDPs}},
	year = {2018}
}
@article{Nilim2005RobustControl,
 author = {Nilim, Arnab and El Ghaoui, Laurent},
 title = {Robust Control of {Markov} Decision Processes with Uncertain Transition Matrices},
 journal = {Operations Research},
 volume = {53},
 number = {5},
 year = {2005},
 pages = {780--798},
 publisher = {INFORMS},
}
@inproceedings{Chow2014CVAR,
 author = {Chow, Yinlam and Ghavamzadeh, Mohammad},
 title = {Algorithms for {CVaR} Optimization in {MDPs}},
 booktitle = {International Conference on Neural Information Processing Systems},
 year = {2014},
 location = {Montreal, Canada},
 pages = {3509--3517},
 publisher = {MIT Press},
}

@Article{boda2006DynamicRisk,
  author={Kang Boda and Jerzy Filar},
  title={{Time Consistent Dynamic Risk Measures}},
  journal={Mathematical Methods of Operations Research},
  year=2006,
  volume={63},
  number={1},
  pages={169-186},
}

@article{Buerle2011MarkovDP,
  title={Markov Decision Processes with Average-Value-at-Risk criteria},
  author={Nicole B{\"a}uerle and Jonathan Ott},
  journal={Mathematical Methods of Operations Research},
  year={2011},
  volume={74},
  pages={361-379}
}
@article{Boucheron2009ABM,
  title={A Bernstein-Von Mises Theorem for discrete probability distributions},
  author={S. Boucheron and {\'E}. Gassiat},
  journal={Electronic Journal of Statistics},
  year={2009},
  volume={3},
  pages={114-148}
}



@inproceedings{Lim2013RMDP,
author = {Lim, Shiau Hong and Xu, Huan and Mannor, Shie},
title = {Reinforcement Learning in Robust Markov Decision Processes},
year = {2013},
booktitle = {International Conference on Neural Information Processing Systems},
volume={1},
pages = {701–709},
series = {NEURIPS'13}
}



@article{iancu2011TightApprox,
author = {Iancu, Dan and Petrik, Marek and Subramanian, Dharmashankar},
year = {2011},
title = {Tight Approximations of Dynamic Risk Measures},
volume = {40},
journal = {Mathematics of Operations Research},
}

@inproceedings{Tirizoni2018PolicyC,
author = {Tirinzoni, Andrea and Chen, Xiangli and Petrik, Marek and Ziebart, Brian D.},
title = {Policy-Conditioned Uncertainty Sets for Robust {M}arkov Decision Processes},
year = {2018},
booktitle = {International Conference on Neural Information Processing Systems},
pages = {8953–8963}
}


@article{Mannor2016KRect,
author = {Mannor, Shie and Mebel, Ofir and Xu, Huan},
title = {Robust {MDPs} with {K}-Rectangular Uncertainty},
year = {2016},
publisher = {INFORMS},
address = {Linthicum, MD, USA},
volume = {41},
number = {4},
journal = {Mathematics of Operations Research},
pages = {1484–1509},
}

@misc{goyal2020robust, series={BIRS Workshop Lecture Videos}, title={Robust Markov Decision Processes: Beyond Rectangularity}, url={https://open.library.ubc.ca/cIRcle/collections/48630/items/1.0379915}, DOI={http://dx.doi.org/10.14288/1.0379915}, note={Author affiliation: Columbia University}, publisher={Banff International Research Station for Mathematical Innovation and Discovery}, author={Grand Clement, Julien}, year={2019},  collection={BIRS Workshop Lecture Videos}}

@ARTICLE{Rockafellar2000optimizationCVAR,
    author = {R. Tyrrell Rockafellar and Stanislav Uryasev},
    title = {Optimization of Conditional Value-at-Risk},
    journal = {Journal of Risk},
    year = {2000},
    volume = {2},
    pages = {21--41}
}

@inproceedings{Thomas2015HighConfidence,
author = {Thomas, Philip S. and Theocharous, Georgios and Ghavamzadeh, Mohammad},
title = {High Confidence Off-Policy Evaluation},
year = {2015},
abstract = {Many reinforcement learning algorithms use trajectories collected from the execution of one or more policies to propose a new policy. Because execution of a bad policy can be costly or dangerous, techniques for evaluating the performance of the new policy without requiring its execution have been of recent interest in industry. Such off-policy evaluation methods, which estimate the performance of a policy using trajectories collected from the execution of other policies, heretofore have not provided confidences regarding the accuracy of their estimates. In this paper we propose an off-policy method for computing a lower confidence bound on the expected return of a policy.},
booktitle = {AAAI Conference on Artificial Intelligence},
pages = {3000–3006}
}

@article{Ben-Tal2019SoftRobust,
 author = {Aharon Ben-Tal and Dimitris Bertsimas and David B. Brown},
 journal = {Operations Research},
 number = {4},
 pages = {1220--1234},
 publisher = {INFORMS},
 title = {A Soft Robust Model for Optimization Under Ambiguity},
 volume = {58},
 year = {2010}
}



@article{Xu2012DistributionallyRobust,
 author = {Huan Xu and Shie Mannor},
 journal = {Mathematics of Operations Research},
 number = {2},
 pages = {288--300},
 publisher = {INFORMS},
 title = {Distributionally Robust Markov Decision Processes},
 volume = {37},
 year = {2012}
}


@inproceedings{Derman2018Soft-RobustAC,
author = {Derman, Esther and Mankowitz, Daniel and Mann, Timothy A and Mannor, Shie},
booktitle = {Conference on Uncertainty in Artificial Intelligence},
title = {{Soft-Robust Actor-Critic Policy-Gradient}},
year = {2018}
}

@inproceedings{Xu2006Robustnesstradeoff,
author = {Xu, Huan and Mannor, Shie},
booktitle = { International Conference on Neural Information Processing Systems
},
title = {The robustness-performance trade-off in {Markov} decision processes},
year = {2006}
}


@article{Chen2012RobustPO,
author = {Chen, Katherine and Bowling, Michael},
journal = {International Conference on Neural Information Processing Systems},
pages = {2069--2077},
title = {{Tractable objectives for robust policy optimization}},
volume = {3},
year = {2012}
}

@inproceedings{Mankowitz2020RobustRL,
title={Robust Reinforcement Learning for Continuous Control with Model Misspecification},
author={Daniel J. Mankowitz and Nir Levine and Rae Jeong and Abbas Abdolmaleki and Jost Tobias Springenberg and Yuanyuan Shi and Jackie Kay and Todd Hester and Timothy Mann and Martin Riedmiller},
booktitle={International Conference on Learning Representations},
year={2020},
url={https://openreview.net/forum?id=HJgC60EtwB}
}

@inproceedings{Strehllittman2004Exploration,
author = {Strehl, Alexander L and Littman, Michael L},
booktitle = {International Conference of Machine Learning},
title = {{Exploration via Model-based Interval Estimation}},
year = {2004}
}
@inproceedings{tamar15PG,
 author = {Tamar, Aviv and Chow, Yinlam and Ghavamzadeh, Mohammad and Mannor, Shie},
 booktitle = {International Conference on Neural Information Processing Systems},
 pages = {1468--1476},
 title = {Policy Gradient for Coherent Risk Measures},
 volume = {28},
 year = {2015}
}


@inproceedings{Tamar2014Scaling,
    author = {Tamar, Aviv and Mannor, Shie and Xu, Huan},
    title = {Scaling up Robust {MDPs} Using Function Approximation},
    year = {2014},
    booktitle = {Proceedings of the 31st International Conference on Machine Learning},
    pages = {181–189}  }



@phdthesis{Bagnell2004LearningDR,
  title={Learning decisions: robustness, uncertainty, and approximation},
  author={J. Bagnell},
  year={2004},
  school = {CMU}
}

@inproceedings{Strehl2004Exploration,
  title={Exploration via Model-based Interval Estimation},
  author={Strehl, A and Littman, M},
  booktitle={International Conference on Machine Learning (ICML)},
  year={2004},
  organization={Citeseer}
}

@inproceedings{Gottesman2019Parametric,
	author = {Gottesman, Omer and Liu, Yao and Sussex, Scott and Brunskill, Emma and Doshi-Velez, Finale},
	booktitle = {Proceedings of the 36th International Conference on Machine Learning},
	eprint = {1905.05787},
	pages = {4236--4245},
	title = {{Combining parametric and non-parametric models for off-policy evaluation}},
	year = {2019}
}

@book{Gelman2014BayesianData,
	author = {Gelman, Andrew and Carlin, John B and Stern, Hal S and Rubin, Donald B},
	booktitle = {Chapman Texts in Statistical Science Series},
	publisher = {Chapman and Hall/CRC},
	title = {{Bayesian} Data Analysis},
	year = {2014}
}
@book{Seeger2004Gaussian,
	author = {Rasmussen, Carl Edward and Williams, Christopher},
	pmid = {15112367},
	title = {{Gaussian processes for machine learning.}},
	publisher = {Springer},
	year = {2004}
}
@article{calamai1987ProjectGD,
author = {Calamai, Paul H. and Mor\'{e}, Jorge J.},
title = {Projected Gradient Methods for Linearly Constrained Problems},
year = {1987},
publisher = {Springer-Verlag},
volume = {39},
number = {1},
journal = {Math. Program.},
pages = {93–116}
}
@article{Ribba2012ATG,
  title={A Tumor Growth Inhibition Model for Low-Grade Glioma Treated with Chemotherapy or Radiotherapy},
  author={B. Ribba and G. Kaloshi and M. Peyre and D. Ricard and V. Calvez and M. Tod and B. Cajavec-Bernard and A. Idbaih and D. Psimaras and L. Dainese and J. Pallud and S. Cartalat-Carel and J. Delattre and J. Honnorat and E. Grenier and F. Ducray},
  journal={Clinical Cancer Research},
  year={2012},
  volume={18},
  pages={5071 - 5080}
}
@article{Ghavamzadeh2016BayesianPG,
	author = {Ghavamzadeh, Mohammad and Engel, Yaakov and Valko, Michal},
	journal = {Journal of Machine Learning Research},
	pages = {1--53},
	title = {{Bayesian} policy gradient and actor-critic algorithms},
	volume = {17},
	year = {2016}
}

@book{Murphy2012ML,
author = {Murphy, Kevin P.},
title = {Machine Learning: A Probabilistic Perspective},
year = {2012},
publisher = {The MIT Press},
}

@book{Follmer2011StochasticF,
author = {Follmer, Hans and Schied, Alexander},
edition = {3rd},
publisher = {Walter de Gruyter},
title = {{Stochastic Finance: An Introduction in Discrete Time}},
year = {2011}
}


@article{smith2006Optimizer,
author = {Smith, James and Winkler, Robert},
year = {2006},
pages = {311-322},
title = {The Optimizer’s Curse: Skepticism and Postdecision Surprise in Decision Analysis},
volume = {52},
journal = {Management Science},
}

@inproceedings{Xu2009Parametric,
author = {Xu, Huan and Mannor, Shie},
booktitle = {IEEE Conference on Decision and Control},
pages = {3606--3613},
title = {Parametric regret in uncertain {Markov} decision processes},
year = {2009}
}
@article{Xu2012Distributionally,
author = {Xu, Huan and Mannor, Shie},
journal = {Mathematics of Operations Research},
number = {2},
pages = {288--300},
title = {{Distributionally robust {Markov} decision processes}},
volume = {37},
year = {2012}
}

@book{Lange2012BatchRL,
author = {Lange, Sasche and Gabel, Thomas and Riedmiller, Martin},
booktitle = {Reinforcement Learning},
pages = {45--73},
pmid = {25719670},
publisher = {Springer},
title = {{Batch Reinforcement Learning}},
year = {2012}
}

@book{Sutton2018RL,
author = {Sutton, Richard S. and Barto, Andrew G.},
title = {Reinforcement Learning: An Introduction},
year = {2018},
publisher = {A Bradford Book},
}

@article{Kaufman2013RobustMPI,
	author = {Kaufman, David L and Schaefer, Andrew J},
	journal = {INFORMS Journal on Computing},
	number = {3},
	pages = {396--410},
	title = {{Robust modified policy iteration}},
	volume = {25},
	year = {2013}
}


@book{Shapiro2014StochasticProgramming,
author = {Shapiro, Alexander and Dentcheva, Darinka and Ruszczyński, Andrzej},
title = {Lectures on Stochastic Programming: Modeling and Theory, Second Edition},
publisher = {Society for Industrial and Applied Mathematics},
year = {2014},
}



@book{rockafellar1970convex,
 author = {R. TYRRELL Rockafellar},
 publisher = {Princeton University Press},
 title = {Convex Analysis},
 year = {1970}
}

@article{Acerbi2002expected,
	author = {Acerbi, Carlo and Tasche, Dirk},
	eprint = {0105191},
	journal = {Economic Notes},
	number = {2},
	pages = {379--388},
	primaryClass = {cond-mat},
	title = {{Expected shortfall: A natural coherent alternative to value at risk}},
	volume = {31},
	year = {2002}
}

@book{stephen1997interiorptmethods,
author = {Wright, Stephen J.},
title = {Primal-Dual Interior-Point Methods},
year = {1997},
publisher = {Society for Industrial and Applied Mathematics},
}

@book{burden1988NumericalAnalysis,
author = {Burden, Richard L. and Faires, J. Douglas},
title = {Numerical Analysis: 4th Ed},
year = {1988},
publisher = {PWS Publishing Co.},
}


@inproceedings{Schied2006Riskmeasures,
	author = {Schied, Alexander},
	booktitle = {Symposium on Probability and Stochastic Processes},
	title = {{Risk measures and robust optimization problems}},
	year = {2004}
}


@inproceedings{Munos2016SafeRL,
	author = {Munos, R{\'{e}}mi and Stepleton, Tom and Harutyunyan, Anna and Bellemare, Marc G.},
	booktitle = {International Conference on Neural Information Processing Systems
},
	eprint = {1606.02647},
	title = {{Safe and Efficient Off-Policy Reinforcement Learning}},
	year = {2016}
}
@InProceedings{pmlr-v48-jiang16, title = {Doubly Robust Off-policy Value Evaluation for Reinforcement Learning}, author = {Nan Jiang and Lihong Li}, booktitle = {Proceedings of The 33rd International Conference on Machine Learning}, pages = {652--661}, year = {2016}, volume = {48}, series = {Proceedings of Machine Learning Research},  publisher = {PMLR}}

@book{Bertsekas2018AbstractDP,
  title={Abstract Dynamic Programming},
  author={D. Bertsekas},
  year={2018},
  publisher={Athena Scientific},
}
@inproceedings{Brown2018EfficientPPB,
	author = {Brown, Daniel S. and Niekum, Scott},
	booktitle = {Thirty-Second AAAI Conference on Artificial Intelligence},
	eprint = {1707.00724},
	pages = {2754--2762},
	title = {{Efficient Probabilistic Performance Bounds for Inverse Reinforcement Learning}},
	year = {2018}
}


@book{Bertsekas1982,
author = {Bertsekas, Dimitri P},
title = {{Constrained Optimization and {Lagrange} Multiplier Methods}},
year = {1982},
publisher={Athena Scientific}
}






@article{Shapiro2013Risk,
author = {Shapiro, A. and Tekaya, W. and da Costa, J. P. and Soares, M. P.},
journal = {European Journal of Operational Research},
number = {2},
pages = {375--391},
title = {{Risk-neutral and risk-averse stochastic dual dynamic programming method}},
year = {2013}
}

@inproceedings{Hiraoka2019LearningRO,
  title={Learning Robust Options by {Conditional Value at Risk} Optimization},
  author={T. Hiraoka and T. Imagawa and T. Mori and T. Onishi and Yoshimasa Tsuruoka},
  booktitle={International Conference on Neural Information Processing Systems},
  year={2019}
}

@book{Kery2012Bayesian,
	author = {Kery, Marc and Schaub, Michael},
	publisher = {Elsevier},
	title = {{Bayesian Population Analysis Using WinBUGS}},
	year = {2012}
}

@article{Yang2017ConvexOpt,
    author = {Yang, Insoon},
    journal = {IEEE Control Systems Letters},
    number = {1},
    pages = {164--169},
    title = {{A Convex Optimization Approach to Distributionally Robust {Markov} Decision Processes With {Wasserstein} Distance}},
    volume = {1},
    year = {2017}
}

@InProceedings{Grand-Clement2020b,
  title = 	 {First-Order Methods for Wasserstein Distributionally Robust MDP},
  author =       {Clement, Julien Grand and Kroer, Christian},
  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
  pages = 	 {2010--2019},
  year = 	 {2021},
  editor = 	 {Meila, Marina and Zhang, Tong},
  volume = 	 {139},
  series = 	 {Proceedings of Machine Learning Research},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v139/clement21a/clement21a.pdf},
  url = 	 {https://proceedings.mlr.press/v139/clement21a.html},
  abstract = 	 {Markov decision processes (MDPs) are known to be sensitive to parameter specification. Distributionally robust MDPs alleviate this issue by allowing for \textit{ambiguity sets} which give a set of possible distributions over parameter sets. The goal is to find an optimal policy with respect to the worst-case parameter distribution. We propose a framework for solving Distributionally robust MDPs via first-order methods, and instantiate it for several types of Wasserstein ambiguity sets. By developing efficient proximal updates, our algorithms achieve a convergence rate of $O\left(NA^{2.5}S^{3.5}\log(S)\log(\epsilon^{-1})\epsilon^{-1.5} \right)$ for the number of kernels $N$ in the support of the nominal distribution, states $S$, and actions $A$; this rate varies slightly based on the Wasserstein setup. Our dependence on $N,A$ and $S$ is significantly better than existing methods, which have a complexity of $O\left(N^{3.5}A^{3.5}S^{4.5}\log^{2}(\epsilon^{-1}) \right)$. Numerical experiments show that our algorithm is significantly more scalable than state-of-the-art approaches across several domains.}
}

@article{Ho2020PartialPI,
  author  = {Chin Pang Ho and Marek Petrik and Wolfram Wiesemann},
  title   = {Partial Policy Iteration for L1-Robust Markov Decision Processes},
  journal = {Journal of Machine Learning Research},
  year    = {2021},
  volume  = {22},
  number  = {275},
  pages   = {1-46},
  url     = {http://jmlr.org/papers/v22/20-445.html}
}

@article{Iancu2014Pareto,
    author = {Iancu, Dan A. and Trichakis, Nikolaos},
    journal = {Management Science},
    number = {1},
    pages = {130--147},
    pmid = {28028460},
    title = {{Pareto Efficiency in Robust Optimization}},
    volume = {60},
    year = {2014}
}




















































@article{Chow18RC,
    author = {Y. Chow and M. Ghavamzadeh and L. Janson and M. Pavone},
    title = {Risk-Constrained Reinforcement Learning with Percentile Risk Criteria},
    journal = {Journal of Machine Learning Research},
    volume = {18},
    number = {167},
    pages = {1-51},
    year = {2018}
}


@inproceedings{vanhasselt2015deep,
    title = {Deep {{Reinforcement Learning}} with {{Double Q}}-Learning},
    booktitle = {{{AAAI Conference}} on {{Artificial Intelligence}}},
    author = {{van Hasselt}, Hado and Guez, Arthur and Silver, David},
    year = {2015}
}


@book{powell2007Approx,
author = {Powell, Warren B.},
title = {Approximate Dynamic Programming: Solving the Curses of Dimensionality (Wiley Series in Probability and Statistics)},
year = {2007},
publisher = {Wiley-Interscience},
}




@inproceedings{Buckman2021Pessimism,
    title = {The Importance of Pessimism in Fixed-Dataset Policy Optimization},
    booktitle = {International {{Conference}} on {{Learning Representations}}},
    author = {Buckman, Jacob and Gelada, Carles and Bellemare, Marc G.},
    year = {2021}
}


@book{Powell2011Approx,
    title = {Approximate Dynamic Programming: Solving the Curses of Dimensionality},
    author = {Powell, Warren B.},
    year = {2011},
    publisher = {{Princeton University Press}}
}


@phdthesis{Petrik2010a,
    title = {Optimization-Based {{Approximate Dynamic Programming}}},
    author = {Petrik, Marek},
    year = {2010},
    school = {University of Massachusetts Amherst}
}


@inproceedings{Petrik2016,
    title = {Interpretable {{Policies}} for {{Dynamic Product Recommendations}}},
    booktitle = {Uncertainty in {{Artificial Intelligence}} ({{UAI}})},
    author = {Petrik, Marek and Luss, Ronny},
    year = {2016}
}


@inproceedings{Kallus2020,
    title = {Confounding-Robust Policy Evaluation in Infinite-Horizon Reinforcement Learning},
    booktitle = {Advances in {{Neural Information Processing Systems}} ({{NeurIPS}})},
    author = {Kallus, Nathan and Zhou, Angela},
    year = {2020}
}
@inproceedings{Mahmood2014weighted,
 author = {Mahmood, A. Rupam and van Hasselt, Hado P and Sutton, Richard S},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {Z. Ghahramani and M. Welling and C. Cortes and N. Lawrence and K. Q. Weinberger},
 publisher = {Curran Associates, Inc.},
 title = {Weighted importance sampling for off-policy learning with linear function approximation},
 volume = {27},
 year = {2014}
}



@article{Koh2018Stronger,
  author    = {Pang Wei Koh and
               Jacob Steinhardt and
               Percy Liang},
  title     = {Stronger Data Poisoning Attacks Break Data Sanitization Defenses},
  journal   = {CoRR},
  volume    = {abs/1811.00741},
  year      = {2018},
  eprinttype = {arXiv},
  eprint    = {1811.00741}
}

@inproceedings{koh2017influence,
author = {Koh, Pang Wei and Liang, Percy},
title = {Understanding Black-Box Predictions via Influence Functions},
year = {2017},
booktitle = {International Conference on Machine Learning},
pages = {1885–1894}
}

@article{Wiesemann2013Pessimistic,
author = {Wiesemann, Wolfram and Tsoukalas, Angelos and Kleniati, Polyxeni-Margarita and Rustem, Berç},
title = {Pessimistic Bilevel Optimization},
journal = {SIAM Journal on Optimization},
volume = {23},
number = {1},
pages = {353-380},
year = {2013}
}


@InProceedings{jiang16doublyrobust, title = {Doubly Robust Off-policy Value Evaluation for Reinforcement Learning}, author = {Nan Jiang and Lihong Li}, booktitle = {International Conference on Machine Learning}, pages = {652--661}, year = {2016}}

@InProceedings{thomas15policyimprovement, 
title = {High Confidence Policy Improvement}, 
author = {Thomas, Philip and Theocharous, Georgios and Ghavamzadeh, Mohammad}, booktitle = {International Conference on Machine Learning}, pages = {2380--2388}, 
year = {2015}, 
series = {Proceedings of Machine Learning Research}, 
address = {Lille, France}, publisher = {PMLR}}





@InProceedings{hanna2019importancesampling,
  title = 	 {Importance Sampling Policy Evaluation with an Estimated Behavior Policy},
  author =       {Hanna, Josiah and Niekum, Scott and Stone, Peter},
  booktitle = 	 {International Conference on Machine Learning},
  pages = 	 {2605--2613},
  year = 	 {2019},
  pdf = 	 {http://proceedings.mlr.press/v97/hanna19a/hanna19a.pdf},
  abstract = 	 {We consider the problem of off-policy evaluation in Markov decision processes. Off-policy evaluation is the task of evaluating the expected return of one policy with data generated by a different, behavior policy. Importance sampling is a technique for off-policy evaluation that re-weights off-policy returns to account for differences in the likelihood of the returns between the two policies. In this paper, we study importance sampling with an estimated behavior policy where the behavior policy estimate comes from the same set of data used to compute the importance sampling estimate. We find that this estimator often lowers the mean squared error of off-policy evaluation compared to importance sampling with the true behavior policy or using a behavior policy that is estimated from a separate data set. Intuitively, estimating the behavior policy in this way corrects for error due to sampling in the action-space. Our empirical results also extend to other popular variants of importance sampling and show that estimating a non-Markovian behavior policy can further lower large-sample mean squared error even when the true behavior policy is Markovian.}
}

@article{hanna2018bootstrapping, title={Bootstrapping with Models: Confidence Intervals for Off-Policy Evaluation}, volume={31}, url={https://ojs.aaai.org/index.php/AAAI/article/view/11123}, abstractNote={ &lt;p&gt; In many reinforcement learning applications, it is desirable to determine confidence interval lower bounds on the performance of any given policy without executing said policy. In this context, we propose two bootstrapping off-policy evaluation methods which use learned MDP transition models in order to estimate lower confidence bounds on policy performance with limited data. We empirically evaluate the proposed methods in a standard policy evaluation tasks. &lt;/p&gt; }, number={1}, journal={Proceedings of the AAAI Conference on Artificial Intelligence}, author={Hanna, Josiah and Stone, Peter and Niekum, Scott}, year={2017}}

misc{gottesman2018evaluating,
      title={Evaluating Reinforcement Learning Algorithms in Observational Health Settings}, 
      author={Omer Gottesman and Fredrik Johansson and Joshua Meier and Jack Dent and Donghun Lee and Srivatsan Srinivasan and Linying Zhang and Yi Ding and David Wihl and Xuefeng Peng and Jiayu Yao and Isaac Lage and Christopher Mosch and Li-wei H. Lehman and Matthieu Komorowski and Matthieu Komorowski and Aldo Faisal and Leo Anthony Celi and David Sontag and Finale Doshi-Velez},
      year={2018},
      eprint={1805.12298},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{voloshin2020empirical,
      title={Empirical Study of Off-Policy Policy Evaluation for Reinforcement Learning}, 
      author={Cameron Voloshin and Hoang M. Le and Nan Jiang and Yisong Yue},
      year={2020},
      eprint={1911.06854},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@inproceedings{thomas2016dataefficient,
author = {Thomas, Philip S. and Brunskill, Emma},
title = {Data-Efficient off-Policy Policy Evaluation for Reinforcement Learning},
year = {2016},
publisher = {JMLR.org},
abstract = {In this paper we present a new way of predicting the performance of a reinforcement learning policy given historical data that may have been generated by a different policy. The ability to evaluate a policy from historical data is important for applications where the deployment of a bad policy can be dangerous or costly. We show empirically that our algorithm produces estimates that often have orders of magnitude lower mean squared error than existing methods--it makes more efficient use of the available data. Our new estimator is based on two advances: an extension of the doubly robust estimator (Jiang &amp; Li, 2015), and a new way to mix between model based and importance sampling based estimates.},
booktitle = {Proceedings of the 33rd International Conference on International Conference on Machine Learning - Volume 48},
pages = {2139–2148},
numpages = {10},
location = {New York, NY, USA},
series = {ICML'16}
}


@InProceedings{gottesman2020interpretable,
  title = 	 {Interpretable Off-Policy Evaluation in Reinforcement Learning by Highlighting Influential Transitions},
  author =       {Gottesman, Omer and Futoma, Joseph and Liu, Yao and Parbhoo, Sonali and Celi, Leo and Brunskill, Emma and Doshi-Velez, Finale},
  booktitle = 	 {International Conference on Machine Learning},
  pages = 	 {3658--3667},
  year = 	 {2020},
  abstract = 	 {Off-policy evaluation in reinforcement learning offers the chance of using observational data to improve future outcomes in domains such as healthcare and education, but safe deployment in high stakes settings requires ways of assessing its validity. Traditional measures such as confidence intervals may be insufficient due to noise, limited data and confounding. In this paper we develop a method that could serve as a hybrid human-AI system, to enable human experts to analyze the validity of policy evaluation estimates. This is accomplished by highlighting observations in the data whose removal will have a large effect on the OPE estimate, and formulating a set of rules for choosing which ones to present to domain experts for validation. We develop methods to compute exactly the influence functions for fitted Q-evaluation with two different function classes: kernel-based and linear least squares, as well as importance sampling methods. Experiments on medical simulations and real-world intensive care unit data demonstrate that our method can be used to identify limitations in the evaluation process and make evaluation more robust.}
}

@Article{kahn1953Methods,
  author={H. Kahn and A. W. Marshall},
  title={{Methods of Reducing Sample Size in Monte Carlo Computations}},
  journal={Operations Research},
  year=1953,
  volume={1},
  number={5}
  }
@book{rubenstein1981MC,
author = {Rubinstein, Reuven Y.},
title = {Simulation and the Monte Carlo Method},
year = {1981},
publisher = {John Wiley \& Sons, Inc.},
address = {USA},
edition = {1st}
}

@inproceedings{Zhang2020Adaptive,
  title = {Adaptive Reward-Poisoning Attacks against Reinforcement Learning},
  author = {Xuezhou Zhang and Yuzhe Ma and Adish Singla and Xiaojin Zhu},
  year = {2020},
  pages = {11225-11234},
  booktitle = {International Conference on Machine Learning}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


@misc{lobo2021softrobust,
      title={Soft-Robust Algorithms for Batch Reinforcement Learning}, 
      author={Elita A. Lobo and Mohammad Ghavamzadeh and Marek Petrik},
      year={2021},
      eprint={2011.14495},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@inbook{ma2019policy,
author = {Ma, Yuzhe and Zhang, Xuezhou and Sun, Wen and Zhu, Xiaojin},
title = {Policy Poisoning in Batch Reinforcement Learning and Control},
year = {2019},
abstract = {We study a security threat to batch reinforcement learning and control where the attacker aims to poison the learned policy. The victim is a reinforcement learner / controller which first estimates the dynamics and the rewards from a batch data set, and then solves for the optimal policy with respect to the estimates. The attacker can modify the data set slightly before learning happens, and wants to force the learner into learning a target policy chosen by the attacker. We present a unified framework for solving batch policy poisoning attacks, and instantiate the attack on two standard victims: tabular certainty equivalence learner in reinforcement learning and linear quadratic regulator in control. We show that both instantiation result in a convex optimization problem on which global optimality is guaranteed, and provide analysis on attack feasibility and attack cost. Experiments show the effectiveness of policy poisoning attacks.},
booktitle = {Proceedings of the 33rd International Conference on Neural Information Processing Systems}
}

@misc{koh2021stronger,
      title={Stronger Data Poisoning Attacks Break Data Sanitization Defenses}, 
      author={Pang Wei Koh and Jacob Steinhardt and Percy Liang},
      year={2021},
      eprint={1811.00741},
      archivePrefix={arXiv},
      primaryClass={stat.ML}
}

@InProceedings{cohen2020detecting,
author = {Cohen, Gilad and Sapiro, Guillermo and Giryes, Raja},
title = {Detecting Adversarial Samples Using Influence Functions and Nearest Neighbors},
booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year = {2020}
}

@inbook{fang2020influence,
author = {Fang, Minghong and Gong, Neil Zhenqiang and Liu, Jia},
title = {Influence Function Based Data Poisoning Attacks to Top-N Recommender Systems},
year = {2020},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
abstract = {Recommender system is an essential component of web services to engage users. Popular recommender systems model user preferences and item properties using a large amount of crowdsourced user-item interaction data, e.g., rating scores; then top-N items that match the best with a user’s preference are recommended to the user. In this work, we show that an attacker can launch a data poisoning attack to a recommender system to make recommendations as the attacker desires via injecting fake users with carefully crafted user-item interaction data. Specifically, an attacker can trick a recommender system to recommend a target item to as many normal users as possible. We focus on matrix factorization based recommender systems because they have been widely deployed in industry. Given the number of fake users the attacker can inject, we formulate the crafting of rating scores for the fake users as an optimization problem. However, this optimization problem is challenging to solve as it is a non-convex integer programming problem. To address the challenge, we develop several techniques to approximately solve the optimization problem. For instance, we leverage influence function to select a subset of normal users who are influential to the recommendations and solve our formulated optimization problem based on these influential users. Our results show that our attacks are effective and outperform existing methods. },
booktitle = {Proceedings of The Web Conference},
pages = {3019–3025},
numpages = {7}
}

@misc{zhang2021corruptionrobust,
      title={Corruption-Robust Offline Reinforcement Learning}, 
      author={Xuezhou Zhang and Yiding Chen and Jerry Zhu and Wen Sun},
      year={2021},
      eprint={2106.06630},
      archivePrefix={arXiv},
}

@InProceedings{zhang2019online,
  title = 	 {Online Data Poisoning Attacks},
  author =       {Zhang, Xuezhou and Zhu, Xiaojin and Lessard, Laurent},
  booktitle = 	 {Conference on Learning for Dynamics and Control},
  pages = 	 {201--210},
  year = 	 {2020},
}

@inproceedings{steinhardt2017certified,
 author = {Steinhardt, Jacob and Koh, Pang Wei W and Liang, Percy S},
 booktitle = {Advances in Neural Information Processing Systems},
 title = {Certified Defenses for Data Poisoning Attacks},
 volume = {30},
 year = {2017}
}

@inproceedings{gleave2021adversarial,
title={Adversarial Policies: Attacking Deep Reinforcement Learning},
author={Adam Gleave and Michael Dennis and Cody Wild and Neel Kant and Sergey Levine and Stuart Russell},
booktitle={International Conference on Learning Representations},
year={2020}
}

@article{Chen2019AdversarialAA,
  title={Adversarial attack and defense in reinforcement learning-from AI security view},
  author={Tong Chen and Jiqiang Liu and Yingxiao Xiang and Wenjia Niu and Endong Tong and Zhen Han},
  journal={Cybersecurity},
  year={2019},
  volume={2},
  pages={1-22}
}

@inproceedings{lin2019tactics,
  author    = {Yen-Chen Lin and Zhang-Wei Hong and Yuan-Hong Liao and Meng-Li Shih and Ming-Yu Liu and Min Sun},
  title     = {Tactics of Adversarial Attack on Deep Reinforcement Learning Agents},
  booktitle = {International Joint Conference on Artificial Intelligence},
  pages     = {3756--3762},
  year      = {2017}
}

@inbook{Wu2021AttackInfluence,
author = {Wu, Chenwang and Lian, Defu and Ge, Yong and Zhu, Zhihao and Chen, Enhong},
title = {Triple Adversarial Learning for Influence Based Poisoning Attack in Recommender Systems},
year = {2021},
publisher = {Association for Computing Machinery},
booktitle = {Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining},
pages = {1830–1840},
}


@inproceedings{Farahmand2008BRM,
author = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv\'{a}ri, Csaba and Mannor, Shie},
title = {Regularized Policy Iteration},
year = {2008},
booktitle = {International Conference on Neural Information Processing Systems},
pages = {441–448},
location = {Vancouver, British Columbia, Canada},
}
@inproceedings{Rubinstein1981SimulationAT,
  title={Simulation and the Monte Carlo method},
  author={Reuven Y. Rubinstein},
  booktitle={Wiley series in probability and mathematical statistics},
  year={1981}
}
@MISC{Precup00temporalabstraction,
    author = {Doina Precup},
    title = {Temporal Abstraction in Reinforcement Learning},
    year = {2000}
}
@MISC{Thomas2015SafeRL,
    author = {Philip Thomas},
    title = {Safe Reinforcement Learning},
    year = {2015}
}
@article{powell1966wis,
    author = {Powell, M. J. D. and Swann, J.},
    title = "Weighted Uniform Sampling: A {Monte Carlo} technique for Reducing Variance",
    journal = {IMA Journal of Applied Mathematics},
    volume = {2},
    number = {3},
    pages = {228-236},
    year = {1966}
}
    
    @misc{brockman2016openai,
      title={OpenAI Gym}, 
      author={Greg Brockman and Vicki Cheung and Ludwig Pettersson and Jonas Schneider and John Schulman and Jie Tang and Wojciech Zaremba},
      year={2016},
      eprint={1606.01540},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}
    
@incollection{torch2019,
title = {PyTorch: An Imperative Style, High-Performance Deep Learning Library},
author = {Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and Desmaison, Alban and Kopf, Andreas and Yang, Edward and DeVito, Zachary and Raison, Martin and Tejani, Alykhan and Chilamkurthy, Sasank and Steiner, Benoit and Fang, Lu and Bai, Junjie and Chintala, Soumith},
booktitle = {Advances in Neural Information Processing Systems 32},
editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
pages = {8024--8035},
year = {2019}
}

@misc{tensorflow2015-whitepaper,
title={ {TensorFlow}: Large-Scale Machine Learning on Heterogeneous Systems},
note={Software available from tensorflow.org},
author={
    Mart\'{i}n~Abadi and
    Ashish~Agarwal and
    Paul~Barham and
    Eugene~Brevdo and
    Zhifeng~Chen and
    Craig~Citro and
    Greg~S.~Corrado and
    Andy~Davis and
    Jeffrey~Dean and
    Matthieu~Devin and
    Sanjay~Ghemawat and
    Ian~Goodfellow and
    Andrew~Harp and
    Geoffrey~Irving and
    Michael~Isard and
    Yangqing Jia and
    Rafal~Jozefowicz and
    Lukasz~Kaiser and
    Manjunath~Kudlur and
    Josh~Levenberg and
    Dandelion~Man\'{e} and
    Rajat~Monga and
    Sherry~Moore and
    Derek~Murray and
    Chris~Olah and
    Mike~Schuster and
    Jonathon~Shlens and
    Benoit~Steiner and
    Ilya~Sutskever and
    Kunal~Talwar and
    Paul~Tucker and
    Vincent~Vanhoucke and
    Vijay~Vasudevan and
    Fernanda~Vi\'{e}gas and
    Oriol~Vinyals and
    Pete~Warden and
    Martin~Wattenberg and
    Martin~Wicke and
    Yuan~Yu and
    Xiaoqiang~Zheng},
  year={2015},
}
@article{cook1980influence,
 journal = {Technometrics},
 number = {4},
 pages = {495--508},
 publisher = {[Taylor & Francis, Ltd., American Statistical Association, American Society for Quality]},
 title = {Characterizations of an Empirical Influence Function for Detecting Influential Cases in Regression},
 volume = {22},
 year = {1980},
 author = {R. Dennis Cook and Sanford Weisberg }
}

@inproceedings{koh2019group,
 author = {Koh, Pang Wei W and Ang, Kai-Siang and Teo, Hubert and Liang, Percy S},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {On the Accuracy of Influence Functions for Measuring Group Effects},
 url = {https://proceedings.neurips.cc/paper/2019/file/a78482ce76496fcf49085f2190e675b4-Paper.pdf},
 volume = {32},
 year = {2019}
}

@inproceedings{Basu2020SecondOrderGI,
  title={Second-Order Group Influence Functions for Black-Box Predictions},
  author={Samyadeep Basu and Xuchen You and Soheil Feizi},
  booktitle={ICML},
  year={2020}
}
@article{hampel1974influence,
 author = {Frank R. Hampel},
 journal = {Journal of the American Statistical Association},
 number = {346},
 pages = {383--393},
 publisher = {[American Statistical Association, Taylor & Francis, Ltd.]},
 title = {The Influence Curve and Its Role in Robust Estimation},
 volume = {69},
 year = {1974}
}
@INPROCEEDINGS{Munos02variableresolution,
    author = {Rémi Munos and Andrew Moore},
    title = {Variable resolution discretization in optimal control},
    booktitle = {Machine Learning},
    year = {2002},
    pages = {291--323},
    publisher = {}
}
@inproceedings{Ting2018optimalsubsampling,
 author = {Ting, Daniel and Brochu, Eric},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
 pages = {},
 title = {Optimal Subsampling with Influence Functions},
 volume = {31},
 year = {2018}
}

@INPROCEEDINGS{Ng99policyinvariance,
    author = {Andrew Y. Ng and Daishi Harada and Stuart Russell},
    title = {Policy invariance under reward transformations: Theory and application to reward shaping},
    booktitle = {International Conference on Machine Learning},
    year = {1999},
    pages = {278--287}
}

@inproceedings{asmuth2008RewardShaping,
author = {Asmuth, John and Littman, Michael L. and Zinkov, Robert},
title = {Potential-Based Shaping in Model-Based Reinforcement Learning},
year = {2008},
booktitle = {National Conference on Artificial Intelligence (AAAI)},
pages = {604–609},
}

@inproceedings{Zhang2008PolicyTeaching,
author = {Zhang, Haoqi and Parkes, David},
title = {Value-Based Policy Teaching with Active Indirect Elicitation},
year = {2008},
booktitle = {National Conference on Artificial Intelligence}
}

@inproceedings{Zhang2009policyteaching,
author = {Zhang, Haoqi and Parkes, David C. and Chen, Yiling},
title = {Policy Teaching through Reward Function Learning},
year = {2009},
booktitle = {ACM Conference on Electronic Commerce}
}

@inproceedings{Kiourti2019TrojDRLTA,
author = {Kiourti, Panagiota and Wardega, Kacper and Jha, Susmit and Li, Wenchao},
title = {TrojDRL: Evaluation of Backdoor Attacks on Deep Reinforcement Learning},
year = {2020},
articleno = {31}
}

@misc{kos2017delving,
      title={Delving into adversarial attacks on deep policies}, 
      author={Jernej Kos and Dawn Song},
      year={2017},
      eprint={1705.06452},
      archivePrefix={arXiv},
      primaryClass={stat.ML}
}
@inproceedings{Behzadan2017VulnerabilityOD,
  title={Vulnerability of Deep Reinforcement Learning to Policy Induction Attacks},
  author={Vahid Behzadan and Arslan Munir},
  booktitle={International Conference on Machine Learning and Data Mining},
  year={2017}
}

@inproceedings {Wu2021Adversarial,
author = {Xian Wu and Wenbo Guo and Hua Wei and Xinyu Xing},
title = {Adversarial Policy Training against Deep Reinforcement Learning},
booktitle = {USENIX Security Symposium},
year = {2021}
}

@misc{huang2017adversarial,
      title={Adversarial Attacks on Neural Network Policies}, 
      author={Sandy Huang and Nicolas Papernot and Ian Goodfellow and Yan Duan and Pieter Abbeel},
      year={2017},
      eprint={1702.02284},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@inproceedings{Jun2018AdversarialAO,
author = {Jun, Kwang-Sung and Li, Lihong and Ma, Yuzhe and Zhu, Xiaojin},
title = {Adversarial Attacks on Stochastic Bandits},
year = {2018},
booktitle = {International Conference on Neural Information Processing Systems}
}

@inproceedings{LiuS19a,
  title = {Data Poisoning Attacks on Stochastic Bandits},
  author = {Fang Liu and Ness B. Shroff},
  year = {2019},
  booktitle = {International Conference on Machine Learning},
  editor = {Kamalika Chaudhuri and Ruslan Salakhutdinov}
}

@inproceedings{ma2018data,
  title={Data poisoning attacks in contextual bandits},
  author={Ma, Yuzhe and Jun, Kwang-Sung and Li, Lihong and Zhu, Xiaojin},
  booktitle={International Conference on Decision and Game Theory for Security},
  pages={186--204},
  year={2018},
  organization={Springer}
}

@INPROCEEDINGS{Calafiore1998samplinguniform,
  author={Calafiore, G. and Dabbene, F. and Tempo, R.},
  booktitle={IEEE Conference on Decision and Control}, 
  title={Uniform sample generation in lp balls for probabilistic robustness analysis}, 
  year={1998},
  volume={3}}
  
@book{boyd2004convex, 
  title={Convex optimization}, 
  author={Boyd, Stephen and Vandenberghe, Lieven}, 
  year={2004}, 
  publisher={Cambridge university press} 
}
@article{agarwal2021deep,
  title={Deep Reinforcement Learning at the Edge of the Statistical Precipice},
  author={Agarwal, Rishabh and Schwarzer, Max and Castro, Pablo Samuel
          and Courville, Aaron and Bellemare, Marc G},
  journal={Advances in Neural Information Processing Systems},
  year={2021}
}
@article{Lagoudakis2003LSPI,
author = {Lagoudakis, Michail G. and Parr, Ronald},
title = {Least-Squares Policy Iteration},
year = {2003},
publisher = {JMLR.org},
volume = {4},
journal = {Journal of Machine Learning Research},
month = {dec},
pages = {1107–1149},
numpages = {43}
}

@article{Liu2008IsolationF,
  title={Isolation Forest},
  author={Fei Tony Liu and Kai Ming Ting and Zhi-Hua Zhou},
  journal={2008 Eighth IEEE International Conference on Data Mining},
  year={2008},
  pages={413-422}
}



@inproceedings{Breunig2000LOF,
author = {Breunig, Markus M. and Kriegel, Hans-Peter and Ng, Raymond T. and Sander, J\"{o}rg},
title = {LOF: Identifying Density-Based Local Outliers},
year = {2000},
publisher = {Association for Computing Machinery},
pages = {93–104},
numpages = {12},
series = {SIGMOD '00}
}

@inproceedings{Emmott2013Systematic,
author = {Emmott, Andrew F. and Das, Shubhomoy and Dietterich, Thomas and Fern, Alan and Wong, Weng-Keen},
title = {Systematic Construction of Anomaly Detection Benchmarks from Real Data},
year = {2013},
publisher = {Association for Computing Machinery},
pages = {16–21},
numpages = {6},
series = {ODD '13}
}

@INPROCEEDINGS{Ernst2006Clinical,
  author={Ernst, Damien and Stan, Guy-Bart and Goncalves, Jorge and Wehenkel, Louis},
  booktitle={Proceedings of the 45th IEEE Conference on Decision and Control}, 
  title={Clinical data based optimal STI strategies for HIV: a reinforcement learning approach}, 
  year={2006},
  volume={},
  number={},
  pages={667-672}}

@article{Yu2021Healthcare,
author = {Yu, Chao and Liu, Jiming and Nemati, Shamim and Yin, Guosheng},
title = {Reinforcement Learning in Healthcare: A Survey},
year = {2021},
issue_date = {January 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {55},
number = {1},
journal = {ACM Computing Survey},
articleno = {5},
numpages = {36}
}

@inproceedings{wu2022copa,
title={{COPA}: Certifying Robust Policies for Offline Reinforcement Learning against Poisoning Attacks},
author={Fan Wu and Linyi Li and Huan Zhang and Bhavya Kailkhura and Krishnaram Kenthapadi and Ding Zhao and Bo Li},
booktitle={International Conference on Learning Representations},
year={2022},
url={https://openreview.net/forum?id=psh0oeMSBiF}
}

@inproceedings{GoodfellowSS2015Explaining,
  author    = {Ian J. Goodfellow and
               Jonathon Shlens and
               Christian Szegedy},
  editor    = {Yoshua Bengio and
               Yann LeCun},
  title     = {Explaining and Harnessing Adversarial Examples},
  booktitle = {3rd International Conference on Learning Representations, {ICLR} 2015,
               San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings},
  year      = {2015},
  timestamp = {25 Jul 2019}
}

@inproceedings{Basu2020SecondOrderGI,
  title={Second-Order Group Influence Functions for Black-Box Predictions},
  author={Samyadeep Basu and Xuchen You and Soheil Feizi},
  booktitle={International {{Conference}} on {{Machine Learning}} ({{ICML}})},
  year={2020}
}


@InProceedings{Jin2020Provably,
  title = 	 {Provably efficient reinforcement learning with linear function approximation},
  author =       {Jin, Chi and Yang, Zhuoran and Wang, Zhaoran and Jordan, Michael I},
  booktitle = 	 {Proceedings of Thirty Third Conference on Learning Theory},
  pages = 	 {2137--2143},
  year = 	 {2020},
  editor = 	 {Abernethy, Jacob and Agarwal, Shivani},
  volume = 	 {125},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {Jul},
  publisher =    {PMLR} }
}

