@article{shalev2010learnability,
  title={Learnability, stability and uniform convergence},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Srebro, Nathan and Sridharan, Karthik},
  journal={The Journal of Machine Learning Research},
  volume={11},
  pages={2635--2670},
  year={2010},
  publisher={JMLR. org}
}


@article{li2022settling,
  title={Settling the sample complexity of model-based offline reinforcement learning},
  author={Li, Gen and Shi, Laixi and Chen, Yuxin and Chi, Yuejie and Wei, Yuting},
  journal={arXiv preprint arXiv:2204.05275},
  year={2022}
}


@article{kallus2020double,
  title={Double Reinforcement Learning for Efficient Off-Policy Evaluation in Markov Decision Processes.},
  author={Kallus, Nathan and Uehara, Masatoshi},
  journal={J. Mach. Learn. Res.},
  volume={21},
  number={167},
  pages={1--63},
  year={2020}
}

@article{yu2013boundedness,
  title={On boundedness of Q-learning iterates for stochastic shortest path problems},
  author={Yu, Huizhen and Bertsekas, Dimitri P},
  journal={Mathematics of Operations Research},
  volume={38},
  number={2},
  pages={209--227},
  year={2013},
  publisher={INFORMS}
}

@article{bertsekas1991analysis,
  title={An analysis of stochastic shortest path problems},
  author={Bertsekas, Dimitri P and Tsitsiklis, John N},
  journal={Mathematics of Operations Research},
  volume={16},
  number={3},
  pages={580--595},
  year={1991},
  publisher={INFORMS}
}

@inproceedings{chen2021finding,
  title={Finding the stochastic shortest path with low regret: The adversarial cost and unknown transition case},
  author={Chen, Liyu and Luo, Haipeng},
  booktitle={International Conference on Machine Learning},
  pages={1651--1660},
  year={2021},
  organization={PMLR}
}

@article{akkaya2019solving,
  title={Solving rubik's cube with a robot hand},
  author={Akkaya, Ilge and Andrychowicz, Marcin and Chociej, Maciek and Litwin, Mateusz and McGrew, Bob and Petron, Arthur and Paino, Alex and Plappert, Matthias and Powell, Glenn and Ribas, Raphael and others},
  journal={arXiv preprint arXiv:1910.07113},
  year={2019}
}

@article{chen2021improved,
  title={Improved No-Regret Algorithms for Stochastic Shortest Path with Linear MDP},
  author={Chen, Liyu and Jain, Rahul and Luo, Haipeng},
  journal={arXiv preprint arXiv:2112.09859},
  year={2021}
}



@article{cohen2021minimax,
  title={Minimax regret for stochastic shortest path},
  author={Cohen, Alon and Efroni, Yonathan and Mansour, Yishay and Rosenberg, Aviv},
  journal={Advances in neural information processing systems},
  year={2021}
}

@inproceedings{tarbouriech2020no,
  title={No-regret exploration in goal-oriented reinforcement learning},
  author={Tarbouriech, Jean and Garcelon, Evrard and Valko, Michal and Pirotta, Matteo and Lazaric, Alessandro},
  booktitle={International Conference on Machine Learning},
  pages={9428--9437},
  year={2020},
  organization={PMLR}
}

@article{tarbouriech2021stochastic,
  title={Stochastic shortest path: Minimax, parameter-free and towards horizon-free regret},
  author={Tarbouriech, Jean and Zhou, Runlong and Du, Simon S and Pirotta, Matteo and Valko, Michal and Lazaric, Alessandro},
  journal={Advances in neural information processing systems},
  year={2021}
}

@article{han,
  title={MULTIPLE HYPOTHESIS TESTING: TREE, FANO AND ASSOUAD},
  author={Han, Yanjun and Fischer-Hwang, Irena},
  url={https://theinformaticists.com/2019/09/16/lecture-8-multiple-hypothesis-testing-tree-fano-and-assoaud/},
  year={2019}
}

@article{wiki,
  title={Sums of independent bounded random variables},
  url={https://en.wikipedia.org/wiki/Chernoff_bound#The_first_step_in_the_proof_of_Chernoff_bounds},
}

@article{diaz1968fixed,
  title={A fixed point theorem of the alternative, for contractions on a generalized complete metric space},
  author={Diaz, JB and Margolis, Beatriz},
  journal={Bulletin of the American Mathematical Society},
  volume={74},
  number={2},
  pages={305--309},
  year={1968},
  publisher={American Mathematical Society}
}


@inproceedings{munos2005error,
  title={Error bounds for approximate value iteration},
  author={Munos, R{\'e}mi},
  booktitle={Proceedings of the National Conference on Artificial Intelligence},
  volume={20},
  number={2},
  pages={1006},
  year={2005},
  organization={Menlo Park, CA; Cambridge, MA; London; AAAI Press; MIT Press; 1999}
}

@article{min2021learning,
  title={Learning stochastic shortest path with linear function approximation},
  author={Min, Yifei and He, Jiafan and Wang, Tianhao and Gu, Quanquan},
  journal={arXiv preprint arXiv:2110.12727},
  year={2021}
}

@inproceedings{yin2022near,
  title={Near-optimal Offline Reinforcement Learning with Linear Representation: Leveraging Variance Information with Pessimism},
  author={Yin, Ming and Duan, Yaqi and Wang, Mengdi and Wang, Yu-Xiang},
  booktitle={International Conference on Learning Representations},
  year={2022}
}

@article{zanette2021provable,
  title={Provable benefits of actor-critic methods for offline reinforcement learning},
  author={Zanette, Andrea and Wainwright, Martin J and Brunskill, Emma},
  journal={Advances in neural information processing systems},
  volume={34},
  year={2021}
}

@inproceedings{jin2021pessimism,
  title={Is pessimism provably efficient for offline rl?},
  author={Jin, Ying and Yang, Zhuoran and Wang, Zhaoran},
  booktitle={International Conference on Machine Learning},
  pages={5084--5096},
  year={2021},
  organization={PMLR}
}

@article{bertsekas2013stochastic,
  title={Stochastic shortest path problems under weak conditions},
  author={Bertsekas, Dimitri P and Yu, Huizhen},
  journal={Lab. for Information and Decision Systems Report LIDS-P-2909, MIT},
  year={2013}
}


@article{wagenmaker2021first,
  title={First-Order Regret in Reinforcement Learning with Linear Function Approximation: A Robust Estimation Approach},
  author={Wagenmaker, Andrew and Chen, Yifang and Simchowitz, Max and Du, Simon S and Jamieson, Kevin},
  journal={arXiv preprint arXiv:2112.03432},
  year={2021}
}

@article{zhang2022off,
  title={Off-Policy Fitted Q-Evaluation with Differentiable Function Approximators: Z-Estimation and Inference Theory},
  author={Zhang, Ruiqi and Zhang, Xuezhou and Ni, Chengzhuo and Wang, Mengdi},
  journal={arXiv preprint arXiv:2202.04970},
  year={2022}
}

@article{jafarnia2021online,
  title={Online learning for stochastic shortest path model via posterior sampling},
  author={Jafarnia-Jahromi, Mehdi and Chen, Liyu and Jain, Rahul and Luo, Haipeng},
  journal={arXiv preprint arXiv:2106.05335},
  year={2021}
}

@article{xie2021policy,
  title={Policy finetuning: Bridging sample-efficient offline and online reinforcement learning},
  author={Xie, Tengyang and Jiang, Nan and Wang, Huan and Xiong, Caiming and Bai, Yu},
  journal={Advances in neural information processing systems},
  volume={34},
  year={2021}
}



@article{vial2021regret,
  title={Regret bounds for stochastic shortest path problems with linear function approximation},
  author={Vial, Daniel and Parulekar, Advait and Shakkottai, Sanjay and Srikant, R},
  journal={arXiv preprint arXiv:2105.01593},
  year={2021}
}


@inproceedings{rosenberg2020near,
  title={Near-optimal regret bounds for stochastic shortest path},
  author={Rosenberg, Aviv and Cohen, Alon and Mansour, Yishay and Kaplan, Haim},
  booktitle={International Conference on Machine Learning},
  pages={8210--8219},
  year={2020},
  organization={PMLR}
}




@article{agarwal2019reinforcement,
  title={Reinforcement learning: Theory and algorithms},
  author={Agarwal, Alekh and Jiang, Nan and Kakade, Sham M},
  journal={CS Dept., UW Seattle, Seattle, WA, USA, Tech. Rep},
  year={2019}
}

@article{xiao2021optimality,
  title={On the Optimality of Batch Policy Optimization Algorithms},
  author={Xiao, Chenjun and Wu, Yifan and Lattimore, Tor and Dai, Bo and Mei, Jincheng and Li, Lihong and Szepesvari, Csaba and Schuurmans, Dale},
  journal={arXiv preprint arXiv:2104.02293},
  year={2021}
}

@article{kidambi2020morel,
  title={Morel: Model-based offline reinforcement learning},
  author={Kidambi, Rahul and Rajeswaran, Aravind and Netrapalli, Praneeth and Joachims, Thorsten},
  journal={arXiv preprint arXiv:2005.05951},
  year={2020}
}

@article{rashidinejad2021bridging,
  title={Bridging Offline Reinforcement Learning and Imitation Learning: A Tale of Pessimism},
  author={Rashidinejad, Paria and Zhu, Banghua and Ma, Cong and Jiao, Jiantao and Russell, Stuart},
  journal={arXiv preprint arXiv:2103.12021},
  year={2021}
}

@article{ren2021nearly,
  title={Nearly Horizon-Free Offline Reinforcement Learning},
  author={Ren, Tongzheng and Li, Jialian and Dai, Bo and Du, Simon S and Sanghavi, Sujay},
  journal={arXiv preprint arXiv:2103.14077},
  year={2021}
}

@article{chandak2021universal,
  title={Universal Off-Policy Evaluation},
  author={Chandak, Yash and Niekum, Scott and da Silva, Bruno Castro and Learned-Miller, Erik and Brunskill, Emma and Thomas, Philip S},
  journal={arXiv preprint arXiv:2104.12820},
  year={2021}
}

@article{feng2021non,
  title={Non-asymptotic confidence intervals of off-policy evaluation: Primal and dual bounds},
  author={Feng, Yihao and Tang, Ziyang and Zhang, Na and Liu, Qiang},
  journal={arXiv preprint arXiv:2103.05741},
  year={2021}
}

@article{hao2021bootstrapping,
  title={Bootstrapping Statistical Inference for Off-Policy Evaluation},
  author={Hao, Botao and Duan, Yaqi and Lu, Hao and Szepesv{\'a}ri, Csaba and Wang, Mengdi and others},
  journal={arXiv preprint arXiv:2102.03607},
  year={2021}
}


@article{nguyen2021finite,
  title={On Finite-Sample Analysis of Offline Reinforcement Learning with Deep ReLU Networks},
  author={Nguyen-Tang, Thanh and Gupta, Sunil and Tran-The, Hung and Venkatesh, Svetha},
  journal={arXiv preprint arXiv:2103.06671},
  year={2021}
}

@article{uehara2021finite,
  title={Finite sample analysis of minimax offline reinforcement learning: Completeness, fast rates and first-order efficiency},
  author={Uehara, Masatoshi and Imaizumi, Masaaki and Jiang, Nan and Kallus, Nathan and Sun, Wen and Xie, Tengyang},
  journal={arXiv preprint arXiv:2102.02981},
  year={2021}
}


@article{wang2020statistical,
  title={What are the Statistical Limits of Offline RL with Linear Function Approximation?},
  author={Wang, Ruosong and Foster, Dean P and Kakade, Sham M},
  journal={arXiv preprint arXiv:2010.11895},
  year={2020}
}

@inproceedings{liu2017deep,
  title={Deep reinforcement learning for dynamic treatment regimes on medical registry data},
  author={Liu, Ying and Logan, Brent and Liu, Ning and Xu, Zhiyuan and Tang, Jian and Wang, Yangzhi},
  booktitle={2017 IEEE International Conference on Healthcare Informatics (ICHI)},
  pages={380--385},
  year={2017},
  organization={IEEE}
}


@inproceedings{kato2020off,
  title={Off-policy evaluation and learning for external validity under a covariate shift},
  author={Kato, Masahiro and Uehara, Masatoshi and Yasui, Shota},
  booktitle={Advances in neural information processing systems},
  year={2020}
}

@inproceedings{ameko2020offline,
  title={Offline contextual multi-armed bandits for mobile health interventions: A case study on emotion regulation},
  author={Ameko, Mawulolo K and Beltzer, Miranda L and Cai, Lihua and Boukhechba, Mehdi and Teachman, Bethany A and Barnes, Laura E},
  booktitle={Fourteenth ACM Conference on Recommender Systems},
  pages={249--258},
  year={2020}
}


@inproceedings{jin2020reward,
  title={Reward-free exploration for reinforcement learning},
  author={Jin, Chi and Krishnamurthy, Akshay and Simchowitz, Max and Yu, Tiancheng},
  booktitle={International Conference on Machine Learning},
  pages={4870--4879},
  year={2020},
  organization={PMLR}
}

@inproceedings{codevilla2018offline,
  title={On offline evaluation of vision-based driving models},
  author={Codevilla, Felipe and Lopez, Antonio M and Koltun, Vladlen and Dosovitskiy, Alexey},
  booktitle={Proceedings of the European Conference on Computer Vision (ECCV)},
  pages={236--251},
  year={2018}
}

@inproceedings{osinski2020simulation,
  title={Simulation-based reinforcement learning for real-world autonomous driving},
  author={Osinski, Blazej and Jakubowski, Adam and Ziecina, Pawel and Milos, Piotr and Galias, Christopher and Homoceanu, Silviu and Michalewski, Henryk},
  booktitle={2020 IEEE International Conference on Robotics and Automation (ICRA)},
  pages={6411--6418},
  year={2020},
  organization={IEEE}
}

@inproceedings{cui2020plug,
  title={Is Plug-in Solver Sample-Efficient for Feature-based Reinforcement Learning?},
  author={Cui, Qiwen and Yang, Lin F},
  booktitle={Advances in neural information processing systems},
  year={2020}
}


@inproceedings{yin2021near,
  title={Near-Optimal Provable Uniform Convergence in Offline Policy Evaluation for Reinforcement Learning},
  author={Yin, Ming and Bai, Yu and Wang, Yu-Xiang},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1567--1575},
  year={2021},
  organization={PMLR}
}

@article{liu2020sharp,
  title={A Sharp Analysis of Model-based Reinforcement Learning with Self-Play},
  author={Liu, Qinghua and Yu, Tiancheng and Bai, Yu and Jin, Chi},
  journal={arXiv preprint arXiv:2010.01604},
  year={2020}
}

@article{zhou2020nearly,
  title={Nearly Minimax Optimal Reinforcement Learning for Linear Mixture Markov Decision Processes},
  author={Zhou, Dongruo and Gu, Quanquan and Szepesvari, Csaba},
  journal={arXiv preprint arXiv:2012.08507},
  year={2020}
}

@inproceedings{jin2020provably,
  title={Provably efficient reinforcement learning with linear function approximation},
  author={Jin, Chi and Yang, Zhuoran and Wang, Zhaoran and Jordan, Michael I},
  booktitle={Conference on Learning Theory},
  pages={2137--2143},
  year={2020},
  organization={PMLR}
}

@book{lattimore2020bandit,
  title={Bandit algorithms},
  author={Lattimore, Tor and Szepesv{\'a}ri, Csaba},
  year={2020},
  publisher={Cambridge University Press}
}

@inproceedings{yang2019sample,
  title={Sample-optimal parametric Q-learning using linearly additive features},
  author={Yang, Lin and Wang, Mengdi},
  booktitle={International Conference on Machine Learning},
  pages={6995--7004},
  year={2019},
  organization={PMLR}
}

@article{wang2020reward,
  title={On reward-free reinforcement learning with linear function approximation},
  author={Wang, Ruosong and Du, Simon S and Yang, Lin F and Salakhutdinov, Ruslan},
  journal={arXiv preprint arXiv:2006.11274},
  year={2020}
}

@article{yu2020mopo,
  title={Mopo: Model-based offline policy optimization},
  author={Yu, Tianhe and Thomas, Garrett and Yu, Lantao and Ermon, Stefano and Zou, James and Levine, Sergey and Finn, Chelsea and Ma, Tengyu},
  journal={arXiv preprint arXiv:2005.13239},
  year={2020}
}

@article{mannor2004sample,
  title={The sample complexity of exploration in the multi-armed bandit problem},
  author={Mannor, Shie and Tsitsiklis, John N},
  journal={Journal of Machine Learning Research},
  volume={5},
  number={Jun},
  pages={623--648},
  year={2004}
}


@article{han2015minimax,
  title={Minimax Estimation of Discrete Distributions Under $l_1$ Loss},
  author={Han, Yanjun and Jiao, Jiantao and Weissman, Tsachy},
  journal={IEEE Transactions on Information Theory},
  volume={61},
  number={11},
  pages={6343-6354},
  year={2015},
  publisher={IEEE}
}


@article{minsker2011some,
  title={On some extensions of Bernstein's inequality for self-adjoint operators},
  author={Minsker, Stanislav},
  journal={arXiv preprint arXiv:1112.5448},
  year={2011}
}

@inproceedings{agarwal2020model,
  title={Model-based reinforcement learning with a generative model is minimax optimal},
  author={Agarwal, Alekh and Kakade, Sham and Yang, Lin F},
  booktitle={Conference on Learning Theory},
  pages={67--83},
  year={2020}
}

@article{kearns2002near,
  title={Near-optimal reinforcement learning in polynomial time},
  author={Kearns, Michael and Singh, Satinder},
  journal={Machine learning},
  volume={49},
  number={2-3},
  pages={209--232},
  year={2002},
  publisher={Springer}
}

@book{vapnik1999overview,
  title={The nature of statistical learning theory},
  author={Vapnik, Vladimir},
  year={2013},
  publisher={Springer science \& business media}
}

@inproceedings{le2019batch,
  title={Batch Policy Learning under Constraints},
  author={Le, Hoang and Voloshin, Cameron and Yue, Yisong},
  booktitle={International Conference on Machine Learning},
  pages={3703--3712},
  year={2019}
}

@inproceedings{munos2003error,
  title={Error bounds for approximate policy iteration},
  author={Munos, R{\'e}mi},
  booktitle={International Conference on Machine Learning},
  pages={560--567},
  year={2003}
}

@inproceedings{dasari2019robonet,
  title={RoboNet: Large-Scale Multi-Robot Learning},
  author={Dasari, Sudeep and Ebert, Frederik and Tian, Stephen and Nair, Suraj and Bucher, Bernadette and Schmeckpeper, Karl and Singh, Siddharth and Levine, Sergey and Finn, Chelsea},
  booktitle={Conference on Robot Learning},
  pages={885--897},
  year={2020}
}

@inproceedings{chen2019information,
  title={Information-Theoretic Considerations in Batch Reinforcement Learning},
  author={Chen, Jinglin and Jiang, Nan},
  booktitle={International Conference on Machine Learning},
  pages={1042--1051},
  year={2019}
}

@article{cheung2020reinforcement,
  title={Reinforcement learning for non-stationary markov decision processes: The blessing of (more) optimism},
  author={Cheung, Wang Chi and Simchi-Levi, David and Zhu, Ruihao},
  journal={arXiv preprint arXiv:2006.14389},
  year={2020}
}

@inproceedings{auer2009near,
  title={Near-optimal regret bounds for reinforcement learning},
  author={Auer, Peter and Jaksch, Thomas and Ortner, Ronald},
  booktitle={Advances in neural information processing systems},
  pages={89--96},
  year={2009}
}

@inproceedings{ortner2020variational,
  title={Variational regret bounds for reinforcement learning},
  author={Ortner, Ronald and Gajane, Pratik and Auer, Peter},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={81--90},
  year={2020},
  organization={PMLR}
}

@article{jaques2019way,
  title={Way off-policy batch deep reinforcement learning of implicit human preferences in dialog},
  author={Jaques, Natasha and Ghandeharioun, Asma and Shen, Judy Hanwen and Ferguson, Craig and Lapedriza, Agata and Jones, Noah and Gu, Shixiang and Picard, Rosalind},
  journal={arXiv preprint arXiv:1907.00456},
  year={2019}
}

@article{gottesman2019guidelines,
  title={Guidelines for reinforcement learning in healthcare},
  author={Gottesman, Omer and Johansson, Fredrik and Komorowski, Matthieu and Faisal, Aldo and Sontag, David and Doshi-Velez, Finale and Celi, Leo Anthony},
  journal={Nat Med},
  volume={25},
  number={1},
  pages={16--18},
  year={2019}
}


@incollection{lange2012batch,
  title={Batch reinforcement learning},
  author={Lange, Sascha and Gabel, Thomas and Riedmiller, Martin},
  booktitle={Reinforcement learning},
  pages={45--73},
  year={2012},
  publisher={Springer}
}


@inproceedings{kallus2019intrinsically,
  title={Intrinsically efficient, stable, and bounded off-policy evaluation for reinforcement learning},
  author={Kallus, Nathan and Uehara, Masatoshi},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3325--3334},
  year={2019}
}

@article{jin2020pessimism,
  title={Is Pessimism Provably Efficient for Offline RL?},
  author={Jin, Ying and Yang, Zhuoran and Wang, Zhaoran},
  journal={arXiv preprint arXiv:2012.15085},
  year={2020}
}


@book{vapnik2013nature,
  title={The nature of statistical learning theory},
  author={Vapnik, Vladimir},
  year={2013},
  publisher={Springer science \& business media}
}

@article{zhang2020model,
  title={Model-based multi-agent rl in zero-sum markov games with near-optimal sample complexity},
  author={Zhang, Kaiqing and Kakade, Sham M and Ba{\c{s}}ar, Tamer and Yang, Lin F},
  journal={arXiv preprint arXiv:2007.07461},
  year={2020}
}

@article{jaksch2010near,
  title={Near-optimal Regret Bounds for Reinforcement Learning.},
  author={Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
  journal={Journal of Machine Learning Research},
  volume={11},
  number={4},
  year={2010}
}

@article{zhang2020reinforcement,
  title={Is reinforcement learning more difficult than bandits? a near-optimal algorithm escaping the curse of horizon},
  author={Zhang, Zihan and Ji, Xiangyang and Du, Simon S},
  journal={arXiv preprint arXiv:2009.13503},
  year={2020}
}
 

@inproceedings{ayoub2020model,
  title={Model-based reinforcement learning with value-targeted regression},
  author={Ayoub, Alex and Jia, Zeyu and Szepesvari, Csaba and Wang, Mengdi and Yang, Lin},
  booktitle={International Conference on Machine Learning},
  pages={463--474},
  year={2020},
  organization={PMLR}
}

@article{zhang2020task,
  title={Task-agnostic exploration in reinforcement learning},
  author={Zhang, Xuezhou and Singla, Adish and others},
  journal={Advances in Neural Information Processing Systems},
  year={2020}
}

@article{zhang2020nearly,
  title={Nearly Minimax Optimal Reward-free Reinforcement Learning},
  author={Zhang, Zihan and Du, Simon S and Ji, Xiangyang},
  journal={arXiv preprint arXiv:2010.05901},
  year={2020}
}

@article{menard2020fast,
  title={Fast active learning for pure exploration in reinforcement learning},
  author={Menard, Pierre and Domingues, Omar Darwiche and Jonsson, Anders and Kaufmann, Emilie and Leurent, Edouard and Valko, Michal},
  journal={arXiv preprint arXiv:2007.13442},
  year={2020}
}

@article{kaufmann2020adaptive,
  title={Adaptive reward-free exploration},
  author={Kaufmann, Emilie and M{\'e}nard, Pierre and Domingues, Omar Darwiche and Jonsson, Anders and Leurent, Edouard and Valko, Michal},
  journal={arXiv preprint arXiv:2006.06294},
  year={2020}
}

@article{li2020breaking,
  title={Breaking the sample size barrier in model-based reinforcement learning with a generative model},
  author={Li, Gen and Wei, Yuting and Chi, Yuejie and Gu, Yuantao and Chen, Yuxin},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}

@inproceedings{szepesvari2005finite,
  title={Finite time bounds for sampling based fitted value iteration},
  author={Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
  booktitle={Proceedings of the 22nd international conference on Machine learning},
  pages={880--887},
  year={2005}
}

@inproceedings{efroni2019tight,
  title={Tight regret bounds for model-based reinforcement learning with greedy policies},
  author={Efroni, Yonathan and Merlis, Nadav and Ghavamzadeh, Mohammad and Mannor, Shie},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}



@article{shalev2010learnability,
  title={Learnability, stability and uniform convergence},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Srebro, Nathan and Sridharan, Karthik},
  journal={The Journal of Machine Learning Research},
  volume={11},
  pages={2635--2670},
  year={2010},
  publisher={JMLR. org}
}

@article{hu2021fast,
  title={Fast Rates for the Regret of Offline Reinforcement Learning},
  author={Hu, Yichun and Kallus, Nathan and Uehara, Masatoshi},
  journal={arXiv preprint arXiv:2102.00479},
  year={2021}
}

@article{yin2021nearoptimal,
  title={Near-Optimal Offline Reinforcement Learning via Double Variance Reduction},
  author={Yin, Ming and Bai, Yu and Wang, Yu-Xiang},
  journal={arXiv preprint arXiv:2102.01748},
  year={2021}
}

@article{hao2020sparse,
  title={Sparse Feature Selection Makes Batch Reinforcement Learning More Sample Efficient},
  author={Hao, Botao and Duan, Yaqi and Lattimore, Tor and Szepesv{\'a}ri, Csaba and Wang, Mengdi},
  journal={arXiv preprint arXiv:2011.04019},
  year={2020}
}




@inproceedings{wang2017optimal,
  title={Optimal and adaptive off-policy evaluation in contextual bandits},
  author={Wang, Yu-Xiang and Agarwal, Alekh and Dud{\i}k, Miroslav},
  booktitle={International Conference on Machine Learning},
  pages={3589--3597},
  year={2017},
  organization={PMLR}
}

@article{zanette2020exponential,
  title={Exponential Lower Bounds for Batch Reinforcement Learning: Batch RL can be Exponentially Harder than Online RL},
  author={Zanette, Andrea},
  journal={arXiv preprint arXiv:2012.08005},
  year={2020}
}

@article{kuzborskij2020confident,
  title={Confident off-policy evaluation and selection through self-normalized importance weighting},
  author={Kuzborskij, Ilja and Vernade, Claire and Gyorgy, Andras and Szepesvari, Csaba},
  journal={arXiv preprint arXiv:2006.10460},
  year={2020}
}

@article{zhang2021average,
  title={Average-Reward Off-Policy Policy Evaluation with Function Approximation},
  author={Zhang, Shangtong and Wan, Yi and Sutton, Richard S and Whiteson, Shimon},
  journal={arXiv preprint arXiv:2101.02808},
  year={2021}
}

@inproceedings{antos2008fitted,
  title={Fitted Q-iteration in continuous action-space MDPs},
  author={Antos, Andras and Munos, Remi and Szepesvari, Csaba},
  booktitle={Advances in Neural Information Processing Systems},
  pages={9-16},
  year={2008}
}

@article{antos2008learning,
  title={Learning near-optimal policies with Bellman-residual minimization based fitted policy iteration and a single sample path},
  author={Antos, Andr{\'a}s and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
  journal={Machine Learning},
  volume={71},
  number={1},
  pages={89--129},
  year={2008},
  publisher={Springer}
}


@inproceedings{feng2020accountable,
  title={Accountable off-policy evaluation with kernel bellman statistics},
  author={Feng, Yihao and Ren, Tongzheng and Tang, Ziyang and Liu, Qiang},
  booktitle={International Conference on Machine Learning},
  pages={3102--3111},
  year={2020},
  organization={PMLR}
}


@article{nachum2019dualdice,
  title={Dualdice: Behavior-agnostic estimation of discounted stationary distribution corrections},
  author={Nachum, Ofir and Chow, Yinlam and Dai, Bo and Li, Lihong},
  journal={arXiv preprint arXiv:1906.04733},
  year={2019}
}

@article{feng2019kernel,
  title={A kernel loss for solving the bellman equation},
  author={Feng, Yihao and Li, Lihong and Liu, Qiang},
  journal={arXiv preprint arXiv:1905.10506},
  year={2019}
}

@article{zhang2020gendice,
  title={Gendice: Generalized offline estimation of stationary values},
  author={Zhang, Ruiyi and Dai, Bo and Li, Lihong and Schuurmans, Dale},
  journal={arXiv preprint arXiv:2002.09072},
  year={2020}
}

@article{yang2020offline,
  title={Offline Policy Selection under Uncertainty},
  author={Yang, Mengjiao and Dai, Bo and Nachum, Ofir and Tucker, George and Schuurmans, Dale},
  journal={arXiv preprint arXiv:2012.06919},
  year={2020}
}

@article{kallus2019efficiently,
  title={Efficiently breaking the curse of horizon: Double reinforcement learning in infinite-horizon processes},
  author={Kallus, Nathan and Uehara, Masatoshi},
  journal={arXiv preprint arXiv:1909.05850},
  year={2019}
}

@inproceedings{duan2020minimax,
  title={Minimax-Optimal Off-Policy Evaluation with Linear Function Approximation},
  author={Duan, Yaqi and Jia, Zeyu and Wang, Mengdi},
  booktitle={International Conference on Machine Learning},
  pages={8334--8342},
  year={2020}
}

@article{jiang2018notes,
  title={Notes on tabular methods},
  author={Jiang, Nan},
  year={2018}
}

@article{levine2020offline,
  title={Offline Reinforcement Learning: Tutorial, Review, and Perspectives on Open Problems},
  author={Levine, Sergey and Kumar, Aviral and Tucker, George and Fu, Justin},
  journal={arXiv preprint arXiv:2005.01643},
  year={2020}
}

@article{chung2006concentration,
  title={Concentration inequalities and martingale inequalities: a survey},
  author={Chung, Fan and Lu, Linyuan},
  journal={Internet Mathematics},
  volume={3},
  number={1},
  pages={79--127},
  year={2006},
  publisher={Taylor \& Francis}
}

@inproceedings{dann2015sample,
  title={Sample complexity of episodic fixed-horizon reinforcement learning},
  author={Dann, Christoph and Brunskill, Emma},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2818--2826},
  year={2015}
}

@article{ye2011simplex,
  title={The simplex and policy-iteration methods are strongly polynomial for the Markov decision problem with a fixed discount rate},
  author={Ye, Yinyu},
  journal={Mathematics of Operations Research},
  volume={36},
  number={4},
  pages={593--603},
  year={2011},
  publisher={INFORMS}
}


@inproceedings{krishnamurthy2016pac,
  title={{PAC} reinforcement learning with rich observations},
  author={Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1840--1848},
  year={2016}
}


@article{tropp2011freedman,
  title={Freedman's inequality for matrix martingales},
  author={Tropp, Joel and others},
  journal={Electronic Communications in Probability},
  volume={16},
  pages={262--270},
  year={2011},
  publisher={The Institute of Mathematical Statistics and the Bernoulli Society}
}


@article{powell1966weighted,
  title={Weighted uniform sampling—a Monte Carlo technique for reducing variance},
  author={Powell, Michael JD and Swann, J},
  journal={IMA Journal of Applied Mathematics},
  volume={2},
  number={3},
  pages={228--236},
  year={1966},
  publisher={Oxford University Press}
}

@article{horvitz1952generalization,
  title={A generalization of sampling without replacement from a finite universe},
  author={Horvitz, Daniel G and Thompson, Donovan J},
  journal={Journal of the American statistical Association},
  volume={47},
  number={260},
  pages={663--685},
  year={1952},
  publisher={Taylor \& Francis Group}
}

@InProceedings{xie2020q,
  title =    {Q* Approximation Schemes for Batch Reinforcement Learning: A Theoretical Comparison},
  author =       {Xie, Tengyang and Jiang, Nan},
  booktitle={Uncertainty in Artificial Intelligence},
  pages =    {550--559},
  year =   {2020},
}

@article{xie2020batch,
  title={Batch Value-function Approximation with Only Realizability},
  author={Xie, Tengyang and Jiang, Nan},
  journal={arXiv preprint arXiv:2008.04990},
  year={2020}
}

@inproceedings{abbasi2011improved,
  title={Improved algorithms for linear stochastic bandits},
  author={Abbasi-Yadkori, Yasin and P{\'a}l, D{\'a}vid and Szepesv{\'a}ri, Csaba},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2312--2320},
  year={2011}
}

@book{tewari2007reinforcement,
  title={Reinforcement learning in large or unknown MDPs},
  author={Tewari, Ambuj},
  year={2007},
  publisher={University of California, Berkeley}
}

@article{liu2020provably,
  title={Provably good batch reinforcement learning without great exploration},
  author={Liu, Yao and Swaminathan, Adith and Agarwal, Alekh and Brunskill, Emma},
  journal={arXiv preprint arXiv:2007.08202},
  year={2020}
}

@inproceedings{bai2020provable,
  title={Provable self-play algorithms for competitive reinforcement learning},
  author={Bai, Yu and Jin, Chi},
  booktitle={International Conference on Machine Learning},
  pages={551--560},
  year={2020},
  organization={PMLR}
}

@inproceedings{NEURIPS2020_0cc6ee01,
 author = {Zhang, Kaiqing and Kakade, Sham and Basar, Tamer and Yang, Lin},
 booktitle = {Advances in Neural Information Processing Systems},
 pages = {1166--1178},
 title = {Model-Based Multi-Agent RL in Zero-Sum Markov Games with Near-Optimal Sample Complexity},
 volume = {33},
 year = {2020}
}


@inproceedings{yin2020asymptotically,
  title={Asymptotically efficient off-policy evaluation for tabular reinforcement learning},
  author={Yin, Ming and Wang, Yu-Xiang},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3948--3958},
  year={2020},
  organization={PMLR}
}

@article{tropp2011freedman,
  title={Freedman's inequality for matrix martingales},
  author={Tropp, Joel and others},
  journal={Electronic Communications in Probability},
  volume={16},
  pages={262--270},
  year={2011},
  publisher={The Institute of Mathematical Statistics and the Bernoulli Society}
}




@book{van2000asymptotic,
  title={Asymptotic statistics},
  author={Van der Vaart, Aad W},
  volume={3},
  year={2000},
  publisher={Cambridge university press}
}




@article{puterman1994markov,
  title={Markov Decision Processes: Discrete Stochastic Dynamic Programming},
  author={Puterman, Martin L},
  year={1994},
  publisher={John Wiley \& Sons, Inc.}
}

@inproceedings{kearns1999finite,
  title={Finite-sample convergence rates for Q-learning and indirect algorithms},
  author={Kearns, Michael J and Singh, Satinder P},
  booktitle={Advances in neural information processing systems},
  pages={996--1002},
  year={1999}
}

@article{maurer2009empirical,
  title={Empirical bernstein bounds and sample variance penalization},
  author={Maurer, Andreas and Pontil, Massimiliano},
  journal={arXiv preprint arXiv:0907.3740},
  year={2009}
}

@article{bertsekas2011temporal,
  title={Temporal difference methods for general projected equations},
  author={Bertsekas, Dimitri P},
  journal={IEEE Transactions on Automatic Control},
  volume={56},
  number={9},
  pages={2128--2139},
  year={2011},
  publisher={IEEE}
}

@inproceedings{dann2017unifying,
  title={Unifying PAC and regret: Uniform PAC bounds for episodic reinforcement learning},
  author={Dann, Christoph and Lattimore, Tor and Brunskill, Emma},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5713--5723},
  year={2017}
}

@inproceedings{xie2019towards,
  title={Towards optimal off-policy evaluation for reinforcement learning with marginalized importance sampling},
  author={Xie, Tengyang and Ma, Yifei and Wang, Yu-Xiang},
  booktitle={Advances in Neural Information Processing Systems},
  pages={9668--9678},
  year={2019}
}

@inproceedings{liu2019off,
  title={Off-Policy Policy Gradient with State Distribution Correction},
  author={Liu, Yao and Swaminathan, Adith and Agarwal, Alekh and Brunskill, Emma},
  booktitle={Uncertainty in Artificial Intelligence},
  year={2019}
}

@article{freedman1975tail,
  title={On tail probabilities for martingales},
  author={Freedman, David A},
  journal={the Annals of Probability},
  pages={100--118},
  year={1975},
  publisher={JSTOR}
}


@inproceedings{mandel2014offline,
  title={Offline policy evaluation across representations with applications to educational games},
  author={Mandel, Travis and Liu, Yun-En and Levine, Sergey and Brunskill, Emma and Popovic, Zoran},
  booktitle={Proceedings of the 2014 international conference on Autonomous agents and multi-agent systems},
  pages={1077--1084},
  year={2014},
  organization={International Foundation for Autonomous Agents and Multiagent Systems}
}

@inproceedings{thomas2017predictive,
  title={Predictive off-policy policy evaluation for nonstationary decision problems, with applications to digital marketing},
  author={Thomas, Philip S and Theocharous, Georgios and Ghavamzadeh, Mohammad and Durugkar, Ishan and Brunskill, Emma},
  booktitle={Twenty-Ninth IAAI Conference},
  year={2017}
}

@inproceedings{theocharous2015personalized,
  title={Personalized ad recommendation systems for life-time value optimization with guarantees},
  author={Theocharous, Georgios and Thomas, Philip S and Ghavamzadeh, Mohammad},
  booktitle={Twenty-Fourth International Joint Conference on Artificial Intelligence},
  year={2015}
}

@article{azar2013minimax,
  title={Minimax PAC bounds on the sample complexity of reinforcement learning with a generative model},
  author={Azar, Mohammad Gheshlaghi and Munos, R{\'e}mi and Kappen, Hilbert J},
  journal={Machine learning},
  volume={91},
  number={3},
  pages={325--349},
  year={2013},
  publisher={Springer}
}

@article{brafman2002r,
  title={R-max-a general polynomial time algorithm for near-optimal reinforcement learning},
  author={Brafman, Ronen I and Tennenholtz, Moshe},
  journal={Journal of Machine Learning Research},
  volume={3},
  number={Oct},
  pages={213--231},
  year={2002}
}


@inproceedings{jiang2017contextual,
  title={Contextual decision processes with low Bellman rank are PAC-learnable},
  author={Jiang, Nan and Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John and Schapire, Robert E},
  booktitle={International Conference on Machine Learning-Volume 70},
  pages={1704--1713},
  year={2017},
}

@article{yin2021towards,
  title={Towards instance-optimal offline reinforcement learning with pessimism},
  author={Yin, Ming and Wang, Yu-Xiang},
  journal={Advances in neural information processing systems},
  volume={34},
  year={2021}
}



@inproceedings{uehara2021pessimistic,
  title={Pessimistic Model-based Offline Reinforcement Learning under Partial Coverage},
  author={Uehara, Masatoshi and Sun, Wen},
  booktitle={International Conference on Learning Representations},
  year={2022},
}


@inproceedings{uehara2020minimax,
  title={Minimax weight and q-function learning for off-policy evaluation},
  author={Uehara, Masatoshi and Huang, Jiawei and Jiang, Nan},
  booktitle={International Conference on Machine Learning},
  pages={9659--9668},
  year={2020},
  organization={PMLR}
}

@article{farajtabar2018more,
  title={More robust doubly robust off-policy evaluation},
  author={Farajtabar, Mehrdad and Chow, Yinlam and Ghavamzadeh, Mohammad},
  journal={arXiv preprint arXiv:1802.03493},
  year={2018}
}

@inproceedings{thomas2016data,
  title={Data-efficient off-policy policy evaluation for reinforcement learning},
  author={Thomas, Philip and Brunskill, Emma},
  booktitle={International Conference on Machine Learning},
  pages={2139--2148},
  year={2016}
}

@article{murphy2001marginal,
  title={Marginal mean models for dynamic regimes},
  author={Murphy, Susan A and van der Laan, Mark J and Robins, James M and Conduct Problems Prevention Research Group},
  journal={Journal of the American Statistical Association},
  volume={96},
  number={456},
  pages={1410--1423},
  year={2001},
  publisher={Taylor \& Francis}
}

@article{hirano2003efficient,
  title={Efficient estimation of average treatment effects using the estimated propensity score},
  author={Hirano, Keisuke and Imbens, Guido W and Ridder, Geert},
  journal={Econometrica},
  volume={71},
  number={4},
  pages={1161--1189},
  year={2003},
  publisher={Wiley Online Library}
}

@article{jiang2015doubly,
  title={Doubly robust off-policy value evaluation for reinforcement learning},
  author={Jiang, Nan and Li, Lihong},
  journal={arXiv preprint arXiv:1511.03722},
  year={2015}
}

@article{raghu2018behaviour,
  title={Behaviour policy estimation in off-policy policy evaluation: Calibration matters},
  author={Raghu, Aniruddh and Gottesman, Omer and Liu, Yao and Komorowski, Matthieu and Faisal, Aldo and Doshi-Velez, Finale and Brunskill, Emma},
  journal={arXiv preprint arXiv:1807.01066},
  year={2018}
}


@inproceedings{quillen2018deep,
  title={Deep reinforcement learning for vision-based robotic grasping: A simulated comparative evaluation of off-policy methods},
  author={Quillen, Deirdre and Jang, Eric and Nachum, Ofir and Finn, Chelsea and Ibarz, Julian and Levine, Sergey},
  booktitle={2018 IEEE International Conference on Robotics and Automation (ICRA)},
  pages={6284--6291},
  year={2018},
  organization={IEEE}
}

@article{bertoluzzo2012testing,
  title={Testing different reinforcement learning configurations for financial trading: Introduction and applications},
  author={Bertoluzzo, Francesco and Corazza, Marco},
  journal={Procedia Economics and Finance},
  volume={3},
  pages={68--77},
  year={2012},
  publisher={Elsevier}
}



@article{sridharan2002gentle,
  title={A gentle introduction to concentration inequalities},
  author={Sridharan, Karthik},
  journal={Dept. Comput. Sci., Cornell Univ., Tech. Rep},
  year={2002},
  publisher={Citeseer}
}


@inproceedings{azar2017minimax,
  title={Minimax regret bounds for reinforcement learning},
  author={Azar, Mohammad Gheshlaghi and Osband, Ian and Munos, R{\'e}mi},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={263--272},
  year={2017},
  organization={JMLR. org}
}

@inproceedings{dann2015sample,
  title={Sample complexity of episodic fixed-horizon reinforcement learning},
  author={Dann, Christoph and Brunskill, Emma},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2818--2826},
  year={2015}
}

@inproceedings{mahmood2014weighted,
  title={Weighted importance sampling for off-policy learning with linear function approximation},
  author={Mahmood, A Rupam and van Hasselt, Hado P and Sutton, Richard S},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3014--3022},
  year={2014}
}

@inproceedings{wang2017optimal,
  title={Optimal and adaptive off-policy evaluation in contextual bandits},
  author={Wang, Yu-Xiang and Agarwal, Alekh and Dudik, Miroslav},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={3589--3597},
  year={2017},
  organization={JMLR. org}
}

@inproceedings{swaminathan2017off,
  title={Off-policy evaluation for slate recommendation},
  author={Swaminathan, Adith and Krishnamurthy, Akshay and Agarwal, Alekh and Dudik, Miro and Langford, John and Jose, Damien and Zitouni, Imed},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3632--3642},
  year={2017}
}

@article{gottesman2019combining,
  title={Combining parametric and nonparametric models for off-policy evaluation},
  author={Gottesman, Omer and Liu, Yao and Sussex, Scott and Brunskill, Emma and Doshi-Velez, Finale},
  journal={arXiv preprint arXiv:1905.05787},
  year={2019}
}

@inproceedings{liu2018representation,
  title={Representation balancing mdps for off-policy policy evaluation},
  author={Liu, Yao and Gottesman, Omer and Raghu, Aniruddh and Komorowski, Matthieu and Faisal, Aldo A and Doshi-Velez, Finale and Brunskill, Emma},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2644--2653},
  year={2018}
}

@inproceedings{li2011unbiased,
  title={Unbiased offline evaluation of contextual-bandit-based news article recommendation algorithms},
  author={Li, Lihong and Chu, Wei and Langford, John and Wang, Xuanhui},
  booktitle={ACM international conference on Web search and data mining},
  pages={297--306},
  year={2011}
}

@book{sutton2018reinforcement,
  title={Reinforcement learning: An introduction},
  author={Sutton, Richard S and Barto, Andrew G},
  year={2018},
  publisher={MIT press}
}



@inproceedings{li2015toward,
  title={Toward Minimax Off-policy Value Estimation},
  author={Li, Lihong and Munos, Remi and Szepesvari, Csaba},
  booktitle={Artificial Intelligence and Statistics},
  pages={608--616},
  year={2015}
}

@inproceedings{sidford2018near,
  title={Near-Optimal Time and Sample Complexities for Solving Markov Decision Processes with a Generative Model},
  author={Sidford, Aaron and Wang, Mengdi and Wu, Xian and Yang, Lin and Ye, Yinyu},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5186--5196},
  year={2018}
}


@article{azar2013minimax,
  title={Minimax PAC bounds on the sample complexity of reinforcement learning with a generative model},
  author={Azar, Mohammad Gheshlaghi and Munos, R{\'e}mi and Kappen, Hilbert J},
  journal={Machine learning},
  volume={91},
  number={3},
  pages={325--349},
  year={2013},
  publisher={Springer}
}

@inproceedings{jiang2018open,
  title={Open problem: The dependence of sample complexity lower bounds on planning horizon},
  author={Jiang, Nan and Agarwal, Alekh},
  booktitle={Conference On Learning Theory},
  pages={3395--3398},
  year={2018}
}

@inproceedings{jin2018q,
  title={Is q-learning provably efficient?},
  author={Jin, Chi and Allen-Zhu, Zeyuan and Bubeck, Sebastien and Jordan, Michael I},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4863--4873},
  year={2018}
}

@article{chao1972negative,
  title={Negative moments of positive random variables},
  author={Chao, Min-Te and Strawderman, WE},
  journal={Journal of the American Statistical Association},
  volume={67},
  number={338},
  pages={429--431},
  year={1972},
  publisher={Taylor \& Francis}
}

@article{chernoff1952measure,
  title={A measure of asymptotic efficiency for tests of a hypothesis based on the sum of observations},
  author={Chernoff, Herman and others},
  journal={The Annals of Mathematical Statistics},
  volume={23},
  number={4},
  pages={493--507},
  year={1952},
  publisher={Institute of Mathematical Statistics}
}

@inproceedings{salakhutdinov2008quantitative,
  title={On the quantitative analysis of deep belief networks},
  author={Salakhutdinov, Ruslan and Murray, Iain},
  booktitle={Proceedings of the 25th international conference on Machine learning},
  pages={872--879},
  year={2008},
  organization={ACM}
}

@article{neal2001annealed,
  title={Annealed importance sampling},
  author={Neal, Radford M},
  journal={Statistics and computing},
  volume={11},
  number={2},
  pages={125--139},
  year={2001},
  publisher={Springer}
}

@inproceedings{degris2012off,
  title={Off-policy actor-critic},
  author={Degris, Thomas and White, Martha and Sutton, Richard S},
  booktitle={Proceedings of the 29th International Coference on International Conference on Machine Learning},
  pages={179--186},
  year={2012},
  organization={Omnipress}
}

@inproceedings{silver2014deterministic,
  title={Deterministic policy gradient algorithms},
  author={Silver, David and Lever, Guy and Heess, Nicolas and Degris, Thomas and Wierstra, Daan and Riedmiller, Martin},
  booktitle={Proceedings of the 31st International Conference on International Conference on Machine Learning-Volume 32},
  pages={I--387},
  year={2014},
  organization={JMLR. org}
}

@inproceedings{sutton2000policy,
  title={Policy gradient methods for reinforcement learning with function approximation},
  author={Sutton, Richard S and McAllester, David A and Singh, Satinder P and Mansour, Yishay},
  booktitle={Advances in neural information processing systems},
  pages={1057--1063},
  year={2000}
}



@inproceedings{precup2000eligibility,
  title={Eligibility Traces for Off-Policy Policy Evaluation},
  author={Precup, Doina and Sutton, Richard S and Singh, Satinder P},
  booktitle={Proceedings of the Seventeenth International Conference on Machine Learning},
  pages={759--766},
  year={2000},
  organization={Morgan Kaufmann Publishers Inc.}
}

@phdthesis{thomas2015safe,
  title={Safe reinforcement learning},
  author={Thomas, Philip S},
  school={University of Massachusetts Amherst},
  year={2015}
}

@book{stein2009real,
  title={Real analysis: measure theory, integration, and Hilbert spaces},
  author={Stein, Elias M and Shakarchi, Rami},
  year={2009},
  publisher={Princeton University Press}
}

@book{rudin1976principles,
  title={Principles of mathematical analysis},
  author={Rudin, Walter and others},
  volume={3},
  year={1976},
  publisher={McGraw-hill New York}
}

@book{sutton1998reinforcement,
  title={Reinforcement learning: An introduction},
  author={Sutton, Richard S and Barto, Andrew G},
  volume={1},
  year={1998},
  publisher={MIT press Cambridge}
}

@book{puterman2014markov,
  title={Markov decision processes: discrete stochastic dynamic programming},
  author={Puterman, Martin L},
  year={2014},
  publisher={John Wiley \& Sons}
}

@inproceedings{jiang2016doubly,
  title={Doubly robust off-policy value evaluation for reinforcement learning},
  author={Jiang, Nan and Li, Lihong},
  booktitle={Proceedings of the 33rd International Conference on International Conference on Machine Learning-Volume 48},
  pages={652--661},
  year={2016},
  organization={JMLR. org}
}

@inproceedings{farajtabar18a,
  title =    {More Robust Doubly Robust Off-policy Evaluation},
  author =   {Farajtabar, Mehrdad and Chow, Yinlam and Ghavamzadeh, Mohammad},
  booktitle =    {International Conference on Machine Learning},
  pages =    {1447--1456},
  year =   {2018}
}

@inproceedings{raghu2017continuous,
  title={Continuous State-Space Models for Optimal Sepsis Treatment: a Deep Reinforcement Learning Approach},
  author={Raghu, Aniruddh and Komorowski, Matthieu and Celi, Leo Anthony and Szolovits, Peter and Ghassemi, Marzyeh},
  booktitle={Machine Learning for Healthcare Conference},
  pages={147--163},
  year={2017}
}

@inproceedings{theocharous2015personalized,
  title={Personalized Ad Recommendation Systems for Life-Time Value Optimization with Guarantees.},
  author={Theocharous, Georgios and Thomas, Philip S and Ghavamzadeh, Mohammad},
  booktitle={IJCAI},
  pages={1806--1812},
  year={2015}
}

@inproceedings{mandel2014offline,
  title={Offline policy evaluation across representations with applications to educational games},
  author={Mandel, Travis and Liu, Yun-En and Levine, Sergey and Brunskill, Emma and Popovic, Zoran},
  booktitle={Proceedings of the 2014 international conference on Autonomous agents and multi-agent systems},
  pages={1077--1084},
  year={2014},
  organization={International Foundation for Autonomous Agents and Multiagent Systems}
}



@inproceedings{thomas2015high,
  title={High-Confidence Off-Policy Evaluation.},
  author={Thomas, Philip S and Theocharous, Georgios and Ghavamzadeh, Mohammad},
  booktitle={AAAI},
  pages={3000--3006},
  year={2015}
}

@inproceedings{guo2017using,
  title={Using Options and Covariance Testing for Long Horizon Off-Policy Policy Evaluation},
  author={Guo, Zhaohan and Thomas, Philip S and Brunskill, Emma},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2492--2501},
  year={2017}
}


@InProceedings{farajtabar2018more,
  title =    {More Robust Doubly Robust Off-policy Evaluation},
  author =   {Farajtabar, Mehrdad and Chow, Yinlam and Ghavamzadeh, Mohammad},
  booktitle =    {Proceedings of the 35th International Conference on Machine Learning},
  pages =    {1447--1456},
  year =   {2018},
  volume =   {80},
  series =   {Proceedings of Machine Learning Research},
  address =    {Stockholmsmässan, Stockholm Sweden},
  month =    {10--15 Jul},
  publisher =    {PMLR},
}

@book{chung2001course,
  title={A course in probability theory},
  author={Chung, Kai Lai},
  year={2001},
  publisher={Academic press}
}

@article{singh1996reinforcement,
  title={Reinforcement learning with replacing eligibility traces},
  author={Singh, Satinder P and Sutton, Richard S},
  journal={Machine learning},
  volume={22},
  number={1-3},
  pages={123--158},
  year={1996},
  publisher={Springer}
}

@article{hoeffding1963probability,
  title={Probability inequalities for sums of bounded random variables},
  author={Hoeffding, Wassily},
  journal={Journal of the American statistical association},
  volume={58},
  number={301},
  pages={13--30},
  year={1963},
  publisher={Taylor \& Francis Group}
}

@article{slutsky1925uber,
  title={Uber stochastische asymptoten und grenzwerte},
  author={Slutsky, Evgeny},
  journal={Metron},
  volume={5},
  number={3},
  pages={3--89},
  year={1925}
}

@article{murphy2001marginal,
  title={Marginal mean models for dynamic regimes},
  author={Murphy, Susan A and van der Laan, Mark J and Robins, James M and Conduct Problems Prevention Research Group},
  journal={Journal of the American Statistical Association},
  volume={96},
  number={456},
  pages={1410--1423},
  year={2001},
  publisher={Taylor \& Francis}
}

@article{hirano2003efficient,
  title={Efficient estimation of average treatment effects using the estimated propensity score},
  author={Hirano, Keisuke and Imbens, Guido W and Ridder, Geert},
  journal={Econometrica},
  volume={71},
  number={4},
  pages={1161--1189},
  year={2003},
  publisher={Wiley Online Library}
}

@inproceedings{dudik2011doubly,
  title={Doubly robust policy evaluation and learning},
  author={Dud{\'\i}k, Miroslav and Langford, John and Li, Lihong},
  booktitle={International Conference on Machine Learning},
  pages={1097--1104},
  year={2011}
}

@inproceedings{wang2017optimal,
  title={Optimal and Adaptive Off-policy Evaluation in Contextual Bandits},
  author={Wang, Yu-Xiang and Agarwal, Alekh and Dud{\i}k, Miroslav},
  booktitle={International Conference on Machine Learning},
  pages={3589--3597},
  year={2017}
}

@inproceedings{strehl2010learning,
  title={Learning from logged implicit exploration data},
  author={Strehl, Alex and Langford, John and Li, Lihong and Kakade, Sham M},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2217--2225},
  year={2010}
}

@article{bottou2013counterfactual,
  title={Counterfactual reasoning and learning systems: The example of computational advertising},
  author={Bottou, L{\'e}on and Peters, Jonas and Qui{\~n}onero-Candela, Joaquin and Charles, Denis X and Chickering, D Max and Portugaly, Elon and Ray, Dipankar and Simard, Patrice and Snelson, Ed},
  journal={The Journal of Machine Learning Research},
  volume={14},
  number={1},
  pages={3207--3260},
  year={2013},
  publisher={JMLR. org}
}

@article{chapelle2015simple,
  title={Simple and scalable response prediction for display advertising},
  author={Chapelle, Olivier and Manavoglu, Eren and Rosales, Romer},
  journal={ACM Transactions on Intelligent Systems and Technology (TIST)},
  volume={5},
  number={4},
  pages={61},
  year={2015},
  publisher={ACM}
}

@inproceedings{tang2013automatic,
  title={Automatic ad format selection via contextual bandits},
  author={Tang, Liang and Rosales, Romer and Singh, Ajit and Agarwal, Deepak},
  booktitle={ACM international conference on information \& knowledge management},
  pages={1587--1594},
  year={2013}
}

@inproceedings{thomas2017predictive,
  title={Predictive Off-Policy Policy Evaluation for Nonstationary Decision Problems, with Applications to Digital Marketing.},
  author={Thomas, Philip S and Theocharous, Georgios and Ghavamzadeh, Mohammad and Durugkar, Ishan and Brunskill, Emma},
  booktitle={AAAI},
  pages={4740--4745},
  year={2017}
}

@article{kallus2019efficiently,
  title={Efficiently Breaking the Curse of Horizon: Double Reinforcement Learning in Infinite-Horizon Processes},
  author={Kallus, Nathan and Uehara, Masatoshi},
  journal={arXiv preprint arXiv:1909.05850},
  year={2019}
}

@inproceedings{kallus2019double,
  title={Double Reinforcement Learning for Efficient Off-Policy Evaluation in Markov Decision Processes},
  author={Kallus, Nathan and Uehara, Masatoshi},
  booktitle={International Conference on Machine Learning},
  pages={1922--1931},
  year={2020}
}

@inproceedings{gelada2019off,
  title={Off-policy deep reinforcement learning by bootstrapping the covariate shift},
  author={Gelada, Carles and Bellemare, Marc G},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={33},
  pages={3647--3655},
  year={2019}
}

@inproceedings{hallak2017consistent,
  title={Consistent on-line off-policy evaluation},
  author={Hallak, Assaf and Mannor, Shie},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={1372--1383},
  year={2017},
  organization={JMLR. org}
}

@inproceedings{liu2018breaking,
  title={Breaking the curse of horizon: Infinite-horizon off-policy estimation},
  author={Liu, Qiang and Li, Lihong and Tang, Ziyang and Zhou, Dengyong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5361--5371},
  year={2018}
}


@inproceedings{liu2018representation,
title = {Representation Balancing MDPs for Off-policy Policy Evaluation},
author = {Liu, Yao and Gottesman, Omer and Raghu, Aniruddh and Komorowski, Matthieu and Faisal, Aldo A and Doshi-Velez, Finale and Brunskill, Emma},
booktitle = {Advances in Neural Information Processing Systems 31},
pages = {2649--2658},
year = {2018}
}



@inproceedings{ernst2006clinical,
  title={Clinical data based optimal STI strategies for HIV: a reinforcement learning approach},
  author={Ernst, Damien and Stan, Guy-Bart and Goncalves, Jorge and Wehenkel, Louis},
  booktitle={Decision and Control, 2006 45th IEEE Conference on},
  pages={667--672},
  year={2006},
  organization={IEEE}
}

@inproceedings{zinkevich2006optimal,
  title={Optimal unbiased estimators for evaluating agent performance},
  author={Zinkevich, Martin and Bowling, Michael and Bard, Nolan and Kan, Morgan and Billings, Darse},
  booktitle={Proceedings of the National Conference on Artificial Intelligence},
  volume={21},
  pages={573},
  year={2006},
  organization={Citeseer}
}

@inproceedings{white2009learning,
  title={Learning a Value Analysis Tool for Agent Evaluation.},
  author={White, Martha and Bowling, Michael H},
  booktitle={IJCAI},
  pages={1976--1981},
  year={2009}
}

@inproceedings{hallak2017consistent,
  title={Consistent on-line off-policy evaluation},
  author={Hallak, Assaf and Mannor, Shie},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={1372--1383},
  year={2017},
  organization={JMLR. org}
}

@article{gelada2019off,
  title={Off-Policy Deep Reinforcement Learning by Bootstrapping the Covariate Shift},
  author={Gelada, Carles and Bellemare, Marc G},
  journal={arXiv preprint arXiv:1901.09455},
  year={2019}
}

@article{gottesman2019combining,
  title={Combining Parametric and Nonparametric Models for Off-Policy Evaluation},
  author={Gottesman, Omer and Liu, Yao and Sussex, Scott and Brunskill, Emma and Doshi-Velez, Finale},
  journal={arXiv preprint arXiv:1905.05787},
  year={2019}
}

@article{bai2019provably,
  title={Provably efficient q-learning with low switching cost},
  author={Bai, Yu and Xie, Tengyang and Jiang, Nan and Wang, Yu-Xiang},
  journal={arXiv preprint arXiv:1905.12849},
  year={2019}
}

@article{zhang2020almost,
  title={Almost optimal model-free reinforcement learning via reference-advantage decomposition},
  author={Zhang, Zihan and Zhou, Yuan and Ji, Xiangyang},
  journal={arXiv preprint arXiv:2004.10019},
  year={2020}
}

@article{gao2021provably,
  title={A provably efficient algorithm for linear markov decision process with low switching cost},
  author={Gao, Minbo and Xie, Tianle and Du, Simon S and Yang, Lin F},
  journal={arXiv preprint arXiv:2101.00494},
  year={2021}
}

 
@article{gao2019batched,
  title={Batched multi-armed bandits problem},
  author={Gao, Zijun and Han, Yanjun and Ren, Zhimei and Zhou, Zhengqing},
  journal={arXiv preprint arXiv:1904.01763},
  year={2019}
}

@inproceedings{zanette2019tighter,
  title={Tighter problem-dependent regret bounds in reinforcement learning without domain knowledge using value function bounds},
  author={Zanette, Andrea and Brunskill, Emma},
  booktitle={International Conference on Machine Learning},
  pages={7304--7312},
  year={2019},
  organization={PMLR}
}

@article{gyorgy2007line,
  title={The On-Line Shortest Path Problem Under Partial Monitoring.},
  author={Gy{\"o}rgy, Andr{\'a}s and Linder, Tam{\'a}s and Lugosi, G{\'a}bor and Ottucs{\'a}k, Gy{\"o}rgy},
  journal={Journal of Machine Learning Research},
  volume={8},
  number={10},
  year={2007}
}

@article{zimin2013online,
  title={Online learning in episodic Markovian decision processes by relative entropy policy search},
  author={Zimin, Alexander and Neu, Gergely},
  journal={Advances in neural information processing systems},
  volume={26},
  year={2013}
}

@inproceedings{chen2021minimax,
  title={Minimax regret for stochastic shortest path with adversarial costs and known transition},
  author={Chen, Liyu and Luo, Haipeng and Wei, Chen-Yu},
  booktitle={Conference on Learning Theory},
  pages={1180--1215},
  year={2021},
  organization={PMLR}
}



@article{rosenberg2019online,
  title={Online stochastic shortest path with bandit feedback and unknown transition function},
  author={Rosenberg, Aviv and Mansour, Yishay},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@inproceedings{neu2012adversarial,
  title={The adversarial stochastic shortest path problem with unknown transition probabilities},
  author={Neu, Gergely and Gyorgy, Andras and Szepesv{\'a}ri, Csaba},
  booktitle={Artificial Intelligence and Statistics},
  pages={805--813},
  year={2012},
  organization={PMLR}
}

@article{talebi2017stochastic,
  title={Stochastic online shortest path routing: The value of feedback},
  author={Talebi, Mohammad Sadegh and Zou, Zhenhua and Combes, Richard and Proutiere, Alexandre and Johansson, Mikael},
  journal={IEEE Transactions on Automatic Control},
  volume={63},
  number={4},
  pages={915--930},
  year={2017},
  publisher={IEEE}
}



@inproceedings{kaufmann2021adaptive,
  title={Adaptive reward-free exploration},
  author={Kaufmann, Emilie and M{\'e}nard, Pierre and Domingues, Omar Darwiche and Jonsson, Anders and Leurent, Edouard and Valko, Michal},
  booktitle={Algorithmic Learning Theory},
  pages={865--891},
  year={2021},
  organization={PMLR}
}


@inproceedings{menard2021fast,
  title={Fast active learning for pure exploration in reinforcement learning},
  author={M{\'e}nard, Pierre and Domingues, Omar Darwiche and Jonsson, Anders and Kaufmann, Emilie and Leurent, Edouard and Valko, Michal},
  booktitle={International Conference on Machine Learning},
  pages={7599--7608},
  year={2021},
  organization={PMLR}
}

@inproceedings{zhang2021near,
  title={Near Optimal Reward-Free Reinforcement Learning},
  author={Zhang, Zihan and Du, Simon and Ji, Xiangyang},
  booktitle={International Conference on Machine Learning},
  pages={12402--12412},
  year={2021},
  organization={PMLR}
}
 
@article{zanette2020provably,
  title={Provably efficient reward-agnostic navigation with linear value iteration},
  author={Zanette, Andrea and Lazaric, Alessandro and Kochenderfer, Mykel J and Brunskill, Emma},
  journal={arXiv preprint arXiv:2008.07737},
  year={2020}
}
 
@article{kong2021online,
  title={Online Sub-Sampling for Reinforcement Learning with General Function Approximation},
  author={Kong, Dingwen and Salakhutdinov, Ruslan and Wang, Ruosong and Yang, Lin F},
  journal={arXiv preprint arXiv:2106.07203},
  year={2021}
}

@inproceedings{cesa2013online,
  title={Online learning with switching costs and other adaptive adversaries},
  author={Cesa-Bianchi, Nicolo and Dekel, Ofer and Shamir, Ohad},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1160--1168},
  year={2013}
}

@inproceedings{dann2019policy,
  title={Policy certificates: Towards accountable reinforcement learning},
  author={Dann, Christoph and Li, Lihong and Wei, Wei and Brunskill, Emma},
  booktitle={International Conference on Machine Learning},
  pages={1507--1516},
  year={2019},
  organization={PMLR}
}
  


@article{osband2013more,
  title={(More) efficient reinforcement learning via posterior sampling},
  author={Osband, Ian and Russo, Daniel and Van Roy, Benjamin},
  journal={arXiv preprint arXiv:1306.0940},
  year={2013}
}

@inproceedings{agrawal2017posterior,
  title={Posterior sampling for reinforcement learning: worst-case regret bounds},
  author={Agrawal, Shipra and Jia, Randy},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1184--1194},
  year={2017}
}

@article{afsar2021reinforcement,
  title={Reinforcement learning based recommender systems: A survey},
  author={Afsar, M Mehdi and Crump, Trafford and Far, Behrouz},
  journal={arXiv preprint arXiv:2101.06286},
  year={2021}
}

@article{yu2021reinforcement,
  title={Reinforcement learning in healthcare: A survey},
  author={Yu, Chao and Liu, Jiming and Nemati, Shamim and Yin, Guosheng},
  journal={ACM Computing Surveys (CSUR)},
  volume={55},
  number={1},
  pages={1--36},
  year={2021},
  publisher={ACM New York, NY}
}

@article{krishnan2018learning,
  title={Learning to optimize join queries with deep reinforcement learning},
  author={Krishnan, Sanjay and Yang, Zongheng and Goldberg, Ken and Hellerstein, Joseph and Stoica, Ion},
  journal={arXiv preprint arXiv:1808.03196},
  year={2018}
}

@inproceedings{xu2018experience,
  title={Experience-driven networking: A deep reinforcement learning based approach},
  author={Xu, Zhiyuan and Tang, Jian and Meng, Jingsong and Zhang, Weiyi and Wang, Yanzhi and Liu, Chi Harold and Yang, Dejun},
  booktitle={IEEE INFOCOM 2018-IEEE Conference on Computer Communications},
  pages={1871--1879},
  year={2018},
  organization={IEEE}
}

@article{perchet2016batched,
  title={Batched bandit problems},
  author={Perchet, Vianney and Rigollet, Philippe and Chassang, Sylvain and Snowberg, Erik},
  journal={The Annals of Statistics},
  volume={44},
  number={2},
  pages={660--681},
  year={2016},
  publisher={Institute of Mathematical Statistics}
}

@article{esfandiari2019regret,
  title={Regret bounds for batched bandits},
  author={Esfandiari, Hossein and Karbasi, Amin and Mehrabian, Abbas and Mirrokni, Vahab},
  journal={arXiv preprint arXiv:1910.04959},
  year={2019}
}