% Encoding: UTF-8
@article{yang2019theoretical,
  title={A Theoretical Analysis of Deep Q-Learning},
  author={Yang, Zhuora and Xie, Yuchen and Wang, Zhaoran},
  journal={arXiv preprint arXiv:1901.00137},
  year={2019}
}

@article{gretton2012kernel,
  title={A kernel two-sample test},
  author={Gretton, Arthur and Borgwardt, Karsten M and Rasch, Malte J and Sch{\"o}lkopf, Bernhard and Smola, Alexander},
  journal={Journal of Machine Learning Research},
  volume={13},
  number={Mar},
  pages={723--773},
  year={2012}
}

@InProceedings{sandholm2012lossy,
  author    = {Sandholm, Tuomas and Singh, Satinder},
  title     = {Lossy stochastic game abstraction with bounds},
  booktitle = {Proceedings of the 13th ACM Conference on Electronic Commerce},
  year      = {2012},
  pages     = {880--897},
}

@InCollection{ravindran2002model,
  author    = {Ravindran, Balaraman and Barto, Andrew},
  title     = {Model minimization in hierarchical reinforcement learning},
  booktitle = {Abstraction, Reformulation, and Approximation},
  year      = {2002},
  pages     = {196--211},
}

@Article{hester2013texplore,
  author  = {Hester, Todd and Stone, Peter},
  title   = {{TEXPLORE}: real-time sample-efficient reinforcement learning for robots},
  journal = {Machine Learning},
  year    = {2013},
  volume  = {90},
  number  = {3},
  pages   = {385--429},
}

@Article{givan2003equivalence,
  author  = {Givan, Robert and Dean, Thomas and Greig, Matthew},
  title   = {Equivalence notions and model minimization in {M}arkov decision processes},
  journal = {Artificial Intelligence},
  year    = {2003},
  volume  = {147},
  number  = {1},
  pages   = {163--223},
}

@InProceedings{li2006towards,
  author    = {Li, Lihong and Walsh, Thomas J and Littman, Michael L},
  title     = {Towards a unified theory of state abstraction for {MDP}s},
  booktitle = {Proceedings of the 9th International Symposium on Artificial Intelligence and Mathematics},
  year      = {2006},
  pages     = {531--539},
}

@PhdThesis{ravindran2004algebraic,
  author = {Ravindran, Balaraman},
  title  = {An algebraic approach to abstraction in reinforcement learning},
  school = {University of Massachusetts Amherst},
  year   = {2004},
}

@InProceedings{sutton1999policy,
  author    = {Sutton, Richard S and McAllester, David A and Singh, Satinder P and Mansour, Yishay},
  title     = {Policy gradient methods for reinforcement learning with function approximation},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {1999},
  volume    = {99},
  pages     = {1057--1063},
}

@Article{dietterich2000hierarchical,
  author  = {Dietterich, Thomas G},
  title   = {Hierarchical reinforcement learning with the {MAXQ} value function decomposition},
  journal = {Journal of Artificial Intelligence Research},
  year    = {2000},
  volume  = {13},
  pages   = {227--303},
}

@InCollection{even-dar2003approximate,
  author    = {Even-Dar, Eyal and Mansour, Yishay},
  title     = {Approximate equivalence of {M}arkov decision processes},
  booktitle = {Learning Theory and Kernel Machines},
  year      = {2003},
  pages     = {581--594},
}

@InProceedings{ferns2004metrics,
  author    = {Ferns, Norm and Panangaden, Prakash and Precup, Doina},
  title     = {Metrics for finite {M}arkov decision processes},
  booktitle = {Proceedings of Uncertainty in Artificial Intelligence},
  year      = {2004},
  pages     = {162--169},
}

@InProceedings{taylor2008bounding,
  author    = {Taylor, Jonathan and Precup, Doina and Panagaden, Prakash},
  title     = {Bounding performance loss in approximate {MDP} homomorphisms},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2008},
  pages     = {1649--1656},
}

@InProceedings{ferns2006methods,
  author    = {Ferns, Norm and Castro, Pablo Samuel and Precup, Doina and Panangaden, Prakash},
  title     = {Methods for computing state similarity in markov decision processes},
  booktitle = {Proceedings of Uncertainty in Artificial Intelligence},
  year      = {2006},
  pages     = {174-181},
}

@InProceedings{narayanamurthy2007efficientlya,
  author    = {Narayanamurthy, Shravan Matthur and Ravindran, Balaraman},
  title     = {Efficiently {E}xploiting {S}ymmetries in {R}eal {T}ime {D}ynamic {P}rogramming},
  booktitle = {International Joint Conference on Artificial Intelligence},
  year      = {2007},
  pages     = {2556--2561},
}

@Article{whitt1978approximations,
  author    = {Whitt, Ward},
  title     = {{Approximations of dynamic programs, I}},
  journal   = {Mathematics of Operations Research},
  year      = {1978},
  volume    = {3},
  number    = {3},
  pages     = {231--243},
  publisher = {INFORMS},
}

@Article{ortner2014selecting,
  author  = {Ortner, Ronald and Maillard, Odalric-Ambrym and Ryabko, Daniil},
  title   = {Selecting {N}ear-{O}ptimal {A}pproximate {S}tate {R}epresentations in {R}einforcement {L}earning},
  journal = {arXiv preprint arXiv:1405.2652},
  year    = {2014},
}

@InProceedings{hallak2013model,
  author    = {Hallak, Assaf and Di-Castro, Dotan and Mannor, Shie},
  title     = {Model selection in markovian processes},
  booktitle = {Proceedings of the 19th ACM SIGKDD Conference on Knowledge Discovery and Data mining},
  year      = {2013},
  pages     = {374--382},
}
@InProceedings{bartlett2009regal,
  author       = {Bartlett, Peter L and Tewari, Ambuj},
  title        = {{REGAL}: {A} regularization based algorithm for reinforcement learning in weakly communicating {MDP}s},
  booktitle    = {Proceedings of the 25th Conference on Uncertainty in Artificial Intelligence},
  year         = {2009},
  pages        = {35--42},
  organization = {AUAI Press},
}

@InProceedings{mandel2014offline,
  author    = {Mandel, Travis and Liu, Yun-En and Levine, Sergey and Brunskill, Emma and Popovic, Zoran},
  title     = {Offline policy evaluation across representations with applications to educational games},
  booktitle = {Proceedings of the 13th International Conference on Autonomous Agents and Multi-Agent Systems},
  year      = {2014},
  pages     = {1077--1084},
}

@InProceedings{parr2008analysis,
  author    = {Parr, Ronald and Li, Lihong and Taylor, Gavin and Painter-Wakefield, Christopher and Littman, Michael L},
  title     = {An analysis of linear models, linear value-function approximation, and feature selection for reinforcement learning},
  booktitle = {Proceedings of the 25th International Conference on Machine Learning},
  year      = {2008},
  pages     = {752--759},
}

@Article{ormoneit2002kernel,
  author    = {Ormoneit, Dirk and Sen, {\'S}aunak},
  title     = {Kernel-based reinforcement learning},
  journal   = {Machine learning},
  year      = {2002},
  volume    = {49},
  number    = {2-3},
  pages     = {161--178},
  publisher = {Kluwer Academic Publishers},
}

@InProceedings{ravindran2004approximate,
  author    = {Ravindran, Balaraman and Barto, Andrew},
  title     = {Approximate homomorphisms: {A} framework for nonexact minimization in {M}arkov decision processes},
  booktitle = {Proceedings of the 5th International Conference on Knowledge-Based Computer Systems},
  year      = {2004},
}

@Article{bartlett2003rademacher,
  author    = {Bartlett, Peter L and Mendelson, Shahar},
  title     = {Rademacher and {G}aussian complexities: {R}isk bounds and structural results},
  journal   = {The Journal of Machine Learning Research},
  year      = {2003},
  volume    = {3},
  pages     = {463--482},
  publisher = {JMLR.org},
}

@Article{boutilier2000stochastic,
  author    = {Boutilier, Craig and Dearden, Richard and Goldszmidt, Mois{\'e}s},
  title     = {{S}tochastic dynamic programming with factored representations},
  journal   = {Artificial Intelligence},
  year      = {2000},
  volume    = {121},
  number    = {1},
  pages     = {49--107},
  publisher = {Elsevier},
}

@Article{brafman2003r,
  author    = {Brafman, Ronen I and Tennenholtz, Moshe},
  title     = {R-max-a general polynomial time algorithm for near-optimal reinforcement learning},
  journal   = {The Journal of Machine Learning Research},
  year      = {2003},
  volume    = {3},
  pages     = {213--231},
  publisher = {JMLR.org},
}

@Article{kaelbling1996reinforcement,
  author  = {Kaelbling, Leslie Pack and Littman, Michael L and Moore, Andrew W},
  title   = {Reinforcement {L}earning: {A} {S}urvey},
  journal = {Journal of Artificial Intelligence Research},
  year    = {1996},
  volume  = {4},
  pages   = {237--285},
}

@Book{kearns1994introduction,
  title     = {An {I}ntroduction to {C}omputational {L}earning {T}heory},
  publisher = {MIT Press},
  year      = {1994},
  author    = {Kearns, Michael J and Vazirani, Umesh V},
  isbn      = {9780262111935},
  lccn      = {lc94016588},
  url       = {http://books.google.com/books?id=vCA01wY6iywC},
}

@Article{kearns2002optimal,
  author    = {Kearns, Michael and Singh, Satinder},
  title     = {Near-optimal reinforcement learning in polynomial time},
  journal   = {Machine Learning},
  year      = {2002},
  volume    = {49},
  number    = {2-3},
  pages     = {209--232},
  publisher = {Springer},
}

@InCollection{koltchinskii2000rademacher,
  author    = {Koltchinskii, Vladimir and Panchenko, Dmitriy},
  title     = {Rademacher processes and bounding the risk of function learning},
  booktitle = {High Dimensional Probability II},
  publisher = {Springer},
  year      = {2000},
  pages     = {443--457},
}

@Article{mannor2007bias,
  author    = {Mannor, Shie and Simester, Duncan and Sun, Peng and Tsitsiklis, John N},
  title     = {Bias and variance approximation in value function estimates},
  journal   = {Management Science},
  year      = {2007},
  volume    = {53},
  number    = {2},
  pages     = {308--322},
  publisher = {INFORMS},
}

@PhdThesis{paduraru2013policy,
  author = {Paduraru, Cosmin},
  title  = {Off-policy {E}valuation in {M}arkov {D}ecision {P}rocesses},
  school = {McGill University},
  year   = {2013},
}

@InProceedings{strehl2005theoretical,
  author       = {Strehl, Alexander L and Littman, Michael L},
  title        = {A theoretical analysis of model-based interval estimation},
  booktitle    = {Proceedings of the 22nd International Conference on Machine learning},
  year         = {2005},
  pages        = {856--863},
  organization = {ACM},
}

@Article{strehl2009reinforcement,
  author    = {Strehl, Alexander L and Li, Lihong and Littman, Michael L},
  title     = {Reinforcement learning in finite {MDP}s: {PAC} analysis},
  journal   = {The Journal of Machine Learning Research},
  year      = {2009},
  volume    = {10},
  pages     = {2413--2444},
  publisher = {JMLR.org},
}

@InProceedings{tewari2006sample,
  author    = {Tewari, Ambuj and Bartlett, Peter L},
  title     = {Sample complexity of policy search with known dynamics},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2006},
  volume    = {19},
  pages     = {97--104},
}

@InProceedings{vapnik1992principles,
  author    = {Vapnik, Vladimir},
  title     = {Principles of risk minimization for learning theory},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {1992},
  pages     = {831--838},
}

@Book{vapnik1998statistical,
  title     = {Statistical learning theory},
  publisher = {Wiley New York},
  year      = {1998},
  author    = {Vapnik, Vladimir},
  volume    = {2},
}

@InProceedings{jiang2014improving,
  author    = {Jiang, Nan and Singh, Satinder and Lewis, Richard},
  title     = {Improving {UCT} planning via approximate homomorphisms},
  booktitle = {Proceedings of the 13th International Conference on Autonomous Agents and Multi-Agent Systems},
  year      = {2014},
  pages     = {1289--1296},
}

@InProceedings{paduraru2008model,
  author    = {Paduraru, Cosmin and Kaplow, Robert and Precup, Doina and Pineau, Joelle},
  title     = {Model-based Reinforcement Learning with State Aggregation},
  booktitle = {8th European Workshop on Reinforcement Learning},
  year      = {2008},
}

@PhdThesis{talvitie2010simple,
  author = {Talvitie, Erik N},
  title  = {Simple Partial Models for Complex Dynamical Systems},
  school = {The University of Michigan},
  year   = {2010},
}

@InProceedings{jong2005state,
  author    = {Jong, Nicholas K and Stone, Peter},
  title     = {State abstraction discovery from irrelevant state variables},
  booktitle = {Proceedings of the 19th International Joint Conference on Artificial Intelligence},
  year      = {2005},
  pages     = {752--757},
}

@Article{singh1994upper,
  author  = {Singh, Satinder and Yee, Richard},
  title   = {An upper bound on the loss from approximate optimal-value functions},
  journal = {Machine Learning},
  year    = {1994},
  volume  = {16},
  number  = {3},
  pages   = {227--233},
}

@InProceedings{dinculescu2010approximate,
  author    = {Dinculescu, Monica and Precup, Doina},
  title     = {Approximate predictive representations of partially observable systems},
  booktitle = {Proceedings of the 27th International Conference on Machine Learning (ICML-10)},
  year      = {2010},
  pages     = {895--902},
}

@Article{talvitie2011learning,
  author  = {Talvitie, Erik and Singh, Satinder},
  title   = {Learning to Make Predictions In Partially Observable Environments Without a Generative Model},
  journal = {Journal of Artificial Intelligence Research (JAIR)},
  year    = {2011},
  volume  = {42},
  pages   = {353--392},
}

@InProceedings{maillard2014how,
  author    = {Maillard, Odalric-Ambrym and Mann, Timothy A and Mannor, Shie},
  title     = {"{H}ow hard is my {MDP}?" {T}he distribution-norm to the rescue},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2014},
  pages     = {1835--1843},
}

@Article{seijen2014efficient,
  author  = {van Seijen, Harm and Whiteson, Shimon and Kester, Leon},
  title   = {Efficient abstraction selection in reinforcement learning},
  journal = {Computational Intelligence},
  year    = {2014},
  volume  = {30},
  number  = {4},
  pages   = {657--699},
}

@InProceedings{konidaris2009efficient,
  author    = {Konidaris, George and Barto, Andrew},
  title     = {Efficient {S}kill {L}earning using {A}bstraction {S}election},
  booktitle = {Proceedings of the 21st International Joint Conference on Artificial Intelligence},
  year      = {2009},
}

@Article{cobo2014abstraction,
  author  = {Cobo, Luis C and Subramanian, Kaushik and Isbell, Charles L and Lanterman, Aaron D and Thomaz, Andrea L},
  title   = {Abstraction from demonstration for efficient reinforcement learning in high-dimensional domains},
  journal = {Artificial Intelligence},
  year    = {2014},
  volume  = {216},
  pages   = {103--128},
}

@Article{farahmand2011regularization,
  author    = {Farahmand, Amir-massoud},
  title     = {Regularization in reinforcement learning},
  year      = {2011},
  publisher = {University of Alberta},
}

@Article{agarwal2016corralling,
  author  = {Agarwal, Alekh and Luo, Haipeng and Neyshabur, Behnam and Schapire, Robert E},
  title   = {Corralling a Band of Bandit Algorithms},
  journal = {arXiv preprint arXiv:1612.06246},
  year    = {2016},
}

@PhdThesis{scott2004dyadic,
  author = {Scott, Clayton D},
  title  = {Dyadic decision trees},
  school = {University of Wisconsin at Madison},
  year   = {2004},
}

@InProceedings{ng2003autonomous,
  author    = {Ng, Andrew Y and Kim, H Jin and Jordan, Michael I and Sastry, Shankar and Ballianda, Shiv},
  title     = {Autonomous helicopter flight via reinforcement learning},
  booktitle = {NIPS},
  year      = {2003},
  volume    = {16},
}

@InProceedings{johnson2016malmo,
  author    = {Johnson, Matthew and Hofmann, Katja and Hutton, Tim and Bignell, David},
  title     = {The malmo platform for artificial intelligence experimentation},
  booktitle = {International joint conference on artificial intelligence (IJCAI)},
  year      = {2016},
  pages     = {4246},
}

@Article{brockman2016openai,
  author  = {Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, John and Tang, Jie and Zaremba, Wojciech},
  title   = {{OpenAI} gym},
  journal = {arXiv preprint arXiv:1606.01540},
  year    = {2016},
}

@InProceedings{li2010contextual,
  author       = {Li, Lihong and Chu, Wei and Langford, John and Schapire, Robert E},
  title        = {A contextual-bandit approach to personalized news article recommendation},
  booktitle    = {Proceedings of the 19th international conference on World wide web},
  year         = {2010},
  pages        = {661--670},
  organization = {ACM},
}

@Article{lei2012a,
  author    = {Lei, H and Nahum-Shani, I and Lynch, K and Oslin, D and Murphy, SA},
  title     = {A" SMART" design for building individualized treatment sequences},
  journal   = {Annual review of clinical psychology},
  year      = {2012},
  volume    = {8},
  pages     = {21--48},
  publisher = {Annual Reviews},
}

@Article{singh2002optimizing,
  author  = {Singh, Satinder and Litman, Diane and Kearns, Michael and Walker, Marilyn},
  title   = {Optimizing dialogue management with reinforcement learning: {E}xperiments with the {NJF}un system},
  journal = {Journal of Artificial Intelligence Research},
  year    = {2002},
  volume  = {16},
  pages   = {105--133},
}

@Article{williams1992simple,
  author    = {Williams, Ronald J},
  title     = {Simple statistical gradient-following algorithms for connectionist reinforcement learning},
  journal   = {Machine learning},
  year      = {1992},
  volume    = {8},
  number    = {3-4},
  pages     = {229--256},
  publisher = {Springer},
}

@PhdThesis{li2009unifying,
  author = {Li, Lihong},
  title  = {A unifying framework for computational reinforcement learning theory},
  school = {Rutgers, The State University of New Jersey},
  year   = {2009},
}

@Book{mohri2012foundations,
  title     = {Foundations of machine learning},
  publisher = {MIT press},
  year      = {2012},
  author    = {Mohri, Mehryar and Rostamizadeh, Afshin and Talwalkar, Ameet},
}

@InProceedings{vanseijen2015deeper,
  author    = {van Seijen, Harm and Sutton, Rich},
  title     = {A Deeper Look at Planning as Learning from Replay},
  booktitle = {Proceedings of the 32nd International Conference on Machine Learning},
  year      = {2015},
  pages     = {2314--2322},
}

@Article{bradtke1996linear,
  author    = {Bradtke, Steven J and Barto, Andrew G},
  title     = {Linear least-squares algorithms for temporal difference learning},
  journal   = {Machine learning},
  year      = {1996},
  volume    = {22},
  number    = {1-3},
  pages     = {33--57},
  publisher = {Springer},
}

@Article{lin1992self,
  author    = {Lin, Long-Ji},
  title     = {Self-improving reactive agents based on reinforcement learning, planning and teaching},
  journal   = {Machine learning},
  year      = {1992},
  volume    = {8},
  number    = {3-4},
  pages     = {293--321},
  publisher = {Springer},
}

@InProceedings{wang2016dueling,
  author    = {Wang, Ziyu and Schaul, Tom and Hessel, Matteo and Hasselt, Hado van and Lanctot, Marc and Freitas, Nando de},
  title     = {Dueling Network Architectures for Deep Reinforcement Learning},
  booktitle = {Proceedings of the 33rd International Conference on Machine Learning},
  year      = {2016},
  pages     = {1995--2003},
}
{sutton1996generalization,
	title={Generalization in reinforcement learning: {S}uccessful examples using sparse coarse coding},
	author={Sutton, Richard S},
	journal={Advances in Neural Information Processing Systems},
	pages={1038--1044},
	year={1996}
%	publisher={MORGAN KAUFMANN PUBLISHERS}
}

@PhdThesis{watkins1989learning,
  author = {Watkins, Christopher John Cornish Hellaby},
  title  = {Learning from delayed rewards},
  school = {University of Cambridge England},
  year   = {1989},
}

@Article{hoeffding1963probability,
  author    = {Hoeffding, Wassily},
  title     = {Probability inequalities for sums of bounded random variables},
  journal   = {Journal of the American statistical association},
  year      = {1963},
  volume    = {58},
  number    = {301},
  pages     = {13--30},
  publisher = {Taylor \& Francis Group},
}

@InProceedings{strehl2006pac,
  author       = {Strehl, Alexander L and Li, Lihong and Wiewiora, Eric and Langford, John and Littman, Michael L},
  title        = {{PAC} model-free reinforcement learning},
  booktitle    = {Proceedings of the 23rd international conference on Machine learning},
  year         = {2006},
  pages        = {881--888},
  organization = {ACM},
}

@Article{auer2007logarithmic,
  author    = {Auer, Peter and Ortner, Ronald},
  title     = {Logarithmic online regret bounds for undiscounted reinforcement learning},
  journal   = {Advances in Neural Information Processing Systems},
  year      = {2007},
  volume    = {19},
  pages     = {49},
  publisher = {MIT; 1998},
}

@Article{jaksch2010optimal,
  author  = {Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
  title   = {Near-optimal regret bounds for reinforcement learning},
  journal = {Journal of Machine Learning Research},
  year    = {2010},
  volume  = {11},
  number  = {Apr},
  pages   = {1563--1600},
}

@PhdThesis{kakade2003sample,
  author = {Kakade, Sham Machandranath},
  title  = {On the sample complexity of reinforcement learning},
  school = {University of College London},
  year   = {2003},
}

@InProceedings{krishnamurthy2016pac,
  author    = {Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John},
  title     = {{PAC} reinforcement learning with rich observations},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2016},
  pages     = {1840--1848},
}


@InProceedings{jiang2017contextual,
  title = 	 {Contextual Decision Processes with low {B}ellman rank are {PAC}-Learnable},
  author = 	 {Nan Jiang and Akshay Krishnamurthy and Alekh Agarwal and John Langford and Robert E. Schapire},
  booktitle = 	 {International Conference on Machine Learning},
  year = 	 {2017}
}

@InProceedings{dann2015sample,
  author    = {Dann, Christoph and Brunskill, Emma},
  title     = {Sample complexity of episodic fixed-horizon reinforcement learning},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2015},
  pages     = {2818--2826},
}

@Article{valiant1984theory,
  author    = {Valiant, Leslie G},
  title     = {A theory of the learnable},
  journal   = {Communications of the ACM},
  year      = {1984},
  volume    = {27},
  number    = {11},
  pages     = {1134--1142},
  publisher = {ACM},
}

@InProceedings{langford2008epoch,
  author    = {Langford, John and Zhang, Tong},
  title     = {The epoch-greedy algorithm for multi-armed bandits with side information},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2008},
  pages     = {817--824},
}

@Article{puterman1994markov,
  author  = {Puterman, ML},
  title   = {Markov decision processes},
  journal = {Jhon Wiley \& Sons, New Jersey},
  year    = {1994},
}

@InProceedings{rosencrantz2004learning,
  author       = {Rosencrantz, Matthew and Gordon, Geoff and Thrun, Sebastian},
  title        = {Learning low dimensional predictive representations},
  booktitle    = {Proceedings of the twenty-first international conference on Machine learning},
  year         = {2004},
  pages        = {88},
  organization = {ACM},
}

@Article{kober2013reinforcement,
  author    = {Kober, Jens and Bagnell, J Andrew and Peters, Jan},
  title     = {Reinforcement learning in robotics: A survey},
  journal   = {The International Journal of Robotics Research},
  year      = {2013},
  pages     = {0278364913495721},
  publisher = {SAGE Publications},
}

@InProceedings{petrik2014raam,
  author    = {Petrik, Marek and Subramanian, Dharmashankar},
  title     = {{RAAM: The Benefits of Robustness in Approximating Aggregated MDPs in Reinforcement Learning}},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2014},
  pages     = {1979--1987},
}

@Article{vanroy2006performance,
  author    = {Van Roy, Benjamin},
  title     = {Performance loss bounds for approximate value iteration with state aggregation},
  journal   = {Mathematics of Operations Research},
  year      = {2006},
  volume    = {31},
  number    = {2},
  pages     = {234--244},
  publisher = {INFORMS},
}

@InProceedings{kulesza2015low,
  author    = {Kulesza, Alex and Jiang, Nan and Singh, Satinder},
  title     = {{Low-Rank Spectral Learning with Weighted Loss Functions}},
  booktitle = {Proceedings of the 18th International Conference on Artificial Intelligence and Statistics},
  year      = {2015},
  pages     = {517--525},
}

@InCollection{riedmiller2005neural,
  author    = {Riedmiller, Martin},
  title     = {{Neural fitted Q iteration--first experiences with a data efficient neural reinforcement learning method}},
  booktitle = {Machine Learning: ECML 2005},
  publisher = {Springer},
  year      = {2005},
  pages     = {317--328},
}

@InProceedings{kobilarov2015sample,
  author    = {Kobilarov, Marin},
  title     = {{Sample Complexity Bounds for Iterative Stochastic Policy Optimization}},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2015},
  pages     = {3096--3104},
}

@Article{bellman1956dynamic,
  author    = {Bellman, Richard},
  title     = {{Dynamic programming and Lagrange multipliers}},
  journal   = {Proceedings of the National Academy of Sciences},
  year      = {1956},
  volume    = {42},
  number    = {10},
  pages     = {767--769},
  publisher = {National Acad Sciences},
}

@Book{kumar2012customer,
  title     = {{Customer relationship management: Concept, strategy, and tools}},
  publisher = {Springer Science \& Business Media},
  year      = {2012},
  author    = {Kumar, Vineet and Reinartz, Werner},
}

@Article{murphy2003optimal,
  author    = {Murphy, Susan A},
  title     = {Optimal dynamic treatment regimes},
  journal   = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
  year      = {2003},
  volume    = {65},
  number    = {2},
  pages     = {331--355},
  publisher = {Wiley Online Library},
}

@InProceedings{hamilton2013modelling,
  author    = {Hamilton, William L and Fard, Mahdi M and Pineau, Joelle},
  title     = {Modelling sparse dynamical systems with compressed predictive state representations},
  booktitle = {Proceedings of the 30th International Conference on Machine Learning (ICML-13)},
  year      = {2013},
  pages     = {178--186},
}

@InProceedings{boots2010closing,
  author    = {Boots, Byron and Siddiqi, Sajid M and Gordon, Geoffrey J},
  title     = {Closing the learning-planning loop with predictive state representations},
  booktitle = {Proceedings of the 9th International Conference on Autonomous Agents and Multiagent Systems},
  year      = {2010},
  pages     = {1369--1370},
}

@InCollection{farahmand2008regularized,
  author    = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
  title     = {{Regularized fitted Q-iteration: Application to planning}},
  booktitle = {Recent Advances in Reinforcement Learning},
  publisher = {Springer Berlin Heidelberg},
  year      = {2008},
  pages     = {55--68},
}

@InProceedings{farahmand2009regularized,
  author       = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
  title        = {{Regularized fitted Q-iteration for planning in continuous-space Markovian decision problems}},
  booktitle    = {American Control Conference.},
  year         = {2009},
  pages        = {725--730},
  organization = {IEEE},
}

@InProceedings{szepesvari2005finite,
  author       = {Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
  title        = {Finite time bounds for sampling based fitted value iteration},
  booktitle    = {Proceedings of the 22nd international conference on Machine learning},
  year         = {2005},
  pages        = {880--887},
  organization = {ACM},
}
@Article{lazaric2012finite,
  author    = {Lazaric, Alessandro and Ghavamzadeh, Mohammad and Munos, R{\'e}mi},
  title     = {Finite-sample analysis of least-squares policy iteration},
  journal   = {The Journal of Machine Learning Research},
  year      = {2012},
  volume    = {13},
  number    = {1},
  pages     = {3041--3074},
  publisher = {JMLR.org},
}

@Article{lagoudakis2003least,
  author    = {Lagoudakis, Michail G and Parr, Ronald},
  title     = {Least-squares policy iteration},
  journal   = {The Journal of Machine Learning Research},
  year      = {2003},
  volume    = {4},
  pages     = {1107--1149},
  publisher = {JMLR.org},
}

@Article{bertsekas2011approximate,
  author    = {Bertsekas, Dimitri P},
  title     = {Approximate policy iteration: A survey and some new methods},
  journal   = {Journal of Control Theory and Applications},
  year      = {2011},
  volume    = {9},
  number    = {3},
  pages     = {310--335},
  publisher = {Springer},
}

@InProceedings{gordon1995stable,
  author    = {Gordon, Geoffrey J},
  title     = {Stable function approximation in dynamic programming},
  booktitle = {Proceedings of the twelfth international conference on machine learning},
  year      = {1995},
  pages     = {261--268},
}

@Book{bertsekas1996neuro,
  title     = {Neuro-Dynamic Programming},
  publisher = {Athena Scientific, Belmont, MA},
  year      = {1996},
  author    = {Bertsekas, Dimitri P and Tsitsiklis, John N},
}

@Article{bellman1959functional,
  author    = {Bellman, Richard and Dreyfus, Stuart},
  title     = {Functional approximations and dynamic programming},
  journal   = {Mathematical Tables and Other Aids to Computation},
  year      = {1959},
  volume    = {13},
  number    = {68},
  pages     = {247--251},
  publisher = {JSTOR},
}

@Article{pires2016policy,
  author  = {Pires, Bernardo {\'A}vila and Szepesv{\'a}ri, Csaba},
  title   = {Policy Error Bounds for Model-Based Reinforcement Learning with Factored Linear Models},
  journal = {arXiv preprint arXiv:1602.06346},
  year    = {2016},
}
@Article{mnih2015human,
  author    = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and others},
  title     = {Human-level control through deep reinforcement learning},
  journal   = {Nature},
  year      = {2015},
  volume    = {518},
  number    = {7540},
  pages     = {529--533},
  publisher = {Nature Publishing Group},
}

@Article{silver2016mastering,
  author    = {Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and van den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
  title     = {Mastering the game of Go with deep neural networks and tree search},
  journal   = {Nature},
  year      = {2016},
  volume    = {529},
  number    = {7587},
  pages     = {484--489},
  publisher = {Nature Publishing Group},
}

@InProceedings{jiang2016doubly,
  author    = {Nan Jiang and Lihong Li},
  title     = {{Doubly Robust Off-policy Value Evaluation for Reinforcement Learning}},
  booktitle = {Proceedings of The 33rd International Conference on Machine Learning},
  year      = {2016},
  volume    = {48},
  pages     = {652--661},
}

@InProceedings{thomas2016data,
  author    = {Thomas, Philip and Brunskill, Emma},
  title     = {{Data-Efficient Off-Policy Policy Evaluation for Reinforcement Learning}},
  booktitle = {Proceedings of the 33rd International Conference on Machine Learning},
  year      = {2016},
}

@InProceedings{li2015minimax,
  author    = {Li, Lihong and Munos, R{\'e}mi and Szepesv{\'a}ri, Csaba},
  title     = {Toward Minimax Off-policy Value Estimation},
  booktitle = {Proceedings of the 18th International Conference on Artificial Intelligence and Statistics},
  year      = {2015},
}

@InProceedings{kakade2002approximately,
  author    = {Kakade, Sham and Langford, John},
  title     = {{Approximately Optimal Approximate Reinforcement Learning}},
  booktitle = {Proceedings of the 19th International Conference on Machine Learning},
  year      = {2002},
  volume    = {2},
  pages     = {267--274},
}

@InProceedings{jiang2015abstraction,
  author    = {Jiang, Nan and Kulesza, Alex and Singh, Satinder},
  title     = {{Abstraction Selection in Model-based Reinforcement Learning}},
  booktitle = {Proceedings of the 32nd International Conference on Machine Learning},
  year      = {2015},
  pages     = {179--188},
}

@InProceedings{thomas2015high,
  author    = {Thomas, Philip and Theocharous, Georgios and Ghavamzadeh, Mohammad},
  title     = {{High Confidence Off-Policy Evaluation}},
  booktitle = {Proceedings of the 29th AAAI Conference on Artificial Intelligence},
  year      = {2015},
}

@InProceedings{dudik2011doubly,
  author    = {Dud{\'\i}k, Miroslav and Langford, John and Li, Lihong},
  title     = {Doubly {R}obust {P}olicy {E}valuation and {L}earning},
  booktitle = {Proceedings of the 28th International Conference on Machine Learning},
  year      = {2011},
  pages     = {1097--1104},
}

@InProceedings{thomas2015higha,
  author    = {Thomas, Philip and Theocharous, Georgios and Ghavamzadeh, Mohammad},
  title     = {{High Confidence Policy Improvement}},
  booktitle = {Proceedings of the 32nd International Conference on Machine Learning},
  year      = {2015},
  pages     = {2380--2388},
}

@InProceedings{kearns2000bias,
  author       = {Kearns, Michael J and Singh, Satinder P},
  title        = {{Bias-Variance Error Bounds for Temporal Difference Updates}},
  booktitle    = {Proceedings of the Thirteenth Annual Conference on Computational Learning Theory},
  year         = {2000},
  pages        = {142--147},
  organization = {Morgan Kaufmann Publishers Inc.},
}

@InProceedings{sutton2014new,
  author    = {Sutton, Rich and Mahmood, Ashique R and Precup, Doina and Hasselt, Hado V},
  title     = {A new {Q}($\lambda$) with interim forward view and {M}onte {C}arlo equivalence},
  booktitle = {Proceedings of the 31st International Conference on Machine Learning},
  year      = {2014},
  pages     = {568--576},
}

@Article{singh1996reinforcement,
  author    = {Singh, Satinder P and Sutton, Richard S},
  title     = {Reinforcement learning with replacing eligibility traces},
  journal   = {Machine learning},
  year      = {1996},
  volume    = {22},
  number    = {1-3},
  pages     = {123--158},
  publisher = {Springer},
}

@InCollection{kocsis2006bandit,
  author    = {Kocsis, Levente and Szepesv{\'a}ri, Csaba},
  title     = {Bandit based monte-carlo planning},
  booktitle = {Machine Learning: ECML 2006},
  publisher = {Springer Berlin Heidelberg},
  year      = {2006},
  pages     = {282--293},
}

@Misc{hettich1999uci,
  author       = {Hettich, S and Bay, S D},
  title        = {{The UCI KDD Archive}},
  howpublished = {\url{http://kdd.ics.uci.edu}},
  year         = {1999},
}

@InProceedings{precup2001policy,
  author    = {Doina Precup and Richard S Sutton and Sanjoy Dasgupta},
  title     = {{Off-Policy Temporal-Difference Learning with Funtion Approximation}},
  booktitle = {Proceedings of the 18th International Conference on Machine Learning},
  year      = {2001},
  pages     = {417--424},
}

@InProceedings{precup2000eligibility,
  author    = {Doina Precup and Richard S Sutton and Satinder P Singh},
  title     = {{Eligibility Traces for Off-Policy Policy Evaluation}},
  booktitle = {Proceedings of the 17th International Conference on Machine Learning},
  year      = {2000},
  pages     = {759--766},
}

@Book{sutton1998reinforcement,
  title     = {Reinforcement Learning: An Introduction},
  publisher = {MIT Press},
  year      = {1998},
  author    = {Richard S Sutton and Andrew G Barto},
  address   = {Cambridge, MA},
  month     = {March},
  isbn      = {0-262-19398-1},
}

@Article{murphy2001marginal,
  author  = {Susan A. Murphy and Mark van der Laan and James M. Robins},
  title   = {{Marginal Mean Models for Dynamic Regimes}},
  journal = {Journal of the American Statistical Association},
  year    = {2001},
  volume  = {96},
  number  = {456},
  pages   = {1410--1423},
}

@InProceedings{li2011unbiased,
  author    = {Lihong Li and Wei Chu and John Langford and Xuanhui Wang},
  title     = {{Unbiased Offline Evaluation of Contextual-bandit-based News Article Recommendation Algorithms}},
  booktitle = {Proceedings of the 4th International Conference on Web Search and Data Mining},
  year      = {2011},
  pages     = {297--306},
}

@Article{bottou2013counterfactual,
  author  = {L\'{e}on Bottou and Jonas Peters and Joaquin Qui{\~n}onero-Candela and Denis Xavier Charles and D. Max Chickering and Elon Portugaly and Dipankar Ray and Patrice Simard and Ed Snelson},
  title   = {{Counterfactual Reasoning and Learning Systems: The Example of Computational Advertising}},
  journal = {Journal of Machine Learning Research},
  year    = {2013},
  volume  = {14},
  pages   = {3207--3260},
}

@InProceedings{sutton1998intra,
  author    = {Richard S Sutton and Doina Precup and Satinder P Singh},
  title     = {{Intra-option Learning about Temporally Abstract Actions}},
  booktitle = {Proceedings of the 15th International Conference on Machine Learning},
  year      = {1998},
  pages     = {556--564},
}

@PhdThesis{precup2000temporal,
  author = {Precup, Doina},
  title  = {Temporal abstraction in reinforcement learning},
  school = {University of Massachusetts Amherst},
  year   = {2000},
}

@PhdThesis{thomas2015safe,
  author = {Thomas, Philip},
  title  = {Safe Reinforcement Learning},
  school = {University of Massachusetts Amherst},
  year   = {2015},
}

@InProceedings{gruenewaelder2012modelling,
  author    = {Gr{\"u}new{\"a}lder, S and Lever, G and Baldassarre, L and Pontil, M and Gretton, A},
  title     = {{Modelling transition dynamics in MDPs with RKHS embeddings}},
  booktitle = {Proceedings of the 29th International Conference on Machine Learning},
  year      = {2012},
  pages     = {535--542},
}

@InProceedings{jong2007model,
  author    = {Jong, Nicholas K and Stone, Peter},
  title     = {Model-based function approximation in reinforcement learning},
  booktitle = {Proceedings of the 6th International Conference on Autonomous Agents and Multiagent Systems},
  year      = {2007},
  pages     = {95},
}

@Article{dann2014policy,
  author  = {Dann, Christoph and Neumann, Gerhard and Peters, Jan},
  title   = {Policy evaluation with temporal differences: {A} survey and comparison},
  journal = {The Journal of Machine Learning Research},
  year    = {2014},
  volume  = {15},
  number  = {1},
  pages   = {809--883},
}

@Article{fonteneau2013batch,
  author  = {Raphael Fonteneau and Susan A. Murphy and Louis Wehenkel and Damien Ernst},
  title   = {Batch Mode Reinforcement Learning based on the Synthesis of Artificial Trajectories},
  journal = {Annals of Operations Research},
  year    = {2013},
  volume  = {208},
  number  = {1},
  pages   = {383--416},
}

@Article{rotnitzky1995semiparametric,
  author  = {Andrea Rotnitzky and James M. Robins},
  title   = {Semiparametric Regression Estimation in the Presence of Dependent Censoring},
  journal = {Biometrika},
  year    = {1995},
  volume  = {82},
  number  = {4},
  pages   = {805--820},
}

@InProceedings{pirotta2013safe,
  author    = {Matteo Pirotta and Marcello Restelli and Alessio Pecorino and Daniele Calandriello},
  title     = {Safe Policy Iteration},
  booktitle = {Proceedings of the 30th International Conference on Machine Learning},
  year      = {2013},
  number    = {3},
  pages     = {307--317},
}

@Book{pearl2009causality,
  title     = {Causality: Models, Reasoning, and Inference},
  publisher = {Cambridge University Press},
  year      = {2009},
  author    = {Judea Pearl},
  edition   = {2nd},
  isbn      = {052189560X},
}

@Article{holland1986statistics,
  author  = {Paul W. Holland},
  title   = {Statistics and Causal Inference},
  journal = {Journal of the American Statistical Association},
  year    = {1986},
  volume  = {81},
  number  = {6},
  pages   = {945--960},
}

@Unpublished{sutton2015emphatic,
  author = {Richard S Sutton and Ashique Rupam Mahmood and Martha White},
  title  = {An Emphatic Approach to the Problem of Off-policy Temporal-Difference Learning},
  note   = {CoRR abs/1503.04269},
  year   = {2015},
}

@Article{hirano2003efficient,
  author  = {Hirano, Keisuke and Imbens, Guido W. and Ridder, Geert},
  title   = {Efficient Estimation of Average Treatment Effects using the Estimated Propensity Score},
  journal = {Econometrica},
  year    = {2003},
  volume  = {71},
  number  = {4},
  pages   = {1161--1189},
}

@InProceedings{pednault2002sequential,
  author     = {Edwin P. D. Pednault and Naoki Abe and Bianca Zadrozny},
  title      = {Sequential Cost-Sensitive Decision-Making with Reinforcement Learning},
  booktitle  = {KDD},
  year       = {2002},
  pages      = {259--268},
  booktitle1 = {Proceedings of the 8th International Conference on Knowledge Discovery and Data Mining},
}

@PhdThesis{jr2010theory,
  author = {Moore Jr, Terrence Joseph},
  title  = {A theory of Cramer-Rao bounds for constrained parametric models},
  school = {University of Maryland, College Park},
  year   = {2010},
}

@PhdThesis{marivate2015improved,
  author  = {Vukosi N. Marivate},
  title   = {Improved Empirical Methods in Reinforcement-Learning Evaluation},
  school  = {Rutgers University},
  year    = {2015},
  address = {New Brunswick, NJ},
}

@InProceedings{marivate2014quantifying,
  author    = {Marivate, Vukosi Ntsakisi and Chemali, Jessica and Brunskill, Emma and Littman, Michael},
  title     = {Quantifying uncertainty in batch personalized sequential decision making},
  booktitle = {Workshops at the 28th AAAI Conference on Artificial Intelligence},
  year      = {2014},
}

@Article{jiang2015doubly,
  author  = {Jiang, Nan and Li, Lihong},
  title   = {Doubly Robust Off-policy Evaluation for Reinforcement Learning},
  journal = {arXiv preprint arXiv:1511.03722},
  year    = {2015},
}

@InProceedings{singh2004predictive,
  author       = {Singh, Satinder and James, Michael R and Rudary, Matthew R},
  title        = {{Predictive state representations: A new theory for modeling dynamical systems}},
  booktitle    = {Proceedings of the 20th Conference on Uncertainty in Artificial Intelligence},
  year         = {2004},
  pages        = {512--519},
  organization = {AUAI Press},
}

@InProceedings{doshi-velez2009infinite,
  author    = {Doshi-Velez, Finale},
  title     = {{The Infinite Partially observable Markov decision process}},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2009},
  pages     = {477--485},
}

@Misc{li2015recurrent,
  author = {Xiujun Li and Lihong Li and Jianfeng Gao and Xiaodong He and Jianshu Chen and Li Deng and Ji He},
  title  = {Recurrent Reinforcement Learning: A Hybrid Approach},
  year   = {2015},
  note   = {arXiv:1509.03044},
}

@InProceedings{asmuth2011approaching,
  author    = {Asmuth, John and Littman, Michael L},
  title     = {Approaching {B}ayes-optimalilty using {M}onte-{C}arlo tree search},
  booktitle = {Proceedings of the 21st International Conference on Automatated Planning and Scheduling},
  year      = {2011},
}

@Article{bagnell2001solving,
  author    = {Bagnell, J Andrew and Ng, Andrew Y and Schneider, Jeff G},
  title     = {Solving uncertain {M}arkov decision processes},
  year      = {2001},
  publisher = {Carnegie Mellon University},
}

@Article{bertuccelli2012robust,
  author    = {Bertuccelli, Luca F and Wu, Albert and How, Jonathan P},
  title     = {Robust Adaptive Markov Decision Processes: Planning with Model Uncertainty},
  journal   = {Control Systems, IEEE},
  year      = {2012},
  volume    = {32},
  number    = {5},
  pages     = {96--109},
  publisher = {IEEE},
}

@InCollection{el-yaniv2007transductive,
  author    = {El-Yaniv, Ran and Pechyony, Dmitry},
  title     = {{T}ransductive rademacher complexity and its applications},
  booktitle = {Learning Theory},
  publisher = {Springer},
  year      = {2007},
  pages     = {157--171},
}

@Article{even-dar2009online,
  author    = {Even-Dar, Eyal and Kakade, Sham M and Mansour, Yishay},
  title     = {Online {M}arkov decision processes},
  journal   = {Mathematics of Operations Research},
  year      = {2009},
  volume    = {34},
  number    = {3},
  pages     = {726--736},
  publisher = {INFORMS},
}

@InProceedings{guez2012efficient,
  author    = {Guez, Arthur and Silver, David and Dayan, Peter},
  title     = {Efficient {B}ayes-{A}daptive {R}einforcement {L}earning using {S}ample-{B}ased {S}earch},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2012},
  pages     = {1034--1042},
}

@Article{browne2012survey,
  author  = {Browne, Cameron B and Powley, Edward and Whitehouse, Daniel and Lucas, Simon M and Cowling, Peter I and Rohlfshagen, Philipp and Tavener, Stephen and Perez, Diego and Samothrakis, Spyridon and Colton, Simon},
  title   = {A survey of {M}onte {C}arlo tree search methods},
  journal = {IEEE Transactions on Computational Intelligence and AI in Games},
  year    = {2012},
  volume  = {4},
  number  = {1},
  pages   = {1--43},
}

@Book{montenegro2006mathematical,
  title     = {Mathematical aspects of mixing times in {M}arkov chains},
  publisher = {Now Publishers Inc},
  year      = {2006},
  author    = {Montenegro, Ravi R and Tetali, Prasad},
}
{ng2000pegasus,
  title =	 {{PEGASUS}: {A} policy search method for large {MDP}s and
                  {POMDP}s},
  author =	 {Ng, Andrew Y and Jordan, Michael},
  booktitle =	 {Proceedings of the16th Conference on
                  Uncertainty in Artificial Intelligence},
  pages =	 {406--415},
  year =	 2000
%  organization = {Morgan Kaufmann Publishers Inc.}
}

@Article{nilim2005robust,
  author  = {Nilim, Arnab and El Ghaoui, Laurent},
  title   = {Robust control of {M}arkov decision processes with uncertain transition matrices},
  journal = {Operations Research},
  year    = {2005},
  volume  = {53},
  number  = {5},
  pages   = {780--798},
}

@InProceedings{silver2010montea,
  author    = {Silver, David and Veness, Joel},
  title     = {{M}onte-{C}arlo planning in large {POMDP}s},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2010},
  pages     = {2164--2172},
}

@InProceedings{smith2004heuristic,
  author    = {Smith, Trey and Simmons, Reid},
  title     = {Heuristic search value iteration for {POMDP}s},
  booktitle = {Proceedings of the 20th Conference on Uncertainty in Artificial Intelligence},
  year      = {2004},
  pages     = {520--527},
}

@Article{ross2011bayesian,
  author    = {Ross, St{\'e}phane and Pineau, Joelle and Chaib-draa, Brahim and Kreitmann, Pierre},
  title     = {A {B}ayesian approach for learning and planning in partially observable {M}arkov decision processes},
  journal   = {The Journal of Machine Learning Research},
  year      = {2011},
  volume    = {12},
  pages     = {1729--1770},
  publisher = {JMLR.org},
}

@InProceedings{bjarnason2009lower,
  author    = {Bjarnason, Ronald and Fern, Alan and Tadepalli, Prasad},
  title     = {Lower bounding {K}londike solitaire with {M}onte-{C}arlo planning},
  booktitle = {Proceedings of International Conference on Automated Planning and Scheduling},
  year      = {2009},
  pages     = {26--33},
}

@InProceedings{zhu2009human,
  author    = {Zhu, Xiaojin and Gibson, Bryan R and Rogers, Timothy T},
  title     = {Human {R}ademacher complexity},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2009},
  pages     = {2322--2330},
}

@Manual{balcan2011cs,
  title        = {{CS} 8803 - {M}achine {L}earning {T}heory: {L}ecture {N}otes},
  author       = {Balcan, Maria-Florina},
  organization = {{G}eorgia {I}nstitute of {T}echnology},
  year         = {2011},
  note         = {\url{http://www.cs.cmu.edu/~ninamf/ML11/lect1115.pdf}},
}

@InProceedings{petrik2009biasing,
  author    = {Petrik, Marek and Scherrer, Bruno},
  title     = {Biasing approximate dynamic programming with a lower discount factor},
  booktitle = {Advances in Neural Information Processing Systems},
  year      = {2009},
  pages     = {1265--1272},
}

@Article{vapnik1999overview,
  author    = {Vapnik, Vladimir N},
  title     = {An overview of statistical learning theory},
  journal   = {Neural Networks, IEEE Transactions on},
  year      = {1999},
  volume    = {10},
  number    = {5},
  pages     = {988--999},
  publisher = {IEEE},
}

@Article{baxter2001infinite,
  author  = {Baxter, Jonathan and Bartlett, Peter L},
  title   = {{Infinite-Horizon Policy-Gradient Estimation}},
  journal = {Journal of Artificial Intelligence Research},
  year    = {2001},
  volume  = {15},
  pages   = {319--350},
}

@Article{amin2017repeated,
  author        = {Amin, Kareem and Jiang, Nan and Singh, Satinder},
  title         = {{Repeated Inverse Reinforcement Learning for AI Safety}},
  journal       = {arXiv preprint arXiv:1705.05427},
  year          = {2017},
  __markedentry = {[najiang:]},
}

@Article{bostrom2003ethical,
  author        = {Bostrom, Nick},
  title         = {Ethical issues in advanced artificial intelligence},
  journal       = {Science Fiction and Philosophy: From Time Travel to Superintelligence},
  year          = {2003},
  pages         = {277--284},
  __markedentry = {[najiang:]},
}

@InProceedings{walsh2010generalizing,
  author        = {Walsh, Thomas J and Subramanian, Kaushik and Littman, Michael L and Diuk, Carlos},
  title         = {Generalizing apprenticeship learning across hypothesis classes},
  booktitle     = {Proceedings of the 27th International Conference on Machine Learning (ICML-10)},
  year          = {2010},
  pages         = {1119--1126},
  __markedentry = {[najiang:]},
}

@Article{li2011knows,
  author        = {Li, Lihong and Littman, Michael L and Walsh, Thomas J and Strehl, Alexander L},
  title         = {Knows what it knows: a framework for self-aware learning},
  journal       = {Machine learning},
  year          = {2011},
  volume        = {82},
  number        = {3},
  pages         = {399--443},
  __markedentry = {[najiang:]},
  publisher     = {Springer},
}

@InProceedings{dani2008stochastic,
  author        = {Dani, Varsha and Hayes, Thomas P and Kakade, Sham M},
  title         = {Stochastic Linear Optimization under Bandit Feedback.},
  booktitle     = {COLT},
  year          = {2008},
  pages         = {355--366},
  __markedentry = {[najiang:]},
}

@InProceedings{abbasi-yadkori2012online,
  author        = {Abbasi-Yadkori, Yasin and Pal, David and Szepesvari, Csaba},
  title         = {Online-to-Confidence-Set Conversions and Application to Sparse Stochastic Bandits.},
  booktitle     = {AISTATS},
  year          = {2012},
  volume        = {22},
  pages         = {1--9},
  __markedentry = {[najiang:]},
}

@Article{mannor2004sample,
  author        = {Mannor, Shie and Tsitsiklis, John N},
  title         = {The sample complexity of exploration in the multi-armed bandit problem},
  journal       = {Journal of Machine Learning Research},
  year          = {2004},
  volume        = {5},
  number        = {Jun},
  pages         = {623--648},
  __markedentry = {[najiang:]},
}

@Article{auer2002using,
  author        = {Auer, Peter},
  title         = {Using confidence bounds for exploitation-exploration trade-offs},
  journal       = {Journal of Machine Learning Research},
  year          = {2002},
  volume        = {3},
  number        = {Nov},
  pages         = {397--422},
  __markedentry = {[najiang:]},
}

@InProceedings{neu2014online,
  author        = {Neu, Gergely and Valko, Michal},
  title         = {Online combinatorial optimization with stochastic decision sets and adversarial losses},
  booktitle     = {Advances in Neural Information Processing Systems},
  year          = {2014},
  pages         = {2780--2788},
  __markedentry = {[najiang:]},
}

@Article{kleinberg2010regret,
  author        = {Kleinberg, Robert and Niculescu-Mizil, Alexandru and Sharma, Yogeshwer},
  title         = {Regret bounds for sleeping experts and bandits},
  journal       = {Machine learning},
  year          = {2010},
  volume        = {80},
  number        = {2-3},
  pages         = {245--272},
  __markedentry = {[najiang:]},
  publisher     = {Springer},
}

@Article{blum2007external,
  author        = {Blum, Avrim and Mansour, Yishay},
  title         = {From external to internal regret},
  journal       = {Journal of Machine Learning Research},
  year          = {2007},
  volume        = {8},
  number        = {Jun},
  pages         = {1307--1324},
  __markedentry = {[najiang:]},
}

@InProceedings{freund1997using,
  author        = {Freund, Yoav and Schapire, Robert E and Singer, Yoram and Warmuth, Manfred K},
  title         = {Using and combining predictors that specialize},
  booktitle     = {Proceedings of the 29th Annual ACM Symposium on Theory of computing},
  year          = {1997},
  pages         = {334--343},
  organization  = {ACM},
  __markedentry = {[najiang:]},
}

@InProceedings{kanade2009sleeping,
  author        = {Kanade, Varun and McMahan, H Brendan and Bryan, Brent},
  title         = {Sleeping Experts and Bandits with Stochastic Action Availability and Adversarial Rewards},
  booktitle     = {AISTATS},
  year          = {2009},
  pages         = {272--279},
  __markedentry = {[najiang:]},
}

@Book{groetschel2012geometric,
  title         = {Geometric algorithms and combinatorial optimization},
  publisher     = {Springer Science \& Business Media},
  year          = {2012},
  author        = {Gr{\"o}tschel, Martin and Lov{\'a}sz, L{\'a}szl{\'o} and Schrijver, Alexander},
  volume        = {2},
  __markedentry = {[najiang:]},
}

@InProceedings{hadfield-menell2016cooperative,
  author        = {Hadfield-Menell, Dylan and Russell, Stuart J and Abbeel, Pieter and Dragan, Anca},
  title         = {Cooperative inverse reinforcement learning},
  booktitle     = {Advances in Neural Information Processing Systems},
  year          = {2016},
  pages         = {3909--3917},
  __markedentry = {[najiang:]},
}

@Article{russell2015research,
  author        = {Russell, Stuart and Dewey, Daniel and Tegmark, Max},
  title         = {Research priorities for robust and beneficial artificial intelligence},
  journal       = {AI Magazine},
  year          = {2015},
  volume        = {36},
  number        = {4},
  pages         = {105--114},
  __markedentry = {[najiang:]},
}

@Article{amodei2016concrete,
  author        = {Amodei, Dario and Olah, Chris and Steinhardt, Jacob and Christiano, Paul and Schulman, John and Man{\'e}, Dan},
  title         = {Concrete problems in {AI} safety},
  journal       = {arXiv preprint arXiv:1606.06565},
  year          = {2016},
  __markedentry = {[najiang:]},
}

@Article{todd1982minimum,
  author        = {Todd, Michael J},
  title         = {On minimum volume ellipsoids containing part of a given ellipsoid},
  journal       = {Mathematics of Operations Research},
  year          = {1982},
  volume        = {7},
  number        = {2},
  pages         = {253--261},
  __markedentry = {[najiang:]},
  publisher     = {INFORMS},
}

@InProceedings{auer1995gambling,
  author        = {Auer, Peter and Cesa-Bianchi, Nicolo and Freund, Yoav and Schapire, Robert E},
  title         = {Gambling in a rigged casino: The adversarial multi-armed bandit problem},
  booktitle     = {Foundations of Computer Science, 1995. Proceedings., 36th Annual Symposium on},
  year          = {1995},
  pages         = {322--331},
  organization  = {IEEE},
  __markedentry = {[najiang:]},
}

@Article{bartlett2008high,
  author        = {Bartlett, Peter L and Dani, Varsha and Hayes, Thomas and Kakade, Sham and Rakhlin, Alexander and Tewari, Ambuj},
  title         = {High-probability regret bounds for bandit online linear optimization},
  year          = {2008},
  pages         = {335–-342},
  __markedentry = {[najiang:]},
  booktitle     = {Proceedings of the 21st Annual Conference on Learning Theory},
}

@InProceedings{abernethy2008competing,
  author        = {Abernethy, Jacob and Hazan, Elad and Rakhlin, Alexander},
  title         = {Competing in the Dark: An Efficient Algorithm for Bandit Linear Optimization},
  booktitle     = {Conference on Learning Theory},
  year          = {2008},
  pages         = {263--274},
  __markedentry = {[najiang:]},
}

@InProceedings{dani2007price,
  author        = {Dani, Varsha and Kakade, Sham M and Hayes, Thomas P},
  title         = {The price of bandit information for online optimization},
  booktitle     = {Advances in Neural Information Processing Systems},
  year          = {2007},
  pages         = {345--352},
  __markedentry = {[najiang:]},
}

@Article{shalev-shwartz2011online,
  author        = {Shalev-Shwartz, Shai},
  title         = {Online learning and online convex optimization},
  journal       = {Foundations and Trends in Machine Learning},
  year          = {2011},
  volume        = {4},
  number        = {2},
  pages         = {107--194},
  __markedentry = {[najiang:]},
}

@InProceedings{hsu2015mixing,
  author        = {Hsu, Daniel J and Kontorovich, Aryeh and Szepesv{\'a}ri, Csaba},
  title         = {Mixing time estimation in reversible Markov chains from a single sample path},
  booktitle     = {Advances in Neural Information Processing Systems},
  year          = {2015},
  pages         = {1459--1467},
  __markedentry = {[najiang:]},
}

@Manual{odonnell201115,
  title         = {15-859({E}) -- linear and semidefinite programming: lecture notes},
  author        = {Ryan O'Donnell},
  organization  = {{Carnegie Mellon University}},
  year          = {2011},
  note          = {\url{https://www.cs.cmu.edu/afs/cs.cmu.edu/academic/class/15859-f11/www/notes/lecture08.pdf}},
  __markedentry = {[najiang:]},
}

@InProceedings{jiang2015dependence,
  author        = {Jiang, Nan and Kulesza, Alex and Singh, Satinder and Lewis, Richard},
  title         = {The Dependence of Effective Planning Horizon on Model Accuracy},
  booktitle     = {Proceedings of the 2015 International Conference on Autonomous Agents and Multiagent Systems},
  year          = {2015},
  pages         = {1181--1189},
  __markedentry = {[najiang:]},
}

@InProceedings{ng2000algorithms,
  author        = {Ng, Andrew Y and Russell, Stuart J},
  title         = {Algorithms for Inverse Reinforcement Learning},
  booktitle     = {Proceedings of the 17th International Conference on Machine Learning},
  year          = {2000},
  pages         = {663--670},
  __markedentry = {[najiang:]},
}

@InProceedings{abbeel2004apprenticeship,
  author        = {Abbeel, Pieter and Ng, Andrew Y},
  title         = {{Apprenticeship Learning via Inverse Reinforcement Learning}},
  booktitle     = {Proceedings of the 21st International Conference on Machine learning},
  year          = {2004},
  pages         = {1},
  organization  = {ACM},
  __markedentry = {[najiang:]},
}

@Article{amin2016towards,
  author        = {Amin, Kareem and Singh, Satinder},
  title         = {Towards Resolving Unidentifiability in Inverse Reinforcement Learning},
  journal       = {arXiv preprint arXiv:1601.06569},
  year          = {2016},
  __markedentry = {[najiang:]},
}

@Article{lovasz1999hit,
  author        = {Lov{\'a}sz, L{\'a}szl{\'o}},
  title         = {Hit-and-run mixes fast},
  journal       = {Mathematical Programming},
  year          = {1999},
  volume        = {86},
  number        = {3},
  pages         = {443--461},
  __markedentry = {[najiang:]},
  publisher     = {Springer},
}

@Article{lovasz2006simulated,
  author        = {Lov{\'a}sz, L{\'a}szl{\'o} and Vempala, Santosh},
  title         = {Simulated annealing in convex bodies and an O*(n4) volume algorithm},
  journal       = {Journal of Computer and System Sciences},
  year          = {2006},
  volume        = {72},
  number        = {2},
  pages         = {392--417},
  __markedentry = {[najiang:]},
  publisher     = {Elsevier},
}

@Article{cousinspractical,
  author        = {Cousins, Ben and Vempala, Santosh},
  title         = {A Practical Volume Algorithm},
  __markedentry = {[najiang:]},
}

@InProceedings{coates2008learning,
  author        = {Coates, Adam and Abbeel, Pieter and Ng, Andrew Y},
  title         = {Learning for control from multiple demonstrations},
  booktitle     = {Proceedings of the 25th international conference on Machine learning},
  year          = {2008},
  pages         = {144--151},
  organization  = {ACM},
  __markedentry = {[najiang:]},
}

@InProceedings{ziebart2008maximum,
  author        = {Ziebart, Brian D and Maas, Andrew L and Bagnell, J Andrew and Dey, Anind K},
  title         = {Maximum Entropy Inverse Reinforcement Learning},
  booktitle     = {AAAI},
  year          = {2008},
  pages         = {1433--1438},
  __markedentry = {[najiang:]},
}

@Article{ramachandran2007bayesian,
  author        = {Ramachandran, Deepak and Amir, Eyal},
  title         = {Bayesian inverse reinforcement learning},
  journal       = {Urbana},
  year          = {2007},
  volume        = {51},
  pages         = {61801},
  __markedentry = {[najiang:]},
}

@InProceedings{syed2007game,
  author        = {Syed, Umar and Schapire, Robert E},
  title         = {A game-theoretic approach to apprenticeship learning},
  booktitle     = {Advances in Neural Information Processing Systems},
  year          = {2007},
  pages         = {1449--1456},
  __markedentry = {[najiang:]},
}

@InProceedings{guillory2010interactive,
  author        = {Guillory, Andrew and Bilmes, Jeff},
  title         = {Interactive submodular set cover},
  booktitle     = {Proceedings of the International Conference on Machine Learning},
  year          = {2010},
  __markedentry = {[najiang:]},
}

@InProceedings{golovin2010adaptive,
  author        = {Golovin, Daniel and Krause, Andreas},
  title         = {Adaptive Submodularity: A New Approach to Active Learning and Stochastic Optimization},
  booktitle     = {COLT},
  year          = {2010},
  pages         = {333--345},
  __markedentry = {[najiang:]},
}

@InProceedings{ratliff2006maximum,
  author        = {Ratliff, Nathan D and Bagnell, J Andrew and Zinkevich, Martin A},
  title         = {Maximum margin planning},
  booktitle     = {Proceedings of the 23rd International Conference on Machine Learning},
  year          = {2006},
  pages         = {729--736},
  organization  = {ACM},
  __markedentry = {[najiang:]},
}

@InProceedings{regan2010robust,
  author        = {Regan, Kevin and Boutilier, Craig},
  title         = {Robust Policy Computation in Reward-Uncertain MDPs Using Nondominated Policies},
  booktitle     = {AAAI},
  year          = {2010},
  __markedentry = {[najiang:]},
}

@InProceedings{smart2002effective,
  author        = {Smart, William D and Kaelbling, Leslie Pack},
  title         = {Effective reinforcement learning for mobile robots},
  booktitle     = {Robotics and Automation, 2002. Proceedings. ICRA'02. IEEE International Conference on},
  year          = {2002},
  volume        = {4},
  pages         = {3404--3410},
  organization  = {IEEE},
  __markedentry = {[najiang:]},
}

@Article{abbeel2007application,
  author        = {Abbeel, Pieter and Coates, Adam and Quigley, Morgan and Ng, Andrew Y},
  title         = {An application of reinforcement learning to aerobatic helicopter flight},
  journal       = {Advances in Neural Information Processing Systems},
  year          = {2007},
  volume        = {19},
  pages         = {1},
  __markedentry = {[najiang:]},
  publisher     = {MIT; 1998},
}

@Article{coates2009apprenticeship,
  author        = {Coates, Adam and Abbeel, Pieter and Ng, Andrew Y},
  title         = {Apprenticeship learning for helicopter control},
  journal       = {Communications of the ACM},
  year          = {2009},
  volume        = {52},
  number        = {7},
  pages         = {97--105},
  __markedentry = {[najiang:]},
  publisher     = {ACM},
}

@Book{vonneumann2007theory,
  title         = {Theory of games and economic behavior (60th Anniversary Commemorative Edition)},
  publisher     = {Princeton university press},
  year          = {2007},
  author        = {Von Neumann, John and Morgenstern, Oskar},
  __markedentry = {[najiang:]},
}

@InProceedings{chajewska2000making,
  author        = {Chajewska, Urszula and Koller, Daphne and Parr, Ronald},
  title         = {Making rational decisions using adaptive utility elicitation},
  booktitle     = {AAAI/IAAI},
  year          = {2000},
  pages         = {363--369},
  __markedentry = {[najiang:]},
}

@InCollection{balcan2007margin,
  author        = {Balcan, Maria-Florina and Broder, Andrei and Zhang, Tong},
  title         = {Margin based active learning},
  booktitle     = {Learning Theory},
  publisher     = {Springer},
  year          = {2007},
  pages         = {35--50},
  __markedentry = {[najiang:]},
}

@Article{balcan2012active,
  author        = {Balcan, Maria Florina and Long, Philip M},
  title         = {Active and passive learning of linear separators under log-concave distributions},
  journal       = {arXiv preprint arXiv:1211.1082},
  year          = {2012},
  __markedentry = {[najiang:]},
}

@InProceedings{dasgupta2005coarse,
  author        = {Dasgupta, Sanjoy},
  title         = {Coarse sample complexity bounds for active learning},
  booktitle     = {Advances in Neural Information Processing Systems},
  year          = {2005},
  pages         = {235--242},
  __markedentry = {[najiang:]},
}

@InProceedings{regan2009regret,
  author        = {Regan, Kevin and Boutilier, Craig},
  title         = {Regret-based reward elicitation for Markov decision processes},
  booktitle     = {Proceedings of the Twenty-Fifth Conference on Uncertainty in Artificial Intelligence},
  year          = {2009},
  pages         = {444--451},
  organization  = {AUAI Press},
  __markedentry = {[najiang:]},
}

@InCollection{lopes2009active,
  author        = {Lopes, Manuel and Melo, Francisco and Montesano, Luis},
  title         = {Active learning for reward estimation in inverse reinforcement learning},
  booktitle     = {Machine Learning and Knowledge Discovery in Databases},
  publisher     = {Springer},
  year          = {2009},
  pages         = {31--46},
  __markedentry = {[najiang:]},
}

@Article{baranes2013active,
  author        = {Baranes, Adrien and Oudeyer, Pierre-Yves},
  title         = {Active learning of inverse models with intrinsically motivated goal exploration in robots},
  journal       = {Robotics and Autonomous Systems},
  year          = {2013},
  volume        = {61},
  number        = {1},
  pages         = {49--73},
  __markedentry = {[najiang:]},
  publisher     = {Elsevier},
}

@InCollection{rothkopf2011preference,
  author        = {Rothkopf, Constantin A and Dimitrakakis, Christos},
  title         = {Preference elicitation and inverse reinforcement learning},
  booktitle     = {Machine Learning and Knowledge Discovery in Databases},
  publisher     = {Springer},
  year          = {2011},
  pages         = {34--48},
  __markedentry = {[najiang:]},
}

@InProceedings{langford2005relating,
  author        = {Langford, John and Zadrozny, Bianca},
  title         = {Relating reinforcement learning performance to classification performance},
  booktitle     = {Proceedings of the 22nd international conference on Machine learning},
  year          = {2005},
  pages         = {473--480},
  organization  = {ACM},
  __markedentry = {[najiang:]},
}

@InProceedings{lagoudakis2003reinforcement,
  author        = {Lagoudakis, Michail and Parr, Ronald},
  title         = {Reinforcement learning as classification: Leveraging modern classifiers},
  booktitle     = {ICML},
  year          = {2003},
  volume        = {3},
  pages         = {424--431},
  __markedentry = {[najiang:]},
}

@Book{boyd2004convex,
  title         = {Convex optimization},
  publisher     = {Cambridge university press},
  year          = {2004},
  author        = {Boyd, Stephen and Vandenberghe, Lieven},
  __markedentry = {[najiang:]},
}

@Article{elghaoui2002inversion,
  author        = {El Ghaoui, Laurent},
  title         = {Inversion error, condition number, and approximate inverses of uncertain matrices},
  journal       = {Linear algebra and its applications},
  year          = {2002},
  volume        = {343},
  pages         = {171--193},
  __markedentry = {[najiang:]},
  publisher     = {Elsevier},
}

@Book{anthony2009neural,
  title         = {Neural network learning: Theoretical foundations},
  publisher     = {cambridge university press},
  year          = {2009},
  author        = {Anthony, Martin and Bartlett, Peter L},
  __markedentry = {[najiang:]},
}

@Article{massart2006risk,
  author        = {Massart, Pascal and N{\'e}d{\'e}lec, {\'E}lodie},
  title         = {Risk bounds for statistical learning},
  journal       = {The Annals of Statistics},
  year          = {2006},
  pages         = {2326--2366},
  __markedentry = {[najiang:]},
  publisher     = {JSTOR},
}

@InProceedings{regan2011eliciting,
  author        = {Regan, Kevin and Boutilier, Craig},
  title         = {Eliciting additive reward functions for Markov decision processes},
  booktitle     = {IJCAI Proceedings-International Joint Conference on Artificial Intelligence},
  year          = {2011},
  volume        = {22},
  number        = {3},
  pages         = {2159},
  __markedentry = {[najiang:]},
}

@Article{kocsis2006improved,
  author        = {Kocsis, Levente and Szepesv{\'a}ri, Csaba and Willemson, Jan},
  title         = {Improved {M}onte-{C}arlo search},
  journal       = {Univ. Tartu, Estonia, Tech. Rep},
  year          = {2006},
  volume        = {1},
  __markedentry = {[najiang:6]},
}

@InProceedings{wang2007modifications,
  author        = {Wang, Yizao and Gelly, Sylvain},
  title         = {Modifications of {UCT} and sequence-like simulations for {M}onte-{C}arlo Go},
  booktitle     = {IEEE Symposium on Computational Intelligence and Games},
  year          = {2007},
  pages         = {175--182},
  __markedentry = {[najiang:6]},
}

@Article{silver2010monte,
  author        = {Silver, David and Veness, Joel},
  title         = {{M}onte-{C}arlo planning in large {POMDP}s},
  journal       = {Advances in Neural Information Processing Systems},
  year          = {2010},
  volume        = {23},
  pages         = {2164--2172},
  __markedentry = {[najiang:6]},
}

@InProceedings{coueetoux2011adding,
  author        = {Cou{\"e}toux, Adrien and Doghmen, Hassen and others},
  title         = {Adding Double Progressive Widening to Upper Confidence Trees to Cope with Uncertainty in Planning Problems},
  booktitle     = {9th European Workshop on Reinforcement Learning},
  year          = {2011},
  __markedentry = {[najiang:6]},
}

@InCollection{coueetoux2011continuous,
  author        = {Cou{\"e}toux, Adrien and Hoock, Jean-Baptiste and Sokolovska, Nataliya and Teytaud, Olivier and Bonnard, Nicolas},
  title         = {Continuous upper confidence trees},
  booktitle     = {Learning and Intelligent Optimization},
  publisher     = {Springer},
  year          = {2011},
  pages         = {433--445},
  __markedentry = {[najiang:6]},
}

@Article{tolpin2012mcts,
  author        = {Tolpin, David and Shimony, Solomon Eyal},
  title         = {{MCTS} based on simple regret},
  journal       = {AAAI. AAAI Press, 2012a. To appear},
  year          = {2012},
  __markedentry = {[najiang:6]},
}

@InProceedings{hingston2007experiments,
  author        = {Hingston, Philip and Masek, Martin},
  title         = {Experiments with {M}onte-{C}arlo {O}thello},
  booktitle     = {Evolutionary Computation},
  year          = {2007},
  pages         = {4059--4064},
  organization  = {IEEE},
  __markedentry = {[najiang:6]},
}

@Article{rasmussen2004gaussian,
  author        = {Rasmussen, Carl Edward and Kuss, Malte},
  title         = {Gaussian processes in reinforcement learning},
  journal       = {Advances in Neural Information Processing Systems},
  year          = {2004},
  volume        = {16},
  __markedentry = {[najiang:6]},
}

@Article{gelly2011monte,
  author        = {Gelly, Sylvain and Silver, David},
  title         = {{M}onte-{C}arlo tree search and rapid action value estimation in computer Go},
  journal       = {Artificial Intelligence},
  year          = {2011},
  volume        = {175},
  number        = {11},
  pages         = {1856--1875},
  __markedentry = {[najiang:6]},
  publisher     = {Elsevier},
}

@InProceedings{narayanamurthy2007efficiently,
  author        = {Narayanamurthy, Shravan Matthur and Ravindran, Balaraman},
  title         = {Efficiently Exploiting Symmetries in Real Time Dynamic Programming},
  booktitle     = {International Joint Conference on Artificial Intelligence},
  year          = {2007},
  pages         = {2556--2561},
  __markedentry = {[najiang:6]},
}

@Article{auer2002finite,
  author    = {Auer, Peter and Cesa-Bianchi, Nicol{\`o} and Fischer, Paul},
  title     = {Finite-time analysis of the multiarmed bandit problem},
  journal   = {Machine learning},
  year      = {2002},
  volume    = {47},
  number    = {2-3},
  pages     = {235--256},
  publisher = {Springer},
}

@Article{kearns2002sparse,
  author    = {Kearns, Michael and Mansour, Yishay and Ng, Andrew Y},
  title     = {A sparse sampling algorithm for near-optimal planning in large {M}arkov decision processes},
  journal   = {Machine Learning},
  year      = {2002},
  volume    = {49},
  number    = {2-3},
  pages     = {193--208},
  publisher = {Springer},
}

@InProceedings{konda2000actor,
  author    = {Konda, Vijay R and Tsitsiklis, John N},
  title     = {Actor-critic algorithms},
  booktitle = {Advances in neural information processing systems},
  year      = {2000},
  pages     = {1008--1014},
}

@article{dai2017boosting,
  title={Boosting the actor with dual critic},
  author={Dai, Bo and Shaw, Albert and He, Niao and Li, Lihong and Song, Le},
  journal={arXiv preprint arXiv:1712.10282},
  year={2017}
}

@article{de2003linear,
  title={The linear programming approach to approximate dynamic programming},
  author={De Farias, Daniela Pucci and Van Roy, Benjamin},
  journal={Operations research},
  volume={51},
  number={6},
  pages={850--865},
  year={2003},
  publisher={INFORMS}
}

@article{de2004constraint,
  title={On constraint sampling in the linear programming approach to approximate dynamic programming},
  author={De Farias, Daniela Pucci and Van Roy, Benjamin},
  journal={Mathematics of operations research},
  volume={29},
  number={3},
  pages={462--478},
  year={2004},
  publisher={INFORMS}
}

@article{lakshminarayanan2018linearly,
  title={A linearly relaxed approximate linear program for markov decision processes},
  author={Lakshminarayanan, Chandrashekar and Bhatnagar, Shalabh and Szepesv{\'a}ri, Csaba},
  journal={IEEE Transactions on Automatic Control},
  volume={63},
  number={4},
  pages={1185--1191},
  year={2018},
  publisher={IEEE}
}

@inproceedings{wang2007dual,
  title={Dual representations for dynamic programming and reinforcement learning},
  author={Wang, Tao and Bowling, Michael and Schuurmans, Dale},
  booktitle={2007 IEEE International Symposium on Approximate Dynamic Programming and Reinforcement Learning},
  pages={44--51},
  year={2007},
  organization={IEEE}
}

@article{wang2007dual1,
  title={Dual representations for dynamic programming},
  author={Wang, Tao and Bowling, Michael and Lizotte, Daniel and Schuurmans, Dale},
  year={2007}
}

@inproceedings{pazis2011non,
  title={Non-parametric approximate linear programming for MDPs},
  author={Pazis, Jason and Parr, Ronald},
  booktitle={Twenty-Fifth AAAI Conference on Artificial Intelligence},
  year={2011}
}

@article{petrik2010feature,
  title={Feature selection using regularization in approximate linear programs for Markov decision processes},
  author={Petrik, Marek and Taylor, Gavin and Parr, Ron and Zilberstein, Shlomo},
  journal={arXiv preprint arXiv:1005.1860},
  year={2010}
}

@Comment{jabref-meta: databaseType:bibtex;}

@inproceedings{hu2021near,
  title={Near-optimal representation learning for linear bandits and linear rl},
  author={Hu, Jiachen and Chen, Xiaoyu and Jin, Chi and Li, Lihong and Wang, Liwei},
  booktitle={International Conference on Machine Learning},
  pages={4349--4358},
  year={2021},
  organization={PMLR}
}

@article{zhang2022efficient,
  title={Efficient Reinforcement Learning in Block MDPs: A Model-free Representation Learning Approach},
  author={Zhang, Xuezhou and Song, Yuda and Uehara, Masatoshi and Wang, Mengdi and Sun, Wen and Agarwal, Alekh},
  journal={arXiv preprint arXiv:2202.00063},
  year={2022}
}

@article{zhang2021provably,
  title={Provably efficient representation learning in low-rank markov decision processes},
  author={Zhang, Weitong and He, Jiafan and Zhou, Dongruo and Zhang, Amy and Gu, Quanquan},
  journal={arXiv preprint arXiv:2106.11935},
  year={2021}
}

@article{uehara2021representation,
  title={Representation Learning for Online and Offline RL in Low-rank MDPs},
  author={Uehara, Masatoshi and Zhang, Xuezhou and Sun, Wen},
  journal={arXiv preprint arXiv:2110.04652},
  year={2021}
}

@article{zhan2022offline,
  title={Offline Reinforcement Learning with Realizability and Single-policy Concentrability},
  author={Zhan, Wenhao and Huang, Baihe and Huang, Audrey and Jiang, Nan and Lee, Jason D},
  journal={arXiv preprint arXiv:2202.04634},
  year={2022}
}

@inproceedings{ross2010efficient,
  title={Efficient reductions for imitation learning},
  author={Ross, St{\'e}phane and Bagnell, Drew},
  booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
  year={2010},
  organization={JMLR Workshop and Conference Proceedings}
}

@phdthesis{ross2013interactive,
  title={Interactive learning for sequential decisions and predictions},
  author={Ross, Stephane},
  year={2013},
  school={Carnegie Mellon University}
}
@article{papini2021reinforcement,
  title={Reinforcement Learning in Linear MDPs: Constant Regret and Representation Selection},
  author={Papini, Matteo and Tirinzoni, Andrea and Pacchiano, Aldo and Restelli, Marcello and Lazaric, Alessandro and Pirotta, Matteo},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@inproceedings{he2021logarithmic,
  title={Logarithmic regret for reinforcement learning with linear function approximation},
  author={He, Jiafan and Zhou, Dongruo and Gu, Quanquan},
  booktitle={International Conference on Machine Learning},
  pages={4171--4180},
  year={2021},
  organization={PMLR}
}
@article{zanette2021provable,
  title={Provable benefits of actor-critic methods for offline reinforcement learning},
  author={Zanette, Andrea and Wainwright, Martin J and Brunskill, Emma},
  journal={Advances in neural information processing systems},
  volume={34},
  year={2021}
}

@article{simchowitz2019non,
  title={Non-asymptotic gap-dependent regret bounds for tabular mdps},
  author={Simchowitz, Max and Jamieson, Kevin G},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@inproceedings{yang2021q,
  title={Q-learning with logarithmic regret},
  author={Yang, Kunhe and Yang, Lin and Du, Simon},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  year={2021},
  organization={PMLR}
}

@inproceedings{uehara2020minimax,
  title={Minimax weight and q-function learning for off-policy evaluation},
  author={Uehara, Masatoshi and Huang, Jiawei and Jiang, Nan},
  booktitle={International Conference on Machine Learning},
  pages={9659--9668},
  year={2020},
  organization={PMLR}
}

@article{jin2021bellman,
  title={Bellman eluder dimension: New rich classes of rl problems, and sample-efficient algorithms},
  author={Jin, Chi and Liu, Qinghua and Miryoosefi, Sobhan},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{yin2021near,
  title={Near-optimal offline reinforcement learning via double variance reduction},
  author={Yin, Ming and Bai, Yu and Wang, Yu-Xiang},
  journal={Advances in neural information processing systems},
  volume={34},
  year={2021}
}

@article{slivkins2019introduction,
  title={Introduction to Multi-Armed Bandits},
  author={Slivkins, Aleksandrs and others},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={12},
  number={1-2},
  pages={1--286},
  year={2019},
  publisher={Now Publishers, Inc.}
}

@article{bubeck2012regret,
  title={Regret Analysis of Stochastic and Nonstochastic Multi-armed Bandit Problems},
  author={Bubeck, S{\'e}bastien and Cesa-Bianchi, Nicolo},
  journal={Machine Learning},
  volume={5},
  number={1},
  pages={1--122},
  year={2012},
  publisher={Citeseer}
}

@book{lattimore2020bandit,
  title={Bandit algorithms},
  author={Lattimore, Tor and Szepesv{\'a}ri, Csaba},
  year={2020},
  publisher={Cambridge University Press}
}

@inproceedings{du2021bilinear,
  title={Bilinear classes: A structural framework for provable generalization in rl},
  author={Du, Simon and Kakade, Sham and Lee, Jason and Lovett, Shachar and Mahajan, Gaurav and Sun, Wen and Wang, Ruosong},
  booktitle={International Conference on Machine Learning},
  pages={2826--2836},
  year={2021},
  organization={PMLR}
}

@article{nachum2019dualdice,
  title={Dualdice: Behavior-agnostic estimation of discounted stationary distribution corrections},
  author={Nachum, Ofir and Chow, Yinlam and Dai, Bo and Li, Lihong},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@article{mou2020sample,
  title={On the sample complexity of reinforcement learning with policy space generalization},
  author={Mou, Wenlong and Wen, Zheng and Chen, Xi},
  journal={arXiv preprint arXiv:2008.07353},
  year={2020}
}

@inproceedings{jin2021pessimism,
  title={Is pessimism provably efficient for offline rl?},
  author={Jin, Ying and Yang, Zhuoran and Wang, Zhaoran},
  booktitle={International Conference on Machine Learning},
  pages={5084--5096},
  year={2021},
  organization={PMLR}
}

@article{yang2020off,
  title={Off-policy evaluation via the regularized lagrangian},
  author={Yang, Mengjiao and Nachum, Ofir and Dai, Bo and Li, Lihong and Schuurmans, Dale},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={6551--6561},
  year={2020}
}
@article{rashidinejad2021bridging,
  title={Bridging offline reinforcement learning and imitation learning: A tale of pessimism},
  author={Rashidinejad, Paria and Zhu, Banghua and Ma, Cong and Jiao, Jiantao and Russell, Stuart},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}
@article{xie2021policy,
  title={Policy finetuning: Bridging sample-efficient offline and online reinforcement learning},
  author={Xie, Tengyang and Jiang, Nan and Wang, Huan and Xiong, Caiming and Bai, Yu},
  journal={Advances in neural information processing systems},
  volume={34},
  year={2021}
}
@inproceedings{scherrer2014approximate,
  title={Approximate policy iteration schemes: a comparison},
  author={Scherrer, Bruno},
  booktitle={International Conference on Machine Learning},
  pages={1314--1322},
  year={2014},
  organization={PMLR}
}
@article{foster2021offline,
  title={Offline Reinforcement Learning: Fundamental Barriers for Value Function Approximation},
  author={Foster, Dylan J and Krishnamurthy, Akshay and Simchi-Levi, David and Xu, Yunzong},
  journal={arXiv preprint arXiv:2111.10919},
  year={2021}
}
@inproceedings{wang2021instabilities,
  title={Instabilities of offline rl with pre-trained neural representation},
  author={Wang, Ruosong and Wu, Yifan and Salakhutdinov, Ruslan and Kakade, Sham},
  booktitle={International Conference on Machine Learning},
  pages={10948--10960},
  year={2021},
  organization={PMLR}
}

@article{kearns2002near,
	title        = {Near-optimal reinforcement learning in polynomial time},
	author       = {Kearns, Michael and Singh, Satinder},
	year         = 2002,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 49,
	number       = 2,
	pages        = {209--232}
}
@inproceedings{precup2000eligibility,
	title        = {Eligibility Traces for Off-Policy Policy Evaluation},
	author       = {Precup, Doina and Sutton, Richard S and Singh, Satinder P},
	year         = 2000,
	booktitle    = {Proceedings of the Seventeenth International Conference on Machine Learning},
	pages        = {759--766}
}
@book{kakade2003sample,
	title        = {On the sample complexity of reinforcement learning},
	author       = {Kakade, Sham Machandranath},
	year         = 2003,
	publisher    = {University of London, University College London (United Kingdom)}
}
@inproceedings{modi2020sample,
	title        = {Sample complexity of reinforcement learning using linearly combined model ensembles},
	author       = {Modi, Aditya and Jiang, Nan and Tewari, Ambuj and Singh, Satinder},
	year         = 2020,
	booktitle    = {International Conference on Artificial Intelligence and Statistics},
	pages        = {2010--2020},
	organization = {PMLR}
}
@article{bai2019provably,
	title        = {Provably efficient Q-learning with low switching cost},
	author       = {Bai, Yu and Xie, Tengyang and Jiang, Nan and Wang, Yu Xiang},
	year         = 2019,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 32
}
@article{matsushima2020deployment,
	title        = {Deployment-efficient reinforcement learning via model-based offline optimization},
	author       = {Matsushima, Tatsuya and Furuta, Hiroki and Matsuo, Yutaka and Nachum, Ofir and Gu, Shixiang},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2006.03647}
}
@article{levine2020offline,
	title        = {Offline reinforcement learning: Tutorial, review, and perspectives on open problems},
	author       = {Levine, Sergey and Kumar, Aviral and Tucker, George and Fu, Justin},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2005.01643}
}
@inproceedings{li2016deep,
	title        = {Deep Reinforcement Learning for Dialogue Generation},
	author       = {Li, Jiwei and Monroe, Will and Ritter, Alan and Jurafsky, Dan and Galley, Michel and Gao, Jianfeng},
	year         = 2016,
	booktitle    = {Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing},
	pages        = {1192--1202}
}
@inproceedings{mandel2014offline,
	title        = {Offline policy evaluation across representations with applications to educational games.},
	author       = {Mandel, Travis and Liu, Yun-En and Levine, Sergey and Brunskill, Emma and Popovic, Zoran},
	year         = 2014,
	booktitle    = {AAMAS},
	volume       = 1077
}
@book{agarwal2012computational,
	title        = {Computational trade-offs in statistical learning},
	author       = {Agarwal, Alekh},
	year         = 2012,
	publisher    = {University of California, Berkeley}
}
@article{xie2021bellman,
  title={Bellman-consistent pessimism for offline reinforcement learning},
  author={Xie, Tengyang and Cheng, Ching-An and Jiang, Nan and Mineiro, Paul and Agarwal, Alekh},
  journal={Advances in neural information processing systems},
  volume={34},
  year={2021}
}
@article{edwards2021automatic,
	title        = {Automatic Tuning for Data-driven Model Predictive Control},
	author       = {Edwards, William and Tang, Gao amd Mamakoukas, Giorgos and Murphey, Todd and Hauser, Kris},
	year         = 2021,
	journal      = {International Conference on Robotics and Automation (ICRA)}
}
@article{modi2021model,
	title        = {Model-free representation learning and exploration in low-rank MDPs},
	author       = {Modi, Aditya and Chen, Jinglin and Krishnamurthy, Akshay and Jiang, Nan and Agarwal, Alekh},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2102.07035}
}
@article{bellemare2020autonomous,
	title        = {Autonomous navigation of stratospheric balloons using reinforcement learning},
	author       = {Bellemare, Marc G and Candido, Salvatore and Castro, Pablo Samuel and Gong, Jun and Machado, Marlos C and Moitra, Subhodeep and Ponda, Sameera S and Wang, Ziyu},
	year         = 2020,
	journal      = {Nature},
	publisher    = {Nature Publishing Group},
	volume       = 588,
	number       = 7836,
	pages        = {77--82}
}
@article{lu2021reinforcement,
	title        = {Reinforcement Learning, Bit by Bit},
	author       = {Lu, Xiuyuan and Van Roy, Benjamin and Dwaracherla, Vikranth and Ibrahimi, Morteza and Osband, Ian and Wen, Zheng},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2103.04047}
}
@article{fu2020d4rl,
	title        = {D4rl: Datasets for deep data-driven reinforcement learning},
	author       = {Fu, Justin and Kumar, Aviral and Nachum, Ofir and Tucker, George and Levine, Sergey},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2004.07219}
}
@inproceedings{gulcehre2020rl,
	title        = {{RL Unplugged: A Suite of Benchmarks for Offline Reinforcement Learning}},
	author       = {Gulcehre, Caglar and Wang, Ziyu and Novikov, Alexander and Paine, Thomas and G\'{o}mez, Sergio and Zolna, Konrad and Agarwal, Rishabh and Merel, Josh S and Mankowitz, Daniel J and Paduraru, Cosmin and Dulac-Arnold, Gabriel and Li, Jerry and Norouzi, Mohammad and Hoffman, Matthew and Heess, Nicolas and de Freitas, Nando},
	year         = 2020,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 33,
	pages        = {7248--7259},
	editor       = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin}
}
@inproceedings{silver2017predictron,
	title        = {The predictron: End-to-end learning and planning},
	author       = {David Silver and Hado van Hasselt and Matteo Hessel and Tom Schaul and Arthur Guez and Tim Harley and Gabriel Dulac-Arnold and David Reichert and Neil Rabinowitz and Andre Barreto and Thomas Degris},
	year         = 2017,
	booktitle    = {International Conference on Machine Learning},
	pages        = {3191--3199},
	organization = {PMLR}
}
@inproceedings{voloshin2021minimax,
	title        = {Minimax Model Learning},
	author       = {Voloshin, Cameron and Jiang, Nan and Yue, Yisong},
	year         = 2021,
	booktitle    = {International Conference on Artificial Intelligence and Statistics},
	pages        = {1612--1620},
	organization = {PMLR}
}
@article{uehara2021finite,
	title        = {Finite sample analysis of minimax offline reinforcement learning: Completeness, fast rates and first-order efficiency},
	author       = {Uehara, Masatoshi and Imaizumi, Masaaki and Jiang, Nan and Kallus, Nathan and Sun, Wen and Xie, Tengyang},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2102.02981}
}
@inproceedings{liu2020understanding,
	title        = {Understanding the curse of horizon in off-policy evaluation via conditional importance sampling},
	author       = {Liu, Yao and Bacon, Pierre-Luc and Brunskill, Emma},
	year         = 2020,
	booktitle    = {International Conference on Machine Learning},
	pages        = {6184--6193},
	organization = {PMLR}
}
@article{weisz2021query,
	title        = {On Query-efficient Planning in MDPs under Linear Realizability of the Optimal State-value Function},
	author       = {Weisz, Gellert and Amortila, Philip and Janzer, Barnab{\'a}s and Abbasi-Yadkori, Yasin and Jiang, Nan and Szepesv{\'a}ri, Csaba},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2102.02049}
}
@article{chen2021infinite,
	title        = {Infinite-Horizon Offline Reinforcement Learning with Linear Function Approximation: Curse of Dimensionality and Algorithm},
	author       = {Chen, Lin and Scherrer, Bruno and Bartlett, Peter L},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2103.09847}
}
@article{de2003linear,
	title        = {The linear programming approach to approximate dynamic programming},
	author       = {De Farias, Daniela Pucci and Van Roy, Benjamin},
	year         = 2003,
	journal      = {Operations research},
	publisher    = {INFORMS},
	volume       = 51,
	number       = 6,
	pages        = {850--865}
}
@article{garg2020batch,
	title        = {Batch-Constrained Distributional Reinforcement Learning for Session-based Recommendation},
	author       = {Garg, Diksha and Gupta, Priyanka and Malhotra, Pankaj and Vig, Lovekesh and Shroff, Gautam},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2012.08984}
}
@article{kumar2020conservative,
	title        = {Conservative q-learning for offline reinforcement learning},
	author       = {Kumar, Aviral and Zhou, Aurick and Tucker, George and Levine, Sergey},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2006.04779}
}
@article{yu2020mopo,
	title        = {Mopo: Model-based offline policy optimization},
	author       = {Yu, Tianhe and Thomas, Garrett and Yu, Lantao and Ermon, Stefano and Zou, James and Levine, Sergey and Finn, Chelsea and Ma, Tengyu},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2005.13239}
}
@article{kidambi2020morel,
	title        = {Morel: Model-based offline reinforcement learning},
	author       = {Kidambi, Rahul and Rajeswaran, Aravind and Netrapalli, Praneeth and Joachims, Thorsten},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2005.05951}
}
@inproceedings{todorov2012mujoco,
	title        = {Mujoco: A physics engine for model-based control},
	author       = {Todorov, Emanuel and Erez, Tom and Tassa, Yuval},
	year         = 2012,
	booktitle    = {2012 IEEE/RSJ International Conference on Intelligent Robots and Systems},
	pages        = {5026--5033},
	organization = {IEEE}
}
@article{zanette2020exponential,
	title        = {{Exponential Lower Bounds for Batch Reinforcement Learning: Batch RL can be Exponentially Harder than Online RL}},
	author       = {Zanette, Andrea},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2012.08005}
}

@article{bellemare2013arcade,
	title        = {The arcade learning environment: An evaluation platform for general agents},
	author       = {Bellemare, Marc G and Naddaf, Yavar and Veness, Joel and Bowling, Michael},
	year         = 2013,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 47,
	pages        = {253--279}
}
@misc{joelletalk,
	title        = {{Reproducible, Reusable, and Robust Reinforcement Learning}},
	author       = {Pineau, Joelle},
	year         = 2018,
	note         = {\url{https://media.neurips.cc/Conferences/NIPS2018/Slides/jpineau-NeurIPS-dec18-fb.pdf}. Accessed: July 2021}
}
@misc{icerm2021workshop,
	title        = {{Workshop on Advances in Theory and Algorithms for Deep Reinforcement Learning}},
	year         = 2021,
	note         = {\url{https://icerm.brown.edu/events/htw-21-drl/}. Accessed: June 2021}
}
@misc{uiucrl,
	title        = {{CS 598: Statistical Reinforcement Learning}},
	author       = {Jiang, Nan},
	note         = {Course website: \url{http://nanjiang.cs.illinois.edu/cs598/}. Accessed: May 2021}
}
@misc{rlsem,
	title        = {{RL Theory Virtual Seminars}},
	note         = {\url{https://sites.google.com/view/rltheoryseminars}. Accessed: May 2021}
}
@misc{uwrl,
	title        = {{CSE 599: Reinforcement Learning and Bandits}},
	author       = {Agarwal, Alekh and Kakade, Sham},
	note         = {Course website: \url{https://courses.cs.washington.edu/courses/cse599m/19sp/}. Accessed: May 2019}
}
@misc{chictech,
	title        = {{ChicTech}},
	author       = {{Women in Computer Science at UIUC}},
	note         = {Program website: \url{http://wcs.illinois.edu/chictech/}. Accessed: July 2021}
}
@misc{games,
	title        = {Girls' Adventures in Mathematics, Engineering, and Science ({GAMES})},
	author       = {{Women at Engineering at Illinois}},
	note         = {Camp website: \url{https://wie.engineering.illinois.edu/k-12-programs-resources/camps-for-k-through-12-students}. Accessed: July 2021}
}
@manual{nan_abstraction_notes,
	title        = {{CS 598: Notes on State Abstractions}},
	author       = {Nan Jiang},
	year         = 2018,
	note         = {\url{http://nanjiang.cs.illinois.edu/files/cs598/note4.pdf}},
	organization = {{University of Illinois at Urbana-Champaign}}
}
@manual{shipranotes,
	title        = {{IEOR 8100: Reinforcement Learning. Lecture 4: Approximate Dynamic Programming}},
	author       = {Shipra Agrawal},
	year         = 2018,
	note         = {\url{https://ieor8100.github.io/rl/docs/Lecture%204%20-%20approximate%20DP.pdf}},
	organization = {{Columbia University}}
}
@inproceedings{pazis2013pac,
	title        = {{PAC Optimal Exploration in Continuous Space Markov Decision Processes}},
	author       = {Pazis, Jason and Parr, Ronald},
	year         = 2013,
	booktitle    = {AAAI}
}
@inproceedings{chen2018scalable,
	title        = {Scalable Bilinear $\pi$ Learning Using State and Action Features},
	author       = {Chen, Yichen and Li, Lihong and Wang, Mengdi},
	year         = 2018,
	booktitle    = {International Conference on Machine Learning},
	pages        = {833--842}
}
@inproceedings{nguyen2013online,
	title        = {Online feature selection for model-based reinforcement learning},
	author       = {Nguyen, Trung and Li, Zhuoru and Silander, Tomi and Leong, Tze Yun},
	year         = 2013,
	booktitle    = {International Conference on Machine Learning},
	pages        = {498--506}
}
@article{guo2017sample,
	title        = {Sample efficient feature selection for factored {MDP}s},
	author       = {Guo, Zhaohan Daniel and Brunskill, Emma},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1703.03454}
}
@inproceedings{diuk2009adaptive,
	title        = {The adaptive k-meteorologists problem and its application to structure learning and feature selection in reinforcement learning},
	author       = {Diuk, Carlos and Li, Lihong and Leffler, Bethany R},
	year         = 2009,
	booktitle    = {Proceedings of the 26th Annual International Conference on Machine Learning},
	pages        = {249--256},
	organization = {ACM}
}
@inproceedings{littman2002predictive,
	title        = {Predictive representations of state},
	author       = {Littman, Michael L and Sutton, Richard S},
	year         = 2002,
	booktitle    = {Advances in neural information processing systems},
	pages        = {1555--1561}
}
@inproceedings{ok2018exploration,
	title        = {Exploration in Structured Reinforcement Learning},
	author       = {Ok, Jungseul and Proutiere, Alexandre and Tranos, Damianos},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems 31},
	pages        = {8888--8896}
}
@inproceedings{jiang2018pac,
	title        = {{PAC} Reinforcement Learning with an Imperfect Model},
	author       = {Jiang, Nan},
	year         = 2018,
	booktitle    = {AAAI Conference on Artificial Intelligence}
}
@inproceedings{kearns1999efficient,
	title        = {Efficient reinforcement learning in factored {MDP}s},
	author       = {Kearns, Michael and Koller, Daphne},
	year         = 1999,
	booktitle    = {International Joint Conference on Artificial Intelligence}
}
@inproceedings{osband2014near,
	title        = {Near-optimal reinforcement learning in factored {MDP}s},
	author       = {Osband, Ian and Van Roy, Benjamin},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems}
}
@article{kearns2002near,
	title        = {Near-optimal reinforcement learning in polynomial time},
	author       = {Kearns, Michael and Singh, Satinder},
	year         = 2002,
	journal      = {Machine learning}
}

}
@book{devroye2012combinatorial,
	title        = {Combinatorial methods in density estimation},
	author       = {Devroye, Luc and Lugosi, G{\'a}bor},
	year         = 2012,
	publisher    = {Springer Science \& Business Media}
}
@inproceedings{koller1999computing,
	title        = {Computing factored value functions for policies in structured {MDP}s},
	author       = {Koller, Daphne and Parr, Ronald},
	year         = 1999,
	booktitle    = {IJCAI},
	volume       = 99,
	pages        = {1332--1339}
}
@inproceedings{krishnamurthy2016pac,
	title        = {{PAC} reinforcement learning with rich observations},
	author       = {Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John},
	year         = 2016,
	booktitle    = {Advances in Neural Information Processing Systems}
}
@inproceedings{szita2010model,
	title        = {Model-based reinforcement learning with nearly tight exploration complexity bounds},
	author       = {Szita, Istv{\'a}n and Szepesv{\'a}ri, Csaba},
	year         = 2010,
	booktitle    = {International Conference on Machine Learning}
}
@article{brafman2002r,
	title        = {R-max-a general polynomial time algorithm for near-optimal reinforcement learning},
	author       = {Brafman, Ronen I and Tennenholtz, Moshe},
	year         = 2002,
	journal      = {Journal of Machine Learning Research},
	volume       = 3,
	number       = {Oct},
	pages        = {213--231}
}
@inproceedings{strehl2006pac,
	title        = {{PAC} model-free reinforcement learning},
	author       = {Strehl, Alexander L and Li, Lihong and Wiewiora, Eric and Langford, John and Littman, Michael L},
	year         = 2006,
	booktitle    = {International Conference on Machine Learning}
}
@phdthesis{kakade2003sample,
	title        = {On the sample complexity of reinforcement learning},
	author       = {Kakade, Sham Machandranath and others},
	year         = 2003,
	school       = {University of London London, England}
}
@inproceedings{jin2018q,
	title        = {Is {Q}-learning Provably Efficient?},
	author       = {Jin, Chi and Allen-Zhu, Zeyuan and Bubeck, Sebastien and Jordan, Michael I},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems}
}
################Model-Based RL with extra exploration distribution###########
@inproceedings{talvitie2015agnostic,
	title        = {Agnostic System Identification for {M}onte {C}arlo Planning.},
	author       = {Talvitie, Erik},
	year         = 2015,
	booktitle    = {AAAI Conference on Artificial Intelligence}
}
@inproceedings{talvitie2017self,
	title        = {Self-Correcting Models for Model-Based Reinforcement Learning.},
	author       = {Talvitie, Erik},
	year         = 2017,
	booktitle    = {AAAI Conference on Artificial Intelligence}
}
################Model-Based RL with extra exploration distribution###########



###########Practical Model-Based RL #####################
@inproceedings{abbeel2007application,
	title        = {An application of reinforcement learning to aerobatic helicopter flight},
	author       = {Abbeel, Pieter and Coates, Adam and Quigley, Morgan and Ng, Andrew Y},
	year         = 2007,
	booktitle    = {Advances in neural information processing systems},
	pages        = {1--8}
}
@article{sun2018dual,
	title        = {Dual Policy Iteration},
	author       = {Sun, Wen and Gordon, Geoffrey J and Boots, Byron and Bagnell, J Andrew},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1805.10755}
}
@article{xie2015model,
	title        = {Model-based reinforcement learning with parametrized physical models and optimism-driven exploration},
	author       = {Xie, Christopher and Patil, Sachin and Moldovan, Teodor and Levine, Sergey and Abbeel, Pieter},
	year         = 2015,
	journal      = {arXiv preprint arXiv:1509.06824}
}
@inproceedings{moldovan2015optimism,
	title        = {Optimism-driven exploration for nonlinear systems},
	author       = {Moldovan, Teodor Mihai and Levine, Sergey and Jordan, Michael I and Abbeel, Pieter},
	year         = 2015,
	booktitle    = {Robotics and Automation (ICRA), 2015 IEEE International Conference on},
	pages        = {3239--3246},
	organization = {IEEE}
}
####################################################
@inproceedings{farahmand2017value,
	title        = {Value-Aware Loss Function for Model-based Reinforcement Learning},
	author       = {Farahmand, Amir-massoud and Barreto, Andre and Nikovski, Daniel},
	year         = 2017,
	booktitle    = {Artificial Intelligence and Statistics}
}
############# Continuous State Space with Lipschitiz models ###########
@inproceedings{lakshmanan2015improved,
	title        = {Improved regret bounds for undiscounted continuous reinforcement learning},
	author       = {Lakshmanan, Kailasam and Ortner, Ronald and Ryabko, Daniil},
	year         = 2015,
	booktitle    = {International Conference on Machine Learning}
}
@inproceedings{ortner2012online,
	title        = {Online regret bounds for undiscounted continuous reinforcement learning},
	author       = {Ortner, Ronald and Ryabko, Daniil},
	year         = 2012,
	booktitle    = {Advances in Neural Information Processing Systems}
}
@inproceedings{kakade2003exploration,
	title        = {Exploration in metric state spaces},
	author       = {Kakade, Sham and Kearns, Michael J and Langford, John},
	year         = 2003,
	booktitle    = {International Conference on Machine Learning}
}
####################################################




=======
@inproceedings{dann2015sample,
	title        = {Sample complexity of episodic fixed-horizon reinforcement learning},
	author       = {Dann, Christoph and Brunskill, Emma},
	year         = 2015,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2818--2826}
}
@inproceedings{wen2013efficient,
	title        = {Efficient exploration and value function generalization in deterministic systems},
	author       = {Wen, Zheng and Van Roy, Benjamin},
	year         = 2013,
	booktitle    = {Advances in Neural Information Processing Systems}
}
@inproceedings{lattimore2013sample,
	title        = {The sample-complexity of general reinforcement learning},
	author       = {Lattimore, Tor and Hutter, Marcus and Sunehag, Peter},
	year         = 2013,
	booktitle    = {International Conference on Machine Learning}
}
@article{bartlett2008high,
	title        = {High-probability regret bounds for bandit online linear optimization},
	author       = {Bartlett, Peter L and Dani, Varsha and Hayes, Thomas and Kakade, Sham and Rakhlin, Alexander and Tewari, Ambuj},
	year         = 2008
}
@article{xu2018algorithmic,
	title        = {Algorithmic Framework for Model-based Reinforcement Learning with Theoretical Guarantees},
	author       = {Xu, Huazhe and Li, Yuanzhi and Tian, Yuandong and Darrell, Trevor and Ma, Tengyu},
	year         = 2018,
	journal      = {arXiv:1807.03858}
}
@inproceedings{pires2016policy,
	title        = {Policy error bounds for model-based reinforcement learning with factored linear models},
	author       = {Pires, Bernardo {\'A}vila and Szepesv{\'a}ri, Csaba},
	year         = 2016,
	booktitle    = {Conference on Learning Theory},
	pages        = {121--151}
}
@inproceedings{jiang2015dependence,
	title        = {The dependence of effective planning horizon on model accuracy},
	author       = {Jiang, Nan and Kulesza, Alex and Singh, Satinder and Lewis, Richard},
	year         = 2015,
	booktitle    = {Proceedings of the 2015 International Conference on Autonomous Agents and Multiagent Systems},
	pages        = {1181--1189},
	organization = {International Foundation for Autonomous Agents and Multiagent Systems}
}
@article{muller1997integral,
	title        = {Integral probability metrics and their generating classes of functions},
	author       = {M{\"u}ller, Alfred},
	year         = 1997,
	journal      = {Advances in Applied Probability}
}
@article{boutilier2000stochastic,
	title        = {Stochastic dynamic programming with factored representations},
	author       = {Boutilier, Craig and Dearden, Richard and Goldszmidt, Mois{\'e}s},
	year         = 2000,
	journal      = {Artificial intelligence},
	publisher    = {Elsevier},
	volume       = 121,
	number       = {1-2},
	pages        = {49--107}
}
@article{guestrin2003efficient,
	title        = {Efficient solution algorithms for factored {MDP}s},
	author       = {Guestrin, Carlos and Koller, Daphne and Parr, Ronald and Venkataraman, Shobha},
	year         = 2003,
	journal      = {Journal of Aprtificial Intelligence Research}
}
@article{arjovsky2017wasserstein,
	title        = {Wasserstein gan},
	author       = {Arjovsky, Martin and Chintala, Soumith and Bottou, L{\'e}on},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1701.07875}
}
@article{azar2017minimax,
	title        = {Minimax regret bounds for reinforcement learning},
	author       = {Azar, Mohammad Gheshlaghi and Osband, Ian and Munos, R{\'e}mi},
	year         = 2017,
	journal      = {arXiv:1703.05449}
}
@article{silver2017mastering,
	title        = {Mastering the game of Go without human knowledge},
	author       = {Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and Chen, Yutian and Lillicrap, Timothy and Hui, Fan and Sifre, Laurent and van den Driessche, George and Graepel, Thore and Hassabis, Demis},
	year         = 2017,
	journal      = {Nature},
	publisher    = {Nature Publishing Group}
}
@inproceedings{levine2014learning,
	title        = {Learning neural network policies with guided policy search under unknown dynamics},
	author       = {Levine, Sergey and Abbeel, Pieter},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems}
}
@article{jaksch2010near,
	title        = {Near-optimal regret bounds for reinforcement learning},
	author       = {Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
	year         = 2010,
	journal      = {Journal of Machine Learning Research}
}
@inproceedings{abbasi2011regret,
	title        = {Regret bounds for the adaptive control of linear quadratic systems},
	author       = {Abbasi-Yadkori, Yasin and Szepesv{\'a}ri, Csaba},
	year         = 2011,
	booktitle    = {Conference on Learning Theory}
}
@inproceedings{dean2018regret,
	title        = {Regret Bounds for Robust Adaptive Control of the Linear Quadratic Regulator},
	author       = {Dean, Sarah and Mania, Horia and Matni, Nikolai and Recht, Benjamin and Tu, Stephen},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems}
}
@inproceedings{osband2014model,
	title        = {Model-based reinforcement learning and the eluder dimension},
	author       = {Osband, Ian and Van Roy, Benjamin},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems}
}
@inproceedings{dann2018polynomial,
	title        = {On oracle-efficient {PAC} Reinforcement Learning with Rich Observations},
	author       = {Dann, Christoph and Jiang, Nan and Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John and Schapire, Robert E},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems}
}
@article{sriperumbudur2012empirical,
	title        = {On the empirical estimation of integral probability metrics},
	author       = {Sriperumbudur, Bharath K and Fukumizu, Kenji and Gretton, Arthur and Sch{\"o}lkopf, Bernhard and Lanckriet, Gert RG and others},
	year         = 2012,
	journal      = {Electronic Journal of Statistics}
}
@book{shalev2014understanding,
	title        = {Understanding machine learning: From theory to algorithms},
	author       = {Shalev-Shwartz, Shai and Ben-David, Shai},
	year         = 2014,
	publisher    = {Cambridge University Press}
}
@article{mundhenk2000complexity,
	title        = {Complexity of finite-horizon {M}arkov decision process problems},
	author       = {Mundhenk, Martin and Goldsmith, Judy and Lusena, Christopher and Allender, Eric},
	year         = 2000,
	journal      = {Journal of the ACM}
}
@inproceedings{allender2003note,
	title        = {A note on the representational incompatibility of function approximation and factored dynamics},
	author       = {Allender, Eric and Arora, Sanjeev and Kearns, Michael and Moore, Cristopher and Russell, Alexander},
	year         = 2003,
	booktitle    = {Advances in Neural Information Processing Systems}
}
@article{watkins1992q,
	title        = {Q-learning},
	author       = {Watkins, Christopher JCH and Dayan, Peter},
	year         = 1992,
	journal      = {Machine learning}
}
@article{levine2016end,
	title        = {End-to-end training of deep visuomotor policies},
	author       = {Levine, Sergey and Finn, Chelsea and Darrell, Trevor and Abbeel, Pieter},
	year         = 2016,
	journal      = {Journal of Machine Learning Research}
}
@inproceedings{deisenroth2011learning,
	title        = {Learning to control a low-cost manipulator using data-efficient reinforcement learning},
	author       = {Deisenroth, Marc P and Rasmussen, Carl E and Fox, Dieter},
	year         = 2011,
	booktitle    = {Robotics: Science and Systems}
}
@inproceedings{aboaf1989task,
	title        = {Task-level robot learning: Juggling a tennis ball more accurately},
	author       = {Aboaf, Eric W and Drucker, Steven M and Atkeson, Christopher G},
	year         = 1989,
	booktitle    = {IEEE International Conference on Robotics and Automation}
}
@article{gao2018robust,
	title        = {Robust Estimation and Generative Adversarial Nets},
	author       = {Gao, Chao and Liu, Jiyi and Yao, Yuan and Zhu, Weizhi},
	year         = 2018,
	journal      = {arXiv:1810.02030}
}
@article{gretton2012kernel,
	title        = {A kernel two-sample test},
	author       = {Gretton, Arthur and Borgwardt, Karsten M and Rasch, Malte J and Sch{\"o}lkopf, Bernhard and Smola, Alexander},
	year         = 2012,
	journal      = {Journal of Machine Learning Research}
}
@inproceedings{nagabandi2018neural,
	title        = {Neural network dynamics for model-based deep reinforcement learning with model-free fine-tuning},
	author       = {Nagabandi, Anusha and Kahn, Gregory and Fearing, Ronald S and Levine, Sergey},
	year         = 2018,
	booktitle    = {IEEE International Conference on Robotics and Automation}
}
@article{weber2017imagination,
	title        = {Imagination-augmented agents for deep reinforcement learning},
	author       = {Weber, Th{\'e}ophane and Racani{\`e}re, S{\'e}bastien and Reichert, David P and Buesing, Lars and Guez, Arthur and Rezende, Danilo Jimenez and Badia, Adria Puigdomenech and Vinyals, Oriol and Heess, Nicolas and Li, Yujia and Pascanu, Razvan and Battaglio, Peter and Hassabis, Demis and Silver, David and Wierstra, Daan},
	year         = 2017,
	journal      = {arXiv:1707.06203}
}
@article{tu2017least,
	title        = {Least-squares temporal difference learning for the linear quadratic regulator},
	author       = {Tu, Stephen and Recht, Benjamin},
	year         = 2017,
	journal      = {arXiv:1712.08642}
}
@article{abbasi2018regret,
	title        = {Model-Free Linear Quadratic Control via Reduction to Expert Prediction},
	author       = {Abbasi-Yadkori, Yasin and Lazic, Nevena and Szepesvari, Csaba},
	year         = 2018,
	journal      = {arXiv:1804.06021}
}
@inproceedings{parr2008analysis,
	title        = {An analysis of linear models, linear value-function approximation, and feature selection for reinforcement learning},
	author       = {Parr, Ronald and Li, Lihong and Taylor, Gavin and Painter-Wakefield, Christopher and Littman, Michael L},
	year         = 2008,
	booktitle    = {International Conference on Machine learning}
}
@article{ross2012agnostic,
	title        = {Agnostic system identification for model-based reinforcement learning},
	author       = {Ross, Stephane and Bagnell, J Andrew},
	year         = 2012,
	journal      = {International Conference on Machine learning}
}
@inproceedings{tu2018gap,
	title        = {The Gap Between Model-Based and Model-Free Methods on the Linear Quadratic Regulator: An Asymptotic Viewpoint},
	author       = {Tu, Stephen and Recht, Benjamin},
	year         = 2019,
	booktitle    = {Conference on Learning Theory}
}
@inproceedings{sutton2000policy,
	title        = {Policy gradient methods for reinforcement learning with function approximation},
	author       = {Sutton, Richard S and McAllester, David A and Singh, Satinder P and Mansour, Yishay},
	year         = 2000,
	booktitle    = {Advances in neural information processing systems},
	pages        = {1057--1063}
}
@article{williams1992simple,
	title        = {Simple statistical gradient-following algorithms for connectionist reinforcement learning},
	author       = {Williams, Ronald J},
	year         = 1992,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 8,
	number       = {3-4},
	pages        = {229--256}
}
@article{huang2019importance,
	title        = {{From Importance Sampling to Doubly Robust Policy Gradient}},
	author       = {Huang, Jiawei and Jiang, Nan},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1910.09066}
}
@inproceedings{uehara2019minimax,
	title        = {{Minimax Weight and Q-Function Learning for Off-Policy Evaluation}},
	author       = {Uehara, Masatoshi and Huang, Jiawei and Jiang, Nan},
	year         = 2020,
	booktitle    = {Proceedings of the 37th International Conference on Machine Learning},
	pages        = {1023--1032}
}
@manual{rlbook2019,
	title        = {{Reinforcement Learning: Theory and Algorithms}},
	author       = {Agarwal, Alekh and Jiang, Nan and Kakade, Sham},
	year         = 2019,
	note         = {\url{https://rltheorybook.github.io/}}
}
@article{HirshbergDavidA.2017AMLE,
	title        = {Augmented Minimax Linear Estimation},
	author       = {Hirshberg, David A. and Wager, Stefan},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1712.00038},
	keywords     = {Statistics - Methodology ; 62f12}
}
@article{KallusNathan2017GOMM,
	title        = {Generalized Optimal Matching Methods for Causal Inference},
	author       = {Kallus, Nathan},
	year         = 2017,
	journal      = {arXiv.org},
	journal      = {arXiv preprint arXiv:1612.08321},
	publisher    = {Cornell University Library, arXiv.org},
	address      = {Ithaca},
	language     = {eng}
}
@article{DietterichT.G.2000HRLw,
	title        = {Hierarchical Reinforcement Learning with the MAXQ Value Function Decomposition},
	author       = {Dietterich, T. G.},
	year         = 2000,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 13,
	pages        = {227--303},
	keywords     = {Computer Science;}
}
@article{HenmiMasayuki2004Apcn,
	title        = {A paradox concerning nuisance parameters and projected estimating functions},
	author       = {Henmi, Masayuki and Eguchi, Shinto},
	year         = 2004,
	journal      = {Biometrika},
	publisher    = {Oxford University Press},
	volume       = 91,
	pages        = {929--941},
	keywords     = {Orthogonality Of Estimating Functions; Propensity Score; Semiparametric Model.}
}
@article{RobinsJM1992Eeeb,
	title        = {Estimating exposure effects by modelling the expectation of exposure conditional on confounders},
	author       = {Robins, J M and Mark, S D and Newey, W K},
	year         = 1992,
	journal      = {Biometrics},
	volume       = 48,
	pages        = 479,
	language     = {eng},
	keywords     = {Epidemiologic Methods ; Forced Expiratory Volume ; Models, Statistical ; Smoking -- Physiopathology}
}
@article{BertsekasDimitriP2009Pemf,
	title        = {Projected equation methods for approximate solution of large linear systems},
	author       = {Bertsekas, Dimitri P and Yu, Huizhen},
	year         = 2009,
	journal      = {Journal of Computational and Applied Mathematics},
	publisher    = {Elsevier B.V},
	volume       = 227,
	pages        = {27--50},
	language     = {eng},
	keywords     = {Linear Equations ; Projected Equations ; Dynamic Programming ; Temporal Differences ; Simulation ; Value Iteration ; Jacobi Method ; Mathematics}
}
@article{jiang2019value,
	title        = {On Value Functions and the Agent-Environment Boundary},
	author       = {Jiang, Nan},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1905.13341}
}
@inproceedings{le2019batch,
	title        = {Batch Policy Learning under Constraints},
	author       = {Le, Hoang and Voloshin, Cameron and Yue, Yisong},
	year         = 2019,
	booktitle    = {International Conference on Machine Learning},
	pages        = {3703--3712}
}
@incollection{mohri2009,
	title        = {Rademacher Complexity Bounds for Non-I.I.D. Processes},
	author       = {Mohri, Mehryar and Rostamizadeh, Afshin},
	year         = 2009,
	booktitle    = {Advances in Neural Information Processing Systems 21},
	pages        = {1097--1104}
}
@article{muller1997integral,
	title        = {Integral probability metrics and their generating classes of functions},
	author       = {M{\"u}ller, Alfred},
	year         = 1997,
	journal      = {Advances in Applied Probability},
	publisher    = {Cambridge University Press},
	volume       = 29,
	number       = 2,
	pages        = {429--443}
}
@inproceedings{sutton2015introduction,
	title        = {Introduction to reinforcement learning with function approximation},
	author       = {Sutton, Rich},
	year         = 2015,
	booktitle    = {Tutorial at the Conference on Neural Information Processing Systems}
}
@inproceedings{lu2018non,
	title        = {Non-delusional Q-learning and value-iteration},
	author       = {Lu, Tyler and Schuurmans, Dale and Boutilier, Craig},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {9971--9981}
}
@article{van2018deep,
	title        = {Deep Reinforcement Learning and the Deadly Triad},
	author       = {Van Hasselt, Hado and Doron, Yotam and Strub, Florian and Hessel, Matteo and Sonnerat, Nicolas and Modayil, Joseph},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1812.02648}
}
@inproceedings{szepesvari2004interpolation,
	title        = {Interpolation-based Q-learning},
	author       = {Szepesv{\'a}ri, Csaba and Smart, William D},
	year         = 2004,
	booktitle    = {Proceedings of the twenty-first international conference on Machine learning},
	pages        = 100,
	organization = {ACM}
}
@inproceedings{lizotte2011convergent,
	title        = {Convergent fitted value iteration with linear function approximation},
	author       = {Lizotte, Daniel J},
	year         = 2011,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2537--2545}
}
@inproceedings{hutter2014extreme,
	title        = {Extreme state aggregation beyond MDPs},
	author       = {Hutter, Marcus},
	year         = 2014,
	booktitle    = {International Conference on Algorithmic Learning Theory},
	pages        = {185--199},
	organization = {Springer}
}
@article{shimodaira2000improving,
	title        = {Improving predictive inference under covariate shift by weighting the log-likelihood function},
	author       = {Shimodaira, Hidetoshi},
	year         = 2000,
	journal      = {Journal of statistical planning and inference},
	publisher    = {Elsevier},
	volume       = 90,
	number       = 2,
	pages        = {227--244}
}
@inproceedings{tosatto2017boosted,
	title        = {Boosted fitted q-iteration},
	author       = {Tosatto, Samuele and Pirotta, Matteo and D'Eramo, Carlo and Restelli, Marcello},
	year         = 2017,
	booktitle    = {Proceedings of the 34th International Conference on Machine Learning-Volume 70},
	pages        = {3434--3443},
	organization = {JMLR. org}
}
@inproceedings{riedmiller2005neural,
	title        = {Neural fitted Q iteration--first experiences with a data efficient neural reinforcement learning method},
	author       = {Riedmiller, Martin},
	year         = 2005,
	booktitle    = {European Conference on Machine Learning},
	pages        = {317--328},
	organization = {Springer}
}
@inproceedings{pires2016policy,
	title        = {Policy error bounds for model-based reinforcement learning with factored linear models},
	author       = {Pires, Bernardo {\'A}vila and Szepesv{\'a}ri, Csaba},
	year         = 2016,
	booktitle    = {Conference on Learning Theory},
	pages        = {121--151}
}
@inproceedings{kulesza2014low,
	title        = {Low-rank spectral learning},
	author       = {Kulesza, Alex and Rao, N Raj and Singh, Satinder},
	year         = 2014,
	booktitle    = {Artificial Intelligence and Statistics},
	pages        = {522--530}
}
@article{chen2018scalable,
	title        = {Scalable Bilinear $\pi$ Learning Using State and Action Features},
	author       = {Chen, Yichen and Li, Lihong and Wang, Mengdi},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1804.10328}
}
@book{shalev2014understanding,
	title        = {Understanding machine learning: From theory to algorithms},
	author       = {Shalev-Shwartz, Shai and Ben-David, Shai},
	year         = 2014,
	publisher    = {Cambridge University Press}
}
@article{szepesvari2010algorithms,
	title        = {Algorithms for reinforcement learning},
	author       = {Szepesv{\'a}ri, Csaba},
	year         = 2010,
	journal      = {Synthesis lectures on artificial intelligence and machine learning},
	publisher    = {Morgan \& Claypool Publishers},
	volume       = 4,
	number       = 1,
	pages        = {1--103}
}
@article{yang2019theoretical,
	title        = {{A Theoretical Analysis of Deep Q-Learning}},
	author       = {Yang, Zhuora and Xie, Yuchen and Wang, Zhaoran},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1901.00137}
}

@phdthesis{van1994feature,
	title        = {Feature-based methods for large scale dynamic programming},
	author       = {Van Roy, Benjamin},
	year         = 1994,
	school       = {Massachusetts Institute of Technology}
}
@incollection{yu1997assouad,
	title        = {{Assouad, Fano, and Le Cam}},
	author       = {Yu, Bin},
	year         = 1997,
	booktitle    = {Festschrift for Lucien Le Cam},
	publisher    = {Springer},
	pages        = {423--435}
}
@inproceedings{dai2018sbeed,
	title        = {Sbeed: Convergent reinforcement learning with nonlinear function approximation},
	author       = {Dai, Bo and Shaw, Albert and Li, Lihong and Xiao, Lin and He, Niao and Liu, Zhen and Chen, Jianshu and Song, Le},
	year         = 2018,
	booktitle    = {International Conference on Machine Learning},
	pages        = {1133--1142}
}
@inproceedings{maillard2010finite,
	title        = {Finite-sample analysis of Bellman residual minimization},
	author       = {Maillard, Odalric-Ambrym and Munos, R{\'e}mi and Lazaric, Alessandro and Ghavamzadeh, Mohammad},
	year         = 2010,
	booktitle    = {Proceedings of 2nd Asian Conference on Machine Learning},
	pages        = {299--314}
}
@inproceedings{munos2003error,
	title        = {Error bounds for approximate policy iteration},
	author       = {Munos, R{\'e}mi},
	year         = 2003,
	booktitle    = {ICML},
	volume       = 3,
	pages        = {560--567}
}
@article{gretton2012kernel,
	title        = {A kernel two-sample test},
	author       = {Gretton, Arthur and Borgwardt, Karsten M and Rasch, Malte J and Sch{\"o}lkopf, Bernhard and Smola, Alexander},
	year         = 2012,
	journal      = {Journal of Machine Learning Research},
	volume       = 13,
	pages        = {723--773}
}
@inproceedings{sandholm2012lossy,
	title        = {Lossy stochastic game abstraction with bounds},
	author       = {Sandholm, Tuomas and Singh, Satinder},
	year         = 2012,
	booktitle    = {Proceedings of the 13th ACM Conference on Electronic Commerce},
	pages        = {880--897}
}
@incollection{ravindran2002model,
	title        = {Model minimization in hierarchical reinforcement learning},
	author       = {Ravindran, Balaraman and Barto, Andrew},
	year         = 2002,
	booktitle    = {Abstraction, Reformulation, and Approximation},
	pages        = {196--211}
}
@article{hester2013texplore,
	title        = {{TEXPLORE}: real-time sample-efficient reinforcement learning for robots},
	author       = {Hester, Todd and Stone, Peter},
	year         = 2013,
	journal      = {Machine Learning},
	volume       = 90,
	number       = 3,
	pages        = {385--429}
}
@article{givan2003equivalence,
	title        = {Equivalence notions and model minimization in {M}arkov decision processes},
	author       = {Givan, Robert and Dean, Thomas and Greig, Matthew},
	year         = 2003,
	journal      = {Artificial Intelligence},
	volume       = 147,
	number       = 1,
	pages        = {163--223}
}
@inproceedings{li2006towards,
	title        = {Towards a unified theory of state abstraction for {MDP}s},
	author       = {Li, Lihong and Walsh, Thomas J and Littman, Michael L},
	year         = 2006,
	booktitle    = {Proceedings of the 9th International Symposium on Artificial Intelligence and Mathematics},
	pages        = {531--539}
}
@phdthesis{ravindran2004algebraic,
	title        = {An algebraic approach to abstraction in reinforcement learning},
	author       = {Ravindran, Balaraman},
	year         = 2004,
	school       = {University of Massachusetts Amherst}
}
@inproceedings{sutton1999policy,
	title        = {Policy gradient methods for reinforcement learning with function approximation},
	author       = {Sutton, Richard S and McAllester, David A and Singh, Satinder P and Mansour, Yishay},
	year         = 1999,
	booktitle    = {Advances in Neural Information Processing Systems},
	volume       = 99,
	pages        = {1057--1063}
}
@article{dietterich2000hierarchical,
	title        = {Hierarchical reinforcement learning with the {MAXQ} value function decomposition},
	author       = {Dietterich, Thomas G},
	year         = 2000,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 13,
	pages        = {227--303}
}
@incollection{even-dar2003approximate,
	title        = {Approximate equivalence of {M}arkov decision processes},
	author       = {Even-Dar, Eyal and Mansour, Yishay},
	year         = 2003,
	booktitle    = {Learning Theory and Kernel Machines},
	pages        = {581--594}
}
@inproceedings{ferns2004metrics,
	title        = {Metrics for finite {M}arkov decision processes},
	author       = {Ferns, Norm and Panangaden, Prakash and Precup, Doina},
	year         = 2004,
	booktitle    = {Proceedings of Uncertainty in Artificial Intelligence},
	pages        = {162--169}
}
@inproceedings{taylor2008bounding,
	title        = {Bounding performance loss in approximate {MDP} homomorphisms},
	author       = {Taylor, Jonathan and Precup, Doina and Panagaden, Prakash},
	year         = 2008,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1649--1656}
}
@inproceedings{ferns2006methods,
	title        = {Methods for computing state similarity in markov decision processes},
	author       = {Ferns, Norm and Castro, Pablo Samuel and Precup, Doina and Panangaden, Prakash},
	year         = 2006,
	booktitle    = {Proceedings of Uncertainty in Artificial Intelligence},
	pages        = {174--181}
}
@inproceedings{narayanamurthy2007efficientlya,
	title        = {Efficiently {E}xploiting {S}ymmetries in {R}eal {T}ime {D}ynamic {P}rogramming},
	author       = {Narayanamurthy, Shravan Matthur and Ravindran, Balaraman},
	year         = 2007,
	booktitle    = {International Joint Conference on Artificial Intelligence},
	pages        = {2556--2561}
}
@article{whitt1978approximations,
	title        = {{Approximations of dynamic programs, I}},
	author       = {Whitt, Ward},
	year         = 1978,
	journal      = {Mathematics of Operations Research},
	publisher    = {INFORMS},
	volume       = 3,
	number       = 3,
	pages        = {231--243}
}
@article{ortner2014selecting,
	title        = {Selecting {N}ear-{O}ptimal {A}pproximate {S}tate {R}epresentations in {R}einforcement {L}earning},
	author       = {Ortner, Ronald and Maillard, Odalric-Ambrym and Ryabko, Daniil},
	year         = 2014,
	journal      = {arXiv preprint arXiv:1405.2652}
}
@inproceedings{hallak2013model,
	title        = {Model selection in markovian processes},
	author       = {Hallak, Assaf and Di-Castro, Dotan and Mannor, Shie},
	year         = 2013,
	booktitle    = {Proceedings of the 19th ACM SIGKDD Conference on Knowledge Discovery and Data mining},
	pages        = {374--382}
}
@inproceedings{bartlett2009regal,
	title        = {{REGAL}: {A} regularization based algorithm for reinforcement learning in weakly communicating {MDP}s},
	author       = {Bartlett, Peter L and Tewari, Ambuj},
	year         = 2009,
	booktitle    = {Proceedings of the 25th Conference on Uncertainty in Artificial Intelligence},
	pages        = {35--42},
	organization = {AUAI Press}
}
@inproceedings{mandel2014offline,
	title        = {Offline policy evaluation across representations with applications to educational games},
	author       = {Mandel, Travis and Liu, Yun-En and Levine, Sergey and Brunskill, Emma and Popovic, Zoran},
	year         = 2014,
	booktitle    = {Proceedings of the 13th International Conference on Autonomous Agents and Multi-Agent Systems},
	pages        = {1077--1084}
}
@inproceedings{parr2008analysis,
	title        = {An analysis of linear models, linear value-function approximation, and feature selection for reinforcement learning},
	author       = {Parr, Ronald and Li, Lihong and Taylor, Gavin and Painter-Wakefield, Christopher and Littman, Michael L},
	year         = 2008,
	booktitle    = {Proceedings of the 25th International Conference on Machine Learning},
	pages        = {752--759}
}
@article{ormoneit2002kernel,
	title        = {Kernel-based reinforcement learning},
	author       = {Ormoneit, Dirk and Sen, {\'S}aunak},
	year         = 2002,
	journal      = {Machine learning},
	publisher    = {Kluwer Academic Publishers},
	volume       = 49,
	number       = {2-3},
	pages        = {161--178}
}
@inproceedings{ravindran2004approximate,
	title        = {Approximate homomorphisms: {A} framework for nonexact minimization in {M}arkov decision processes},
	author       = {Ravindran, Balaraman and Barto, Andrew},
	year         = 2004,
	booktitle    = {Proceedings of the 5th International Conference on Knowledge-Based Computer Systems}
}
@article{bartlett2003rademacher,
	title        = {Rademacher and {G}aussian complexities: {R}isk bounds and structural results},
	author       = {Bartlett, Peter L and Mendelson, Shahar},
	year         = 2003,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 3,
	pages        = {463--482}
}
@article{boutilier2000stochastic,
	title        = {{S}tochastic dynamic programming with factored representations},
	author       = {Boutilier, Craig and Dearden, Richard and Goldszmidt, Mois{\'e}s},
	year         = 2000,
	journal      = {Artificial Intelligence},
	publisher    = {Elsevier},
	volume       = 121,
	number       = 1,
	pages        = {49--107}
}
@article{brafman2003r,
	title        = {R-max-a general polynomial time algorithm for near-optimal reinforcement learning},
	author       = {Brafman, Ronen I and Tennenholtz, Moshe},
	year         = 2003,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 3,
	pages        = {213--231}
}
@article{kaelbling1996reinforcement,
	title        = {Reinforcement {L}earning: {A} {S}urvey},
	author       = {Kaelbling, Leslie Pack and Littman, Michael L and Moore, Andrew W},
	year         = 1996,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 4,
	pages        = {237--285}
}
@book{kearns1994introduction,
	title        = {An {I}ntroduction to {C}omputational {L}earning {T}heory},
	author       = {Kearns, Michael J and Vazirani, Umesh V},
	year         = 1994,
	publisher    = {MIT Press},
	isbn         = 9780262111935,
	url          = {http://books.google.com/books?id=vCA01wY6iywC},
	lccn         = {lc94016588}
}
@article{kearns2002optimal,
	title        = {Near-optimal reinforcement learning in polynomial time},
	author       = {Kearns, Michael and Singh, Satinder},
	year         = 2002,
	journal      = {Machine Learning},
	publisher    = {Springer},
	volume       = 49,
	number       = {2-3},
	pages        = {209--232}
}
@incollection{koltchinskii2000rademacher,
	title        = {Rademacher processes and bounding the risk of function learning},
	author       = {Koltchinskii, Vladimir and Panchenko, Dmitriy},
	year         = 2000,
	booktitle    = {High Dimensional Probability II},
	publisher    = {Springer},
	pages        = {443--457}
}
@article{mannor2007bias,
	title        = {Bias and variance approximation in value function estimates},
	author       = {Mannor, Shie and Simester, Duncan and Sun, Peng and Tsitsiklis, John N},
	year         = 2007,
	journal      = {Management Science},
	publisher    = {INFORMS},
	volume       = 53,
	number       = 2,
	pages        = {308--322}
}
@phdthesis{paduraru2013policy,
	title        = {Off-policy {E}valuation in {M}arkov {D}ecision {P}rocesses},
	author       = {Paduraru, Cosmin},
	year         = 2013,
	school       = {McGill University}
}
@inproceedings{strehl2005theoretical,
	title        = {A theoretical analysis of model-based interval estimation},
	author       = {Strehl, Alexander L and Littman, Michael L},
	year         = 2005,
	booktitle    = {Proceedings of the 22nd International Conference on Machine learning},
	pages        = {856--863},
	organization = {ACM}
}
@article{strehl2009reinforcement,
	title        = {Reinforcement learning in finite {MDP}s: {PAC} analysis},
	author       = {Strehl, Alexander L and Li, Lihong and Littman, Michael L},
	year         = 2009,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 10,
	pages        = {2413--2444}
}
@inproceedings{tewari2006sample,
	title        = {Sample complexity of policy search with known dynamics},
	author       = {Tewari, Ambuj and Bartlett, Peter L},
	year         = 2006,
	booktitle    = {Advances in Neural Information Processing Systems},
	volume       = 19,
	pages        = {97--104}
}
@inproceedings{vapnik1992principles,
	title        = {Principles of risk minimization for learning theory},
	author       = {Vapnik, Vladimir},
	year         = 1992,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {831--838}
}
@book{vapnik1998statistical,
	title        = {Statistical learning theory},
	author       = {Vapnik, Vladimir},
	year         = 1998,
	publisher    = {Wiley New York},
	volume       = 2
}
@inproceedings{jiang2014improving,
	title        = {Improving {UCT} planning via approximate homomorphisms},
	author       = {Jiang, Nan and Singh, Satinder and Lewis, Richard},
	year         = 2014,
	booktitle    = {Proceedings of the 13th International Conference on Autonomous Agents and Multi-Agent Systems},
	pages        = {1289--1296}
}
@inproceedings{paduraru2008model,
	title        = {Model-based Reinforcement Learning with State Aggregation},
	author       = {Paduraru, Cosmin and Kaplow, Robert and Precup, Doina and Pineau, Joelle},
	year         = 2008,
	booktitle    = {8th European Workshop on Reinforcement Learning}
}
@phdthesis{talvitie2010simple,
	title        = {Simple Partial Models for Complex Dynamical Systems},
	author       = {Talvitie, Erik N},
	year         = 2010,
	school       = {The University of Michigan}
}
@inproceedings{jong2005state,
	title        = {State abstraction discovery from irrelevant state variables},
	author       = {Jong, Nicholas K and Stone, Peter},
	year         = 2005,
	booktitle    = {Proceedings of the 19th International Joint Conference on Artificial Intelligence},
	pages        = {752--757}
}
@article{singh1994upper,
	title        = {An upper bound on the loss from approximate optimal-value functions},
	author       = {Singh, Satinder and Yee, Richard},
	year         = 1994,
	journal      = {Machine Learning},
	volume       = 16,
	number       = 3,
	pages        = {227--233}
}
@inproceedings{dinculescu2010approximate,
	title        = {Approximate predictive representations of partially observable systems},
	author       = {Dinculescu, Monica and Precup, Doina},
	year         = 2010,
	booktitle    = {Proceedings of the 27th International Conference on Machine Learning (ICML-10)},
	pages        = {895--902}
}
@article{talvitie2011learning,
	title        = {Learning to Make Predictions In Partially Observable Environments Without a Generative Model},
	author       = {Talvitie, Erik and Singh, Satinder},
	year         = 2011,
	journal      = {Journal of Artificial Intelligence Research (JAIR)},
	volume       = 42,
	pages        = {353--392}
}
@inproceedings{maillard2014how,
	title        = {"{H}ow hard is my {MDP}?" {T}he distribution-norm to the rescue},
	author       = {Maillard, Odalric-Ambrym and Mann, Timothy A and Mannor, Shie},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1835--1843}
}
@article{seijen2014efficient,
	title        = {Efficient abstraction selection in reinforcement learning},
	author       = {van Seijen, Harm and Whiteson, Shimon and Kester, Leon},
	year         = 2014,
	journal      = {Computational Intelligence},
	volume       = 30,
	number       = 4,
	pages        = {657--699}
}
@inproceedings{konidaris2009efficient,
	title        = {Efficient {S}kill {L}earning using {A}bstraction {S}election},
	author       = {Konidaris, George and Barto, Andrew},
	year         = 2009,
	booktitle    = {Proceedings of the 21st International Joint Conference on Artificial Intelligence}
}
@article{cobo2014abstraction,
	title        = {Abstraction from demonstration for efficient reinforcement learning in high-dimensional domains},
	author       = {Cobo, Luis C and Subramanian, Kaushik and Isbell, Charles L and Lanterman, Aaron D and Thomaz, Andrea L},
	year         = 2014,
	journal      = {Artificial Intelligence},
	volume       = 216,
	pages        = {103--128}
}
@article{farahmand2011regularization,
	title        = {Regularization in reinforcement learning},
	author       = {Farahmand, Amir-massoud},
	year         = 2011,
	publisher    = {University of Alberta}
}
@article{agarwal2016corralling,
	title        = {Corralling a Band of Bandit Algorithms},
	author       = {Agarwal, Alekh and Luo, Haipeng and Neyshabur, Behnam and Schapire, Robert E},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1612.06246}
}
@phdthesis{scott2004dyadic,
	title        = {Dyadic decision trees},
	author       = {Scott, Clayton D},
	year         = 2004,
	school       = {University of Wisconsin at Madison}
}
@inproceedings{ng2003autonomous,
	title        = {Autonomous helicopter flight via reinforcement learning},
	author       = {Ng, Andrew Y and Kim, H Jin and Jordan, Michael I and Sastry, Shankar and Ballianda, Shiv},
	year         = 2003,
	booktitle    = {NIPS},
	volume       = 16
}
@inproceedings{johnson2016malmo,
	title        = {The malmo platform for artificial intelligence experimentation},
	author       = {Johnson, Matthew and Hofmann, Katja and Hutton, Tim and Bignell, David},
	year         = 2016,
	booktitle    = {International joint conference on artificial intelligence (IJCAI)},
	pages        = 4246
}
@article{brockman2016openai,
	title        = {{OpenAI} gym},
	author       = {Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, John and Tang, Jie and Zaremba, Wojciech},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1606.01540}
}
@inproceedings{li2010contextual,
	title        = {A contextual-bandit approach to personalized news article recommendation},
	author       = {Li, Lihong and Chu, Wei and Langford, John and Schapire, Robert E},
	year         = 2010,
	booktitle    = {Proceedings of the 19th international conference on World wide web},
	pages        = {661--670},
	organization = {ACM}
}
@article{lei2012a,
	title        = {A" SMART" design for building individualized treatment sequences},
	author       = {Lei, H and Nahum-Shani, I and Lynch, K and Oslin, D and Murphy, SA},
	year         = 2012,
	journal      = {Annual review of clinical psychology},
	publisher    = {Annual Reviews},
	volume       = 8,
	pages        = {21--48}
}
@article{singh2002optimizing,
	title        = {Optimizing dialogue management with reinforcement learning: {E}xperiments with the {NJF}un system},
	author       = {Singh, Satinder and Litman, Diane and Kearns, Michael and Walker, Marilyn},
	year         = 2002,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 16,
	pages        = {105--133}
}
@article{williams1992simple,
	title        = {Simple statistical gradient-following algorithms for connectionist reinforcement learning},
	author       = {Williams, Ronald J},
	year         = 1992,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 8,
	number       = {3-4},
	pages        = {229--256}
}
@phdthesis{li2009unifying,
	title        = {A unifying framework for computational reinforcement learning theory},
	author       = {Li, Lihong},
	year         = 2009,
	school       = {Rutgers, The State University of New Jersey}
}
@book{mohri2012foundations,
	title        = {Foundations of machine learning},
	author       = {Mohri, Mehryar and Rostamizadeh, Afshin and Talwalkar, Ameet},
	year         = 2012,
	publisher    = {MIT press}
}
@inproceedings{vanseijen2015deeper,
	title        = {A Deeper Look at Planning as Learning from Replay},
	author       = {van Seijen, Harm and Sutton, Rich},
	year         = 2015,
	booktitle    = {Proceedings of the 32nd International Conference on Machine Learning},
	pages        = {2314--2322}
}
@article{bradtke1996linear,
	title        = {Linear least-squares algorithms for temporal difference learning},
	author       = {Bradtke, Steven J and Barto, Andrew G},
	year         = 1996,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 22,
	number       = {1-3},
	pages        = {33--57}
}
@article{lin1992self,
	title        = {Self-improving reactive agents based on reinforcement learning, planning and teaching},
	author       = {Lin, Long-Ji},
	year         = 1992,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 8,
	number       = {3-4},
	pages        = {293--321}
}
@inproceedings{wang2016dueling,
	title        = {Dueling Network Architectures for Deep Reinforcement Learning},
	author       = {Wang, Ziyu and Schaul, Tom and Hessel, Matteo and Hasselt, Hado van and Lanctot, Marc and Freitas, Nando de},
	year         = 2016,
	booktitle    = {Proceedings of the 33rd International Conference on Machine Learning},
	pages        = {1995--2003}
}
{sutton1996generalization,
	title={Generalization in reinforcement learning: {S}uccessful examples using sparse coarse coding},
	author={Sutton, Richard S},
	journal={Advances in Neural Information Processing Systems},
	pages={1038--1044},
	year={1996}
%	publisher={MORGAN KAUFMANN PUBLISHERS}
}
@phdthesis{watkins1989learning,
	title        = {Learning from delayed rewards},
	author       = {Watkins, Christopher John Cornish Hellaby},
	year         = 1989,
	school       = {University of Cambridge England}
}
@article{hoeffding1963probability,
	title        = {Probability inequalities for sums of bounded random variables},
	author       = {Hoeffding, Wassily},
	year         = 1963,
	journal      = {Journal of the American statistical association},
	publisher    = {Taylor \& Francis Group},
	volume       = 58,
	number       = 301,
	pages        = {13--30}
}
@inproceedings{strehl2006pac,
	title        = {{PAC} model-free reinforcement learning},
	author       = {Strehl, Alexander L and Li, Lihong and Wiewiora, Eric and Langford, John and Littman, Michael L},
	year         = 2006,
	booktitle    = {Proceedings of the 23rd international conference on Machine learning},
	pages        = {881--888},
	organization = {ACM}
}
@article{auer2007logarithmic,
	title        = {Logarithmic online regret bounds for undiscounted reinforcement learning},
	author       = {Auer, Peter and Ortner, Ronald},
	year         = 2007,
	journal      = {Advances in Neural Information Processing Systems},
	publisher    = {MIT; 1998},
	volume       = 19,
	pages        = 49
}
@article{jaksch2010optimal,
	title        = {Near-optimal regret bounds for reinforcement learning},
	author       = {Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
	year         = 2010,
	journal      = {Journal of Machine Learning Research},
	volume       = 11,
	number       = {Apr},
	pages        = {1563--1600}
}
@phdthesis{kakade2003sample,
	title        = {On the sample complexity of reinforcement learning},
	author       = {Kakade, Sham Machandranath},
	year         = 2003,
	school       = {University of College London}
}
@inproceedings{krishnamurthy2016pac,
	title        = {{PAC} reinforcement learning with rich observations},
	author       = {Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John},
	year         = 2016,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1840--1848}
}

@inproceedings{dann2015sample,
	title        = {Sample complexity of episodic fixed-horizon reinforcement learning},
	author       = {Dann, Christoph and Brunskill, Emma},
	year         = 2015,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2818--2826}
}
@article{valiant1984theory,
	title        = {A theory of the learnable},
	author       = {Valiant, Leslie G},
	year         = 1984,
	journal      = {Communications of the ACM},
	publisher    = {ACM},
	volume       = 27,
	number       = 11,
	pages        = {1134--1142}
}
@inproceedings{langford2008epoch,
	title        = {The epoch-greedy algorithm for multi-armed bandits with side information},
	author       = {Langford, John and Zhang, Tong},
	year         = 2008,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {817--824}
}
@article{puterman1994markov,
	title        = {Markov decision processes},
	author       = {Puterman, ML},
	year         = 1994,
	journal      = {Jhon Wiley \& Sons, New Jersey}
}
@inproceedings{rosencrantz2004learning,
	title        = {Learning low dimensional predictive representations},
	author       = {Rosencrantz, Matthew and Gordon, Geoff and Thrun, Sebastian},
	year         = 2004,
	booktitle    = {Proceedings of the twenty-first international conference on Machine learning},
	pages        = 88,
	organization = {ACM}
}
@article{kober2013reinforcement,
	title        = {Reinforcement learning in robotics: A survey},
	author       = {Kober, Jens and Bagnell, J Andrew and Peters, Jan},
	year         = 2013,
	journal      = {The International Journal of Robotics Research},
	publisher    = {SAGE Publications},
	pages        = {0278364913495721}
}
@inproceedings{petrik2014raam,
	title        = {{RAAM: The Benefits of Robustness in Approximating Aggregated MDPs in Reinforcement Learning}},
	author       = {Petrik, Marek and Subramanian, Dharmashankar},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1979--1987}
}
@article{vanroy2006performance,
	title        = {Performance loss bounds for approximate value iteration with state aggregation},
	author       = {Van Roy, Benjamin},
	year         = 2006,
	journal      = {Mathematics of Operations Research},
	publisher    = {INFORMS},
	volume       = 31,
	number       = 2,
	pages        = {234--244}
}
@manual{nan_abstraction_notes,
	title        = {{CS 598: Notes on State Abstractions}},
	author       = {Nan Jiang},
	year         = 2018,
	note         = {\url{http://nanjiang.cs.illinois.edu/files/cs598/note4.pdf}},
	organization = {{University of Illinois at Urbana-Champaign}}
}
@manual{nan_mdp_notes,
	title        = {{CS 598: MDP Preliminaries}},
	author       = {Nan Jiang},
	year         = 2018,
	note         = {\url{http://nanjiang.cs.illinois.edu/files/cs598/note1.pdf}},
	organization = {{University of Illinois at Urbana-Champaign}}
}
@manual{shipranotes,
	title        = {{IEOR 8100: Reinforcement Learning. Lecture 4: Approximate Dynamic Programming}},
	author       = {Shipra Agrawal},
	year         = 2018,
	note         = {\url{https://ieor8100.github.io/rl/docs/Lecture%204%20-%20approximate%20DP.pdf}},
	organization = {{Columbia University}}
}
@phdthesis{van1994feature,
	title        = {Feature-based methods for large scale dynamic programming},
	author       = {Van Roy, Benjamin},
	year         = 1994,
	school       = {Massachusetts Institute of Technology}
}
@inproceedings{kulesza2015low,
	title        = {{Low-Rank Spectral Learning with Weighted Loss Functions}},
	author       = {Kulesza, Alex and Jiang, Nan and Singh, Satinder},
	year         = 2015,
	booktitle    = {Proceedings of the 18th International Conference on Artificial Intelligence and Statistics},
	pages        = {517--525}
}
@incollection{riedmiller2005neural,
	title        = {{Neural fitted Q iteration--first experiences with a data efficient neural reinforcement learning method}},
	author       = {Riedmiller, Martin},
	year         = 2005,
	booktitle    = {Machine Learning: ECML 2005},
	publisher    = {Springer},
	pages        = {317--328}
}
@inproceedings{kobilarov2015sample,
	title        = {{Sample Complexity Bounds for Iterative Stochastic Policy Optimization}},
	author       = {Kobilarov, Marin},
	year         = 2015,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {3096--3104}
}
@inproceedings{abel2016near,
	title        = {Near optimal behavior via approximate state abstraction},
	author       = {Abel, David and Hershkowitz, D Ellis and Littman, Michael L},
	year         = 2016,
	booktitle    = {Proceedings of the 33rd International Conference on International Conference on Machine Learning-Volume 48},
	pages        = {2915--2923},
	organization = {JMLR. org}
}
@article{bellman1956dynamic,
	title        = {{Dynamic programming and Lagrange multipliers}},
	author       = {Bellman, Richard},
	year         = 1956,
	journal      = {Proceedings of the National Academy of Sciences},
	publisher    = {National Acad Sciences},
	volume       = 42,
	number       = 10,
	pages        = {767--769}
}
@book{kumar2012customer,
	title        = {{Customer relationship management: Concept, strategy, and tools}},
	author       = {Kumar, Vineet and Reinartz, Werner},
	year         = 2012,
	publisher    = {Springer Science \& Business Media}
}
@article{murphy2003optimal,
	title        = {Optimal dynamic treatment regimes},
	author       = {Murphy, Susan A},
	year         = 2003,
	journal      = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
	publisher    = {Wiley Online Library},
	volume       = 65,
	number       = 2,
	pages        = {331--355}
}
@inproceedings{hamilton2013modelling,
	title        = {Modelling sparse dynamical systems with compressed predictive state representations},
	author       = {Hamilton, William L and Fard, Mahdi M and Pineau, Joelle},
	year         = 2013,
	booktitle    = {Proceedings of the 30th International Conference on Machine Learning (ICML-13)},
	pages        = {178--186}
}
@inproceedings{boots2010closing,
	title        = {Closing the learning-planning loop with predictive state representations},
	author       = {Boots, Byron and Siddiqi, Sajid M and Gordon, Geoffrey J},
	year         = 2010,
	booktitle    = {Proceedings of the 9th International Conference on Autonomous Agents and Multiagent Systems},
	pages        = {1369--1370}
}
@incollection{farahmand2008regularized,
	title        = {{Regularized fitted Q-iteration: Application to planning}},
	author       = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
	year         = 2008,
	booktitle    = {Recent Advances in Reinforcement Learning},
	publisher    = {Springer Berlin Heidelberg},
	pages        = {55--68}
}
@inproceedings{farahmand2009regularized,
	title        = {{Regularized fitted Q-iteration for planning in continuous-space Markovian decision problems}},
	author       = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
	year         = 2009,
	booktitle    = {American Control Conference.},
	pages        = {725--730},
	organization = {IEEE}
}
@article{tsitsiklis1997analysis,
	title        = {An Analysis of Temporal-Difference Learning with Function Approximation},
	author       = {Tsitsiklis, John N and Van Roy, Benjamin},
	year         = 1997,
	journal      = {IEEE TRANSACTIONS ON AUTOMATIC CONTROL},
	volume       = 42
}
@inproceedings{szepesvari2005finite,
	title        = {Finite time bounds for sampling based fitted value iteration},
	author       = {Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
	year         = 2005,
	booktitle    = {Proceedings of the 22nd international conference on Machine learning},
	pages        = {880--887},
	organization = {ACM}
}
@article{lazaric2012finite,
	title        = {Finite-sample analysis of least-squares policy iteration},
	author       = {Lazaric, Alessandro and Ghavamzadeh, Mohammad and Munos, R{\'e}mi},
	year         = 2012,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 13,
	number       = 1,
	pages        = {3041--3074}
}
@article{bertsekas2011approximate,
	title        = {Approximate policy iteration: A survey and some new methods},
	author       = {Bertsekas, Dimitri P},
	year         = 2011,
	journal      = {Journal of Control Theory and Applications},
	publisher    = {Springer},
	volume       = 9,
	number       = 3,
	pages        = {310--335}
}
@inproceedings{gordon1995stable,
	title        = {Stable function approximation in dynamic programming},
	author       = {Gordon, Geoffrey J},
	year         = 1995,
	booktitle    = {Proceedings of the twelfth international conference on machine learning},
	pages        = {261--268}
}
@book{bertsekas1996neuro,
	title        = {Neuro-Dynamic Programming},
	author       = {Bertsekas, Dimitri P and Tsitsiklis, John N},
	year         = 1996,
	publisher    = {Athena Scientific, Belmont, MA}
}
@article{bellman1959functional,
	title        = {Functional approximations and dynamic programming},
	author       = {Bellman, Richard and Dreyfus, Stuart},
	year         = 1959,
	journal      = {Mathematical Tables and Other Aids to Computation},
	publisher    = {JSTOR},
	volume       = 13,
	number       = 68,
	pages        = {247--251}
}
@article{pires2016policy,
	title        = {Policy Error Bounds for Model-Based Reinforcement Learning with Factored Linear Models},
	author       = {Pires, Bernardo {\'A}vila and Szepesv{\'a}ri, Csaba},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1602.06346}
}
@article{mnih2015human,
	title        = {Human-level control through deep reinforcement learning},
	author       = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and others},
	year         = 2015,
	journal      = {Nature},
	publisher    = {Nature Publishing Group},
	volume       = 518,
	number       = 7540,
	pages        = {529--533}
}
@article{silver2016mastering,
	title        = {Mastering the game of Go with deep neural networks and tree search},
	author       = {Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and van den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
	year         = 2016,
	journal      = {Nature},
	publisher    = {Nature Publishing Group},
	volume       = 529,
	number       = 7587,
	pages        = {484--489}
}
@inproceedings{jiang2016doubly,
	title        = {{Doubly Robust Off-policy Value Evaluation for Reinforcement Learning}},
	author       = {Nan Jiang and Lihong Li},
	year         = 2016,
	booktitle    = {Proceedings of the 33rd International Conference on Machine Learning},
	volume       = 48,
	pages        = {652--661}
}
@inproceedings{thomas2016data,
	title        = {{Data-Efficient Off-Policy Policy Evaluation for Reinforcement Learning}},
	author       = {Thomas, Philip and Brunskill, Emma},
	year         = 2016,
	booktitle    = {Proceedings of the 33rd International Conference on Machine Learning}
}
@inproceedings{li2015minimax,
	title        = {Toward Minimax Off-policy Value Estimation},
	author       = {Li, Lihong and Munos, R{\'e}mi and Szepesv{\'a}ri, Csaba},
	year         = 2015,
	booktitle    = {Proceedings of the 18th International Conference on Artificial Intelligence and Statistics}
}
@inproceedings{kakade2002approximately,
	title        = {{Approximately Optimal Approximate Reinforcement Learning}},
	author       = {Kakade, Sham and Langford, John},
	year         = 2002,
	booktitle    = {Proceedings of the 19th International Conference on Machine Learning},
	volume       = 2,
	pages        = {267--274}
}
@inproceedings{jiang2015abstraction,
	title        = {{Abstraction Selection in Model-based Reinforcement Learning}},
	author       = {Jiang, Nan and Kulesza, Alex and Singh, Satinder},
	year         = 2015,
	booktitle    = {Proceedings of the 32nd International Conference on Machine Learning},
	pages        = {179--188}
}
@inproceedings{thomas2015high,
	title        = {{High Confidence Off-Policy Evaluation}},
	author       = {Thomas, Philip and Theocharous, Georgios and Ghavamzadeh, Mohammad},
	year         = 2015,
	booktitle    = {Proceedings of the 29th AAAI Conference on Artificial Intelligence}
}
@inproceedings{dudik2011doubly,
	title        = {Doubly {R}obust {P}olicy {E}valuation and {L}earning},
	author       = {Dud{\'\i}k, Miroslav and Langford, John and Li, Lihong},
	year         = 2011,
	booktitle    = {Proceedings of the 28th International Conference on Machine Learning},
	pages        = {1097--1104}
}
@inproceedings{thomas2015higha,
	title        = {{High Confidence Policy Improvement}},
	author       = {Thomas, Philip and Theocharous, Georgios and Ghavamzadeh, Mohammad},
	year         = 2015,
	booktitle    = {Proceedings of the 32nd International Conference on Machine Learning},
	pages        = {2380--2388}
}
@inproceedings{kearns2000bias,
	title        = {{Bias-Variance Error Bounds for Temporal Difference Updates}},
	author       = {Kearns, Michael J and Singh, Satinder P},
	year         = 2000,
	booktitle    = {Proceedings of the Thirteenth Annual Conference on Computational Learning Theory},
	pages        = {142--147},
	organization = {Morgan Kaufmann Publishers Inc.}
}
@inproceedings{sutton2014new,
	title        = {A new {Q}($\lambda$) with interim forward view and {M}onte {C}arlo equivalence},
	author       = {Sutton, Rich and Mahmood, Ashique R and Precup, Doina and Hasselt, Hado V},
	year         = 2014,
	booktitle    = {Proceedings of the 31st International Conference on Machine Learning},
	pages        = {568--576}
}
@article{singh1996reinforcement,
	title        = {Reinforcement learning with replacing eligibility traces},
	author       = {Singh, Satinder P and Sutton, Richard S},
	year         = 1996,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 22,
	number       = {1-3},
	pages        = {123--158}
}
@incollection{kocsis2006bandit,
	title        = {Bandit based monte-carlo planning},
	author       = {Kocsis, Levente and Szepesv{\'a}ri, Csaba},
	year         = 2006,
	booktitle    = {Machine Learning: ECML 2006},
	publisher    = {Springer Berlin Heidelberg},
	pages        = {282--293}
}
@misc{hettich1999uci,
	title        = {{The UCI KDD Archive}},
	author       = {Hettich, S and Bay, S D},
	year         = 1999,
	howpublished = {\url{http://kdd.ics.uci.edu}}
}
@inproceedings{precup2001policy,
	title        = {{Off-Policy Temporal-Difference Learning with Funtion Approximation}},
	author       = {Doina Precup and Richard S Sutton and Sanjoy Dasgupta},
	year         = 2001,
	booktitle    = {Proceedings of the 18th International Conference on Machine Learning},
	pages        = {417--424}
}
@inproceedings{precup2000eligibility,
	title        = {{Eligibility Traces for Off-Policy Policy Evaluation}},
	author       = {Doina Precup and Richard S Sutton and Satinder P Singh},
	year         = 2000,
	booktitle    = {Proceedings of the 17th International Conference on Machine Learning},
	pages        = {759--766}
}
@book{sutton1998reinforcement,
	title        = {Reinforcement Learning: An Introduction},
	author       = {Richard S Sutton and Andrew G Barto},
	year         = 1998,
	month        = {March},
	publisher    = {MIT Press},
	address      = {Cambridge, MA},
	isbn         = {0-262-19398-1}
}
@book{sutton2018reinforcement,
	title        = {Reinforcement learning: An introduction},
	author       = {Sutton, Richard S and Barto, Andrew G},
	year         = 2018,
	publisher    = {MIT press}
}
@article{murphy2001marginal,
	title        = {{Marginal Mean Models for Dynamic Regimes}},
	author       = {Susan A. Murphy and Mark van der Laan and James M. Robins},
	year         = 2001,
	journal      = {Journal of the American Statistical Association},
	volume       = 96,
	number       = 456,
	pages        = {1410--1423}
}
@inproceedings{li2011unbiased,
	title        = {{Unbiased Offline Evaluation of Contextual-bandit-based News Article Recommendation Algorithms}},
	author       = {Lihong Li and Wei Chu and John Langford and Xuanhui Wang},
	year         = 2011,
	booktitle    = {Proceedings of the 4th International Conference on Web Search and Data Mining},
	pages        = {297--306}
}
@article{bottou2013counterfactual,
	title        = {{Counterfactual Reasoning and Learning Systems: The Example of Computational Advertising}},
	author       = {L\'{e}on Bottou and Jonas Peters and Joaquin Qui{\~n}onero-Candela and Denis Xavier Charles and D. Max Chickering and Elon Portugaly and Dipankar Ray and Patrice Simard and Ed Snelson},
	year         = 2013,
	journal      = {Journal of Machine Learning Research},
	volume       = 14,
	pages        = {3207--3260}
}
@inproceedings{sutton1998intra,
	title        = {{Intra-option Learning about Temporally Abstract Actions}},
	author       = {Richard S Sutton and Doina Precup and Satinder P Singh},
	year         = 1998,
	booktitle    = {Proceedings of the 15th International Conference on Machine Learning},
	pages        = {556--564}
}
@phdthesis{precup2000temporal,
	title        = {Temporal abstraction in reinforcement learning},
	author       = {Precup, Doina},
	year         = 2000,
	school       = {University of Massachusetts Amherst}
}
@phdthesis{thomas2015safe,
	title        = {Safe Reinforcement Learning},
	author       = {Thomas, Philip},
	year         = 2015,
	school       = {University of Massachusetts Amherst}
}
@inproceedings{gruenewaelder2012modelling,
	title        = {{Modelling transition dynamics in MDPs with RKHS embeddings}},
	author       = {Gr{\"u}new{\"a}lder, S and Lever, G and Baldassarre, L and Pontil, M and Gretton, A},
	year         = 2012,
	booktitle    = {Proceedings of the 29th International Conference on Machine Learning},
	pages        = {535--542}
}
@inproceedings{jong2007model,
	title        = {Model-based function approximation in reinforcement learning},
	author       = {Jong, Nicholas K and Stone, Peter},
	year         = 2007,
	booktitle    = {Proceedings of the 6th International Conference on Autonomous Agents and Multiagent Systems},
	pages        = 95
}
@article{dann2014policy,
	title        = {Policy evaluation with temporal differences: {A} survey and comparison},
	author       = {Dann, Christoph and Neumann, Gerhard and Peters, Jan},
	year         = 2014,
	journal      = {The Journal of Machine Learning Research},
	volume       = 15,
	pages        = {809--883}
}
@article{fonteneau2013batch,
	title        = {Batch Mode Reinforcement Learning based on the Synthesis of Artificial Trajectories},
	author       = {Raphael Fonteneau and Susan A. Murphy and Louis Wehenkel and Damien Ernst},
	year         = 2013,
	journal      = {Annals of Operations Research},
	volume       = 208,
	number       = 1,
	pages        = {383--416}
}
@article{rotnitzky1995semiparametric,
	title        = {Semiparametric Regression Estimation in the Presence of Dependent Censoring},
	author       = {Andrea Rotnitzky and James M. Robins},
	year         = 1995,
	journal      = {Biometrika},
	volume       = 82,
	number       = 4,
	pages        = {805--820}
}
@inproceedings{pirotta2013safe,
	title        = {Safe Policy Iteration},
	author       = {Matteo Pirotta and Marcello Restelli and Alessio Pecorino and Daniele Calandriello},
	year         = 2013,
	booktitle    = {Proceedings of the 30th International Conference on Machine Learning},
	number       = 3,
	pages        = {307--317}
}
@book{pearl2009causality,
	title        = {Causality: Models, Reasoning, and Inference},
	author       = {Judea Pearl},
	year         = 2009,
	publisher    = {Cambridge University Press},
	isbn         = {052189560X},
	edition      = {2nd}
}
@article{holland1986statistics,
	title        = {Statistics and Causal Inference},
	author       = {Paul W. Holland},
	year         = 1986,
	journal      = {Journal of the American Statistical Association},
	volume       = 81,
	number       = 6,
	pages        = {945--960}
}
@unpublished{sutton2015emphatic,
	title        = {An Emphatic Approach to the Problem of Off-policy Temporal-Difference Learning},
	author       = {Richard S Sutton and Ashique Rupam Mahmood and Martha White},
	year         = 2015,
	note         = {CoRR abs/1503.04269}
}
@article{hirano2003efficient,
	title        = {Efficient Estimation of Average Treatment Effects using the Estimated Propensity Score},
	author       = {Hirano, Keisuke and Imbens, Guido W. and Ridder, Geert},
	year         = 2003,
	journal      = {Econometrica},
	volume       = 71,
	number       = 4,
	pages        = {1161--1189}
}
@inproceedings{pednault2002sequential,
	title        = {Sequential Cost-Sensitive Decision-Making with Reinforcement Learning},
	author       = {Edwin P. D. Pednault and Naoki Abe and Bianca Zadrozny},
	year         = 2002,
	booktitle    = {KDD},
	pages        = {259--268},
	booktitle1   = {Proceedings of the 8th International Conference on Knowledge Discovery and Data Mining}
}
@phdthesis{jr2010theory,
	title        = {A theory of Cramer-Rao bounds for constrained parametric models},
	author       = {Moore Jr, Terrence Joseph},
	year         = 2010,
	school       = {University of Maryland, College Park}
}
@phdthesis{marivate2015improved,
	title        = {Improved Empirical Methods in Reinforcement-Learning Evaluation},
	author       = {Vukosi N. Marivate},
	year         = 2015,
	address      = {New Brunswick, NJ},
	school       = {Rutgers University}
}
@inproceedings{marivate2014quantifying,
	title        = {Quantifying uncertainty in batch personalized sequential decision making},
	author       = {Marivate, Vukosi Ntsakisi and Chemali, Jessica and Brunskill, Emma and Littman, Michael},
	year         = 2014,
	booktitle    = {Workshops at the 28th AAAI Conference on Artificial Intelligence}
}
@article{jiang2015doubly,
	title        = {Doubly Robust Off-policy Evaluation for Reinforcement Learning},
	author       = {Jiang, Nan and Li, Lihong},
	year         = 2015,
	journal      = {arXiv preprint arXiv:1511.03722}
}
@inproceedings{singh2004predictive,
	title        = {{Predictive state representations: A new theory for modeling dynamical systems}},
	author       = {Singh, Satinder and James, Michael R and Rudary, Matthew R},
	year         = 2004,
	booktitle    = {Proceedings of the 20th Conference on Uncertainty in Artificial Intelligence},
	pages        = {512--519},
	organization = {AUAI Press}
}
@inproceedings{doshi-velez2009infinite,
	title        = {{The Infinite Partially observable Markov decision process}},
	author       = {Doshi-Velez, Finale},
	year         = 2009,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {477--485}
}
@misc{li2015recurrent,
	title        = {Recurrent Reinforcement Learning: A Hybrid Approach},
	author       = {Xiujun Li and Lihong Li and Jianfeng Gao and Xiaodong He and Jianshu Chen and Li Deng and Ji He},
	year         = 2015,
	note         = {arXiv:1509.03044}
}
@inproceedings{asmuth2011approaching,
	title        = {Approaching {B}ayes-optimalilty using {M}onte-{C}arlo tree search},
	author       = {Asmuth, John and Littman, Michael L},
	year         = 2011,
	booktitle    = {Proceedings of the 21st International Conference on Automatated Planning and Scheduling}
}
@article{bagnell2001solving,
	title        = {Solving uncertain {M}arkov decision processes},
	author       = {Bagnell, J Andrew and Ng, Andrew Y and Schneider, Jeff G},
	year         = 2001,
	publisher    = {Carnegie Mellon University}
}
@article{bertuccelli2012robust,
	title        = {Robust Adaptive Markov Decision Processes: Planning with Model Uncertainty},
	author       = {Bertuccelli, Luca F and Wu, Albert and How, Jonathan P},
	year         = 2012,
	journal      = {Control Systems, IEEE},
	publisher    = {IEEE},
	volume       = 32,
	number       = 5,
	pages        = {96--109}
}
@incollection{el-yaniv2007transductive,
	title        = {{T}ransductive rademacher complexity and its applications},
	author       = {El-Yaniv, Ran and Pechyony, Dmitry},
	year         = 2007,
	booktitle    = {Learning Theory},
	publisher    = {Springer},
	pages        = {157--171}
}
@article{even-dar2009online,
	title        = {Online {M}arkov decision processes},
	author       = {Even-Dar, Eyal and Kakade, Sham M and Mansour, Yishay},
	year         = 2009,
	journal      = {Mathematics of Operations Research},
	publisher    = {INFORMS},
	volume       = 34,
	number       = 3,
	pages        = {726--736}
}
@inproceedings{guez2012efficient,
	title        = {Efficient {B}ayes-{A}daptive {R}einforcement {L}earning using {S}ample-{B}ased {S}earch},
	author       = {Guez, Arthur and Silver, David and Dayan, Peter},
	year         = 2012,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1034--1042}
}
@article{browne2012survey,
	title        = {A survey of {M}onte {C}arlo tree search methods},
	author       = {Browne, Cameron B and Powley, Edward and Whitehouse, Daniel and Lucas, Simon M and Cowling, Peter I and Rohlfshagen, Philipp and Tavener, Stephen and Perez, Diego and Samothrakis, Spyridon and Colton, Simon},
	year         = 2012,
	journal      = {IEEE Transactions on Computational Intelligence and AI in Games},
	volume       = 4,
	number       = 1,
	pages        = {1--43}
}
@book{montenegro2006mathematical,
	title        = {Mathematical aspects of mixing times in {M}arkov chains},
	author       = {Montenegro, Ravi R and Tetali, Prasad},
	year         = 2006,
	publisher    = {Now Publishers Inc}
}
{ng2000pegasus,
  title =	 {{PEGASUS}: {A} policy search method for large {MDP}s and
                  {POMDP}s},
  author =	 {Ng, Andrew Y and Jordan, Michael},
  booktitle =	 {Proceedings of the16th Conference on
                  Uncertainty in Artificial Intelligence},
  pages =	 {406--415},
  year =	 2000
%  organization = {Morgan Kaufmann Publishers Inc.}
}
@article{nilim2005robust,
	title        = {Robust control of {M}arkov decision processes with uncertain transition matrices},
	author       = {Nilim, Arnab and El Ghaoui, Laurent},
	year         = 2005,
	journal      = {Operations Research},
	volume       = 53,
	number       = 5,
	pages        = {780--798}
}
@inproceedings{silver2010montea,
	title        = {{M}onte-{C}arlo planning in large {POMDP}s},
	author       = {Silver, David and Veness, Joel},
	year         = 2010,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2164--2172}
}
@inproceedings{smith2004heuristic,
	title        = {Heuristic search value iteration for {POMDP}s},
	author       = {Smith, Trey and Simmons, Reid},
	year         = 2004,
	booktitle    = {Proceedings of the 20th Conference on Uncertainty in Artificial Intelligence},
	pages        = {520--527}
}
@article{ross2011bayesian,
	title        = {A {B}ayesian approach for learning and planning in partially observable {M}arkov decision processes},
	author       = {Ross, St{\'e}phane and Pineau, Joelle and Chaib-draa, Brahim and Kreitmann, Pierre},
	year         = 2011,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 12,
	pages        = {1729--1770}
}
@inproceedings{bjarnason2009lower,
	title        = {Lower bounding {K}londike solitaire with {M}onte-{C}arlo planning},
	author       = {Bjarnason, Ronald and Fern, Alan and Tadepalli, Prasad},
	year         = 2009,
	booktitle    = {Proceedings of International Conference on Automated Planning and Scheduling},
	pages        = {26--33}
}
@inproceedings{zhu2009human,
	title        = {Human {R}ademacher complexity},
	author       = {Zhu, Xiaojin and Gibson, Bryan R and Rogers, Timothy T},
	year         = 2009,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2322--2330}
}
@manual{balcan2011cs,
	title        = {{CS} 8803 - {M}achine {L}earning {T}heory: {L}ecture {N}otes},
	author       = {Balcan, Maria-Florina},
	year         = 2011,
	note         = {\url{http://www.cs.cmu.edu/~ninamf/ML11/lect1115.pdf}},
	organization = {{G}eorgia {I}nstitute of {T}echnology}
}
@inproceedings{petrik2009biasing,
	title        = {Biasing approximate dynamic programming with a lower discount factor},
	author       = {Petrik, Marek and Scherrer, Bruno},
	year         = 2009,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1265--1272}
}
@article{vapnik1999overview,
	title        = {An overview of statistical learning theory},
	author       = {Vapnik, Vladimir N},
	year         = 1999,
	journal      = {Neural Networks, IEEE Transactions on},
	publisher    = {IEEE},
	volume       = 10,
	number       = 5,
	pages        = {988--999}
}
@article{baxter2001infinite,
	title        = {{Infinite-Horizon Policy-Gradient Estimation}},
	author       = {Baxter, Jonathan and Bartlett, Peter L},
	year         = 2001,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 15,
	pages        = {319--350}
}
@article{amin2017repeated,
	title        = {{Repeated Inverse Reinforcement Learning for AI Safety}},
	author       = {Amin, Kareem and Jiang, Nan and Singh, Satinder},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1705.05427},
	__markedentry = {[najiang:]}
}
@article{bostrom2003ethical,
	title        = {Ethical issues in advanced artificial intelligence},
	author       = {Bostrom, Nick},
	year         = 2003,
	journal      = {Science Fiction and Philosophy: From Time Travel to Superintelligence},
	pages        = {277--284},
	__markedentry = {[najiang:]}
}
@inproceedings{walsh2010generalizing,
	title        = {Generalizing apprenticeship learning across hypothesis classes},
	author       = {Walsh, Thomas J and Subramanian, Kaushik and Littman, Michael L and Diuk, Carlos},
	year         = 2010,
	booktitle    = {Proceedings of the 27th International Conference on Machine Learning (ICML-10)},
	pages        = {1119--1126},
	__markedentry = {[najiang:]}
}
@article{li2011knows,
	title        = {Knows what it knows: a framework for self-aware learning},
	author       = {Li, Lihong and Littman, Michael L and Walsh, Thomas J and Strehl, Alexander L},
	year         = 2011,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 82,
	number       = 3,
	pages        = {399--443},
	__markedentry = {[najiang:]}
}
@inproceedings{dani2008stochastic,
	title        = {Stochastic Linear Optimization under Bandit Feedback.},
	author       = {Dani, Varsha and Hayes, Thomas P and Kakade, Sham M},
	year         = 2008,
	booktitle    = {COLT},
	pages        = {355--366},
	__markedentry = {[najiang:]}
}
@inproceedings{abbasi-yadkori2012online,
	title        = {Online-to-Confidence-Set Conversions and Application to Sparse Stochastic Bandits.},
	author       = {Abbasi-Yadkori, Yasin and Pal, David and Szepesvari, Csaba},
	year         = 2012,
	booktitle    = {AISTATS},
	volume       = 22,
	pages        = {1--9},
	__markedentry = {[najiang:]}
}
@article{mannor2004sample,
	title        = {The sample complexity of exploration in the multi-armed bandit problem},
	author       = {Mannor, Shie and Tsitsiklis, John N},
	year         = 2004,
	journal      = {Journal of Machine Learning Research},
	volume       = 5,
	number       = {Jun},
	pages        = {623--648},
	__markedentry = {[najiang:]}
}
@article{auer2002using,
	title        = {Using confidence bounds for exploitation-exploration trade-offs},
	author       = {Auer, Peter},
	year         = 2002,
	journal      = {Journal of Machine Learning Research},
	volume       = 3,
	number       = {Nov},
	pages        = {397--422},
	__markedentry = {[najiang:]}
}
@inproceedings{neu2014online,
	title        = {Online combinatorial optimization with stochastic decision sets and adversarial losses},
	author       = {Neu, Gergely and Valko, Michal},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2780--2788},
	__markedentry = {[najiang:]}
}
@article{kleinberg2010regret,
	title        = {Regret bounds for sleeping experts and bandits},
	author       = {Kleinberg, Robert and Niculescu-Mizil, Alexandru and Sharma, Yogeshwer},
	year         = 2010,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 80,
	number       = {2-3},
	pages        = {245--272},
	__markedentry = {[najiang:]}
}
@article{blum2007external,
	title        = {From external to internal regret},
	author       = {Blum, Avrim and Mansour, Yishay},
	year         = 2007,
	journal      = {Journal of Machine Learning Research},
	volume       = 8,
	number       = {Jun},
	pages        = {1307--1324},
	__markedentry = {[najiang:]}
}
@inproceedings{freund1997using,
	title        = {Using and combining predictors that specialize},
	author       = {Freund, Yoav and Schapire, Robert E and Singer, Yoram and Warmuth, Manfred K},
	year         = 1997,
	booktitle    = {Proceedings of the 29th Annual ACM Symposium on Theory of computing},
	pages        = {334--343},
	organization = {ACM},
	__markedentry = {[najiang:]}
}
@inproceedings{kanade2009sleeping,
	title        = {Sleeping Experts and Bandits with Stochastic Action Availability and Adversarial Rewards},
	author       = {Kanade, Varun and McMahan, H Brendan and Bryan, Brent},
	year         = 2009,
	booktitle    = {AISTATS},
	pages        = {272--279},
	__markedentry = {[najiang:]}
}
@book{groetschel2012geometric,
	title        = {Geometric algorithms and combinatorial optimization},
	author       = {Gr{\"o}tschel, Martin and Lov{\'a}sz, L{\'a}szl{\'o} and Schrijver, Alexander},
	year         = 2012,
	publisher    = {Springer Science \& Business Media},
	volume       = 2,
	__markedentry = {[najiang:]}
}
@inproceedings{hadfield-menell2016cooperative,
	title        = {Cooperative inverse reinforcement learning},
	author       = {Hadfield-Menell, Dylan and Russell, Stuart J and Abbeel, Pieter and Dragan, Anca},
	year         = 2016,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {3909--3917},
	__markedentry = {[najiang:]}
}
@article{russell2015research,
	title        = {Research priorities for robust and beneficial artificial intelligence},
	author       = {Russell, Stuart and Dewey, Daniel and Tegmark, Max},
	year         = 2015,
	journal      = {AI Magazine},
	volume       = 36,
	number       = 4,
	pages        = {105--114},
	__markedentry = {[najiang:]}
}
@article{amodei2016concrete,
	title        = {Concrete problems in {AI} safety},
	author       = {Amodei, Dario and Olah, Chris and Steinhardt, Jacob and Christiano, Paul and Schulman, John and Man{\'e}, Dan},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1606.06565},
	__markedentry = {[najiang:]}
}
@article{todd1982minimum,
	title        = {On minimum volume ellipsoids containing part of a given ellipsoid},
	author       = {Todd, Michael J},
	year         = 1982,
	journal      = {Mathematics of Operations Research},
	publisher    = {INFORMS},
	volume       = 7,
	number       = 2,
	pages        = {253--261},
	__markedentry = {[najiang:]}
}
@inproceedings{auer1995gambling,
	title        = {Gambling in a rigged casino: The adversarial multi-armed bandit problem},
	author       = {Auer, Peter and Cesa-Bianchi, Nicolo and Freund, Yoav and Schapire, Robert E},
	year         = 1995,
	booktitle    = {Foundations of Computer Science, 1995. Proceedings., 36th Annual Symposium on},
	pages        = {322--331},
	organization = {IEEE},
	__markedentry = {[najiang:]}
}
@article{bartlett2008high,
	title        = {High-probability regret bounds for bandit online linear optimization},
	author       = {Bartlett, Peter L and Dani, Varsha and Hayes, Thomas and Kakade, Sham and Rakhlin, Alexander and Tewari, Ambuj},
	year         = 2008,
	booktitle    = {Proceedings of the 21st Annual Conference on Learning Theory},
	pages        = {335--342},
	__markedentry = {[najiang:]}
}
@inproceedings{abernethy2008competing,
	title        = {Competing in the Dark: An Efficient Algorithm for Bandit Linear Optimization},
	author       = {Abernethy, Jacob and Hazan, Elad and Rakhlin, Alexander},
	year         = 2008,
	booktitle    = {Conference on Learning Theory},
	pages        = {263--274},
	__markedentry = {[najiang:]}
}
@inproceedings{dani2007price,
	title        = {The price of bandit information for online optimization},
	author       = {Dani, Varsha and Kakade, Sham M and Hayes, Thomas P},
	year         = 2007,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {345--352},
	__markedentry = {[najiang:]}
}
@article{shalev-shwartz2011online,
	title        = {Online learning and online convex optimization},
	author       = {Shalev-Shwartz, Shai},
	year         = 2011,
	journal      = {Foundations and Trends in Machine Learning},
	volume       = 4,
	number       = 2,
	pages        = {107--194},
	__markedentry = {[najiang:]}
}
@inproceedings{hsu2015mixing,
	title        = {Mixing time estimation in reversible Markov chains from a single sample path},
	author       = {Hsu, Daniel J and Kontorovich, Aryeh and Szepesv{\'a}ri, Csaba},
	year         = 2015,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1459--1467},
	__markedentry = {[najiang:]}
}
@manual{odonnell201115,
	title        = {15-859({E}) -- linear and semidefinite programming: lecture notes},
	author       = {Ryan O'Donnell},
	year         = 2011,
	note         = {\url{https://www.cs.cmu.edu/afs/cs.cmu.edu/academic/class/15859-f11/www/notes/lecture08.pdf}},
	organization = {{Carnegie Mellon University}},
	__markedentry = {[najiang:]}
}
@inproceedings{jiang2015dependence,
	title        = {{The Dependence of Effective Planning Horizon on Model Accuracy}},
	author       = {Jiang, Nan and Kulesza, Alex and Singh, Satinder and Lewis, Richard},
	year         = 2015,
	booktitle    = {Proceedings of the 2015 International Conference on Autonomous Agents and Multiagent Systems},
	pages        = {1181--1189},
	__markedentry = {[najiang:]}
}
@inproceedings{ng2000algorithms,
	title        = {Algorithms for Inverse Reinforcement Learning},
	author       = {Ng, Andrew Y and Russell, Stuart J},
	year         = 2000,
	booktitle    = {Proceedings of the 17th International Conference on Machine Learning},
	pages        = {663--670},
	__markedentry = {[najiang:]}
}
@inproceedings{abbeel2004apprenticeship,
	title        = {{Apprenticeship Learning via Inverse Reinforcement Learning}},
	author       = {Abbeel, Pieter and Ng, Andrew Y},
	year         = 2004,
	booktitle    = {Proceedings of the 21st International Conference on Machine learning},
	pages        = 1,
	organization = {ACM},
	__markedentry = {[najiang:]}
}
@article{amin2016towards,
	title        = {Towards Resolving Unidentifiability in Inverse Reinforcement Learning},
	author       = {Amin, Kareem and Singh, Satinder},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1601.06569},
	__markedentry = {[najiang:]}
}
@article{lovasz1999hit,
	title        = {Hit-and-run mixes fast},
	author       = {Lov{\'a}sz, L{\'a}szl{\'o}},
	year         = 1999,
	journal      = {Mathematical Programming},
	publisher    = {Springer},
	volume       = 86,
	number       = 3,
	pages        = {443--461},
	__markedentry = {[najiang:]}
}
@article{lovasz2006simulated,
	title        = {Simulated annealing in convex bodies and an O*(n4) volume algorithm},
	author       = {Lov{\'a}sz, L{\'a}szl{\'o} and Vempala, Santosh},
	year         = 2006,
	journal      = {Journal of Computer and System Sciences},
	publisher    = {Elsevier},
	volume       = 72,
	number       = 2,
	pages        = {392--417},
	__markedentry = {[najiang:]}
}
@article{cousinspractical,
	title        = {A Practical Volume Algorithm},
	author       = {Cousins, Ben and Vempala, Santosh},
	__markedentry = {[najiang:]}
}
@inproceedings{coates2008learning,
	title        = {Learning for control from multiple demonstrations},
	author       = {Coates, Adam and Abbeel, Pieter and Ng, Andrew Y},
	year         = 2008,
	booktitle    = {Proceedings of the 25th international conference on Machine learning},
	pages        = {144--151},
	organization = {ACM},
	__markedentry = {[najiang:]}
}
@inproceedings{ziebart2008maximum,
	title        = {Maximum Entropy Inverse Reinforcement Learning},
	author       = {Ziebart, Brian D and Maas, Andrew L and Bagnell, J Andrew and Dey, Anind K},
	year         = 2008,
	booktitle    = {AAAI},
	pages        = {1433--1438},
	__markedentry = {[najiang:]}
}
@article{ramachandran2007bayesian,
	title        = {Bayesian inverse reinforcement learning},
	author       = {Ramachandran, Deepak and Amir, Eyal},
	year         = 2007,
	journal      = {Urbana},
	volume       = 51,
	pages        = 61801,
	__markedentry = {[najiang:]}
}
@inproceedings{syed2007game,
	title        = {A game-theoretic approach to apprenticeship learning},
	author       = {Syed, Umar and Schapire, Robert E},
	year         = 2007,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1449--1456},
	__markedentry = {[najiang:]}
}
@inproceedings{guillory2010interactive,
	title        = {Interactive submodular set cover},
	author       = {Guillory, Andrew and Bilmes, Jeff},
	year         = 2010,
	booktitle    = {Proceedings of the International Conference on Machine Learning},
	__markedentry = {[najiang:]}
}
@inproceedings{golovin2010adaptive,
	title        = {Adaptive Submodularity: A New Approach to Active Learning and Stochastic Optimization},
	author       = {Golovin, Daniel and Krause, Andreas},
	year         = 2010,
	booktitle    = {COLT},
	pages        = {333--345},
	__markedentry = {[najiang:]}
}
@inproceedings{ratliff2006maximum,
	title        = {Maximum margin planning},
	author       = {Ratliff, Nathan D and Bagnell, J Andrew and Zinkevich, Martin A},
	year         = 2006,
	booktitle    = {Proceedings of the 23rd International Conference on Machine Learning},
	pages        = {729--736},
	organization = {ACM},
	__markedentry = {[najiang:]}
}
@inproceedings{regan2010robust,
	title        = {Robust Policy Computation in Reward-Uncertain MDPs Using Nondominated Policies},
	author       = {Regan, Kevin and Boutilier, Craig},
	year         = 2010,
	booktitle    = {AAAI},
	__markedentry = {[najiang:]}
}
@inproceedings{smart2002effective,
	title        = {Effective reinforcement learning for mobile robots},
	author       = {Smart, William D and Kaelbling, Leslie Pack},
	year         = 2002,
	booktitle    = {Robotics and Automation, 2002. Proceedings. ICRA'02. IEEE International Conference on},
	volume       = 4,
	pages        = {3404--3410},
	organization = {IEEE},
	__markedentry = {[najiang:]}
}
@article{abbeel2007application,
	title        = {An application of reinforcement learning to aerobatic helicopter flight},
	author       = {Abbeel, Pieter and Coates, Adam and Quigley, Morgan and Ng, Andrew Y},
	year         = 2007,
	journal      = {Advances in Neural Information Processing Systems},
	publisher    = {MIT; 1998},
	volume       = 19,
	pages        = 1,
	__markedentry = {[najiang:]}
}
@article{coates2009apprenticeship,
	title        = {Apprenticeship learning for helicopter control},
	author       = {Coates, Adam and Abbeel, Pieter and Ng, Andrew Y},
	year         = 2009,
	journal      = {Communications of the ACM},
	publisher    = {ACM},
	volume       = 52,
	number       = 7,
	pages        = {97--105},
	__markedentry = {[najiang:]}
}
@book{vonneumann2007theory,
	title        = {Theory of games and economic behavior (60th Anniversary Commemorative Edition)},
	author       = {Von Neumann, John and Morgenstern, Oskar},
	year         = 2007,
	publisher    = {Princeton university press},
	__markedentry = {[najiang:]}
}
@inproceedings{chajewska2000making,
	title        = {Making rational decisions using adaptive utility elicitation},
	author       = {Chajewska, Urszula and Koller, Daphne and Parr, Ronald},
	year         = 2000,
	booktitle    = {AAAI/IAAI},
	pages        = {363--369},
	__markedentry = {[najiang:]}
}
@incollection{balcan2007margin,
	title        = {Margin based active learning},
	author       = {Balcan, Maria-Florina and Broder, Andrei and Zhang, Tong},
	year         = 2007,
	booktitle    = {Learning Theory},
	publisher    = {Springer},
	pages        = {35--50},
	__markedentry = {[najiang:]}
}
@article{balcan2012active,
	title        = {Active and passive learning of linear separators under log-concave distributions},
	author       = {Balcan, Maria Florina and Long, Philip M},
	year         = 2012,
	journal      = {arXiv preprint arXiv:1211.1082},
	__markedentry = {[najiang:]}
}
@inproceedings{dasgupta2005coarse,
	title        = {Coarse sample complexity bounds for active learning},
	author       = {Dasgupta, Sanjoy},
	year         = 2005,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {235--242},
	__markedentry = {[najiang:]}
}
@inproceedings{regan2009regret,
	title        = {Regret-based reward elicitation for Markov decision processes},
	author       = {Regan, Kevin and Boutilier, Craig},
	year         = 2009,
	booktitle    = {Proceedings of the Twenty-Fifth Conference on Uncertainty in Artificial Intelligence},
	pages        = {444--451},
	organization = {AUAI Press},
	__markedentry = {[najiang:]}
}
@incollection{lopes2009active,
	title        = {Active learning for reward estimation in inverse reinforcement learning},
	author       = {Lopes, Manuel and Melo, Francisco and Montesano, Luis},
	year         = 2009,
	booktitle    = {Machine Learning and Knowledge Discovery in Databases},
	publisher    = {Springer},
	pages        = {31--46},
	__markedentry = {[najiang:]}
}
@article{baranes2013active,
	title        = {Active learning of inverse models with intrinsically motivated goal exploration in robots},
	author       = {Baranes, Adrien and Oudeyer, Pierre-Yves},
	year         = 2013,
	journal      = {Robotics and Autonomous Systems},
	publisher    = {Elsevier},
	volume       = 61,
	number       = 1,
	pages        = {49--73},
	__markedentry = {[najiang:]}
}
@incollection{rothkopf2011preference,
	title        = {Preference elicitation and inverse reinforcement learning},
	author       = {Rothkopf, Constantin A and Dimitrakakis, Christos},
	year         = 2011,
	booktitle    = {Machine Learning and Knowledge Discovery in Databases},
	publisher    = {Springer},
	pages        = {34--48},
	__markedentry = {[najiang:]}
}
@inproceedings{langford2005relating,
	title        = {Relating reinforcement learning performance to classification performance},
	author       = {Langford, John and Zadrozny, Bianca},
	year         = 2005,
	booktitle    = {Proceedings of the 22nd international conference on Machine learning},
	pages        = {473--480},
	organization = {ACM},
	__markedentry = {[najiang:]}
}
@inproceedings{lagoudakis2003reinforcement,
	title        = {Reinforcement learning as classification: Leveraging modern classifiers},
	author       = {Lagoudakis, Michail and Parr, Ronald},
	year         = 2003,
	booktitle    = {ICML},
	volume       = 3,
	pages        = {424--431},
	__markedentry = {[najiang:]}
}
@book{boyd2004convex,
	title        = {Convex optimization},
	author       = {Boyd, Stephen and Vandenberghe, Lieven},
	year         = 2004,
	publisher    = {Cambridge university press},
	__markedentry = {[najiang:]}
}
@article{elghaoui2002inversion,
	title        = {Inversion error, condition number, and approximate inverses of uncertain matrices},
	author       = {El Ghaoui, Laurent},
	year         = 2002,
	journal      = {Linear algebra and its applications},
	publisher    = {Elsevier},
	volume       = 343,
	pages        = {171--193},
	__markedentry = {[najiang:]}
}
@book{anthony2009neural,
	title        = {Neural network learning: Theoretical foundations},
	author       = {Anthony, Martin and Bartlett, Peter L},
	year         = 2009,
	publisher    = {cambridge university press},
	__markedentry = {[najiang:]}
}
@article{massart2006risk,
	title        = {Risk bounds for statistical learning},
	author       = {Massart, Pascal and N{\'e}d{\'e}lec, {\'E}lodie},
	year         = 2006,
	journal      = {The Annals of Statistics},
	publisher    = {JSTOR},
	pages        = {2326--2366},
	__markedentry = {[najiang:]}
}
@inproceedings{regan2011eliciting,
	title        = {Eliciting additive reward functions for Markov decision processes},
	author       = {Regan, Kevin and Boutilier, Craig},
	year         = 2011,
	booktitle    = {IJCAI Proceedings-International Joint Conference on Artificial Intelligence},
	volume       = 22,
	number       = 3,
	pages        = 2159,
	__markedentry = {[najiang:]}
}
@article{kocsis2006improved,
	title        = {Improved {M}onte-{C}arlo search},
	author       = {Kocsis, Levente and Szepesv{\'a}ri, Csaba and Willemson, Jan},
	year         = 2006,
	journal      = {Univ. Tartu, Estonia, Tech. Rep},
	volume       = 1,
	__markedentry = {[najiang:6]}
}
@inproceedings{wang2007modifications,
	title        = {Modifications of {UCT} and sequence-like simulations for {M}onte-{C}arlo Go},
	author       = {Wang, Yizao and Gelly, Sylvain},
	year         = 2007,
	booktitle    = {IEEE Symposium on Computational Intelligence and Games},
	pages        = {175--182},
	__markedentry = {[najiang:6]}
}
@article{silver2010monte,
	title        = {{M}onte-{C}arlo planning in large {POMDP}s},
	author       = {Silver, David and Veness, Joel},
	year         = 2010,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 23,
	pages        = {2164--2172},
	__markedentry = {[najiang:6]}
}
@inproceedings{coueetoux2011adding,
	title        = {Adding Double Progressive Widening to Upper Confidence Trees to Cope with Uncertainty in Planning Problems},
	author       = {Cou{\"e}toux, Adrien and Doghmen, Hassen and others},
	year         = 2011,
	booktitle    = {9th European Workshop on Reinforcement Learning},
	__markedentry = {[najiang:6]}
}
@incollection{coueetoux2011continuous,
	title        = {Continuous upper confidence trees},
	author       = {Cou{\"e}toux, Adrien and Hoock, Jean-Baptiste and Sokolovska, Nataliya and Teytaud, Olivier and Bonnard, Nicolas},
	year         = 2011,
	booktitle    = {Learning and Intelligent Optimization},
	publisher    = {Springer},
	pages        = {433--445},
	__markedentry = {[najiang:6]}
}
@article{tolpin2012mcts,
	title        = {{MCTS} based on simple regret},
	author       = {Tolpin, David and Shimony, Solomon Eyal},
	year         = 2012,
	journal      = {AAAI. AAAI Press, 2012a. To appear},
	__markedentry = {[najiang:6]}
}
@inproceedings{hingston2007experiments,
	title        = {Experiments with {M}onte-{C}arlo {O}thello},
	author       = {Hingston, Philip and Masek, Martin},
	year         = 2007,
	booktitle    = {Evolutionary Computation},
	pages        = {4059--4064},
	organization = {IEEE},
	__markedentry = {[najiang:6]}
}
@article{rasmussen2004gaussian,
	title        = {Gaussian processes in reinforcement learning},
	author       = {Rasmussen, Carl Edward and Kuss, Malte},
	year         = 2004,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 16,
	__markedentry = {[najiang:6]}
}
@article{gelly2011monte,
	title        = {{M}onte-{C}arlo tree search and rapid action value estimation in computer Go},
	author       = {Gelly, Sylvain and Silver, David},
	year         = 2011,
	journal      = {Artificial Intelligence},
	publisher    = {Elsevier},
	volume       = 175,
	number       = 11,
	pages        = {1856--1875},
	__markedentry = {[najiang:6]}
}
@inproceedings{narayanamurthy2007efficiently,
	title        = {Efficiently Exploiting Symmetries in Real Time Dynamic Programming},
	author       = {Narayanamurthy, Shravan Matthur and Ravindran, Balaraman},
	year         = 2007,
	booktitle    = {International Joint Conference on Artificial Intelligence},
	pages        = {2556--2561},
	__markedentry = {[najiang:6]}
}
@article{auer2002finite,
	title        = {Finite-time analysis of the multiarmed bandit problem},
	author       = {Auer, Peter and Cesa-Bianchi, Nicol{\`o} and Fischer, Paul},
	year         = 2002,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 47,
	number       = {2-3},
	pages        = {235--256}
}
@article{kearns2002sparse,
	title        = {A sparse sampling algorithm for near-optimal planning in large {M}arkov decision processes},
	author       = {Kearns, Michael and Mansour, Yishay and Ng, Andrew Y},
	year         = 2002,
	journal      = {Machine Learning},
	publisher    = {Springer},
	volume       = 49,
	number       = {2-3},
	pages        = {193--208}
}
@inproceedings{konda2000actor,
	title        = {Actor-critic algorithms},
	author       = {Konda, Vijay R and Tsitsiklis, John N},
	year         = 2000,
	booktitle    = {Advances in neural information processing systems},
	pages        = {1008--1014}
}
@article{lazaric2012finite,
	title        = {Finite-sample analysis of least-squares policy iteration},
	author       = {Lazaric, Alessandro and Ghavamzadeh, Mohammad and Munos, R{\'e}mi},
	year         = 2012,
	journal      = {Journal of Machine Learning Research},
	volume       = 13,
	number       = {Oct},
	pages        = {3041--3074}
}
@article{pires2012statistical,
	title        = {Statistical linear estimation with penalized estimators: an application to reinforcement learning},
	author       = {Pires, Bernardo Avila and Szepesv{\'a}ri, Csaba},
	year         = 2012,
	journal      = {arXiv preprint arXiv:1206.6444}
}
@article{farahmand2016regularized,
	title        = {Regularized policy iteration with nonparametric function spaces},
	author       = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
	year         = 2016,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR. org},
	volume       = 17,
	number       = 1,
	pages        = {4809--4874}
}
@article{barreto2014policy,
	title        = {Policy iteration based on stochastic factorization},
	author       = {Barreto, Andr{\'e} da Motta Salles and Pineau, Joelle and Precup, Doina},
	year         = 2014,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 50,
	pages        = {763--803}
}
@article{wu2019behavior,
	title        = {Behavior regularized offline reinforcement learning},
	author       = {Wu, Yifan and Tucker, George and Nachum, Ofir},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1911.11361}
}
@article{agarwal2020flambe,
	title        = {FLAMBE: Structural Complexity and Representation Learning of Low Rank MDPs},
	author       = {Agarwal, Alekh and Kakade, Sham and Krishnamurthy, Akshay and Sun, Wen},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2006.10814}
}
@inproceedings{jacot2018neural,
	title        = {Neural tangent kernel: Convergence and generalization in neural networks},
	author       = {Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
	year         = 2018,
	booktitle    = {Advances in neural information processing systems},
	pages        = {8571--8580}
}
@inproceedings{littman1996generalized,
	title        = {A generalized reinforcement-learning model: Convergence and applications},
	author       = {Littman, Michael L and Szepesv{\'a}ri, Csaba},
	year         = 1996,
	booktitle    = {ICML},
	volume       = 96,
	pages        = {310--318}
}
@article{jiang2019value,
	title        = {On value functions and the agent-environment boundary},
	author       = {Jiang, Nan},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1905.13341}
}
@book{devroye2012combinatorial,
	title        = {Combinatorial methods in density estimation},
	author       = {Devroye, Luc and Lugosi, G{\'a}bor},
	year         = 2012,
	publisher    = {Springer Science \& Business Media}
}
@article{paine2020hyperparameter,
	title        = {Hyperparameter Selection for Offline Reinforcement Learning},
	author       = {Paine, Tom Le and Paduraru, Cosmin and Michi, Andrea and Gulcehre, Caglar and Zolna, Konrad and Novikov, Alexander and Wang, Ziyu and de Freitas, Nando},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2007.09055}
}
@article{liu2020provably,
	title        = {Provably Good Batch Reinforcement Learning Without Great Exploration},
	author       = {Liu, Yao and Swaminathan, Adith and Agarwal, Alekh and Brunskill, Emma},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2007.08202}
}
@article{jiang2020minimax,
	title        = {Minimax Value Interval for Off-Policy Evaluation and Policy Optimization},
	author       = {Jiang, Nan and Huang, Jiawei},
	year         = 2020,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 33
}
@inproceedings{feng2020accountable,
	title        = {Accountable Off-Policy Evaluation With Kernel Bellman Statistics},
	author       = {Feng, Yihao and Ren, Tongzheng and Tang, Ziyang and Liu, Qiang},
	year         = 2020,
	booktitle    = {Proceedings of the 37th International Conference on Machine Learning (ICML-20)}
}
@manual{nan_abstraction_notes,
	title        = {{CS 598: Notes on State Abstractions}},
	author       = {Nan Jiang},
	year         = 2018,
	note         = {\url{http://nanjiang.cs.illinois.edu/files/cs598/note4.pdf}},
	organization = {{University of Illinois at Urbana-Champaign}}
}
@manual{agarwal2011covering,
	title        = {{E0 370 Statistical Learning Theory: Covering Numbers, Pseudo-Dimension, and Fat-Shattering Dimension}},
	author       = {Shivani Agarwal},
	year         = 2011,
	note         = {\url{https://www.shivani-agarwal.net/Teaching/E0370/Aug-2011/Lectures/5.pdf}},
	organization = {{Indian Institute of Science}}
}
@inproceedings{xie2020q,
	title        = {Q* approximation schemes for batch reinforcement learning: A theoretical comparison},
	author       = {Xie, Tengyang and Jiang, Nan},
	year         = 2020,
	booktitle    = {Conference on Uncertainty in Artificial Intelligence},
	pages        = {550--559},
	organization = {PMLR}
}
@article{voloshin2019empirical,
	title        = {Empirical Study of Off-Policy Policy Evaluation for Reinforcement Learning},
	author       = {Voloshin, Cameron and Le, Hoang M and Jiang, Nan and Yue, Yisong},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1911.06854}
}
@article{liao2019off,
	title        = {Off-Policy Estimation of Long-Term Average Outcomes with Applications to Mobile Health},
	author       = {Liao, Peng and Klasnja, Predrag and Murphy, Susan},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1912.13088}
}
@inproceedings{feng2019kernel,
	title        = {A kernel loss for solving the bellman equation},
	author       = {Feng, Yihao and Li, Lihong and Liu, Qiang},
	year         = 2019,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {15430--15441}
}
@article{nilim2005robust,
	title        = {Robust control of Markov decision processes with uncertain transition matrices},
	author       = {Nilim, Arnab and El Ghaoui, Laurent},
	year         = 2005,
	journal      = {Operations Research},
	publisher    = {INFORMS},
	volume       = 53,
	number       = 5,
	pages        = {780--798}
}
@manual{nan_rmax_notes,
	title        = {{CS 598: Notes on Rmax exploration}},
	author       = {Nan Jiang},
	year         = 2018,
	note         = {\url{http://nanjiang.cs.illinois.edu/files/cs598/note7.pdf}},
	organization = {{University of Illinois at Urbana-Champaign}}
}
@inproceedings{zhang2019gendice,
	title        = {GenDICE: Generalized Offline Estimation of Stationary Values},
	author       = {Zhang, Ruiyi and Dai, Bo and Li, Lihong and Schuurmans, Dale},
	year         = 2019,
	booktitle    = {International Conference on Learning Representations}
}
@article{neumann1928theorie,
	title        = {Zur theorie der gesellschaftsspiele},
	author       = {Neumann, J v},
	year         = 1928,
	journal      = {Mathematische annalen},
	publisher    = {Springer},
	volume       = 100,
	number       = 1,
	pages        = {295--320}
}
@article{sion1958general,
	title        = {On general minimax theorems.},
	author       = {Sion, Maurice and others},
	year         = 1958,
	journal      = {Pacific Journal of mathematics},
	publisher    = {Pacific Journal of Mathematics},
	volume       = 8,
	number       = 1,
	pages        = {171--176}
}
@article{zhang2020gradientdice,
	title        = {GradientDICE: Rethinking Generalized Offline Estimation of Stationary Values},
	author       = {Zhang, Shangtong and Liu, Bo and Whiteson, Shimon},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1910.12809}
}
@article{rowland2019conditional,
	title        = {Conditional Importance Sampling for Off-Policy Learning},
	author       = {Rowland, Mark and Harutyunyan, Anna and van Hasselt, Hado and Borsa, Diana and Schaul, Tom and Munos, R{\'e}mi and Dabney, Will},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1910.07479}
}
@article{liu2019understanding,
	title        = {Understanding the Curse of Horizon in Off-Policy Evaluation via Conditional Importance Sampling},
	author       = {Liu, Yao and Bacon, Pierre-Luc and Brunskill, Emma},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1910.06508}
}
@article{liu2019off,
	title        = {Off-policy policy gradient with state distribution correction},
	author       = {Liu, Yao and Swaminathan, Adith and Agarwal, Alekh and Brunskill, Emma},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1904.08473}
}
@inproceedings{fujimoto2019off,
	title        = {Off-Policy Deep Reinforcement Learning without Exploration},
	author       = {Fujimoto, Scott and Meger, David and Precup, Doina},
	year         = 2019,
	booktitle    = {International Conference on Machine Learning},
}
@article{nachum2020reinforcement,
	title        = {Reinforcement Learning via Fenchel-Rockafellar Duality},
	author       = {Nachum, Ofir and Dai, Bo},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2001.01866}
}
@article{wolpert1996lack,
	title        = {The lack of a priori distinctions between learning algorithms},
	author       = {Wolpert, David H},
	year         = 1996,
	journal      = {Neural computation},
	publisher    = {MIT Press},
	volume       = 8,
	number       = 7,
	pages        = {1341--1390}
}
@incollection{baird1995residual,
	title        = {Residual algorithms: Reinforcement learning with function approximation},
	author       = {Baird, Leemon},
	year         = 1995,
	booktitle    = {Machine Learning Proceedings 1995},
	publisher    = {Elsevier},
	pages        = {30--37}
}
@inproceedings{chen2019information,
	title        = {Information-Theoretic Considerations in Batch Reinforcement Learning},
	author       = {Chen, Jinglin and Jiang, Nan},
	year         = 2019,
	booktitle    = {Proceedings of the 36th International Conference on Machine Learning},
	pages        = {1042--1051}
}
@inproceedings{fu2019diagnosing,
	title        = {Diagnosing Bottlenecks in Deep Q-learning Algorithms},
	author       = {Fu, Justin and Kumar, Aviral and Soh, Matthew and Levine, Sergey},
	year         = 2019,
	booktitle    = {Proceedings of the 36th International Conference on Machine Learning},
	pages        = {2021--2030}
}
@inproceedings{agarwal2012contextual,
	title        = {Contextual bandit learning with predictable rewards},
	author       = {Agarwal, Alekh and Dud{\'\i}k, Miroslav and Kale, Satyen and Langford, John and Schapire, Robert},
	year         = 2012,
	booktitle    = {Artificial Intelligence and Statistics},
	pages        = {19--26}
}
@inproceedings{agarwal2014taming,
	title        = {Taming the monster: A fast and simple algorithm for contextual bandits},
	author       = {Agarwal, Alekh and Hsu, Daniel and Kale, Satyen and Langford, John and Li, Lihong and Schapire, Robert},
	year         = 2014,
	booktitle    = {International Conference on Machine Learning},
	pages        = {1638--1646}
}
@article{haussler1992decision,
	title        = {Decision theoretic generalizations of the {PAC} model for neural net and other learning applications},
	author       = {Haussler, David},
	year         = 1992,
	journal      = {Information and computation},
	publisher    = {Elsevier}
}
@inproceedings{ng2000pegasus,
	title        = {{PEGASUS: A policy search method for large MDPs and POMDPs}},
	author       = {Ng, Andrew Y and Jordan, Michael},
	year         = 2000,
	booktitle    = {Proceedings of the Sixteenth conference on Uncertainty in artificial intelligence},
	pages        = {406--415},
	organization = {Morgan Kaufmann Publishers Inc.}
}
@inproceedings{szepesvari2005finite,
	title        = {Finite time bounds for sampling based fitted value iteration},
	author       = {Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
	year         = 2005,
	booktitle    = {Proceedings of the 22nd international conference on Machine learning},
	pages        = {880--887},
	organization = {ACM}
}
@inproceedings{sutton2000policy,
	title        = {Policy gradient methods for reinforcement learning with function approximation},
	author       = {Sutton, Richard S and McAllester, David A and Singh, Satinder P and Mansour, Yishay},
	year         = 2000,
	booktitle    = {Advances in neural information processing systems},
	pages        = {1057--1063}
}
@article{muller1997integral,
	title        = {Integral probability metrics and their generating classes of functions},
	author       = {M{\"u}ller, Alfred},
	year         = 1997,
	journal      = {Advances in Applied Probability},
	publisher    = {Cambridge University Press},
	volume       = 29,
	number       = 2,
	pages        = {429--443}
}
@inproceedings{sutton2015introduction,
	title        = {Introduction to reinforcement learning with function approximation},
	author       = {Sutton, Rich},
	year         = 2015,
	booktitle    = {Tutorial at the Conference on Neural Information Processing Systems}
}
@inproceedings{lu2018non,
	title        = {Non-delusional Q-learning and value-iteration},
	author       = {Lu, Tyler and Schuurmans, Dale and Boutilier, Craig},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {9971--9981}
}
@article{van2018deep,
	title        = {Deep Reinforcement Learning and the Deadly Triad},
	author       = {Van Hasselt, Hado and Doron, Yotam and Strub, Florian and Hessel, Matteo and Sonnerat, Nicolas and Modayil, Joseph},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1812.02648}
}
@inproceedings{szepesvari2004interpolation,
	title        = {Interpolation-based Q-learning},
	author       = {Szepesv{\'a}ri, Csaba and Smart, William D},
	year         = 2004,
	booktitle    = {Proceedings of the twenty-first international conference on Machine learning},
	pages        = 100,
	organization = {ACM}
}
@inproceedings{lizotte2011convergent,
	title        = {Convergent fitted value iteration with linear function approximation},
	author       = {Lizotte, Daniel J},
	year         = 2011,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2537--2545}
}
@inproceedings{hutter2014extreme,
	title        = {Extreme state aggregation beyond MDPs},
	author       = {Hutter, Marcus},
	year         = 2014,
	booktitle    = {International Conference on Algorithmic Learning Theory},
	pages        = {185--199},
	organization = {Springer}
}
@article{shimodaira2000improving,
	title        = {Improving predictive inference under covariate shift by weighting the log-likelihood function},
	author       = {Shimodaira, Hidetoshi},
	year         = 2000,
	journal      = {Journal of statistical planning and inference},
	publisher    = {Elsevier},
	volume       = 90,
	number       = 2,
	pages        = {227--244}
}
@inproceedings{tosatto2017boosted,
	title        = {Boosted fitted q-iteration},
	author       = {Tosatto, Samuele and Pirotta, Matteo and D'Eramo, Carlo and Restelli, Marcello},
	year         = 2017,
	booktitle    = {Proceedings of the 34th International Conference on Machine Learning-Volume 70},
	pages        = {3434--3443},
	organization = {JMLR. org}
}
@inproceedings{riedmiller2005neural,
	title        = {Neural fitted Q iteration--first experiences with a data efficient neural reinforcement learning method},
	author       = {Riedmiller, Martin},
	year         = 2005,
	booktitle    = {European Conference on Machine Learning},
	pages        = {317--328},
	organization = {Springer}
}
@inproceedings{pires2016policy,
	title        = {Policy error bounds for model-based reinforcement learning with factored linear models},
	author       = {Pires, Bernardo {\'A}vila and Szepesv{\'a}ri, Csaba},
	year         = 2016,
	booktitle    = {Conference on Learning Theory},
	pages        = {121--151}
}
@inproceedings{liu2018breaking,
	title        = {Breaking the curse of horizon: Infinite-horizon off-policy estimation},
	author       = {Liu, Qiang and Li, Lihong and Tang, Ziyang and Zhou, Dengyong},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {5361--5371}
}
@article{chen2018scalable,
	title        = {Scalable Bilinear $\pi$ Learning Using State and Action Features},
	author       = {Chen, Yichen and Li, Lihong and Wang, Mengdi},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1804.10328}
}
@book{shalev2014understanding,
	title        = {Understanding machine learning: From theory to algorithms},
	author       = {Shalev-Shwartz, Shai and Ben-David, Shai},
	year         = 2014,
	publisher    = {Cambridge University Press}
}
@article{szepesvari2010algorithms,
	title        = {Algorithms for reinforcement learning},
	author       = {Szepesv{\'a}ri, Csaba},
	year         = 2010,
	journal      = {Synthesis lectures on artificial intelligence and machine learning},
	publisher    = {Morgan \& Claypool Publishers},
	volume       = 4,
	number       = 1,
	pages        = {1--103}
}
@article{yang2019theoretical,
	title        = {A Theoretical Analysis of Deep Q-Learning},
	author       = {Yang, Zhuora and Xie, Yuchen and Wang, Zhaoran},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1901.00137}
}
@inproceedings{sun2019model,
	title        = {{Model-based RL in Contextual Decision Processes: PAC bounds and Exponential Improvements over Model-free Approaches}},
	author       = {Sun, Wen and Jiang, Nan and Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John},
	year         = 2019,
	booktitle    = {Conference on Learning Theory}
}
@phdthesis{van1994feature,
	title        = {Feature-based methods for large scale dynamic programming},
	author       = {Van Roy, Benjamin},
	year         = 1994,
	school       = {Massachusetts Institute of Technology}
}
@incollection{yu1997assouad,
	title        = {{Assouad, Fano, and Le Cam}},
	author       = {Yu, Bin},
	year         = 1997,
	booktitle    = {Festschrift for Lucien Le Cam},
	publisher    = {Springer},
	pages        = {423--435}
}
@inproceedings{dai2018sbeed,
	title        = {Sbeed: Convergent reinforcement learning with nonlinear function approximation},
	author       = {Dai, Bo and Shaw, Albert and Li, Lihong and Xiao, Lin and He, Niao and Liu, Zhen and Chen, Jianshu and Song, Le},
	year         = 2018,
	booktitle    = {International Conference on Machine Learning},
	pages        = {1133--1142}
}
@inproceedings{maillard2010finite,
	title        = {Finite-sample analysis of Bellman residual minimization},
	author       = {Maillard, Odalric-Ambrym and Munos, R{\'e}mi and Lazaric, Alessandro and Ghavamzadeh, Mohammad},
	year         = 2010,
	booktitle    = {Proceedings of 2nd Asian Conference on Machine Learning},
	pages        = {299--314}
}
@article{munos2008finite,
	title        = {Finite-time bounds for fitted value iteration},
	author       = {Munos, R{\'e}mi and Szepesv{\'a}ri, Csaba},
	year         = 2008,
	journal      = {Journal of Machine Learning Research},
	volume       = 9,
	number       = {May},
	pages        = {815--857}
}
@article{jiang2016contextual,
	title        = {Contextual decision processes with low Bellman rank are PAC-learnable},
	author       = {Jiang, Nan and Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John and Schapire, Robert E},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1610.09512}
}
@inproceedings{dann2018oracle,
	title        = {On Oracle-Efficient {PAC} Reinforcement Learning with Rich Observations},
	author       = {Dann, Christoph and Jiang, Nan and Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John and Schapire, Robert E.},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems 31}
}
@article{gretton2012kernel,
	title        = {A kernel two-sample test},
	author       = {Gretton, Arthur and Borgwardt, Karsten M and Rasch, Malte J and Sch{\"o}lkopf, Bernhard and Smola, Alexander},
	year         = 2012,
	journal      = {Journal of Machine Learning Research},
	volume       = 13,
	number       = {Mar},
	pages        = {723--773}
}
@inproceedings{sandholm2012lossy,
	title        = {Lossy stochastic game abstraction with bounds},
	author       = {Sandholm, Tuomas and Singh, Satinder},
	year         = 2012,
	booktitle    = {Proceedings of the 13th ACM Conference on Electronic Commerce},
	pages        = {880--897}
}
@incollection{ravindran2002model,
	title        = {Model minimization in hierarchical reinforcement learning},
	author       = {Ravindran, Balaraman and Barto, Andrew},
	year         = 2002,
	booktitle    = {Abstraction, Reformulation, and Approximation},
	pages        = {196--211}
}
@article{hester2013texplore,
	title        = {{TEXPLORE}: real-time sample-efficient reinforcement learning for robots},
	author       = {Hester, Todd and Stone, Peter},
	year         = 2013,
	journal      = {Machine Learning},
	volume       = 90,
	number       = 3,
	pages        = {385--429}
}
@article{givan2003equivalence,
	title        = {Equivalence notions and model minimization in {M}arkov decision processes},
	author       = {Givan, Robert and Dean, Thomas and Greig, Matthew},
	year         = 2003,
	journal      = {Artificial Intelligence},
	volume       = 147,
	number       = 1,
	pages        = {163--223}
}
@inproceedings{li2006towards,
	title        = {Towards a unified theory of state abstraction for {MDP}s},
	author       = {Li, Lihong and Walsh, Thomas J and Littman, Michael L},
	year         = 2006,
	booktitle    = {Proceedings of the 9th International Symposium on Artificial Intelligence and Mathematics},
	pages        = {531--539}
}
@phdthesis{ravindran2004algebraic,
	title        = {An algebraic approach to abstraction in reinforcement learning},
	author       = {Ravindran, Balaraman},
	year         = 2004,
	school       = {University of Massachusetts Amherst}
}
@inproceedings{sutton1999policy,
	title        = {Policy gradient methods for reinforcement learning with function approximation},
	author       = {Sutton, Richard S and McAllester, David A and Singh, Satinder P and Mansour, Yishay},
	year         = 1999,
	booktitle    = {Advances in Neural Information Processing Systems},
	volume       = 99,
	pages        = {1057--1063}
}
@article{dietterich2000hierarchical,
	title        = {Hierarchical reinforcement learning with the {MAXQ} value function decomposition},
	author       = {Dietterich, Thomas G},
	year         = 2000,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 13,
	pages        = {227--303}
}
@incollection{even-dar2003approximate,
	title        = {Approximate equivalence of {M}arkov decision processes},
	author       = {Even-Dar, Eyal and Mansour, Yishay},
	year         = 2003,
	booktitle    = {Learning Theory and Kernel Machines},
	publisher    = {Springer},
	pages        = {581--594}
}
@inproceedings{ferns2004metrics,
	title        = {Metrics for finite {M}arkov decision processes},
	author       = {Ferns, Norm and Panangaden, Prakash and Precup, Doina},
	year         = 2004,
	booktitle    = {Proceedings of Uncertainty in Artificial Intelligence},
	pages        = {162--169}
}
@inproceedings{taylor2008bounding,
	title        = {Bounding performance loss in approximate {MDP} homomorphisms},
	author       = {Taylor, Jonathan and Precup, Doina and Panagaden, Prakash},
	year         = 2008,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1649--1656}
}
@inproceedings{ferns2006methods,
	title        = {Methods for computing state similarity in markov decision processes},
	author       = {Ferns, Norm and Castro, Pablo Samuel and Precup, Doina and Panangaden, Prakash},
	year         = 2006,
	booktitle    = {Proceedings of Uncertainty in Artificial Intelligence},
	pages        = {174--181}
}
@inproceedings{narayanamurthy2007efficientlya,
	title        = {Efficiently {E}xploiting {S}ymmetries in {R}eal {T}ime {D}ynamic {P}rogramming},
	author       = {Narayanamurthy, Shravan Matthur and Ravindran, Balaraman},
	year         = 2007,
	booktitle    = {International Joint Conference on Artificial Intelligence},
	pages        = {2556--2561}
}
@article{whitt1978approximations,
	title        = {{Approximations of dynamic programs, I}},
	author       = {Whitt, Ward},
	year         = 1978,
	journal      = {Mathematics of Operations Research},
	publisher    = {INFORMS},
	volume       = 3,
	number       = 3,
	pages        = {231--243}
}
@article{ortner2014selecting,
	title        = {Selecting {N}ear-{O}ptimal {A}pproximate {S}tate {R}epresentations in {R}einforcement {L}earning},
	author       = {Ortner, Ronald and Maillard, Odalric-Ambrym and Ryabko, Daniil},
	year         = 2014,
	journal      = {arXiv preprint arXiv:1405.2652}
}
@inproceedings{hallak2013model,
	title        = {Model selection in markovian processes},
	author       = {Hallak, Assaf and Di-Castro, Dotan and Mannor, Shie},
	year         = 2013,
	booktitle    = {Proceedings of the 19th ACM SIGKDD Conference on Knowledge Discovery and Data mining},
	pages        = {374--382}
}
@article{farahmand2011model,
	title        = {Model selection in reinforcement learning},
	author       = {Farahmand, Amir-massoud and Szepesv{\'a}ri, Csaba},
	year         = 2011,
	journal      = {Machine learning},
	publisher    = {Springer US},
	volume       = 85,
	number       = 3,
	pages        = {299--332}
}
@inproceedings{ross2011reduction,
	title        = {A reduction of imitation learning and structured prediction to no-regret online learning},
	author       = {Ross, St{\'e}phane and Gordon, Geoffrey and Bagnell, Drew},
	year         = 2011,
	booktitle    = {Proceedings of the fourteenth international conference on artificial intelligence and statistics},
	pages        = {627--635}
}
@article{ross2014reinforcement,
	title        = {Reinforcement and imitation learning via interactive no-regret learning},
	author       = {Ross, Stephane and Bagnell, J Andrew},
	year         = 2014,
	journal      = {arXiv preprint arXiv:1406.5979}
}
@inproceedings{bartlett2009regal,
	title        = {{REGAL}: {A} regularization based algorithm for reinforcement learning in weakly communicating {MDP}s},
	author       = {Bartlett, Peter L and Tewari, Ambuj},
	year         = 2009,
	booktitle    = {Proceedings of the 25th Conference on Uncertainty in Artificial Intelligence},
	pages        = {35--42},
	organization = {AUAI Press}
}
@inproceedings{mandel2014offline,
	title        = {Offline policy evaluation across representations with applications to educational games},
	author       = {Mandel, Travis and Liu, Yun-En and Levine, Sergey and Brunskill, Emma and Popovic, Zoran},
	year         = 2014,
	booktitle    = {Proceedings of the 13th International Conference on Autonomous Agents and Multi-Agent Systems},
	pages        = {1077--1084}
}
@inproceedings{parr2008analysis,
	title        = {An analysis of linear models, linear value-function approximation, and feature selection for reinforcement learning},
	author       = {Parr, Ronald and Li, Lihong and Taylor, Gavin and Painter-Wakefield, Christopher and Littman, Michael L},
	year         = 2008,
	booktitle    = {Proceedings of the 25th International Conference on Machine Learning},
	pages        = {752--759}
}
@article{ormoneit2002kernel,
	title        = {Kernel-based reinforcement learning},
	author       = {Ormoneit, Dirk and Sen, {\'S}aunak},
	year         = 2002,
	journal      = {Machine learning},
	publisher    = {Kluwer Academic Publishers},
	volume       = 49,
	number       = {2-3},
	pages        = {161--178}
}
@inproceedings{ravindran2004approximate,
	title        = {Approximate homomorphisms: {A} framework for nonexact minimization in {M}arkov decision processes},
	author       = {Ravindran, Balaraman and Barto, Andrew},
	year         = 2004,
	booktitle    = {Proceedings of the 5th International Conference on Knowledge-Based Computer Systems}
}
@article{bartlett2003rademacher,
	title        = {Rademacher and {G}aussian complexities: {R}isk bounds and structural results},
	author       = {Bartlett, Peter L and Mendelson, Shahar},
	year         = 2003,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 3,
	pages        = {463--482}
}
@article{boutilier2000stochastic,
	title        = {{S}tochastic dynamic programming with factored representations},
	author       = {Boutilier, Craig and Dearden, Richard and Goldszmidt, Mois{\'e}s},
	year         = 2000,
	journal      = {Artificial Intelligence},
	publisher    = {Elsevier},
	volume       = 121,
	number       = 1,
	pages        = {49--107}
}
@article{brafman2003r,
	title        = {R-max-a general polynomial time algorithm for near-optimal reinforcement learning},
	author       = {Brafman, Ronen I and Tennenholtz, Moshe},
	year         = 2003,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 3,
	pages        = {213--231}
}
@article{kaelbling1996reinforcement,
	title        = {Reinforcement {L}earning: {A} {S}urvey},
	author       = {Kaelbling, Leslie Pack and Littman, Michael L and Moore, Andrew W},
	year         = 1996,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 4,
	pages        = {237--285}
}
@book{kearns1994introduction,
	title        = {An {I}ntroduction to {C}omputational {L}earning {T}heory},
	author       = {Kearns, Michael J and Vazirani, Umesh V},
	year         = 1994,
	publisher    = {MIT Press},
	isbn         = 9780262111935,
	url          = {http://books.google.com/books?id=vCA01wY6iywC},
	lccn         = {lc94016588}
}
@article{kearns2002optimal,
	title        = {Near-optimal reinforcement learning in polynomial time},
	author       = {Kearns, Michael and Singh, Satinder},
	year         = 2002,
	journal      = {Machine Learning},
	publisher    = {Springer},
	volume       = 49,
	number       = {2-3},
	pages        = {209--232}
}
@incollection{koltchinskii2000rademacher,
	title        = {Rademacher processes and bounding the risk of function learning},
	author       = {Koltchinskii, Vladimir and Panchenko, Dmitriy},
	year         = 2000,
	booktitle    = {High Dimensional Probability II},
	publisher    = {Springer},
	pages        = {443--457}
}
@article{mannor2007bias,
	title        = {Bias and variance approximation in value function estimates},
	author       = {Mannor, Shie and Simester, Duncan and Sun, Peng and Tsitsiklis, John N},
	year         = 2007,
	journal      = {Management Science},
	publisher    = {INFORMS},
	volume       = 53,
	number       = 2,
	pages        = {308--322}
}
@phdthesis{paduraru2013policy,
	title        = {Off-policy {E}valuation in {M}arkov {D}ecision {P}rocesses},
	author       = {Paduraru, Cosmin},
	year         = 2013,
	school       = {McGill University}
}
@inproceedings{strehl2005theoretical,
	title        = {A theoretical analysis of model-based interval estimation},
	author       = {Strehl, Alexander L and Littman, Michael L},
	year         = 2005,
	booktitle    = {Proceedings of the 22nd International Conference on Machine learning},
	pages        = {856--863},
	organization = {ACM}
}
@article{strehl2009reinforcement,
	title        = {Reinforcement learning in finite {MDP}s: {PAC} analysis},
	author       = {Strehl, Alexander L and Li, Lihong and Littman, Michael L},
	year         = 2009,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 10,
	pages        = {2413--2444}
}
@inproceedings{tewari2006sample,
	title        = {Sample complexity of policy search with known dynamics},
	author       = {Tewari, Ambuj and Bartlett, Peter L},
	year         = 2006,
	booktitle    = {Advances in Neural Information Processing Systems},
	volume       = 19,
	pages        = {97--104}
}
@inproceedings{vapnik1992principles,
	title        = {Principles of risk minimization for learning theory},
	author       = {Vapnik, Vladimir},
	year         = 1992,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {831--838}
}
@book{vapnik1998statistical,
	title        = {Statistical learning theory},
	author       = {Vapnik, Vladimir},
	year         = 1998,
	publisher    = {Wiley New York},
	volume       = 2
}
@inproceedings{jiang2014improving,
	title        = {Improving {UCT} planning via approximate homomorphisms},
	author       = {Jiang, Nan and Singh, Satinder and Lewis, Richard},
	year         = 2014,
	booktitle    = {Proceedings of the 13th International Conference on Autonomous Agents and Multi-Agent Systems},
	pages        = {1289--1296}
}
@inproceedings{paduraru2008model,
	title        = {Model-based Reinforcement Learning with State Aggregation},
	author       = {Paduraru, Cosmin and Kaplow, Robert and Precup, Doina and Pineau, Joelle},
	year         = 2008,
	booktitle    = {8th European Workshop on Reinforcement Learning}
}
@phdthesis{talvitie2010simple,
	title        = {Simple Partial Models for Complex Dynamical Systems},
	author       = {Talvitie, Erik N},
	year         = 2010,
	school       = {The University of Michigan}
}
@inproceedings{jong2005state,
	title        = {State abstraction discovery from irrelevant state variables},
	author       = {Jong, Nicholas K and Stone, Peter},
	year         = 2005,
	booktitle    = {Proceedings of the 19th International Joint Conference on Artificial Intelligence},
	pages        = {752--757}
}
@article{singh1994upper,
	title        = {An upper bound on the loss from approximate optimal-value functions},
	author       = {Singh, Satinder and Yee, Richard},
	year         = 1994,
	journal      = {Machine Learning},
	volume       = 16,
	number       = 3,
	pages        = {227--233}
}
@inproceedings{dinculescu2010approximate,
	title        = {Approximate predictive representations of partially observable systems},
	author       = {Dinculescu, Monica and Precup, Doina},
	year         = 2010,
	booktitle    = {Proceedings of the 27th International Conference on Machine Learning (ICML-10)},
	pages        = {895--902}
}
@article{talvitie2011learning,
	title        = {Learning to Make Predictions In Partially Observable Environments Without a Generative Model},
	author       = {Talvitie, Erik and Singh, Satinder},
	year         = 2011,
	journal      = {Journal of Artificial Intelligence Research (JAIR)},
	volume       = 42,
	pages        = {353--392}
}
@inproceedings{maillard2014how,
	title        = {"{H}ow hard is my {MDP}?" {T}he distribution-norm to the rescue},
	author       = {Maillard, Odalric-Ambrym and Mann, Timothy A and Mannor, Shie},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1835--1843}
}
@article{seijen2014efficient,
	title        = {Efficient abstraction selection in reinforcement learning},
	author       = {van Seijen, Harm and Whiteson, Shimon and Kester, Leon},
	year         = 2014,
	journal      = {Computational Intelligence},
	volume       = 30,
	number       = 4,
	pages        = {657--699}
}
@inproceedings{konidaris2009efficient,
	title        = {Efficient {S}kill {L}earning using {A}bstraction {S}election},
	author       = {Konidaris, George and Barto, Andrew},
	year         = 2009,
	booktitle    = {Proceedings of the 21st International Joint Conference on Artificial Intelligence}
}
@article{cobo2014abstraction,
	title        = {Abstraction from demonstration for efficient reinforcement learning in high-dimensional domains},
	author       = {Cobo, Luis C and Subramanian, Kaushik and Isbell, Charles L and Lanterman, Aaron D and Thomaz, Andrea L},
	year         = 2014,
	journal      = {Artificial Intelligence},
	volume       = 216,
	pages        = {103--128}
}
@article{farahmand2011regularization,
	title        = {Regularization in reinforcement learning},
	author       = {Farahmand, Amir-massoud},
	year         = 2011,
	publisher    = {University of Alberta}
}
@article{agarwal2016corralling,
	title        = {Corralling a Band of Bandit Algorithms},
	author       = {Agarwal, Alekh and Luo, Haipeng and Neyshabur, Behnam and Schapire, Robert E},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1612.06246}
}
@phdthesis{scott2004dyadic,
	title        = {Dyadic decision trees},
	author       = {Scott, Clayton D},
	year         = 2004,
	school       = {University of Wisconsin at Madison}
}
@inproceedings{ng2003autonomous,
	title        = {Autonomous helicopter flight via reinforcement learning},
	author       = {Ng, Andrew Y and Kim, H Jin and Jordan, Michael I and Sastry, Shankar and Ballianda, Shiv},
	year         = 2003,
	booktitle    = {NIPS},
	volume       = 16
}
@inproceedings{johnson2016malmo,
	title        = {The malmo platform for artificial intelligence experimentation},
	author       = {Johnson, Matthew and Hofmann, Katja and Hutton, Tim and Bignell, David},
	year         = 2016,
	booktitle    = {International joint conference on artificial intelligence (IJCAI)},
	pages        = 4246
}
@article{brockman2016openai,
	title        = {{OpenAI} gym},
	author       = {Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, John and Tang, Jie and Zaremba, Wojciech},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1606.01540}
}
@inproceedings{li2010contextual,
	title        = {A contextual-bandit approach to personalized news article recommendation},
	author       = {Li, Lihong and Chu, Wei and Langford, John and Schapire, Robert E},
	year         = 2010,
	booktitle    = {Proceedings of the 19th international conference on World wide web},
	pages        = {661--670},
	organization = {ACM}
}
@article{lei2012a,
	title        = {A" SMART" design for building individualized treatment sequences},
	author       = {Lei, H and Nahum-Shani, I and Lynch, K and Oslin, D and Murphy, SA},
	year         = 2012,
	journal      = {Annual review of clinical psychology},
	publisher    = {Annual Reviews},
	volume       = 8,
	pages        = {21--48}
}
@article{singh2002optimizing,
	title        = {Optimizing dialogue management with reinforcement learning: {E}xperiments with the {NJF}un system},
	author       = {Singh, Satinder and Litman, Diane and Kearns, Michael and Walker, Marilyn},
	year         = 2002,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 16,
	pages        = {105--133}
}
@article{williams1992simple,
	title        = {Simple statistical gradient-following algorithms for connectionist reinforcement learning},
	author       = {Williams, Ronald J},
	year         = 1992,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 8,
	number       = {3-4},
	pages        = {229--256}
}
@phdthesis{li2009unifying,
	title        = {A unifying framework for computational reinforcement learning theory},
	author       = {Li, Lihong},
	year         = 2009,
	school       = {Rutgers, The State University of New Jersey}
}
@book{mohri2012foundations,
	title        = {Foundations of machine learning},
	author       = {Mohri, Mehryar and Rostamizadeh, Afshin and Talwalkar, Ameet},
	year         = 2012,
	publisher    = {MIT press}
}
@inproceedings{vanseijen2015deeper,
	title        = {A Deeper Look at Planning as Learning from Replay},
	author       = {van Seijen, Harm and Sutton, Rich},
	year         = 2015,
	booktitle    = {Proceedings of the 32nd International Conference on Machine Learning},
	pages        = {2314--2322}
}
@article{bradtke1996linear,
	title        = {Linear least-squares algorithms for temporal difference learning},
	author       = {Bradtke, Steven J and Barto, Andrew G},
	year         = 1996,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 22,
	number       = {1-3},
	pages        = {33--57}
}
@article{lin1992self,
	title        = {Self-improving reactive agents based on reinforcement learning, planning and teaching},
	author       = {Lin, Long-Ji},
	year         = 1992,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 8,
	number       = {3-4},
	pages        = {293--321}
}
@inproceedings{wang2016dueling,
	title        = {Dueling Network Architectures for Deep Reinforcement Learning},
	author       = {Wang, Ziyu and Schaul, Tom and Hessel, Matteo and Hasselt, Hado van and Lanctot, Marc and Freitas, Nando de},
	year         = 2016,
	booktitle    = {Proceedings of the 33rd International Conference on Machine Learning},
	pages        = {1995--2003}
}
{sutton1996generalization,
	title={Generalization in reinforcement learning: {S}uccessful examples using sparse coarse coding},
	author={Sutton, Richard S},
	journal={Advances in Neural Information Processing Systems},
	pages={1038--1044},
	year={1996}
%	publisher={MORGAN KAUFMANN PUBLISHERS}
}
@phdthesis{watkins1989learning,
	title        = {Learning from delayed rewards},
	author       = {Watkins, Christopher John Cornish Hellaby},
	year         = 1989,
	school       = {University of Cambridge England}
}
@article{hoeffding1963probability,
	title        = {Probability inequalities for sums of bounded random variables},
	author       = {Hoeffding, Wassily},
	year         = 1963,
	journal      = {Journal of the American statistical association},
	publisher    = {Taylor \& Francis Group},
	volume       = 58,
	number       = 301,
	pages        = {13--30}
}
@inproceedings{strehl2006pac,
	title        = {{PAC} model-free reinforcement learning},
	author       = {Strehl, Alexander L and Li, Lihong and Wiewiora, Eric and Langford, John and Littman, Michael L},
	year         = 2006,
	booktitle    = {Proceedings of the 23rd international conference on Machine learning},
	pages        = {881--888},
	organization = {ACM}
}
@article{auer2007logarithmic,
	title        = {Logarithmic online regret bounds for undiscounted reinforcement learning},
	author       = {Auer, Peter and Ortner, Ronald},
	year         = 2007,
	journal      = {Advances in Neural Information Processing Systems},
	publisher    = {MIT; 1998},
	volume       = 19,
	pages        = 49
}
@article{jaksch2010optimal,
	title        = {Near-optimal regret bounds for reinforcement learning},
	author       = {Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
	year         = 2010,
	journal      = {Journal of Machine Learning Research},
	volume       = 11,
	number       = {Apr},
	pages        = {1563--1600}
}
@phdthesis{kakade2003sample,
	title        = {On the sample complexity of reinforcement learning},
	author       = {Kakade, Sham Machandranath},
	year         = 2003,
	school       = {University of College London}
}
@inproceedings{krishnamurthy2016pac,
	title        = {{PAC} reinforcement learning with rich observations},
	author       = {Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John},
	year         = 2016,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1840--1848}
}

@inproceedings{dann2015sample,
	title        = {Sample complexity of episodic fixed-horizon reinforcement learning},
	author       = {Dann, Christoph and Brunskill, Emma},
	year         = 2015,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2818--2826}
}
@article{valiant1984theory,
	title        = {A theory of the learnable},
	author       = {Valiant, Leslie G},
	year         = 1984,
	journal      = {Communications of the ACM},
	publisher    = {ACM},
	volume       = 27,
	number       = 11,
	pages        = {1134--1142}
}
@inproceedings{langford2008epoch,
	title        = {The epoch-greedy algorithm for multi-armed bandits with side information},
	author       = {Langford, John and Zhang, Tong},
	year         = 2008,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {817--824}
}
@article{puterman1994markov,
	title        = {Markov {D}ecision {P}rocesses},
	author       = {Puterman, ML},
	year         = 1994,
	journal      = {Jhon Wiley \& Sons, New Jersey}
}
@inproceedings{rosencrantz2004learning,
	title        = {Learning low dimensional predictive representations},
	author       = {Rosencrantz, Matthew and Gordon, Geoff and Thrun, Sebastian},
	year         = 2004,
	booktitle    = {Proceedings of the twenty-first international conference on Machine learning},
	pages        = 88,
	organization = {ACM}
}
@article{kober2013reinforcement,
	title        = {Reinforcement learning in robotics: A survey},
	author       = {Kober, Jens and Bagnell, J Andrew and Peters, Jan},
	year         = 2013,
	journal      = {The International Journal of Robotics Research},
	publisher    = {SAGE Publications},
	pages        = {0278364913495721}
}
@inproceedings{petrik2014raam,
	title        = {{RAAM: The Benefits of Robustness in Approximating Aggregated MDPs in Reinforcement Learning}},
	author       = {Petrik, Marek and Subramanian, Dharmashankar},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1979--1987}
}
@article{vanroy2006performance,
	title        = {Performance loss bounds for approximate value iteration with state aggregation},
	author       = {Van Roy, Benjamin},
	year         = 2006,
	journal      = {Mathematics of Operations Research},
	publisher    = {INFORMS},
	volume       = 31,
	number       = 2,
	pages        = {234--244}
}
@manual{shipranotes,
	title        = {{IEOR 8100: Reinforcement Learning. Lecture 4: Approximate Dynamic Programming}},
	author       = {Shipra Agrawal},
	year         = 2018,
	note         = {\url{https://ieor8100.github.io/rl/docs/Lecture%204%20-%20approximate%20DP.pdf}},
	organization = {{Columbia University}}
}
@phdthesis{van1994feature,
	title        = {Feature-based methods for large scale dynamic programming},
	author       = {Van Roy, Benjamin},
	year         = 1994,
	school       = {Massachusetts Institute of Technology}
}
@inproceedings{kulesza2015low,
	title        = {{Low-Rank Spectral Learning with Weighted Loss Functions}},
	author       = {Kulesza, Alex and Jiang, Nan and Singh, Satinder},
	year         = 2015,
	booktitle    = {Proceedings of the 18th International Conference on Artificial Intelligence and Statistics},
	pages        = {517--525}
}
@incollection{riedmiller2005neural,
	title        = {{Neural fitted Q iteration--first experiences with a data efficient neural reinforcement learning method}},
	author       = {Riedmiller, Martin},
	year         = 2005,
	booktitle    = {Machine Learning: ECML 2005},
	publisher    = {Springer},
	pages        = {317--328}
}
@inproceedings{kobilarov2015sample,
	title        = {{Sample Complexity Bounds for Iterative Stochastic Policy Optimization}},
	author       = {Kobilarov, Marin},
	year         = 2015,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {3096--3104}
}
@inproceedings{abel2016near,
	title        = {Near optimal behavior via approximate state abstraction},
	author       = {Abel, David and Hershkowitz, D Ellis and Littman, Michael L},
	year         = 2016,
	booktitle    = {Proceedings of the 33rd International Conference on International Conference on Machine Learning-Volume 48},
	pages        = {2915--2923},
	organization = {JMLR. org}
}
@article{bellman1956dynamic,
	title        = {{Dynamic programming and Lagrange multipliers}},
	author       = {Bellman, Richard},
	year         = 1956,
	journal      = {Proceedings of the National Academy of Sciences},
	publisher    = {National Acad Sciences},
	volume       = 42,
	number       = 10,
	pages        = {767--769}
}
@book{kumar2012customer,
	title        = {{Customer relationship management: Concept, strategy, and tools}},
	author       = {Kumar, Vineet and Reinartz, Werner},
	year         = 2012,
	publisher    = {Springer Science \& Business Media}
}
@article{murphy2003optimal,
	title        = {Optimal dynamic treatment regimes},
	author       = {Murphy, Susan A},
	year         = 2003,
	journal      = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
	publisher    = {Wiley Online Library},
	volume       = 65,
	number       = 2,
	pages        = {331--355}
}
@inproceedings{hamilton2013modelling,
	title        = {Modelling sparse dynamical systems with compressed predictive state representations},
	author       = {Hamilton, William L and Fard, Mahdi M and Pineau, Joelle},
	year         = 2013,
	booktitle    = {Proceedings of the 30th International Conference on Machine Learning (ICML-13)},
	pages        = {178--186}
}
@inproceedings{boots2010closing,
	title        = {Closing the learning-planning loop with predictive state representations},
	author       = {Boots, Byron and Siddiqi, Sajid M and Gordon, Geoffrey J},
	year         = 2010,
	booktitle    = {Proceedings of the 9th International Conference on Autonomous Agents and Multiagent Systems},
	pages        = {1369--1370}
}
@incollection{farahmand2008regularized,
	title        = {{Regularized fitted Q-iteration: Application to planning}},
	author       = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
	year         = 2008,
	booktitle    = {Recent Advances in Reinforcement Learning},
	publisher    = {Springer Berlin Heidelberg},
	pages        = {55--68}
}
@inproceedings{farahmand2009regularized,
	title        = {{Regularized fitted Q-iteration for planning in continuous-space Markovian decision problems}},
	author       = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
	year         = 2009,
	booktitle    = {American Control Conference.},
	pages        = {725--730},
	organization = {IEEE}
}
@article{tsitsiklis1997analysis,
	title        = {An Analysis of Temporal-Difference Learning with Function Approximation},
	author       = {Tsitsiklis, John N and Van Roy, Benjamin},
	year         = 1997,
	journal      = {IEEE TRANSACTIONS ON AUTOMATIC CONTROL},
	volume       = 42,
	number       = 5
}
@inproceedings{szepesvari2005finite,
	title        = {Finite time bounds for sampling based fitted value iteration},
	author       = {Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
	year         = 2005,
	booktitle    = {Proceedings of the 22nd international conference on Machine learning},
	pages        = {880--887},
	organization = {ACM}
}
@article{lazaric2012finite,
	title        = {Finite-sample analysis of least-squares policy iteration},
	author       = {Lazaric, Alessandro and Ghavamzadeh, Mohammad and Munos, R{\'e}mi},
	year         = 2012,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 13,
	number       = 1,
	pages        = {3041--3074}
}
@article{lagoudakis2003least,
	title        = {Least-squares policy iteration},
	author       = {Lagoudakis, Michail G and Parr, Ronald},
	year         = 2003,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 4,
	pages        = {1107--1149}
}
@article{bertsekas2011approximate,
	title        = {Approximate policy iteration: A survey and some new methods},
	author       = {Bertsekas, Dimitri P},
	year         = 2011,
	journal      = {Journal of Control Theory and Applications},
	publisher    = {Springer},
	volume       = 9,
	number       = 3,
	pages        = {310--335}
}
@inproceedings{gordon1995stable,
	title        = {Stable function approximation in dynamic programming},
	author       = {Gordon, Geoffrey J},
	year         = 1995,
	booktitle    = {Proceedings of the twelfth international conference on machine learning},
	pages        = {261--268}
}
@book{bertsekas1996neuro,
	title        = {Neuro-Dynamic Programming},
	author       = {Bertsekas, Dimitri P and Tsitsiklis, John N},
	year         = 1996,
	publisher    = {Athena Scientific, Belmont, MA}
}
@article{bellman1959functional,
	title        = {Functional approximations and dynamic programming},
	author       = {Bellman, Richard and Dreyfus, Stuart},
	year         = 1959,
	journal      = {Mathematical Tables and Other Aids to Computation},
	publisher    = {JSTOR},
	volume       = 13,
	number       = 68,
	pages        = {247--251}
}
@article{bellemare2013arcade,
	title        = {The arcade learning environment: {A}n evaluation platform for general agents},
	author       = {Bellemare, Marc G and Naddaf, Yavar and Veness, Joel and Bowling, Michael},
	year         = 2013,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 47,
	pages        = {253--279}
}
@article{pires2016policy,
	title        = {Policy Error Bounds for Model-Based Reinforcement Learning with Factored Linear Models},
	author       = {Pires, Bernardo {\'A}vila and Szepesv{\'a}ri, Csaba},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1602.06346}
}
@article{ernst2005tree,
	title        = {Tree-based batch mode reinforcement learning},
	author       = {Ernst, Damien and Geurts, Pierre and Wehenkel, Louis},
	year         = 2005,
	journal      = {Journal of Machine Learning Research},
	publisher    = {Microtome Publishing},
	volume       = 6,
	pages        = {503--556}
}
@inproceedings{jiang2016doubly,
	title        = {{Doubly Robust Off-policy Value Evaluation for Reinforcement Learning}},
	author       = {Nan Jiang and Lihong Li},
	year         = 2016,
	booktitle    = {Proceedings of The 33rd International Conference on Machine Learning},
	volume       = 48,
	pages        = {652--661}
}
@inproceedings{thomas2016data,
	title        = {{Data-Efficient Off-Policy Policy Evaluation for Reinforcement Learning}},
	author       = {Thomas, Philip and Brunskill, Emma},
	year         = 2016,
	booktitle    = {Proceedings of the 33rd International Conference on Machine Learning}
}
@inproceedings{li2015minimax,
	title        = {Toward Minimax Off-policy Value Estimation},
	author       = {Li, Lihong and Munos, R{\'e}mi and Szepesv{\'a}ri, Csaba},
	year         = 2015,
	booktitle    = {Proceedings of the 18th International Conference on Artificial Intelligence and Statistics}
}
@inproceedings{kakade2002approximately,
	title        = {{Approximately Optimal Approximate Reinforcement Learning}},
	author       = {Kakade, Sham and Langford, John},
	year         = 2002,
	booktitle    = {Proceedings of the 19th International Conference on Machine Learning},
	volume       = 2,
	pages        = {267--274}
}
@inproceedings{jiang2015abstraction,
	title        = {{Abstraction Selection in Model-based Reinforcement Learning}},
	author       = {Jiang, Nan and Kulesza, Alex and Singh, Satinder},
	year         = 2015,
	booktitle    = {Proceedings of the 32nd International Conference on Machine Learning},
	pages        = {179--188}
}
@inproceedings{thomas2015high,
	title        = {{High Confidence Off-Policy Evaluation}},
	author       = {Thomas, Philip and Theocharous, Georgios and Ghavamzadeh, Mohammad},
	year         = 2015,
	booktitle    = {Proceedings of the 29th AAAI Conference on Artificial Intelligence}
}
@inproceedings{dudik2011doubly,
	title        = {Doubly {R}obust {P}olicy {E}valuation and {L}earning},
	author       = {Dud{\'\i}k, Miroslav and Langford, John and Li, Lihong},
	year         = 2011,
	booktitle    = {Proceedings of the 28th International Conference on Machine Learning},
	pages        = {1097--1104}
}
@inproceedings{thomas2015higha,
	title        = {{High Confidence Policy Improvement}},
	author       = {Thomas, Philip and Theocharous, Georgios and Ghavamzadeh, Mohammad},
	year         = 2015,
	booktitle    = {Proceedings of the 32nd International Conference on Machine Learning},
	pages        = {2380--2388}
}
@inproceedings{kearns2000bias,
	title        = {{Bias-Variance Error Bounds for Temporal Difference Updates}},
	author       = {Kearns, Michael J and Singh, Satinder P},
	year         = 2000,
	booktitle    = {Proceedings of the Thirteenth Annual Conference on Computational Learning Theory},
	pages        = {142--147},
	organization = {Morgan Kaufmann Publishers Inc.}
}
@inproceedings{sutton2014new,
	title        = {A new {Q}($\lambda$) with interim forward view and {M}onte {C}arlo equivalence},
	author       = {Sutton, Rich and Mahmood, Ashique R and Precup, Doina and Hasselt, Hado V},
	year         = 2014,
	booktitle    = {Proceedings of the 31st International Conference on Machine Learning},
	pages        = {568--576}
}
@inproceedings{guo2014deep,
	title        = {Deep learning for real-time Atari game play using offline Monte-Carlo tree search planning},
	author       = {Guo, Xiaoxiao and Singh, Satinder and Lee, Honglak and Lewis, Richard L and Wang, Xiaoshi},
	year         = 2014,
	booktitle    = {Advances in neural information processing systems},
	pages        = {3338--3346}
}
@article{singh1996reinforcement,
	title        = {Reinforcement learning with replacing eligibility traces},
	author       = {Singh, Satinder P and Sutton, Richard S},
	year         = 1996,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 22,
	number       = {1-3},
	pages        = {123--158}
}
@incollection{kocsis2006bandit,
	title        = {Bandit based monte-carlo planning},
	author       = {Kocsis, Levente and Szepesv{\'a}ri, Csaba},
	year         = 2006,
	booktitle    = {Machine Learning: ECML 2006},
	publisher    = {Springer Berlin Heidelberg},
	pages        = {282--293}
}
@misc{hettich1999uci,
	title        = {{The UCI KDD Archive}},
	author       = {Hettich, S and Bay, S D},
	year         = 1999,
	howpublished = {\url{http://kdd.ics.uci.edu}}
}
@inproceedings{precup2001policy,
	title        = {{Off-Policy Temporal-Difference Learning with Funtion Approximation}},
	author       = {Doina Precup and Richard S Sutton and Sanjoy Dasgupta},
	year         = 2001,
	booktitle    = {Proceedings of the 18th International Conference on Machine Learning},
	pages        = {417--424}
}
@inproceedings{precup2000eligibility,
	title        = {{Eligibility Traces for Off-Policy Policy Evaluation}},
	author       = {Doina Precup and Richard S Sutton and Satinder P Singh},
	year         = 2000,
	booktitle    = {Proceedings of the 17th International Conference on Machine Learning},
	pages        = {759--766}
}
@book{sutton1998reinforcement,
	title        = {Reinforcement Learning: An Introduction},
	author       = {Richard S Sutton and Andrew G Barto},
	year         = 1998,
	month        = {March},
	publisher    = {MIT Press},
	address      = {Cambridge, MA},
	isbn         = {0-262-19398-1}
}
@article{murphy2001marginal,
	title        = {{Marginal Mean Models for Dynamic Regimes}},
	author       = {Susan A. Murphy and Mark van der Laan and James M. Robins},
	year         = 2001,
	journal      = {Journal of the American Statistical Association},
	volume       = 96,
	number       = 456,
	pages        = {1410--1423}
}
@inproceedings{li2011unbiased,
	title        = {{Unbiased Offline Evaluation of Contextual-bandit-based News Article Recommendation Algorithms}},
	author       = {Lihong Li and Wei Chu and John Langford and Xuanhui Wang},
	year         = 2011,
	booktitle    = {Proceedings of the 4th International Conference on Web Search and Data Mining},
	pages        = {297--306}
}
@article{bottou2013counterfactual,
	title        = {{Counterfactual Reasoning and Learning Systems: The Example of Computational Advertising}},
	author       = {L\'{e}on Bottou and Jonas Peters and Joaquin Qui{\~n}onero-Candela and Denis Xavier Charles and D. Max Chickering and Elon Portugaly and Dipankar Ray and Patrice Simard and Ed Snelson},
	year         = 2013,
	journal      = {Journal of Machine Learning Research},
	volume       = 14,
	pages        = {3207--3260}
}
@inproceedings{sutton1998intra,
	title        = {{Intra-option Learning about Temporally Abstract Actions}},
	author       = {Richard S Sutton and Doina Precup and Satinder P Singh},
	year         = 1998,
	booktitle    = {Proceedings of the 15th International Conference on Machine Learning},
	pages        = {556--564}
}
@phdthesis{precup2000temporal,
	title        = {Temporal abstraction in reinforcement learning},
	author       = {Precup, Doina},
	year         = 2000,
	school       = {University of Massachusetts Amherst}
}
@phdthesis{thomas2015safe,
	title        = {Safe Reinforcement Learning},
	author       = {Thomas, Philip},
	year         = 2015,
	school       = {University of Massachusetts Amherst}
}
@inproceedings{gruenewaelder2012modelling,
	title        = {{Modelling transition dynamics in MDPs with RKHS embeddings}},
	author       = {Gr{\"u}new{\"a}lder, S and Lever, G and Baldassarre, L and Pontil, M and Gretton, A},
	year         = 2012,
	booktitle    = {Proceedings of the 29th International Conference on Machine Learning},
	pages        = {535--542}
}
@inproceedings{jong2007model,
	title        = {Model-based function approximation in reinforcement learning},
	author       = {Jong, Nicholas K and Stone, Peter},
	year         = 2007,
	booktitle    = {Proceedings of the 6th International Conference on Autonomous Agents and Multiagent Systems},
	pages        = 95
}
@article{dann2014policy,
	title        = {Policy evaluation with temporal differences: {A} survey and comparison},
	author       = {Dann, Christoph and Neumann, Gerhard and Peters, Jan},
	year         = 2014,
	journal      = {The Journal of Machine Learning Research},
	volume       = 15,
	number       = 1,
	pages        = {809--883}
}
@article{fonteneau2013batch,
	title        = {Batch Mode Reinforcement Learning based on the Synthesis of Artificial Trajectories},
	author       = {Raphael Fonteneau and Susan A. Murphy and Louis Wehenkel and Damien Ernst},
	year         = 2013,
	journal      = {Annals of Operations Research},
	volume       = 208,
	number       = 1,
	pages        = {383--416}
}
@article{rotnitzky1995semiparametric,
	title        = {Semiparametric Regression Estimation in the Presence of Dependent Censoring},
	author       = {Andrea Rotnitzky and James M. Robins},
	year         = 1995,
	journal      = {Biometrika},
	volume       = 82,
	number       = 4,
	pages        = {805--820}
}
@inproceedings{pirotta2013safe,
	title        = {Safe Policy Iteration},
	author       = {Matteo Pirotta and Marcello Restelli and Alessio Pecorino and Daniele Calandriello},
	year         = 2013,
	booktitle    = {Proceedings of the 30th International Conference on Machine Learning},
	number       = 3,
	pages        = {307--317}
}
@book{pearl2009causality,
	title        = {Causality: Models, Reasoning, and Inference},
	author       = {Judea Pearl},
	year         = 2009,
	publisher    = {Cambridge University Press},
	isbn         = {052189560X},
	edition      = {2nd}
}
@article{holland1986statistics,
	title        = {Statistics and Causal Inference},
	author       = {Paul W. Holland},
	year         = 1986,
	journal      = {Journal of the American Statistical Association},
	volume       = 81,
	number       = 6,
	pages        = {945--960}
}
@unpublished{sutton2015emphatic,
	title        = {An Emphatic Approach to the Problem of Off-policy Temporal-Difference Learning},
	author       = {Richard S Sutton and Ashique Rupam Mahmood and Martha White},
	year         = 2015,
	note         = {CoRR abs/1503.04269}
}
@article{hirano2003efficient,
	title        = {Efficient Estimation of Average Treatment Effects using the Estimated Propensity Score},
	author       = {Hirano, Keisuke and Imbens, Guido W. and Ridder, Geert},
	year         = 2003,
	journal      = {Econometrica},
	volume       = 71,
	number       = 4,
	pages        = {1161--1189}
}
@inproceedings{pednault2002sequential,
	title        = {Sequential Cost-Sensitive Decision-Making with Reinforcement Learning},
	author       = {Edwin P. D. Pednault and Naoki Abe and Bianca Zadrozny},
	year         = 2002,
	booktitle    = {KDD},
	pages        = {259--268},
	booktitle1   = {Proceedings of the 8th International Conference on Knowledge Discovery and Data Mining}
}
@phdthesis{jr2010theory,
	title        = {A theory of Cramer-Rao bounds for constrained parametric models},
	author       = {Moore Jr, Terrence Joseph},
	year         = 2010,
	school       = {University of Maryland, College Park}
}
@phdthesis{marivate2015improved,
	title        = {Improved Empirical Methods in Reinforcement-Learning Evaluation},
	author       = {Vukosi N. Marivate},
	year         = 2015,
	address      = {New Brunswick, NJ},
	school       = {Rutgers University}
}
@inproceedings{marivate2014quantifying,
	title        = {Quantifying uncertainty in batch personalized sequential decision making},
	author       = {Marivate, Vukosi Ntsakisi and Chemali, Jessica and Brunskill, Emma and Littman, Michael},
	year         = 2014,
	booktitle    = {Workshops at the 28th AAAI Conference on Artificial Intelligence}
}
@inproceedings{singh2004predictive,
	title        = {{Predictive state representations: A new theory for modeling dynamical systems}},
	author       = {Singh, Satinder and James, Michael R and Rudary, Matthew R},
	year         = 2004,
	booktitle    = {Proceedings of the 20th Conference on Uncertainty in Artificial Intelligence},
	pages        = {512--519},
	organization = {AUAI Press}
}
@inproceedings{doshi-velez2009infinite,
	title        = {{The Infinite Partially observable Markov decision process}},
	author       = {Doshi-Velez, Finale},
	year         = 2009,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {477--485}
}
@misc{li2015recurrent,
	title        = {Recurrent Reinforcement Learning: A Hybrid Approach},
	author       = {Xiujun Li and Lihong Li and Jianfeng Gao and Xiaodong He and Jianshu Chen and Li Deng and Ji He},
	year         = 2015,
	note         = {arXiv:1509.03044}
}
@inproceedings{asmuth2011approaching,
	title        = {Approaching {B}ayes-optimalilty using {M}onte-{C}arlo tree search},
	author       = {Asmuth, John and Littman, Michael L},
	year         = 2011,
	booktitle    = {Proceedings of the 21st International Conference on Automatated Planning and Scheduling}
}
@article{bagnell2001solving,
	title        = {Solving uncertain {M}arkov decision processes},
	author       = {Bagnell, J Andrew and Ng, Andrew Y and Schneider, Jeff G},
	year         = 2001,
	publisher    = {Carnegie Mellon University}
}
@article{bertuccelli2012robust,
	title        = {Robust Adaptive Markov Decision Processes: Planning with Model Uncertainty},
	author       = {Bertuccelli, Luca F and Wu, Albert and How, Jonathan P},
	year         = 2012,
	journal      = {Control Systems, IEEE},
	publisher    = {IEEE},
	volume       = 32,
	number       = 5,
	pages        = {96--109}
}
@incollection{el-yaniv2007transductive,
	title        = {{T}ransductive rademacher complexity and its applications},
	author       = {El-Yaniv, Ran and Pechyony, Dmitry},
	year         = 2007,
	booktitle    = {Learning Theory},
	publisher    = {Springer},
	pages        = {157--171}
}
@article{even-dar2009online,
	title        = {Online {M}arkov decision processes},
	author       = {Even-Dar, Eyal and Kakade, Sham M and Mansour, Yishay},
	year         = 2009,
	journal      = {Mathematics of Operations Research},
	publisher    = {INFORMS},
	volume       = 34,
	number       = 3,
	pages        = {726--736}
}
@inproceedings{guez2012efficient,
	title        = {Efficient {B}ayes-{A}daptive {R}einforcement {L}earning using {S}ample-{B}ased {S}earch},
	author       = {Guez, Arthur and Silver, David and Dayan, Peter},
	year         = 2012,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1034--1042}
}
@article{browne2012survey,
	title        = {A survey of {M}onte {C}arlo tree search methods},
	author       = {Browne, Cameron B and Powley, Edward and Whitehouse, Daniel and Lucas, Simon M and Cowling, Peter I and Rohlfshagen, Philipp and Tavener, Stephen and Perez, Diego and Samothrakis, Spyridon and Colton, Simon},
	year         = 2012,
	journal      = {IEEE Transactions on Computational Intelligence and AI in Games},
	volume       = 4,
	number       = 1,
	pages        = {1--43}
}
@book{montenegro2006mathematical,
	title        = {Mathematical aspects of mixing times in {M}arkov chains},
	author       = {Montenegro, Ravi R and Tetali, Prasad},
	year         = 2006,
	publisher    = {Now Publishers Inc}
}
{ng2000pegasus,
  title =	 {{PEGASUS}: {A} policy search method for large {MDP}s and
                  {POMDP}s},
  author =	 {Ng, Andrew Y and Jordan, Michael},
  booktitle =	 {Proceedings of the16th Conference on
                  Uncertainty in Artificial Intelligence},
  pages =	 {406--415},
  year =	 2000
%  organization = {Morgan Kaufmann Publishers Inc.}
}
@article{nilim2005robust,
	title        = {Robust control of {M}arkov decision processes with uncertain transition matrices},
	author       = {Nilim, Arnab and El Ghaoui, Laurent},
	year         = 2005,
	journal      = {Operations Research},
	volume       = 53,
	number       = 5,
	pages        = {780--798}
}
@inproceedings{silver2010montea,
	title        = {{M}onte-{C}arlo planning in large {POMDP}s},
	author       = {Silver, David and Veness, Joel},
	year         = 2010,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2164--2172}
}
@inproceedings{smith2004heuristic,
	title        = {Heuristic search value iteration for {POMDP}s},
	author       = {Smith, Trey and Simmons, Reid},
	year         = 2004,
	booktitle    = {Proceedings of the 20th Conference on Uncertainty in Artificial Intelligence},
	pages        = {520--527}
}
@article{ross2011bayesian,
	title        = {A {B}ayesian approach for learning and planning in partially observable {M}arkov decision processes},
	author       = {Ross, St{\'e}phane and Pineau, Joelle and Chaib-draa, Brahim and Kreitmann, Pierre},
	year         = 2011,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR.org},
	volume       = 12,
	pages        = {1729--1770}
}
@inproceedings{bjarnason2009lower,
	title        = {Lower bounding {K}londike solitaire with {M}onte-{C}arlo planning},
	author       = {Bjarnason, Ronald and Fern, Alan and Tadepalli, Prasad},
	year         = 2009,
	booktitle    = {Proceedings of International Conference on Automated Planning and Scheduling},
	pages        = {26--33}
}
@inproceedings{zhu2009human,
	title        = {Human {R}ademacher complexity},
	author       = {Zhu, Xiaojin and Gibson, Bryan R and Rogers, Timothy T},
	year         = 2009,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2322--2330}
}
@manual{balcan2011cs,
	title        = {{CS} 8803 - {M}achine {L}earning {T}heory: {L}ecture {N}otes},
	author       = {Balcan, Maria-Florina},
	year         = 2011,
	note         = {\url{http://www.cs.cmu.edu/~ninamf/ML11/lect1115.pdf}},
	organization = {{G}eorgia {I}nstitute of {T}echnology}
}
@inproceedings{petrik2009biasing,
	title        = {Biasing approximate dynamic programming with a lower discount factor},
	author       = {Petrik, Marek and Scherrer, Bruno},
	year         = 2009,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1265--1272}
}
@article{vapnik1999overview,
	title        = {An overview of statistical learning theory},
	author       = {Vapnik, Vladimir N},
	year         = 1999,
	journal      = {Neural Networks, IEEE Transactions on},
	publisher    = {IEEE},
	volume       = 10,
	number       = 5,
	pages        = {988--999}
}
@article{baxter2001infinite,
	title        = {{Infinite-Horizon Policy-Gradient Estimation}},
	author       = {Baxter, Jonathan and Bartlett, Peter L},
	year         = 2001,
	journal      = {Journal of Artificial Intelligence Research},
	volume       = 15,
	pages        = {319--350}
}
@article{amin2017repeated,
	title        = {{Repeated Inverse Reinforcement Learning for AI Safety}},
	author       = {Amin, Kareem and Jiang, Nan and Singh, Satinder},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1705.05427},
	__markedentry = {[najiang:]}
}
@article{bostrom2003ethical,
	title        = {Ethical issues in advanced artificial intelligence},
	author       = {Bostrom, Nick},
	year         = 2003,
	journal      = {Science Fiction and Philosophy: From Time Travel to Superintelligence},
	pages        = {277--284},
	__markedentry = {[najiang:]}
}
@inproceedings{walsh2010generalizing,
	title        = {Generalizing apprenticeship learning across hypothesis classes},
	author       = {Walsh, Thomas J and Subramanian, Kaushik and Littman, Michael L and Diuk, Carlos},
	year         = 2010,
	booktitle    = {Proceedings of the 27th International Conference on Machine Learning (ICML-10)},
	pages        = {1119--1126},
	__markedentry = {[najiang:]}
}
@article{li2011knows,
	title        = {Knows what it knows: a framework for self-aware learning},
	author       = {Li, Lihong and Littman, Michael L and Walsh, Thomas J and Strehl, Alexander L},
	year         = 2011,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 82,
	number       = 3,
	pages        = {399--443},
	__markedentry = {[najiang:]}
}
@inproceedings{dani2008stochastic,
	title        = {Stochastic Linear Optimization under Bandit Feedback.},
	author       = {Dani, Varsha and Hayes, Thomas P and Kakade, Sham M},
	year         = 2008,
	booktitle    = {COLT},
	pages        = {355--366},
	__markedentry = {[najiang:]}
}
@inproceedings{abbasi-yadkori2012online,
	title        = {Online-to-Confidence-Set Conversions and Application to Sparse Stochastic Bandits.},
	author       = {Abbasi-Yadkori, Yasin and Pal, David and Szepesvari, Csaba},
	year         = 2012,
	booktitle    = {AISTATS},
	volume       = 22,
	pages        = {1--9},
	__markedentry = {[najiang:]}
}
@article{mannor2004sample,
	title        = {The sample complexity of exploration in the multi-armed bandit problem},
	author       = {Mannor, Shie and Tsitsiklis, John N},
	year         = 2004,
	journal      = {Journal of Machine Learning Research},
	volume       = 5,
	number       = {Jun},
	pages        = {623--648},
	__markedentry = {[najiang:]}
}
@article{auer2002using,
	title        = {Using confidence bounds for exploitation-exploration trade-offs},
	author       = {Auer, Peter},
	year         = 2002,
	journal      = {Journal of Machine Learning Research},
	volume       = 3,
	number       = {Nov},
	pages        = {397--422},
	__markedentry = {[najiang:]}
}
@inproceedings{neu2014online,
	title        = {Online combinatorial optimization with stochastic decision sets and adversarial losses},
	author       = {Neu, Gergely and Valko, Michal},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {2780--2788},
	__markedentry = {[najiang:]}
}
@article{kleinberg2010regret,
	title        = {Regret bounds for sleeping experts and bandits},
	author       = {Kleinberg, Robert and Niculescu-Mizil, Alexandru and Sharma, Yogeshwer},
	year         = 2010,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 80,
	number       = {2-3},
	pages        = {245--272},
	__markedentry = {[najiang:]}
}
@article{blum2007external,
	title        = {From external to internal regret},
	author       = {Blum, Avrim and Mansour, Yishay},
	year         = 2007,
	journal      = {Journal of Machine Learning Research},
	volume       = 8,
	number       = {Jun},
	pages        = {1307--1324},
	__markedentry = {[najiang:]}
}
@inproceedings{freund1997using,
	title        = {Using and combining predictors that specialize},
	author       = {Freund, Yoav and Schapire, Robert E and Singer, Yoram and Warmuth, Manfred K},
	year         = 1997,
	booktitle    = {Proceedings of the 29th Annual ACM Symposium on Theory of computing},
	pages        = {334--343},
	organization = {ACM},
	__markedentry = {[najiang:]}
}
@inproceedings{kanade2009sleeping,
	title        = {Sleeping Experts and Bandits with Stochastic Action Availability and Adversarial Rewards},
	author       = {Kanade, Varun and McMahan, H Brendan and Bryan, Brent},
	year         = 2009,
	booktitle    = {AISTATS},
	pages        = {272--279},
	__markedentry = {[najiang:]}
}
@book{groetschel2012geometric,
	title        = {Geometric algorithms and combinatorial optimization},
	author       = {Gr{\"o}tschel, Martin and Lov{\'a}sz, L{\'a}szl{\'o} and Schrijver, Alexander},
	year         = 2012,
	publisher    = {Springer Science \& Business Media},
	volume       = 2,
	__markedentry = {[najiang:]}
}
@inproceedings{hadfield-menell2016cooperative,
	title        = {Cooperative inverse reinforcement learning},
	author       = {Hadfield-Menell, Dylan and Russell, Stuart J and Abbeel, Pieter and Dragan, Anca},
	year         = 2016,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {3909--3917},
	__markedentry = {[najiang:]}
}
@article{russell2015research,
	title        = {Research priorities for robust and beneficial artificial intelligence},
	author       = {Russell, Stuart and Dewey, Daniel and Tegmark, Max},
	year         = 2015,
	journal      = {AI Magazine},
	volume       = 36,
	number       = 4,
	pages        = {105--114},
	__markedentry = {[najiang:]}
}
@article{amodei2016concrete,
	title        = {Concrete problems in {AI} safety},
	author       = {Amodei, Dario and Olah, Chris and Steinhardt, Jacob and Christiano, Paul and Schulman, John and Man{\'e}, Dan},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1606.06565},
	__markedentry = {[najiang:]}
}
@article{todd1982minimum,
	title        = {On minimum volume ellipsoids containing part of a given ellipsoid},
	author       = {Todd, Michael J},
	year         = 1982,
	journal      = {Mathematics of Operations Research},
	publisher    = {INFORMS},
	volume       = 7,
	number       = 2,
	pages        = {253--261},
	__markedentry = {[najiang:]}
}
@inproceedings{auer1995gambling,
	title        = {Gambling in a rigged casino: The adversarial multi-armed bandit problem},
	author       = {Auer, Peter and Cesa-Bianchi, Nicolo and Freund, Yoav and Schapire, Robert E},
	year         = 1995,
	booktitle    = {Foundations of Computer Science, 1995. Proceedings., 36th Annual Symposium on},
	pages        = {322--331},
	organization = {IEEE},
	__markedentry = {[najiang:]}
}
@article{bartlett2008high,
	title        = {High-probability regret bounds for bandit online linear optimization},
	author       = {Bartlett, Peter L and Dani, Varsha and Hayes, Thomas and Kakade, Sham and Rakhlin, Alexander and Tewari, Ambuj},
	year         = 2008,
	booktitle    = {Proceedings of the 21st Annual Conference on Learning Theory},
	pages        = {335--342},
	__markedentry = {[najiang:]}
}
@inproceedings{abernethy2008competing,
	title        = {Competing in the Dark: An Efficient Algorithm for Bandit Linear Optimization},
	author       = {Abernethy, Jacob and Hazan, Elad and Rakhlin, Alexander},
	year         = 2008,
	booktitle    = {Conference on Learning Theory},
	pages        = {263--274},
	__markedentry = {[najiang:]}
}
@inproceedings{dani2007price,
	title        = {The price of bandit information for online optimization},
	author       = {Dani, Varsha and Kakade, Sham M and Hayes, Thomas P},
	year         = 2007,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {345--352},
	__markedentry = {[najiang:]}
}
@article{shalev-shwartz2011online,
	title        = {Online learning and online convex optimization},
	author       = {Shalev-Shwartz, Shai},
	year         = 2011,
	journal      = {Foundations and Trends in Machine Learning},
	volume       = 4,
	number       = 2,
	pages        = {107--194},
	__markedentry = {[najiang:]}
}
@inproceedings{hsu2015mixing,
	title        = {Mixing time estimation in reversible Markov chains from a single sample path},
	author       = {Hsu, Daniel J and Kontorovich, Aryeh and Szepesv{\'a}ri, Csaba},
	year         = 2015,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1459--1467},
	__markedentry = {[najiang:]}
}
@manual{odonnell201115,
	title        = {15-859({E}) -- linear and semidefinite programming: lecture notes},
	author       = {Ryan O'Donnell},
	year         = 2011,
	note         = {\url{https://www.cs.cmu.edu/afs/cs.cmu.edu/academic/class/15859-f11/www/notes/lecture08.pdf}},
	organization = {{Carnegie Mellon University}},
	__markedentry = {[najiang:]}
}
@inproceedings{jiang2015dependence,
	title        = {The Dependence of Effective Planning Horizon on Model Accuracy},
	author       = {Jiang, Nan and Kulesza, Alex and Singh, Satinder and Lewis, Richard},
	year         = 2015,
	booktitle    = {Proceedings of the 2015 International Conference on Autonomous Agents and Multiagent Systems},
	pages        = {1181--1189},
	__markedentry = {[najiang:]}
}
@inproceedings{ng2000algorithms,
	title        = {Algorithms for Inverse Reinforcement Learning},
	author       = {Ng, Andrew Y and Russell, Stuart J},
	year         = 2000,
	booktitle    = {Proceedings of the 17th International Conference on Machine Learning},
	pages        = {663--670},
	__markedentry = {[najiang:]}
}
@inproceedings{abbeel2004apprenticeship,
	title        = {{Apprenticeship Learning via Inverse Reinforcement Learning}},
	author       = {Abbeel, Pieter and Ng, Andrew Y},
	year         = 2004,
	booktitle    = {Proceedings of the 21st International Conference on Machine learning},
	pages        = 1,
	organization = {ACM},
	__markedentry = {[najiang:]}
}
@article{amin2016towards,
	title        = {Towards Resolving Unidentifiability in Inverse Reinforcement Learning},
	author       = {Amin, Kareem and Singh, Satinder},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1601.06569},
	__markedentry = {[najiang:]}
}
@article{lovasz1999hit,
	title        = {Hit-and-run mixes fast},
	author       = {Lov{\'a}sz, L{\'a}szl{\'o}},
	year         = 1999,
	journal      = {Mathematical Programming},
	publisher    = {Springer},
	volume       = 86,
	number       = 3,
	pages        = {443--461},
	__markedentry = {[najiang:]}
}
@article{lovasz2006simulated,
	title        = {Simulated annealing in convex bodies and an O*(n4) volume algorithm},
	author       = {Lov{\'a}sz, L{\'a}szl{\'o} and Vempala, Santosh},
	year         = 2006,
	journal      = {Journal of Computer and System Sciences},
	publisher    = {Elsevier},
	volume       = 72,
	number       = 2,
	pages        = {392--417},
	__markedentry = {[najiang:]}
}
@article{cousinspractical,
	title        = {A Practical Volume Algorithm},
	author       = {Cousins, Ben and Vempala, Santosh},
	__markedentry = {[najiang:]}
}
@inproceedings{coates2008learning,
	title        = {Learning for control from multiple demonstrations},
	author       = {Coates, Adam and Abbeel, Pieter and Ng, Andrew Y},
	year         = 2008,
	booktitle    = {Proceedings of the 25th international conference on Machine learning},
	pages        = {144--151},
	organization = {ACM},
	__markedentry = {[najiang:]}
}
@inproceedings{ziebart2008maximum,
	title        = {Maximum Entropy Inverse Reinforcement Learning},
	author       = {Ziebart, Brian D and Maas, Andrew L and Bagnell, J Andrew and Dey, Anind K},
	year         = 2008,
	booktitle    = {AAAI},
	pages        = {1433--1438},
	__markedentry = {[najiang:]}
}
@article{ramachandran2007bayesian,
	title        = {Bayesian inverse reinforcement learning},
	author       = {Ramachandran, Deepak and Amir, Eyal},
	year         = 2007,
	journal      = {Urbana},
	volume       = 51,
	pages        = 61801,
	__markedentry = {[najiang:]}
}
@inproceedings{syed2007game,
	title        = {A game-theoretic approach to apprenticeship learning},
	author       = {Syed, Umar and Schapire, Robert E},
	year         = 2007,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1449--1456},
	__markedentry = {[najiang:]}
}
@inproceedings{guillory2010interactive,
	title        = {Interactive submodular set cover},
	author       = {Guillory, Andrew and Bilmes, Jeff},
	year         = 2010,
	booktitle    = {Proceedings of the International Conference on Machine Learning},
	__markedentry = {[najiang:]}
}
@inproceedings{golovin2010adaptive,
	title        = {Adaptive Submodularity: A New Approach to Active Learning and Stochastic Optimization},
	author       = {Golovin, Daniel and Krause, Andreas},
	year         = 2010,
	booktitle    = {COLT},
	pages        = {333--345},
	__markedentry = {[najiang:]}
}
@inproceedings{ratliff2006maximum,
	title        = {Maximum margin planning},
	author       = {Ratliff, Nathan D and Bagnell, J Andrew and Zinkevich, Martin A},
	year         = 2006,
	booktitle    = {Proceedings of the 23rd International Conference on Machine Learning},
	pages        = {729--736},
	organization = {ACM},
	__markedentry = {[najiang:]}
}
@inproceedings{regan2010robust,
	title        = {Robust Policy Computation in Reward-Uncertain MDPs Using Nondominated Policies},
	author       = {Regan, Kevin and Boutilier, Craig},
	year         = 2010,
	booktitle    = {AAAI},
	__markedentry = {[najiang:]}
}
@inproceedings{smart2002effective,
	title        = {Effective reinforcement learning for mobile robots},
	author       = {Smart, William D and Kaelbling, Leslie Pack},
	year         = 2002,
	booktitle    = {Robotics and Automation, 2002. Proceedings. ICRA'02. IEEE International Conference on},
	volume       = 4,
	pages        = {3404--3410},
	organization = {IEEE},
	__markedentry = {[najiang:]}
}
@article{abbeel2007application,
	title        = {An application of reinforcement learning to aerobatic helicopter flight},
	author       = {Abbeel, Pieter and Coates, Adam and Quigley, Morgan and Ng, Andrew Y},
	year         = 2007,
	journal      = {Advances in Neural Information Processing Systems},
	publisher    = {MIT; 1998},
	volume       = 19,
	pages        = 1,
	__markedentry = {[najiang:]}
}
@article{coates2009apprenticeship,
	title        = {Apprenticeship learning for helicopter control},
	author       = {Coates, Adam and Abbeel, Pieter and Ng, Andrew Y},
	year         = 2009,
	journal      = {Communications of the ACM},
	publisher    = {ACM},
	volume       = 52,
	number       = 7,
	pages        = {97--105},
	__markedentry = {[najiang:]}
}
@book{vonneumann2007theory,
	title        = {Theory of games and economic behavior (60th Anniversary Commemorative Edition)},
	author       = {Von Neumann, John and Morgenstern, Oskar},
	year         = 2007,
	publisher    = {Princeton university press},
	__markedentry = {[najiang:]}
}
@inproceedings{chajewska2000making,
	title        = {Making rational decisions using adaptive utility elicitation},
	author       = {Chajewska, Urszula and Koller, Daphne and Parr, Ronald},
	year         = 2000,
	booktitle    = {AAAI/IAAI},
	pages        = {363--369},
	__markedentry = {[najiang:]}
}
@incollection{balcan2007margin,
	title        = {Margin based active learning},
	author       = {Balcan, Maria-Florina and Broder, Andrei and Zhang, Tong},
	year         = 2007,
	booktitle    = {Learning Theory},
	publisher    = {Springer},
	pages        = {35--50},
	__markedentry = {[najiang:]}
}
@article{balcan2012active,
	title        = {Active and passive learning of linear separators under log-concave distributions},
	author       = {Balcan, Maria Florina and Long, Philip M},
	year         = 2012,
	journal      = {arXiv preprint arXiv:1211.1082},
	__markedentry = {[najiang:]}
}
@inproceedings{dasgupta2005coarse,
	title        = {Coarse sample complexity bounds for active learning},
	author       = {Dasgupta, Sanjoy},
	year         = 2005,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {235--242},
	__markedentry = {[najiang:]}
}
@inproceedings{regan2009regret,
	title        = {Regret-based reward elicitation for Markov decision processes},
	author       = {Regan, Kevin and Boutilier, Craig},
	year         = 2009,
	booktitle    = {Proceedings of the Twenty-Fifth Conference on Uncertainty in Artificial Intelligence},
	pages        = {444--451},
	organization = {AUAI Press},
	__markedentry = {[najiang:]}
}
@incollection{lopes2009active,
	title        = {Active learning for reward estimation in inverse reinforcement learning},
	author       = {Lopes, Manuel and Melo, Francisco and Montesano, Luis},
	year         = 2009,
	booktitle    = {Machine Learning and Knowledge Discovery in Databases},
	publisher    = {Springer},
	pages        = {31--46},
	__markedentry = {[najiang:]}
}
@article{baranes2013active,
	title        = {Active learning of inverse models with intrinsically motivated goal exploration in robots},
	author       = {Baranes, Adrien and Oudeyer, Pierre-Yves},
	year         = 2013,
	journal      = {Robotics and Autonomous Systems},
	publisher    = {Elsevier},
	volume       = 61,
	number       = 1,
	pages        = {49--73},
	__markedentry = {[najiang:]}
}
@incollection{rothkopf2011preference,
	title        = {Preference elicitation and inverse reinforcement learning},
	author       = {Rothkopf, Constantin A and Dimitrakakis, Christos},
	year         = 2011,
	booktitle    = {Machine Learning and Knowledge Discovery in Databases},
	publisher    = {Springer},
	pages        = {34--48},
	__markedentry = {[najiang:]}
}
@inproceedings{langford2005relating,
	title        = {Relating reinforcement learning performance to classification performance},
	author       = {Langford, John and Zadrozny, Bianca},
	year         = 2005,
	booktitle    = {Proceedings of the 22nd international conference on Machine learning},
	pages        = {473--480},
	organization = {ACM},
	__markedentry = {[najiang:]}
}
@inproceedings{lagoudakis2003reinforcement,
	title        = {Reinforcement learning as classification: Leveraging modern classifiers},
	author       = {Lagoudakis, Michail and Parr, Ronald},
	year         = 2003,
	booktitle    = {ICML},
	volume       = 3,
	pages        = {424--431},
	__markedentry = {[najiang:]}
}
@book{boyd2004convex,
	title        = {Convex optimization},
	author       = {Boyd, Stephen and Vandenberghe, Lieven},
	year         = 2004,
	publisher    = {Cambridge university press},
	__markedentry = {[najiang:]}
}
@article{elghaoui2002inversion,
	title        = {Inversion error, condition number, and approximate inverses of uncertain matrices},
	author       = {El Ghaoui, Laurent},
	year         = 2002,
	journal      = {Linear algebra and its applications},
	publisher    = {Elsevier},
	volume       = 343,
	pages        = {171--193},
	__markedentry = {[najiang:]}
}
@book{anthony2009neural,
	title        = {Neural network learning: Theoretical foundations},
	author       = {Anthony, Martin and Bartlett, Peter L},
	year         = 2009,
	publisher    = {cambridge university press},
	__markedentry = {[najiang:]}
}
@article{massart2006risk,
	title        = {Risk bounds for statistical learning},
	author       = {Massart, Pascal and N{\'e}d{\'e}lec, {\'E}lodie},
	year         = 2006,
	journal      = {The Annals of Statistics},
	publisher    = {JSTOR},
	pages        = {2326--2366},
	__markedentry = {[najiang:]}
}
@inproceedings{regan2011eliciting,
	title        = {Eliciting additive reward functions for Markov decision processes},
	author       = {Regan, Kevin and Boutilier, Craig},
	year         = 2011,
	booktitle    = {IJCAI Proceedings-International Joint Conference on Artificial Intelligence},
	volume       = 22,
	number       = 3,
	pages        = 2159,
	__markedentry = {[najiang:]}
}
@article{kocsis2006improved,
	title        = {Improved {M}onte-{C}arlo search},
	author       = {Kocsis, Levente and Szepesv{\'a}ri, Csaba and Willemson, Jan},
	year         = 2006,
	journal      = {Univ. Tartu, Estonia, Tech. Rep},
	volume       = 1,
	__markedentry = {[najiang:6]}
}
@inproceedings{wang2007modifications,
	title        = {Modifications of {UCT} and sequence-like simulations for {M}onte-{C}arlo Go},
	author       = {Wang, Yizao and Gelly, Sylvain},
	year         = 2007,
	booktitle    = {IEEE Symposium on Computational Intelligence and Games},
	pages        = {175--182},
	__markedentry = {[najiang:6]}
}
@article{silver2010monte,
	title        = {{M}onte-{C}arlo planning in large {POMDP}s},
	author       = {Silver, David and Veness, Joel},
	year         = 2010,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 23,
	pages        = {2164--2172},
	__markedentry = {[najiang:6]}
}
@inproceedings{coueetoux2011adding,
	title        = {Adding Double Progressive Widening to Upper Confidence Trees to Cope with Uncertainty in Planning Problems},
	author       = {Cou{\"e}toux, Adrien and Doghmen, Hassen and others},
	year         = 2011,
	booktitle    = {9th European Workshop on Reinforcement Learning},
	__markedentry = {[najiang:6]}
}
@incollection{coueetoux2011continuous,
	title        = {Continuous upper confidence trees},
	author       = {Cou{\"e}toux, Adrien and Hoock, Jean-Baptiste and Sokolovska, Nataliya and Teytaud, Olivier and Bonnard, Nicolas},
	year         = 2011,
	booktitle    = {Learning and Intelligent Optimization},
	publisher    = {Springer},
	pages        = {433--445},
	__markedentry = {[najiang:6]}
}
@article{tolpin2012mcts,
	title        = {{MCTS} based on simple regret},
	author       = {Tolpin, David and Shimony, Solomon Eyal},
	year         = 2012,
	journal      = {AAAI. AAAI Press, 2012a. To appear},
	__markedentry = {[najiang:6]}
}
@inproceedings{hingston2007experiments,
	title        = {Experiments with {M}onte-{C}arlo {O}thello},
	author       = {Hingston, Philip and Masek, Martin},
	year         = 2007,
	booktitle    = {Evolutionary Computation},
	pages        = {4059--4064},
	organization = {IEEE},
	__markedentry = {[najiang:6]}
}
@article{rasmussen2004gaussian,
	title        = {Gaussian processes in reinforcement learning},
	author       = {Rasmussen, Carl Edward and Kuss, Malte},
	year         = 2004,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 16,
	__markedentry = {[najiang:6]}
}
@article{gelly2011monte,
	title        = {{M}onte-{C}arlo tree search and rapid action value estimation in computer Go},
	author       = {Gelly, Sylvain and Silver, David},
	year         = 2011,
	journal      = {Artificial Intelligence},
	publisher    = {Elsevier},
	volume       = 175,
	number       = 11,
	pages        = {1856--1875},
	__markedentry = {[najiang:6]}
}
@inproceedings{narayanamurthy2007efficiently,
	title        = {Efficiently Exploiting Symmetries in Real Time Dynamic Programming},
	author       = {Narayanamurthy, Shravan Matthur and Ravindran, Balaraman},
	year         = 2007,
	booktitle    = {International Joint Conference on Artificial Intelligence},
	pages        = {2556--2561},
	__markedentry = {[najiang:6]}
}
@article{auer2002finite,
	title        = {Finite-time analysis of the multiarmed bandit problem},
	author       = {Auer, Peter and Cesa-Bianchi, Nicol{\`o} and Fischer, Paul},
	year         = 2002,
	journal      = {Machine learning},
	publisher    = {Springer},
	volume       = 47,
	number       = {2-3},
	pages        = {235--256}
}
@article{kearns2002sparse,
	title        = {A sparse sampling algorithm for near-optimal planning in large {M}arkov decision processes},
	author       = {Kearns, Michael and Mansour, Yishay and Ng, Andrew Y},
	year         = 2002,
	journal      = {Machine Learning},
	publisher    = {Springer},
	volume       = 49,
	number       = {2-3},
	pages        = {193--208}
}
@inproceedings{konda2000actor,
	title        = {Actor-critic algorithms},
	author       = {Konda, Vijay R and Tsitsiklis, John N},
	year         = 2000,
	booktitle    = {Advances in neural information processing systems},
	pages        = {1008--1014}
}
@article{lazaric2012finite,
	title        = {Finite-sample analysis of least-squares policy iteration},
	author       = {Lazaric, Alessandro and Ghavamzadeh, Mohammad and Munos, R{\'e}mi},
	year         = 2012,
	journal      = {Journal of Machine Learning Research},
	volume       = 13,
	number       = {Oct},
	pages        = {3041--3074}
}
@article{pires2012statistical,
	title        = {Statistical linear estimation with penalized estimators: an application to reinforcement learning},
	author       = {Pires, Bernardo Avila and Szepesv{\'a}ri, Csaba},
	year         = 2012,
	journal      = {arXiv preprint arXiv:1206.6444}
}
@article{farahmand2016regularized,
	title        = {Regularized policy iteration with nonparametric function spaces},
	author       = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
	year         = 2016,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR. org},
	volume       = 17,
	number       = 1,
	pages        = {4809--4874}
}
@article{cheng2019trajectory,
	title        = {Trajectory-wise control variates for variance reduction in policy gradient methods},
	author       = {Cheng, Ching-An and Yan, Xinyan and Boots, Byron},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1908.03263}
}
@article{wang2020statistical,
	title        = {What are the Statistical Limits of Offline RL with Linear Function Approximation?},
	author       = {Wang, Ruosong and Foster, Dean P and Kakade, Sham M},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2010.11895}
}
@article{amortila2020variant,
	title        = {A Variant of the Wang-Foster-Kakade Lower Bound for the Discounted Setting},
	author       = {Amortila, Philip and Jiang, Nan and Xie, Tengyang},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2011.01075}
}
@inproceedings{laroche2019safe,
	title        = {Safe policy improvement with baseline bootstrapping},
	author       = {Laroche, Romain and Trichelair, Paul and Des Combes, Remi Tachet},
	year         = 2019,
	booktitle    = {International Conference on Machine Learning},
	pages        = {3652--3661},
	organization = {PMLR}
}
@inproceedings{farajtabar2018more,
	title        = {More robust doubly robust off-policy evaluation},
	author       = {Farajtabar, Mehrdad and Chow, Yinlam and Ghavamzadeh, Mohammad},
	year         = 2018,
	booktitle    = {International Conference on Machine Learning},
	pages        = {1447--1456},
	organization = {PMLR}
}
@phdthesis{paduraru2012off,
	title        = {Off-policy evaluation in Markov decision processes},
	author       = {Paduraru, Cosmin},
	year         = 2012
}
@inproceedings{xie2020batch,
	title        = {Batch value-function approximation with only realizability},
	author       = {Xie, Tengyang and Jiang, Nan},
	year         = 2021,
	booktitle    = {International Conference on Machine Learning},
	organization = {PMLR}
}
@article{vermeulen2015bias,
	title        = {Bias-reduced doubly robust estimation},
	author       = {Vermeulen, Karel and Vansteelandt, Stijn},
	year         = 2015,
	journal      = {Journal of the American Statistical Association},
	publisher    = {Taylor \& Francis},
	volume       = 110,
	number       = 511,
	pages        = {1024--1036}
}
@inproceedings{cutler2015efficient,
	title        = {Efficient reinforcement learning for robots using informative simulated priors},
	author       = {Cutler, Mark and How, Jonathan P},
	year         = 2015,
	booktitle    = {2015 IEEE International Conference on Robotics and Automation (ICRA)},
	pages        = {2605--2612},
	organization = {IEEE}
}
@inproceedings{jiang2018completing,
	title        = {Completing State representations using spectral learning},
	author       = {Jiang, Nan and Kulesza, Alex and Singh, Satinder},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {4328--4337}
}
@inproceedings{du2019provably,
	title        = {{Provably efficient RL with Rich Observations via Latent State Decoding}},
	author       = {Du, Simon and Krishnamurthy, Akshay and Jiang, Nan and Agarwal, Alekh and Dudik, Miroslav and Langford, John},
	year         = 2019,
	booktitle    = {International Conference on Machine Learning},
	pages        = {1665--1674}
}
@article{voloshin2019empirical,
	title        = {Empirical Study of Off-Policy Policy Evaluation for Reinforcement Learning},
	author       = {Voloshin, Cameron and Le, Hoang M and Jiang, Nan and Yue, Yisong},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1911.06854}
}
@inproceedings{ng2000algorithms,
	title        = {Algorithms for inverse reinforcement learning},
	author       = {Ng, Andrew Y and Russell, Stuart J},
	year         = 2000,
	booktitle    = {Proceedings of the 17th International Conference on Machine Learning},
	pages        = {663--670}
}
@article{amodei2016concrete,
	title        = {Concrete problems in {AI} safety},
	author       = {Amodei, Dario and Olah, Chris and Steinhardt, Jacob and Christiano, Paul and Schulman, John and Man{\'e}, Dan},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1606.06565}
}
@inproceedings{thomas2016data,
	title        = {Data-Efficient Off-Policy Policy Evaluation for Reinforcement Learning},
	author       = {Thomas, Philip S and Brunskill, Emma},
	year         = 2016,
	booktitle    = {Proceedings of the 33rd International Conference on Machine Learning},
	pages        = {2139--2148}
}
@article{farahmand2011regularization,
	title        = {Regularization in reinforcement learning},
	author       = {Farahmand, Amir-massoud},
	year         = 2011,
	publisher    = {University of Alberta}
}
@inproceedings{schaul2013better,
	title        = {Better generalization with forecasts},
	author       = {Schaul, Tom and Ring, Mark},
	year         = 2013,
	booktitle    = {Proceedings of the 23rd International Joint Conference on Artificial Intelligence},
	pages        = {1656--1662},
	organization = {AAAI Press}
}
@article{weston2014memory,
	title        = {Memory networks},
	author       = {Weston, Jason and Chopra, Sumit and Bordes, Antoine},
	year         = 2014,
	journal      = {arXiv preprint arXiv:1410.3916}
}
@inproceedings{boots2010hilbert,
	title        = {{Hilbert Space Embeddings of Hidden Markov Models}},
	author       = {Boots, Byron and Siddiqi, Sajid M and Gordon, Geoffrey and Smola, Alex},
	year         = 2010,
	booktitle    = {Proceedings of the 27th International Conference on Machine Learning}
}
@inproceedings{wen2013efficient,
	title        = {Efficient exploration and value function generalization in deterministic systems},
	author       = {Wen, Zheng and Van Roy, Benjamin},
	year         = 2013,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {3021--3029}
}
@inproceedings{abbeel2004apprenticeship,
	title        = {{Apprenticeship Learning via Inverse Reinforcement Learning}},
	author       = {Abbeel, Pieter and Ng, Andrew Y},
	year         = 2004,
	booktitle    = {Proceedings of the 21st International Conference on Machine learning},
	pages        = 1,
	organization = {ACM}
}
@article{krishnamurthy2016contextual,
	title        = {{Contextual-MDPs for PAC-Reinforcement Learning with Rich Observations}},
	author       = {Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1602.02722}
}
@inproceedings{zhang2014spectral,
	title        = {{Spectral methods meet EM: A provably optimal algorithm for crowdsourcing}},
	author       = {Zhang, Yuchen and Chen, Xi and Zhou, Denny and Jordan, Michael I},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1260--1268}
}
@inproceedings{rosencrantz2004learning,
	title        = {Learning low dimensional predictive representations},
	author       = {Rosencrantz, Matthew and Gordon, Geoff and Thrun, Sebastian},
	year         = 2004,
	booktitle    = {Proceedings of the 21st International Conference on Machine Learning},
	pages        = 88,
	organization = {ACM}
}
@inproceedings{kulesza2015spectral,
	title        = {Spectral learning of predictive state representations with insufficient statistics},
	author       = {Kulesza, Alex and Jiang, Nan and Singh, Satinder},
	year         = 2015,
	booktitle    = {Proceedings of the 29th AAAI Conference on Artificial Intelligence}
}
@inproceedings{jiang2016improving,
	title        = {Improving Predictive State Representations via Gradient Descent},
	author       = {Jiang, Nan and Kulesza, Alex and Singh, Satinder},
	year         = 2016,
	booktitle    = {Proceedings of the 30th AAAI Conference on Artificial Intelligence}
}
@inproceedings{jiang2016structural,
	title        = {On Structural Properties of MDPs that Bound Loss due to Shallow Planning},
	author       = {Jiang, Nan and Singh, Satinder and Tewari, Ambuj},
	year         = 2016,
	booktitle    = {Proceedings of the 25th International Joint Conference on Artificial Intelligence}
}
@article{boots2011closing,
	title        = {Closing the learning-planning loop with predictive state representations},
	author       = {Boots, Byron and Siddiqi, Sajid M and Gordon, Geoffrey J},
	year         = 2011,
	journal      = {The International Journal of Robotics Research},
	publisher    = {SAGE Publications},
	volume       = 30,
	number       = 7,
	pages        = {954--966}
}
@inproceedings{kulesza2014low,
	title        = {Low-Rank Spectral Learning},
	author       = {Kulesza, Alex and Rao, N Raj and Singh, Satinder},
	year         = 2014,
	booktitle    = {Proceedings of the 17th International Conference on Artificial Intelligence and Statistics},
	pages        = {522--530}
}
@inproceedings{singh2004predictive,
	title        = {Predictive state representations: A new theory for modeling dynamical systems},
	author       = {Singh, Satinder and James, Michael R and Rudary, Matthew R},
	year         = 2004,
	booktitle    = {Proceedings of the 20th Conference on Uncertainty in Artificial Intelligence},
	pages        = {512--519},
	organization = {AUAI Press}
}
@inproceedings{jiang2015abstraction,
	title        = {Abstraction Selection in Model-based Reinforcement Learning},
	author       = {Jiang, Nan and Kulesza, Alex and Singh, Satinder},
	year         = 2015,
	booktitle    = {Proceedings of the 32nd International Conference on Machine Learning},
	pages        = {179--188}
}
@inproceedings{jiang2014improving,
	title        = {Improving {UCT} Planning via Approximate Homomorphisms},
	author       = {Jiang, Nan and Singh, Satinder and Lewis, Richard},
	year         = 2014,
	booktitle    = {Proceedings of the 2014 International Conference on Autonomous Agents and Multiagent Systems},
	pages        = {1289--1296}
}
@inproceedings{jiang2015dependence,
	title        = {The Dependence of Effective Planning Horizon on Model Accuracy},
	author       = {Jiang, Nan and Kulesza, Alex and Singh, Satinder and Lewis, Richard},
	year         = 2015,
	booktitle    = {Proceedings of the 2015 International Conference on Autonomous Agents and Multiagent Systems},
	pages        = {1181--1189}
}
@article{kober2013reinforcement,
	title        = {{Reinforcement learning in robotics: A survey}},
	author       = {Kober, Jens and Bagnell, J Andrew and Peters, Jan},
	year         = 2013,
	journal      = {The International Journal of Robotics Research},
	publisher    = {SAGE Publications},
	pages        = {0278364913495721}
}
@inproceedings{petrik2014raam,
	title        = {{RAAM: The Benefits of Robustness in Approximating Aggregated MDPs in Reinforcement Learning}},
	author       = {Petrik, Marek and Subramanian, Dharmashankar},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {1979--1987}
}
@article{van2006performance,
	title        = {Performance loss bounds for approximate value iteration with state aggregation},
	author       = {Van Roy, Benjamin},
	year         = 2006,
	journal      = {Mathematics of Operations Research},
	publisher    = {INFORMS},
	volume       = 31,
	number       = 2,
	pages        = {234--244}
}
@inproceedings{kulesza2015low,
	title        = {Low-Rank Spectral Learning with Weighted Loss Functions},
	author       = {Kulesza, Alex and Jiang, Nan and Singh, Satinder},
	year         = 2015,
	booktitle    = {Proceedings of the 18th International Conference on Artificial Intelligence and Statistics},
	pages        = {517--525}
}
@incollection{riedmiller2005neural,
	title        = {{Neural fitted Q iteration--first experiences with a data efficient neural reinforcement learning method}},
	author       = {Riedmiller, Martin},
	year         = 2005,
	booktitle    = {Machine Learning: ECML 2005},
	publisher    = {Springer},
	pages        = {317--328}
}
@inproceedings{kobilarov2015sample,
	title        = {{Sample Complexity Bounds for Iterative Stochastic Policy Optimization}},
	author       = {Kobilarov, Marin},
	year         = 2015,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {3096--3104}
}
@phdthesis{scott2004dyadic,
	title        = {Dyadic decision trees},
	author       = {Scott, Clayton D},
	year         = 2004,
	school       = {University of Wisconsin at Madison}
}
@article{bellman1956dynamic,
	title        = {{Dynamic programming and Lagrange multipliers}},
	author       = {Bellman, Richard},
	year         = 1956,
	journal      = {Proceedings of the National Academy of Sciences},
	publisher    = {National Acad Sciences},
	volume       = 42,
	number       = 10,
	pages        = {767--769}
}
@book{kumar2012customer,
	title        = {{Customer relationship management: Concept, strategy, and tools}},
	author       = {Kumar, Vineet and Reinartz, Werner},
	year         = 2012,
	publisher    = {Springer Science \& Business Media}
}
@article{singh2002optimizing,
	title        = {{Optimizing dialogue management with reinforcement learning: Experiments with the NJFun system}},
	author       = {Singh, Satinder and Litman, Diane and Kearns, Michael and Walker, Marilyn},
	year         = 2002,
	journal      = {Journal of Artificial Intelligence Research},
	pages        = {105--133}
}
@article{murphy2003optimal,
	title        = {Optimal dynamic treatment regimes},
	author       = {Murphy, Susan A},
	year         = 2003,
	journal      = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
	publisher    = {Wiley Online Library},
	volume       = 65,
	number       = 2,
	pages        = {331--355}
}
@inproceedings{hamilton2013modelling,
	title        = {Modelling sparse dynamical systems with compressed predictive state representations},
	author       = {Hamilton, William L and Fard, Mahdi M and Pineau, Joelle},
	year         = 2013,
	booktitle    = {Proceedings of the 30th International Conference on Machine Learning},
	pages        = {178--186}
}
@inproceedings{boots2010closing,
	title        = {Closing the learning-planning loop with predictive state representations},
	author       = {Boots, Byron and Siddiqi, Sajid M and Gordon, Geoffrey J},
	year         = 2010,
	booktitle    = {Proceedings of the 9th International Conference on Autonomous Agents and Multiagent Systems},
	pages        = {1369--1370}
}
@incollection{farahmand2008regularized,
	title        = {{Regularized fitted Q-iteration: Application to planning}},
	author       = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
	year         = 2008,
	booktitle    = {Recent Advances in Reinforcement Learning},
	publisher    = {Springer Berlin Heidelberg},
	pages        = {55--68}
}
@inproceedings{farahmand2009regularized,
	title        = {{Regularized fitted Q-iteration for planning in continuous-space Markovian decision problems}},
	author       = {Farahmand, Amir-massoud and Ghavamzadeh, Mohammad and Szepesv{\'a}ri, Csaba and Mannor, Shie},
	year         = 2009,
	booktitle    = {American Control Conference, 2009. ACC'09.},
	pages        = {725--730},
	organization = {IEEE}
}
@inproceedings{szepesvari2005finite,
	title        = {Finite time bounds for sampling based fitted value iteration},
	author       = {Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
	year         = 2005,
	booktitle    = {Proceedings of the 22nd International Conference on Machine Learning},
	pages        = {880--887},
	organization = {ACM}
}
@article{antos2008learning,
	title        = {Learning near-optimal policies with Bellman-residual minimization based fitted policy iteration and a single sample path},
	author       = {Antos, Andr{\'a}s and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
	year         = 2008,
	journal      = {Machine Learning},
	publisher    = {Springer},
	volume       = 71,
	number       = 1,
	pages        = {89--129}
}
@article{lazaric2012finite,
	title        = {Finite-sample analysis of least-squares policy iteration},
	author       = {Lazaric, Alessandro and Ghavamzadeh, Mohammad and Munos, R{\'e}mi},
	year         = 2012,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR. org},
	volume       = 13,
	number       = 1,
	pages        = {3041--3074}
}
@article{lagoudakis2003least,
	title        = {Least-squares policy iteration},
	author       = {Lagoudakis, Michail G and Parr, Ronald},
	year         = 2003,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR. org},
	volume       = 4,
	pages        = {1107--1149}
}
@article{bertsekas2011approximate,
	title        = {Approximate policy iteration: A survey and some new methods},
	author       = {Bertsekas, Dimitri P},
	year         = 2011,
	journal      = {Journal of Control Theory and Applications},
	publisher    = {Springer},
	volume       = 9,
	number       = 3,
	pages        = {310--335}
}
@inproceedings{gordon1995stable,
	title        = {Stable function approximation in dynamic programming},
	author       = {Gordon, Geoffrey J},
	year         = 1995,
	booktitle    = {Proceedings of the 12th International Conference on Machine Learning},
	pages        = {261--268}
}
@book{bertsekas1996neuro,
	title        = {Neuro-Dynamic Programming},
	author       = {Bertsekas, Dimitri P and Tsitsiklis, John N},
	year         = 1996,
	publisher    = {Athena Scientific, Belmont, MA}
}
@article{bellman1959functional,
	title        = {Functional approximations and dynamic programming},
	author       = {Bellman, Richard and Dreyfus, Stuart},
	year         = 1959,
	journal      = {Mathematical Tables and Other Aids to Computation},
	publisher    = {JSTOR},
	volume       = 13,
	number       = 68,
	pages        = {247--251}
}
@article{munos2007performance,
	title        = {Performance bounds in l\_p-norm for approximate value iteration},
	author       = {Munos, R{\'e}mi},
	year         = 2007,
	journal      = {SIAM journal on control and optimization},
	publisher    = {SIAM},
	volume       = 46,
	number       = 2,
	pages        = {541--561}
}
@inproceedings{farahmand2010error,
	title        = {Error propagation for approximate policy and value iteration},
	author       = {Farahmand, Amir-massoud and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
	year         = 2010,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {568--576}
}
@article{pires2016policy,
	title        = {Policy Error Bounds for Model-Based Reinforcement Learning with Factored Linear Models},
	author       = {Pires, Bernardo {\'A}vila and Szepesv{\'a}ri, Csaba},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1602.06346}
}
@article{mnih2015human,
	title        = {Human-level control through deep reinforcement learning},
	author       = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and Petersen, Stig and Beattie, Charles and Sadik, Amir and Antonoglou, Ioannis and King, Helen and Kumaran, Dharshan and Wierstra, Daan and Legg, Shane and Hassabis, Demis},
	year         = 2015,
	journal      = {Nature},
	publisher    = {Nature Publishing Group},
	volume       = 518,
	number       = 7540,
	pages        = {529--533}
}
@article{silver2016mastering,
	title        = {{Mastering the game of Go with deep neural networks and tree search}},
	author       = {Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and van den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and Dieleman, Sander and Grewe, Dominik and Nham, John and Kalchbrenner, Nal and Sutskever, Ilya and Lillicrap, Timothy and Leach, Madeleine and Kavukcuoglu, Koray and Graepel, Thore and Hassabis, Demis},
	year         = 2016,
	journal      = {Nature},
	publisher    = {Nature Publishing Group},
	volume       = 529,
	number       = 7587,
	pages        = {484--489}
}
@article{kearns2002near,
	title        = {Near-optimal reinforcement learning in polynomial time},
	author       = {Kearns, Michael and Singh, Satinder},
	year         = 2002,
	journal      = {Machine Learning},
	publisher    = {Springer},
	volume       = 49,
	number       = {2-3},
	pages        = {209--232}
}
@inproceedings{jiang2016doubly,
	title        = {Doubly Robust Off-policy Value Evaluation for Reinforcement Learning},
	author       = {Jiang, Nan and Li, Lihong},
	year         = 2016,
	booktitle    = {Proceedings of the 33rd International Conference on Machine Learning},
	pages        = {652--661}
}
@inproceedings{doshi2009infinite,
	title        = {The infinite partially observable Markov decision process},
	author       = {Doshi-Velez, Finale},
	year         = 2009,
	booktitle    = {Advances in Neural Information Processing Systems},
	pages        = {477--485}
}

@inproceedings{hu2021fast,
  title={Fast Rates for the Regret of Offline Reinforcement Learning},
  author={Hu, Yichun and Kallus, Nathan and Uehara, Masatoshi},
  booktitle={Conference on Learning Theory},
  year={2021},
  organization={PMLR}
}

@article{wang2021exponential,
  title={An Exponential Lower Bound for Linearly Realizable MDP with Constant Suboptimality Gap},
  author={Wang, Yuanhao and Wang, Ruosong and Kakade, Sham},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{antos2007fitted,
  title={Fitted Q-iteration in continuous action-space MDPs},
  author={Antos, Andr{\'a}s and Szepesv{\'a}ri, Csaba and Munos, R{\'e}mi},
  journal={Advances in neural information processing systems},
  volume={20},
  year={2007}
}

@article{wu2021gap,
  title={Gap-dependent unsupervised exploration for reinforcement learning},
  author={Wu, Jingfeng and Braverman, Vladimir and Yang, Lin F},
  journal={arXiv preprint arXiv:2108.05439},
  year={2021}
}

@inproceedings{ayoub2020model,
  title={Model-based reinforcement learning with value-targeted regression},
  author={Ayoub, Alex and Jia, Zeyu and Szepesvari, Csaba and Wang, Mengdi and Yang, Lin},
  booktitle={International Conference on Machine Learning},
  year={2020},
}

