@string{JASA = {Journal of the American Statistical Association}}
@string{JC  = {Journal of Classification}}
@string{JSPI  = {Journal of Statistical Planning and Inference}}
@string{JRSSB  = {Journal of the Royal Statistical Society, Series B}}
@string{SAGMB  = {Statistical Applications in Genetics and Molecular Biology}}
@string{NIPS  = {Advances in Neural Information Processing Systems}}
@string{AOS  = {Annals of Statistics}}
@string{AOAS  = {Annals of Applied Statistics}}
@string{JMLR  = {Journal of Machine Learning Research}}
@string{EJS  = {Electronic Journal of Statistics}}
@string{AISTATS  = {International Conference on Artificial Intelligence and Statistics}}
@string{UAI  = {Conference on Uncertainty in Artificial Intelligence}}
@string{ICML  = {International Conference on Machine Learning}}
@string{COLT  = {Conference on Learning Theory}}
@string{TIT = {IEEE Transactions on Information Theory}}
@string{TCS ={Theoretical Computer Science}}






@inproceedings{agrawal2017optimistic,
  title={Optimistic posterior sampling for reinforcement learning: worst-case regret bounds},
  author={Agrawal, Shipra and Jia, Randy},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1184--1194},
  year={2017}
}

@inproceedings{azar2017minimax,
  title={Minimax regret bounds for reinforcement learning},
  author={Azar, Mohammad Gheshlaghi and Osband, Ian and Munos, R{\'e}mi},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={263--272},
  year={2017},
  organization={JMLR. org}
}

@article{kearns2002near,
  title={Near-optimal reinforcement learning in polynomial time},
  author={Kearns, Michael and Singh, Satinder},
  journal={Machine learning},
  volume={49},
  number={2-3},
  pages={209--232},
  year={2002},
  publisher={Springer}
}


@inproceedings{strehl2006pac,
  title={PAC model-free reinforcement learning},
  author={Strehl, Alexander L and Li, Lihong and Wiewiora, Eric and Langford, John and Littman, Michael L},
  booktitle={Proceedings of the 23rd international conference on Machine learning},
  pages={881--888},
  year={2006},
  organization={ACM}
}

@article{dong2019q,
  title={Q-learning with UCB Exploration is Sample Efficient for Infinite-Horizon MDP},
  author={Dong, Kefan and Wang, Yuanhao and Chen, Xiaoyu and Wang, Liwei},
  journal={arXiv preprint arXiv:1901.09311},
  year={2019}
}

@article{sidford2018near,
  title={Near-Optimal Time and Sample Complexities for for Solving Discounted Markov Decision Process with a Generative Model},
  author={Sidford, Aaron and Wang, Mengdi and Wu, Xian and Yang, Lin F and Ye, Yinyu},
  journal={arXiv preprint arXiv:1806.01492},
  year={2018}
}

@inproceedings{lattimore2012pac,
  title={PAC bounds for discounted MDPs},
  author={Lattimore, Tor and Hutter, Marcus},
  booktitle={International Conference on Algorithmic Learning Theory},
  pages={320--334},
  year={2012},
  organization={Springer}
}




@inproceedings{zhang2016online,
  title={Online stochastic linear optimization under one-bit feedback},
  author={Zhang, Lijun and Yang, Tianbao and Jin, Rong and Xiao, Yichi and Zhou, Zhi-Hua},
  booktitle={International Conference on Machine Learning},
  pages={392--401},
  year={2016}
}

@misc{liu2020regret,
    title={Regret Bounds for Discounted MDPs},
    author={Shuang Liu and Hao Su},
    year={2020},
    eprint={2002.05138},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}
@article{szita2010model,
  title={Model-based reinforcement learning with nearly tight exploration complexity bounds},
  author={Szita, Istv{\'a}n and Szepesv{\'a}ri, Csaba},
  year={2010}
}



@string{JASA = {Journal of the American Statistical Association}}
@string{JC  = {Journal of Classification}}
@string{JSPI  = {Journal of Statistical Planning and Inference}}
@string{JRSSB  = {Journal of the Royal Statistical Society, Series B}}
@string{SAGMB  = {Statistical Applications in Genetics and Molecular Biology}}
@string{NIPS  = {Advances in Neural Information Processing Systems}}
@string{AOS  = {Annals of Statistics}}
@string{AOAS  = {Annals of Applied Statistics}}
@string{JMLR  = {Journal of Machine Learning Research}}
@string{EJS  = {Electronic Journal of Statistics}}
@string{AISTATS  = {International Conference on Artificial Intelligence and Statistics}}
@string{UAI  = {Conference on Uncertainty in Artificial Intelligence}}
@string{ICML  = {International Conference on Machine Learning}}
@string{COLT  = {Conference on Learning Theory}}
@string{TIT = {IEEE Transactions on Information Theory}}
@string{TCS ={Theoretical Computer Science}}





@inproceedings{agrawal2017optimistic,
  title={Optimistic posterior sampling for reinforcement learning: worst-case regret bounds},
  author={Agrawal, Shipra and Jia, Randy},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1184--1194},
  year={2017}
}



@article{kearns2002near,
  title={Near-optimal reinforcement learning in polynomial time},
  author={Kearns, Michael and Singh, Satinder},
  journal={Machine learning},
  volume={49},
  number={2-3},
  pages={209--232},
  year={2002},
  publisher={Springer}
}








@inproceedings{zhang2016online,
  title={Online stochastic linear optimization under one-bit feedback},
  author={Zhang, Lijun and Yang, Tianbao and Jin, Rong and Xiao, Yichi and Zhou, Zhi-Hua},
  booktitle={International Conference on Machine Learning},
  pages={392--401},
  year={2016}
}



@misc{2002.05138,
Author = {Shuang Liu and Hao Su},
Title = {Regret Bounds for Discounted MDPs},
Year = {2020},
Eprint = {arXiv:2002.05138},
}








@article{azar2013minimax,
  title={Minimax PAC bounds on the sample complexity of reinforcement learning with a generative model},
  author={Azar, Mohammad Gheshlaghi and Munos, R{\'e}mi and Kappen, Hilbert J},
  journal={Machine learning},
  volume={91},
  number={3},
  pages={325--349},
  year={2013},
  publisher={Springer}
}



@article{brafman2002r,
  title={R-max-a general polynomial time algorithm for near-optimal reinforcement learning},
  author={Brafman, Ronen I and Tennenholtz, Moshe},
  journal={Journal of Machine Learning Research},
  volume={3},
  number={Oct},
  pages={213--231},
  year={2002}
}

@inproceedings{sidford2018variance,
  title={Variance reduced value iteration and faster algorithms for solving markov decision processes},
  author={Sidford, Aaron and Wang, Mengdi and Wu, Xian and Ye, Yinyu},
  booktitle={Proceedings of the Twenty-Ninth Annual ACM-SIAM Symposium on Discrete Algorithms},
  pages={770--787},
  year={2018},
  organization={SIAM}
}
@inproceedings{kearns1999finite,
  title={Finite-sample convergence rates for Q-learning and indirect algorithms},
  author={Kearns, Michael J and Singh, Satinder P},
  booktitle={Advances in neural information processing systems},
  pages={996--1002},
  year={1999}
}

@article{wang2017randomized,
  title={Randomized linear programming solves the discounted Markov decision problem in nearly-linear running time},
  author={Wang, Mengdi},
  journal={arXiv preprint arXiv:1704.01869},
  year={2017}
}





@inproceedings{dann2015sample,
  title={Sample complexity of episodic fixed-horizon reinforcement learning},
  author={Dann, Christoph and Brunskill, Emma},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2818--2826},
  year={2015}
}


@inproceedings{zheng2018drn,
  title={DRN: A deep reinforcement learning framework for news recommendation},
  author={Zheng, Guanjie and Zhang, Fuzheng and Zheng, Zihan and Xiang, Yang and Yuan, Nicholas Jing and Xie, Xing and Li, Zhenhui},
  booktitle={Proceedings of the 2018 World Wide Web Conference},
  pages={167--176},
  year={2018}
}

@article{mnih2015human,
  title={Human-level control through deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and others},
  journal={Nature},
  volume={518},
  number={7540},
  pages={529--533},
  year={2015},
  publisher={Nature Publishing Group}
}
@article{silver2016mastering,
  title={Mastering the game of Go with deep neural networks and tree search},
  author={Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
  journal={nature},
  volume={529},
  number={7587},
  pages={484},
  year={2016},
  publisher={Nature Publishing Group}
}




@inproceedings{jiang2017contextual,
  title={Contextual decision processes with low Bellman rank are PAC-learnable},
  author={Jiang, Nan and Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John and Schapire, Robert E},
  booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
  pages={1704--1713},
  year={2017},
  organization={JMLR. org}
}

@article{zanette2020learning,
  title={Learning Near Optimal Policies with Low Inherent Bellman Error},
  author={Zanette, Andrea and Lazaric, Alessandro and Kochenderfer, Mykel and Brunskill, Emma},
  journal={arXiv preprint arXiv:2003.00153},
  year={2020}
}

@article{du2019good,
  title={Is a Good Representation Sufficient for Sample Efficient Reinforcement Learning?},
  author={Du, Simon S and Kakade, Sham M and Wang, Ruosong and Yang, Lin F},
  journal={arXiv preprint arXiv:1910.03016},
  year={2019}
}

@inproceedings{du2019provably,
  title={Provably efficient Q-learning with function approximation via distribution shift error checking oracle},
  author={Du, Simon S and Luo, Yuping and Wang, Ruosong and Zhang, Hanrui},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8058--8068},
  year={2019}
}

@article{besson2018doubling,
  title={What doubling tricks can and can't do for multi-armed bandits},
  author={Besson, Lilian and Kaufmann, Emilie},
  journal={arXiv preprint arXiv:1803.06971},
  year={2018}
}






@article{lattimore2018bandit,
  title={Bandit algorithms},
  author={Lattimore, Tor and Szepesv{\'a}ri, Csaba},
  journal={preprint},
  pages={28},
  year={2018}
}


@book{nesterov2013introductory,
  title={Introductory lectures on convex optimization: A basic course},
  author={Nesterov, Yurii},
  volume={87},
  year={2013},
  publisher={Springer Science \& Business Media}
}


@article{bertsekas2018feature,
  title={Feature-based aggregation and deep reinforcement learning: A survey and some new implementations},
  author={Bertsekas, Dimitri P},
  journal={IEEE/CAA Journal of Automatica Sinica},
  volume={6},
  number={1},
  pages={1--31},
  year={2018},
  publisher={IEEE}
}

@article{bhandari2018finite,
  title={A finite time analysis of temporal difference learning with linear function approximation},
  author={Bhandari, Jalaj and Russo, Daniel and Singal, Raghav},
  journal={arXiv preprint arXiv:1806.02450},
  year={2018}
}


@article{watkins1989learning,
  title={Learning from delayed rewards},
  author={Watkins, Christopher John Cornish Hellaby},
  year={1989},
  publisher={King's College, Cambridge}
}


@inproceedings{duan2019state,
  title={State aggregation learning from markov transition data},
  author={Duan, Yaqi and Ke, Tracy and Wang, Mengdi},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4488--4497},
  year={2019}
}


@article{arulkumaran2017deep,
  title={Deep reinforcement learning: A brief survey},
  author={Arulkumaran, Kai and Deisenroth, Marc Peter and Brundage, Miles and Bharath, Anil Anthony},
  journal={IEEE Signal Processing Magazine},
  volume={34},
  number={6},
  pages={26--38},
  year={2017},
  publisher={IEEE}
}


@inproceedings{zou2019finite,
  title={Finite-sample analysis for sarsa with linear function approximation},
  author={Zou, Shaofeng and Xu, Tengyu and Liang, Yingbin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8665--8675},
  year={2019}
}


@article{kober2013reinforcement,
  title={Reinforcement learning in robotics: A survey},
  author={Kober, Jens and Bagnell, J Andrew and Peters, Jan},
  journal={The International Journal of Robotics Research},
  volume={32},
  number={11},
  pages={1238--1274},
  year={2013},
  publisher={SAGE Publications Sage UK: London, England}
}


@inproceedings{kwok2004reinforcement,
  title={Reinforcement learning for sensing strategies},
  author={Kwok, Cody and Fox, Dieter},
  booktitle={2004 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)(IEEE Cat. No. 04CH37566)},
  volume={4},
  pages={3158--3163},
  organization={IEEE}
}


@inproceedings{singh1995reinforcement,
  title={Reinforcement learning with soft state aggregation},
  author={Singh, Satinder P and Jaakkola, Tommi and Jordan, Michael I},
  booktitle={Advances in neural information processing systems},
  pages={361--368},
  year={1995}
}






@article{lattimore2019learning,
  title={Learning with Good Feature Representations in Bandits and in RL with a Generative Model},
  author={Lattimore, Tor and Szepesvari, Csaba},
  journal={arXiv preprint arXiv:1911.07676},
  year={2019}
}

@article{van2019comments,
  title={Comments on the Du-Kakade-Wang-Yang Lower Bounds},
  author={Van Roy, Benjamin and Dong, Shi},
  journal={arXiv preprint arXiv:1911.07910},
  year={2019}
}


@book{boyd2004convex,
  title={Convex optimization},
  author={Boyd, Stephen and Boyd, Stephen P and Vandenberghe, Lieven},
  year={2004},
  publisher={Cambridge university press}
}


@article{ayoub2020model,
  title={Model-Based Reinforcement Learning with Value-Targeted Regression},
  author={Ayoub, Alex and Jia, Zeyu and Szepesvari, Csaba and Wang, Mengdi and Yang, Lin F},
  journal={arXiv preprint arXiv:2006.01107},
  year={2020}
}

@inproceedings{jia2020model,
  title={Model-based reinforcement learning with value-targeted regression},
  author={Jia, Zeyu and Yang, Lin and Szepesvari, Csaba and Wang, Mengdi},
  booktitle={Learning for Dynamics and Control},
  pages={666--686},
  year={2020},
  organization={PMLR}
}

@article{agarwal2020flambe,
  title={Flambe: Structural complexity and representation learning of low rank mdps},
  author={Agarwal, Alekh and Kakade, Sham and Krishnamurthy, Akshay and Sun, Wen},
  journal={arXiv preprint arXiv:2006.10814},
  year={2020}
}



@inproceedings{chu2011contextual,
  title={Contextual bandits with linear payoff functions},
  author={Chu, Wei and Li, Lihong and Reyzin, Lev and Schapire, Robert},
  booktitle={Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics},
  pages={208--214},
  year={2011},
  organization={JMLR Workshop and Conference Proceedings}
}




@inproceedings{modi2020sample,
  title={Sample complexity of reinforcement learning using linearly combined model ensembles},
  author={Modi, Aditya and Jiang, Nan and Tewari, Ambuj and Singh, Satinder},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2010--2020},
  year={2020},
  organization={PMLR}
}

@article{auer2002using,
  title={Using confidence bounds for exploitation-exploration trade-offs},
  author={Auer, Peter},
  journal={Journal of Machine Learning Research},
  volume={3},
  number={Nov},
  pages={397--422},
  year={2002}
}

@inproceedings{li2010contextual,
  title={A contextual-bandit approach to personalized news article recommendation},
  author={Li, Lihong and Chu, Wei and Langford, John and Schapire, Robert E},
  booktitle={Proceedings of the 19th international conference on World wide web},
  pages={661--670},
  year={2010}
}

@inproceedings{li2021tight,
  title={Tight regret bounds for infinite-armed linear contextual bandits},
  author={Li, Yingkai and Wang, Yining and Chen, Xi and Zhou, Yuan},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={370--378},
  year={2021},
  organization={PMLR}
}

@inproceedings{zhou2020nearly,
  title={Nearly minimax optimal reinforcement learning for linear mixture markov decision processes},
  author={Zhou, Dongruo and Gu, Quanquan and Szepesvari, Csaba},
  booktitle={Conference on Learning Theory},
  pages={4532--4576},
  year={2021},
  organization={PMLR}
}

@article{zhang2021variance,
  title={Variance-Aware Confidence Set: Variance-Dependent Bound for Linear Bandits and Horizon-Free Bound for Linear Mixture MDP},
  author={Zhang, Zihan and Yang, Jiaqi and Ji, Xiangyang and Du, Simon S},
  journal={arXiv preprint arXiv:2101.12745},
  year={2021}
}

@inproceedings{li2019nearly,
  title={Nearly minimax-optimal regret for linearly parameterized bandits},
  author={Li, Yingkai and Wang, Yining and Zhou, Yuan},
  booktitle={Conference on Learning Theory},
  pages={2173--2174},
  year={2019},
  organization={PMLR}
}

@article{jaksch2010near,
  title={Near-optimal regret bounds for reinforcement learning},
  author={Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
  journal={Journal of Machine Learning Research},
  volume={11},
  number={Apr},
  pages={1563--1600},
  year={2010}
}

@inproceedings{agrawal2017optimistic,
  title={Optimistic posterior sampling for reinforcement learning: worst-case regret bounds},
  author={Agrawal, Shipra and Jia, Randy},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1184--1194},
  year={2017}
}



@article{kearns2002near,
  title={Near-optimal reinforcement learning in polynomial time},
  author={Kearns, Michael and Singh, Satinder},
  journal={Machine learning},
  volume={49},
  number={2-3},
  pages={209--232},
  year={2002},
  publisher={Springer}
}


@article{dong2019q,
  title={Q-learning with UCB Exploration is Sample Efficient for Infinite-Horizon MDP},
  author={Dong, Kefan and Wang, Yuanhao and Chen, Xiaoyu and Wang, Liwei},
  journal={arXiv preprint arXiv:1901.09311},
  year={2019}
}

@article{sidford2018near,
  title={Near-Optimal Time and Sample Complexities for for Solving Discounted Markov Decision Process with a Generative Model},
  author={Sidford, Aaron and Wang, Mengdi and Wu, Xian and Yang, Lin F and Ye, Yinyu},
  journal={arXiv preprint arXiv:1806.01492},
  year={2018}
}

@inproceedings{lattimore2012pac,
  title={PAC bounds for discounted MDPs},
  author={Lattimore, Tor and Hutter, Marcus},
  booktitle={International Conference on Algorithmic Learning Theory},
  pages={320--334},
  year={2012},
  organization={Springer}
}


@article{jin2019provably,
  title={Provably efficient reinforcement learning with linear function approximation},
  author={Jin, Chi and Yang, Zhuoran and Wang, Zhaoran and Jordan, Michael I},
  journal={arXiv preprint arXiv:1907.05388},
  year={2019}
}

@inproceedings{zhang2016online,
  title={Online stochastic linear optimization under one-bit feedback},
  author={Zhang, Lijun and Yang, Tianbao and Jin, Rong and Xiao, Yichi and Zhou, Zhi-Hua},
  booktitle={International Conference on Machine Learning},
  pages={392--401},
  year={2016}
}



@article{szita2010model,
  title={Model-based reinforcement learning with nearly tight exploration complexity bounds},
  author={Szita, Istv{\'a}n and Szepesv{\'a}ri, Csaba},
  year={2010}
}





@string{JASA = {Journal of the American Statistical Association}}
@string{JC  = {Journal of Classification}}
@string{JSPI  = {Journal of Statistical Planning and Inference}}
@string{JRSSB  = {Journal of the Royal Statistical Society, Series B}}
@string{SAGMB  = {Statistical Applications in Genetics and Molecular Biology}}
@string{NIPS  = {Advances in Neural Information Processing Systems}}
@string{AOS  = {Annals of Statistics}}
@string{AOAS  = {Annals of Applied Statistics}}
@string{JMLR  = {Journal of Machine Learning Research}}
@string{EJS  = {Electronic Journal of Statistics}}
@string{AISTATS  = {International Conference on Artificial Intelligence and Statistics}}
@string{UAI  = {Conference on Uncertainty in Artificial Intelligence}}
@string{ICML  = {International Conference on Machine Learning}}
@string{COLT  = {Conference on Learning Theory}}
@string{TIT = {IEEE Transactions on Information Theory}}
@string{TCS ={Theoretical Computer Science}}


@inproceedings{jin2018q,
  title={Is q-learning provably efficient?},
  author={Jin, Chi and Allen-Zhu, Zeyuan and Bubeck, Sebastien and Jordan, Michael I},
  booktitle={Advances in Neural Information Processing Systems},
  pages={4868--4878},
  year={2018}
}


@inproceedings{agrawal2017optimistic,
  title={Optimistic posterior sampling for reinforcement learning: worst-case regret bounds},
  author={Agrawal, Shipra and Jia, Randy},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1184--1194},
  year={2017}
}



@article{kearns2002near,
  title={Near-optimal reinforcement learning in polynomial time},
  author={Kearns, Michael and Singh, Satinder},
  journal={Machine learning},
  volume={49},
  number={2-3},
  pages={209--232},
  year={2002},
  publisher={Springer}
}




@inproceedings{zhang2016online,
  title={Online stochastic linear optimization under one-bit feedback},
  author={Zhang, Lijun and Yang, Tianbao and Jin, Rong and Xiao, Yichi and Zhou, Zhi-Hua},
  booktitle={International Conference on Machine Learning},
  pages={392--401},
  year={2016}
}

@inproceedings{abbasi2011improved,
  title={Improved algorithms for linear stochastic bandits},
  author={Abbasi-Yadkori, Yasin and P{\'a}l, D{\'a}vid and Szepesv{\'a}ri, Csaba},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2312--2320},
  year={2011}
}


@article{yang2019reinforcement,
  title={Reinforcement leaning in feature space: Matrix bandit, kernels, and regret bound},
  author={Yang, Lin F and Wang, Mengdi},
  journal={arXiv preprint arXiv:1905.10389},
  year={2019}
}

@inproceedings{yang2019sample,
  title={Sample-optimal parametric q-learning using linearly additive features},
  author={Yang, Lin and Wang, Mengdi},
  booktitle={International Conference on Machine Learning},
  pages={6995--7004},
  year={2019}
}

@article{azar2013minimax,
  title={Minimax PAC bounds on the sample complexity of reinforcement learning with a generative model},
  author={Azar, Mohammad Gheshlaghi and Munos, R{\'e}mi and Kappen, Hilbert J},
  journal={Machine learning},
  volume={91},
  number={3},
  pages={325--349},
  year={2013},
  publisher={Springer}
}

@article{azar2011speedy,
  title={Speedy Q-learning},
  author={Ghavamzadeh, Mohammad and Kappen, Hilbert and Azar, Mohammad and Munos, R{\'e}mi},
  journal={Advances in neural information processing systems},
  volume={24},
  pages={2411--2419},
  year={2011}
}

@article{brafman2002r,
  title={R-max-a general polynomial time algorithm for near-optimal reinforcement learning},
  author={Brafman, Ronen I and Tennenholtz, Moshe},
  journal={Journal of Machine Learning Research},
  volume={3},
  number={Oct},
  pages={213--231},
  year={2002}
}

@inproceedings{sidford2018variance,
  title={Variance reduced value iteration and faster algorithms for solving markov decision processes},
  author={Sidford, Aaron and Wang, Mengdi and Wu, Xian and Ye, Yinyu},
  booktitle={Proceedings of the Twenty-Ninth Annual ACM-SIAM Symposium on Discrete Algorithms},
  pages={770--787},
  year={2018},
  organization={SIAM}
}
@inproceedings{kearns1999finite,
  title={Finite-sample convergence rates for Q-learning and indirect algorithms},
  author={Kearns, Michael J and Singh, Satinder P},
  booktitle={Advances in neural information processing systems},
  pages={996--1002},
  year={1999}
}

@article{wang2017randomized,
  title={Randomized linear programming solves the discounted Markov decision problem in nearly-linear running time},
  author={Wang, Mengdi},
  journal={arXiv preprint arXiv:1704.01869},
  year={2017}
}



@inproceedings{strehl2006pac,
  title={PAC model-free reinforcement learning},
  author={Strehl, Alexander L and Li, Lihong and Wiewiora, Eric and Langford, John and Littman, Michael L},
  booktitle={Proceedings of the 23rd international conference on Machine learning},
  pages={881--888},
  year={2006}
}




@inproceedings{zheng2018drn,
  title={DRN: A deep reinforcement learning framework for news recommendation},
  author={Zheng, Guanjie and Zhang, Fuzheng and Zheng, Zihan and Xiang, Yang and Yuan, Nicholas Jing and Xie, Xing and Li, Zhenhui},
  booktitle={Proceedings of the 2018 World Wide Web Conference},
  pages={167--176},
  year={2018}
}

@article{mnih2015human,
  title={Human-level control through deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and others},
  journal={Nature},
  volume={518},
  number={7540},
  pages={529--533},
  year={2015},
  publisher={Nature Publishing Group}
}
@article{silver2016mastering,
  title={Mastering the game of Go with deep neural networks and tree search},
  author={Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
  journal={nature},
  volume={529},
  number={7587},
  pages={484},
  year={2016},
  publisher={Nature Publishing Group}
}

@article{wang2019optimism,
  title={Optimism in Reinforcement Learning with Generalized Linear Function Approximation},
  author={Wang, Yining and Wang, Ruosong and Du, Simon S and Krishnamurthy, Akshay},
  journal={arXiv preprint arXiv:1912.04136},
  year={2019}
}








@inproceedings{du2019provably,
  title={Provably efficient Q-learning with function approximation via distribution shift error checking oracle},
  author={Du, Simon S and Luo, Yuping and Wang, Ruosong and Zhang, Hanrui},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8058--8068},
  year={2019}
}

@article{besson2018doubling,
  title={What doubling tricks can and can't do for multi-armed bandits},
  author={Besson, Lilian and Kaufmann, Emilie},
  journal={arXiv preprint arXiv:1803.06971},
  year={2018}
}


@article{osband2016lower,
  title={On lower bounds for regret in reinforcement learning},
  author={Osband, Ian and Van Roy, Benjamin},
  journal={arXiv preprint arXiv:1608.02732},
  year={2016}
}


@article{dani2008stochastic,
  title={Stochastic linear optimization under bandit feedback},
  author={Dani, Varsha and Hayes, Thomas P and Kakade, Sham M},
  year={2008}
}



@book{nesterov2013introductory,
  title={Introductory lectures on convex optimization: A basic course},
  author={Nesterov, Yurii},
  volume={87},
  year={2013},
  publisher={Springer Science \& Business Media}
}


@article{xu2020reanalysis,
  title={Reanalysis of Variance Reduced Temporal Difference Learning},
  author={Xu, Tengyu and Wang, Zhe and Zhou, Yi and Liang, Yingbin},
  journal={arXiv preprint arXiv:2001.01898},
  year={2020}
}


@article{bradtke1996linear,
  title={Linear least-squares algorithms for temporal difference learning},
  author={Bradtke, Steven J and Barto, Andrew G},
  journal={Machine learning},
  volume={22},
  number={1-3},
  pages={33--57},
  year={1996},
  publisher={Springer}
}

@inproceedings{melo2007q,
  title={Q-learning with linear function approximation},
  author={Melo, Francisco S and Ribeiro, M Isabel},
  booktitle={International Conference on Computational Learning Theory},
  pages={308--322},
  year={2007},
  organization={Springer}
}

@article{maurer2009empirical,
  title={Empirical Bernstein bounds and sample variance penalization},
  author={Maurer, Andreas and Pontil, Massimiliano},
  journal={arXiv preprint arXiv:0907.3740},
  year={2009}
}


@article{agarwal2019model,
  title={Model-based reinforcement learning with a generative model is minimax optimal},
  author={Agarwal, Alekh and Kakade, Sham and Yang, Lin F},
  journal={arXiv preprint arXiv:1906.03804},
  year={2019}
}
@phdthesis{kakade2003sample,
  title={On the sample complexity of reinforcement learning},
  author={Kakade, Sham Machandranath and others},
  year={2003},
  school={University of London London, England}
}



@article{wainwright2019variance,
  title={Variance-reduced $ Q $-learning is minimax optimal},
  author={Wainwright, Martin J},
  journal={arXiv preprint arXiv:1906.04697},
  year={2019}
}


@article{zhang2020model,
  title={Model-Free Reinforcement Learning: from Clipped Pseudo-Regret to Sample Complexity},
  author={Zhang, Zihan and Zhou, Yuan and Ji, Xiangyang},
  journal={arXiv preprint arXiv:2006.03864},
  year={2020}
}


@inproceedings{osband2013more,
  title={(More) efficient reinforcement learning via posterior sampling},
  author={Osband, Ian and Russo, Daniel and Van Roy, Benjamin},
  booktitle={Advances in Neural Information Processing Systems},
  pages={3003--3011},
  year={2013}
}


@article{kearns2002near,
  title={Near-optimal reinforcement learning in polynomial time},
  author={Kearns, Michael and Singh, Satinder},
  journal={Machine learning},
  volume={49},
  number={2-3},
  pages={209--232},
  year={2002},
  publisher={Springer}
}

%new

@article{strehl2008analysis,
  title={An analysis of model-based interval estimation for Markov decision processes},
  author={Strehl, Alexander L and Littman, Michael L},
  journal={Journal of Computer and System Sciences},
  volume={74},
  number={8},
  pages={1309--1331},
  year={2008},
  publisher={Academic Press}
}


@inproceedings{kolter2009near,
  title={Near-Bayesian exploration in polynomial time},
  author={Kolter, J Zico and Ng, Andrew Y},
  booktitle={Proceedings of the 26th annual international conference on machine learning},
  pages={513--520},
  year={2009}
}

@inproceedings{auer2009near,
  title={Near-optimal regret bounds for reinforcement learning},
  author={Auer, Peter and Jaksch, Thomas and Ortner, Ronald},
  booktitle={Advances in neural information processing systems},
  pages={89--96},
  year={2009}
}




@inproceedings{dann2017unifying,
  title={Unifying PAC and regret: Uniform PAC bounds for episodic reinforcement learning},
  author={Dann, Christoph and Lattimore, Tor and Brunskill, Emma},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5713--5723},
  year={2017}
}

@inproceedings{osband2017posterior,
  title={Why is posterior sampling better than optimism for reinforcement learning?},
  author={Osband, Ian and Van Roy, Benjamin},
  booktitle={International Conference on Machine Learning},
  pages={2701--2710},
  year={2017}
}



@inproceedings{dann2019policy,
  title={Policy certificates: Towards accountable reinforcement learning},
  author={Dann, Christoph and Li, Lihong and Wei, Wei and Brunskill, Emma},
  booktitle={International Conference on Machine Learning},
  pages={1507--1516},
  year={2019},
  organization={PMLR}
}

@inproceedings{simchowitz2019non,
  title={Non-asymptotic gap-dependent regret bounds for tabular mdps},
  author={Simchowitz, Max and Jamieson, Kevin G},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1153--1162},
  year={2019}
}

@inproceedings{russo2019worst,
  title={Worst-case regret bounds for exploration via randomized value functions},
  author={Russo, Daniel},
  booktitle={Advances in Neural Information Processing Systems},
  pages={14433--14443},
  year={2019}
}



@article{cai2019provably,
  title={Provably efficient exploration in policy optimization},
  author={Cai, Qi and Yang, Zhuoran and Jin, Chi and Wang, Zhaoran},
  journal={arXiv preprint arXiv:1912.05830},
  year={2019}
}

@article{zhang2020almost,
  title={Almost Optimal Model-Free Reinforcement Learning via Reference-Advantage Decomposition},
  author={Zhang, Zihan and Zhou, Yuan and Ji, Xiangyang},
  journal={arXiv preprint arXiv:2004.10019},
  year={2020}
}

@article{yang2020q,
  title={$ Q $-learning with Logarithmic Regret},
  author={Yang, Kunhe and Yang, Lin F and Du, Simon S},
  journal={arXiv preprint arXiv:2006.09118},
  year={2020}
}

@article{pacchiano2020optimism,
  title={On Optimism in Model-Based Reinforcement Learning},
  author={Pacchiano, Aldo and Ball, Philip and Parker-Holder, Jack and Choromanski, Krzysztof and Roberts, Stephen},
  journal={arXiv preprint arXiv:2006.11911},
  year={2020}
}

@article{neu2020unifying,
  title={A Unifying View of Optimism in Episodic Reinforcement Learning},
  author={Neu, Gergely and Pike-Burke, Ciara},
  journal={arXiv preprint arXiv:2007.01891},
  year={2020}
}

@article{zanette2019tighter,
  title={Tighter problem-dependent regret bounds in reinforcement learning without domain knowledge using value function bounds},
  author={Zanette, Andrea and Brunskill, Emma},
  journal={arXiv preprint arXiv:1901.00210},
  year={2019}
}



@book{cesa2006prediction,
  title={Prediction, learning, and games},
  author={Cesa-Bianchi, Nicolo and Lugosi, G{\'a}bor},
  year={2006},
  publisher={Cambridge university press}
}

@article{zhang2020reinforcement,
  title={Is reinforcement learning more difficult than bandits? a near-optimal algorithm escaping the curse of horizon},
  author={Zhang, Zihan and Ji, Xiangyang and Du, Simon S},
  journal={arXiv preprint arXiv:2009.13503},
  year={2020}
}

@inproceedings{he2021logarithmic,
    title={Logarithmic regret for reinforcement learning with linear function approximation},
    author={He, Jiafan and Zhou, Dongruo and Gu, Quanquan},
    booktitle={International Conference on Machine Learning},
    pages={4171--4180},
    year={2021},
    organization={PMLR}
}


@article{soare2014best,
  title={Best-arm identification in linear bandits},
  author={Soare, Marta and Lazaric, Alessandro and Munos, R{\'e}mi},
  journal={arXiv preprint arXiv:1409.6110},
  year={2014}
}


@inproceedings{karnin2016verification,
  title={Verification based solution for structured mab problems},
  author={Karnin, Zohar},
  booktitle={Proceedings of the 30th International Conference on Neural Information Processing Systems},
  pages={145--153},
  year={2016}
}

@inproceedings{xu2018fully,
  title={A fully adaptive algorithm for pure exploration in linear bandits},
  author={Xu, Liyuan and Honda, Junya and Sugiyama, Masashi},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={843--851},
  year={2018},
  organization={PMLR}
}

@inproceedings{tao2018best,
  title={Best arm identification in linear bandits with linear dimension dependency},
  author={Tao, Chao and Blanco, Sa{\'u}l and Zhou, Yuan},
  booktitle={International Conference on Machine Learning},
  pages={4877--4886},
  year={2018},
  organization={PMLR}
}

@article{fiez2019sequential,
  title={Sequential experimental design for transductive linear bandits},
  author={Fiez, Tanner and Jain, Lalit and Jamieson, Kevin and Ratliff, Lillian},
  journal={arXiv preprint arXiv:1906.08399},
  year={2019}
}


@inproceedings{russo2013eluder,
  title={Eluder Dimension and the Sample Complexity of Optimistic Exploration.},
  author={Russo, Daniel and Van Roy, Benjamin},
  booktitle={NIPS},
  pages={2256--2264},
  year={2013},
  organization={Citeseer}
}

@article{wang2020reinforcement,
  title={Reinforcement Learning with General Value Function Approximation: Provably Efficient Approach via Bounded Eluder Dimension},
  author={Wang, Ruosong and Salakhutdinov, Russ R and Yang, Lin},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}

@article{jin2021bellman,
  title={Bellman Eluder dimension: New rich classes of RL problems, and sample-efficient algorithms},
  author={Jin, Chi and Liu, Qinghua and Miryoosefi, Sobhan},
  journal={arXiv preprint arXiv:2102.00815},
  year={2021}
}

@article{du2021bilinear,
  title={Bilinear classes: A structural framework for provable generalization in rl},
  author={Du, Simon S and Kakade, Sham M and Lee, Jason D and Lovett, Shachar and Mahajan, Gaurav and Sun, Wen and Wang, Ruosong},
  journal={arXiv preprint arXiv:2103.10897},
  year={2021}
}

@inproceedings{sun2019model,
  title={Model-based rl in contextual decision processes: Pac bounds and exponential improvements over model-free approaches},
  author={Sun, Wen and Jiang, Nan and Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John},
  booktitle={Conference on learning theory},
  pages={2898--2933},
  year={2019},
  organization={PMLR}
}

@inproceedings{dong2020root,
  title={Root-n-regret for learning in markov decision processes with function approximation and low bellman rank},
  author={Dong, Kefan and Peng, Jian and Wang, Yining and Zhou, Yuan},
  booktitle={Conference on Learning Theory},
  pages={1554--1557},
  year={2020},
  organization={PMLR}
}

@article{yang2020function,
  title={On function approximation in reinforcement learning: Optimism in the face of large state spaces},
  author={Yang, Zhuoran and Jin, Chi and Wang, Zhaoran and Wang, Mengdi and Jordan, Michael I},
  journal={arXiv preprint arXiv:2011.04622},
  year={2020}
}

@inproceedings{he2021uniform,
    title={Uniform-{PAC} Bounds for Reinforcement Learning with Linear Function Approximation},
    author={Jiafan He and Dongruo Zhou and Quanquan Gu},
    booktitle={Thirty-Fifth Conference on Neural Information Processing Systems},
    year={2021}
}

@inproceedings{zhou2020provably,
  title={Provably Efficient Reinforcement Learning for Discounted MDPs with Feature Mapping},
  author={Zhou, Dongruo and He, Jiafan and Gu, Quanquan},
  booktitle={International Conference on Machine Learning},
  year={2021},
  organization={PMLR}
}


@article{osband2014model,
  title={Model-based Reinforcement Learning and the Eluder Dimension},
  author={Osband, Ian and Van Roy, Benjamin},
  journal={Advances in Neural Information Processing Systems},
  volume={27},
  pages={1466--1474},
  year={2014}
}

@article{wagenmaker2022reward,
  title={Reward-Free RL is No Harder Than Reward-Aware RL in Linear Markov Decision Processes},
  author={Wagenmaker, Andrew and Chen, Yifang and Simchowitz, Max and Du, Simon S and Jamieson, Kevin},
  journal={arXiv preprint arXiv:2201.11206},
  year={2022}
}

@article{chen2022general,
  title={A General Framework for Sample-Efficient Function Approximation in Reinforcement Learning},
  author={Chen, Zixiang and Li, Chris Junchi and Yuan, Angela and Gu, Quanquan and Jordan, Michael I},
  journal={arXiv preprint arXiv:2209.15634},
  year={2022}
}


@InProceedings{pmlr-v162-chen22f,
  title = 	 {On the Sample Complexity of Learning Infinite-horizon Discounted Linear Kernel {MDP}s},
  author =       {Chen, Yuanzhou and He, Jiafan and Gu, Quanquan},
  booktitle = 	 {Proceedings of the 39th International Conference on Machine Learning},
  pages = 	 {3149--3183},
  year = 	 {2022}
}
