@inproceedings{hao2020adaptive,
  title={Adaptive exploration in linear contextual bandit},
  author={Hao, Botao and Lattimore, Tor and Szepesvari, Csaba},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3536--3545},
  year={2020},
  organization={PMLR}
}
@inproceedings{papini2021leveraging,
  title={Leveraging Good Representations in Linear Contextual Bandits},
  author={Papini, Matteo and Tirinzoni, Andrea and Restelli, Marcello and Lazaric, Alessandro and Pirotta, Matteo},
  booktitle={International Conference on Machine Learning},
  year={2021},
  organization={PMLR}
}
@inproceedings{he2020logarithmic,
  title={Logarithmic regret for reinforcement learning with linear function approximation},
  author={He, Jiafan and Zhou, Dongruo and Gu, Quanquan},
  booktitle={International Conference on Machine Learning},
  year={2021},
  organization={PMLR}
}
@inproceedings{wei2021learning,
  title={Learning infinite-horizon average-reward mdps with linear function approximation},
  author={Wei, Chen-Yu and Jahromi, Mehdi Jafarnia and Luo, Haipeng and Jain, Rahul},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3007--3015},
  year={2021},
  organization={PMLR}
}
@article{tropp2012user,
  title={User-friendly tail bounds for sums of random matrices},
  author={Tropp, Joel A},
  journal={Foundations of computational mathematics},
  volume={12},
  number={4},
  pages={389--434},
  year={2012},
  publisher={Springer}
}
@inproceedings{abbasi2011improved,
  title={Improved Algorithms for Linear Stochastic Bandits.},
  author={Abbasi-Yadkori, Yasin and P{\'a}l, D{\'a}vid and Szepesv{\'a}ri, Csaba},
  booktitle={NIPS},
  volume={11},
  pages={2312--2320},
  year={2011}
}
@inproceedings{jin2020provably,
  title={Provably efficient reinforcement learning with linear function approximation},
  author={Jin, Chi and Yang, Zhuoran and Wang, Zhaoran and Jordan, Michael I},
  booktitle={Conference on Learning Theory},
  pages={2137--2143},
  year={2020},
  organization={PMLR}
}
@article{mnih2013playing,
  title={Playing atari with deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},
  journal={arXiv preprint arXiv:1312.5602},
  year={2013}
}
@article{kober2013reinforcement,
  title={Reinforcement learning in robotics: A survey},
  author={Kober, Jens and Bagnell, J Andrew and Peters, Jan},
  journal={The International Journal of Robotics Research},
  volume={32},
  number={11},
  pages={1238--1274},
  year={2013},
  publisher={SAGE Publications Sage UK: London, England}
}
@article{de2018integrating,
  title={Integrating state representation learning into deep reinforcement learning},
  author={de Bruin, Tim and Kober, Jens and Tuyls, Karl and Babu{\v{s}}ka, Robert},
  journal={IEEE Robotics and Automation Letters},
  volume={3},
  number={3},
  pages={1394--1401},
  year={2018},
  publisher={IEEE}
}
@inproceedings{yang2021impact,
  title={Impact of representation learning in linear bandits},
  author={Yang, Jiaqi and Hu, Wei and Lee, Jason D and Du, Simon S},
  booktitle={International Conference on Learning Representations},
  year={2021}
}
@inproceedings{yang2019sample,
  title={Sample-optimal parametric Q-learning using linearly additive features},
  author={Yang, Lin and Wang, Mengdi},
  booktitle={International Conference on Machine Learning},
  pages={6995--7004},
  year={2019},
  organization={PMLR}
}
@inproceedings{zanette2020learning,
  title={Learning near optimal policies with low inherent bellman error},
  author={Zanette, Andrea and Lazaric, Alessandro and Kochenderfer, Mykel and Brunskill, Emma},
  booktitle={International Conference on Machine Learning},
  pages={10978--10989},
  year={2020},
  organization={PMLR}
}
@inproceedings{jiang2017contextual,
  title={Contextual decision processes with low bellman rank are pac-learnable},
  author={Jiang, Nan and Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John and Schapire, Robert E},
  booktitle={International Conference on Machine Learning},
  pages={1704--1713},
  year={2017},
  organization={PMLR}
}
@inproceedings{ayoub2020model,
  title={Model-based reinforcement learning with value-targeted regression},
  author={Ayoub, Alex and Jia, Zeyu and Szepesvari, Csaba and Wang, Mengdi and Yang, Lin},
  booktitle={International Conference on Machine Learning},
  pages={463--474},
  year={2020},
  organization={PMLR}
}
@inproceedings{jia2020model,
  title={Model-based reinforcement learning with value-targeted regression},
  author={Jia, Zeyu and Yang, Lin and Szepesvari, Csaba and Wang, Mengdi},
  booktitle={Learning for Dynamics and Control},
  pages={666--686},
  year={2020},
  organization={PMLR}
}
@inproceedings{zhou2020provably,
  title={Provably Efficient Reinforcement Learning for Discounted MDPs with Feature Mapping},
  author={Zhou, Dongruo and He, Jiafan and Gu, Quanquan},
  booktitle={International Conference on Machine Learning},
  year={2021},
  organization={PMLR}
}
@inproceedings{Du2020Is,
  title={Is a Good Representation Sufficient for Sample Efficient Reinforcement Learning?},
  author={Du, Simon S and Kakade, Sham M and Wang, Ruosong and Yang, Lin F},
  booktitle={International Conference on Learning Representations},
  year={2020}
}
@inproceedings{nachum2018nearoptimal,
  title={Near-Optimal Representation Learning for Hierarchical Reinforcement Learning},
  author={Nachum, Ofir and Gu, Shixiang and Lee, Honglak and Levine, Sergey},
  booktitle={International Conference on Learning Representations},
  year={2018}
}
@article{michael1995reinforcement,
  title={Reinforcement learning with soft state aggregation},
  author={Michael, Satinder P Singh Tommi Jaakkola and Jordan, I},
  journal={Advances in neural information processing systems 7},
  volume={7},
  pages={361},
  year={1995},
  publisher={MIT Press}
}
@inproceedings{dean1997model,
  title={Model minimization in Markov decision processes},
  author={Dean, Thomas and Givan, Robert},
  booktitle={AAAI/IAAI},
  pages={106--111},
  year={1997}
}
@inproceedings{ravindran2002model,
  title={Model minimization in hierarchical reinforcement learning},
  author={Ravindran, Balaraman and Barto, Andrew G},
  booktitle={International Symposium on Abstraction, Reformulation, and Approximation},
  pages={196--211},
  year={2002},
  organization={Springer}
}
@inproceedings{abel2016near,
  title={Near optimal behavior via approximate state abstraction},
  author={Abel, David and Hershkowitz, David and Littman, Michael},
  booktitle={International Conference on Machine Learning},
  pages={2915--2923},
  year={2016},
  organization={PMLR}
}
@inproceedings{diuk2008object,
  title={An object-oriented representation for efficient reinforcement learning},
  author={Diuk, Carlos and Cohen, Andre and Littman, Michael L},
  booktitle={Proceedings of the 25th international conference on Machine learning},
  pages={240--247},
  year={2008}
}
@article{stooke2020decoupling,
  title={Decoupling representation learning from reinforcement learning},
  author={Stooke, Adam and Lee, Kimin and Abbeel, Pieter and Laskin, Michael},
  journal={arXiv preprint arXiv:2009.08319},
  year={2020}
}
@inproceedings{chatterji2020osom,
  title={Osom: A simultaneously optimal algorithm for multi-armed and linear contextual bandits},
  author={Chatterji, Niladri and Muthukumar, Vidya and Bartlett, Peter},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1844--1854},
  year={2020},
  organization={PMLR}
}
@article{simchowitz2019non,
  title={Non-Asymptotic Gap-Dependent Regret Bounds for Tabular MDPs},
  journal={Advances in neural information processing systems},
  author={Simchowitz, Max and Jamieson, Kevin},
  year={2019}
}
@article{du2019provably,
  title={Provably Efficient Q-learning with Function Approximation via Distribution Shift Error Checking Oracle},
  author={Du, Simon S and Luo, Yuping and Wang, Ruosong and Zhang, Hanrui},
  journal={Advances in neural information processing systems},
  year={2019}
}
@article{du2020agnostic,
  title={Agnostic Q-learning with function approximation in deterministic systems: Tight bounds on approximation error and sample complexity},
  author={Du, Simon S and Lee, Jason D and Mahajan, Gaurav and Wang, Ruosong},
  journal={arXiv preprint arXiv:2002.07125},
  year={2020}
}
@inproceedings{yang2021q,
  title={Q-learning with Logarithmic Regret},
  author={Yang, Kunhe and Yang, Lin and Du, Simon},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1576--1584},
  year={2021},
  organization={PMLR}
}
@article{mou2020sample,
  title={On the sample complexity of reinforcement learning with policy space generalization},
  author={Mou, Wenlong and Wen, Zheng and Chen, Xi},
  journal={arXiv preprint arXiv:2008.07353},
  year={2020}
}
@article{foster2019model,
  title={Model Selection for Contextual Bandits},
  author={Foster, Dylan and Krishnamurthy, Akshay and Luo, Haipeng},
  journal={Advances in neural information processing systems},
  year={2019}
}
@article{pacchiano2020model,
  title={Model Selection in Contextual Stochastic Bandit Problems},
  author={Pacchiano, Aldo and Phan, My and Abbasi Yadkori, Yasin and Rao, Anup and Zimmert, Julian and Lattimore, Tor and Szepesvari, Csaba},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}

@inproceedings{odalric2011adaptive,
  title={Adaptive bandits: Towards the best history-dependent strategy},
  author={Odalric, Maillard and Munos, R{\'e}mi},
  booktitle={Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics},
  pages={570--578},
  year={2011},
  organization={JMLR Workshop and Conference Proceedings}
}
@inproceedings{agarwal2017corralling,
  title={Corralling a band of bandit algorithms},
  author={Agarwal, Alekh and Luo, Haipeng and Neyshabur, Behnam and Schapire, Robert E},
  booktitle={Conference on Learning Theory},
  pages={12--38},
  year={2017},
  organization={PMLR}
}
@article{auer2002nonstochastic,
  title={The nonstochastic multiarmed bandit problem},
  author={Auer, Peter and Cesa-Bianchi, Nicolo and Freund, Yoav and Schapire, Robert E},
  journal={SIAM journal on computing},
  volume={32},
  number={1},
  pages={48--77},
  year={2002},
  publisher={SIAM}
}
@inproceedings{modi2020sample,
  title={Sample complexity of reinforcement learning using linearly combined model ensembles},
  author={Modi, Aditya and Jiang, Nan and Tewari, Ambuj and Singh, Satinder},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2010--2020},
  year={2020},
  organization={PMLR}
}
@inproceedings{ghosh2021problem,
  title={Problem-complexity adaptive model selection for stochastic linear bandits},
  author={Ghosh, Avishek and Sankararaman, Abishek and Kannan, Ramchandran},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1396--1404},
  year={2021},
  organization={PMLR}
}
@article{abbasi2020regret,
  title={Regret balancing for bandit and rl model selection},
  author={Abbasi-Yadkori, Yasin and Pacchiano, Aldo and Phan, My},
  journal={arXiv preprint arXiv:2006.05491},
  year={2020}
}
@inproceedings{foster2020open,
  title={Open Problem: Model Selection for Contextual Bandits},
  author={Foster, Dylan J and Krishnamurthy, Akshay and Luo, Haipeng},
  booktitle={Conference on Learning Theory},
  pages={3842--3846},
  year={2020},
  organization={PMLR}
}
@article{yang2020offline,
  title={Offline Policy Selection under Uncertainty},
  author={Yang, Mengjiao and Dai, Bo and Nachum, Ofir and Tucker, George and Schuurmans, Dale},
  journal={arXiv preprint arXiv:2012.06919},
  year={2020}
}
@article{pacchiano2020regret,
  title={Regret Bound Balancing and Elimination for Model Selection in Bandits and RL},
  author={Pacchiano, Aldo and Dann, Christoph and Gentile, Claudio and Bartlett, Peter},
  journal={arXiv preprint arXiv:2012.13045},
  year={2020}
}
@inproceedings{wu2020stochastic,
  title={Stochastic Linear Contextual Bandits with Diverse Contexts},
  author={Wu, Weiqiang and Yang, Jing and Shen, Cong},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2392--2401},
  year={2020},
  organization={PMLR}
}
@article{agarwal2020flambe,
  title={FLAMBE: Structural Complexity and Representation Learning of Low Rank MDPs},
  author={Agarwal, Alekh and Kakade, Sham M and Krishnamurthy, Akshay and Sun, Wen},
  journal={Advances in neural information processing systems},
  year={2020}
}
@article{modi2021model,
  title={Model-free Representation Learning and Exploration in Low-rank MDPs},
  author={Modi, Aditya and Chen, Jinglin and Krishnamurthy, Akshay and Jiang, Nan and Agarwal, Alekh},
  journal={arXiv preprint arXiv:2102.07035},
  year={2021}
}
@inproceedings{yang2020reinforcement,
  title={Reinforcement learning in feature space: Matrix bandit, kernels, and regret bound},
  author={Yang, Lin and Wang, Mengdi},
  booktitle={International Conference on Machine Learning},
  pages={10746--10756},
  year={2020},
  organization={PMLR}
}
@inproceedings{chu2011contextual,
  title={Contextual bandits with linear payoff functions},
  author={Chu, Wei and Li, Lihong and Reyzin, Lev and Schapire, Robert},
  booktitle={Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics},
  pages={208--214},
  year={2011},
  organization={JMLR Workshop and Conference Proceedings}
}
@inproceedings{du2019provably1,
  title={Provably efficient RL with rich observations via latent state decoding},
  author={Du, Simon and Krishnamurthy, Akshay and Jiang, Nan and Agarwal, Alekh and Dudik, Miroslav and Langford, John},
  booktitle={International Conference on Machine Learning},
  pages={1665--1674},
  year={2019},
  organization={PMLR}
}
@inproceedings{misra2020kinematic,
  title={Kinematic state abstraction and provably efficient rich-observation reinforcement learning},
  author={Misra, Dipendra and Henaff, Mikael and Krishnamurthy, Akshay and Langford, John},
  booktitle={International conference on machine learning},
  pages={6961--6971},
  year={2020},
  organization={PMLR}
}
@article{foster2020instance,
  title={Instance-dependent complexity of contextual bandits and reinforcement learning: A disagreement-based perspective},
  author={Foster, Dylan J and Rakhlin, Alexander and Simchi-Levi, David and Xu, Yunzong},
  journal={arXiv preprint arXiv:2010.03104},
  year={2020}
}
@inproceedings {zhou2020nearly,
  title={Nearly Minimax Optimal Reinforcement Learning for Linear Mixture Markov Decision Processes},
  author={Zhou, Dongruo and Gu, Quanquan and Szepesvari, Csaba},
  booktitle={Conference on Learning Theory},
  year={2021},
  organization={PMLR}
}
@article{azuma1967weighted,
  title={Weighted sums of certain dependent random variables},
  author={Azuma, Kazuoki},
  journal={Tohoku Mathematical Journal, Second Series},
  volume={19},
  number={3},
  pages={357--367},
  year={1967},
  publisher={Mathematical Institute, Tohoku University}
}
@article{bartlett2005local,
  title={Local rademacher complexities},
  author={Bartlett, Peter L and Bousquet, Olivier and Mendelson, Shahar and others},
  journal={The Annals of Statistics},
  volume={33},
  number={4},
  pages={1497--1537},
  year={2005},
  publisher={Institute of Mathematical Statistics}
}
@article{papinireinforcement,
  title={Reinforcement Learning in Linear MDPs: Constant Regret and Representation Selection},
  author={Papini, Matteo and Tirinzoni, Andrea and Pacchiano, Aldo and Restelli, Marcello and Lazaric, Alessandro and Pirotta, Matteo},
  year={2021}
}
@inproceedings{ghosh2017misspecified,
  title={Misspecified linear bandits},
  author={Ghosh, Avishek and Chowdhury, Sayak Ray and Gopalan, Aditya},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={31},
  number={1},
  year={2017}
}
@article{foster2021adapting,
  title={Adapting to misspecification in contextual bandits},
  author={Foster, Dylan J and Gentile, Claudio and Mohri, Mehryar and Zimmert, Julian},
  journal={arXiv preprint arXiv:2107.05745},
  year={2021}
}
@inproceedings{
yin2022nearoptimal,
title={Near-optimal Offline Reinforcement Learning with Linear Representation: Leveraging Variance Information with Pessimism},
author={Ming Yin and Yaqi Duan and Mengdi Wang and Yu-Xiang Wang},
booktitle={International Conference on Learning Representations},
year={2022},
}
@article{uehara2021representation,
  title={Representation Learning for Online and Offline RL in Low-rank MDPs},
  author={Uehara, Masatoshi and Zhang, Xuezhou and Sun, Wen},
  journal={arXiv preprint arXiv:2110.04652},
  year={2021}
}
@inproceedings{jin2021pessimism,
  title={Is pessimism provably efficient for offline rl?},
  author={Jin, Ying and Yang, Zhuoran and Wang, Zhaoran},
  booktitle={International Conference on Machine Learning},
  pages={5084--5096},
  year={2021},
  organization={PMLR}
}

@article{dann2017unifying,
  title={Unifying PAC and regret: Uniform PAC bounds for episodic reinforcement learning},
  author={Dann, Christoph and Lattimore, Tor and Brunskill, Emma},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  year={2017}
}
@inproceedings{cai2020provably,
  title={Provably efficient exploration in policy optimization},
  author={Cai, Qi and Yang, Zhuoran and Jin, Chi and Wang, Zhaoran},
  booktitle={International Conference on Machine Learning},
  pages={1283--1294},
  year={2020},
  organization={PMLR}
}

@inproceedings{wang2020statistical,
  title={What are the Statistical Limits of Offline RL with Linear Function Approximation?},
  author={Wang, Ruosong and Foster, Dean and Kakade, Sham M},
  booktitle={International Conference on Learning Representations},
  year={2020}
}
@article{min2021variance,
  title={Variance-aware off-policy evaluation with linear function approximation},
  author={Min, Yifei and Wang, Tianhao and Zhou, Dongruo and Gu, Quanquan},
  journal={Advances in neural information processing systems},
  volume={34},
  year={2021}
}
@article{xie2021bellman,
  title={Bellman-consistent pessimism for offline reinforcement learning},
  author={Xie, Tengyang and Cheng, Ching-An and Jiang, Nan and Mineiro, Paul and Agarwal, Alekh},
  journal={Advances in neural information processing systems},
  volume={34},
  year={2021}
}
@article{yin2021near,
  title={Near-optimal offline reinforcement learning via double variance reduction},
  author={Yin, Ming and Bai, Yu and Wang, Yu-Xiang},
  journal={Advances in neural information processing systems},
  volume={34},
  year={2021}
}
@article{uehara2021pessimistic,
  title={Pessimistic model-based offline rl: Pac bounds and posterior sampling under partial coverage},
  author={Uehara, Masatoshi and Sun, Wen},
  journal={arXiv e-prints},
  pages={arXiv--2107},
  year={2021}
}
@article{foster2021offline,
  title={Offline Reinforcement Learning: Fundamental Barriers for Value Function Approximation},
  author={Foster, Dylan J and Krishnamurthy, Akshay and Simchi-Levi, David and Xu, Yunzong},
  journal={arXiv preprint arXiv:2111.10919},
  year={2021}
}
@article{wang2022gap,
  title={On Gap-dependent Bounds for Offline Reinforcement Learning},
  author={Wang, Xinqi and Cui, Qiwen and Du, Simon S},
  journal={arXiv preprint arXiv:2206.00177},
  year={2022}
}
@inproceedings{qiu2022contrastive,
  title={Contrastive ucb: Provably efficient contrastive self-supervised learning in online reinforcement learning},
  author={Qiu, Shuang and Wang, Lingxiao and Bai, Chenjia and Yang, Zhuoran and Wang, Zhaoran},
  booktitle={International Conference on Machine Learning},
  pages={18168--18210},
  year={2022},
  organization={PMLR}
}
@inproceedings{zhang2022making,
  title={Making linear mdps practical via contrastive representation learning},
  author={Zhang, Tianjun and Ren, Tongzheng and Yang, Mengjiao and Gonzalez, Joseph and Schuurmans, Dale and Dai, Bo},
  booktitle={International Conference on Machine Learning},
  pages={26447--26466},
  year={2022},
  organization={PMLR}
}


@inproceedings{sun2019model,
  title={Model-based rl in contextual decision processes: Pac bounds and exponential improvements over model-free approaches},
  author={Sun, Wen and Jiang, Nan and Krishnamurthy, Akshay and Agarwal, Alekh and Langford, John},
  booktitle={Conference on learning theory},
  pages={2898--2933},
  year={2019},
  organization={PMLR}
}

@inproceedings{du2021bilinear,
  title={Bilinear classes: A structural framework for provable generalization in rl},
  author={Du, Simon and Kakade, Sham and Lee, Jason and Lovett, Shachar and Mahajan, Gaurav and Sun, Wen and Wang, Ruosong},
  booktitle={International Conference on Machine Learning},
  pages={2826--2836},
  year={2021},
  organization={PMLR}
}
@article{uehara2022provably,
  title={Provably efficient reinforcement learning in partially observable dynamical systems},
  author={Uehara, Masatoshi and Sekhari, Ayush and Lee, Jason D and Kallus, Nathan and Sun, Wen},
  journal={arXiv preprint arXiv:2206.12020},
  year={2022}
}