%% This BibTeX bibliography file was created using BibDesk.
%% http://bibdesk.sourceforge.net/

%% Saved with string encoding Unicode (UTF-8) 


@string{aap = {Advances in Applied Probability}}

@string{acmsig = {Proc. of ACM SIGMETRICS}}

@string{annap = {The Annals of Applied Probability}}

@string{annprob = {The Annals of Probability}}

@string{ap = {Academic Press}}

@string{aw = {Addison-Wesley}}

@string{belltech = {Bell System Technical Journal}}

@string{cdc = {{IEEE Conf. on Decision and Control}}}

@string{columbia = {Columbia University, New York, NY}}

@string{cssm = {Comm. Statist. Stochastic Models}}

@string{ctr = {Center for Telecommunications Research}}

@string{dp = {Duxbury Press}}

@string{focs = {FOCS}}

@string{globecom = {{GLOBECOM}}}

@string{gtelabs = {GTE Laboratories}}

@string{icdcs = {ICDCS}}

@string{icnp = {ICNP}}

@string{ims = {Institute of Mathematical Statistics}}

@string{infocom = {{INFOCOM}}}

@string{intjcomm = {Int'l. Journal of Communication Systems}}

@string{intl = {Int'l.}}

@string{jap = {Journal of Applied Probability}}

@string{jb = {Jones and Bartlett}}

@string{jsac = {IEEE Journal on Selected Areas in Communications}}

@string{jws = {John Wiley \& Sons}}



@string{lncs = {Lecture Notes in Computer Science}}

@string{mac = {Macmillan}}

@string{mh = {McGraw-Hill}}

@string{mobicom = {{MobiCom}}}

@string{mobihoc = {{MobiHoc}}}

@string{mobisys = {{MobiSys}}}

@string{ncsu = {North Carolina State University, Raleigh, NC}}

@string{netmag = {IEEE Network}}

@string{or = {Operations Research}}

@string{osu = {The Ohio State University, Columbus, OH}}

@string{pcps = {Proc. Cambridge Philos. Soc.}}

@string{perfeval = {Performance Evaluation}}

@string{ph = {Prentice-Hall}}

@string{proc = {Proc.}}

@string{proieee = {Proc. IEEE}}

@string{purdue = {Purdue University, West Lafayette, IN}}

@string{qs = {Queueing Systems}}

@string{sensys = {{SenSys}}}

@string{siamjapm = {SIAM Journal on Applied Mathematics}}

@string{siamjdism = {SIAM Journal of Discrete Mathematics}}

@string{siamreview = {SIAM Review}}

@string{sigcomm = {{SIGCOMM}}}

@string{sigmetrics = {{SIGMETRICS}}}

@string{stoc = {STOC}}

@string{stomodel = {Stochastic Models}}

@string{stoproapp = {Stochastic Processes and their Applications}}

@string{sv = {Springer-Verlag}}

@string{symp = {Symp.}}

@string{trans = {Trans.}}

@string{transcomm = {IEEE Trans. on Communications}}

@string{transcont = {IEEE Trans. on Automatic Control}}

@string{transinfo = {IEEE Trans. on Information Theory}}

@string{transnet = {IEEE/ACM Trans. on Networking}}

@string{twc = {IEEE Trans. on Wireless Computing}}

@string{wbs = {W. B. Saunders}}

@article{robbins1952some,
  title={Some aspects of the sequential design of experiments},
  author={Robbins, Herbert},
  journal={Bulletin of the American Mathematical Society},
  volume={58},
  number={5},
  pages={527--535},
  year={1952}
}

@article{berry1985bandit,
  title={Bandit problems: sequential allocation of experiments (Monographs on statistics and applied probability)},
  author={Berry, Donald A and Fristedt, Bert},
  journal={London: Chapman and Hall},
  volume={5},
  number={71-87},
  pages={7--7},
  year={1985},
  publisher={Springer}
}

@inproceedings{rbmle,
  title={Exploration Through Reward Biasing: Reward-Biased Maximum Likelihood Estimation for Stochastic Multi-Armed Bandits},
  author={Liu, Xi and Hsieh, Ping-Chun and Hung, Yu-Heng and Bhattacharya, Anirban and Kumar, PR},
  booktitle={International Conference on Machine Learning},
  year={2020}
}
@inproceedings{singh2015decentralized,
  title={Decentralized throughput maximizing policies for deadline-constrained wireless networks},
  author={Singh, Rahul and Kumar, P.~R.},
  booktitle={2015 54th IEEE Conference on Decision and Control (CDC)},
  pages={3759--3766},
  year={2015},
  organization={IEEE}
}

@article{singh2018throughput,
  title={Throughput optimal decentralized scheduling of multihop networks with end-to-end deadline constraints: Unreliable links},
  author={Singh, Rahul and Kumar, {P. R.}},
  journal={IEEE Transactions on Automatic Control},
  volume={64},
  number={1},
  pages={127--142},
  year={2018},
  publisher={IEEE}
}

@inproceedings{li2010contextual,
	title={A contextual-bandit approach to personalized news article recommendation},
	author={Li, Lihong and Chu, Wei and Langford, John and Schapire, Robert E},
	booktitle={Proceedings of the 19th international conference on World wide web},
	pages={661--670},
	year={2010}
}


@article{mitrophanov2005sensitivity,
	title={Sensitivity and convergence of uniformly ergodic {M}arkov chains},
	author={Mitrophanov, A Yu},
	journal={Journal of Applied Probability},
	volume={42},
	number={4},
	pages={1003--1014},
	year={2005},
	publisher={Cambridge University Press}
}


@book{gittins2011multi,
  title={Multi-armed bandit allocation indices},
  author={Gittins, John and Glazebrook, Kevin and Weber, Richard},
  year={2011},
  publisher={John Wiley \& Sons}
}

@article{thompson1933likelihood,
  title={On the likelihood that one unknown probability exceeds another in view of the evidence of two samples},
  author={Thompson, William R},
  journal={Biometrika},
  volume={25},
  number={3/4},
  pages={285--294},
  year={1933},
  publisher={JSTOR}
}

@book{sutton2018reinforcement,
	title={Reinforcement learning: An introduction},
	author={Sutton, Richard S and Barto, Andrew G},
	year={2018},
	publisher={MIT press}
}

@article{lai1985asymptotically,
  title={Asymptotically efficient adaptive allocation rules},
  author={Lai, Tze Leung and Robbins, Herbert},
  journal={Advances in applied mathematics},
  volume={6},
  number={1},
  pages={4--22},
  year={1985},
  publisher={Elsevier}
}



@inproceedings{auer2000using,
	title={Using upper confidence bounds for online learning},
	author={Auer, Peter},
	booktitle={Proceedings 41st Annual Symposium on Foundations of Computer Science},
	pages={270--279},
	year={2000},
	organization={IEEE}
}


@article{auer2002finite,
  title={Finite-time analysis of the multiarmed bandit problem},
  author={Auer, Peter and Cesa-Bianchi, Nicolo and Fischer, Paul},
  journal={Machine learning},
  volume={47},
  number={2-3},
  pages={235--256},
  year={2002},
  publisher={Springer}
}

@inproceedings{agrawal2012analysis,
  title={Analysis of Thompson Sampling for the Multi-armed Bandit Problem.},
  author={Agrawal, Shipra and Goyal, Navin},
  booktitle={COLT},
  pages={39.1--39.26},
  year={2012}
}
@article{agrawal1995sample,
	title={Sample mean based index policies by o (log n) regret for the multi-armed bandit problem},
	author={Agrawal, Rajeev},
	journal={Advances in Applied Probability},
	volume={27},
	number={4},
	pages={1054--1078},
	year={1995},
	publisher={Cambridge University Press}
}

@article{bubeck2012regret,
  title={Regret analysis of stochastic and nonstochastic multi-armed bandit problems},
  author={Bubeck, S{\'e}bastien and Cesa-Bianchi, Nicolo},
  journal={arXiv preprint arXiv:1204.5721},
  year={2012}
}
@article{azuma1967weighted,
  title={Weighted sums of certain dependent random variables},
  author={Azuma, Kazuoki},
  journal={Tohoku Mathematical Journal, Second Series},
  volume={19},
  number={3},
  pages={357--367},
  year={1967},
  publisher={Mathematical Institute, Tohoku University}
}

@article{garivier2008upper,
  title={On upper-confidence bound policies for non-stationary bandit problems},
  author={Garivier, Aur{\'e}lien and Moulines, Eric},
  journal={arXiv preprint arXiv:0805.3415},
  year={2008}
}

@inproceedings{garivier2011upper,
  title={On upper-confidence bound policies for switching bandit problems},
  author={Garivier, Aur{\'e}lien and Moulines, Eric},
  booktitle={International Conference on Algorithmic Learning Theory},
  pages={174--188},
  year={2011},
  organization={Springer}
}

@inproceedings{yu2009piecewise,
  title={Piecewise-stationary bandit problems with side observations},
  author={Yu, Jia Yuan and Mannor, Shie},
  booktitle={Proceedings of the 26th Annual International Conference on Machine Learning},
  pages={1177--1184},
  year={2009},
  organization={ACM}
}

@inproceedings{besbes2014stochastic,
  title={Stochastic multi-armed-bandit problem with non-stationary rewards},
  author={Besbes, Omar and Gur, Yonatan and Zeevi, Assaf},
  booktitle={Advances in neural information processing systems},
  pages={199--207},
  year={2014}
}

@inproceedings{mellor2013thompson,
  title={Thompson sampling in switching environments with Bayesian online change detection},
  author={Mellor, Joseph and Shapiro, Jonathan},
  booktitle={Proceedings of the Sixteenth International Conference on Artificial Intelligence and Statistics},
  pages={442--450},
  year={2013}
}

@article{hartland2007change,
  title={Change point detection and meta-bandits for online learning in dynamic environments},
  author={Hartland, C{\'e}dric and Baskiotis, Nicolas and Gelly, Sylvain and Sebag, Michele and Teytaud, Olivier},
  journal={CAp},
  pages={237--250},
  year={2007}
}

@inproceedings{alaya2008dynamic,
  title={Dynamic spectrum access with non-stationary multi-armed bandit},
  author={Alaya-Feki, Afef Ben Hadj and Moulines, Eric and LeCornec, Alain},
  booktitle={2008 IEEE 9th Workshop on Signal Processing Advances in Wireless Communications},
  pages={416--420},
  year={2008},
  organization={IEEE}
}

@inproceedings{wei2016tracking,
  title={Tracking the Best Expert in Non-stationary Stochastic Environments},
  author={Wei, Chen-Yu and Hong, Yi-Te and Lu, Chi-Jen},
  booktitle={Advances In Neural Information Processing Systems},
  pages={3972--3980},
  year={2016}
}

@article{hinkley1971inference,
  title={Inference about the change-point from cumulative sum tests},
  author={Hinkley, David V},
  journal={Biometrika},
  pages={509--523},
  year={1971},
  publisher={JSTOR}
}


@article{doi:10.1137/S0097539701398375,
author = {Peter Auer and Nicolò Cesa-Bianchi and Yoav Freund and Robert E. Schapire},
title = {The Nonstochastic Multiarmed Bandit Problem},
journal = {SIAM Journal on Computing},
volume = {32},
number = {1},
pages = {48-77},
year = {2002}
}

@book{basseville1993detection,
  title={Detection of abrupt changes: theory and application},
  author={Basseville, Mich{\`e}le and Nikiforov, Igor V},
  volume={104},
  year={1993},
  publisher={Prentice Hall Englewood Cliffs}
}

@article{lorden1971procedures,
  title={Procedures for reacting to a change in distribution},
  author={Lorden, Gary},
  journal={The Annals of Mathematical Statistics},
  pages={1897--1908},
  year={1971},
  publisher={JSTOR}
}

@article{page1954continuous,
  title={Continuous inspection schemes},
  author={Page, E. S.},
  journal={Biometrika},
  volume={41},
  number={1/2},
  pages={100--115},
  year={1954},
  publisher={JSTOR}
}

@article{khan1981note,
  title={A note on Page's two-sided cumulative sum procedure},
  author={Khan, Rasul A},
  journal={Biometrika},
  pages={717--719},
  year={1981},
  publisher={JSTOR}
}

@misc{YahooWebScope,
  title= {Webscope program},
  author= {Yahoo!},
  howpublished={\url{http://webscope.sandbox.yahoo.com/catalog.php?datatype=r&did=49}},
  note={{[Online; accessed 18-Oct-2016]}}
}

@book{pollard1984convergence,
  title={Convergence of Stochastic Processes},
  author={Pollard, D},
  year={1984},
  publisher={Springer}
}

@book{gallager2012discrete,
  title={Discrete stochastic processes},
  author={Gallager, Robert G},
  volume={321},
  year={2012},
  publisher={Springer Science \& Business Media}
}

@inproceedings{li2011unbiased,
  title={Unbiased offline evaluation of contextual-bandit-based news article recommendation algorithms},
  author={Li, Lihong and Chu, Wei and Langford, John and Wang, Xuanhui},
  booktitle={Proceedings of the fourth ACM international conference on Web search and data mining},
  pages={297--306},
  year={2011},
  organization={ACM}
}

@inproceedings{li2016collaborative,
  title={Collaborative filtering bandits},
  author={Li, Shuai and Karatzoglou, Alexandros and Gentile, Claudio},
  booktitle={Proceedings of the 39th International ACM SIGIR conference on Research and Development in Information Retrieval},
  pages={539--548},
  year={2016},
  organization={ACM}
}

@article{buccapatnam2014stochastic,
  title={Stochastic bandits with side observations on networks},
  author={Buccapatnam, Swapna and Eryilmaz, Atilla and Shroff, Ness B},
  journal={ACM SIGMETRICS Performance Evaluation Review},
  volume={42},
  number={1},
  pages={289--300},
  year={2014},
  publisher={ACM}
}

@inproceedings{kaufmann2012thompson,
  title={Thompson sampling: An asymptotically optimal finite-time analysis},
  author={Kaufmann, Emilie and Korda, Nathaniel and Munos, R{\'e}mi},
  booktitle={International Conference on Algorithmic Learning Theory},
  pages={199--213},
  year={2012},
  organization={Springer}
}

@inproceedings{kocsis2006discounted,
  title={Discounted UCB},
  author={Kocsis, Levente and Szepesv{\'a}ri, Csaba},
  booktitle={2nd PASCAL Challenges Workshop},
  pages={784--791},
  year={2006}
}

@inproceedings{allesiardo2015exp3,
  title={EXP3 with drift detection for the switching bandit problem},
  author={Allesiardo, Robin and F{\'e}raud, Rapha{\"e}l},
  booktitle={Data Science and Advanced Analytics (DSAA), 2015. 36678 2015. IEEE International Conference on},
  pages={1--7},
  year={2015},
  organization={IEEE}
}

@book{cesa2006prediction,
  title={Prediction, learning, and games},
  author={Cesa-Bianchi, Nicolo and Lugosi, G{\'a}bor},
  year={2006},
  publisher={Cambridge university press}
}

@inproceedings{srivastava2014surveillance,
  title={Surveillance in an abruptly changing world via multiarmed bandits},
  author={Srivastava, Vaibhav and Reverdy, Paul and Leonard, Naomi E},
  booktitle={Decision and Control (CDC), 2014 IEEE 53rd Annual Conference on},
  pages={692--697},
  year={2014},
  organization={IEEE}
}

@article{buccapatnam2017reward,
  title={Reward Maximization Under Uncertainty: Leveraging Side-Observations on Networks},
  author={Buccapatnam, Swapna and Liu, Fang and Eryilmaz, Atilla and Shroff, Ness B},
  journal={arXiv preprint arXiv:1704.07943},
  year={2017}
}

@article{liu2017change,
  title={A Change-Detection based Framework for Piecewise-stationary Multi-Armed Bandit Problem},
  author={Liu, Fang and Lee, Joohyun and Shroff, Ness},
  journal={arXiv preprint arXiv:1711.03539},
  year={2017}
}
@article{altman1994rate,
  title={Rate of convergence of empirical measures and costs in controlled {M}arkov chains and transient optimality},
  author={Altman, Eitan and Zeitouni, Ofer},
  journal={Mathematics of operations Research},
  volume={19},
  number={4},
  pages={955--974},
  year={1994},
  publisher={INFORMS}
}
@book{ross2014introduction,
  title={Introduction to stochastic dynamic programming},
  author={Ross, Sheldon M},
  year={2014},
  publisher={Academic press}
}
@article{jaksch2010near,
  title={Near-optimal regret bounds for reinforcement learning},
  author={Jaksch, Thomas and Ortner, Ronald and Auer, Peter},
  journal={Journal of Machine Learning Research},
  volume={11},
  number={Apr},
  pages={1563--1600},
  year={2010}
}
@article{liu2017change,
  title={A change-detection based framework for piecewise-stationary multi-armed bandit problem},
  author={Liu, Fang and Lee, Joohyun and Shroff, Ness},
  journal={arXiv preprint arXiv:1711.03539},
  year={2017}
}
@inproceedings{ouyang2017learning,
  title={Learning unknown {M}arkov decision processes: A thompson sampling approach},
  author={Ouyang, Yi and Gagrani, Mukul and Nayyar, Ashutosh and Jain, Rahul},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1333--1342},
  year={2017}
}

@book{folland2013real,
  title={Real analysis: modern techniques and their applications},
  author={Folland, Gerald B},
  year={2013},
  publisher={John Wiley \& Sons}
}

@book{puterman2014markov,
  title={{M}arkov decision processes: discrete stochastic dynamic programming},
  author={Puterman, Martin L},
  year={2014},
  publisher={John Wiley \& Sons}
}

@article{hajek1982hitting,
  title={Hitting-time and occupation-time bounds implied by drift analysis with applications},
  author={Hajek, Bruce},
  journal={Advances in Applied probability},
  pages={502--525},
  year={1982},
  publisher={JSTOR}
}

@book{shiryaev2007optimal,
  title={Optimal stopping rules},
  author={Shiryaev, Albert N},
  volume={8},
  year={2007},
  publisher={Springer Science \& Business Media}
}

% recent NS

@article{russac2020algorithms,
  title={Algorithms for Non-Stationary Generalized Linear Bandits},
  author={Russac, Yoan and Capp{\'e}, Olivier and Garivier, Aur{\'e}lien},
  journal={arXiv preprint arXiv:2003.10113},
  year={2020}
}

@article{domingues2020kernel,
  title={A kernel-based approach to non-stationary reinforcement learning in metric spaces},
  author={Domingues, Omar Darwiche and M{\'e}nard, Pierre and Pirotta, Matteo and Kaufmann, Emilie and Valko, Michal},
  journal={arXiv preprint arXiv:2007.05078},
  year={2020}
}

@article{gajane2018sliding,
  title={A sliding-window algorithm for {M}arkov decision processes with arbitrarily changing rewards and transitions},
  author={Gajane, Pratik and Ortner, Ronald and Auer, Peter},
  journal={arXiv preprint arXiv:1805.10066},
  year={2018}
}

@inproceedings{ortner2020variational,
  title={Variational regret bounds for reinforcement learning},
  author={Ortner, Ronald and Gajane, Pratik and Auer, Peter},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={81--90},
  year={2020},
  organization={PMLR}
}

@article{cheung2020reinforcement,
  title={Reinforcement learning for non-stationary {M}arkov decision processes: The blessing of (more) optimism},
  author={Cheung, Wang Chi and Simchi-Levi, David and Zhu, Ruihao},
  journal={arXiv preprint arXiv:2006.14389},
  year={2020}
}


@article{vega2003average,
	title={The average cost optimality equation: a fixed point approach},
	author={Vega-Amaya, Oscar},
	journal={Bol. Soc. Mat. Mexicana},
	volume={9},
	number={1},
	pages={185--195},
	year={2003}
}

@article{gordienko1995average,
	title={Average cost {M}arkov control processes with weighted norms: existence of canonical policies},
	author={Gordienko, Evgueni and Hern{\'a}ndez-Lerma, On{\'e}simo},
	journal={Applicationes Mathematicae},
	volume={23},
	pages={199--218},
	year={1995},
	publisher={Citeseer}
}
@article{jaskiewicz2006optimality,
	title={On the optimality equation for average cost {M}arkov control processes with Feller transition probabilities},
	author={Ja{\'s}kiewicz, Anna and Nowak, Andrzej S},
	journal={Journal of mathematical analysis and applications},
	volume={316},
	number={2},
	pages={495--509},
	year={2006},
	publisher={Elsevier}
}


@article{osband2014model,
  title={Model-based reinforcement learning and the eluder dimension},
  author={Osband, Ian and Van Roy, Benjamin},
  journal={Advances in Neural Information Processing Systems},
  volume={27},
  year={2014}
}

@inproceedings{abbasi2011regret,
	title={Regret bounds for the adaptive control of linear quadratic systems},
	author={Abbasi-Yadkori, Yasin and Szepesv{\'a}ri, Csaba},
	booktitle={Proceedings of the 24th Annual Conference on Learning Theory},
	pages={1--26},
	year={2011},
	organization={JMLR Workshop and Conference Proceedings}
}

@article{ortner2013online,
	title={Online regret bounds for undiscounted continuous reinforcement learning},
	author={Ortner, Ronald and Ryabko, Daniil},
	journal={arXiv preprint arXiv:1302.2550},
	year={2013}
}

@inproceedings{lakshmanan2015improved,
	title={Improved regret bounds for undiscounted continuous reinforcement learning},
	author={Lakshmanan, Kailasam and Ortner, Ronald and Ryabko, Daniil},
	booktitle={International Conference on Machine Learning},
	pages={524--532},
	year={2015},
	organization={PMLR}
}

@article{ortner2012online,
	title={Online regret bounds for undiscounted continuous reinforcement learning},
	author={Ortner, Ronald and Ryabko, Daniil},
	journal={Advances in Neural Information Processing Systems},
	volume={25},
	year={2012}
}

@article{kakade2020information,
  title={Information theoretic regret bounds for online nonlinear control},
  author={Kakade, Sham and Krishnamurthy, Akshay and Lowrey, Kendall and Ohnishi, Motoya and Sun, Wen},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={15312--15325},
  year={2020}
}

@article{mania2019certainty,
	title={Certainty equivalent control of lqr is efficient},
	author={Mania, Horia and Tu, Stephen and Recht, Benjamin},
	journal={arXiv preprint arXiv:1902.07826},
	year={2019},
	publisher={Technical Report}
}

@article{saldi2017asymptotic,
	title={On the asymptotic optimality of finite approximations to {M}arkov decision processes with Borel spaces},
	author={Saldi, Naci and Y{\"u}ksel, Serdar and Linder, Tam{\'a}s},
	journal={Mathematics of Operations Research},
	volume={42},
	number={4},
	pages={945--978},
	year={2017},
	publisher={INFORMS}
}

@book{hernandez2012discrete,
	title={Discrete-time {M}arkov control processes: basic optimality criteria},
	author={Hern{\'a}ndez-Lerma, On{\'e}simo and Lasserre, Jean B},
	volume={30},
	year={2012},
	publisher={Springer Science \& Business Media}
}

@book{lattimore2020bandit,
	title={Bandit algorithms},
	author={Lattimore, Tor and Szepesv{\'a}ri, Csaba},
	year={2020},
	publisher={Cambridge University Press}
}

@book{hernandez2012adaptive,
	title={Adaptive {M}arkov control processes},
	author={Hern{\'a}ndez-Lerma, On{\'e}simo},
	volume={79},
	year={2012},
	publisher={Springer Science \& Business Media}
}

@article{arapostathis1993discrete,
	title={Discrete-time controlled {M}arkov processes with average cost criterion: a survey},
	author={Arapostathis, Aristotle and Borkar, Vivek S. and Fern{\'a}ndez-Gaucherand, Emmanuel and Ghosh, Mrinal K and Marcus, Steven I},
	journal={SIAM Journal on Control and Optimization},
	volume={31},
	number={2},
	pages={282--344},
	year={1993},
	publisher={SIAM}
}
@article{hernandez1991recurrence,
	title={Recurrence conditions for {M}arkov decision processes with {B}orel state space: a survey},
	author={Hern{\'a}ndez-Lerma, On{\'e}simo and Montes-de-Oca, Ra{\'u}l and Cavazos-Cadena, Rolando},
	journal={Annals of Operations Research},
	volume={28},
	number={1},
	pages={29--46},
	year={1991},
	publisher={Springer}
}

@article{roberts2004general,
	title={General state space {M}arkov chains and MCMC algorithms},
	author={Roberts, Gareth O and Rosenthal, Jeffrey S},
	journal={Probability surveys},
	volume={1},
	pages={20--71},
	year={2004},
	publisher={Institute of Mathematical Statistics and Bernoulli Society}
}

@article{nummelin1982geometric,
	title={Geometric ergodicity of Harris recurrent Marcov chains with applications to renewal theory},
	author={Nummelin, Esa and Tuominen, Pekka},
	journal={Stochastic Processes and Their Applications},
	volume={12},
	number={2},
	pages={187--202},
	year={1982},
	publisher={Elsevier}
}

@article{nummelin1978geometric,
	title={Geometric ergodicity and R-positivity for general {M}arkov chains},
	author={Nummelin, Esa and Tweedie, Richard L},
	journal={The Annals of Probability},
	volume={6},
	number={3},
	pages={404--420},
	year={1978},
	publisher={Institute of Mathematical Statistics}
}
@article{nummelin1983rate,
	title={The rate of convergence in Orey's theorem for Harris recurrent {M}arkov chains with applications to renewal theory},
	author={Nummelin, Esa and Tuominen, Pekka},
	journal={Stochastic Processes and Their Applications},
	volume={15},
	number={3},
	pages={295--311},
	year={1983},
	publisher={Elsevier}
}


@book{hernandez2012further,
	title={Further topics on discrete-time {M}arkov control processes},
	author={Hern{\'a}ndez-Lerma, On{\'e}simo and Lasserre, Jean B},
	volume={42},
	year={2012},
	publisher={Springer Science \& Business Media}
}

@inproceedings{mete2021reward,
	title={Reward biased maximum likelihood estimation for reinforcement learning},
	author={Mete, Akshay and Singh, Rahul and Liu, Xi and Kumar, {P. R.}},
	booktitle={Learning for Dynamics and Control},
	pages={815--827},
	year={2021},
	organization={PMLR}
}

@article{kumar1982adaptive,
	title={Adaptive control with a compact parameter set},
	author={Kumar, {P. R.}},
	journal={SIAM Journal on Control and Optimization},
	volume={20},
	number={1},
	pages={9--13},
	year={1982},
	publisher={SIAM}
}


@article{kumar1983simultaneous,
	title={Simultaneous identification and adaptive control of unknown systems over finite parameter sets},
	author={Kumar, {P. R.}},
	journal={IEEE Transactions on Automatic Control},
	volume={28},
	number={1},
	pages={68--76},
	year={1983},
	publisher={IEEE}
}


@article{kumar1982optimal,
	title={Optimal adaptive controllers for unknown {M}arkov chains},
	author={Kumar, {P. R.} and Lin, Woei},
	journal={IEEE Transactions on Automatic Control},
	volume={27},
	number={4},
	pages={765--774},
	year={1982},
	publisher={IEEE}
}


@article{kumar1982new,
	title={A new family of optimal adaptive controllers for {M}arkov chains},
	author={Kumar, {P. R.} and Becker, A},
	journal={IEEE Transactions on Automatic Control},
	volume={27},
	number={1},
	pages={137--146},
	year={1982},
	publisher={IEEE}
}

@book{resnick2019probability,
	title={A Probability Path},
	author={Resnick, Sidney},
	year={2019},
	publisher={Springer}
}

@book{sennott2009stochastic,
	title={Stochastic dynamic programming and the control of queueing systems},
	author={Sennott, Linn I},
	volume={504},
	year={2009},
	publisher={John Wiley \& Sons}
}

@article{sennott1986new,
	title={A new condition for the existence of optimal stationary policies in average cost {M}arkov decision processes},
	author={Sennott, Linn I},
	journal={Operations research letters},
	volume={5},
	number={1},
	pages={17--23},
	year={1986},
	publisher={Elsevier}
}
@article{sennott1989average,
	title={Average cost optimal stationary policies in infinite state {M}arkov decision processes with unbounded costs},
	author={Sennott, Linn I},
	journal={Operations Research},
	volume={37},
	number={4},
	pages={626--633},
	year={1989},
	publisher={INFORMS}
}


@book{aastrom2013adaptive,
	title={Adaptive control},
	author={{\AA}str{\"o}m, Karl J and Wittenmark, Bj{\"o}rn},
	year={2013},
	publisher={Courier Corporation}
}


@book{ioannou2006adaptive,
	title={Adaptive control tutorial},
	author={Ioannou, Petros and Fidan, Bari{\c{s}}},
	year={2006},
	publisher={SIAM}
}

@article{1957markovian,
	title={A {M}arkovian decision process},
	author={Bellman, Richard},
	journal={Journal of mathematics and mechanics},
	volume={6},
	number={5},
	pages={679--684},
	year={1957},
	publisher={JSTOR}
}


@book{meyn2012markov,
	title={{M}arkov chains and stochastic stability},
	author={Meyn, Sean P and Tweedie, Richard L},
	year={2012},
	publisher={Springer Science \& Business Media}
}
@book{kumar2015stochastic,
	title={Stochastic systems: Estimation, identification, and adaptive control},
	author={Kumar, {P. R.} and Varaiya, Pravin},
	year={2015},
	publisher={SIAM}
}
@book{rudin1964principles,
	title={Principles of mathematical analysis},
	author={Rudin, Walter and others},
	volume={3},
	year={1964},
	publisher={McGraw-hill New York}
}

@article{feldbaum1960dual1,
	title={Dual control theory. I},
	author={Feldbaum, A. A.},
	journal={Avtomatika i Telemekhanika},
	volume={21},
	number={9},
	pages={1240--1249},
	year={1960}
}

@article{feldbaum1960dual2,
	title={Dual control theory. II},
	author={Feldbaum, A. A.},
	journal={Avtomatika i Telemekhanika},
	volume={21},
	number={11},
	pages={1453--1464},
	year={1960}
}

@article{kumar1983optimal,
	title={Optimal adaptive control of linear-quadratic-{G}aussian systems},
	author={Kumar, {P. R.}},
	journal={SIAM Journal on Control and Optimization},
	volume={21},
	number={2},
	pages={163--178},
	year={1983},
	publisher={SIAM}
}

@article{borkar1991self,
	title={Self-tuning control of diffusions without the identifiability condition},
	author={Borkar, V. S.},
	journal={Journal of optimization theory and applications},
	volume={68},
	number={1},
	pages={117--138},
	year={1991},
	publisher={Springer}
}

@article{stettner1993nearly,
	title={On nearly self-optimizing strategies for a discrete-time uniformly ergodic adaptive model},
	author={Stettner, Lukasz},
	journal={Applied Mathematics and Optimization},
	volume={27},
	number={2},
	pages={161--177},
	year={1993},
	publisher={Springer}
}

@techreport{BeckerKumar81b,
	title = {{Optimal strategies for the N-armed bandit problem}},
	author = {Becker, Arthur and Kumar, {P. R.}},
	group = {Department of Mathematics},
	year = {1981},
	institution = {Mathematics Research Report No. 81-1, Department of Mathematics, University of Maryland Baltimore County},
	month = {Jan},
}

@ARTICLE{kumar_becker_82,  
	author={Kumar, {P. R.} and Becker, A.},  
	journal={IEEE Transactions on Automatic Control},   
	title={A new family of optimal adaptive controllers for {M}arkov chains},   
	year={1982},  
	volume={27},  
	number={1},  
	pages={137-146},
}
@article{mandl_74,
	title={Estimation and control in {M}arkov chains},
	author={Mandl, P.},
	journal={Advances in Applied Probability},
	pages={40--60},
	year={1974},
	publisher={JSTOR}
}
@ARTICLE{borkar_varaiya_79, 
	author={V. {Borkar} and P. {Varaiya}},  
	journal={IEEE Transactions on Automatic Control},   
	title={{Adaptive control of {M}arkov chains, I: Finite parameter set}},   
	year={1979},  
	volume={24},  
	number={6},  
	pages={953-957}
}
@article{borkar1990kumar,
	title={The {K}umar-{B}ecker-{L}in scheme revisited},
	author={Borkar, V. S.},
	journal={Journal of Optimization Theory and Applications},
	volume={66},
	number={2},
	pages={289--309},
	year={1990},
	publisher={Springer}
}

@article{kumar_83,
	title={Optimal adaptive control of linear-quadratic-Gaussian systems},
	author={Kumar, {P. R.}},
	journal={SIAM Journal on Control and Optimization},
	volume={21},
	number={2},
	pages={163--178},
	year={1983},
	publisher={SIAM}
}
@article{prandini_00,
	title={Adaptive {LQG} Control of Input-Output Systems---A Cost-biased Approach},
	author={Prandini, Maria and Campi, Marco C.},
	journal={SIAM Journal on Control and Optimization},
	volume={39},
	number={5},
	pages={1499--1519},
	year={2000},
	publisher={SIAM}
}
@article{campi_98,
	title={Adaptive linear quadratic Gaussian control: the cost-biased approach revisited},
	author={Campi, Marco C. and Kumar, {P. R.}},
	journal={SIAM Journal on Control and Optimization},
	volume={36},
	number={6},
	pages={1890--1907},
	year={1998},
	publisher={SIAM}
}
@article{bittanti2006adaptive,
	title={Adaptive control of linear time invariant systems: the ``bet on the best" principle},
	author={Bittanti, Sergio and Campi, Marco C. and others},
	journal={Communications in Information \& Systems},
	volume={6},
	number={4},
	pages={299--320},
	year={2006},
	publisher={International Press of Boston}
}
@article{duncan1994almost,
	title={Almost self-optimizing strategies for the adaptive control of diffusion processes},
	author={Duncan, T. E. and Pasik-Duncan, B. and Stettner, L.},
	journal={Journal of optimization theory and applications},
	volume={81},
	number={3},
	pages={479--507},
	year={1994},
	publisher={Springer}
}

@article{ortner2007logarithmic,
	title={Logarithmic online regret bounds for undiscounted reinforcement learning},
	author={Ortner, P and Auer, R},
	journal={Advances in Neural Information Processing Systems},
	volume={19},
	pages={49},
	year={2007}
}
@article{liu_20,
	title={Exploration Through Reward Biasing: Reward-Biased Maximum Likelihood Estimation for Stochastic Multi-Armed Bandits},
	author={Liu, Xi and Hsieh, Ping-Chun and Hung, Yu-Heng and Bhattacharya, Anirban and Kumar, {P. R.}},
	booktitle={International Conference on Machine Learning},
	year={2020}
}

@article{hung2020reward,
	title={Reward-biased maximum likelihood estimation for linear stochastic bandits},
	author={Hung, Yu-Heng and Hsieh, Ping-Chun and Liu, Xi and Kumar, {P. R.}},
	journal={arXiv preprint arXiv:2010.04091},
	year={2020}
}

@article{whitt1978approximations,
	title={Approximations of dynamic programs, I},
	author={Whitt, Ward},
	journal={Mathematics of Operations Research},
	volume={3},
	number={3},
	pages={231--243},
	year={1978},
	publisher={INFORMS}
}

@article{chow1991optimal,
	title={An optimal one-way multigrid algorithm for discrete-time stochastic control},
	author={Chow, CHEE-S and Tsitsiklis, John N},
	journal={IEEE transactions on automatic control},
	volume={36},
	number={8},
	pages={898--914},
	year={1991}
}
@book{shalev2014understanding,
	title={Understanding machine learning: From theory to algorithms},
	author={Shalev-Shwartz, Shai and Ben-David, Shai},
	year={2014},
	publisher={Cambridge university press}
}
@incollection{mahajan2008multi,
	title={Multi-armed bandit problems},
	author={Mahajan, Aditya and Teneketzis, Demosthenis},
	booktitle={Foundations and applications of sensor management},
	pages={121--151},
	year={2008},
	publisher={Springer}
}

@book{orey1971lecture,
	title={Lecture notes on limit theorems for {M}arkov chain transition probabilities},
	author={Orey, Steven},
	volume={34},
	year={1971},
	publisher={Van Nostrand Reinhold Company}
}
@book{khalil,
	author  = {Khalil, Hassan K},
	title = {Nonlinear systems; 3rd ed.},
	publisher  = {Prentice-Hall},
	address  = {Upper Saddle River, NJ},
	year  = {2002}
}

@article{borkar1994ergodic,
	title={Ergodic control of {M}arkov chains with constraints—the general case},
	author={Borkar, Vivek S},
	journal={SIAM journal on control and optimization},
	volume={32},
	number={1},
	pages={176--186},
	year={1994},
	publisher={SIAM}
}
@book{altman1999constrained,
	title={Constrained {M}arkov decision processes},
	author={Altman, Eitan},
	volume={7},
	year={1999},
	publisher={CRC Press}
}
@article{milgrom2002envelope,
	title={Envelope theorems for arbitrary choice sets},
	author={Milgrom, Paul and Segal, Ilya},
	journal={Econometrica},
	volume={70},
	number={2},
	pages={583--601},
	year={2002},
	publisher={Wiley Online Library}
}

@book{bardi1997optimal,
	title={Optimal control and viscosity solutions of {H}amilton-{J}acobi-{B}ellman equations},
	author={Bardi, Martino and Dolcetta, Italo Capuzzo and others},
	volume={12},
	year={1997},
	publisher={Springer}
}
@article{kallenberg2011markov,
	title={{M}arkov decision processes},
	author={Kallenberg, Lodewijk},
	journal={Lecture Notes. University of Leiden},
	pages={2--5},
	year={2011}
}
@article{marbach2001simulation,
	title={Simulation-based optimization of {M}arkov reward processes},
	author={Marbach, Peter and Tsitsiklis, John N},
	journal={IEEE Transactions on Automatic Control},
	volume={46},
	number={2},
	pages={191--209},
	year={2001},
	publisher={IEEE}
}

@book{levin2017markov,
	title={{M}arkov chains and mixing times},
	author={Levin, David A and Peres, Yuval},
	volume={107},
	year={2017},
	publisher={American Mathematical Soc.}
}

@book{haggstrom2002finite,
	title={Finite {M}arkov chains and algorithmic applications},
	author={H{\"a}ggstr{\"o}m, Olle and others},
	number={52},
	year={2002},
	publisher={Cambridge University Press}
}
@book{boyd2004convex,
	title={Convex optimization},
	author={Boyd, Stephen and Boyd, Stephen P and Vandenberghe, Lieven},
	year={2004},
	publisher={Cambridge university press}
}

@article{abbasi2011improved,
	title={Improved algorithms for linear stochastic bandits},
	author={Abbasi-Yadkori, Yasin and P{\'a}l, D{\'a}vid and Szepesv{\'a}ri, Csaba},
	journal={Advances in neural information processing systems},
	volume={24},
	pages={2312--2320},
	year={2011}
}

@incollection{makowski,
	title={The {P}oisson {E}quation for {C}ountable {M}arkov {C}hains: {P}robabilistic {M}ethods and {I}nterpretations},
	author={Makowski, A.M. and Shwartz, Adam},
	booktitle ={Handbook of {M}arkov Decision Processes -- Methods and Applications},
	pages={269--303},
	year={2002},
	publisher = {Kluwer Publishers}, 
	address = {Norwell (MA)},
	editor = {E. Feinberg and A. Shwartz}
}


@InProceedings{domingues2021kernel,
  title = 	 {Kernel-Based Reinforcement Learning: A Finite-Time Analysis},
  author =       {Domingues, Omar Darwiche and Menard, Pierre and Pirotta, Matteo and Kaufmann, Emilie and Valko, Michal},
  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
  pages = 	 {2783--2792},
  year = 	 {2021},
  editor = 	 {Meila, Marina and Zhang, Tong},
  volume = 	 {139},
  series = 	 {Proceedings of Machine Learning Research},
  publisher =    {PMLR}
}


@book{wellner2013weak,
  title={Weak convergence and empirical processes: with applications to statistics},
  author={Wellner, Jon and others},
  year={2013},
  publisher={Springer Science \& Business Media}
}

@article{kleinberg2019bandits,
  title={Bandits and experts in metric spaces},
  author={Kleinberg, Robert and Slivkins, Aleksandrs and Upfal, Eli},
  journal={Journal of the ACM (JACM)},
  volume={66},
  number={4},
  pages={1--77},
  year={2019},
  publisher={ACM New York, NY, USA}
}

@phdthesis{kleinberg2005online,
  title={Online decision problems with large strategy sets},
  author={Kleinberg, Robert David},
  year={2005},
  school={Massachusetts Institute of Technology}
}

@book{cover1999elements,
  title={Elements of information theory},
  author={Cover, Thomas M},
  year={1999},
  publisher={John Wiley \& Sons}
}

@book{billingsley2017probability,
  title={Probability and measure},
  author={Billingsley, Patrick},
  year={2017},
  publisher={John Wiley \& Sons}
}

@book{wasserman2004all,
  title={All of statistics: a concise course in statistical inference},
  author={Wasserman, Larry},
  volume={26},
  year={2004},
  publisher={Springer}
}

#episodic rl

@inproceedings{domingues2021episodic,
  title={Episodic reinforcement learning in finite {MDP}s: Minimax lower bounds revisited},
  author={Domingues, Omar Darwiche and M{\'e}nard, Pierre and Kaufmann, Emilie and Valko, Michal},
  booktitle={Algorithmic Learning Theory},
  pages={578--598},
  year={2021},
  organization={PMLR}
}


@inproceedings{zanette2019tighter,
  title={Tighter problem-dependent regret bounds in reinforcement learning without domain knowledge using value function bounds},
  author={Zanette, Andrea and Brunskill, Emma},
  booktitle={International Conference on Machine Learning},
  pages={7304--7312},
  year={2019},
  organization={PMLR}
}

@article{jin2018q,
  title={Is Q-learning provably efficient?},
  author={Jin, Chi and Allen-Zhu, Zeyuan and Bubeck, Sebastien and Jordan, Michael I},
  journal={Advances in neural information processing systems},
  volume={31},
  year={2018}
}

@inproceedings{azar2017minimax,
  title={Minimax regret bounds for reinforcement learning},
  author={Azar, Mohammad Gheshlaghi and Osband, Ian and Munos, R{\'e}mi},
  booktitle={International Conference on Machine Learning},
  pages={263--272},
  year={2017},
  organization={PMLR}
}

@article{dann2017unifying,
  title={Unifying PAC and regret: Uniform PAC bounds for episodic reinforcement learning},
  author={Dann, Christoph and Lattimore, Tor and Brunskill, Emma},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  year={2017}
}

@article{dann2015sample,
  title={Sample complexity of episodic fixed-horizon reinforcement learning},
  author={Dann, Christoph and Brunskill, Emma},
  journal={Advances in Neural Information Processing Systems},
  volume={28},
  year={2015}
}

@article{simchowitz2019non,
  title={Non-asymptotic gap-dependent regret bounds for tabular {MDP}s},
  author={Simchowitz, Max and Jamieson, Kevin G},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

# zooming bandit

@inproceedings{kleinberg2008multi,
  title={Multi-armed bandits in metric spaces},
  author={Kleinberg, Robert and Slivkins, Aleksandrs and Upfal, Eli},
  booktitle={Proceedings of the fortieth annual ACM symposium on Theory of computing},
  pages={681--690},
  year={2008}
}

# continuous space average reward mdp

@article{ormoneit2002kernel,
  title={Kernel-based reinforcement learning in average-cost problems},
  author={Ormoneit, Dirk and Glynn, Peter},
  journal={IEEE Transactions on Automatic Control},
  volume={47},
  number={10},
  pages={1624--1636},
  year={2002},
  publisher={IEEE}
}

@article{glynn2002hoeffding,
	title={Hoeffding's inequality for uniformly ergodic {M}arkov chains},
	author={Glynn, Peter W and Ormoneit, Dirk},
	journal={Statistics \& probability letters},
	volume={56},
	number={2},
	pages={143--146},
	year={2002},
	publisher={Elsevier}
}

@article{dedecker2015subgaussian,
	title={Subgaussian concentration inequalities for geometrically ergodic {M}arkov chains},
	author={Dedecker, J{\'e}r{\^o}me and Gou{\"e}zel, S{\'e}bastien},
	journal={Electronic Communications in Probability},
	volume={20},
	number={article 64},
	year={2015}
}

@article{zou2009learning,
	title={Learning from uniformly ergodic {M}arkov chains},
	author={Zou, Bin and Zhang, Hai and Xu, Zongben},
	journal={Journal of Complexity},
	volume={25},
	number={2},
	pages={188--200},
	year={2009},
	publisher={Elsevier}
}

@article{fan2021hoeffding,
	title={Hoeffding's inequality for general {M}arkov chains and its applications to statistical learning},
	author={Fan, Jianqing and Jiang, Bai and Sun, Qiang},
	journal={Journal of Machine Learning Research},
	volume={22},
	number={139},
	pages={1--35},
	year={2021}
}

@article{ormoneitsen2002kernel,
	title={Kernel-based reinforcement learning},
	author={Ormoneit, Dirk and Sen, {\'S}aunak},
	journal={Machine learning},
	volume={49},
	pages={161--178},
	year={2002},
	publisher={Springer}
}

# zoomrl

@article{sinclair2019adaptive,
  title={Adaptive discretization for episodic reinforcement learning in metric spaces},
  author={Sinclair, Sean R and Banerjee, Siddhartha and Yu, Christina Lee},
  journal={Proceedings of the ACM on Measurement and Analysis of Computing Systems},
  volume={3},
  number={3},
  pages={1--44},
  year={2019},
  publisher={ACM New York, NY, USA}
}

@article{touati2020zooming,
  title={Zooming for efficient model-free reinforcement learning in metric spaces},
  author={Touati, Ahmed and Taiga, Adrien Ali and Bellemare, Marc G},
  journal={arXiv preprint arXiv:2003.04069},
  year={2020}
}


@article{sinclair2023adaptive,
  title={Adaptive discretization in online reinforcement learning},
  author={Sinclair, Sean R and Banerjee, Siddhartha and Yu, Christina Lee},
  journal={Operations Research},
  volume={71},
  number={5},
  pages={1636--1652},
  year={2023},
  publisher={INFORMS}
}

@article{cao2020provably,
  title={Provably adaptive reinforcement learning in metric spaces},
  author={Cao, Tongyi and Krishnamurthy, Akshay},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={9736--9744},
  year={2020}
}


@article{slivkins2019introduction,
  title={Introduction to multi-armed bandits},
  author={Slivkins, Aleksandrs and others},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={12},
  number={1-2},
  pages={1--286},
  year={2019},
  publisher={Now Publishers, Inc.}
}

#Episodic MDP, restrictive assumptions

@article{zhou2022computationally,
  title={Computationally efficient horizon-free reinforcement learning for linear mixture {MDP}s},
  author={Zhou, Dongruo and Gu, Quanquan},
  journal={Advances in neural information processing systems},
  volume={35},
  pages={36337--36349},
  year={2022}
}

@inproceedings{zhou2021nearly,
  title={Nearly minimax optimal reinforcement learning for linear mixture {M}arkov decision processes},
  author={Zhou, Dongruo and Gu, Quanquan and Szepesvari, Csaba},
  booktitle={Conference on Learning Theory},
  pages={4532--4576},
  year={2021},
  organization={PMLR}
}

@inproceedings{jin2020provably,
  title={Provably efficient reinforcement learning with linear function approximation},
  author={Jin, Chi and Yang, Zhuoran and Wang, Zhaoran and Jordan, Michael I},
  booktitle={Conference on Learning Theory},
  pages={2137--2143},
  year={2020},
  organization={PMLR}
}

@inproceedings{zanette2020learning,
  title={Learning near optimal policies with low inherent {B}ellman error},
  author={Zanette, Andrea and Lazaric, Alessandro and Kochenderfer, Mykel and Brunskill, Emma},
  booktitle={International Conference on Machine Learning},
  pages={10978--10989},
  year={2020},
  organization={PMLR}
}

@article{wang2020reinforcement,
  title={Reinforcement learning with general value function approximation: Provably efficient approach via bounded eluder dimension},
  author={Wang, Ruosong and Salakhutdinov, Russ R and Yang, Lin},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={6123--6135},
  year={2020}
}

@article{jin2021bellman,
  title={{B}ellman eluder dimension: New rich classes of rl problems, and sample-efficient algorithms},
  author={Jin, Chi and Liu, Qinghua and Miryoosefi, Sobhan},
  journal={Advances in neural information processing systems},
  volume={34},
  pages={13406--13418},
  year={2021}
}

@inproceedings{agarwal2023vo,
  title={VO $ Q $ L: Towards Optimal Regret in Model-free RL with Nonlinear Function Approximation},
  author={Agarwal, Alekh and Jin, Yujia and Zhang, Tong},
  booktitle={The Thirty Sixth Annual Conference on Learning Theory},
  pages={987--1063},
  year={2023},
  organization={PMLR}
}


@inproceedings{chowdhury2019online,
  title={Online learning in kernelized {M}arkov decision processes},
  author={Chowdhury, Sayak Ray and Gopalan, Aditya},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={3197--3205},
  year={2019},
  organization={PMLR}
}

#average reward works

@article{burnetas1997optimal,
  title={Optimal adaptive policies for {M}arkov decision processes},
  author={Burnetas, Apostolos N and Katehakis, Michael N},
  journal={Mathematics of Operations Research},
  volume={22},
  number={1},
  pages={222--255},
  year={1997},
  publisher={INFORMS}
}

# nicer policy class example

@techreport{moore1990efficient,
  title={Efficient memory-based learning for robot control},
  author={Moore, Andrew William},
  year={1990},
  institution={University of Cambridge, Computer Laboratory}
}

@article{mardia2020concentration,
  title={Concentration inequalities for the empirical distribution of discrete distributions: beyond the method of types},
  author={Mardia, Jay and Jiao, Jiantao and T{\'a}nczos, Ervin and Nowak, Robert D and Weissman, Tsachy},
  journal={Information and Inference: A Journal of the IMA},
  volume={9},
  number={4},
  pages={813--850},
  year={2020},
  publisher={Oxford University Press}
}

@inproceedings{ayoub2020model,
  title={Model-based reinforcement learning with value-targeted regression},
  author={Ayoub, Alex and Jia, Zeyu and Szepesvari, Csaba and Wang, Mengdi and Yang, Lin},
  booktitle={International Conference on Machine Learning},
  pages={463--474},
  year={2020},
  organization={PMLR}
}

@article{srinivas2009gaussian,
  title={Gaussian process optimization in the bandit setting: No regret and experimental design},
  author={Srinivas, Niranjan and Krause, Andreas and Kakade, Sham M and Seeger, Matthias},
  journal={arXiv preprint arXiv:0912.3995},
  year={2009}
}

@article{russo2013eluder,
  title={Eluder dimension and the sample complexity of optimistic exploration},
  author={Russo, Daniel and Van Roy, Benjamin},
  journal={Advances in Neural Information Processing Systems},
  volume={26},
  year={2013}
}

@article{bartlett2012regal,
  title={REGAL: A regularization based algorithm for reinforcement learning in weakly communicating {MDP}s},
  author={Bartlett, Peter L and Tewari, Ambuj},
  journal={arXiv preprint arXiv:1205.2661},
  year={2012}
}

@inproceedings{fruit2018efficient,
  title={Efficient bias-span-constrained exploration-exploitation in reinforcement learning},
  author={Fruit, Ronan and Pirotta, Matteo and Lazaric, Alessandro and Ortner, Ronald},
  booktitle={International Conference on Machine Learning},
  pages={1578--1586},
  year={2018},
  organization={PMLR}
}

@article{qian2019exploration,
	title={Exploration bonus for regret minimization in discrete and continuous average reward {MDP}s},
	author={Qian, Jian and Fruit, Ronan and Pirotta, Matteo and Lazaric, Alessandro},
	journal={Advances in Neural Information Processing Systems},
	volume={32},
	year={2019}
}

@inproceedings{wu2022nearly,
  title={Nearly minimax optimal regret for learning infinite-horizon average-reward {MDP}s with linear function approximation},
  author={Wu, Yue and Zhou, Dongruo and Gu, Quanquan},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3883--3913},
  year={2022},
  organization={PMLR}
}

@inproceedings{wei2021learning,
  title={Learning infinite-horizon average-reward {MDP}s with linear function approximation},
  author={Wei, Chen-Yu and Jahromi, Mehdi Jafarnia and Luo, Haipeng and Jain, Rahul},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3007--3015},
  year={2021},
  organization={PMLR}
}

@inproceedings{he2023sample,
  title={Sample-efficient Learning of Infinite-horizon Average-reward {MDP}s with General Function Approximation},
  author={He, Jianliang and Zhong, Han and Yang, Zhuoran},
  booktitle={The Twelfth International Conference on Learning Representations},
  year={2023}
}

@book{van1996weak,
  title={Weak convergence},
  author={Van Der Vaart, Aad W and Wellner, Jon A and van der Vaart, Aad W and Wellner, Jon A},
  year={1996},
  publisher={Springer}
}

@article{bercu2008exponential,
	title={Exponential inequalities for self-normalized martingales with applications},
	author={Bercu, Bernard and Touati, Abderrahmen},
	year={2008},
        journal={-}
}

@article{de1999general,
	title={A general class of exponential inequalities for martingales and ratios},
	author={de la Pena, Victor H},
	journal={The Annals of Probability},
	volume={27},
	number={1},
	pages={537--564},
	year={1999},
	publisher={Institute of Mathematical Statistics}
}

@inproceedings{wang2023optimal,
title={Optimal Sample Complexity for Average Reward {M}arkov Decision Processes},
author={Shengbo Wang and Jose Blanchet and Peter Glynn},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024},
url={https://openreview.net/forum?id=jOm5p3q7c7}
}

@article{tossou2019near,
	title={Near-optimal optimistic reinforcement learning using empirical bernstein inequalities},
	author={Tossou, Aristide and Basu, Debabrota and Dimitrakakis, Christos},
	journal={arXiv preprint arXiv:1905.12425},
	year={2019}
}

@inproceedings{he2021logarithmic,
	title={Logarithmic regret for reinforcement learning with linear function approximation},
	author={He, Jiafan and Zhou, Dongruo and Gu, Quanquan},
	booktitle={International Conference on Machine Learning},
	pages={4171--4180},
	year={2021},
	organization={PMLR}
}

@inproceedings{moulos2020hoeffding,
  title={A hoeffding inequality for finite state {M}arkov chains and its applications to {M}arkovian bandits},
  author={Moulos, Vrettos},
  booktitle={2020 IEEE International Symposium on Information Theory (ISIT)},
  pages={2777--2782},
  year={2020},
  organization={IEEE}
}

@inproceedings{ortner2008online,
  title={Online regret bounds for {M}arkov decision processes with deterministic transitions},
  author={Ortner, Ronald},
  booktitle={International Conference on Algorithmic Learning Theory},
  pages={123--137},
  year={2008},
  organization={Springer}
}

@article{pirotta2015policy,
  title={Policy gradient in {L}ipschitz {M}arkov decision processes},
  author={Pirotta, Matteo and Restelli, Marcello and Bascetta, Luca},
  journal={Machine Learning},
  volume={100},
  pages={255--283},
  year={2015},
  publisher={Springer}
}

@article{nair2022r3m,
  title={R3m: A universal visual representation for robot manipulation},
  author={Nair, Suraj and Rajeswaran, Aravind and Kumar, Vikash and Finn, Chelsea and Gupta, Abhinav},
  journal={Confernce on Robot Learning},
  year={2023}
}

@article{kumar2021rma,
  title={{RMA}: {R}apid motor adaptation for legged robots},
  author={Kumar, Ashish and Fu, Zipeng and Pathak, Deepak and Malik, Jitendra},
  journal={arXiv preprint arXiv:2107.04034},
  year={2021}
}

@article{maran2024no,
  title={No-Regret Reinforcement Learning in Smooth {MDP}s},
  author={Maran, Davide and Metelli, Alberto Maria and Papini, Matteo and Restell, Marcello},
  journal={The Forty-first International Conference on Machine Learning},
  year={2024}
}


@article{maran2024projection,
  title={Projection by Convolution: Optimal Sample Complexity for Reinforcement Learning in Continuous-Space {MDP}s},
  author={Maran, Davide and Metelli, Alberto Maria and Papini, Matteo and Restelli, Marcello},
  journal={The 37th Annual Conference on Learning Theory},
  year={2024}
}

@article{song2019efficient,
  title={Efficient model-free reinforcement learning in metric spaces},
  author={Song, Zhao and Sun, Wen},
  journal={arXiv preprint arXiv:1905.00475},
  year={2019}
}

@article{graves1997asymptotically,
  title={Asymptotically efficient adaptive choice of control laws incontrolled {M}arkov chains},
  author={Graves, Todd L and Lai, Tze Leung},
  journal={SIAM journal on control and optimization},
  volume={35},
  number={3},
  pages={715--743},
  year={1997},
  publisher={SIAM}
}

@article{kar2024adaptive,
  title={Adaptive Discretization-based Non-Episodic Reinforcement Learning in Metric Spaces},
  author={Kar, Avik and Singh, Rahul},
  journal={arXiv preprint arXiv:2405.18793v3},
  year={2024}
}

@article{kar2024policy,
  title={Policy Zooming: Adaptive Discretization-based Infinite-Horizon Average-Reward Reinforcement Learning},
  author={Kar, Avik and Singh, Rahul},
  journal={arXiv preprint arXiv:2405.18793v1},
  year={2024}
}
@article{kar2024linear,
  title={Linear Bandits With Side Observations on Networks},
  author={Kar, Avik and Singh, Rahul and Liu, Fang and Liu, Xin and Shroff, Ness B},
  journal={IEEE/ACM Transactions on Networking},
  year={2024},
  publisher={IEEE}
}

@article{metivier1984applications,
  title={Applications of a {K}ushner and {C}lark lemma to general classes of stochastic algorithms},
  author={Metivier, Michel and Priouret, Pierre},
  journal={IEEE Transactions on Information Theory},
  volume={30},
  number={2},
  pages={140--151},
  year={1984},
  publisher={IEEE}
}

@article{bertsekas2011dynamic,
  title={Dynamic Programming and Optimal Control 3rd Edition, Volume II},
  author={Bertsekas, Dimitri P},
  year={2011}
}

@article{raginsky2013concentration,
  title={Concentration of measure inequalities in information theory, communications, and coding},
  author={Raginsky, Maxim and Sason, Igal and others},
  journal={Foundations and Trends{\textregistered} in Communications and Information Theory},
  volume={10},
  number={1-2},
  pages={1--246},
  year={2013},
  publisher={Now Publishers, Inc.}
}


@article{ibarz2021train,
  title={How to train your robot with deep reinforcement learning: lessons we have learned},
  author={Ibarz, Julian and Tan, Jie and Finn, Chelsea and Kalakrishnan, Mrinal and Pastor, Peter and Levine, Sergey},
  journal={The International Journal of Robotics Research},
  volume={40},
  number={4-5},
  pages={698--721},
  year={2021},
  publisher={SAGE Publications Sage UK: London, England}
}

@inproceedings{sodhi2023effectiveness,
  title={On the effectiveness of offline {RL} for dialogue response generation},
  author={Sodhi, Paloma and Wu, Felix and Elenberg, Ethan R and Weinberger, Kilian Q and McDonald, Ryan},
  booktitle={International Conference on Machine Learning},
  pages={32088--32104},
  year={2023},
  organization={PMLR}
}

@article{shah2018q,
  title={Q-learning with nearest neighbors},
  author={Shah, Devavrat and Xie, Qiaomin},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}

@inproceedings{guntuboyina2012l1,
  title={L1 covering numbers for uniformly bounded convex functions},
  author={Guntuboyina, Adityanand and Sen, Bodhisattva},
  booktitle={Conference on Learning Theory},
  pages={12--1},
  year={2012},
  organization={JMLR Workshop and Conference Proceedings}
}

@article{strehl2008analysis,
  title={An analysis of model-based interval estimation for {M}arkov decision processes},
  author={Strehl, Alexander L and Littman, Michael L},
  journal={Journal of Computer and System Sciences},
  volume={74},
  number={8},
  pages={1309--1331},
  year={2008},
  publisher={Elsevier}
}

@article{borkar2000ode,
  title={The {ODE} method for convergence of stochastic approximation and reinforcement learning},
  author={Borkar, Vivek S and Meyn, Sean P},
  journal={SIAM Journal on Control and Optimization},
  volume={38},
  number={2},
  pages={447--469},
  year={2000},
  publisher={SIAM}
}

@book{douc2018markov,
  title={Markov chains: Basic definitions},
  author={Douc, Randal and Moulines, Eric and Priouret, Pierre and Soulier, Philippe},
  year={2018},
  publisher={Springer}
}

@inproceedings{magureanu2014lipschitz,
  title={Lipschitz bandits: Regret lower bound and optimal algorithms},
  author={Magureanu, Stefan and Combes, Richard and Proutiere, Alexandre},
  booktitle={Conference on Learning Theory},
  pages={975--999},
  year={2014},
  organization={PMLR}
}
@article{agrawal1995continuum,
  title={The continuum-armed bandit problem},
  author={Agrawal, Rajeev},
  journal={SIAM journal on control and optimization},
  volume={33},
  number={6},
  pages={1926--1951},
  year={1995},
  publisher={SIAM}
}
@article{kleinberg2004nearly,
  title={Nearly tight bounds for the continuum-armed bandit problem},
  author={Kleinberg, Robert},
  journal={Advances in Neural Information Processing Systems},
  volume={17},
  year={2004}
}
@article{bubeck2011x,
  title={X-Armed Bandits.},
  author={Bubeck, S{\'e}bastien and Munos, R{\'e}mi and Stoltz, Gilles and Szepesv{\'a}ri, Csaba},
  journal={Journal of Machine Learning Research},
  volume={12},
  number={5},
  year={2011}
}

@article{ortner2020regret,
  title={Regret bounds for reinforcement learning via {M}arkov chain concentration},
  author={Ortner, Ronald},
  journal={Journal of Artificial Intelligence Research},
  volume={67},
  pages={115--128},
  year={2020}
}

@inproceedings{hao2021adaptive,
  title={Adaptive approximate policy iteration},
  author={Hao, Botao and Lazic, Nevena and Abbasi-Yadkori, Yasin and Joulani, Pooria and Szepesv{\'a}ri, Csaba},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={523--531},
  year={2021},
  organization={PMLR}
}

@inproceedings{zhang2023sharper,
  title={Sharper model-free reinforcement learning for average-reward {M}arkov decision processes},
  author={Zhang, Zihan and Xie, Qiaomin},
  booktitle={The Thirty Sixth Annual Conference on Learning Theory},
  pages={5476--5477},
  year={2023},
  organization={PMLR}
}