@inproceedings{10.5555/3020652.3020666,
author = {Arora, Raman and Dekel, Ofer and Tewari, Ambuj},
title = {Deterministic MDPs with Adversarial Rewards and Bandit Feedback},
year = {2012},
isbn = {9780974903989},
publisher = {AUAI Press},
address = {Arlington, Virginia, USA},
abstract = {We consider a Markov decision process with deterministic state transition dynamics,
adversarially generated rewards that change arbitrarily from round to round, and a
bandit feedback model in which the decision maker only observes the rewards it receives.
In this setting, we present a novel and efficient online decision making algorithm
named MarcoPolo. Under mild assumptions on the structure of the transition dynamics,
we prove that MarcoPolo enjoys a regret of O(T3/4 √log T) against the best deterministic
policy in hindsight. Specifically, our analysis does not rely on the stringent unichain
assumption, which dominates much of the previous work on this topic.},
booktitle = {Proceedings of the Twenty-Eighth Conference on Uncertainty in Artificial Intelligence},
pages = {93–101},
numpages = {9},
location = {Catalina Island, CA},
series = {UAI'12}
}
@book{markov_chains,
author = {Bremaud, P.},
year = {2000},
publisher="Springer",
title = {Markov Chains: Gibbs Fields, Monte Carlo Simulation, and Queues},
}


@inproceedings{10.5555/3042817.3043012,
author = {Dekel, Ofer and Hazan, Elad},
title = {Better Rates for Any Adversarial Deterministic MDP},
year = {2013},
publisher = {JMLR.org},
abstract = {We consider regret minimization in adversarial deterministic Markov Decision Processes
(ADMDPs) with bandit feedback. We devise a new algorithm that pushes the state-of-the-art
forward in two ways: First, it attains a regret of O(T2/3) with respect to the best
fixed policy in hindsight, whereas the previous best regret bound was O(T3/4). Second,
the algorithm and its analysis are compatible with any feasible ADMDP graph topology,
while all previous approaches required additional restrictions on the graph topology.},
booktitle = {Proceedings of the 30th International Conference on International Conference on Machine Learning - Volume 28},
pages = {III–675–III–683},
location = {Atlanta, GA, USA},
series = {ICML'13}
}

@article{DBLP:journals/tac/NeuGSA14,
  author    = {Gergely Neu and
               Andr{\'{a}}s Gy{\"{o}}rgy and
               Csaba Szepesv{\'{a}}ri and
               Andr{\'{a}}s Antos},
  title     = {Online Markov Decision Processes Under Bandit Feedback},
  journal   = {{IEEE} Trans. Autom. Control.},
  volume    = {59},
  number    = {3},
  pages     = {676--691},
  year      = {2014},
  url       = {https://doi.org/10.1109/TAC.2013.2292137},
  doi       = {10.1109/TAC.2013.2292137},
  timestamp = {Mon, 26 Oct 2020 08:48:32 +0100},
  biburl    = {https://dblp.org/rec/journals/tac/NeuGSA14.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}
@article{10.2307/40538442,
 ISSN = {0364765X, 15265471},
 URL = {http://www.jstor.org/stable/40538442},
 abstract = {We consider a Markov decision process (MDP) setting in which the reward function is allowed to change after each time step (possibly in an adversarial manner), yet the dynamics remain fixed. Similar to the experts setting, we address the question of how well an agent can do when compared to the reward achieved under the best stationary policy over time. We provide efficient algorithms, which have regret bounds with no dependence on the size of state space. Instead, these bounds depend only on a certain horizon time of the process and logarithmically on the number of actions.},
 author = {Eyal Even-Dar and Sham. M. Kakade and Yishay Mansour},
 journal = {Mathematics of Operations Research},
 number = {3},
 pages = {726--736},
 publisher = {INFORMS},
 title = {Online Markov Decision Processes},
 volume = {34},
 year = {2009}
}

@article{switch,
author = {Dekel, Ofer and Ding, Jian and Koren, Tomer and Peres, Yuval},
year = {2013},
month = {10},
pages = {},
title = {Bandits with Switching Costs: $T^{2/3}$ Regret},
journal = {Proceedings of the Annual ACM Symposium on Theory of Computing},
doi = {10.1145/2591796.2591868}
}

@article{10.2307/3689120,
 ISSN = {0364765X, 15265471},
 URL = {http://www.jstor.org/stable/3689120},
 abstract = {Let P be an irreducible N Ã— N matrix having nonnegative entries, and consider the (directed) network containing arc (i, j) if and only if Pij is positive. Theorem 1 expresses the period of this network in terms of any of its arborescences. Theorem 2 shows that if the network's period is one and if node i is contained in a cycle of length n, then the ith row and column of $P^{t}$ are positive for t â‰¥ (N - 1)n. Theorem 3 shows how to compute the network's period with work proportional to the number of its arcs.},
 author = {Eric V. Denardo},
 journal = {Mathematics of Operations Research},
 number = {1},
 pages = {20--24},
 publisher = {INFORMS},
 title = {Periods of Connected Networks and Powers of Nonnegative Matrices},
 volume = {2},
 year = {1977}
}

@article{ORTNER20102684,
title = {Online regret bounds for Markov decision processes with deterministic transitions},
journal = {Theoretical Computer Science},
volume = {411},
number = {29},
pages = {2684-2695},
year = {2010},
note = {Algorithmic Learning Theory (ALT 2008)},
issn = {0304-3975},
doi = {https://doi.org/10.1016/j.tcs.2010.04.005},
url = {https://www.sciencedirect.com/science/article/pii/S0304397510002008},
author = {Ronald Ortner},
keywords = {Markov decision process, Regret, Labeled digraph},
abstract = {We consider an upper confidence bound algorithm for learning in Markov decision processes with deterministic transitions. For this algorithm we derive upper bounds on the online regret with respect to an (ε-)optimal policy that are logarithmic in the number of steps taken. We also present a corresponding lower bound. As an application, multi-armed bandits with switching cost are considered.}
}
@book{10.5555/1062374,
author = {Bazaraa, Mokhtar S. and Jarvis, John J. and Sherali, Hanif D.},
title = {Linear Programming and Network Flows},
year = {2004},
isbn = {0471485993},
publisher = {Wiley-Interscience},
address = {USA}
}
@article{KALAI2005291,
title = {Efficient algorithms for online decision problems},
journal = {Journal of Computer and System Sciences},
volume = {71},
number = {3},
pages = {291-307},
year = {2005},
note = {Learning Theory 2003},
issn = {0022-0000},
doi = {https://doi.org/10.1016/j.jcss.2004.10.016},
url = {https://www.sciencedirect.com/science/article/pii/S0022000004001394},
author = {Adam Kalai and Santosh Vempala},
keywords = {Online algorithms, Hannan's algorithm, Optimization, Decision theory},
abstract = {In an online decision problem, one makes a sequence of decisions without knowledge of the future. Each period, one pays a cost based on the decision and observed state. We give a simple approach for doing nearly as well as the best single decision, where the best is chosen with the benefit of hindsight. A natural idea is to follow the leader, i.e. each period choose the decision which has done best so far. We show that by slightly perturbing the totals and then choosing the best decision, the expected performance is nearly as good as the best decision in hindsight. Our approach, which is very much like Hannan's original game-theoretic approach from the 1950s, yields guarantees competitive with the more modern exponential weighting algorithms like Weighted Majority. More importantly, these follow-the-leader style algorithms extend naturally to a large class of structured online problems for which the exponential algorithms are inefficient.}
}

@book{book,
author = {Cesa-Bianchi, Nicolò and Lugosi, Gábor},
year = {2006},
title = {Prediction, Learning, and Games},
isbn = {978-0-521-84108-5},
publisher = {Cambridge University Press}
}
@INPROCEEDINGS{EXP3,
  author={Auer, P. and Cesa-Bianchi, N. and Freund, Y. and Schapire, R.E.},
  booktitle={Proceedings of IEEE 36th Annual Foundations of Computer Science}, 
  title={Gambling in a rigged casino: The adversarial multi-armed bandit problem}, 
  year={1995},
  volume={},
  number={},
  pages={322-331},
  doi={10.1109/SFCS.1995.492488}}
  
  @article{ftpl_mdp_bandit,
  title={Follow-the-Perturbed-Leader for Adversarial Markov Decision Processes with Bandit Feedback},
  author={Dai, Yan and Luo, Haipeng and Chen, Liyu},
  year={2022},
  journal={Advances in Neural Information Processing Systems, NeurIPS 2022}
}
@article{switching_ub,
author = {Arora, Raman and Dekel, Ofer and Tewari, Ambuj},
year = {2012},
month = {06},
pages = {},
title = {Online Bandit Learning against an Adaptive Adversary: from Regret to
Policy Regret},
volume = {2},
journal = {Proceedings of the 29th International Conference on Machine Learning, ICML 2012}
}

@misc{Haghtalab,
  doi = {10.48550/ARXIV.2202.08549},
  
  url = {https://arxiv.org/abs/2202.08549},
  
  author = {Haghtalab, Nika and Han, Yanjun and Shetty, Abhishek and Yang, Kunhe},
  
  keywords = {Machine Learning (cs.LG), Data Structures and Algorithms (cs.DS), Machine Learning (stat.ML), FOS: Computer and information sciences, FOS: Computer and information sciences},
  
  title = {Oracle-Efficient Online Learning for Beyond Worst-Case Adversaries},
  
  publisher = {arXiv},
  
  year = {2022},
  
  copyright = {Creative Commons Attribution 4.0 International}
}
@misc{block,
  doi = {10.48550/ARXIV.2202.04690},
  
  url = {https://arxiv.org/abs/2202.04690},
  
  author = {Block, Adam and Dagan, Yuval and Golowich, Noah and Rakhlin, Alexander},
  
  keywords = {Machine Learning (stat.ML), Machine Learning (cs.LG), FOS: Computer and information sciences, FOS: Computer and information sciences},
  
  title = {Smoothed Online Learning is as Easy as Statistical Learning},
  
  publisher = {arXiv},
  
  year = {2022},
  
  copyright = {Creative Commons Attribution 4.0 International}
}
@misc{dudik,
  doi = {10.48550/ARXIV.1611.01688},
  
  url = {https://arxiv.org/abs/1611.01688},
  
  author = {Dudík, Miroslav and Haghtalab, Nika and Luo, Haipeng and Schapire, Robert E. and Syrgkanis, Vasilis and Vaughan, Jennifer Wortman},
  
  keywords = {Machine Learning (cs.LG), Data Structures and Algorithms (cs.DS), Computer Science and Game Theory (cs.GT), FOS: Computer and information sciences, FOS: Computer and information sciences},
  
  title = {Oracle-Efficient Online Learning and Auction Design},
  
  publisher = {arXiv},
  
  year = {2016},
  
  copyright = {arXiv.org perpetual, non-exclusive license}
}
@article{complexity_allender,
  author       = {Martin Mundhenk and
                  Judy Goldsmith and
                  Christopher Lusena and
                  Eric Allender},
  title        = {Complexity of finite-horizon Markov decision process problems},
  journal      = {J. {ACM}},
  volume       = {47},
  number       = {4},
  pages        = {681--720},
  year         = {2000},
  url          = {https://doi.org/10.1145/347476.347480},
  doi          = {10.1145/347476.347480},
  timestamp    = {Sun, 02 Jun 2019 20:59:13 +0200},
  biburl       = {https://dblp.org/rec/journals/jacm/MundhenkGLA00.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}