    @article{einstein,
	author =       "Albert Einstein",
	title =        "{Zur Elektrodynamik bewegter Körper}",
	journal =      "Annalen der Physik",
	volume =       "322",
	number =       "10",
	pages =        "891--921",
	year =         "1905"}
@book{durrett2019probability,
  title={Probability: theory and examples},
  author={Durrett, Rick},
  volume={49},
  year={2019},
  publisher={Cambridge university press}
}
@article{tsitsiklis1994asynchronous,
  title={{Asynchronous stochastic approximation and $Q$-learning}},
  author={Tsitsiklis, John N},
  journal={Machine learning},
  volume={16},
  pages={185--202},
  year={1994},
  publisher={Springer}
}
@book{latexcompanion,
	author    = "Michel Goossens and Frank Mittelbach and Alexander Samarin",
	title     = "The \LaTeX\ Companion",
	year      = "1993",
	publisher = "Addison-Wesley"}
@article{gao2017properties,
  title={On the properties of the softmax function with application in game theory and reinforcement learning},
  author={Gao, Bolin and Pavel, Lacra},
  journal={Preprint arXiv:1704.00805},
  year={2017}
}
@article{mehta_convex_2020,
	title = {Convex {Q}-{Learning}, {Part} 1: {Deterministic} {Optimal} {Control}},
	shorttitle = {Convex {Q}-{Learning}, {Part} 1},
	url = {http://arxiv.org/abs/2008.03559},
	abstract = {It is well known that the extension of Watkins' algorithm to general function approximation settings is challenging: does the projected Bellman equation have a solution? If so, is the solution useful in the sense of generating a good policy? And, if the preceding questions are answered in the affirmative, is the algorithm consistent? These questions are unanswered even in the special case of Q-function approximations that are linear in the parameter. The challenge seems paradoxical, given the long history of convex analytic approaches to dynamic programming. The paper begins with a brief survey of linear programming approaches to optimal control, leading to a particular over parameterization that lends itself to applications in reinforcement learning. The main conclusions are summarized as follows: (i) The new class of convex Q-learning algorithms is introduced based on the convex relaxation of the Bellman equation. Convergence is established under general conditions, including a linear function approximation for the Q-function. (ii) A batch implementation appears similar to the famed DQN algorithm (one engine behind AlphaZero). It is shown that in fact the algorithms are very different: while convex Q-learning solves a convex program that approximates the Bellman equation, theory for DQN is no stronger than for Watkins' algorithm with function approximation: (a) it is shown that both seek solutions to the same fixed point equation, and (b) the ODE approximations for the two algorithms coincide, and little is known about the stability of this ODE. These results are obtained for deterministic nonlinear systems with total cost criterion. Many extensions are proposed, including kernel implementation, and extension to MDP models.},
	urldate = {2020-08-11},
	journal = {arXiv:2008.03559 [cs, math]},
	author = {Mehta, Prashant G. and Meyn, Sean P.},
	month = aug,
	year = {2020},
	note = {arXiv: 2008.03559},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, 68T05 (Primary) 93E35, 49L20 (Secondary)},
	annote = {Comment: This pre-print is written in a tutorial style so it is accessible to new-comers. It will be a part of a handout for upcoming short courses on RL. A more compact version suitable for journal submission is in preparation},
	file = {arXiv Fulltext PDF:/Users/guannanqu/Zotero/storage/BSDJWLBA/Mehta and Meyn - 2020 - Convex Q-Learning, Part 1 Deterministic Optimal C.pdf:application/pdf;arXiv.org Snapshot:/Users/guannanqu/Zotero/storage/S6XV8VAU/2008.html:text/html}
}
@article{chen2021lyapunov,
	title={{A Lyapunov theory for finite-sample guarantees of asynchronous $Q$-learning and TD-learning variants}},
	author={Chen, Zaiwei and Maguluri, Siva Theja and Shakkottai, Sanjay and Shanmugam, Karthikeyan},
	journal={Preprint arXiv:2102.01567},
	year={2021}
}
@article{wang_reward-free_2020,
	title = {On {Reward}-{Free} {Reinforcement} {Learning} with {Linear} {Function} {Approximation}},
	url = {http://arxiv.org/abs/2006.11274},
	abstract = {Reward-free reinforcement learning (RL) is a framework which is suitable for both the batch RL setting and the setting where there are many reward functions of interest. During the exploration phase, an agent collects samples without using a pre-specified reward function. After the exploration phase, a reward function is given, and the agent uses samples collected during the exploration phase to compute a near-optimal policy. Jin et al. [2020] showed that in the tabular setting, the agent only needs to collect polynomial number of samples (in terms of the number states, the number of actions, and the planning horizon) for reward-free RL. However, in practice, the number of states and actions can be large, and thus function approximation schemes are required for generalization. In this work, we give both positive and negative results for reward-free RL with linear function approximation. We give an algorithm for reward-free RL in the linear Markov decision process setting where both the transition and the reward admit linear representations. The sample complexity of our algorithm is polynomial in the feature dimension and the planning horizon, and is completely independent of the number of states and actions. We further give an exponential lower bound for reward-free RL in the setting where only the optimal \$Q\$-function admits a linear representation. Our results imply several interesting exponential separations on the sample complexity of reward-free RL.},
	urldate = {2020-07-04},
	journal = {arXiv:2006.11274 [cs, math, stat]},
	author = {Wang, Ruosong and Du, Simon S. and Yang, Lin F. and Salakhutdinov, Ruslan},
	month = jun,
	year = {2020},
	note = {arXiv: 2006.11274},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:files/52/Wang et al. - 2020 - On Reward-Free Reinforcement Learning with Linear .pdf:application/pdf;arXiv.org Snapshot:files/53/2006.html:text/html}
}

@article{wang_reinforcement_2020,
	title = {Reinforcement {Learning} with {General} {Value} {Function} {Approximation}: {Provably} {Efficient} {Approach} via {Bounded} {Eluder} {Dimension}},
	shorttitle = {Reinforcement {Learning} with {General} {Value} {Function} {Approximation}},
	url = {http://arxiv.org/abs/2005.10804},
	abstract = {Value function approximation has demonstrated phenomenal empirical success in reinforcement learning (RL). Nevertheless, despite a handful of recent progress on developing theory for RL with linear function approximation, the understanding of general function approximation schemes largely remains missing. In this paper, we establish a provably efficient RL algorithm with general value function approximation. We show that if the value functions admit an approximation with a function class \${\textbackslash}mathcal\{F\}\$, our algorithm achieves a regret bound of \${\textbackslash}widetilde\{O\}({\textbackslash}mathrm\{poly\}(dH){\textbackslash}sqrt\{T\})\$ where \$d\$ is a complexity measure of \${\textbackslash}mathcal\{F\}\$ that depends on the eluder dimension [Russo and Van Roy, 2013] and log-covering numbers, \$H\$ is the planning horizon, and \$T\$ is the number interactions with the environment. Our theory generalizes recent progress on RL with linear value function approximation and does not make explicit assumptions on the model of the environment. Moreover, our algorithm is model-free and provides a framework to justify the effectiveness of algorithms used in practice.},
	urldate = {2020-07-04},
	journal = {arXiv:2005.10804 [cs, math, stat]},
	author = {Wang, Ruosong and Salakhutdinov, Ruslan and Yang, Lin F.},
	month = jun,
	year = {2020},
	note = {arXiv: 2005.10804},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/62/2MW3BMYJ.pdf:application/pdf;arXiv.org Snapshot:files/63/2005.html:text/html}
}

@article{jin_sample-efficient_2020,
	title = {Sample-{Efficient} {Reinforcement} {Learning} of {Undercomplete} {POMDPs}},
	url = {http://arxiv.org/abs/2006.12484},
	abstract = {Partial observability is a common challenge in many reinforcement learning applications, which requires an agent to maintain memory, infer latent states, and integrate this past information into exploration. This challenge leads to a number of computational and statistical hardness results for learning general Partially Observable Markov Decision Processes (POMDPs). This work shows that these hardness barriers do not preclude efficient reinforcement learning for rich and interesting subclasses of POMDPs. In particular, we present a sample-efficient algorithm, OOM-UCB, for episodic finite undercomplete POMDPs, where the number of observations is larger than the number of latent states and where exploration is essential for learning, thus distinguishing our results from prior works. OOM-UCB achieves an optimal sample complexity of \$O(1/{\textbackslash}epsilon{\textasciicircum}2)\$ for finding an \${\textbackslash}epsilon\$-optimal policy, along with being polynomial in all other relevant quantities. As an interesting special case, we also provide a computationally and statistically efficient algorithm for POMDPs with deterministic state transitions.},
	urldate = {2020-07-04},
	journal = {arXiv:2006.12484 [cs, math, stat]},
	author = {Jin, Chi and Kakade, Sham M. and Krishnamurthy, Akshay and Liu, Qinghua},
	month = jun,
	year = {2020},
	note = {arXiv: 2006.12484},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:files/108/AKH7X9SH.pdf:application/pdf;arXiv.org Snapshot:files/109/2006.html:text/html}
}

@article{zhou_provably_2020,
	title = {Provably {Efficient} {Reinforcement} {Learning} for {Discounted} {MDPs} with {Feature} {Mapping}},
	url = {http://arxiv.org/abs/2006.13165},
	abstract = {Modern tasks in reinforcement learning are always with large state and action spaces. To deal with them efficiently, one often uses predefined feature mapping to represents states and actions in a low dimensional space. In this paper, we study reinforcement learning with feature mapping for discounted Markov Decision Processes (MDPs). We propose a novel algorithm which makes use of the feature mapping and obtains a \${\textbackslash}tilde O(d{\textbackslash}sqrt\{T\}/(1-{\textbackslash}gamma){\textasciicircum}2)\$ regret, where \$d\$ is the dimension of the feature space, \$T\$ is the time horizon and \${\textbackslash}gamma\$ is the discount factor of the MDP. To the best of our knowledge, this is the first polynomial regret bound without accessing to a generative model or making strong assumptions such as ergodicity of the MDP. By constructing a special class of MDPs, we also show that for any algorithms, the regret is lower bounded by \${\textbackslash}Omega(d{\textbackslash}sqrt\{T\}/(1-{\textbackslash}gamma){\textasciicircum}\{1.5\})\$. Our upper and lower bound results together suggest that the proposed reinforcement learning algorithm is near-optimal up to a \$(1-{\textbackslash}gamma){\textasciicircum}\{-0.5\}\$ factor.},
	urldate = {2020-07-04},
	journal = {arXiv:2006.13165 [cs, math, stat]},
	author = {Zhou, Dongruo and He, Jiafan and Gu, Quanquan},
	month = jun,
	year = {2020},
	note = {arXiv: 2006.13165},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:files/120/Zhou et al. - 2020 - Provably Efficient Reinforcement Learning for Disc.pdf:application/pdf;arXiv.org Snapshot:files/121/2006.html:text/html}
}

@article{fei_risk-sensitive_2020,
	title = {Risk-{Sensitive} {Reinforcement} {Learning}: {Near}-{Optimal} {Risk}-{Sample} {Tradeoff} in {Regret}},
	shorttitle = {Risk-{Sensitive} {Reinforcement} {Learning}},
	url = {http://arxiv.org/abs/2006.13827},
	abstract = {We study risk-sensitive reinforcement learning in episodic Markov decision processes with unknown transition kernels, where the goal is to optimize the total reward under the risk measure of exponential utility. We propose two provably efficient model-free algorithms, Risk-Sensitive Value Iteration (RSVI) and Risk-Sensitive Q-learning (RSQ). These algorithms implement a form of risk-sensitive optimism in the face of uncertainty, which adapts to both risk-seeking and risk-averse modes of exploration. We prove that RSVI attains an \${\textbackslash}tilde\{O\}{\textbackslash}big({\textbackslash}lambda({\textbar}{\textbackslash}beta{\textbar} H{\textasciicircum}2) {\textbackslash}cdot {\textbackslash}sqrt\{H{\textasciicircum}\{3\} S{\textasciicircum}\{2\}AT\} {\textbackslash}big)\$ regret, while RSQ attains an \${\textbackslash}tilde\{O\}{\textbackslash}big({\textbackslash}lambda({\textbar}{\textbackslash}beta{\textbar} H{\textasciicircum}2) {\textbackslash}cdot {\textbackslash}sqrt\{H{\textasciicircum}\{4\} SAT\} {\textbackslash}big)\$ regret, where \${\textbackslash}lambda(u) = (e{\textasciicircum}\{3u\}-1)/u\$ for \$u{\textgreater}0\$. In the above, \${\textbackslash}beta\$ is the risk parameter of the exponential utility function, \$S\$ the number of states, \$A\$ the number of actions, \$T\$ the total number of timesteps, and \$H\$ the episode length. On the flip side, we establish a regret lower bound showing that the exponential dependence on \${\textbar}{\textbackslash}beta{\textbar}\$ and \$H\$ is unavoidable for any algorithm with an \${\textbackslash}tilde\{O\}({\textbackslash}sqrt\{T\})\$ regret (even when the risk objective is on the same scale as the original reward), thus certifying the near-optimality of the proposed algorithms. Our results demonstrate that incorporating risk awareness into reinforcement learning necessitates an exponential cost in \${\textbar}{\textbackslash}beta{\textbar}\$ and \$H\$, which quantifies the fundamental tradeoff between risk sensitivity (related to aleatoric uncertainty) and sample efficiency (related to epistemic uncertainty). To the best of our knowledge, this is the first regret analysis of risk-sensitive reinforcement learning with the exponential utility.},
	urldate = {2020-07-04},
	journal = {arXiv:2006.13827 [cs, math, stat]},
	author = {Fei, Yingjie and Yang, Zhuoran and Chen, Yudong and Wang, Zhaoran and Xie, Qiaomin},
	month = jun,
	year = {2020},
	note = {arXiv: 2006.13827},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/159/Fei et al. - 2020 - Risk-Sensitive Reinforcement Learning Near-Optima.pdf:application/pdf;arXiv.org Snapshot:files/160/2006.html:text/html}
}

@article{xie_learning_2020,
	title = {Learning {Zero}-{Sum} {Simultaneous}-{Move} {Markov} {Games} {Using} {Function} {Approximation} and {Correlated} {Equilibrium}},
	url = {http://arxiv.org/abs/2002.07066},
	abstract = {We develop provably efficient reinforcement learning algorithms for two-player zero-sum finite-horizon Markov games with simultaneous moves. To incorporate function approximation, we consider a family of Markov games where the reward function and transition kernel possess a linear structure. Both the offline and online settings of the problems are considered. In the offline setting, we control both players and aim to find the Nash Equilibrium by minimizing the duality gap. In the online setting, we control a single player playing against an arbitrary opponent and aim to minimize the regret. For both settings, we propose an optimistic variant of the least-squares minimax value iteration algorithm. We show that our algorithm is computationally efficient and provably achieves an \${\textbackslash}tilde O({\textbackslash}sqrt\{d{\textasciicircum}3 H{\textasciicircum}3 T\} )\$ upper bound on the duality gap and regret, where \$d\$ is the linear dimension, \$H\$ the horizon and \$T\$ the total number of timesteps. Our results do not require additional assumptions on the sampling model. Our setting requires overcoming several new challenges that are absent in Markov decision processes or turn-based Markov games. In particular, to achieve optimism with simultaneous moves, we construct both upper and lower confidence bounds of the value function, and then compute the optimistic policy by solving a general-sum matrix game with these bounds as the payoff matrices. As finding the Nash Equilibrium of a general-sum game is computationally hard, our algorithm instead solves for a Coarse Correlated Equilibrium (CCE), which can be obtained efficiently. To our best knowledge, such a CCE-based scheme for optimism has not appeared in the literature and might be of interest in its own right.},
	urldate = {2020-07-04},
	journal = {arXiv:2002.07066 [cs, math, stat]},
	author = {Xie, Qiaomin and Chen, Yudong and Wang, Zhaoran and Yang, Zhuoran},
	month = jun,
	year = {2020},
	note = {arXiv: 2002.07066},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Computer Science - Computer Science and Game Theory, Computer Science - Multiagent Systems},
	file = {arXiv Fulltext PDF:files/182/Xie et al. - 2020 - Learning Zero-Sum Simultaneous-Move Markov Games U.pdf:application/pdf;arXiv.org Snapshot:files/183/2002.html:text/html}
}

@article{shar_lookahead-bounded_2020,
	title = {Lookahead-{Bounded} {Q}-{Learning}},
	url = {http://arxiv.org/abs/2006.15690},
	abstract = {We introduce the lookahead-bounded Q-learning (LBQL) algorithm, a new, provably convergent variant of Q-learning that seeks to improve the performance of standard Q-learning in stochastic environments through the use of ``lookahead'' upper and lower bounds. To do this, LBQL employs previously collected experience and each iteration's state-action values as dual feasible penalties to construct a sequence of sampled information relaxation problems. The solutions to these problems provide estimated upper and lower bounds on the optimal value, which we track via stochastic approximation. These quantities are then used to constrain the iterates to stay within the bounds at every iteration. Numerical experiments on benchmark problems show that LBQL exhibits faster convergence and more robustness to hyperparameters when compared to standard Q-learning and several related techniques. Our approach is particularly appealing in problems that require expensive simulations or real-world interactions.},
	urldate = {2020-07-05},
	journal = {arXiv:2006.15690 [cs, math, stat]},
	author = {Shar, Ibrahim El and Jiang, Daniel R.},
	month = jun,
	year = {2020},
	note = {arXiv: 2006.15690},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/258/Shar and Jiang - 2020 - Lookahead-Bounded Q-Learning.pdf:application/pdf;arXiv.org Snapshot:files/259/2006.html:text/html}
}

@article{fei_dynamic_2020,
	title = {Dynamic {Regret} of {Policy} {Optimization} in {Non}-stationary {Environments}},
	url = {http://arxiv.org/abs/2007.00148},
	abstract = {We consider reinforcement learning (RL) in episodic MDPs with adversarial full-information reward feedback and unknown fixed transition kernels. We propose two model-free policy optimization algorithms, POWER and POWER++, and establish guarantees for their dynamic regret. Compared with the classical notion of static regret, dynamic regret is a stronger notion as it explicitly accounts for the non-stationarity of environments. The dynamic regret attained by the proposed algorithms interpolates between different regimes of non-stationarity, and moreover satisfies a notion of adaptive (near-)optimality, in the sense that it matches the (near-)optimal static regret under slow-changing environments. The dynamic regret bound features two components, one arising from exploration, which deals with the uncertainty of transition kernels, and the other arising from adaptation, which deals with non-stationary environments. Specifically, we show that POWER++ improves over POWER on the second component of the dynamic regret by actively adapting to non-stationarity through prediction. To the best of our knowledge, our work is the first dynamic regret analysis of model-free RL algorithms in non-stationary environments.},
	urldate = {2020-07-05},
	journal = {arXiv:2007.00148 [cs, math, stat]},
	author = {Fei, Yingjie and Yang, Zhuoran and Wang, Zhaoran and Xie, Qiaomin},
	month = jun,
	year = {2020},
	note = {arXiv: 2007.00148},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/268/Fei et al. - 2020 - Dynamic Regret of Policy Optimization in Non-stati.pdf:application/pdf;arXiv.org Snapshot:files/269/2007.html:text/html}
}

@article{bai_model-free_2020,
	title = {Model-{Free} {Algorithm} and {Regret} {Analysis} for {MDPs} with {Peak} {Constraints}},
	url = {http://arxiv.org/abs/2003.05555},
	abstract = {In the optimization of dynamic systems, the variables typically have constraints. Such problems can be modeled as a constrained Markov Decision Process (MDP). This paper considers a model-free approach to the problem, where the transition probabilities are not known. In the presence of peak constraints, the agent has to choose the policy to maximize the long-term average reward as well as satisfy the constraints at each time. We propose a novel algorithm that coverts the constrained problem to an unconstrained problem using a modification of the reward function, and a Q-learming based approach is used on the unconstrained problem. The proposed algorithm is shown to achieve \$O({\textbackslash}sqrt\{H{\textasciicircum}6SAT{\textbackslash}ell\})\$ bound for both the obtained reward and constraint violations with probability at-least \$1-2p\$, where \$T\$ is the time-horizon, \$A\$ is the number of actions, \$S\$ is the number of states, \$H\$ is the number of steps in each episode, and \${\textbackslash}ell={\textbackslash}log({\textbackslash}frac\{2SAT\}\{p\})\$. We note that these are the first results on regret analysis for constrained MDP, where the transition problems are not known apriori. We demonstrate the proposed algorithm on an energy harvesting problem where it outperforms state-of-the-art and performs close to the theoretical upper bound of the studied optimization problem.},
	urldate = {2020-07-05},
	journal = {arXiv:2003.05555 [cs, eess, math, stat]},
	author = {Bai, Qinbo and Aggarwal, Vaneet and Gattami, Ather},
	month = jul,
	year = {2020},
	note = {arXiv: 2003.05555},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Electrical Engineering and Systems Science - Systems and Control},
	file = {arXiv Fulltext PDF:files/278/Bai et al. - 2020 - Model-Free Algorithm and Regret Analysis for MDPs .pdf:application/pdf;arXiv.org Snapshot:files/279/2003.html:text/html}
}

@article{cai_provably_2020,
	title = {Provably {Efficient} {Exploration} in {Policy} {Optimization}},
	url = {http://arxiv.org/abs/1912.05830},
	abstract = {While policy-based reinforcement learning (RL) achieves tremendous successes in practice, it is significantly less understood in theory, especially compared with value-based RL. In particular, it remains elusive how to design a provably efficient policy optimization algorithm that incorporates exploration. To bridge such a gap, this paper proposes an {\textbackslash}underline\{O\}ptimistic variant of the {\textbackslash}underline\{P\}roximal {\textbackslash}underline\{P\}olicy {\textbackslash}underline\{O\}ptimization algorithm (OPPO), which follows an "optimistic version" of the policy gradient direction. This paper proves that, in the problem of episodic Markov decision process with unknown transition and full-information feedback of adversarial reward, OPPO achieves an \${\textbackslash}tilde\{O\}({\textbackslash}sqrt\{{\textbar}{\textbackslash}mathcal\{S\}{\textbar}{\textasciicircum}2{\textbar}{\textbackslash}mathcal\{A\}{\textbar}H{\textasciicircum}3 T\})\$ regret. Here, \${\textbar}{\textbackslash}mathcal\{S\}{\textbar}\$ is the size of the state space, \${\textbar}{\textbackslash}mathcal\{A\}{\textbar}\$ is the size of the action space, \$H\$ is the episode horizon, and \$T\$ is the total number of steps. To the best of our knowledge, OPPO is the first provably efficient{\textasciitilde}policy optimization algorithm that explores.},
	urldate = {2020-07-05},
	journal = {arXiv:1912.05830 [cs, math, stat]},
	author = {Cai, Qi and Yang, Zhuoran and Jin, Chi and Wang, Zhaoran},
	month = jul,
	year = {2020},
	note = {arXiv: 1912.05830},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/282/ZB64RQ4A.pdf:application/pdf;arXiv.org Snapshot:files/283/1912.html:text/html}
}

@article{jin_provably_2019,
	title = {Provably {Efficient} {Reinforcement} {Learning} with {Linear} {Function} {Approximation}},
	url = {http://arxiv.org/abs/1907.05388},
	abstract = {Modern Reinforcement Learning (RL) is commonly applied to practical problems with an enormous number of states, where function approximation must be deployed to approximate either the value function or the policy. The introduction of function approximation raises a fundamental set of challenges involving computational and statistical efficiency, especially given the need to manage the exploration/exploitation tradeoff. As a result, a core RL question remains open: how can we design provably efficient RL algorithms that incorporate function approximation? This question persists even in a basic setting with linear dynamics and linear rewards, for which only linear function approximation is needed. This paper presents the first provable RL algorithm with both polynomial runtime and polynomial sample complexity in this linear setting, without requiring a "simulator" or additional assumptions. Concretely, we prove that an optimistic modification of Least-Squares Value Iteration (LSVI)---a classical algorithm frequently studied in the linear setting---achieves \${\textbackslash}tilde\{{\textbackslash}mathcal\{O\}\}({\textbackslash}sqrt\{d{\textasciicircum}3H{\textasciicircum}3T\})\$ regret, where \$d\$ is the ambient dimension of feature space, \$H\$ is the length of each episode, and \$T\$ is the total number of steps. Importantly, such regret is independent of the number of states and actions.},
	urldate = {2020-07-07},
	journal = {arXiv:1907.05388 [cs, math, stat]},
	author = {Jin, Chi and Yang, Zhuoran and Wang, Zhaoran and Jordan, Michael I.},
	month = aug,
	year = {2019},
	note = {arXiv: 1907.05388},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv.org Snapshot:files/375/1907.html:text/html;arXiv Fulltext PDF:files/376/98LDRNGG.pdf:application/pdf}
}

@article{agarwal2021theory,
	title={{On the theory of policy gradient methods: Optimality, approximation, and distribution shift}},
	author={Agarwal, Alekh and Kakade, Sham M and Lee, Jason D and Mahajan, Gaurav},
	journal={The Journal of Machine Learning Research},
	volume={22},
	number={1},
	pages={4431--4506},
	year={2021},
	publisher={JMLRORG}
}

@article{wang_is_2020,
	title = {Is {Long} {Horizon} {Reinforcement} {Learning} {More} {Difficult} {Than} {Short} {Horizon} {Reinforcement} {Learning}?},
	url = {http://arxiv.org/abs/2005.00527},
	abstract = {Learning to plan for long horizons is a central challenge in episodic reinforcement learning problems. A fundamental question is to understand how the difficulty of the problem scales as the horizon increases. Here the natural measure of sample complexity is a normalized one: we are interested in the number of episodes it takes to provably discover a policy whose value is \${\textbackslash}varepsilon\$ near to that of the optimal value, where the value is measured by the normalized cumulative reward in each episode. In a COLT 2018 open problem, Jiang and Agarwal conjectured that, for tabular, episodic reinforcement learning problems, there exists a sample complexity lower bound which exhibits a polynomial dependence on the horizon -- a conjecture which is consistent with all known sample complexity upper bounds. This work refutes this conjecture, proving that tabular, episodic reinforcement learning is possible with a sample complexity that scales only logarithmically with the planning horizon. In other words, when the values are appropriately normalized (to lie in the unit interval), this results shows that long horizon RL is no more difficult than short horizon RL, at least in a minimax sense. Our analysis introduces two ideas: (i) the construction of an \${\textbackslash}varepsilon\$-net for optimal policies whose log-covering number scales only logarithmically with the planning horizon, and (ii) the Online Trajectory Synthesis algorithm, which adaptively evaluates all policies in a given policy class using sample complexity that scales with the log-covering number of the given policy class. Both may be of independent interest.},
	urldate = {2020-07-10},
	journal = {arXiv:2005.00527 [cs, math, stat]},
	author = {Wang, Ruosong and Du, Simon S. and Yang, Lin F. and Kakade, Sham M.},
	month = jul,
	year = {2020},
	note = {arXiv: 2005.00527},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:files/552/Wang et al. - 2020 - Is Long Horizon Reinforcement Learning More Diffic.pdf:application/pdf;arXiv.org Snapshot:files/553/2005.html:text/html}
}

@article{cen_fast_2020,
	title = {Fast {Global} {Convergence} of {Natural} {Policy} {Gradient} {Methods} with {Entropy} {Regularization}},
	url = {http://arxiv.org/abs/2007.06558},
	abstract = {Natural policy gradient (NPG) methods are among the most widely used policy optimization algorithms in contemporary reinforcement learning. This class of methods is often applied in conjunction with entropy regularization -- an algorithmic scheme that helps encourage exploration -- and is closely related to soft policy iteration and trust region policy optimization. Despite the empirical success, the theoretical underpinnings for NPG methods remain severely limited even for the tabular setting. This paper develops \${\textbackslash}textit\{non-asymptotic\}\$ convergence guarantees for entropy-regularized NPG methods under softmax parameterization, focusing on discounted Markov decision processes (MDPs). Assuming access to exact policy evaluation, we demonstrate that the algorithm converges linearly -- or even quadratically once it enters a local region around the optimal policy -- when computing optimal value functions of the regularized MDP. Moreover, the algorithm is provably stable vis-{\textbackslash}`a-vis inexactness of policy evaluation, and is able to find an \${\textbackslash}epsilon\$-optimal policy for the original MDP when applied to a slightly perturbed MDP. Our convergence results outperform the ones established for unregularized NPG methods (arXiv:1908.00261), and shed light upon the role of entropy regularization in accelerating convergence .},
	urldate = {2020-07-14},
	journal = {arXiv:2007.06558 [cs, math, stat]},
	author = {Cen, Shicong and Cheng, Chen and Chen, Yuxin and Wei, Yuting and Chi, Yuejie},
	month = jul,
	year = {2020},
	note = {arXiv: 2007.06558},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Computer Science - Information Theory},
	file = {arXiv Fulltext PDF:files/605/Cen et al. - 2020 - Fast Global Convergence of Natural Policy Gradient.pdf:application/pdf;arXiv.org Snapshot:files/606/2007.html:text/html}
}

@article{park_structured_2020,
	title = {Structured {Policy} {Iteration} for {Linear} {Quadratic} {Regulator}},
	url = {http://arxiv.org/abs/2007.06202},
	abstract = {Linear quadratic regulator (LQR) is one of the most popular frameworks to tackle continuous Markov decision process tasks. With its fundamental theory and tractable optimal policy, LQR has been revisited and analyzed in recent years, in terms of reinforcement learning scenarios such as the model-free or model-based setting. In this paper, we introduce the {\textbackslash}textit\{Structured Policy Iteration\} (S-PI) for LQR, a method capable of deriving a structured linear policy. Such a structured policy with (block) sparsity or low-rank can have significant advantages over the standard LQR policy: more interpretable, memory-efficient, and well-suited for the distributed setting. In order to derive such a policy, we first cast a regularized LQR problem when the model is known. Then, our Structured Policy Iteration (S-PI) algorithm, which takes a policy evaluation step and a policy improvement step in an iterative manner, can solve this regularized LQR efficiently. We further extend the S-PI algorithm to the model-free setting where a smoothing procedure is adopted to estimate the gradient. In both the known-model and model-free setting, we prove convergence analysis under the proper choice of parameters. Finally, the experiments demonstrate the advantages of S-PI in terms of balancing the LQR performance and level of structure by varying the weight parameter.},
	urldate = {2020-07-14},
	journal = {arXiv:2007.06202 [cs, math]},
	author = {Park, Youngsuk and Rossi, Ryan A. and Wen, Zheng and Wu, Gang and Zhao, Handong},
	month = jul,
	year = {2020},
	note = {arXiv: 2007.06202},
	keywords = {Mathematics - Optimization and Control, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:files/608/Park et al. - 2020 - Structured Policy Iteration for Linear Quadratic R.pdf:application/pdf;arXiv.org Snapshot:files/609/2007.html:text/html}
}

@article{kumar_zeroth-order_2020,
	title = {Zeroth-order {Deterministic} {Policy} {Gradient}},
	url = {http://arxiv.org/abs/2006.07314},
	abstract = {Deterministic Policy Gradient (DPG) removes a level of randomness from standard randomized-action Policy Gradient (PG), and demonstrates substantial empirical success for tackling complex dynamic problems involving Markov decision processes. At the same time, though, DPG loses its ability to learn in a model-free (i.e., actor-only) fashion, frequently necessitating the use of critics in order to obtain consistent estimates of the associated policy-reward gradient. In this work, we introduce Zeroth-order Deterministic Policy Gradient (ZDPG), which approximates policy-reward gradients via two-point stochastic evaluations of the \$Q\$-function, constructed by properly designed low-dimensional action-space perturbations. Exploiting the idea of random horizon rollouts for obtaining unbiased estimates of the \$Q\$-function, ZDPG lifts the dependence on critics and restores true model-free policy learning, while enjoying built-in and provable algorithmic stability. Additionally, we present new finite sample complexity bounds for ZDPG, which improve upon existing results by up to two orders of magnitude. Our findings are supported by several numerical experiments, which showcase the effectiveness of ZDPG in a practical setting, and its advantages over both PG and Baseline PG.},
	urldate = {2020-07-14},
	journal = {arXiv:2006.07314 [cs, math, stat]},
	author = {Kumar, Harshat and Kalogerias, Dionysios S. and Pappas, George J. and Ribeiro, Alejandro},
	month = jul,
	year = {2020},
	note = {arXiv: 2006.07314},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/615/Kumar et al. - 2020 - Zeroth-order Deterministic Policy Gradient.pdf:application/pdf;arXiv.org Snapshot:files/616/2006.html:text/html}
}

@article{huang_momentum-based_2020,
	title = {Momentum-{Based} {Policy} {Gradient} {Methods}},
	url = {http://arxiv.org/abs/2007.06680},
	abstract = {In the paper, we propose a class of efficient momentum-based policy gradient methods for the model-free reinforcement learning, which use adaptive learning rates and do not require any large batches. Specifically, we propose a fast important-sampling momentum-based policy gradient (IS-MBPG) method based on a new momentum-based variance reduced technique and the importance sampling technique. We also propose a fast Hessian-aided momentum-based policy gradient (HA-MBPG) method based on the momentum-based variance reduced technique and the Hessian-aided technique. Moreover, we prove that both the IS-MBPG and HA-MBPG methods reach the best known sample complexity of \$O({\textbackslash}epsilon{\textasciicircum}\{-3\})\$ for finding an \${\textbackslash}epsilon\$-stationary point of the nonconcave performance function, which only require one trajectory at each iteration. In particular, we present a non-adaptive version of IS-MBPG method, i.e., IS-MBPG*, which also reaches the best known sample complexity of \$O({\textbackslash}epsilon{\textasciicircum}\{-3\})\$ without any large batches. In the experiments, we apply four benchmark tasks to demonstrate the effectiveness of our algorithms.},
	urldate = {2020-07-15},
	journal = {arXiv:2007.06680 [cs, eess, math, stat]},
	author = {Huang, Feihu and Gao, Shangqian and Pei, Jian and Huang, Heng},
	month = jul,
	year = {2020},
	note = {arXiv: 2007.06680},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Computer Science - Robotics, Electrical Engineering and Systems Science - Systems and Control},
	file = {arXiv Fulltext PDF:files/623/Huang et al. - 2020 - Momentum-Based Policy Gradient Methods.pdf:application/pdf;arXiv.org Snapshot:files/624/2007.html:text/html}
}

@article{derman_distributional_2020,
	title = {Distributional {Robustness} and {Regularization} in {Reinforcement} {Learning}},
	url = {http://arxiv.org/abs/2003.02894},
	abstract = {Distributionally Robust Optimization (DRO) has enabled to prove the equivalence between robustness and regularization in classification and regression, thus providing an analytical reason why regularization generalizes well in statistical learning. Although DRO's extension to sequential decision-making overcomes \${\textbackslash}textit\{external uncertainty\}\$ through the robust Markov Decision Process (MDP) setting, the resulting formulation is hard to solve, especially on large domains. On the other hand, existing regularization methods in reinforcement learning only address \${\textbackslash}textit\{internal uncertainty\}\$ due to stochasticity. Our study aims to facilitate robust reinforcement learning by establishing a dual relation between robust MDPs and regularization. We introduce Wasserstein distributionally robust MDPs and prove that they hold out-of-sample performance guarantees. Then, we introduce a new regularizer for empirical value functions and show that it lower bounds the Wasserstein distributionally robust value function. We extend the result to linear value function approximation for large state spaces. Our approach provides an alternative formulation of robustness with guaranteed finite-sample performance. Moreover, it suggests using regularization as a practical tool for dealing with \${\textbackslash}textit\{external uncertainty\}\$ in reinforcement learning methods.},
	urldate = {2020-07-15},
	journal = {arXiv:2003.02894 [cs, math, stat]},
	author = {Derman, Esther and Mannor, Shie},
	month = jul,
	year = {2020},
	note = {arXiv: 2003.02894},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/627/Derman and Mannor - 2020 - Distributional Robustness and Regularization in Re.pdf:application/pdf;arXiv.org Snapshot:files/628/2003.html:text/html}
}

@article{feng_provably_2020,
	title = {Provably {Efficient} {Exploration} for {RL} with {Unsupervised} {Learning}},
	url = {http://arxiv.org/abs/2003.06898},
	abstract = {Motivated by the prevailing paradigm of using unsupervised learning for efficient exploration in reinforcement learning (RL) problems [tang2017exploration,bellemare2016unifying], we investigate when this paradigm is provably efficient. We study episodic Markov decision processes with rich observations generated from a small number of latent states. We present a general algorithmic framework that is built upon two components: an unsupervised learning algorithm and a no-regret tabular RL algorithm. Theoretically, we prove that as long as the unsupervised learning algorithm enjoys a polynomial sample complexity guarantee, we can find a near-optimal policy with sample complexity polynomial in the number of latent states, which is significantly smaller than the number of observations. Empirically, we instantiate our framework on a class of hard exploration problems to demonstrate the practicality of our theory.},
	urldate = {2020-07-15},
	journal = {arXiv:2003.06898 [cs, math, stat]},
	author = {Feng, Fei and Wang, Ruosong and Yin, Wotao and Du, Simon S. and Yang, Lin F.},
	month = jul,
	year = {2020},
	note = {arXiv: 2003.06898},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:files/630/Feng et al. - 2020 - Provably Efficient Exploration for RL with Unsuper.pdf:application/pdf;arXiv.org Snapshot:files/631/2003.html:text/html}
}

@article{weng_analysis_2020,
	title = {Analysis of {Q}-learning with {Adaptation} and {Momentum} {Restart} for {Gradient} {Descent}},
	url = {http://arxiv.org/abs/2007.07422},
	doi = {10.24963/ijcai.2020/422},
	abstract = {Existing convergence analyses of Q-learning mostly focus on the vanilla stochastic gradient descent (SGD) type of updates. Despite the Adaptive Moment Estimation (Adam) has been commonly used for practical Q-learning algorithms, there has not been any convergence guarantee provided for Q-learning with such type of updates. In this paper, we first characterize the convergence rate for Q-AMSGrad, which is the Q-learning algorithm with AMSGrad update (a commonly adopted alternative of Adam for theoretical analysis). To further improve the performance, we propose to incorporate the momentum restart scheme to Q-AMSGrad, resulting in the so-called Q-AMSGradR algorithm. The convergence rate of Q-AMSGradR is also established. Our experiments on a linear quadratic regulator problem show that the two proposed Q-learning algorithms outperform the vanilla Q-learning with SGD updates. The two algorithms also exhibit significantly better performance than the DQN learning method over a batch of Atari 2600 games.},
	urldate = {2020-07-16},
	journal = {Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence},
	author = {Weng, Bowen and Xiong, Huaqing and Liang, Yingbin and Zhang, Wei},
	month = jul,
	year = {2020},
	note = {arXiv: 2007.07422},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control},
	pages = {3051--3057}
}

@article{jansch-porto_policy_2020,
	title = {Policy {Learning} of {MDPs} with {Mixed} {Continuous}/{Discrete} {Variables}: {A} {Case} {Study} on {Model}-{Free} {Control} of {Markovian} {Jump} {Systems}},
	shorttitle = {Policy {Learning} of {MDPs} with {Mixed} {Continuous}/{Discrete} {Variables}},
	url = {http://arxiv.org/abs/2006.03116},
	abstract = {Markovian jump linear systems (MJLS) are an important class of dynamical systems that arise in many control applications. In this paper, we introduce the problem of controlling unknown (discrete-time) MJLS as a new benchmark for policy-based reinforcement learning of Markov decision processes (MDPs) with mixed continuous/discrete state variables. Compared with the traditional linear quadratic regulator (LQR), our proposed problem leads to a special hybrid MDP (with mixed continuous and discrete variables) and poses significant new challenges due to the appearance of an underlying Markov jump parameter governing the mode of the system dynamics. Specifically, the state of a MJLS does not form a Markov chain and hence one cannot study the MJLS control problem as a MDP with solely continuous state variable. However, one can augment the state and the jump parameter to obtain a MDP with a mixed continuous/discrete state space. We discuss how control theory sheds light on the policy parameterization of such hybrid MDPs. Then we modify the widely used natural policy gradient method to directly learn the optimal state feedback control policy for MJLS without identifying either the system dynamics or the transition probability of the switching parameter. We implement the (data-driven) natural policy gradient method on different MJLS examples. Our simulation results suggest that the natural gradient method can efficiently learn the optimal controller for MJLS with unknown dynamics.},
	urldate = {2020-07-16},
	journal = {arXiv:2006.03116 [cs, eess, math]},
	author = {Jansch-Porto, Joao Paulo and Hu, Bin and Dullerud, Geir},
	month = jul,
	year = {2020},
	note = {arXiv: 2006.03116},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Electrical Engineering and Systems Science - Systems and Control},
	file = {arXiv Fulltext PDF:files/643/Jansch-Porto et al. - 2020 - Policy Learning of MDPs with Mixed ContinuousDisc.pdf:application/pdf;arXiv.org Snapshot:files/644/2006.html:text/html}
}

@article{self_online_2020,
	title = {Online inverse reinforcement learning with unknown disturbances},
	url = {http://arxiv.org/abs/2003.03912},
	abstract = {This paper addresses the problem of online inverse reinforcement learning for nonlinear systems with modeling uncertainties while in the presence of unknown disturbances. The developed approach observes state and input trajectories for an agent and identifies the unknown reward function online. Sub-optimality introduced in the observed trajectories by the unknown external disturbance is compensated for using a novel model-based inverse reinforcement learning approach. The observer estimates the external disturbances and uses the resulting estimates to learn the dynamic model of the demonstrator. The learned demonstrator model along with the observed suboptimal trajectories are used to implement inverse reinforcement learning. Theoretical guarantees are provided using Lyapunov theory and a simulation example is shown to demonstrate the effectiveness of the proposed technique.},
	urldate = {2020-07-20},
	journal = {arXiv:2003.03912 [cs, eess, math]},
	author = {Self, Ryan and Abudia, Moad and Kamalapurkar, Rushikesh},
	month = mar,
	year = {2020},
	note = {arXiv: 2003.03912},
	keywords = {Mathematics - Optimization and Control, Electrical Engineering and Systems Science - Systems and Control}
}

@article{bhandari_global_2019,
	title = {Global {Optimality} {Guarantees} {For} {Policy} {Gradient} {Methods}},
	url = {http://arxiv.org/abs/1906.01786},
	abstract = {Policy gradients methods are perhaps the most widely used class of reinforcement learning algorithms. These methods apply to complex, poorly understood, control problems by performing stochastic gradient descent over a parameterized class of polices. Unfortunately, even for simple control problems solvable by classical techniques, policy gradient algorithms face non-convex optimization problems and are widely understood to converge only to local minima. This work identifies structural properties -- shared by finite MDPs and several classic control problems -- which guarantee that policy gradient objective function has no suboptimal local minima despite being non-convex. When these assumptions are relaxed, our work gives conditions under which any local minimum is near-optimal, where the error bound depends on a notion of the expressive capacity of the policy class.},
	urldate = {2020-07-21},
	journal = {arXiv:1906.01786 [cs, stat]},
	author = {Bhandari, Jalaj and Russo, Daniel},
	month = jun,
	year = {2019},
	note = {arXiv: 1906.01786},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/670/Bhandari and Russo - 2019 - Global Optimality Guarantees For Policy Gradient M.pdf:application/pdf;arXiv.org Snapshot:files/671/1906.html:text/html}
}

@article{wang_neural_2019,
	title = {Neural {Policy} {Gradient} {Methods}: {Global} {Optimality} and {Rates} of {Convergence}},
	shorttitle = {Neural {Policy} {Gradient} {Methods}},
	url = {http://arxiv.org/abs/1909.01150},
	abstract = {Policy gradient methods with actor-critic schemes demonstrate tremendous empirical successes, especially when the actors and critics are parameterized by neural networks. However, it remains less clear whether such "neural" policy gradient methods converge to globally optimal policies and whether they even converge at all. We answer both the questions affirmatively in the overparameterized regime. In detail, we prove that neural natural policy gradient converges to a globally optimal policy at a sublinear rate. Also, we show that neural vanilla policy gradient converges sublinearly to a stationary point. Meanwhile, by relating the suboptimality of the stationary points to the representation power of neural actor and critic classes, we prove the global optimality of all stationary points under mild regularity conditions. Particularly, we show that a key to the global optimality and convergence is the "compatibility" between the actor and critic, which is ensured by sharing neural architectures and random initializations across the actor and critic. To the best of our knowledge, our analysis establishes the first global optimality and convergence guarantees for neural policy gradient methods.},
	urldate = {2020-07-21},
	journal = {arXiv:1909.01150 [cs, math, stat]},
	author = {Wang, Lingxiao and Cai, Qi and Yang, Zhuoran and Wang, Zhaoran},
	month = nov,
	year = {2019},
	note = {arXiv: 1909.01150},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/674/Wang et al. - 2019 - Neural Policy Gradient Methods Global Optimality .pdf:application/pdf;arXiv.org Snapshot:files/675/1909.html:text/html}
}

@article{zhang_global_2020,
	title = {Global {Convergence} of {Policy} {Gradient} {Methods} to ({Almost}) {Locally} {Optimal} {Policies}},
	url = {http://arxiv.org/abs/1906.08383},
	abstract = {Policy gradient (PG) methods are a widely used reinforcement learning methodology in many applications such as video games, autonomous driving, and robotics. In spite of its empirical success, a rigorous understanding of the global convergence of PG methods is lacking in the literature. In this work, we close the gap by viewing PG methods from a nonconvex optimization perspective. In particular, we propose a new variant of PG methods for infinite-horizon problems that uses a random rollout horizon for the Monte-Carlo estimation of the policy gradient. This method then yields an unbiased estimate of the policy gradient with bounded variance, which enables the tools from nonconvex optimization to be applied to establish global convergence. Employing this perspective, we first recover the convergence results with rates to the stationary-point policies in the literature. More interestingly, motivated by advances in nonconvex optimization, we modify the proposed PG method by introducing periodically enlarged stepsizes. The modified algorithm is shown to escape saddle points under mild assumptions on the reward and the policy parameterization. Under a further strict saddle points assumption, this result establishes convergence to essentially locally-optimal policies of the underlying problem, and thus bridges the gap in existing literature on the convergence of PG methods. Results from experiments on the inverted pendulum are then provided to corroborate our theory, namely, by slightly reshaping the reward function to satisfy our assumption, unfavorable saddle points can be avoided and better limit points can be attained. Intriguingly, this empirical finding justifies the benefit of reward-reshaping from a nonconvex optimization perspective.},
	urldate = {2020-07-21},
	journal = {arXiv:1906.08383 [cs, eess, math, stat]},
	author = {Zhang, Kaiqing and Koppel, Alec and Zhu, Hao and Başar, Tamer},
	month = jun,
	year = {2020},
	note = {arXiv: 1906.08383},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Electrical Engineering and Systems Science - Systems and Control, Mathematics - Statistics Theory},
	file = {arXiv Fulltext PDF:files/678/Zhang et al. - 2020 - Global Convergence of Policy Gradient Methods to (.pdf:application/pdf;arXiv.org Snapshot:files/679/1906.html:text/html}
}

@article{cai_neural_2020,
	title = {Neural {Temporal}-{Difference} and {Q}-{Learning} {Provably} {Converge} to {Global} {Optima}},
	url = {http://arxiv.org/abs/1905.10027},
	abstract = {Temporal-difference learning (TD), coupled with neural networks, is among the most fundamental building blocks of deep reinforcement learning. However, due to the nonlinearity in value function approximation, such a coupling leads to nonconvexity and even divergence in optimization. As a result, the global convergence of neural TD remains unclear. In this paper, we prove for the first time that neural TD converges at a sublinear rate to the global optimum of the mean-squared projected Bellman error for policy evaluation. In particular, we show how such global convergence is enabled by the overparametrization of neural networks, which also plays a vital role in the empirical success of neural TD. Beyond policy evaluation, we establish the global convergence of neural (soft) Q-learning, which is further connected to that of policy gradient algorithms.},
	urldate = {2020-07-21},
	journal = {arXiv:1905.10027 [cs, math, stat]},
	author = {Cai, Qi and Yang, Zhuoran and Lee, Jason D. and Wang, Zhaoran},
	month = apr,
	year = {2020},
	note = {arXiv: 1905.10027},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:files/683/USJKIMPK.pdf:application/pdf;arXiv.org Snapshot:files/684/1905.html:text/html}
}

@article{dong_expressivity_2020,
	title = {On the {Expressivity} of {Neural} {Networks} for {Deep} {Reinforcement} {Learning}},
	url = {http://arxiv.org/abs/1910.05927},
	abstract = {We compare the model-free reinforcement learning with the model-based approaches through the lens of the expressive power of neural networks for policies, \$Q\$-functions, and dynamics. We show, theoretically and empirically, that even for one-dimensional continuous state space, there are many MDPs whose optimal \$Q\$-functions and policies are much more complex than the dynamics. We hypothesize many real-world MDPs also have a similar property. For these MDPs, model-based planning is a favorable algorithm, because the resulting policies can approximate the optimal policy significantly better than a neural network parameterization can, and model-free or model-based policy optimization rely on policy parameterization. Motivated by the theory, we apply a simple multi-step model-based bootstrapping planner (BOOTS) to bootstrap a weak \$Q\$-function into a stronger policy. Empirical results show that applying BOOTS on top of model-based or model-free policy optimization algorithms at the test time improves the performance on MuJoCo benchmark tasks.},
	urldate = {2020-07-21},
	journal = {arXiv:1910.05927 [cs, stat]},
	author = {Dong, Kefan and Luo, Yuping and Ma, Tengyu},
	month = jun,
	year = {2020},
	note = {arXiv: 1910.05927},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:files/687/Dong et al. - 2020 - On the Expressivity of Neural Networks for Deep Re.pdf:application/pdf;arXiv.org Snapshot:files/688/1910.html:text/html}
}

@article{pasula_lagrangian_2020,
	title = {Lagrangian {Duality} in {Reinforcement} {Learning}},
	url = {http://arxiv.org/abs/2007.09998},
	abstract = {Although duality is used extensively in certain fields, such as supervised learning in machine learning, it has been much less explored in others, such as reinforcement learning (RL). In this paper, we show show how duality is involved in a variety of RL work, from that which spearheaded the field, such as Richard Bellman's value iteration, to that which was done within just the past few years yet has already had significant impact, such as TRPO, A3C, and GAIL. We show that duality is not uncommon in reinforcement learning, especially when value iteration, or dynamic programming, is used or when first or second order approximations are made to transform initially intractable problems into tractable convex programs.},
	urldate = {2020-07-22},
	journal = {arXiv:2007.09998 [cs, math, stat]},
	author = {Pasula, Pranay},
	month = jul,
	year = {2020},
	note = {arXiv: 2007.09998},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:files/694/Pasula - 2020 - Lagrangian Duality in Reinforcement Learning.pdf:application/pdf;arXiv.org Snapshot:files/695/2007.html:text/html}
}

@article{liu_convergence_2020,
	title = {On the {Convergence} of {Reinforcement} {Learning} with {Monte} {Carlo} {Exploring} {Starts}},
	url = {http://arxiv.org/abs/2007.10916},
	abstract = {A basic simulation-based reinforcement learning algorithm is the Monte Carlo Exploring States (MCES) method, also known as optimistic policy iteration, in which the value function is approximated by simulated returns and a greedy policy is selected at each iteration. The convergence of this algorithm in the general setting has been an open question. In this paper, we investigate the convergence of this algorithm for the case with undiscounted costs, also known as the stochastic shortest path problem. The results complement existing partial results on this topic and thereby helps further settle the open problem. As a side result, we also provide a proof of a version of the supermartingale convergence theorem commonly used in stochastic approximation.},
	urldate = {2020-07-22},
	journal = {arXiv:2007.10916 [cs, math]},
	author = {Liu, Jun},
	month = jul,
	year = {2020},
	note = {arXiv: 2007.10916},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control},
	file = {arXiv Fulltext PDF:files/698/Liu - 2020 - On the Convergence of Reinforcement Learning with .pdf:application/pdf;arXiv.org Snapshot:files/699/2007.html:text/html}
}

@article{bhandari_note_2020,
	title = {A {Note} on the {Linear} {Convergence} of {Policy} {Gradient} {Methods}},
	url = {http://arxiv.org/abs/2007.11120},
	abstract = {We revisit the finite time analysis of policy gradient methods in the simplest setting: finite state and action problems with a policy class consisting of all stochastic policies and with exact gradient evaluations. Some recent works have viewed these problems as instances of smooth nonlinear optimization problems, suggesting suggest small stepsizes and showing sublinear convergence rates. This note instead takes a policy iteration perspective and highlights that many versions of policy gradient succeed with extremely large stepsizes and attain a linear rate of convergence.},
	urldate = {2020-07-23},
	journal = {arXiv:2007.11120 [cs, math, stat]},
	author = {Bhandari, Jalaj and Russo, Daniel},
	month = jul,
	year = {2020},
	note = {arXiv: 2007.11120},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/701/Bhandari and Russo - 2020 - A Note on the Linear Convergence of Policy Gradien.pdf:application/pdf;arXiv.org Snapshot:files/702/2007.html:text/html}
}

@article{russo_approximation_2020,
	title = {Approximation {Benefits} of {Policy} {Gradient} {Methods} with {Aggregated} {States}},
	url = {http://arxiv.org/abs/2007.11684},
	abstract = {Folklore suggests that policy gradient can be more robust to misspecification than its relative, approximate policy iteration. This paper studies the case of state-aggregation, where the state space is partitioned and either the policy or value function approximation is held constant over partitions. This paper shows a policy gradient method converges to a policy whose regret per-period is bounded by \${\textbackslash}epsilon\$, the largest difference between two elements of the state-action value function belonging to a common partition. With the same representation, both approximate policy iteration and approximate value iteration can produce policies whose per-period regret scales as \${\textbackslash}epsilon/(1-{\textbackslash}gamma)\$, where \${\textbackslash}gamma\$ is a discount factor. Theoretical results synthesize recent analysis of policy gradient methods with insights of Van Roy (2006) into the critical role of state-relevance weights in approximate dynamic programming.},
	urldate = {2020-07-24},
	journal = {arXiv:2007.11684 [cs, math, stat]},
	author = {Russo, Daniel},
	month = jul,
	year = {2020},
	note = {arXiv: 2007.11684},
	keywords = {Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning}
}

@article{jin_is_2018,
	title = {Is {Q}-learning {Provably} {Efficient}?},
	url = {http://arxiv.org/abs/1807.03765},
	abstract = {Model-free reinforcement learning (RL) algorithms, such as Q-learning, directly parameterize and update value functions or policies without explicitly modeling the environment. They are typically simpler, more flexible to use, and thus more prevalent in modern deep RL than model-based approaches. However, empirical work has suggested that model-free algorithms may require more samples to learn [Deisenroth and Rasmussen 2011, Schulman et al. 2015]. The theoretical question of "whether model-free algorithms can be made sample efficient" is one of the most fundamental questions in RL, and remains unsolved even in the basic scenario with finitely many states and actions. We prove that, in an episodic MDP setting, Q-learning with UCB exploration achieves regret \${\textbackslash}tilde\{O\}({\textbackslash}sqrt\{H{\textasciicircum}3 SAT\})\$, where \$S\$ and \$A\$ are the numbers of states and actions, \$H\$ is the number of steps per episode, and \$T\$ is the total number of steps. This sample efficiency matches the optimal regret that can be achieved by any model-based approach, up to a single \${\textbackslash}sqrt\{H\}\$ factor. To the best of our knowledge, this is the first analysis in the model-free setting that establishes \${\textbackslash}sqrt\{T\}\$ regret without requiring access to a "simulator."},
	urldate = {2020-07-25},
	journal = {arXiv:1807.03765 [cs, math, stat]},
	author = {Jin, Chi and Allen-Zhu, Zeyuan and Bubeck, Sebastien and Jordan, Michael I.},
	month = jul,
	year = {2018},
	note = {arXiv: 1807.03765},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Mathematics - Optimization and Control, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:files/741/Jin et al. - 2018 - Is Q-learning Provably Efficient.pdf:application/pdf;arXiv.org Snapshot:files/742/1807.html:text/html}
}

@article{cen2021fast,
	title={Fast policy extragradient methods for competitive games with entropy regularization},
	author={Cen, Shicong and Wei, Yuting and Chi, Yuejie},
	journal={Advances in Neural Information Processing Systems},
	volume={34},
	year={2021}
}
@article{chen2019nonlinearSA,
	title={Finite-sample analysis of nonlinear stochastic approximation with applications in reinforcement learning},
	author={Chen, Zaiwei and Zhang, Sheng and Doan, Thinh T and Clarke, John-Paul and Maguluri, Siva Theja},
	journal={Automatica},
	volume={146},
	pages={110623},
	year={2022},
	publisher={Elsevier}
}

inproceedings{boyan1999lstd,
	author = {Boyan, Justin A.},
	title = {Least-Squares Temporal Difference Learning},
	year = {1999},
	isbn = {1558606122},
	publisher = {Morgan Kaufmann Publishers Inc.},
	address = {San Francisco, CA, USA},
	booktitle = {Proceedings of the Sixteenth International Conference on Machine Learning},
	pages = {49–56},
	numpages = {8},
	series = {ICML '99}
}

@conference {zhang2021gradientStochasticGame,
	title = {Gradient Play in Stochastic Games: Stationary Points and Local Geometry},
	booktitle = {25th International Symposium on Mathematical Theory of Networks and Systems (MTNS 2022)},
	year = {2022},
	//url = {https://drive.google.com/file/d/1quEJpEUGGvc5zqCdkbUBaqwTzG3saiRb/view?usp=sharing},
	author = {Runyu Zhang and Zhaolin Ren and Li., Na}
}


ARTICLE{zhang2021gradientStochasticGame,
	author = {{Zhang}, Runyu and {Ren}, Zhaolin and {Li}, Na},
	title = "{Gradient play in stochastic games: stationary points, convergence, and sample complexity}",
	journal = {arXiv e-prints},
	keywords = {Computer Science - Machine Learning, Computer Science - Computer Science and Game Theory, Computer Science - Multiagent Systems, Mathematics - Optimization and Control},
	year = 2021,
	month = may,
	eid = {arXiv:2106.00198},
	pages = {arXiv:2106.00198},
	archivePrefix = {arXiv},
	eprint = {2106.00198},
	primaryClass = {cs.LG},
	adsurl = {https://ui.adsabs.harvard.edu/abs/2021arXiv210600198Z},
	adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}
@article{yu2009convergence,
  title={Convergence results for some temporal difference methods based on least squares},
  author={Yu, Huizhen and Bertsekas, Dimitri P},
  journal={IEEE Transactions on Automatic Control},
  volume={54},
  number={7},
  pages={1515--1531},
  year={2009},
  publisher={IEEE}
}
@inproceedings{
	leo2021convergeMPG,
	title={Global Convergence of Multi-Agent Policy Gradient in Markov Potential Games},
	author={Stefanos Leonardos and Will Overman and Ioannis Panageas and Georgios Piliouras},
	booktitle={International Conference on Learning Representations},
	year={2022},
	//url={https://openreview.net/forum?id=gfwON7rAm4}
}
@inproceedings{dalal2018finite,
  title={Finite sample analysis of two-timescale stochastic approximation with applications to reinforcement learning},
  author={Dalal, Gal and Thoppe, Gugan and Sz{\"o}r{\'e}nyi, Bal{\'a}zs and Mannor, Shie},
  booktitle={Conference On Learning Theory},
  pages={1199--1233},
  year={2018},
  organization={PMLR}
}
inproceedings{leo2021convergeMPG,
	title={Global Convergence of Multi-Agent Policy Gradient in Markov Potential Games},
	author={Leonardos, Stefanos and Overman, Will and Panageas, Ioannis and Piliouras, Georgios},
	booktitle={ICLR 2022 Workshop on Gamification and Multiagent Solutions}
}

% ARTICLE{leo2021convergeMPG,
	%        author = {{Leonardos}, Stefanos and {Overman}, Will and {Panageas}, Ioannis and {Piliouras}, Georgios},
	%         title = "{Global Convergence of Multi-Agent Policy Gradient in Markov Potential Games}",
	%       journal = {arXiv e-prints},
	%      keywords = {Computer Science - Machine Learning, Computer Science - Computer Science and Game Theory},
	%          year = 2021,
	%         month = jun,
	%           eid = {arXiv:2106.01969},
	%         pages = {arXiv:2106.01969},
	% archivePrefix = {arXiv},
	%        eprint = {2106.01969},
	%  primaryClass = {cs.LG},
	%        adsurl = {https://ui.adsabs.harvard.edu/abs/2021arXiv210601969L},
	%       adsnote = {Provided by the SAO/NASA Astrophysics Data System}
	% }

@inproceedings{pmlr-v162-ding22b,
	title={{Independent policy gradient for large-scale markov potential games: Sharper rates, function approximation, and game-agnostic convergence}},
	author={Ding, Dongsheng and Wei, Chen-Yu and Zhang, Kaiqing and Jovanovic, Mihailo},
	booktitle={International Conference on Machine Learning},
	pages={5166--5220},
	year={2022},
	organization={PMLR}
}
@article{tsitsiklis1997analysis,
  title={{An Analysis of Temporal-Difference Learning with Function Approximation}},
  author={Tsitsiklis, John N and Van Roy, Benjamin},
  journal={IEEE Transactions on Automatic Control},
  volume={42},
  number={5},
  pages={674--690},
  year={1997},
  publisher={IEEE-Institute of Electrical and Electronics Engineers, Inc., New York, USA}
}
InProceedings{pmlr-v162-ding22b,
	title = 	 {Independent Policy Gradient for Large-Scale {M}arkov Potential Games: Sharper Rates, Function Approximation, and Game-Agnostic Convergence},
	author =       {Ding, Dongsheng and Wei, Chen-Yu and Zhang, Kaiqing and Jovanovic, Mihailo},
	booktitle = 	 {Proceedings of the 39th International Conference on Machine Learning},
	pages = 	 {5166--5220},
	year = 	 {2022},
	editor = 	 {Chaudhuri, Kamalika and Jegelka, Stefanie and Song, Le and Szepesvari, Csaba and Niu, Gang and Sabato, Sivan},
	volume = 	 {162},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {17--23 Jul},
	publisher =    {PMLR},
	pdf = 	 {https://proceedings.mlr.press/v162/ding22b/ding22b.pdf},
	url = 	 {https://proceedings.mlr.press/v162/ding22b.html},
	abstract = 	 {We examine global non-asymptotic convergence properties of policy gradient methods for multi-agent reinforcement learning (RL) problems in Markov potential games (MPGs). To learn a Nash equilibrium of an MPG in which the size of state space and/or the number of players can be very large, we propose new independent policy gradient algorithms that are run by all players in tandem. When there is no uncertainty in the gradient evaluation, we show that our algorithm finds an $\epsilon$-Nash equilibrium with $O(1/\epsilon^2)$ iteration complexity which does not explicitly depend on the state space size. When the exact gradient is not available, we establish $O(1/\epsilon^5)$ sample complexity bound in a potentially infinitely large state space for a sample-based algorithm that utilizes function approximation. Moreover, we identify a class of independent policy gradient algorithms that enjoy convergence for both zero-sum Markov games and Markov cooperative games with the players that are oblivious to the types of games being played. Finally, we provide computational experiments to corroborate the merits and the effectiveness of our theoretical developments.}
}
@inproceedings{Srikant2019FiniteTimeEB,
  title={{Finite-time error bounds for linear stochastic approximation and TD learning}},
  author={Srikant, Rayadurgam and Ying, Lei},
  booktitle={Conference on Learning Theory},
  pages={2803--2830},
  year={2019},
  organization={PMLR}
}

@article{lin2021multi,
	title={Multi-agent reinforcement learning in stochastic networked systems},
	author={Lin, Yiheng and Qu, Guannan and Huang, Longbo and Wierman, Adam},
	journal={Advances in Neural Information Processing Systems},
	volume={34},
	pages={7825--7837},
	year={2021}
}

InProceedings{pmlr-v151-fox22a,
	title = 	 { Independent Natural Policy Gradient always converges in Markov Potential Games },
	author =       {Fox, Roy and Mcaleer, Stephen M. and Overman, Will and Panageas, Ioannis},
	booktitle = 	 {Proceedings of The 25th International Conference on Artificial Intelligence and Statistics},
	pages = 	 {4414--4425},
	year = 	 {2022},
	editor = 	 {Camps-Valls, Gustau and Ruiz, Francisco J. R. and Valera, Isabel},
	volume = 	 {151},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {28--30 Mar},
	publisher =    {PMLR},
	pdf = 	 {https://proceedings.mlr.press/v151/fox22a/fox22a.pdf},
	url = 	 {https://proceedings.mlr.press/v151/fox22a.html},
	abstract = 	 { Natural policy gradient has emerged as one of the most successful algorithms for computing optimal policies in challenging Reinforcement Learning (RL) tasks, yet, very little was known about its convergence properties until recently. The picture becomes more blurry when it comes to multi-agent RL (MARL); the line of works that have theoretical guarantees for convergence to Nash policies are very limited. In this paper, we focus on a particular class of multi-agent stochastic games called Markov Potential Games and we prove that Independent Natural Policy Gradient always converges using constant learning rates. The proof deviates from the existing approaches and the main challenge lies in the fact that Markov potential Games do not have unique optimal values (as single-agent settings exhibit) so different initializations can lead to different limit point values. We complement our theoretical results with experiments that indicate that Natural Policy Gradient outperforms Policy Gradient in MARL settings (our process benchmark is multi-state congestion games). }
}

@ARTICLE{cen2022INPG4PG,
	author = {{Cen}, Shicong and {Chen}, Fan and {Chi}, Yuejie},
	title = "{Independent Natural Policy Gradient Methods for Potential Games: Finite-time Global Convergence with Entropy Regularization}",
	journal = {arXiv e-prints},
	keywords = {Mathematics - Optimization and Control, Computer Science - Computer Science and Game Theory, Computer Science - Information Theory, Computer Science - Machine Learning},
	year = 2022,
	month = apr,
	eid = {arXiv:2204.05466},
	pages = {arXiv:2204.05466},
	archivePrefix = {arXiv},
	eprint = {2204.05466},
	primaryClass = {math.OC},
	adsurl = {https://ui.adsabs.harvard.edu/abs/2022arXiv220405466C},
	adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@inproceedings{zhang2022logBarrierSoftmax,
	title={On the Global Convergence Rates of Decentralized Softmax Gradient Play in Markov Potential Games},
	author={Zhang, Runyu and Mei, Jincheng and Dai, Bo and Schuurmans, Dale and Li, Na},
	booktitle={Advances in Neural Information Processing Systems},
	year=2022
}
@inproceedings{jaakkola1994convergence,
	title        = {Convergence of stochastic iterative dynamic programming algorithms},
	author       = {Jaakkola, Tommi and Jordan, Michael I and Singh, Satinder P},
	year         = 1994,
	booktitle    = {Advances in neural information processing systems},
	pages        = {703--710}
}
@article{zhang2022global,
	title={{Global Convergence of Localized Policy Iteration in Networked Multi-Agent Reinforcement Learning}},
	author={Zhang, Yizhou and Qu, Guannan and Xu, Pan and Lin, Yiheng and Chen, Zaiwei and Wierman, Adam},
	journal={Preprint arXiv:2211.17116},
	year={2022}
}
conference {zhang2022logBarrierSoftmax,
	title = {On the Global Convergence Rates of Decentralized Softmax Gradient Play in Markov Potential Games},
	booktitle = {Thirty-sixth Conference on Neural Information Processing Systems (NeurIPS 2022)},
	year = {2022},
	url = {https://arxiv.org/abs/2202.00872},
	author = {Runyu Zhang and Jincheng Mei and Bo Dai and Dale Schuurmans and Li., Na}
}


@inproceedings{qu2019scalableMARL,
	title={{Scalable reinforcement learning of localized policies for multi-agent networked systems}},
	author={Qu, Guannan and Wierman, Adam and Li, Na},
	booktitle={Learning for Dynamics and Control},
	pages={256--266},
	year={2020},
	organization={PMLR}
}
@inproceedings{bhandari2018finite,
	title        = {{A Finite Time Analysis of Temporal Difference Learning With Linear Function Approximation}},
	author       = {Bhandari, Jalaj and Russo, Daniel and Singal, Raghav},
	year         = 2018,
	booktitle    = {Conference On Learning Theory},
	pages        = {1691--1692}
}

ARTICLE{qu2019scalableMARL,
	author = {{Qu}, Guannan and {Wierman}, Adam and {Li}, Na},
	title = "{Scalable Reinforcement Learning for Multi-Agent Networked Systems}",
	journal = {arXiv e-prints},
	keywords = {Mathematics - Optimization and Control, Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
	year = 2019,
	month = dec,
	eid = {arXiv:1912.02906},
	pages = {arXiv:1912.02906},
	archivePrefix = {arXiv},
	eprint = {1912.02906},
	primaryClass = {math.OC},
	adsurl = {https://ui.adsabs.harvard.edu/abs/2019arXiv191202906Q},
	adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@article{sutton1999PGT,
	title={{Policy gradient methods for reinforcement learning with function approximation}},
	author={Sutton, Richard S and McAllester, David and Singh, Satinder and Mansour, Yishay},
	journal={Advances in neural information processing systems},
	volume={12},
	year={1999}
}
inproceedings{sutton1999PGT,
	author = {Sutton, Richard S and McAllester, David and Singh, Satinder and Mansour, Yishay},
	booktitle = {Advances in Neural Information Processing Systems},
	editor = {S. Solla and T. Leen and K. M\"{u}ller},
	pages = {},
	publisher = {MIT Press},
	title = {Policy Gradient Methods for Reinforcement Learning with Function Approximation},
	url = {https://proceedings.neurips.cc/paper/1999/file/464d828b85b0bed98e80ade0a5c43b0f-Paper.pdf},
	volume = {12},
	year = {1999}
}
@article{roughgarden2004bounding,
	title={Bounding the inefficiency of equilibria in nonatomic congestion games},
	author={Roughgarden, Tim and Tardos, {\'E}va},
	journal={Games and economic behavior},
	volume={47},
	number={2},
	pages={389--403},
	year={2004},
	publisher={Elsevier}
}
@inproceedings{shi2022stability,
	title={Stability constrained reinforcement learning for real-time voltage control},
	author={Shi, Yuanyuan and Qu, Guannan and Low, Steven and Anandkumar, Anima and Wierman, Adam},
	booktitle={2022 American Control Conference (ACC)},
	pages={2715--2721},
	year={2022},
	organization={IEEE}
}
@inproceedings{mao2022Decent-MARL,
	title={{On improving model-free algorithms for decentralized multi-agent reinforcement learning}},
	author={Mao, Weichao and Yang, Lin and Zhang, Kaiqing and Basar, Tamer},
	booktitle={International Conference on Machine Learning},
	pages={15007--15049},
	year={2022},
	organization={PMLR}
}


InProceedings{mao2022Decent-MARL,
	title = 	 {On Improving Model-Free Algorithms for Decentralized Multi-Agent Reinforcement Learning},
	author =       {Mao, Weichao and Yang, Lin and Zhang, Kaiqing and Basar, Tamer},
	booktitle = 	 {Proceedings of the 39th International Conference on Machine Learning},
	pages = 	 {15007--15049},
	year = 	 {2022},
	editor = 	 {Chaudhuri, Kamalika and Jegelka, Stefanie and Song, Le and Szepesvari, Csaba and Niu, Gang and Sabato, Sivan},
	volume = 	 {162},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {17--23 Jul},
	publisher =    {PMLR},
	pdf = 	 {https://proceedings.mlr.press/v162/mao22a/mao22a.pdf},
	url = 	 {https://proceedings.mlr.press/v162/mao22a.html},
	abstract = 	 {Multi-agent reinforcement learning (MARL) algorithms often suffer from an exponential sample complexity dependence on the number of agents, a phenomenon known as <em>the curse of multiagents</em>. We address this challenge by investigating sample-efficient model-free algorithms in <em>decentralized</em> MARL, and aim to improve existing algorithms along this line. For learning (coarse) correlated equilibria in general-sum Markov games, we propose <em>stage-based</em> V-learning algorithms that significantly simplify the algorithmic design and analysis of recent works, and circumvent a rather complicated no-<em>weighted</em>-regret bandit subroutine. For learning Nash equilibria in Markov potential games, we propose an independent policy gradient algorithm with a decentralized momentum-based variance reduction technique. All our algorithms are decentralized in that each agent can make decisions based on only its local information. Neither communication nor centralized coordination is required during learning, leading to a natural generalization to a large number of agents. Finally, we provide numerical simulations to corroborate our theoretical findings.}
}

@inproceedings{pmlr-v119-mei20b,
	title={{On the global convergence rates of softmax policy gradient methods}},
	author={Mei, Jincheng and Xiao, Chenjun and Szepesvari, Csaba and Schuurmans, Dale},
	booktitle={International Conference on Machine Learning},
	pages={6820--6829},
	year={2020},
	organization={PMLR}
}

InProceedings{pmlr-v119-mei20b,
	title = 	 {On the Global Convergence Rates of Softmax Policy Gradient Methods},
	author =       {Mei, Jincheng and Xiao, Chenjun and Szepesvari, Csaba and Schuurmans, Dale},
	booktitle = 	 {Proceedings of the 37th International Conference on Machine Learning},
	pages = 	 {6820--6829},
	year = 	 {2020},
	editor = 	 {III, Hal Daumé and Singh, Aarti},
	volume = 	 {119},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {13--18 Jul},
	publisher =    {PMLR},
	pdf = 	 {http://proceedings.mlr.press/v119/mei20b/mei20b.pdf},
	url = 	 {https://proceedings.mlr.press/v119/mei20b.html},
	abstract = 	 {We make three contributions toward better understanding policy gradient methods in the tabular setting. First, we show that with the true gradient, policy gradient with a softmax parametrization converges at a $O(1/t)$ rate, with constants depending on the problem and initialization. This result significantly expands the recent asymptotic convergence results. The analysis relies on two findings: that the softmax policy gradient satisfies a Ł{}ojasiewicz inequality, and the minimum probability of an optimal action during optimization can be bounded in terms of its initial value. Second, we analyze entropy regularized policy gradient and show that it enjoys a significantly faster linear convergence rate $O(e^{-t})$ toward softmax optimal policy. This result resolves an open question in the recent literature. Finally, combining the above two results and additional new $\Omega(1/t)$ lower bound results, we explain how entropy regularization improves policy optimization, even with the true gradient, from the perspective of convergence rate. The separation of rates is further explained using the notion of non-uniform Ł{}ojasiewicz degree. These results provide a theoretical understanding of the impact of entropy and corroborate existing empirical studies.}
}
@inproceedings{qu2020finite,
	title={{Finite-time analysis of asynchronous stochastic approximation and $ Q $-learning}},
	author={Qu, Guannan and Wierman, Adam},
	booktitle={Conference on Learning Theory},
	pages={3185--3205},
	year={2020},
	organization={PMLR}
}
@article{Xu2020AC,
	title={{Improving sample complexity bounds for (natural) actor-critic algorithms}},
	author={Xu, Tengyu and Wang, Zhe and Liang, Yingbin},
	journal={Advances in Neural Information Processing Systems},
	volume={33},
	pages={4358--4369},
	year={2020}
}

inproceedings{Xu2020AC,
	author = {Xu, Tengyu and Wang, Zhe and Liang, Yingbin},
	booktitle = {Advances in Neural Information Processing Systems},
	editor = {H. Larochelle and M. Ranzato and R. Hadsell and M.F. Balcan and H. Lin},
	pages = {4358--4369},
	publisher = {Curran Associates, Inc.},
	title = {Improving Sample Complexity Bounds for (Natural) Actor-Critic Algorithms},
	url = {https://proceedings.neurips.cc/paper/2020/file/2e1b24a664f5e9c18f407b2f9c73e821-Paper.pdf},
	volume = {33},
	year = {2020}
}

@article{Zou2019SARSA,
	title={{Finite-sample analysis for sarsa with linear function approximation}},
	author={Zou, Shaofeng and Xu, Tengyu and Liang, Yingbin},
	journal={Advances in neural information processing systems},
	volume={32},
	year={2019}
}

inproceedings{Zou2019SARSA,
	author = {Zou, Shaofeng and Xu, Tengyu and Liang, Yingbin},
	booktitle = {Advances in Neural Information Processing Systems},
	editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
	pages = {},
	publisher = {Curran Associates, Inc.},
	title = {Finite-Sample Analysis for SARSA with Linear Function Approximation},
	//url = {https://proceedings.neurips.cc/paper/2019/file/9f9e8cba3700df6a947a8cf91035ab84-Paper.pdf},
	volume = {32},
	year = {2019}
}

@inproceedings{
	Xu2020Reanalysis,
	title={Reanalysis of Variance Reduced Temporal Difference Learning},
	author={Tengyu Xu and Zhe Wang and Yi Zhou and Yingbin Liang},
	booktitle={International Conference on Learning Representations},
	year={2020},
	//url={https://openreview.net/forum?id=S1ly10EKDS}
}

@inproceedings{pmlr-v75-bhandari18a,
	title={{A finite time analysis of temporal difference learning with linear function approximation}},
	author={Bhandari, Jalaj and Russo, Daniel and Singal, Raghav},
	booktitle={Conference on learning theory},
	pages={1691--1692},
	year={2018},
	organization={PMLR}
}

InProceedings{pmlr-v75-bhandari18a,
	title = 	 {A Finite Time Analysis of Temporal Difference Learning With Linear Function Approximation},
	author =       {Bhandari, Jalaj and Russo, Daniel and Singal, Raghav},
	booktitle = 	 {Proceedings of the 31st  Conference On Learning Theory},
	pages = 	 {1691--1692},
	year = 	 {2018},
	editor = 	 {Bubeck, Sébastien and Perchet, Vianney and Rigollet, Philippe},
	volume = 	 {75},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {06--09 Jul},
	publisher =    {PMLR},
	pdf = 	 {http://proceedings.mlr.press/v75/bhandari18a/bhandari18a.pdf},
	url = 	 {https://proceedings.mlr.press/v75/bhandari18a.html},
	abstract = 	 {Temporal difference learning (TD) is a simple iterative algorithm used to estimate the value function corresponding to a given policy in a Markov decision process. Although TD is one of the most widely used algorithms in reinforcement learning, its theoretical analysis has proved challenging and few guarantees on its statistical efficiency are available. In this work, we provide a \emph{simple and explicit finite time analysis} of temporal difference learning with linear function approximation. Except for a few key insights, our analysis mirrors standard techniques for  analyzing stochastic gradient descent algorithms, and therefore inherits the simplicity and elegance of that literature. A final section of the paper shows that all of our main results extend to the study of a variant of Q-learning applied to optimal stopping problems.}
}
@inproceedings{fox2022independent,
	title={Independent natural policy gradient always converges in Markov potential games},
	author={Fox, Roy and Mcaleer, Stephen M and Overman, Will and Panageas, Ioannis},
	booktitle={International Conference on Artificial Intelligence and Statistics},
	pages={4414--4425},
	year={2022},
	organization={PMLR}
}
@incollection{littman1994markov,
	title={Markov games as a framework for multi-agent reinforcement learning},
	author={Littman, Michael L},
	booktitle={Machine learning proceedings 1994},
	pages={157--163},
	year={1994},
	publisher={Elsevier}
}

@article{gu2022mean,
	title={{Mean-Field Multi-Agent Reinforcement Learning: A Decentralized Network Approach}},
	author={Gu, Haotian and Guo, Xin and Wei, Xiaoli and Xu, Renyuan},
	journal={Available at SSRN 3900139},
	year={2022}
}

@book{bertsekas1996neuro,
	title={Neuro-dynamic programming},
	author={Bertsekas, Dimitri and Tsitsiklis, John N},
	year={1996},
	publisher={Athena Scientific}
}
@article{silver2017mastering,
	title={Mastering the game of go without human knowledge},
	author={Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and others},
	journal={nature},
	volume={550},
	number={7676},
	pages={354--359},
	year={2017},
	publisher={Nature Publishing Group}
}
@book{levin2017markov,
	title={{Markov chains and mixing times}},
	author={Levin, David A and Peres, Yuval},
	volume={107},
	year={2017},
	publisher={American Mathematical Soc.}
}


@article{chen2023colt,
	title={???},
	author={???},
	journal={???}
}

@article{marden2013distributed,
	title={Distributed welfare games},
	author={Marden, Jason R and Wierman, Adam},
	journal={Operations Research},
	volume={61},
	number={1},
	pages={155--168},
	year={2013},
	publisher={INFORMS}
}
@article{lan2022policy,
	title={Policy mirror descent for reinforcement learning: Linear convergence, new sampling complexity, and generalized problem classes},
	author={Lan, Guanghui},
	journal={Mathematical programming},
	pages={1--48},
	year={2022},
	publisher={Springer}
}

@inproceedings{boyan1999least,
	title={{Least-Squares Temporal Difference Learning}},
	author={Boyan, Justin A},
	booktitle={Proceedings of the Sixteenth International Conference on Machine Learning},
	pages={49--56},
	year={1999}
}

@inproceedings{
	macua2018learning,
	title={Learning parametric closed-loop policies for markov potential games},
	author={Macua, Sergio Valcarcel and Zazo, Javier and Zazo, Santiago},
	booktitle={International Conference on Learning Representations},
	year={2018}
}

@article{dechert2006stochastic,
	title={The stochastic lake game: A numerical solution},
	author={Dechert, W Davis and O’Donnell, SI},
	journal={Journal of Economic Dynamics and Control},
	volume={30},
	number={9-10},
	pages={1569--1587},
	year={2006},
	publisher={Elsevier}
}

@misc{lee2022systems,
	title={Systems and methods for adaptive EV charging},
	author={Lee, Zachary J and Li, Tongxin and Low, Steven H and Sharma, Sunash B},
	year={2022},
	month=jul # "~5",
	publisher={Google Patents},
	note={US Patent 11,376,981}
}

@article{zhang2016control,
	title={Control of robotic mobility-on-demand systems: a queueing-theoretical perspective},
	author={Zhang, Rick and Pavone, Marco},
	journal={The International Journal of Robotics Research},
	volume={35},
	number={1-3},
	pages={186--203},
	year={2016},
	publisher={SAGE Publications Sage UK: London, England}
}

@article{application_chakrabarti2008epidemic,
	title={Epidemic thresholds in real networks},
	author={Chakrabarti, Deepayan and Wang, Yang and Wang, Chenxi and Leskovec, Jurij and Faloutsos, Christos},
	journal={ACM Transactions on Information and System Security (TISSEC)},
	volume={10},
	number={4},
	pages={1},
	year={2008},
	publisher={ACM}
}

@article{application_communication,
	author = {Vogels, Werner and van Renesse, Robbert and Birman, Ken},
	title = {The Power of Epidemics: Robust Communication for Large-scale Distributed Systems},
	journal = {SIGCOMM Comput. Commun. Rev.},
	issue_date = {January 2003},
	volume = {33},
	number = {1},
	month = jan,
	year = {2003},
	issn = {0146-4833},
	pages = {131--135},
	numpages = {5},
	//url = {http://doi.acm.org/10.1145/774763.774784},
	doi = {10.1145/774763.774784},
	acmid = {774784},
	publisher = {ACM},
	address = {New York, NY, USA},
}

@article{gu2021mean,
  title={Mean-field controls with Q-learning for cooperative MARL: convergence and complexity analysis},
  author={Gu, Haotian and Guo, Xin and Wei, Xiaoli and Xu, Renyuan},
  journal={SIAM Journal on Mathematics of Data Science},
  volume={3},
  number={4},
  pages={1168--1196},
  year={2021},
  publisher={SIAM}
}

@inproceedings{mondal2022can,
  title={Can mean field control (mfc) approximate cooperative multi agent reinforcement learning (marl) with non-uniform interaction?},
  author={Mondal, Washim Uddin and Aggarwal, Vaneet and Ukkusuri, Satish V},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={1371--1380},
  year={2022},
  organization={PMLR}
}

@article{mondal2022on,
title={On the Near-Optimality of Local Policies in Large Cooperative Multi-Agent Reinforcement Learning},
author={Washim Uddin Mondal and Vaneet Aggarwal and Satish Ukkusuri},
journal={Transactions on Machine Learning Research},
issn={2835-8856},
year={2022},
note={}
}

@article{kalogiannis2022efficiently,
  title={Efficiently Computing Nash Equilibria in Adversarial Team Markov Games},
  author={Kalogiannis, Fivos and Anagnostides, Ioannis and Panageas, Ioannis and Vlatakis-Gkaragkounis, Emmanouil-Vasileios and Chatziafratis, Vaggos and Stavroulakis, Stelios},
  journal={Preprint arXiv:2208.02204},
  year={2022}
}

@article{zhang2021multi,
  title={Multi-agent reinforcement learning: A selective overview of theories and algorithms},
  author={Zhang, Kaiqing and Yang, Zhuoran and Ba{\c{s}}ar, Tamer},
  journal={Handbook of reinforcement learning and control},
  pages={321--384},
  year={2021},
  publisher={Springer}
}