\section{Preliminaries}\label{pre}

\subsection{Game theory}

\textbf{Zero-sum game.} A two players zero-sum game consists of two agents $\CN = \{1,2\}$, and the losses of both players are determined by a payoff matrix $A  \in \BR^{m \times n}$. At each time $t$, the player 1 selects
a mixed strategy $\tb{x}^t_1$ from the simplex constrains 
\begin{align*}
    \Delta_m = \left\{\tb{x} \in \BR^m | \sum^m_{i=1} \tb{x}_{i} = 1, \tb{x}_i \ge 0 \right\},
\end{align*}
and similarly, the player 2 selects a mixed strategy $\tb{x}^t_2$ from the simplex constrains $\Delta_n$. 
Given that player $1$ selects strategy $\tb{x}_1 \in \Delta_m$ and player $2$ selects strategy $\tb{x}_2 \in \Delta_n$, player 1 receives loss $u_1(x,y) = -\langle \tb{x}_1, A \tb{x}_2 \rangle$, and player 2 receives loss $u_2(x,y) = \langle \tb{x}_2, A^{\top} \tb{x}_1 \rangle$.  Naturally, players want to minimize their loss resulting the following min-max problem: 
\begin{equation}
\max_{\tb{x}_1 \in \Delta_m} \min_{\tb{x}_2 \in \Delta_n} \tb{x}_1^{\top} A \tb{x}_2 \tag{Zero-Sum Game}
\end{equation}

A mixed strategy $\tb{x} \in \Delta_m$ is called fully mixed if $\tb{x}_i > 0$ for all $i \in [m]$. The $\KL$-divergence between two pairs of fully mixed strategies $(\tb{x},\tb{y})$ and $(\tb{x}',\tb{y}') \in \Delta_m \times \Delta_n$ is defined as
\begin{align*}
    \KL \left((\tb{x},\tb{y}),(\tb{x}',\tb{y}')\right) = \sum_{i \in [m]} \tb{x}_i \ln \left( \frac{\tb{x}_i}{\tb{x}'_i} \right) + \sum_{j \in [n]} \tb{y}_j \ln \left( \frac{\tb{y}_j}{\tb{y}'_j} \right).
\end{align*}

The KL-divergence can be considered as a measurement of the distance between two pairs of mixed strategies. Note that for fixed $(\tb{x},\tb{y})$, the KL-divergence will diverge to infinity when $(\tb{x}',\tb{y}')$ approaches the boundary of the simplex constrains, i,e., when some components of $(\tb{x}',\tb{y}')$ tends to zero.


\paragraph{Periodic zero-sum game.}  In this paper, we study games in which the payoff matrices vary over time periodically.

\begin{defn}[Periodic zero-sum games]\label{pg2} A periodic game with period $\CT$ is an infinite sequence of zero-sum bilinear games 
$\{A_t\}^{\infty}_{t = 0} \subset  \BR^{n \times m}$, and $A_{t+\CT} = A_t$ for all $t \ge 1$. 
\end{defn}
Note that the \textit{time-independent game} is a special case of the periodic game with $\CT=1$. The periodic game defined here is the same as \citep{feng2023last}, except they consider the unconstrained case while we consider the constrained case.  A continuous-time counterpart of the periodic zero-sum game was also studied in \citep{fiez2021online}.

\subsection{Learning dynamics in games} 
In this paper we consider two types of learning dynamics : Optimistic Multiplicative Weights Updates  (OMWU) and Extra-gradient Multiplicative Weights Updates (Extra-MWU), which are variants of the Multiplicative Weights Updates algorithms (MWU). Both (OMWU) and (Extra-MWU) possess the last-iterate convergence property in repeated game with a time-independent payoff matrix and simplex constrains, as demonstrated in previous literature \citep{daskalakis2018last,mertikopoulos2018optimistic,wei2020linear,fasoulakis2022forward}.  Here we state their forms within a time-varying context.
\paragraph{MWU.}
The dynamics of MWU is
\begin{align*}\label{OMWU}
&\tb{x}_1^{t+1} = 
\left(\frac{\tb{x}_{1,i}^t e^{\eta (A_t \tb{x}_{2}^t)^i}} { \sum^m_{s=1}\tb{x}_{1,s}^t e^{ \eta (A_t \tb{x}_{2}^t)^s}}\right)^{m}_{ i =1},\\
& \\
&\tb{x}_2^{t+1} = 
\left(\frac{\tb{x}_{2,j}^t e^{- \eta (A^{\top}_t \tb{x}_1^t)^j }} { \sum^m_{s=1}\tb{x}_{2,s}^t e^{- \eta (A^{\top}_t \tb{x}_1^t)^s}}\right)^{n}_{ j =1}.\tag{MWU}
\end{align*}
Here $\eta$ represents the step size. (MWU) belongs to the general class of Follow-the-Regularized-Leader algorithms (FTRL), which play a central role in the online learning problems \citep{shalev2012online}. It is known that when two players both use (MWU) to update their strategies in a time-independent zero-sum game, the trajectories of their strategies will not converge and may diverge to the boundary of the simplex \citep{bailey2018multiplicative}. 

Recently, there are also works that study the dynamical behaviors of continuous-time partner of (MWU) and more general FTRL dynamics in periodic game. It is shown that these dynamics exhibit the Poincar\'{e} recurrence property in a periodic game \citep{fiez2021online}.



\paragraph{Optimistic MWU.}
The dynamics of Optimistic-MWU is
\begin{align*}\label{OMWU}
&\tb{x}_1^{t+1} = 
\left(\frac{\tb{x}_{1,i}^t e^{2 \eta (A_t \tb{x}_{2}^t)^i - \eta (A_{t-1} \tb{x}_2^{t-1})^{i}}} { \sum^m_{s=1}\tb{x}_{1,s}^t e^{2 \eta (A_t \tb{x}_{2}^t)^s - \eta (A_{t-1} \tb{x}_2^{t-1})^s}}\right)^{m}_{ i =1},\\
& \\
&\tb{x}_2^{t+1} = 
\left(\frac{\tb{x}_{2,j}^t e^{-2 \eta (A^{\top}_t \tb{x}_1^t)^j + \eta (A^{\top}_{t-1} \tb{x}_1^{t-1})^j}} { \sum^m_{s=1}\tb{x}_{2,s}^t e^{-2 \eta (A^{\top}_t \tb{x}_1^t)^s +\eta (A^{\top}_{t-1} \tb{x}_1^{t-1})^s}}\right)^{n}_{ j =1}.\tag{OMWU}
\end{align*}
Note that in (OMWU), $t$ and $t-1$ steps are used together to update the step at time $t+1$. 
We will use  $(\tb{x}_1^0, \tb{x}_2^0), (\tb{x}_1^{-1}, \tb{x}_2^{-1})$ to denote the initial conditions for (OMWU). 

Optimistic method was proposed in \citep{popov1980modification} as a variant of gradient descent ascent method in saddle-point optimization problem. The last iterate convergence property of Optimistic Gradient Descent-Ascent (OGDA) in unconstrained bilinear game with a time-independent payoff was proved in \citep{daskalakis2017training}. Recently, there are also works analyzing the regret behaviors of OGDA under a time varying setting \citep{anagnostides2023convergence}. However, the study of (OMWU) in the time-varying setting is still missing in the literature, and the current work partially fills that gap.

\paragraph{Extra-gradient MWU.} 

In Extra-MWU dynamics with a step size of $\eta$, each iteration consists of two steps. In the first step, a half step strategies vectors $(\tb{x}_1^{t+\frac{1}{2}},\tb{x}_2^{t+\frac{1}{2}})$ is calculated based on the payoff vectors in the t-th round as follows :
\begin{align*}
&\tb{x}_1^{t+\frac{1}{2}} = 
\left(\frac{\tb{x}_{1,i}^t e^{\eta(A_t \tb{x}_2^t)^i}} { \sum^m_{s=1}\tb{x}_{1,s}^t e^{\eta(A_t \tb{x}_2^t)^s}}\right)^{m}_{ i =1},\\
&\tb{x}_2^{t+\frac{1}{2}} = 
\left(\frac{\tb{x}_{2,j}^t e^{-\eta(A^{\top}_t \tb{x}_1^t)^j}} { \sum^n_{s=1}\tb{x}_{2,s}^t e^{-\eta(A^{\top}_t \tb{x}_1^t)^s}}\right)^{n}_{ j =1}. 
\end{align*}
The second step for calculating the strategies $(\tb{x}_1^{t+1},\tb{x}_2^{t+1})$ is as follows :
\begin{align*}\label{EMWU}
&\tb{x}_1^{t+1} = 
\left(\frac{\tb{x}_{1,i}^t e^{\eta(A_t \tb{x}_2^{t+\frac{1}{2}})^i}} { \sum^m_{s=1}\tb{x}_{1,s}^t e^{\eta(A_t \tb{x}_2^{t+\frac{1}{2}})^s}}\right)^{m}_{i =1},\\
&\tb{x}_2^{t+1} = 
\left(\frac{\tb{x}_{2,j}^t e^{-\eta(A^{\top}_t \tb{x}_1^{t+\frac{1}{2}})^j}} { \sum^n_{s=1}\tb{x}_{2,s}^t e^{-\eta(A_t^\top \tb{x}_1^{t+\frac{1}{2}})^s}}\right)^{n}_{ j =1}.\tag{Extra-MWU}
\end{align*}

Extra-gradient was firstly proposed in \citep{korpelevich1976extragradient} as a modification of the gradient method in saddle-point optimization problem. It is known that  Extra-gradient Descent-Ascent (Extra-GDA) method converge to the equilibrium in the time-independent bilinear zero-sum game with a linear convergence rate \citep{LiangS18}. Convergence of (Extra-GDA) on convex-concave game was analyzed in \citep{nemirovski2004prox,monteiro2010complexity}, and convergence guarantees for special non-convex-non-concave time-independent game of the more general Extra-gradient Mirror Descent was provided in \citep{mertikopoulos2018optimistic}.

%\begin{rem}
%In recent literature, there is a confused use of the terms optimistic and extra-gradient, despite their different origins and distinct update methods. For example,  the algorithms discussed in \citep{mertikopoulos2018optimistic} are called optimistic methods, while these algorithms require a two-step update in each iteration, thus aligning with the paradigm of Extra-gradient method. In fact, in the context of time-independent games, these two methods usually exhibit the same behaviors and can even be analyzed in a unified way \citep{mokhtari2020unified}.
%This also makes our results on the convergence behaviors of these two methods in the setting of periodic game interesting.
%\end{rem}


\subsection{Results from dynamical systems}

In this paper, we analyze the last-iterate behavior of learning algorithms in periodic games by modeling them as dynamical systems. The resulting systems possess two characteristics that make their analysis challenging: firstly, they are non-autonomous, i.e., the evolution of the system not only depends on its current state but also on the temporal variables; secondly, they are non-linear.  In this section we introduce the necessary backgrounds on this kind of dynamical systems. 

\begin{defn}[Periodic dynamical system]Let $\CX$ be a subset of $\BR^n$. A discrete,
$\CT$-\textit{periodic dynamical} system is a finite sequence ${f_0,...,f_{\CT-1}}$ of maps
where $f_i : \CX \to \CX$ for $i =0,...,\CT-1$. The sequence can be extended to
a periodic infinite by defining $f_i = f_{i \mod \CT}$ for $i \ge \CT$.
The \textit{trajectory} $\{x_n\}$ of a point $x$ is given by the n-fold composition of these p maps, i.e., $x_n = f_{n-1}\circ \cdots \circ f_1 \circ f_0 (x)$.  
\end{defn}

Periodic dynamical systems are non-autonomous. The dynamical behaviors exhibited by non-autonomous systems can be highly intricate, and typically only results pertaining to linear systems are available \citep{carvalho2015non}. However, the study of periodic dynamical systems can be simplified by analyzing an autonomous system derived from the underlying periodic system \citep{franke2003attractors,colonius2014dynamical}. For simplicity, we present a proposition concerning the convergence behaviors of a periodic system that is useful for our analysis.

\begin{prop}[\citep{franke2003attractors}]\label{attrat} Let $\Tilde{f}_i = f_{i+\CT-1} \circ ... \circ f_{i}$, for $ i \in [\CT]$. Then $\Tilde{f}_i$ is a time-independent dynamical system. If for all $x \in \CX$ and each $i\in [\CT]$, it holds that $\lim_{n \to \infty} \Tilde{f}_i^n(x) = x^*$ for some $x^* \in \CX$, then the periodic system defined by
$\{f_i\}^{\CT}_{i=1}$ will converge to $x^*$ for arbitrary initial points $x \in \CX$.
\end{prop}

%\yi{This is not the original statements in their paper, should we state their  original form ? It is a bit long.}

The proposition above demonstrates that in order to establish the convergence of a periodic system, it suffices to demonstrate the convergence of each corresponding autonomous systems $\Tilde{f}_i$.


In the following, we consider the second characteristic of the dynamical systems arising from our learning algorithms : non-linearity. In general, non-linear dynamical systems exhibit complex behaviors such as chaos \citep{hirsch2012differential}, thereby rendering the understanding of their global behavior challenging. Subsequently, we present results pertaining to the local behaviors of non-linear dynamical systems $\phi$ using the technique of \textit{linearization} \citep{galor2007discrete}.
 
\begin{defn}[Stable, Unstable, and Center eigenspaces.] Let $\phi : \BR^n \to \BR^n$ be a continuous differentiable
function, and $\bar{x}$ be a fixed point of $\phi$, i.e., $\phi(x)=x$. let $\mathcal{D} \phi(\bar{x})$ be the Jacobian matrix of $\phi$ at point $\bar{x}$.
 The stable eigenspace of $\bar{x}$ is defined as
    \begin{align*}
        E^s(\bar{x}) = \text{span}  \{ \text{Eigenvectors of} \ & \mathcal{D}(\bar{x}) \ \text{whose eigenvalues} \\
        & \text{have modules} < 1 \}.
    \end{align*}
Similarly, the unstable (rep. center ) eigenspace $E^u(\bar{x})$ (rep. $E^c(\bar{x})$) of $\bar{x}$ is the subspace spanned by eigenvectors of $\mathcal{D}(\bar{x})$ whose eigenvalues have modules $>1$ (rep. $=1$).
\end{defn}

\begin{prop}[ \citep{galor2007discrete}]\label{decomposition} Let $\phi : \BR^n \to \BR^n$ be a continuous differentiable function, and with the concepts defined as above, we have
\begin{align*}
    \dim E^s(\bar{x}) + \dim E^u(\bar{x}) + \dim E^c(\bar{x}) = n.
\end{align*}
\end{prop}

Proposition \ref{decomposition} implies that any point in $\BR^n$ can be decomposed to linear combination of the vectors belonging to the three eigenspaces defined above. These three eigenspaces provide a full characterization on the local behavior of $\phi$ near the fixed point $\bar{x}$ : if a point $x$ is close to $\bar{x}$, and lies in the stable space $E^s(\bar{x})$, it will converge to $\bar{x}$ after sufficient number of iterations of $\phi$. On the other hand, vectors in $E^u(\bar{x})$ or $E^c(\bar{x})$ will not converge to $\bar{x}$.






