Objective \[\mu^\ast = \argmin_{\mu^\ast\in \Pi_{\max}} \sumH \Psi_h(p_h^\ast\cdot \mu_h^\ast)\,.\]
\begin{proof}
    For $h=H$, 
    \begin{align*}
        \sumH \Psi_h(p_h^\ast\cdot \mu) &= \sum_{h=1}^{H-1} \Psi_h(p_h^\ast\cdot \mu_h) + \sum_{x_H,a_H\in\gX_H\times\gA} p^\ast (x_H) \mu(x_H,a_H) \left[\log p^\ast (x_H) \mu(x_H) + \log \mu(a_H|X_H) \right]\\
        & = \sum_{h=1}^{H-1} \Psi_h(p_h^\ast\cdot \mu_h) + \sum_{x_H\in\gX_H} p^\ast (x_H) \mu(x_H) \left[\log p^\ast (x_H) \mu(x_H)\right] \\
        &\qquad + \sum_{x_H\in\gX_H} p^\ast (x_H) \mu(x_H) \sum_{a_H\in\gA} \mu(a_H|x_H)\log \mu(a_H|x_H)\,.
    \end{align*}
    Take the derivative for $\mu(\cdot|x_H)$, we obtain $\mu^\ast (\cdot|x_H)\sim Uni(\gA)\,.$
    For $h=H-1$, denote $\frac{\partial }{\partial \mu_{H-1}} \coloneqq\frac{\partial }{\partial \mu(|x_{H-1})} $
    \begin{align*}
        &\qquad \frac{\partial }{\partial \mu_{H-1}}\sumH \Psi_h(p_h^\ast\cdot \mu) \\
        &= \frac{\partial }{\partial \mu_{H-1}}\sum_{x_{H-1}\in \gX_{H-1}}p^\ast(x_{H-1})\mu(x_{H-1})\sum_{a\in\gA}\mu(a|x_{H-1})\log \mu (a|x_{H-1}) \\
        &\qquad + \frac{\partial }{\partial \mu_{H-1}}\sum_{x_H\in\gX_H} p^\ast (x_H) \mu(x_H) \log \left(p^\ast (x_H) \mu(x_H) A\right)\\
        &= \frac{\partial }{\partial \mu_{H-1}}\sum_{x_{H-1}\in \gX_{H-1}}p^\ast(x_{H-1})\mu(x_{H-1})\sum_{a\in\gA}\mu(a|x_{H-1})\log \mu (a|x_{H-1}) \\
        &\qquad + \frac{\partial }{\partial \mu_{H-1}}\sum_{x_{H-1}\in\gX_{H-1}}\sum_{a\in\gA}\sum_{x_H\in C(x_{H-1},a)} p^\ast (x_H) \mu(x_{H-1})\mu(a|x_{H-1}) \left[\log \left(p^\ast (x_H) \mu(x_{H-1}) A\right)+\log \mu(a|x_{H-1})\right]\\
    \end{align*}
    The zero gradient gives to 
    \begin{align*}
        \left(p^\ast (x_{H-1}) + \sum_{x_H\in C(x_{H-1},a)}p^\ast (x_{H})\right) (1+\log \mu (a|x)) + \sum_{x_H\in C(x_{H-1},a)} p^\ast (x_H) \log \left(p^\ast (x_H) \mu(x_{H-1}) A\right) = 0
    \end{align*}
    which gives 
    \begin{align*}
        \mu (a|x) &\propto \exp \left(-\frac{p^\ast (x_{H-1}) + \sum_{x_H\in C(x_{H-1},a)}p^\ast (x_{H})+\sum_{x_H\in C(x_{H-1},a)} p^\ast (x_H) \log \left(p^\ast (x_H) \mu(x_{H-1}) A\right)}{p^\ast (x_{H-1}) + \sum_{x_H\in C(x_{H-1},a)}p^\ast (x_{H})}\right)\\
        &\propto \exp \left(-\frac{\sum_{x_H\in C(x_{H-1},a)} p^\ast (x_H) \log \left(p^\ast (x_H) \mu(x_{H-1}) A\right)}{p^\ast (x_{H-1}) + \sum_{x_H\in C(x_{H-1},a)}p^\ast (x_{H})}\right)\\
    \end{align*}
    Let \begin{align*}
        Z_{H-1}(x_{H-1}) = \sum_{a\in\gA} \exp \left(-\frac{\sum_{x_H\in C(x_{H-1},a)} p^\ast (x_H) \log \left(p^\ast (x_H) \mu(x_{H-1}) A\right)}{p^\ast (x_{H-1}) + \sum_{x_H\in C(x_{H-1},a)}p^\ast (x_{H})}\right)
    \end{align*}
    The original objective is 
    \begin{align*}
    \sum_{h=1}^{H-2} \Psi_h(p_h^\ast\cdot \mu_h) + \sum_{x_{H-1},a} \left(p^\ast (x_{H-1}) + \sum_{x_H\in C(x_{H-1},a)}p^\ast (x_{H})\right) \mu(x_{H-1},a)\log \frac{1}{Z_{H-1}(x_{H-1})}
    \end{align*}
\end{proof}