\section{Experiment}\label{sec:experiment}

% \LL{This Section still needs some polishing. For instance, I would introduce the various systems we consider directly in the paragraph below. Also, currently, Section 6.1 is not very appealing, as it only shows a toy example and I do not see the convergence analysis the title of the subsection seems to refer to. In Section 6.2, I would try to emphasize the advantages compared to Monte Carlo simulations and existing approaches. Also, Figures 2 and 3 should be discussed in more detail as they are nice}

We demonstrate our proposed error bound approach via several numerical experiments. First, we illustrate a synthetic second-order error bound on a simple linear system. Then, we present our main applications of the first-order error bound for non-trivial systems including nonlinear, chaotic, and high dimensional SDEs. The details of these systems are provided in Appendix~\ref{appendix:system}.
% A heat PDE example is provided in the Appendix~\ref{appendix:1D-HEAT-sys} and~\ref{appendix:1D-HEAT-exp} to illustrate potential extensions to other linear PDEs. 
For all the experiments, fully connected neural networks are used for both $\hat{p}(x,t)$ and $\hat{e}_1(x,t)$. 
Throughout all experiments, we employ a fixed weighting scheme for training (see Appendix \ref{appendix:train_and_results}). 
More sophisticated weight-tuning strategies (e.g., \citet{basir2023adaptive} and references therein) have been shown to enhance PINN accuracy and could further improve the results reported here.
The code implemented in Python and Pytorch is available on GitHub \citep{PINN_Error:github}. All the experiments are conducted on a MacBook Pro with Apple M2 processor and 24GB RAM.

\begin{table*}[t]
    \centering
    \caption{First-order error bound results categorized into three groups: (1) Linear, where $p$ is obtained analytically, (2) Nonlinear, where `true' $p$ is obtained by Monte-Carlo (M.C.) since no analytical solutions exist, and (3) High Dimensional, where `true' $p$ is obtained by semi-analytical numerical integration. Here, time $p$ reports the computation time \textbf{in seconds} to obtain the `true' $p$ via M.C., time $\hat{p}$ and $\hat{e}_1$ are the training time for PINNs \textbf{in seconds}, $\alpha_1^{\max}:=\max_t \alpha_1(t)$, $\alpha_1$ reports the mean and standard deviation of $\alpha_1(t)$ over $t$.
     $\text{Gap}^{\min}:=\min_t((B_1(t)-e_1^*(t))/\max_x p(x,t))$ is the minimum gap (over time) between the error bound and maximum error normalized by the true solution, and
     $B_1^{\mathcal{N}}$ reports the average and standard deviation (over time) of the normalized error bound $B_1(t)/\max_x p(x,t)$.}
    \renewcommand{\arraystretch}{0.9} 
    \setlength{\tabcolsep}{4.0pt}
    \begin{tabular}{p{1.8cm}|lcccccccc}  
      \toprule
      \bfseries Category & \bfseries Experiment & \bfseries time $p$ & \bfseries time $\hat{p}$ & \bfseries time $\hat{e}_1$ & \bfseries $\alpha_1^{\max}$ & \bfseries $\alpha_1$ & \bfseries $\text{Gap}^{\min}$ & \bfseries $B_1^{\mathcal{N}}$ \\
      \midrule
      \multirow{1}{2cm}{\textbf{Linear}} 
      & 1D Linear & Analytical & 5 & 17 & 0.37 & 0.23 $\pm$ 0.04 & 7e-2 & 0.18 $\pm$ 6e-3 \\
      \midrule
      \multirow{3}{2cm}{\textbf{Nonlinear}}  
      & 1D Nonlinear & 37634 (M.C.) & 1031 & 1840 & 0.63 & 0.24 $\pm$ 0.16 & 2e-2 & 0.12 $\pm$ 6e-2 \\
      & 2D Inverted Pendulum & 45409 (M.C.). & 1054 & 4484 & 0.69 & 0.46 $\pm$ 0.14 & 2e-2 & 0.14 $\pm$ 7e-2 \\
      & 2D Duffing Oscillator & 39421 (M.C.) & 358 & 3727 & 0.42 & 0.32 $\pm$ 0.11 & 8e-2 & 0.21 $\pm$ 3e-2 \\
      \midrule
      \multirow{3}{2cm}{\textbf{High Dimensional}} 
      & 3D Time-varying OU & Semi-Analytical & 267 & 943 & 0.34 & 0.29 $\pm$ 0.03 & 2e-2 & 0.05 $\pm$ 1e-2 \\
      & 7D Time-varying OU & Semi-Analytical & 478 & 1475 & 0.49 & 0.43 $\pm$ 0.05 & 6e-2 & 0.18 $\pm$ 6e-2 \\
      & 10D Time-varying OU & Semi-Analytical & 803 & 5954 & 0.83 & 0.64 $\pm$ 0.16 & 5e-2 & 0.19 $\pm$ 8e-2 \\
      \bottomrule
    \end{tabular}
    \label{tab:main_result}
\end{table*}

\textbf{Second-order error bound illustration \quad}\label{sec:arb_tight_error_bound_example}
% \ck{adapted from the original experiment}
% To illustrate the second order error bound introduced in Section~\ref{sec:theory}, 
We consider a FP-PDE for a 1D linear SDE. This simple system has analytical PDF $p(x,t)$, which allows us to synthesize $\hat{e}_2(x,t)$ and validate second-order error bound $B_2(t)$.
Specifically, we first train two PINNs in sequence: one $\hat{p}(x,t)$ and the other for $\hat{e}_1(x,t)$. Due to the practical challenge discussed in Section~\ref{sec:theory}, we synthesize $\hat{e}_2(x,t)$ from the analytical solution and the trained PINNs, i.e., $\hat{e}_2(x,t)=p(x,t)-\hat{p}(x,t)-\hat{e}_1(x,t)+\delta(x,t)$, where $\delta(x,t)$ is a chosen sinusoidal perturbation such that Conditions~\ref{eq:alpha_conditions} are satisfied.
    With the learned $\hat{e}_1(x,t)$ and synthesized $\hat{e}_2(x,t)$, we construct  second- and first-order error bound $B_2(t)$  and $B_1(t)$ from Eqs.~\eqref{eq: temporal error bound} and \eqref{eq:tight_error_bound}, respectively. Fig.~\ref{fig:sub1a} validates that both $B_2(t)$ and $B_1(t)$ upper-bound the worst-case error for all time. Furthermore, it shows that $B_2(t)$ is tighter than $B_1(t)$, and the relative tightness $B_1(t)/B_2(t)$ is at most $1.63<2$ as predicted by Corollary~\ref{corollary:special_error_bound}. Fig.~\ref{fig:sub1b} visualizes the satisfaction of the sufficient conditions on $\alpha_1(t)$ and $\alpha_2(t)$ in Eqs.~\eqref{eq:alpha_conditions} for all $t \in T'$. These results validate the soundness of our 
second-order error bound under the proposed conditions, and its tightness relative to the first-order bound.
% derivations for these error bounds.

\textbf{First-order error bound application \quad}\label{sec:exp_tight_error_bound}
% \ck{adapted from the original experiment}
We apply the approach in Section~\ref{sec:tight_error_bound} to construct error bound $B_1(t)$ for several FP-PDEs that do not have closed-form solutions. First, we consider FP-PDEs associated with nonlinear SDEs (see rows labeled as Nonlinear in Table~\ref{tab:main_result}). We note that 2D Duffing Oscillator is from \citet{anderson2024fisher} and exhibits chaotic behaviors. Due to complexity of these systems, highly detailed Monte Carlo (M.C.) simulations with extremely small time steps and very large numbers of samples were necessary to generate high-fidelity `ground-truth' PDF distributions at specific discrete time instances (see Appendix~\ref{appendix:system}). 
The final set of experiments consider high-dimensional (up to 10-D) time-varying Ornstein-Uhlenbeck (OU) processes (see rows labeled as High Dimensional in Table~\ref{tab:main_result}). 
We note that running M.C. to obtain `ground-truth' PDF for fully nonlinear and high-dimensional systems is not tractable. Hence, we choose these time-varying OU because they are non-trivial (i.e., no closed-form solutions) but allow us to efficiently estimate accurate $p(x,t)$ via semi-analytical method that numerically integrates the Gaussian distributions by exploiting the time-varying linear dynamics \citep{sarkka2019applied}. 
Thus, unlike our PINN-based method or standard M.C. simulations, this semi-analytical approach cannot handle general nonlinear dynamics.

\begin{figure*}[t!] % Use figure* for wide figures spanning both columns
    \centering
    \begin{subfigure}{0.32\textwidth} % First row, first column
        \includegraphics[width=\textwidth]{figs/1dnl_phatsurface.pdf}
        \caption{}
        \label{fig:1dnl_paper_p}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth} % First row, second column
        \includegraphics[width=\textwidth]{figs/1dnl_e1hatsurface.pdf}
        \caption{}
        \label{fig:1dnl_paper_e}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth} % First row, first column
        \includegraphics[width=\textwidth]{figs/phat_eS.pdf} % Replace with your image
        \caption{}
        \label{fig:1dnl_paper}
    \end{subfigure}
    
    \begin{subfigure}{\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figs/2dduff_errorbound.pdf}
        \caption{}
        \label{fig:2dduff_paper}
    \end{subfigure}
    
    \begin{subfigure}{0.24\textwidth}
        \includegraphics[width=\textwidth]{figs/2dnl_p.pdf} % Replace with your image
        \caption{}
        \label{fig:2dip_paper_1}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.24\textwidth} % First row, second column
        \includegraphics[width=\textwidth]{figs/2dnl_e.pdf} % Replace with your image
        \caption{}
        \label{fig:2dip_paper_2}
    \end{subfigure}
    \hfill
        \begin{subfigure}{0.24\textwidth}
        \includegraphics[width=\textwidth]{figs/3dtv/e_t1.0.pdf}
        \caption{}
        \label{fig:3d_e1t1.0}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.24\textwidth}
        \includegraphics[width=\textwidth]{figs/3dtv/ehat_t1.0.pdf}
        \caption{}
        \label{fig:3d_e1hatt1.0}
    \end{subfigure}
    \caption{First-order error bound results. (a)-(c) 1D Nonlinear PDF $p$ vs PINN $\hat{p}$, error $e_1$ vs error PINN $\hat{e}_1$, and error bound $B_1$ compared to the classical Gaussian mixture method $\hat{p}_{GM}$, illustrated at three time points. 
    (d) 2D Duffing Oscillator true error $|e_1|$, error PINN $|\hat{e}_1|$, and error bound $B_1 \geq |e_1|$ over time.
    (e)-(f) 2D Inverted Pendulum PDF $p$, PINN $\hat{p}$, true error $|e_1|$, error PINN $|\hat{e}_1|$, and error bound $B_1 \geq |e_1|$ at $t=3$. 
    (g)-(h) 3D Time-varying OU error $e_1$ and error PINN $\hat{e}_1$ at $t=1$.
    }
    \label{fig:representative_results}
\end{figure*}

\begin{figure}[htbp!]
    \centering
    \begin{subfigure}[b]{0.23\textwidth} % First row, first column
        \includegraphics[width=\textwidth]{figs/1dheat_phatsurface.pdf}
        \caption{$\hat{u}$ vs $u$}
    \end{subfigure}
    \hfill
    \begin{subfigure}[b]{0.23\textwidth} % First row, second column
        \includegraphics[width=\textwidth]{figs/1dheat_e1hatsurface.pdf}
        \caption{$\hat{e}_1$ vs $e_1$}
    \end{subfigure}
    \caption{1D Heat PINN solution $\hat{u}(x,t)$ and error $\hat{e}_1(x,t)$ v.s. true solution $u(x,t)$ and error $e_1(x,t)$.}
    \label{fig:1dheat_surfaces_main}
\end{figure}

Table~\ref{tab:main_result} summarizes the results on all systems. 
The metrics include computation time for $p$ and training times for $\hat{p}(x,t)$ and $\hat{e}_1$ PINNs in seconds. 
% , because it is the best approach, though expensive, to achieve extremely accurate PDF without sophisticated parameter/weight tuning. 
Also, we show training feasibility of $\hat{e}_1(x,t)$ by reporting $\alpha_1(t)$ and its maximum value over time ($\alpha^{\max}$).  To show soundness of error bound $B_1$, we calculate the minimum gap $\text{Gap}^{\min}$ between $B_1$ and the true error, i.e., positive $\text{Gap}^{\min}$ indicate correctness of $B_1$. To assess the tightness of $B_1(t)$, we report the statistics of its magnitude normalized by $\max_x p(x,t)$, denoted by $B_1^\mathcal{N}$.
Overall, the results show: 
(i) \textit{soundness \& feasibility:} $B_1$ correctly bounds the approximation error ($\text{Gap}^{\min} > 0$) and the learning condition is satisfied ($\alpha_1^{\max}<1$) though it becomes increasingly challenging to meet as dimensionality grows,
(ii) \textit{efficiency:} the time comparisons between $p$ vs $\hat{p}$ illustrate significant speedup (two orders of magnitude: $38\times$ to $65\times$) in obtaining accurate PDF with PINNs for systems that do not have analytical solutions (nonlinear systems),
and
(iii) \textit{scalability:} our PINN method is able to scale to 10-dimensional systems with fairly tight error bounds across all cases (on average 5\%-21\% with respect to the true solution). 

We plot $\alpha_1(t)$ distributions (over time) vs training loss of $\hat{e}_1$ in Fig.~\ref{fig:meta} to show that the condition in Corollary~\ref{corollary:special_error_bound} is satisfied for all systems as training loss decreases.
We visualize the error bounds for some representative cases in Fig.~\ref{fig:representative_results}. 
Specifically, Fig.~\ref{fig:1dnl_paper_p} plots the PDF $p(x,t)$ and its PINN approximation $\hat{p}(x,t)$ of the 1D Nonlinear experiment. Note that $\hat{p}(x,t)$ is a continuous surface over time and space, while $p(x,t)$ is illustrated by curves at discrete time instances according to those used in Monte-Carlo simulation.
Fig.~\ref{fig:1dnl_paper_e} shows the `true' error $e_1(x,t)$ at discrete time instances and its PINN approximation $\hat{e}_1(x,t)$ as a continuous surface of the 1D Nonlinear experiment.
Observe that both $\hat{p}$ and $\hat{e}$ closely approximate $p$ and $e$, respectively, over all space and time, respectively.
In addition, Fig.~\ref{fig:1dnl_paper} shows the PINN‐learned density $\hat p$ alongside the GMM approximation $\tilde p_{\rm GM}$ (125 Gaussian mixtures integrated over $\Delta t=0.001$ time step). 
We include the GMM---a common tool for uncertainty propagation in nonlinear dynamics \citep{archambeau2007gaussian,terejanu2008uncertainty,vittaldev2016spacecraft,10886416}---as a classical alternative.
% In Fig.~\ref{fig:1dnl_paper}, we compare $\hat{p}$ learned by PINN and $\tilde{p}_{GM}$ integrated over 0.001 time steps with 125 Gaussian mixtures. 
% We used a Gaussian mixture model (GMM) in Figure 3a as a complementary comparison since GMMs are a common tool for uncertainty propagation in nonlinear dynamics (e.g., Terejanu et al., 2008 and Vittaldev et al. 2016); note that as compared to our proposed method, GMMs typically do not provide quantified error bounds.
Compared to the PINN $\hat{p}$, observe that $\tilde{p}_{GM}$ gradually deviates from $p$ as time increases.
More importantly, the true PDF $p$ lies within the first-order error bound $B_1$ (illustrated by green regions) of approximate PDF $\hat{p}$, while $\tilde{p}_{GM}$ does not provide rigorous error bound.
For the 2D Duffing Oscillator experiment, Fig.~\ref{fig:2dduff_paper} plots the time evolution of the true error, PINN error approximation, and the first-order error bound. Observe that the magnitude of the error bound does not necessarily grow over time, suggesting that the error bound may remain tight even over extended horizons.
Figs.~\ref{fig:2dip_paper_1} and~\ref{fig:2dip_paper_2} visualize PINNs results and the constructed $B_1$ of the 2D Inverted Pendulum experiment at a given time ($t=3$). 
Observe that both $\hat{p}$ and $\hat{e}_1$ closely approximate the unknown complicated distributions of $p$ and $e_1$, respectively. 
For the 3D Time-varying OU, Figs.~\ref{fig:3d_e1t1.0} and~\ref{fig:3d_e1hatt1.0} visualize the true error $e_1$ and PINN error $\hat{e}_1$ at $t=1$, showing good approximation (i.e., $\alpha_1 < 1)$ for constructing the first-order error bound $B_1$.
Lastly, PINN predictions $\hat u$ and $\hat e_1$ for the 1D Heat PDE (Fig.~\ref{fig:1dheat_surfaces_main}) closely match the true solution $u$ and error $e_1$ across space and time, respectively, demonstrating straightforward extension to other linear PDEs (Remark~\ref{remark:3}).
See Appendix~\ref{appendix:train_and_results} for complete training details and additional results of the conducted experiments.

