
@inproceedings{hamid_marginalising_2022,
	title = {Marginalising over {Stationary} {Kernels} with {Bayesian} {Quadrature}},
	url = {https://proceedings.mlr.press/v151/hamid22a.html},
	abstract = {Marginalising over families of Gaussian Process kernels produces flexible model classes with well-calibrated uncertainty estimates. Existing approaches require likelihood evaluations of many kernels, rendering them prohibitively expensive for larger datasets. We propose a Bayesian Quadrature scheme to make this marginalisation more efficient and thereby more practical. Through use of maximum mean discrepancies between distributions, we define a kernel over kernels that captures invariances between Spectral Mixture (SM) Kernels. Kernel samples are selected by generalising an information-theoretic acquisition function for warped Bayesian Quadrature. We show that our framework achieves more accurate predictions with better calibrated uncertainty than state-of-the-art baselines, especially when given limited (wall-clock) time budgets.},
	language = {en},
	urldate = {2024-02-10},
	booktitle = {Proceedings of {The} 25th {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Hamid, Saad and Schulze, Sebastian and Osborne, Michael A. and Roberts, Stephen},
	month = may,
	year = {2022},
	note = {ISSN: 2640-3498},
	pages = {9776--9792},
}

@article{tan_variational_2016,
	title = {Variational inference for sparse spectrum {Gaussian} process regression},
	volume = {26},
	issn = {1573-1375},
	url = {https://doi.org/10.1007/s11222-015-9600-7},
	doi = {10.1007/s11222-015-9600-7},
	abstract = {We develop a fast variational approximation scheme for Gaussian process (GP) regression, where the spectrum of the covariance function is subjected to a sparse approximation. Our approach enables uncertainty in covariance function hyperparameters to be treated without using Monte Carlo methods and is robust to overfitting. Our article makes three contributions. First, we present a variational Bayes algorithm for fitting sparse spectrum GP regression models that uses nonconjugate variational message passing to derive fast and efficient updates. Second, we propose a novel adaptive neighbourhood technique for obtaining predictive inference that is effective in dealing with nonstationarity. Regression is performed locally at each point to be predicted and the neighbourhood is determined using a measure defined based on lengthscales estimated from an initial fit. Weighting dimensions according to lengthscales, this downweights variables of little relevance, leading to automatic variable selection and improved prediction. Third, we introduce a technique for accelerating convergence in nonconjugate variational message passing by adapting step sizes in the direction of the natural gradient of the lower bound. Our adaptive strategy can be easily implemented and empirical results indicate significant speedups.},
	language = {en},
	number = {6},
	urldate = {2024-02-10},
	journal = {Statistics and Computing},
	author = {Tan, Linda S. L. and Ong, Victor M. H. and Nott, David J. and Jasra, Ajay},
	month = nov,
	year = {2016},
	keywords = {Adaptive neighbourhood, Bound optimization, Local Gaussian process, Nonconjugate variational message passing, Sparse approximation},
	pages = {1243--1261},
}

@inproceedings{hensman_scalable_2015,
	title = {Scalable {Variational} {Gaussian} {Process} {Classification}},
	url = {https://proceedings.mlr.press/v38/hensman15.html},
	abstract = {Gaussian process classification is a popular method with a number of appealing properties. We show how to scale the model within a variational inducing point framework, out-performing the state of the art on benchmark datasets. Importantly, the variational formulation an be exploited to allow classification in problems with millions of data points, as we demonstrate in experiments.},
	language = {en},
	urldate = {2024-02-10},
	booktitle = {Proceedings of the {Eighteenth} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Hensman, James and Matthews, Alexander and Ghahramani, Zoubin},
	month = feb,
	year = {2015},
	note = {ISSN: 1938-7228},
	pages = {351--360},
}

@inproceedings{liu_stein_2016,
	title = {Stein {Variational} {Gradient} {Descent}: {A} {General} {Purpose} {Bayesian} {Inference} {Algorithm}},
	volume = {29},
	shorttitle = {Stein {Variational} {Gradient} {Descent}},
	url = {https://proceedings.neurips.cc/paper_files/paper/2016/hash/b3ba8f1bee1238a2f37603d90b58898d-Abstract.html},
	abstract = {We propose a general purpose variational inference algorithm that forms a natural counterpart of gradient descent for optimization. Our method iteratively transports a set of particles to match the target distribution, by applying a form of functional gradient descent that minimizes the KL divergence. Empirical studies are performed on various real world models and datasets, on which our method is competitive with existing state-of-the-art methods. The derivation of our method is based on a new theoretical result that connects the derivative of KL divergence under smooth transforms with Stein’s identity and a recently proposed kernelized Stein discrepancy, which is of independent interest.},
	urldate = {2024-02-10},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Liu, Qiang and Wang, Dilin},
	year = {2016},
}

@inproceedings{chen_projected_2020,
	title = {Projected {Stein} {Variational} {Gradient} {Descent}},
	volume = {33},
	url = {https://proceedings.neurips.cc/paper_files/paper/2020/hash/14faf969228fc18fcd4fcf59437b0c97-Abstract.html},
	abstract = {The curse of dimensionality is a longstanding challenge in Bayesian
inference in high dimensions. In this work, we propose a \{\vphantom{\}}projected
  Stein variational gradient descent\vphantom{\{}\} (pSVGD) method to overcome this
challenge by exploiting the fundamental property of intrinsic low
dimensionality of the data informed subspace stemming from
ill-posedness of such problems. We adaptively construct the subspace
using a gradient information matrix of the log-likelihood, and apply
pSVGD to the much lower-dimensional coefficients of the parameter
projection. The method is demonstrated to be more accurate and
efficient than SVGD. It is also shown to be more scalable with respect
to the number of parameters, samples, data points, and processor cores
via experiments with parameters dimensions ranging from the
hundreds to the tens of thousands.},
	urldate = {2024-02-10},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Chen, Peng and Ghattas, Omar},
	year = {2020},
	pages = {1947--1958},
}

@inproceedings{ma_functional_2021,
	title = {Functional {Variational} {Inference} based on {Stochastic} {Process} {Generators}},
	volume = {34},
	url = {https://proceedings.neurips.cc/paper/2021/hash/b613e70fd9f59310cf0a8d33de3f2800-Abstract.html},
	abstract = {Bayesian inference in the space of functions has been an important topic for Bayesian modeling in the past. In this paper, we propose a new solution to this problem called Functional Variational Inference (FVI). In FVI, we minimize a divergence in function space between the variational distribution and the posterior process. This is done by using as functional variational family a new class of flexible distributions called Stochastic Process Generators (SPGs), which are cleverly designed so that the functional ELBO can be estimated efficiently using analytic solutions and mini-batch sampling. FVI can be applied to stochastic process priors when random function samples from those priors are available. Our experiments show that FVI consistently outperforms weight-space and function space VI methods on several tasks, which validates the effectiveness of our approach.},
	urldate = {2024-02-09},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Ma, Chao and Hernández-Lobato, José Miguel},
	year = {2021},
	pages = {21795--21807},
}

@article{miller_bayesian_2022,
	title = {Bayesian spatial modeling using random {Fourier} frequencies},
	volume = {48},
	issn = {2211-6753},
	url = {https://www.sciencedirect.com/science/article/pii/S2211675322000069},
	doi = {10.1016/j.spasta.2022.100598},
	abstract = {Spectral methods are important for both theory and computation in spatial data analysis. When data lie on a grid, spectral approaches can take advantage of the discrete Fourier transform for fast computation. If data are not on a grid, then low-rank processes with Fourier basis functions may be sufficient approximations. However, deciding which basis functions to use is difficult and can depend on unknown parameters. Here, we introduce Bayesian Random Fourier Frequencies (BRFF), a fully Bayesian extension of the random Fourier features approach. BRFF treats the spectral frequencies as random parameters, which unlike fixed frequency approximations allows the frequencies to be data-adaptive and averages over uncertainty in frequency selection. We apply this method to non-gridded continuous, binary, and count data. We compare BRFF using simulated and observed data to another popular low-rank method, the predictive processes (PP) model. BRFF is faster than PP, and outperforms or matches the predictive performance of the PP model in settings with high numbers of observations.},
	urldate = {2024-02-08},
	journal = {Spatial Statistics},
	author = {Miller, Matthew J. and Reich, Brian J.},
	month = apr,
	year = {2022},
	keywords = {Bayesian statistics, Hamiltonian Monte Carlo, Low-rank approximations, Spectral methods},
	pages = {100598},
}

@misc{dangelo_annealed_2021,
	title = {Annealed {Stein} {Variational} {Gradient} {Descent}},
	url = {http://arxiv.org/abs/2101.09815},
	doi = {10.48550/arXiv.2101.09815},
	abstract = {Particle based optimization algorithms have recently been developed as sampling methods that iteratively update a set of particles to approximate a target distribution. In particular Stein variational gradient descent has gained attention in the approximate inference literature for its flexibility and accuracy. We empirically explore the ability of this method to sample from multi-modal distributions and focus on two important issues: (i) the inability of the particles to escape from local modes and (ii) the inefficacy in reproducing the density of the different regions. We propose an annealing schedule to solve these issues and show, through various experiments, how this simple solution leads to significant improvements in mode coverage, without invalidating any theoretical properties of the original algorithm.},
	urldate = {2024-02-04},
	publisher = {arXiv},
	author = {D'Angelo, Francesco and Fortuin, Vincent},
	month = mar,
	year = {2021},
	note = {arXiv:2101.09815 [cs]},
	keywords = {Computer Science - Machine Learning},
}

@misc{han_stein_2018,
	title = {Stein {Variational} {Gradient} {Descent} {Without} {Gradient}},
	url = {http://arxiv.org/abs/1806.02775},
	doi = {10.48550/arXiv.1806.02775},
	abstract = {Stein variational gradient decent (SVGD) has been shown to be a powerful approximate inference algorithm for complex distributions. However, the standard SVGD requires calculating the gradient of the target density and cannot be applied when the gradient is unavailable. In this work, we develop a gradient-free variant of SVGD (GF-SVGD), which replaces the true gradient with a surrogate gradient, and corrects the induced bias by re-weighting the gradients in a proper form. We show that our GF-SVGD can be viewed as the standard SVGD with a special choice of kernel, and hence directly inherits the theoretical properties of SVGD. We shed insights on the empirical choice of the surrogate gradient and propose an annealed GF-SVGD that leverages the idea of simulated annealing to improve the performance on high dimensional complex distributions. Empirical studies show that our method consistently outperforms a number of recent advanced gradient-free MCMC methods.},
	urldate = {2024-02-04},
	publisher = {arXiv},
	author = {Han, Jun and Liu, Qiang},
	month = jun,
	year = {2018},
	note = {arXiv:1806.02775 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@misc{chang_kernel_2020,
	title = {Kernel {Stein} {Generative} {Modeling}},
	url = {http://arxiv.org/abs/2007.03074},
	doi = {10.48550/arXiv.2007.03074},
	abstract = {We are interested in gradient-based Explicit Generative Modeling where samples can be derived from iterative gradient updates based on an estimate of the score function of the data distribution. Recent advances in Stochastic Gradient Langevin Dynamics (SGLD) demonstrates impressive results with energy-based models on high-dimensional and complex data distributions. Stein Variational Gradient Descent (SVGD) is a deterministic sampling algorithm that iteratively transports a set of particles to approximate a given distribution, based on functional gradient descent that decreases the KL divergence. SVGD has promising results on several Bayesian inference applications. However, applying SVGD on high dimensional problems is still under-explored. The goal of this work is to study high dimensional inference with SVGD. We first identify key challenges in practical kernel SVGD inference in high-dimension. We propose noise conditional kernel SVGD (NCK-SVGD), that works in tandem with the recently introduced Noise Conditional Score Network estimator. NCK is crucial for successful inference with SVGD in high dimension, as it adapts the kernel to the noise level of the score estimate. As we anneal the noise, NCK-SVGD targets the real data distribution. We then extend the annealed SVGD with an entropic regularization. We show that this offers a flexible control between sample quality and diversity, and verify it empirically by precision and recall evaluations. The NCK-SVGD produces samples comparable to GANs and annealed SGLD on computer vision benchmarks, including MNIST and CIFAR-10.},
	urldate = {2024-02-01},
	publisher = {arXiv},
	author = {Chang, Wei-Cheng and Li, Chun-Liang and Mroueh, Youssef and Yang, Yiming},
	month = jul,
	year = {2020},
	note = {arXiv:2007.03074 [cs, stat]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning},
}

@inproceedings{jung_efficient_2022,
	title = {Efficient {Approximate} {Inference} for {Stationary} {Kernel} on {Frequency} {Domain}},
	url = {https://proceedings.mlr.press/v162/jung22b.html},
	abstract = {Based on the Fourier duality between a stationary kernel and its spectral density, modeling the spectral density using a Gaussian mixture density enables one to construct a flexible kernel, known as a Spectral Mixture kernel, that can model any stationary kernel. However, despite its expressive power, training this kernel is typically difficult because scalability and overfitting issues often arise due to a large number of training parameters. To resolve these issues, we propose an approximate inference method for estimating the Spectral mixture kernel hyperparameters. Specifically, we approximate this kernel by using the finite random spectral points based on Random Fourier Feature and optimize the parameters for the distribution of spectral points by sampling-based variational inference. To improve this inference procedure, we analyze the training loss and propose two special methods: a sampling method of spectral points to reduce the error of the approximate kernel in training, and an approximate natural gradient to accelerate the convergence of parameter inference.},
	language = {en},
	urldate = {2024-01-29},
	booktitle = {Proceedings of the 39th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Jung, Yohan and Song, Kyungwoo and Park, Jinkyoo},
	month = jun,
	year = {2022},
	note = {ISSN: 2640-3498},
	pages = {10502--10538},
}

@inproceedings{rossi_sparse_2021,
	title = {Sparse {Gaussian} {Processes} {Revisited}: {Bayesian} {Approaches} to {Inducing}-{Variable} {Approximations}},
	shorttitle = {Sparse {Gaussian} {Processes} {Revisited}},
	url = {https://proceedings.mlr.press/v130/rossi21a.html},
	abstract = {Variational inference techniques based on inducing variables provide an elegant framework for scalable posterior estimation in Gaussian process (GP) models. Besides enabling scalability, one of their main advantages over sparse approximations using direct marginal likelihood maximization is that they provide a robust alternative for point estimation of the inducing inputs, i.e. the location of the inducing variables. In this work we challenge the common wisdom that optimizing the inducing inputs in the variational framework yields optimal performance. We show that, by revisiting old model approximations such as the fully-independent training conditionals endowed with powerful sampling-based inference methods, treating both inducing locations and GP hyper-parameters in a Bayesian way can improve performance significantly. Based on stochastic gradient Hamiltonian Monte Carlo, we develop a fully Bayesian approach to scalable GP and deep GP models, and demonstrate its state-of-the-art performance through an extensive experimental campaign across several regression and classification problems.},
	language = {en},
	urldate = {2024-01-29},
	booktitle = {Proceedings of {The} 24th {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Rossi, Simone and Heinonen, Markus and Bonilla, Edwin and Shen, Zheyang and Filippone, Maurizio},
	month = mar,
	year = {2021},
	note = {ISSN: 2640-3498},
	pages = {1837--1845},
}

@misc{remes_non-stationary_2017,
	title = {Non-{Stationary} {Spectral} {Kernels}},
	url = {http://arxiv.org/abs/1705.08736},
	doi = {10.48550/arXiv.1705.08736},
	abstract = {We propose non-stationary spectral kernels for Gaussian process regression. We propose to model the spectral density of a non-stationary kernel function as a mixture of input-dependent Gaussian process frequency density surfaces. We solve the generalised Fourier transform with such a model, and present a family of non-stationary and non-monotonic kernels that can learn input-dependent and potentially long-range, non-monotonic covariances between inputs. We derive efficient inference using model whitening and marginalized posterior, and show with case studies that these kernels are necessary when modelling even rather simple time series, image or geospatial data with non-stationary characteristics.},
	urldate = {2024-01-29},
	publisher = {arXiv},
	author = {Remes, Sami and Heinonen, Markus and Kaski, Samuel},
	month = may,
	year = {2017},
	note = {arXiv:1705.08736 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@incollection{benton_function-space_2019,
	address = {Red Hook, NY, USA},
	title = {Function-space distributions over kernels},
	abstract = {Gaussian processes are flexible function approximators, with inductive biases controlled by a covariance kernel. Learning the kernel is the key to representation learning and strong predictive performance. In this paper, we develop functional kernel learning (FKL) to directly infer functional posteriors over kernels. In particular, we place a transformed Gaussian process over a spectral density, to induce a non-parametric distribution over kernel functions. The resulting approach enables learning of rich representations, with support for any stationary kernel, uncertainty over the values of the kernel, and an interpretable specification of a prior directly over kernels, without requiring sophisticated initialization or manual intervention. We perform inference through elliptical slice sampling, which is especially well suited to marginalizing posteriors with the strongly correlated priors typical to function space modeling. We develop our approach for non-uniform, large-scale, multi-task, and multidimensional data, and show promising performance in a wide range of settings, including interpolation, extrapolation, and kernel recovery experiments.},
	number = {1340},
	urldate = {2024-01-26},
	booktitle = {Proceedings of the 33rd {International} {Conference} on {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates Inc.},
	author = {Benton, Gregory W. and Maddox, Wesley J. and Salkey, Jayson P. and Albinati, Júlio and Wilson, Andrew Gordon},
	month = dec,
	year = {2019},
	pages = {14965--14976},
}

@misc{jakkala_deep_2021,
	title = {Deep {Gaussian} {Processes}: {A} {Survey}},
	shorttitle = {Deep {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/2106.12135},
	doi = {10.48550/arXiv.2106.12135},
	abstract = {Gaussian processes are one of the dominant approaches in Bayesian learning. Although the approach has been applied to numerous problems with great success, it has a few fundamental limitations. Multiple methods in literature have addressed these limitations. However, there has not been a comprehensive survey of the topics as of yet. Most existing surveys focus on only one particular variant of Gaussian processes and their derivatives. This survey details the core motivations for using Gaussian processes, their mathematical formulations, limitations, and research themes that have flourished over the years to address said limitations. Furthermore, one particular research area is Deep Gaussian Processes (DGPs), it has improved substantially in the past decade. The significant publications that advanced the forefront of this research area are outlined in their survey. Finally, a brief discussion on open problems and research directions for future work is presented at the end.},
	urldate = {2024-01-24},
	publisher = {arXiv},
	author = {Jakkala, Kalvik},
	month = jun,
	year = {2021},
	note = {arXiv:2106.12135 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@inproceedings{salimbeni_doubly_2017,
	address = {Red Hook, NY, USA},
	series = {{NIPS}'17},
	title = {Doubly stochastic variational inference for deep {Gaussian} processes},
	isbn = {978-1-5108-6096-4},
	abstract = {Gaussian processes (GPs) are a good choice for function approximation as they are flexible, robust to overfitting, and provide well-calibrated predictive uncertainty. Deep Gaussian processes (DGPs) are multi-layer generalizations of GPs, but inference in these models has proved challenging. Existing approaches to inference in DGP models assume approximate posteriors that force independence between the layers, and do not work well in practice. We present a doubly stochastic variational inference algorithm that does not force independence between layers. With our method of inference we demonstrate that a DGP model can be used effectively on data ranging in size from hundreds to a billion points. We provide strong empirical evidence that our inference scheme for DGPs works well in practice in both classification and regression.},
	urldate = {2024-01-24},
	booktitle = {Proceedings of the 31st {International} {Conference} on {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates Inc.},
	author = {Salimbeni, Hugh and Deisenroth, Marc Peter},
	month = dec,
	year = {2017},
	pages = {4591--4602},
}

@inproceedings{shi_spectral_2018,
	title = {A {Spectral} {Approach} to {Gradient} {Estimation} for {Implicit} {Distributions}},
	url = {https://proceedings.mlr.press/v80/shi18a.html},
	abstract = {Recently there have been increasing interests in learning and inference with implicit distributions (i.e., distributions without tractable densities). To this end, we develop a gradient estimator for implicit distributions based on Stein’s identity and a spectral decomposition of kernel operators, where the eigenfunctions are approximated by the Nystr\{ö\}m method. Unlike the previous works that only provide estimates at the sample points, our approach directly estimates the gradient function, thus allows for a simple and principled out-of-sample extension. We provide theoretical results on the error bound of the estimator and discuss the bias-variance tradeoff in practice. The effectiveness of our method is demonstrated by applications to gradient-free Hamiltonian Monte Carlo and variational inference with implicit distributions. Finally, we discuss the intuition behind the estimator by drawing connections between the Nystr\{ö\}m method and kernel PCA, which indicates that the estimator can automatically adapt to the geometry of the underlying distribution.},
	language = {en},
	urldate = {2024-01-24},
	booktitle = {Proceedings of the 35th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Shi, Jiaxin and Sun, Shengyang and Zhu, Jun},
	month = jul,
	year = {2018},
	note = {ISSN: 2640-3498},
	pages = {4644--4653},
}

@misc{leibfried_tutorial_2022,
	title = {A {Tutorial} on {Sparse} {Gaussian} {Processes} and {Variational} {Inference}},
	url = {http://arxiv.org/abs/2012.13962},
	doi = {10.48550/arXiv.2012.13962},
	abstract = {Gaussian processes (GPs) provide a framework for Bayesian inference that can offer principled uncertainty estimates for a large range of problems. For example, if we consider regression problems with Gaussian likelihoods, a GP model enjoys a posterior in closed form. However, identifying the posterior GP scales cubically with the number of training examples and requires to store all examples in memory. In order to overcome these obstacles, sparse GPs have been proposed that approximate the true posterior GP with pseudo-training examples. Importantly, the number of pseudo-training examples is user-defined and enables control over computational and memory complexity. In the general case, sparse GPs do not enjoy closed-form solutions and one has to resort to approximate inference. In this context, a convenient choice for approximate inference is variational inference (VI), where the problem of Bayesian inference is cast as an optimization problem -- namely, to maximize a lower bound of the log marginal likelihood. This paves the way for a powerful and versatile framework, where pseudo-training examples are treated as optimization arguments of the approximate posterior that are jointly identified together with hyperparameters of the generative model (i.e. prior and likelihood). The framework can naturally handle a wide scope of supervised learning problems, ranging from regression with heteroscedastic and non-Gaussian likelihoods to classification problems with discrete labels, but also problems with multidimensional labels. The purpose of this tutorial is to provide access to the basic matter for readers without prior knowledge in both GPs and VI. A proper exposition to the subject enables also access to more recent advances (like importance-weighted VI as well as interdomain, multioutput and deep GPs) that can serve as an inspiration for new research ideas.},
	urldate = {2024-01-24},
	publisher = {arXiv},
	author = {Leibfried, Felix and Dutordoir, Vincent and John, S. T. and Durrande, Nicolas},
	month = dec,
	year = {2022},
	note = {arXiv:2012.13962 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@misc{van_der_wilk_framework_2020,
	title = {A {Framework} for {Interdomain} and {Multioutput} {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/2003.01115},
	doi = {10.48550/arXiv.2003.01115},
	abstract = {One obstacle to the use of Gaussian processes (GPs) in large-scale problems, and as a component in deep learning system, is the need for bespoke derivations and implementations for small variations in the model or inference. In order to improve the utility of GPs we need a modular system that allows rapid implementation and testing, as seen in the neural network community. We present a mathematical and software framework for scalable approximate inference in GPs, which combines interdomain approximations and multiple outputs. Our framework, implemented in GPflow, provides a unified interface for many existing multioutput models, as well as more recent convolutional structures. This simplifies the creation of deep models with GPs, and we hope that this work will encourage more interest in this approach.},
	urldate = {2024-01-24},
	publisher = {arXiv},
	author = {van der Wilk, Mark and Dutordoir, Vincent and John, S. T. and Artemev, Artem and Adam, Vincent and Hensman, James},
	month = mar,
	year = {2020},
	note = {arXiv:2003.01115 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@inproceedings{dutordoir_deep_2021,
	title = {Deep {Neural} {Networks} as {Point} {Estimates} for {Deep} {Gaussian} {Processes}},
	volume = {34},
	url = {https://papers.nips.cc/paper_files/paper/2021/hash/4e6cd95227cb0c280e99a195be5f6615-Abstract.html},
	abstract = {Neural networks and Gaussian processes are complementary in their strengths and weaknesses. Having a better understanding of their relationship comes with the promise to make each method benefit from the strengths of the other. In this work, we establish an equivalence between the forward passes of neural networks and (deep) sparse Gaussian process models. The theory we develop is based on interpreting activation functions as interdomain inducing features through a rigorous analysis of the interplay between activation functions and kernels. This results in models that can either be seen as neural networks with improved uncertainty prediction or deep Gaussian processes with increased prediction accuracy. These claims are supported by experimental results on regression and classification datasets.},
	urldate = {2024-01-24},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Dutordoir, Vincent and Hensman, James and van der Wilk, Mark and Ek, Carl Henrik and Ghahramani, Zoubin and Durrande, Nicolas},
	year = {2021},
	pages = {9443--9455},
}

@inproceedings{dutordoir_sparse_2020,
	title = {Sparse {Gaussian} {Processes} with {Spherical} {Harmonic} {Features}},
	url = {https://proceedings.mlr.press/v119/dutordoir20a.html},
	abstract = {We introduce a new class of inter-domain variational Gaussian processes (GP) where data is mapped onto the unit hypersphere in order to use spherical harmonic representations. Our inference scheme is comparable to variational Fourier features, but it does not suffer from the curse of dimensionality, and leads to diagonal covariance matrices between inducing variables. This enables a speed-up in inference, because it bypasses the need to invert large covariance matrices. Our experiments show that our model is able to fit a regression model for a dataset with 6 million entries two orders of magnitude faster compared to standard sparse GPs, while retaining state of the art accuracy. We also demonstrate competitive performance on classification with non-conjugate likelihoods.},
	language = {en},
	urldate = {2024-01-24},
	booktitle = {Proceedings of the 37th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Dutordoir, Vincent and Durrande, Nicolas and Hensman, James},
	month = nov,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {2793--2802},
}

@misc{tiao_spherical_2023,
	title = {Spherical {Inducing} {Features} for {Orthogonally}-{Decoupled} {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/2304.14034},
	doi = {10.48550/arXiv.2304.14034},
	abstract = {Despite their many desirable properties, Gaussian processes (GPs) are often compared unfavorably to deep neural networks (NNs) for lacking the ability to learn representations. Recent efforts to bridge the gap between GPs and deep NNs have yielded a new class of inter-domain variational GPs in which the inducing variables correspond to hidden units of a feedforward NN. In this work, we examine some practical issues associated with this approach and propose an extension that leverages the orthogonal decomposition of GPs to mitigate these limitations. In particular, we introduce spherical inter-domain features to construct more flexible data-dependent basis functions for both the principal and orthogonal components of the GP approximation and show that incorporating NN activation features under this framework not only alleviates these shortcomings but is more scalable than alternative strategies. Experiments on multiple benchmark datasets demonstrate the effectiveness of our approach.},
	urldate = {2024-01-24},
	publisher = {arXiv},
	author = {Tiao, Louis C. and Dutordoir, Vincent and Picheny, Victor},
	month = jun,
	year = {2023},
	note = {arXiv:2304.14034 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@inproceedings{wang_nonlinear_2019,
	title = {Nonlinear {Stein} {Variational} {Gradient} {Descent} for {Learning} {Diversified} {Mixture} {Models}},
	url = {https://proceedings.mlr.press/v97/wang19h.html},
	abstract = {Diversification has been shown to be a powerful mechanism for learning robust models in non-convex settings. A notable example is learning mixture models, in which enforcing diversity between the different mixture components allows us to prevent the model collapsing phenomenon and capture more patterns from the observed data. In this work, we present a variational approach for diversity-promoting learning, which leverages the entropy functional as a natural mechanism for enforcing diversity. We develop a simple and efficient functional gradient-based algorithm for optimizing the variational objective function, which provides a significant generalization of Stein variational gradient descent (SVGD). We test our method on various challenging real world problems, including deep embedded clustering and deep anomaly detection. Empirical results show that our method provides an effective mechanism for diversity-promoting learning, achieving substantial improvement over existing methods.},
	language = {en},
	urldate = {2024-01-19},
	booktitle = {Proceedings of the 36th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Wang, Dilin and Liu, Qiang},
	month = may,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {6576--6585},
}

@inproceedings{pielok_approximate_2022,
	title = {Approximate {Bayesian} {Inference} with {Stein} {Functional} {Variational} {Gradient} {Descent}},
	url = {https://openreview.net/forum?id=a2-aoqmeYM4},
	abstract = {We propose a general-purpose variational algorithm that forms a natural analogue of Stein variational gradient descent (SVGD) in function space. While SVGD successively updates a set of particles to match a target density, the method introduced here of Stein functional variational gradient descent (SFVGD) updates a set of particle functions to match a target stochastic process (SP). The update step is found by minimizing the functional derivative of the Kullback-Leibler divergence between SPs. SFVGD can either be used to train Bayesian neural networks (BNNs) or for ensemble gradient boosting. We show the efficacy of training BNNs with SFVGD on various real-world datasets.},
	language = {en},
	urldate = {2024-01-18},
	author = {Pielok, Tobias and Bischl, Bernd and Rügamer, David},
	month = sep,
	year = {2022},
}

@inproceedings{cutajar_random_2017,
	title = {Random {Feature} {Expansions} for {Deep} {Gaussian} {Processes}},
	url = {https://proceedings.mlr.press/v70/cutajar17a.html},
	abstract = {The composition of multiple Gaussian Processes as a Deep Gaussian Process DGP enables a deep probabilistic nonparametric approach to flexibly tackle complex machine learning problems with sound quantification of uncertainty. Existing inference approaches for DGP models have limited scalability and are notoriously cumbersome to construct. In this work we introduce a novel formulation of DGPs based on random feature expansions that we train using stochastic variational inference. This yields a practical learning framework which significantly advances the state-of-the-art in inference for DGPs, and enables accurate quantification of uncertainty. We extensively showcase the scalability and performance of our proposal on several datasets with up to 8 million observations, and various DGP architectures with up to 30 hidden layers.},
	language = {en},
	urldate = {2024-01-18},
	booktitle = {Proceedings of the 34th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Cutajar, Kurt and Bonilla, Edwin V. and Michiardi, Pietro and Filippone, Maurizio},
	month = jul,
	year = {2017},
	note = {ISSN: 2640-3498},
	pages = {884--893},
}

@misc{beukema_satellite_2023,
	title = {Satellite {Imagery} and {AI}: {A} {New} {Era} in {Ocean} {Conservation}, from {Research} to {Deployment} and {Impact}},
	shorttitle = {Satellite {Imagery} and {AI}},
	url = {http://arxiv.org/abs/2312.03207},
	doi = {10.48550/arXiv.2312.03207},
	abstract = {Illegal, unreported, and unregulated (IUU) fishing poses a global threat to ocean habitats. Publicly available satellite data offered by NASA and the European Space Agency (ESA) provide an opportunity to actively monitor this activity. Effectively leveraging satellite data for maritime conservation requires highly reliable machine learning models operating globally with minimal latency. This paper introduces three specialized computer vision models designed for synthetic aperture radar (Sentinel-1), optical imagery (Sentinel-2), and nighttime lights (Suomi-NPP/NOAA-20). It also presents best practices for developing and delivering real-time computer vision services for conservation. These models have been deployed in Skylight, a real time maritime monitoring platform, which is provided at no cost to users worldwide.},
	urldate = {2024-01-17},
	publisher = {arXiv},
	author = {Beukema, Patrick and Bastani, Favyen and Wolters, Piper and Herzog, Henry and Ferdinando, Joe},
	month = dec,
	year = {2023},
	note = {arXiv:2312.03207 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}

@article{lam_learning_2023,
	title = {Learning skillful medium-range global weather forecasting},
	copyright = {Copyright © 2023 The Authors, some rights reserved; exclusive licensee American Association for the Advancement of Science. No claim to original U.S. Government Works},
	url = {https://www.science.org/doi/10.1126/science.adi2336},
	doi = {10.1126/science.adi2336},
	abstract = {Machine learning leads to better, faster, and cheaper weather forecasting.},
	language = {EN},
	urldate = {2024-01-17},
	journal = {Science},
	author = {Lam, Remi and Sanchez-Gonzalez, Alvaro and Willson, Matthew and Wirnsberger, Peter and Fortunato, Meire and Alet, Ferran and Ravuri, Suman and Ewalds, Timo and Eaton-Rosen, Zach and Hu, Weihua and Merose, Alexander and Hoyer, Stephan and Holland, George and Vinyals, Oriol and Stott, Jacklynn and Pritzel, Alexander and Mohamed, Shakir and Battaglia, Peter},
	month = dec,
	year = {2023},
	note = {Publisher: American Association for the Advancement of Science},
}

@article{wang_attention_2022,
	title = {Attention based spatiotemporal graph attention networks for traffic flow forecasting},
	volume = {607},
	issn = {0020-0255},
	url = {https://www.sciencedirect.com/science/article/pii/S0020025522005679},
	doi = {10.1016/j.ins.2022.05.127},
	abstract = {Traffic flow forecasting is a crucial task in transportation and necessary for congestion mitigation, traffic control, and intelligent traffic management. Deep learning models can aid in high-accuracy traffic flow forecasting; however, the current research focuses only the ability of the model to capture dynamic spatiotemporal features, and studies on the effect of deeper network layers on spatiotemporal features—a critical factor affecting traffic flow forecasting accuracy—are limited. In this paper, we propose an attention-based spatiotemporal graph attention network (ASTGAT) model designed for network degradation and over-smoothing problems to investigate in-depth spatiotemporal information. Compared to other networks, ASTGAT can capture dynamic spatiotemporal correlations in data and deepen the network to improve prediction accuracy through multiple residual convolution and high-low feature concat. ASTGAT comprises three components that separately model the temporal relationships of the recent, daily, and weekly periods. Each component stacks multiple spatiotemporal blocks constructed using the attention mechanism, dilated gated convolution, and graph attention network. The graph and temporal attention layers capture spatiotemporal information dynamically, and the graph attention layer alleviates the over-smoothing phenomenon to deepen the network. The combined utilization of the attention mechanism and dilated gated convolution layer improves the medium and long temporal span prediction ability. We validated ASTGAT using two open highway datasets, and the results demonstrated that our ASTGAT model effectively extracts in-depth spatiotemporal information and the prediction results outperform those predicted by the current eight baselines. Our research is dedicated to establishing a better scientific basis for intelligent traffic management that can assist in decision making.},
	urldate = {2023-10-25},
	journal = {Information Sciences},
	author = {Wang, Yi and Jing, Changfeng and Xu, Shishuo and Guo, Tao},
	month = aug,
	year = {2022},
	keywords = {Dynamic spatiotemporal correlation, Intelligent transportation systems, Network deepening, Network degradation, Spatiotemporal graph neural network, Traffic flow forecasting},
	pages = {869--883},
}

@misc{fu_monarch_2023,
	title = {Monarch {Mixer}: {A} {Simple} {Sub}-{Quadratic} {GEMM}-{Based} {Architecture}},
	shorttitle = {Monarch {Mixer}},
	url = {http://arxiv.org/abs/2310.12109},
	doi = {10.48550/arXiv.2310.12109},
	abstract = {Machine learning models are increasingly being scaled in both sequence length and model dimension to reach longer contexts and better performance. However, existing architectures such as Transformers scale quadratically along both these axes. We ask: are there performant architectures that can scale sub-quadratically along sequence length and model dimension? We introduce Monarch Mixer (M2), a new architecture that uses the same sub-quadratic primitive along both sequence length and model dimension: Monarch matrices, a simple class of expressive structured matrices that captures many linear transforms, achieves high hardware efficiency on GPUs, and scales sub-quadratically. As a proof of concept, we explore the performance of M2 in three domains: non-causal BERT-style language modeling, ViT-style image classification, and causal GPT-style language modeling. For non-causal BERT-style modeling, M2 matches BERT-base and BERT-large in downstream GLUE quality with up to 27\% fewer parameters, and achieves up to 9.1\${\textbackslash}times\$ higher throughput at sequence length 4K. On ImageNet, M2 outperforms ViT-b by 1\% in accuracy, with only half the parameters. Causal GPT-style models introduce a technical challenge: enforcing causality via masking introduces a quadratic bottleneck. To alleviate this bottleneck, we develop a novel theoretical view of Monarch matrices based on multivariate polynomial evaluation and interpolation, which lets us parameterize M2 to be causal while remaining sub-quadratic. Using this parameterization, M2 matches GPT-style Transformers at 360M parameters in pretraining perplexity on The PILE--showing for the first time that it may be possible to match Transformer quality without attention or MLPs.},
	urldate = {2023-10-25},
	publisher = {arXiv},
	author = {Fu, Daniel Y. and Arora, Simran and Grogan, Jessica and Johnson, Isys and Eyuboglu, Sabri and Thomas, Armin W. and Spector, Benjamin and Poli, Michael and Rudra, Atri and Ré, Christopher},
	month = oct,
	year = {2023},
	note = {arXiv:2310.12109 [cs]},
	keywords = {Computer Science - Machine Learning},
}

@inproceedings{gong_quantile_2019,
	title = {Quantile {Stein} {Variational} {Gradient} {Descent} for {Batch} {Bayesian} {Optimization}},
	url = {https://proceedings.mlr.press/v97/gong19b.html},
	abstract = {Batch Bayesian optimization has been shown to be an efficient and successful approach for black-box function optimization, especially when the evaluation of cost function is highly expensive but can be efficiently parallelized. In this paper, we introduce a novel variational framework for batch query optimization, based on the argument that the query batch should be selected to have both high diversity and good worst case performance. This motivates us to introduce a variational objective that combines a quantile-based risk measure (for worst case performance) and entropy regularization (for enforcing diversity). We derive a gradient-based particle-based algorithm for solving our quantile-based variational objective, which generalizes Stein variational gradient descent (SVGD). We evaluate our method on a number of real-world applications and show that it consistently outperforms other recent state-of-the-art batch Bayesian optimization methods. Extensive experimental results indicate that our method achieves better or comparable performance, compared to the existing methods.},
	language = {en},
	urldate = {2024-01-15},
	booktitle = {Proceedings of the 36th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Gong, Chengyue and Peng, Jian and Liu, Qiang},
	month = may,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {2347--2356},
}

@inproceedings{liu_understanding_2019,
	title = {Understanding and {Accelerating} {Particle}-{Based} {Variational} {Inference}},
	url = {https://proceedings.mlr.press/v97/liu19i.html},
	abstract = {Particle-based variational inference methods (ParVIs) have gained attention in the Bayesian inference literature, for their capacity to yield flexible and accurate approximations. We explore ParVIs from the perspective of Wasserstein gradient flows, and make both theoretical and practical contributions. We unify various finite-particle approximations that existing ParVIs use, and recognize that the approximation is essentially a compulsory smoothing treatment, in either of two equivalent forms. This novel understanding reveals the assumptions and relations of existing ParVIs, and also inspires new ParVIs. We propose an acceleration framework and a principled bandwidth-selection method for general ParVIs; these are based on the developed theory and leverage the geometry of the Wasserstein space. Experimental results show the improved convergence by the acceleration framework and enhanced sample accuracy by the bandwidth-selection method.},
	language = {en},
	urldate = {2024-01-09},
	booktitle = {Proceedings of the 36th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Liu, Chang and Zhuo, Jingwei and Cheng, Pengyu and Zhang, Ruiyi and Zhu, Jun},
	month = may,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {4082--4092},
}

@article{anastasiou_steins_2023,
	title = {Stein’s {Method} {Meets} {Computational} {Statistics}: {A} {Review} of {Some} {Recent} {Developments}},
	volume = {38},
	issn = {0883-4237, 2168-8745},
	shorttitle = {Stein’s {Method} {Meets} {Computational} {Statistics}},
	url = {https://projecteuclid.org/journals/statistical-science/volume-38/issue-1/Steins-Method-Meets-Computational-Statistics--A-Review-of-Some/10.1214/22-STS863.full},
	doi = {10.1214/22-STS863},
	abstract = {Stein’s method compares probability distributions through the study of a class of linear operators called Stein operators. While mainly studied in probability and used to underpin theoretical statistics, Stein’s method has led to significant advances in computational statistics in recent years. The goal of this survey is to bring together some of these recent developments, and in doing so, to stimulate further research into the successful field of Stein’s method and statistics. The topics we discuss include tools to benchmark and compare sampling methods such as approximate Markov chain Monte Carlo, deterministic alternatives to sampling methods, control variate techniques, parameter estimation and goodness-of-fit testing.},
	number = {1},
	urldate = {2024-01-08},
	journal = {Statistical Science},
	author = {Anastasiou, Andreas and Barp, Alessandro and Briol, François-Xavier and Ebner, Bruno and Gaunt, Robert E. and Ghaderinezhad, Fatemeh and Gorham, Jackson and Gretton, Arthur and Ley, Christophe and Liu, Qiang and Mackey, Lester and Oates, Chris J. and Reinert, Gesine and Swan, Yvik},
	month = feb,
	year = {2023},
	note = {Publisher: Institute of Mathematical Statistics},
	keywords = {Stein’s method, approximate Markov chain Monte Carlo, control variates, goodness-of-fit testing, likelihood ratio, maximum likelihood estimator, prior sensitivity, sample quality, variational inference},
	pages = {120--139},
}

@misc{rahman_truly_2023,
	title = {Truly {Scale}-{Equivariant} {Deep} {Nets} with {Fourier} {Layers}},
	url = {http://arxiv.org/abs/2311.02922},
	doi = {10.48550/arXiv.2311.02922},
	abstract = {In computer vision, models must be able to adapt to changes in image resolution to effectively carry out tasks such as image segmentation; This is known as scale-equivariance. Recent works have made progress in developing scale-equivariant convolutional neural networks, e.g., through weight-sharing and kernel resizing. However, these networks are not truly scale-equivariant in practice. Specifically, they do not consider anti-aliasing as they formulate the down-scaling operation in the continuous domain. To address this shortcoming, we directly formulate down-scaling in the discrete domain with consideration of anti-aliasing. We then propose a novel architecture based on Fourier layers to achieve truly scale-equivariant deep nets, i.e., absolute zero equivariance-error. Following prior works, we test this model on MNIST-scale and STL-10 datasets. Our proposed model achieves competitive classification performance while maintaining zero equivariance-error.},
	urldate = {2023-12-18},
	publisher = {arXiv},
	author = {Rahman, Md Ashiqur and Yeh, Raymond A.},
	month = nov,
	year = {2023},
	note = {arXiv:2311.02922 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
}

@misc{cheema_integrated_2023,
	title = {Integrated {Variational} {Fourier} {Features} for {Fast} {Spatial} {Modelling} with {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/2308.14142},
	doi = {10.48550/arXiv.2308.14142},
	abstract = {Sparse variational approximations are popular methods for scaling up inference and learning in Gaussian processes to larger datasets. For \$N\$ training points, exact inference has \$O(N{\textasciicircum}3)\$ cost; with \$M {\textbackslash}ll N\$ features, state of the art sparse variational methods have \$O(NM{\textasciicircum}2)\$ cost. Recently, methods have been proposed using more sophisticated features; these promise \$O(M{\textasciicircum}3)\$ cost, with good performance in low dimensional tasks such as spatial modelling, but they only work with a very limited class of kernels, excluding some of the most commonly used. In this work, we propose integrated Fourier features, which extends these performance benefits to a very broad class of stationary covariance functions. We motivate the method and choice of parameters from a convergence analysis and empirical exploration, and show practical speedup in synthetic and real world spatial regression tasks.},
	urldate = {2023-12-18},
	publisher = {arXiv},
	author = {Cheema, Talay M. and Rasmussen, Carl Edward},
	month = aug,
	year = {2023},
	note = {arXiv:2308.14142 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@inproceedings{zhen_learning_2020,
	title = {Learning to {Learn} {Kernels} with {Variational} {Random} {Features}},
	url = {https://proceedings.mlr.press/v119/zhen20a.html},
	abstract = {We introduce kernels with random Fourier features in the meta-learning framework for few-shot learning. We propose meta variational random features (MetaVRF) to learn adaptive kernels for the base-learner, which is developed in a latent variable model by treating the random feature basis as the latent variable. We formulate the optimization of MetaVRF as a variational inference problem by deriving an evidence lower bound under the meta-learning framework. To incorporate shared knowledge from related tasks, we propose a context inference of the posterior, which is established by an LSTM architecture. The LSTM-based inference network can effectively integrate the context information of previous tasks with task-specific information, generating informative and adaptive features. The learned MetaVRF can produce kernels of high representational power with a relatively low spectral sampling rate and also enables fast adaptation to new tasks. Experimental results on a variety of few-shot regression and classification tasks demonstrate that MetaVRF delivers much better, or at least competitive, performance compared to existing meta-learning alternatives.},
	language = {en},
	urldate = {2023-12-18},
	booktitle = {Proceedings of the 37th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Zhen, Xiantong and Sun, Haoliang and Du, Yingjun and Xu, Jun and Yin, Yilong and Shao, Ling and Snoek, Cees},
	month = nov,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {11409--11419},
}

@misc{pinder_stein_2020,
	title = {Stein {Variational} {Gaussian} {Processes}},
	url = {https://arxiv.org/abs/2009.12141v3},
	abstract = {We show how to use Stein variational gradient descent (SVGD) to carry out inference in Gaussian process (GP) models with non-Gaussian likelihoods and large data volumes. Markov chain Monte Carlo (MCMC) is extremely computationally intensive for these situations, but the parametric assumptions required for efficient variational inference (VI) result in incorrect inference when they encounter the multi-modal posterior distributions that are common for such models. SVGD provides a non-parametric alternative to variational inference which is substantially faster than MCMC. We prove that for GP models with Lipschitz gradients the SVGD algorithm monotonically decreases the Kullback-Leibler divergence from the sampling distribution to the true posterior. Our method is demonstrated on benchmark problems in both regression and classification, a multimodal posterior, and an air quality example with 550,134 spatiotemporal observations, showing substantial performance improvements over MCMC and VI.},
	language = {en},
	urldate = {2024-01-08},
	author = {Pinder, Thomas and Nemeth, Christopher and Leslie, David},
	month = sep,
	year = {2020},
}

@misc{dangelo_stein_2021,
	title = {On {Stein} {Variational} {Neural} {Network} {Ensembles}},
	url = {https://arxiv.org/abs/2106.10760v2},
	abstract = {Ensembles of deep neural networks have achieved great success recently, but they do not offer a proper Bayesian justification. Moreover, while they allow for averaging of predictions over several hypotheses, they do not provide any guarantees for their diversity, leading to redundant solutions in function space. In contrast, particle-based inference methods, such as Stein variational gradient descent (SVGD), offer a Bayesian framework, but rely on the choice of a kernel to measure the similarity between ensemble members. In this work, we study different SVGD methods operating in the weight space, function space, and in a hybrid setting. We compare the SVGD approaches to other ensembling-based methods in terms of their theoretical properties and assess their empirical performance on synthetic and real-world tasks. We find that SVGD using functional and hybrid kernels can overcome the limitations of deep ensembles. It improves on functional diversity and uncertainty estimation and approaches the true Bayesian posterior more closely. Moreover, we show that using stochastic SVGD updates, as opposed to the standard deterministic ones, can further improve the performance.},
	language = {en},
	urldate = {2024-01-08},
	author = {D'Angelo, Francesco and Fortuin, Vincent and Wenzel, Florian},
	month = jun,
	year = {2021},
}

@inproceedings{xue_deep_2019,
	address = {Macao, China},
	series = {{IJCAI}'19},
	title = {Deep spectral kernel learning},
	isbn = {978-0-9992411-4-1},
	abstract = {Recently, spectral kernels have attracted wide attention in complex dynamic environments. These advanced kernels mainly focus on breaking through the crucial limitation on locality, that is, the stationarity and the monotonicity. But actually, owing to the inefficiency of shallow models in computational elements, they are more likely unable to accurately reveal dynamic and potential variations. In this paper, we propose a novel deep spectral kernel network (DSKN) to naturally integrate nonstationary and non-monotonic spectral kernels into elegant deep architectures in an interpretable way, which can be further generalized to cover most kernels. Concretely, we firstly deal with the general form of spectral kernels by the inverse Fourier transform. Secondly, DSKN is constructed by embedding the preeminent spectral kernels into each layer to boost the efficiency in computational elements, which can effectively reveal the dynamic input-dependent characteristics and potential longrange correlations by compactly representing complex advanced concepts. Thirdly, detailed analyses of DSKN are presented. Owing to its universality, we propose a unified spectral transform technique to flexibly extend and reasonably initialize domainrelated DSKN. Furthermore, the representer theorem of DSKN is given. Systematical experiments demonstrate the superiority of DSKN compared to state-of-the-art relevant algorithms on varieties of standard real-world tasks.},
	urldate = {2024-01-07},
	booktitle = {Proceedings of the 28th {International} {Joint} {Conference} on {Artificial} {Intelligence}},
	publisher = {AAAI Press},
	author = {Xue, Hui and Wu, Zheng-Fan and Sun, Wei-Xiang},
	month = aug,
	year = {2019},
	pages = {4019--4025},
}

@inproceedings{cunningham_actually_2023,
	title = {Actually {Sparse} {Variational} {Gaussian} {Processes}},
	url = {https://proceedings.mlr.press/v206/cunningham23a.html},
	abstract = {Gaussian processes (GPs) are typically criticised for their unfavourable scaling in both computational and memory requirements. For large datasets, sparse GPs reduce these demands by conditioning on a small set of inducing variables designed to summarise the data. In practice however, for large datasets requiring many inducing variables, such as low-lengthscale spatial data, even sparse GPs can become computationally expensive, limited by the number of inducing variables one can use. In this work, we propose a new class of inter-domain variational GP, constructed by projecting a GP onto a set of compactly supported B-spline basis functions. The key benefit of our approach is that the compact support of the B-spline basis functions admits the use of sparse linear algebra to significantly speed up matrix operations and drastically reduce the memory footprint. This allows us to very efficiently model fast-varying spatial phenomena with tens of thousands of inducing variables, where previous approaches failed.},
	language = {en},
	urldate = {2024-01-07},
	booktitle = {Proceedings of {The} 26th {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Cunningham, Harry Jake and Souza, Daniel Augusto de and Takao, So and Wilk, Mark van der and Deisenroth, Marc Peter},
	month = apr,
	year = {2023},
	note = {ISSN: 2640-3498},
	pages = {10395--10408},
}

@inproceedings{gal_improving_2015,
	title = {Improving the {Gaussian} {Process} {Sparse} {Spectrum} {Approximation} by {Representing} {Uncertainty} in {Frequency} {Inputs}},
	url = {https://proceedings.mlr.press/v37/galb15.html},
	abstract = {Standard sparse pseudo-input approximations to the Gaussian process (GP) cannot handle complex functions well. Sparse spectrum alternatives attempt to answer this but are known to over-fit. We suggest the use of variational inference for the sparse spectrum approximation to avoid both issues. We model the covariance function with a finite Fourier series approximation and treat it as a random variable. The random covariance function has a posterior, on which a variational distribution is placed. The variational distribution transforms the random covariance function to fit the data. We study the properties of our approximate inference, compare it to alternative ones, and extend it to the distributed and stochastic domains. Our approximation captures complex functions better than standard approaches and avoids over-fitting.},
	language = {en},
	urldate = {2024-01-03},
	booktitle = {Proceedings of the 32nd {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Gal, Yarin and Turner, Richard},
	month = jun,
	year = {2015},
	note = {ISSN: 1938-7228},
	pages = {655--664},
}

@inproceedings{shen_harmonizable_2019,
	title = {Harmonizable mixture kernels with variational {Fourier} features},
	url = {https://proceedings.mlr.press/v89/shen19c.html},
	abstract = {The expressive power of Gaussian processes depends heavily on the choice of kernel. In this work we propose the novel harmonizable mixture kernel (HMK), a family of expressive, interpretable, non-stationary kernels derived from mixture models on the generalized spectral representation. As a theoretically sound treatment of non-stationary kernels, HMK supports harmonizable covariances, a wide subset of kernels including all stationary and many non-stationary covariances. We also propose variational Fourier features, an inter-domain sparse GP inference framework that offers a representative set of ’inducing frequencies’. We show that harmonizable mixture kernels interpolate between local patterns, and that variational Fourier features offers a robust kernel learning framework for the new kernel family.},
	language = {en},
	urldate = {2024-01-02},
	booktitle = {Proceedings of the {Twenty}-{Second} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Shen, Zheyang and Heinonen, Markus and Kaski, Samuel},
	month = apr,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {3273--3282},
}

@misc{zhang_bayesian_2023,
	title = {Bayesian {Non}-linear {Latent} {Variable} {Modeling} via {Random} {Fourier} {Features}},
	url = {http://arxiv.org/abs/2306.08352},
	doi = {10.48550/arXiv.2306.08352},
	abstract = {The Gaussian process latent variable model (GPLVM) is a popular probabilistic method used for nonlinear dimension reduction, matrix factorization, and state-space modeling. Inference for GPLVMs is computationally tractable only when the data likelihood is Gaussian. Moreover, inference for GPLVMs has typically been restricted to obtaining maximum a posteriori point estimates, which can lead to overfitting, or variational approximations, which mischaracterize the posterior uncertainty. Here, we present a method to perform Markov chain Monte Carlo (MCMC) inference for generalized Bayesian nonlinear latent variable modeling. The crucial insight necessary to generalize GPLVMs to arbitrary observation models is that we approximate the kernel function in the Gaussian process mappings with random Fourier features; this allows us to compute the gradient of the posterior in closed form with respect to the latent variables. We show that we can generalize GPLVMs to non-Gaussian observations, such as Poisson, negative binomial, and multinomial distributions, using our random feature latent variable model (RFLVM). Our generalized RFLVMs perform on par with state-of-the-art latent variable models on a wide range of applications, including motion capture, images, and text data for the purpose of estimating the latent structure and imputing the missing data of these complex data sets.},
	urldate = {2023-12-18},
	publisher = {arXiv},
	author = {Zhang, Michael Minyi and Gundersen, Gregory W. and Engelhardt, Barbara E.},
	month = jun,
	year = {2023},
	note = {arXiv:2306.08352 [cs, stat]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Statistics - Machine Learning},
}

@article{chang_data-driven_2017,
	title = {Data-driven {Random} {Fourier} {Features} using {Stein} {Effect}},
	url = {http://arxiv.org/abs/1705.08525},
	abstract = {Large-scale kernel approximation is an important problem in machine learning research. Approaches using random Fourier features have become increasingly popular [Rahimi and Recht, 2007], where kernel approximation is treated as empirical mean estimation via Monte Carlo (MC) or Quasi-Monte Carlo (QMC) integration [Yang et al., 2014]. A limitation of the current approaches is that all the features receive an equal weight summing to 1. In this paper, we propose a novel shrinkage estimator from "Stein effect", which provides a data-driven weighting strategy for random features and enjoys theoretical justifications in terms of lowering the empirical risk. We further present an efficient randomized algorithm for large-scale applications of the proposed method. Our empirical results on six benchmark data sets demonstrate the advantageous performance of this approach over representative baselines in both kernel approximation and supervised learning tasks.},
	journal = {arXiv:1705.08525 [cs, stat]},
	author = {Chang, Wei-Cheng and Li, Chun-Liang and Yang, Yiming and Poczos, Barnabas},
	month = may,
	year = {2017},
	note = {arXiv: 1705.08525},
	keywords = {kernel learning, random fourier features, stein svgd, 🔥},
}

@article{marmin_deep_2022,
	title = {Deep {Gaussian} {Processes} for {Calibration} of {Computer} {Models} (with {Discussion})},
	volume = {17},
	issn = {1936-0975, 1931-6690},
	url = {https://projecteuclid.org/journals/bayesian-analysis/volume-17/issue-4/Deep-Gaussian-Processes-for-Calibration-of-Computer-Models-with-Discussion/10.1214/21-BA1293.full},
	doi = {10.1214/21-BA1293},
	abstract = {Bayesian calibration of black-box computer models offers an established framework for quantification of uncertainty of model parameters and predictions. Traditional Bayesian calibration involves the emulation of the computer model and an additive model discrepancy term using Gaussian processes; inference is then carried out using Markov chain Monte Carlo. This calibration approach is limited by the poor scalability of Gaussian processes and by the need to specify a sensible covariance function to deal with the complexity of the computer model and the discrepancy. In this work, we propose a novel calibration framework, where these challenges are addressed by means of compositions of Gaussian processes into Deep Gaussian processes and scalable variational inference techniques. Thanks to this formulation, it is possible to obtain a flexible calibration approach, which is easy to implement in development environments featuring automatic differentiation and exploiting GPU-type hardware. We show how our proposal yields a powerful alternative to the state-of-the-art by means of experimental validations on various calibration problems. We conclude the paper by showing how we can carry out adaptive experimental design, and by discussing the identifiability properties of the proposed calibration model.},
	number = {4},
	urldate = {2023-12-18},
	journal = {Bayesian Analysis},
	author = {Marmin, Sébastien and Filippone, Maurizio},
	month = dec,
	year = {2022},
	note = {Publisher: International Society for Bayesian Analysis},
	keywords = {62F15, 62K99, 62M45, Bayesian inference, Neural nets, computer experiments, deep Gaussian process, 🔥},
	pages = {1301--1350},
}

@article{hensman_variational_2018,
	title = {Variational {Fourier} {Features} for {Gaussian} {Processes}},
	volume = {18},
	issn = {1533-7928},
	url = {http://jmlr.org/papers/v18/16-579.html},
	abstract = {This work brings together two powerful concepts in Gaussian processes: the variational approach to sparse approximation and the spectral representation of Gaussian processes. This gives rise to an approximation that inherits the benefits of the variational approach but with the representational power and computational scalability of spectral representations. The work hinges on a key result that there exist spectral features related to a finite domain of the Gaussian process which exhibit almost-independent covariances. We derive these expressions for Matérn kernels in one dimension, and generalize to more dimensions using kernels with specific structures. Under the assumption of additive Gaussian noise, our method requires only a single pass through the data set, making for very fast and accurate computation. We fit a model to 4 million training points in just a few minutes on a standard laptop. With non- conjugate likelihoods, our MCMC scheme reduces the cost of computation from 
(N
M
2
)
O(NM2)
 (for a sparse Gaussian process) to 
(NM)
O(NM)
 per iteration, where 
N
N
 is the number of data and 
M
M
 is the number of features.},
	number = {151},
	urldate = {2023-11-01},
	journal = {Journal of Machine Learning Research},
	author = {Hensman, James and Durrande, Nicolas and Solin, Arno},
	year = {2018},
	keywords = {🔥},
	pages = {1--52},
}

@misc{graves_bayesian_2023,
	title = {Bayesian {Flow} {Networks}},
	url = {http://arxiv.org/abs/2308.07037},
	doi = {10.48550/arXiv.2308.07037},
	abstract = {This paper introduces Bayesian Flow Networks (BFNs), a new class of generative model in which the parameters of a set of independent distributions are modified with Bayesian inference in the light of noisy data samples, then passed as input to a neural network that outputs a second, interdependent distribution. Starting from a simple prior and iteratively updating the two distributions yields a generative procedure similar to the reverse process of diffusion models; however it is conceptually simpler in that no forward process is required. Discrete and continuous-time loss functions are derived for continuous, discretised and discrete data, along with sample generation procedures. Notably, the network inputs for discrete data lie on the probability simplex, and are therefore natively differentiable, paving the way for gradient-based sample guidance and few-step generation in discrete domains such as language modelling. The loss function directly optimises data compression and places no restrictions on the network architecture. In our experiments BFNs achieve competitive log-likelihoods for image modelling on dynamically binarized MNIST and CIFAR-10, and outperform all known discrete diffusion models on the text8 character-level language modelling task.},
	urldate = {2023-12-13},
	publisher = {arXiv},
	author = {Graves, Alex and Srivastava, Rupesh Kumar and Atkinson, Timothy and Gomez, Faustino},
	month = nov,
	year = {2023},
	note = {arXiv:2308.07037 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
}

@misc{yu_climsim_2023,
	title = {{ClimSim}: {A} large multi-scale dataset for hybrid physics-{ML} climate emulation},
	shorttitle = {{ClimSim}},
	url = {http://arxiv.org/abs/2306.08754},
	doi = {10.48550/arXiv.2306.08754},
	abstract = {Modern climate projections lack adequate spatial and temporal resolution due to computational constraints. A consequence is inaccurate and imprecise predictions of critical processes such as storms. Hybrid methods that combine physics with machine learning (ML) have introduced a new generation of higher fidelity climate simulators that can sidestep Moore's Law by outsourcing compute-hungry, short, high-resolution simulations to ML emulators. However, this hybrid ML-physics simulation approach requires domain-specific treatment and has been inaccessible to ML experts because of lack of training data and relevant, easy-to-use workflows. We present ClimSim, the largest-ever dataset designed for hybrid ML-physics research. It comprises multi-scale climate simulations, developed by a consortium of climate scientists and ML researchers. It consists of 5.7 billion pairs of multivariate input and output vectors that isolate the influence of locally-nested, high-resolution, high-fidelity physics on a host climate simulator's macro-scale physical state. The dataset is global in coverage, spans multiple years at high sampling frequency, and is designed such that resulting emulators are compatible with downstream coupling into operational climate simulators. We implement a range of deterministic and stochastic regression baselines to highlight the ML challenges and their scoring. The data (https://huggingface.co/datasets/LEAP/ClimSim\_high-res) and code (https://leap-stc.github.io/ClimSim) are released openly to support the development of hybrid ML-physics and high-fidelity climate simulations for the benefit of science and society.},
	urldate = {2023-12-06},
	publisher = {arXiv},
	author = {Yu, Sungduk and Hannah, Walter and Peng, Liran and Lin, Jerry and Bhouri, Mohamed Aziz and Gupta, Ritwik and Lütjens, Björn and Will, Justus Christopher and Behrens, Gunnar and Busecke, Julius and Loose, Nora and Stern, Charles I. and Beucler, Tom and Harrop, Bryce and Hillman, Benjamin R. and Jenney, Andrea and Ferretti, Savannah and Liu, Nana and Anandkumar, Anima and Brenowitz, Noah D. and Eyring, Veronika and Geneva, Nicholas and Gentine, Pierre and Mandt, Stephan and Pathak, Jaideep and Subramaniam, Akshay and Vondrick, Carl and Yu, Rose and Zanna, Laure and Zheng, Tian and Abernathey, Ryan and Ahmed, Fiaz and Bader, David C. and Baldi, Pierre and Barnes, Elizabeth and Bretherton, Christopher and Caldwell, Peter and Chuang, Wayne and Han, Yilun and Huang, Yu and Iglesias-Suarez, Fernando and Jantre, Sanket and Kashinath, Karthik and Khairoutdinov, Marat and Kurth, Thorsten and Lutsko, Nicholas and Ma, Po-Lun and Mooers, Griffin and Neelin, J. David and Randall, David and Shamekh, Sara and Taylor, Mark A. and Urban, Nathan and Yuval, Janni and Zhang, Guang and Pritchard, Michael},
	month = oct,
	year = {2023},
	note = {arXiv:2306.08754 [physics]},
	keywords = {Computer Science - Machine Learning, Physics - Atmospheric and Oceanic Physics},
}

@inproceedings{mallick_deep_2021,
	title = {Deep kernels with probabilistic embeddings for small-data learning},
	url = {https://proceedings.mlr.press/v161/mallick21a.html},
	abstract = {Gaussian Processes (GPs) are known to provide accurate predictions and uncertainty estimates even with small amounts of labeled data by capturing similarity between data points through their kernel function. However traditional GP kernels are not very effective at capturing similarity between high dimensional data points. Neural networks can be used to learn good representations that encode intricate structures in high dimensional data, and can be used as inputs to the GP kernel. However the huge data requirement of neural networks makes this approach ineffective in small data settings. To solves the conflicting problems of representation learning and data efficiency, we propose to learn deep kernels on probabilistic embeddings by using a probabilistic neural network. Our approach maps high-dimensional data to a probability distribution in a low dimensional subspace and then computes a kernel between these distributions to capture similarity. To enable end-to-end learning, we derive a functional gradient descent procedure for training the model. Experiments on a variety of datasets show that our approach outperforms the state-of-the-art in GP kernel learning in both supervised and semi-supervised settings. We also extend our approach to other small-data paradigms such as few-shot classification where it outperforms previous approaches on mini-Imagenet and CUB datasets.},
	language = {en},
	urldate = {2023-11-01},
	booktitle = {Proceedings of the {Thirty}-{Seventh} {Conference} on {Uncertainty} in {Artificial} {Intelligence}},
	publisher = {PMLR},
	author = {Mallick, Ankur and Dwivedi, Chaitanya and Kailkhura, Bhavya and Joshi, Gauri and Han, T. Yong-Jin},
	month = dec,
	year = {2021},
	note = {ISSN: 2640-3498},
	pages = {918--928},
}

@inproceedings{pu_vae_2017,
	title = {{VAE} {Learning} via {Stein} {Variational} {Gradient} {Descent}},
	volume = {30},
	url = {https://proceedings.neurips.cc/paper_files/paper/2017/hash/443dec3062d0286986e21dc0631734c9-Abstract.html},
	abstract = {A new method for learning variational autoencoders (VAEs) is developed, based on Stein variational gradient descent. A key advantage of this approach is that one need not make parametric assumptions about the form of the encoder distribution. Performance is further enhanced by integrating the proposed encoder with importance sampling. Excellent performance is demonstrated across multiple unsupervised and semi-supervised problems, including semi-supervised analysis of the ImageNet data, demonstrating the scalability of the model to large datasets.},
	urldate = {2023-11-01},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Pu, Yuchen and Gan, Zhe and Henao, Ricardo and Li, Chunyuan and Han, Shaobo and Carin, Lawrence},
	year = {2017},
}

@misc{fan_continuous-time_2021,
	title = {Continuous-{Time} {Sequential} {Recommendation} with {Temporal} {Graph} {Collaborative} {Transformer}},
	url = {http://arxiv.org/abs/2108.06625},
	doi = {10.48550/arXiv.2108.06625},
	abstract = {In order to model the evolution of user preference, we should learn user/item embeddings based on time-ordered item purchasing sequences, which is defined as Sequential Recommendation (SR) problem. Existing methods leverage sequential patterns to model item transitions. However, most of them ignore crucial temporal collaborative signals, which are latent in evolving user-item interactions and coexist with sequential patterns. Therefore, we propose to unify sequential patterns and temporal collaborative signals to improve the quality of recommendation, which is rather challenging. Firstly, it is hard to simultaneously encode sequential patterns and collaborative signals. Secondly, it is non-trivial to express the temporal effects of collaborative signals. Hence, we design a new framework Temporal Graph Sequential Recommender (TGSRec) upon our defined continuous-time bi-partite graph. We propose a novel Temporal Collaborative Trans-former (TCT) layer in TGSRec, which advances the self-attention mechanism by adopting a novel collaborative attention. TCT layer can simultaneously capture collaborative signals from both users and items, as well as considering temporal dynamics inside sequential patterns. We propagate the information learned fromTCTlayerover the temporal graph to unify sequential patterns and temporal collaborative signals. Empirical results on five datasets show that TGSRec significantly outperforms other baselines, in average up to 22.5\% and 22.1\%absolute improvements in Recall@10and MRR, respectively.},
	urldate = {2023-10-27},
	publisher = {arXiv},
	author = {Fan, Ziwei and Liu, Zhiwei and Zhang, Jiawei and Xiong, Yun and Zheng, Lei and Yu, Philip S.},
	month = aug,
	year = {2021},
	note = {arXiv:2108.06625 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Information Retrieval, Computer Science - Machine Learning, 🔥},
}

@article{galychyn_organic_2017,
	series = {Green {Urbanism} ({GU})},
	title = {Organic {Urbanism}: {Human}-oriented {Design} for {Metropolises}},
	volume = {37},
	issn = {1878-0296},
	shorttitle = {Organic {Urbanism}},
	url = {https://www.sciencedirect.com/science/article/pii/S1878029617300051},
	doi = {10.1016/j.proenv.2017.03.005},
	abstract = {Existing theories of sustainable development have failed to address three common structural elementsof the human-oriented transportation system: healthy neighborhoods, Transit-Oriented Developments (TODs) connected by integrated public-soft transportation network. The newly emerged conceptual model for zero-emission and zero-waste urban design named Green Urbanism like the previous theories has failed to develop those three structural elements of organic city. In this paper, approaches regarding the successful integration of soft transportation to the transit network will be discussed. Secondly, the best physical standard for TOD will be determined by comparing TODs in Japan, Denmark, Ukraine and Turkey by socio-economic indicators of each TOD type. Finally, the concept of healthy neighborhood will be developed based on the Lehmann's healthy community\&district definitions, concept of human oriented transportation, and the latest eco-mobility study from Barcelona. Thus, new conceptual model for sustainable city will be proposed.},
	urldate = {2023-10-27},
	journal = {Procedia Environmental Sciences},
	author = {Galychyn, Oleksandr and Üstundağ, Kevser},
	month = jan,
	year = {2017},
	pages = {396--407},
}

@misc{luo_who_2023,
	title = {Who {You} {Play} {Affects} {How} {You} {Play}: {Predicting} {Sports} {Performance} {Using} {Graph} {Attention} {Networks} {With} {Temporal} {Convolution}},
	shorttitle = {Who {You} {Play} {Affects} {How} {You} {Play}},
	url = {http://arxiv.org/abs/2303.16741},
	doi = {10.48550/arXiv.2303.16741},
	abstract = {This study presents a novel deep learning method, called GATv2-GCN, for predicting player performance in sports. To construct a dynamic player interaction graph, we leverage player statistics and their interactions during gameplay. We use a graph attention network to capture the attention that each player pays to each other, allowing for more accurate modeling of the dynamic player interactions. To handle the multivariate player statistics time series, we incorporate a temporal convolution layer, which provides the model with temporal predictive power. We evaluate the performance of our model using real-world sports data, demonstrating its effectiveness in predicting player performance. Furthermore, we explore the potential use of our model in a sports betting context, providing insights into profitable strategies that leverage our predictive power. The proposed method has the potential to advance the state-of-the-art in player performance prediction and to provide valuable insights for sports analytics and betting industries.},
	urldate = {2023-10-04},
	publisher = {arXiv},
	author = {Luo, Rui and Krishnamurthy, Vikram},
	month = mar,
	year = {2023},
	note = {arXiv:2303.16741 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Social and Information Networks, 🔥},
}

@book{hennig_probabilistic_2022,
	title = {Probabilistic numerics: computation as machine learning},
	isbn = {978-1-107-16344-7},
	shorttitle = {Probabilistic numerics},
	abstract = {Probabilistic numerical computation formalises the connection between machine learning and applied mathematics. Numerical algorithms approximate intractable quantities from computable ones. They estimate integrals from evaluations of the integrand, or the path of a dynamical system described by differential equations from evaluations of the vector field. In other words, they infer a latent quantity from data. This book shows that it is thus formally possible to think of computational routines as learning machines, and to use the notion of Bayesian inference to build more flexible, efficient, or customised algorithms for computation. The text caters for Masters' and PhD students, as well as postgraduate researchers in artificial intelligence, computer science, statistics, and applied mathematics. Extensive background material is provided along with a wealth of figures, worked examples, and exercises (with solutions) to develop intuition},
	language = {eng},
	publisher = {Cambridge University Press},
	author = {Hennig, Philipp and Osborne, Michael A. and Kersting, Hans},
	year = {2022},
	doi = {10.1017/9781316681411},
}

@misc{kingma_adam_2017,
	title = {Adam: {A} {Method} for {Stochastic} {Optimization}},
	shorttitle = {Adam},
	url = {http://arxiv.org/abs/1412.6980},
	doi = {10.48550/arXiv.1412.6980},
	abstract = {We introduce Adam, an algorithm for first-order gradient-based optimization of stochastic objective functions, based on adaptive estimates of lower-order moments. The method is straightforward to implement, is computationally efficient, has little memory requirements, is invariant to diagonal rescaling of the gradients, and is well suited for problems that are large in terms of data and/or parameters. The method is also appropriate for non-stationary objectives and problems with very noisy and/or sparse gradients. The hyper-parameters have intuitive interpretations and typically require little tuning. Some connections to related algorithms, on which Adam was inspired, are discussed. We also analyze the theoretical convergence properties of the algorithm and provide a regret bound on the convergence rate that is comparable to the best known results under the online convex optimization framework. Empirical results demonstrate that Adam works well in practice and compares favorably to other stochastic optimization methods. Finally, we discuss AdaMax, a variant of Adam based on the infinity norm.},
	urldate = {2023-10-24},
	publisher = {arXiv},
	author = {Kingma, Diederik P. and Ba, Jimmy},
	month = jan,
	year = {2017},
	note = {arXiv:1412.6980 [cs]},
	keywords = {Computer Science - Machine Learning},
}

@inproceedings{ott_baysian_2023,
	title = {Baysian numerical integration with neural networks},
	url = {https://proceedings.mlr.press/v216/ott23a.html},
	abstract = {Bayesian probabilistic numerical methods for numerical integration offer significant advantages over their non-Bayesian counterparts: they can encode prior information about the integrand, and can quantify uncertainty over estimates of an integral. However, the most popular algorithm in this class, Bayesian quadrature, is based on Gaussian process models and is therefore associated with a high computational cost. To improve scalability, we propose an alternative approach based on Bayesian neural networks which we call Bayesian Stein networks. The key ingredients are a neural network architecture based on Stein operators, and an approximation of the Bayesian posterior based on the Laplace approximation. We show that this leads to orders of magnitude speed-ups on the popular Genz functions benchmark, and on challenging problems arising in the Bayesian analysis of dynamical systems, and the prediction of energy production for a large-scale wind farm.},
	language = {en},
	urldate = {2023-10-17},
	booktitle = {Proceedings of the {Thirty}-{Ninth} {Conference} on {Uncertainty} in {Artificial} {Intelligence}},
	author = {Ott, Katharina and Tiemann, Michael and Hennig, Philipp and Briol, François-Xavier},
	month = jul,
	year = {2023},
	note = {ISSN: 2640-3498},
	pages = {1606--1617},
}

@inproceedings{zhu_bayesian_2020,
	title = {Bayesian {Probabilistic} {Numerical} {Integration} with {Tree}-{Based} {Models}},
	volume = {33},
	url = {https://proceedings.neurips.cc/paper/2020/hash/3fe94a002317b5f9259f82690aeea4cd-Abstract.html},
	abstract = {Bayesian quadrature (BQ) is a method for solving numerical integration problems in a Bayesian manner, which allows users to quantify their uncertainty about the solution. The standard approach to BQ is based on a Gaussian process (GP) approximation of the integrand. As a result, BQ is inherently limited to cases where GP approximations can be done in an efficient manner, thus often prohibiting very high-dimensional or non-smooth target functions. This paper proposes to tackle this issue with a new Bayesian numerical integration algorithm based on Bayesian Additive Regression Trees (BART) priors, which we call BART-Int. BART priors are easy to tune and well-suited for discontinuous functions. We demonstrate that they also lend themselves naturally to a sequential design setting and that explicit convergence rates can be obtained in a variety of settings. The advantages and disadvantages of this new methodology are highlighted on a set of benchmark tests including the Genz functions, on a rare-event simulation problem and on a Bayesian survey design problem.},
	urldate = {2023-10-17},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	author = {Zhu, Harrison and Liu, Xing and Kang, Ruya and Shen, Zhichao and Flaxman, Seth and Briol, Francois-Xavier},
	year = {2020},
	pages = {5837--5849},
}

@article{fritsch_method_1984,
	title = {A {Method} for {Constructing} {Local} {Monotone} {Piecewise} {Cubic} {Interpolants}},
	volume = {5},
	issn = {0196-5204, 2168-3417},
	url = {http://epubs.siam.org/doi/10.1137/0905021},
	doi = {10.1137/0905021},
	language = {en},
	number = {2},
	urldate = {2023-10-17},
	journal = {SIAM Journal on Scientific and Statistical Computing},
	author = {Fritsch, F. N. and Butland, J.},
	month = jun,
	year = {1984},
	pages = {300--304},
}

@article{cooley_algorithm_1965,
	title = {An {Algorithm} for the {Machine} {Calculation} of {Complex} {Fourier} {Series}},
	volume = {19},
	issn = {0025-5718},
	url = {https://www.jstor.org/stable/2003354},
	doi = {10.2307/2003354},
	number = {90},
	urldate = {2023-10-17},
	journal = {Mathematics of Computation},
	author = {Cooley, James W. and Tukey, John W.},
	year = {1965},
	note = {Publisher: American Mathematical Society},
	pages = {297--301},
}

@book{mcgillem_continuous_1991,
	edition = {3rd ed},
	title = {Continuous and discrete signal and system analysis},
	author = {McGillem, Clare D. and Cooper, George R.},
	year = {1991},
	keywords = {Digital filters (Mathematics), Discrete-time systems, State-space methods, System analysis},
}

@inproceedings{sutherland_error_2015,
	series = {{UAI}'15},
	title = {On the error of random fourier features},
	isbn = {978-0-9966431-0-8},
	abstract = {Kernel methods give powerful, flexible, and theoretically grounded approaches to solving many problems in machine learning. The standard approach, however, requires pairwise evaluations of a kernel function, which can lead to scalability issues for very large datasets. Rahimi and Recht (2007) suggested a popular approach to handling this problem, known as random Fourier features. The quality of this approximation, however, is not well understood. We improve the uniform error bound of that paper, as well as giving novel understandings of the embedding's variance, approximation error, and use in some machine learning methods. We also point out that surprisingly, of the two main variants of those features, the more widely used is strictly higher-variance for the Gaussian kernel and has worse bounds.},
	urldate = {2021-09-27},
	booktitle = {Proceedings of the {Thirty}-{First} {Conference} on {Uncertainty} in {Artificial} {Intelligence}},
	author = {Sutherland, Danica J. and Schneider, Jeff},
	month = jul,
	year = {2015},
	pages = {862--871},
}

@inproceedings{tompkins_black_2019,
	title = {Black {Box} {Quantiles} for {Kernel} {Learning}},
	url = {http://proceedings.mlr.press/v89/tompkins19a.html},
	abstract = {Kernel methods have been successfully used in various domains to model nonlinear patterns. However, the structure of the kernels is typically handcrafted for each dataset based on the experience of...},
	language = {en},
	urldate = {2021-04-07},
	booktitle = {The 22nd {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	author = {Tompkins, Anthony and Senanayake, Ransalu and Morere, Philippe and Ramos, Fabio},
	month = apr,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {1427--1437},
}

@inproceedings{le_fastfood_2013,
	title = {Fastfood - {Computing} {Hilbert} {Space} {Expansions} in loglinear time},
	url = {https://proceedings.mlr.press/v28/le13.html},
	abstract = {Fast nonlinear function classes are crucial for nonparametric estimation, such as in kernel methods. This paper proposes an improvement to random kitchen sinks that offers significantly faster computation in log-linear time without sacrificing accuracy. Furthermore, we show how one may adjust the regularization properties of the kernel simply by changing the spectral distribution of the projection matrix. We provide experimental results which show that even for for moderately small problems we already achieve two orders of magnitude faster computation and three orders of magnitude lower memory footprint.},
	language = {en},
	urldate = {2023-05-17},
	booktitle = {Proceedings of the 30th {International} {Conference} on {Machine} {Learning}},
	author = {Le, Quoc and Sarlos, Tamas and Smola, Alexander},
	month = may,
	year = {2013},
	note = {ISSN: 1938-7228},
	pages = {244--252},
}

@inproceedings{rahimi_weighted_2008,
	title = {Weighted {Sums} of {Random} {Kitchen} {Sinks}: {Replacing} minimization with randomization in learning},
	volume = {21},
	shorttitle = {Weighted {Sums} of {Random} {Kitchen} {Sinks}},
	url = {https://papers.nips.cc/paper_files/paper/2008/hash/0efe32849d230d7f53049ddc4a4b0c60-Abstract.html},
	abstract = {Randomized neural networks are immortalized in this AI Koan: In the days when Sussman was a novice, Minsky once came to him as he sat hacking at the PDP-6. What are you doing?'' asked Minsky.I am training a randomly wired neural net to play tic-tac-toe,'' Sussman replied. Why is the net wired randomly?'' asked Minsky. Sussman replied,I do not want it to have any preconceptions of how to play.'' Minsky then shut his eyes. Why do you close your eyes?'' Sussman asked his teacher.So that the room will be empty,'' replied Minsky. At that moment, Sussman was enlightened. We analyze shallow random networks with the help of concentration of measure inequalities. Specifically, we consider architectures that compute a weighted sum of their inputs after passing them through a bank of arbitrary randomized nonlinearities. We identify conditions under which these networks exhibit good classification performance, and bound their test error in terms of the size of the dataset and the number of random nonlinearities.},
	urldate = {2023-05-17},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	author = {Rahimi, Ali and Recht, Benjamin},
	year = {2008},
}

@inproceedings{sellier_sparse_2023,
	title = {Sparse {Spectral} {Bayesian} {Permanental} {Process} with {Generalized} {Kernel}},
	url = {https://proceedings.mlr.press/v206/sellier23a.html},
	abstract = {We introduce a novel scheme for Bayesian inference on permanental processes which models the Poisson intensity as the square of a Gaussian process. Combining generalized kernels and a Fourier features-based representation of the Gaussian process with a Laplace approximation to the posterior, we achieve a fast and efficient inference that does not require numerical integration over the input space, allows kernel design and scales linearly with the number of events. Our method builds and improves upon the state-of-theart Laplace Bayesian point process benchmark of Walder and Bishop (2017), demonstrated on both synthetic, real-world temporal and large spatial data sets.},
	language = {en},
	urldate = {2023-05-16},
	booktitle = {Proceedings of {The} 26th {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	author = {Sellier, Jeremy and Dellaportas, Petros},
	month = apr,
	year = {2023},
	note = {ISSN: 2640-3498},
	pages = {2769--2791},
}

@inproceedings{rahimi_random_2008,
	title = {Random {Features} for {Large}-{Scale} {Kernel} {Machines}},
	volume = {20},
	url = {https://papers.nips.cc/paper/2007/hash/013a006f03dbc5392effeb8f18fda755-Abstract.html},
	urldate = {2021-10-01},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	author = {Rahimi, Ali and Recht, Benjamin},
	year = {2008},
}

@inproceedings{ober_promises_2021,
	title = {The promises and pitfalls of deep kernel learning},
	abstract = {Deep kernel learning and related techniques promise to combine the representational power of neural networks with the reliable uncertainty estimates of Gaussian processes. One crucial aspect of these models is an expectation that, because they are treated as Gaussian process models optimized using the marginal likelihood, they are protected from overfitting. However, we identify pathological behavior, including overfitting, on a simple toy example. We explore this pathology, explaining its origins and considering how it applies to real datasets. Through careful experimentation on UCI datasets, CIFAR-10, and the UTKFace dataset, we find that the overfitting from overparameterized deep kernel learning, in which the model is “somewhat Bayesian”, can in certain scenarios be worse than that from not being Bayesian at all. However, we find that a fully Bayesian treatment of deep kernel learning can rectify this overfitting and obtain the desired performance improvements over standard neural networks and Gaussian processes.},
	language = {en},
	urldate = {2023-10-17},
	booktitle = {Proceedings of the {Thirty}-{Seventh} {Conference} on {Uncertainty} in {Artificial} {Intelligence}},
	author = {Ober, Sebastian W. and Rasmussen, Carl E. and Wilk, Mark van der},
	month = dec,
	year = {2021},
	note = {ISSN: 2640-3498},
	pages = {1206--1216},
}

@article{lazaro-gredilla_sparse_2010,
	title = {Sparse {Spectrum} {Gaussian} {Process} {Regression}},
	volume = {11},
	url = {http://jmlr.org/papers/v11/lazaro-gredilla10a.html},
	abstract = {We present a new sparse Gaussian Process (GP) model for
regression. The key novel idea is to sparsify the spectral
  representation of the GP. This leads to a simple, practical
algorithm for regression tasks. We compare the achievable trade-offs
between predictive accuracy and computational requirements, and show
that these are typically superior to existing state-of-the-art
sparse approximations. We discuss both the weight space and function
space representations, and note that the new construction implies
priors over functions which are always stationary, and can
approximate any covariance function in this class.},
	number = {63},
	urldate = {2021-06-28},
	journal = {Journal of Machine Learning Research},
	author = {Lázaro-Gredilla, Miguel and Quiñnero-Candela, Joaquin and Rasmussen, Carl E. and Figueiras-Vidal, Aníbal R.},
	year = {2010},
	pages = {1865--1881},
}

@book{rasmussen_gaussian_2006,
	address = {Cambridge, Mass},
	series = {Adaptive computation and machine learning},
	title = {Gaussian processes for machine learning},
	isbn = {978-0-262-18253-9},
	publisher = {MIT Press},
	author = {Rasmussen, Carl E. and Williams, Christopher},
	year = {2006},
	note = {OCLC: ocm61285753},
	keywords = {gaussian processes, machine learning},
}

@inproceedings{ghahramani_bayesian_2003,
	title = {Bayesian {Monte} {Carlo}},
	volume = {15},
	url = {https://papers.nips.cc/paper/2002/hash/24917db15c4e37e421866448c9ab23d8-Abstract.html},
	urldate = {2021-12-01},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	author = {Ghahramani, Zoubin and Rasmussen, Carl E.},
	year = {2003},
}

@inproceedings{warren_generalized_2022,
	title = {Generalized {Bayesian} {Quadrature} with {Spectral} {Kernels}},
	abstract = {We present a method based on random Fourier features by which any shift-invariant kernel can be used in an analytically tractable form of Bayesian quadrature.},
	language = {en},
	booktitle = {Proceedings of the 38th {Conference} on {Uncertainty} in {Artificial} {Intelligence}},
	author = {Warren, Houston and Oliveira, Rafael and Ramos, Fabio},
	month = jun,
	year = {2022},
	keywords = {bayesian quadrature, physics-informed learning, random fourier features, simulation},
}

@inproceedings{oliva_bayesian_2016,
	title = {Bayesian {Nonparametric} {Kernel}-{Learning}},
	url = {https://proceedings.mlr.press/v51/oliva16.html},
	abstract = {Kernel methods are ubiquitous tools in machine learning. They have proven to be effective in many domains and tasks. Yet, kernel methods often require the user to select a predefined kernel to build an estimator with. However, there is often little reason for the common practice of selecting a kernel a priori. Even if a universal approximating kernel is selected, the quality of the finite sample estimator may be greatly affected by the choice of kernel. Furthermore, when directly applying kernel methods, one typically needs to compute a N {\textbackslash}times N Gram matrix of pairwise kernel evaluations to work with a dataset of N instances. The computation of this Gram matrix precludes the direct application of kernel methods on large datasets, and makes kernel learning especially difficult. In this paper we introduce Bayesian nonparmetric kernel-learning (BaNK), a generic, data-driven framework for scalable learning of kernels. BaNK places a nonparametric prior on the spectral distribution of random frequencies allowing it to both learn kernels and scale to large datasets. We show that this framework can be used for large scale regression and classification tasks. Furthermore, we show that BaNK outperforms several other scalable approaches for kernel learning on a variety of real world datasets.},
	language = {en},
	urldate = {2021-10-14},
	booktitle = {Proceedings of the 19th {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Oliva, Junier B. and Dubey, Avinava and Wilson, Andrew G. and Poczos, Barnabas and Schneider, Jeff and Xing, Eric P.},
	month = may,
	year = {2016},
	note = {ISSN: 1938-7228},
	pages = {1078--1086},
}

@article{he_bayesian_2020,
	title = {Bayesian {Deep} {Ensembles} via the {Neural} {Tangent} {Kernel}},
	volume = {33},
	url = {https://papers.nips.cc/paper/2020/hash/0b1ec366924b26fc98fa7b71a9c249cf-Abstract.html},
	language = {en},
	journal = {Advances in Neural Information Processing Systems},
	author = {He, Bobby and Lakshminarayanan, Balaji and Teh, Yee Whye},
	year = {2020},
	pages = {1010--1022},
}

@inproceedings{genz_testing_1984,
	title = {Testing multidimensional integration routines},
	isbn = {978-0-444-87570-9},
	urldate = {2023-10-16},
	booktitle = {Proc. of international conference on {Tools}, methods and languages for scientific and engineering computation},
	author = {Genz, Alan},
	month = sep,
	year = {1984},
	pages = {81--94},
}

@article{briol_probabilistic_2019,
	title = {Probabilistic {Integration}: {A} {Role} in {Statistical} {Computation}?},
	volume = {34},
	issn = {0883-4237},
	shorttitle = {Probabilistic {Integration}},
	number = {1},
	journal = {Statistical Science},
	author = {Briol, François-Xavier and Oates, Chris J. and Girolami, Mark and Osborne, Michael A. and Sejdinovic, Dino},
	year = {2019},
	pages = {1--22},
}

@inproceedings{zhu_deep_2021,
	title = {Deep {Fourier} {Kernel} for {Self}-{Attentive} {Point} {Processes}},
	url = {http://proceedings.mlr.press/v130/zhu21b.html},
	abstract = {We present a novel attention-based model for discrete event data to capture complex non-linear temporal dependence structures. We borrow the idea from the attention mechanism and incorporate it in...},
	language = {en},
	urldate = {2021-06-07},
	booktitle = {International {Conference} on {Artificial} {Intelligence} and {Statistics}},
	author = {Zhu, Shixiang and Zhang, Minghe and Ding, Ruyi and Xie, Yao},
	month = mar,
	year = {2021},
	note = {ISSN: 2640-3498},
	pages = {856--864},
}

@techreport{ober_promises_nodate,
	title = {The {Promises} and {Pitfalls} of {Deep} {Kernel} {Learning}},
	url = {http://arxiv.org/abs/2102.12108},
	abstract = {Deep kernel learning (DKL) and related techniques aim to combine the representational power of neural networks with the reliable uncertainty estimates of Gaussian processes. One crucial aspect of these models is an expectation that, because they are treated as Gaussian process models optimized using the marginal likelihood, they are protected from overfitting. However, we identify situations where this is not the case. We explore this behavior, explain its origins and consider how it applies to real datasets. Through careful experimentation on the UCI, CIFAR-10, and the UTKFace datasets, we find that the overfitting from overparameterized maximum marginal likelihood, in which the model is "somewhat Bayesian", can in certain scenarios be worse than that from not being Bayesian at all. We explain how and when DKL can still be successful by investigating optimization dynamics. We also find that failures of DKL can be rectified by a fully Bayesian treatment, which leads to the desired performance improvements over standard neural networks and Gaussian processes.},
	number = {arXiv:2102.12108},
	urldate = {2022-05-31},
	institution = {arXiv},
	author = {Ober, Sebastian W. and Rasmussen, Carl Edward and van der Wilk, Mark},
	doi = {10.48550/arXiv.2102.12108},
	note = {arXiv:2102.12108 [cs, stat]
type: article},
	keywords = {machine learning},
}

@article{robert_accelerating_2018,
	title = {Accelerating {MCMC} algorithms},
	volume = {10},
	issn = {1939-5108},
	url = {https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6108397/},
	doi = {10.1002/wics.1435},
	abstract = {Markov chain Monte Carlo algorithms are used to simulate from complex statistical distributions by way of a local exploration of these distributions. This local feature avoids heavy requests on understanding the nature of the target, but it also potentially induces a lengthy exploration of this target, with a requirement on the number of simulations that grows with the dimension of the problem and with the complexity of the data behind it. Several techniques are available toward accelerating the convergence of these Monte Carlo algorithms, either at the exploration level (as in tempering, Hamiltonian Monte Carlo and partly deterministic methods) or at the exploitation level (with Rao–Blackwellization and scalable methods)., This article is categorized under:
Statistical and Graphical Methods of Data Analysis {\textgreater} Markov Chain Monte Carlo (MCMC)Algorithms and Computational Methods {\textgreater} AlgorithmsStatistical and Graphical Methods of Data Analysis {\textgreater} Monte Carlo Methods},
	number = {5},
	urldate = {2023-05-17},
	journal = {Wiley Interdisciplinary Reviews. Computational Statistics},
	author = {Robert, Christian P. and Elvira, Víctor and Tawn, Nick and Wu, Changye},
	year = {2018},
}

@inproceedings{melkumyan_sparse_2009,
	address = {San Francisco, CA, USA},
	series = {{IJCAI}'09},
	title = {A sparse covariance function for exact {Gaussian} process inference in large datasets},
	abstract = {Despite the success of Gaussian processes (GPs) in modelling spatial stochastic processes, dealing with large datasets is still challenging. The problem arises by the need to invert a potentially large covariance matrix during inference. In this paper we address the complexity problem by constructing a new stationary covariance function (Mercer kernel) that naturally provides a sparse covariance matrix. The sparseness of the matrix is defined by hyperparameters optimised during learning. The new covariance function enables exact GP inference and performs comparatively to the squared-exponential one, at a lower computational cost. This allows the application of GPs to large-scale problems such as ore grade prediction in mining or 3D surface modelling. Experiments show that using the proposed covariance function, very sparse covariance matrices are normally obtained which can be effectively used for faster inference and less memory usage.},
	urldate = {2023-10-05},
	booktitle = {Proceedings of the 21st {International} {Joint} {Conference} on {Artificial} {Intelligence}},
	publisher = {Morgan Kaufmann Publishers Inc.},
	author = {Melkumyan, Arman and Ramos, Fabio},
	month = jul,
	year = {2009},
	pages = {1936--1942},
}

@inproceedings{wang_stein_2019,
	title = {Stein {Variational} {Gradient} {Descent} {With} {Matrix}-{Valued} {Kernels}},
	volume = {32},
	url = {https://proceedings.neurips.cc/paper/2019/hash/5dcd0ddd3d918c70d380d32bce4e733a-Abstract.html},
	abstract = {Stein variational gradient descent (SVGD) is a particle-based inference algorithm that  leverages gradient information for efficient approximate inference. In this work, we enhance SVGD by leveraging preconditioning matrices, such as the Hessian and Fisher information matrix, to incorporate geometric information into SVGD updates. We achieve this by presenting a generalization of SVGD that replaces the scalar-valued kernels in vanilla SVGD with more general matrix-valued kernels. This yields a significant extension of SVGD, and more importantly, allows us to flexibly incorporate various preconditioning matricesto accelerate the exploration in the probability landscape. Empirical results show that our method outperforms vanilla SVGD and a variety of baseline approaches over a range of real-world Bayesian inference tasks.},
	urldate = {2023-09-26},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Wang, Dilin and Tang, Ziyang and Bajaj, Chandrajit and Liu, Qiang},
	year = {2019},
}

@inproceedings{yu_orthogonal_2016,
	title = {Orthogonal {Random} {Features}},
	volume = {29},
	url = {https://papers.nips.cc/paper_files/paper/2016/hash/53adaf494dc89ef7196d73636eb2451b-Abstract.html},
	urldate = {2023-09-25},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Yu, Felix Xinnan X and Suresh, Ananda Theertha and Choromanski, Krzysztof M and Holtmann-Rice, Daniel N and Kumar, Sanjiv},
	year = {2016},
}

@inproceedings{yang_nystrom_2012,
	title = {Nyström {Method} vs {Random} {Fourier} {Features}: {A} {Theoretical} and {Empirical} {Comparison}},
	volume = {25},
	shorttitle = {Nyström {Method} vs {Random} {Fourier} {Features}},
	url = {https://proceedings.neurips.cc/paper_files/paper/2012/hash/621bf66ddb7c962aa0d22ac97d69b793-Abstract.html},
	abstract = {Both random Fourier features and the Nyström method have been successfully applied to efficient kernel learning. In this work, we investigate the fundamental difference between these two approaches, and how the difference could affect their generalization performances. Unlike approaches based on random Fourier features  where the basis functions (i.e., cosine and sine functions) are sampled from a distribution  \{{\textbackslash}it independent\} from the training data, basis functions used by the Nyström method are randomly sampled from the training examples and are therefore \{{\textbackslash}it data dependent\}. By exploring this difference, we show that when there is a large gap in the eigen-spectrum of the kernel matrix, approaches based the Nyström method can yield  impressively  better generalization error bound than random Fourier features based approach. We empirically verify our theoretical findings on a wide range of large data sets.},
	urldate = {2023-09-25},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Yang, Tianbao and Li, Yu-feng and Mahdavi, Mehrdad and Jin, Rong and Zhou, Zhi-Hua},
	year = {2012},
}

@misc{feng_learning_2017,
	title = {Learning to {Draw} {Samples} with {Amortized} {Stein} {Variational} {Gradient} {Descent}},
	url = {http://arxiv.org/abs/1707.06626},
	doi = {10.48550/arXiv.1707.06626},
	abstract = {We propose a simple algorithm to train stochastic neural networks to draw samples from given target distributions for probabilistic inference. Our method is based on iteratively adjusting the neural network parameters so that the output changes along a Stein variational gradient direction (Liu \& Wang, 2016) that maximally decreases the KL divergence with the target distribution. Our method works for any target distribution specified by their unnormalized density function, and can train any black-box architectures that are differentiable in terms of the parameters we want to adapt. We demonstrate our method with a number of applications, including variational autoencoder (VAE) with expressive encoders to model complex latent space structures, and hyper-parameter learning of MCMC samplers that allows Bayesian inference to adaptively improve itself when seeing more data.},
	urldate = {2023-09-19},
	publisher = {arXiv},
	author = {Feng, Yihao and Wang, Dilin and Liu, Qiang},
	month = oct,
	year = {2017},
	note = {arXiv:1707.06626 [stat]},
	keywords = {Statistics - Machine Learning},
}

@misc{holmgren_pvlibpvlib-python_2023,
	title = {pvlib/pvlib-python: v0.10.1},
	copyright = {Open Access},
	shorttitle = {pvlib/pvlib-python},
	url = {https://zenodo.org/record/8117210},
	abstract = {https://pvlib-python.readthedocs.io/en/v0.10.1/whatsnew.html},
	urldate = {2023-08-25},
	publisher = {Zenodo},
	author = {Holmgren, Will and Anderson, Kevin and Hansen, Cliff and {Calama-Consulting} and Mikofski, Mark and Lorenzo, Antonio and Jensen, Adam R. and Krien, Uwe and {Bmu} and Stark, Cameron and Driesse, Anton and {DaCoEx} and De León Peque, Miguel Sánchez and {Kt} and {Naman Priyadarshi} and {Taos Transue} and {Mayudong} and {Heliolytics} and Miller, Ed and Anoma, Marc A. and Guo, Veronica and Boeman, Leland and Stein, Joshua and Vining, Will and {Jforbess} and Lunel, Tanguy and Leroy, Cedric and Morgan, Alexander and {Echedey Luis} and Ranalli, Joe},
	month = jul,
	year = {2023},
	doi = {10.5281/ZENODO.8117210},
}

@misc{nguyen_climax_2023,
	title = {{ClimaX}: {A} foundation model for weather and climate},
	shorttitle = {{ClimaX}},
	url = {http://arxiv.org/abs/2301.10343},
	doi = {10.48550/arXiv.2301.10343},
	abstract = {Most state-of-the-art approaches for weather and climate modeling are based on physics-informed numerical models of the atmosphere. These approaches aim to model the non-linear dynamics and complex interactions between multiple variables, which are challenging to approximate. Additionally, many such numerical models are computationally intensive, especially when modeling the atmospheric phenomenon at a fine-grained spatial and temporal resolution. Recent data-driven approaches based on machine learning instead aim to directly solve a downstream forecasting or projection task by learning a data-driven functional mapping using deep neural networks. However, these networks are trained using curated and homogeneous climate datasets for specific spatiotemporal tasks, and thus lack the generality of numerical models. We develop and demonstrate ClimaX, a flexible and generalizable deep learning model for weather and climate science that can be trained using heterogeneous datasets spanning different variables, spatio-temporal coverage, and physical groundings. ClimaX extends the Transformer architecture with novel encoding and aggregation blocks that allow effective use of available compute while maintaining general utility. ClimaX is pre-trained with a self-supervised learning objective on climate datasets derived from CMIP6. The pre-trained ClimaX can then be fine-tuned to address a breadth of climate and weather tasks, including those that involve atmospheric variables and spatio-temporal scales unseen during pretraining. Compared to existing data-driven baselines, we show that this generality in ClimaX results in superior performance on benchmarks for weather forecasting and climate projections, even when pretrained at lower resolutions and compute budgets. The source code is available at https://github.com/microsoft/ClimaX.},
	urldate = {2023-08-17},
	publisher = {arXiv},
	author = {Nguyen, Tung and Brandstetter, Johannes and Kapoor, Ashish and Gupta, Jayesh K. and Grover, Aditya},
	month = jul,
	year = {2023},
	note = {arXiv:2301.10343 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
}

@misc{muller_transformers_2023,
	title = {Transformers {Can} {Do} {Bayesian} {Inference}},
	url = {http://arxiv.org/abs/2112.10510},
	doi = {10.48550/arXiv.2112.10510},
	abstract = {Currently, it is hard to reap the benefits of deep learning for Bayesian methods, which allow the explicit specification of prior knowledge and accurately capture model uncertainty. We present Prior-Data Fitted Networks (PFNs). PFNs leverage large-scale machine learning techniques to approximate a large set of posteriors. The only requirement for PFNs to work is the ability to sample from a prior distribution over supervised learning tasks (or functions). Our method restates the objective of posterior approximation as a supervised classification problem with a set-valued input: it repeatedly draws a task (or function) from the prior, draws a set of data points and their labels from it, masks one of the labels and learns to make probabilistic predictions for it based on the set-valued input of the rest of the data points. Presented with a set of samples from a new supervised learning task as input, PFNs make probabilistic predictions for arbitrary other data points in a single forward propagation, having learned to approximate Bayesian inference. We demonstrate that PFNs can near-perfectly mimic Gaussian processes and also enable efficient Bayesian inference for intractable problems, with over 200-fold speedups in multiple setups compared to current methods. We obtain strong results in very diverse areas such as Gaussian process regression, Bayesian neural networks, classification for small tabular data sets, and few-shot image classification, demonstrating the generality of PFNs. Code and trained PFNs are released at https://github.com/automl/TransformersCanDoBayesianInference.},
	urldate = {2023-07-12},
	publisher = {arXiv},
	author = {Müller, Samuel and Hollmann, Noah and Arango, Sebastian Pineda and Grabocka, Josif and Hutter, Frank},
	month = feb,
	year = {2023},
	note = {arXiv:2112.10510 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@inproceedings{liang_transformer_2022,
	title = {Transformer {Adapters} for {Robot} {Learning}},
	url = {https://openreview.net/forum?id=H--wvRYBmF},
	abstract = {Large transformer-based architectures are capable of complex robot task planning and low-level control. In the natural language processing (NLP) community, fine-tuning large pretrained models (PTMs) such as GPT-3 and PaLM is the de-facto standard. With the scalability of transformer models and growing availability of large-scale multimodal robot data, we investigate pretraining large backbone models to capture useful behavioral priors that enable efficient few-shot transfer to downstream robot tasks. We explore the setting of modular reinforcement learning (RL) in which each downstream task is encapsulated by an independently learned module. With many downstream tasks, fine-tuning or training separate copies of these large PTMs become computationally and memory intensive. We propose to pretrain a large transformer backbone on task-agnostic data and learn small task-specific adapters using few-shot imitation learning to quickly adapt to downstream tasks. We evaluate on complex robot manipulation tasks in the Metaworld environment and demonstrate that adapter training is a parameter-efficient approach for modular RL.},
	language = {en},
	urldate = {2023-06-28},
	author = {Liang, Anthony and Singh, Ishika and Pertsch, Karl and Thomason, Jesse},
	month = nov,
	year = {2022},
}

@misc{chen_decision_2021,
	title = {Decision {Transformer}: {Reinforcement} {Learning} via {Sequence} {Modeling}},
	shorttitle = {Decision {Transformer}},
	url = {http://arxiv.org/abs/2106.01345},
	doi = {10.48550/arXiv.2106.01345},
	abstract = {We introduce a framework that abstracts Reinforcement Learning (RL) as a sequence modeling problem. This allows us to draw upon the simplicity and scalability of the Transformer architecture, and associated advances in language modeling such as GPT-x and BERT. In particular, we present Decision Transformer, an architecture that casts the problem of RL as conditional sequence modeling. Unlike prior approaches to RL that fit value functions or compute policy gradients, Decision Transformer simply outputs the optimal actions by leveraging a causally masked Transformer. By conditioning an autoregressive model on the desired return (reward), past states, and actions, our Decision Transformer model can generate future actions that achieve the desired return. Despite its simplicity, Decision Transformer matches or exceeds the performance of state-of-the-art model-free offline RL baselines on Atari, OpenAI Gym, and Key-to-Door tasks.},
	urldate = {2023-06-28},
	publisher = {arXiv},
	author = {Chen, Lili and Lu, Kevin and Rajeswaran, Aravind and Lee, Kimin and Grover, Aditya and Laskin, Michael and Abbeel, Pieter and Srinivas, Aravind and Mordatch, Igor},
	month = jun,
	year = {2021},
	note = {arXiv:2106.01345 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
}

@misc{he_towards_2022,
	title = {Towards a {Unified} {View} of {Parameter}-{Efficient} {Transfer} {Learning}},
	url = {http://arxiv.org/abs/2110.04366},
	doi = {10.48550/arXiv.2110.04366},
	abstract = {Fine-tuning large pre-trained language models on downstream tasks has become the de-facto learning paradigm in NLP. However, conventional approaches fine-tune all the parameters of the pre-trained model, which becomes prohibitive as the model size and the number of tasks grow. Recent work has proposed a variety of parameter-efficient transfer learning methods that only fine-tune a small number of (extra) parameters to attain strong performance. While effective, the critical ingredients for success and the connections among the various methods are poorly understood. In this paper, we break down the design of state-of-the-art parameter-efficient transfer learning methods and present a unified framework that establishes connections between them. Specifically, we re-frame them as modifications to specific hidden states in pre-trained models, and define a set of design dimensions along which different methods vary, such as the function to compute the modification and the position to apply the modification. Through comprehensive empirical studies across machine translation, text summarization, language understanding, and text classification benchmarks, we utilize the unified view to identify important design choices in previous methods. Furthermore, our unified framework enables the transfer of design elements across different approaches, and as a result we are able to instantiate new parameter-efficient fine-tuning methods that tune less parameters than previous methods while being more effective, achieving comparable results to fine-tuning all parameters on all four tasks.},
	urldate = {2023-06-21},
	publisher = {arXiv},
	author = {He, Junxian and Zhou, Chunting and Ma, Xuezhe and Berg-Kirkpatrick, Taylor and Neubig, Graham},
	month = feb,
	year = {2022},
	note = {arXiv:2110.04366 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@misc{hu_lora_2021,
	title = {{LoRA}: {Low}-{Rank} {Adaptation} of {Large} {Language} {Models}},
	shorttitle = {{LoRA}},
	url = {http://arxiv.org/abs/2106.09685},
	doi = {10.48550/arXiv.2106.09685},
	abstract = {An important paradigm of natural language processing consists of large-scale pre-training on general domain data and adaptation to particular tasks or domains. As we pre-train larger models, full fine-tuning, which retrains all model parameters, becomes less feasible. Using GPT-3 175B as an example -- deploying independent instances of fine-tuned models, each with 175B parameters, is prohibitively expensive. We propose Low-Rank Adaptation, or LoRA, which freezes the pre-trained model weights and injects trainable rank decomposition matrices into each layer of the Transformer architecture, greatly reducing the number of trainable parameters for downstream tasks. Compared to GPT-3 175B fine-tuned with Adam, LoRA can reduce the number of trainable parameters by 10,000 times and the GPU memory requirement by 3 times. LoRA performs on-par or better than fine-tuning in model quality on RoBERTa, DeBERTa, GPT-2, and GPT-3, despite having fewer trainable parameters, a higher training throughput, and, unlike adapters, no additional inference latency. We also provide an empirical investigation into rank-deficiency in language model adaptation, which sheds light on the efficacy of LoRA. We release a package that facilitates the integration of LoRA with PyTorch models and provide our implementations and model checkpoints for RoBERTa, DeBERTa, and GPT-2 at https://github.com/microsoft/LoRA.},
	urldate = {2023-06-14},
	publisher = {arXiv},
	author = {Hu, Edward J. and Shen, Yelong and Wallis, Phillip and Allen-Zhu, Zeyuan and Li, Yuanzhi and Wang, Shean and Wang, Lu and Chen, Weizhu},
	month = oct,
	year = {2021},
	note = {arXiv:2106.09685 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@misc{honsberg_photovoltaics_2019,
	title = {Photovoltaics {Education} {Website}},
	url = {www.pveducation.org},
	author = {Honsberg, C.B. and Bowden, S.G},
	year = {2019},
}

@article{bouraiou_field_2022,
	title = {Field investigation of {PV} pumping system ageing failures operation under {Saharan} environment},
	volume = {243},
	issn = {0038-092X},
	url = {https://www.sciencedirect.com/science/article/pii/S0038092X22005527},
	doi = {10.1016/j.solener.2022.08.005},
	abstract = {Understanding photovoltaic modules' failures and degradation modes is an essential task for improving their useful lifetime and manufacturing materials. This paper focuses on the assessment and analysis of the different PV module defects that occurred in a solar pumping system after a long period of exposure (since 2008) to a harsh (hot and dry) climate. Additional issues that have been observed in these systems are highlighted to enhance their operation and maintenance. A detailed visual inspection of 21 mono-crystalline silicon PV modules revealed that EVA discoloration and metallization (bus bar, finger) corrosion are the most prevalent defects, accounting for 100\% of the total, followed by front side delamination and bubbles at 80\%, and cracking in solar cells at 11.77\%.},
	language = {en},
	urldate = {2023-05-26},
	journal = {Solar Energy},
	author = {Bouraiou, Ahmed and Bekraoui, Amina and Necaibia, Ammar and Rouabhia, Abdelkrim and Boutasseta, Nadir and Khelifi, Seyfallah and Padmanaban, Sanjeevikumar and Khan, Baseem and Bouakkaz, Mohammed Salah and Attoui, Issam and Dabou, Rachid},
	month = sep,
	year = {2022},
	keywords = {Bubbles, Cell cracking, Delamination, EVA discoloration, Hot and dry climate, Metallization corrosion, Photovoltaic (PV) module defect},
	pages = {142--152},
}

@article{leon-garzon_thermodynamic_2018,
	title = {Thermodynamic analysis of the degradation of polyethylene subjected to internal partial discharges},
	volume = {180},
	issn = {0009-2509},
	url = {https://www.sciencedirect.com/science/article/pii/S000925091830023X},
	doi = {10.1016/j.ces.2018.01.023},
	abstract = {The field of electrical insulation has improved considerably after the introduction of the first synthetic polymeric dielectrics. However, these materials are often considered the “weakest link” as they tend to degrade and eventually fail, thus compromising the reliability of devices and equipment. Such degradation is related to the occurrence of partial discharges (PDs), a phenomenon that is difficult to measure in actual applications. In the present work, the chemical evolution of the gas phase composition and polyethylene degradation in a polyethylene microcavity subject to a PD were studied using a thermodynamic model. Thermodynamic parameters for the depolymerization of polyethylene into ethylene were obtained using density functional theory calculations. The system evolution is studied minimizing the Gibbs free energy subject to the constraints of energy conservation explicitly considering the polyethylene surface among the reacting species. The model results can be interpreted in terms of the density of energy introduced by the partial discharge. At low discharge energy densities methane and carbon monoxide are the most abundant species produced by the discharge. As the energy density increases the model predicts that ethylene is formed in the gas phase because of the depolymerization of the polymer. At high energy densities carbon may be formed at the surface. In these conditions, the most abundant gas phase species predicted by the model are methane, carbon monoxide, hydrogen, water, and carbon dioxide. This is in good agreement with experimental observations reported in the polyethylene PD literature. In addition, the model is able to correlate the energy of the partial discharge to the increase of volume of the microcavity, thus providing a tool that may be used to predict polymer degradation due to PDs dissipated power.},
	language = {en},
	urldate = {2023-05-23},
	journal = {Chemical Engineering Science},
	author = {Leon-Garzon, Andres R. and Dotelli, Giovanni and Villa, Andrea and Barbieri, Luca and Gondola, Marco and Cavallotti, Carlo},
	month = apr,
	year = {2018},
	keywords = {Partial discharges, Polyethylene, Polymer degradation, Thermodynamic analysis},
	pages = {1--10},
}

@techreport{freeman_system_2018,
	title = {System {Advisor} {Model} ({SAM}) {General} {Description} ({Version} 2017.9.5)},
	url = {https://www.osti.gov/biblio/1440404},
	abstract = {This document describes the capabilities of the System Advisor Model (SAM) developed and distributed by the U.S. Department of Energy's National Renewable Energy Laboratory. The document is for potential users and others wanting to learn about the model's capabilities. SAM is a techno-economic computer model that calculates performance and financial metrics of renewable energy projects. Project developers, policy makers, equipment manufacturers, and researchers use graphs and tables of SAM results in the process of evaluating financial, technology, and incentive options for renewable energy projects. SAM simulates the performance of photovoltaic, concentrating solar power, solar water heating, wind, geothermal, biomass, and conventional power systems. The financial models are for projects that either buy and sell electricity at retail rates (residential and commercial) or sell electricity at a price determined in a power purchase agreement (PPA). SAM's simulation tools facilitate parametric and sensitivity analyses, Monte Carlo simulation and weather variability (P50/P90) studies. SAM can also read input variables from Microsoft Excel worksheets. For software developers, the SAM software development kit (SDK) makes it possible to use SAM simulation modules in their applications written in C/C plus plus, C sharp, Java, Python, MATLAB, and other languages.},
	language = {English},
	number = {NREL/TP-6A20-70414},
	urldate = {2023-05-23},
	institution = {National Renewable Energy Lab. (NREL), Golden, CO (United States)},
	author = {Freeman, Janine M. and DiOrio, Nicholas A. and Blair, Nathan J. and Neises, Ty W. and Wagner, Michael J. and Gilman, Paul and Janzou, Steven},
	month = may,
	year = {2018},
	doi = {10.2172/1440404},
}

@inproceedings{wilson_deep_2016,
	title = {Deep {Kernel} {Learning}},
	url = {https://proceedings.mlr.press/v51/wilson16.html},
	abstract = {We introduce scalable deep kernels, which combine the structural properties of deep learning architectures with the non-parametric flexibility of kernel methods.  Specifically, we transform the inputs of a spectral mixture base kernel with a deep architecture, using local kernel interpolation, inducing points, and structure exploiting (Kronecker and Toeplitz) algebra for a scalable kernel representation.  These closed-form kernels can be used as drop-in replacements for standard kernels, with benefits in expressive power and scalability.  We jointly learn the properties of these kernels through the marginal likelihood of a Gaussian process.  Inference and learning cost O(n) for n training points, and predictions cost O(1) per test point.  On a large and diverse collection of applications, including a dataset with 2 million examples, we show improved performance over scalable Gaussian processes with flexible kernel learning models, and stand-alone deep architectures.},
	language = {en},
	urldate = {2023-05-17},
	booktitle = {Proceedings of the 19th {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Wilson, Andrew Gordon and Hu, Zhiting and Salakhutdinov, Ruslan and Xing, Eric P.},
	month = may,
	year = {2016},
	note = {ISSN: 1938-7228},
	pages = {370--378},
}

@inproceedings{wilson_gaussian_2013,
	title = {Gaussian {Process} {Kernels} for {Pattern} {Discovery} and {Extrapolation}},
	url = {https://proceedings.mlr.press/v28/wilson13.html},
	abstract = {Gaussian processes are rich distributions over functions, which provide a Bayesian nonparametric approach to smoothing and interpolation.  We introduce simple closed form kernels that can be used with Gaussian processes to discover patterns and enable extrapolation.  These kernels are derived by modelling a spectral density – the Fourier transform of a kernel – with a Gaussian mixture.  The proposed kernels support a broad class of stationary covariances, but Gaussian process inference remains simple and analytic.  We demonstrate the proposed kernels by discovering patterns and performing long range extrapolation on synthetic examples, as well as atmospheric CO2 trends and airline passenger data.  We also show that it is possible to reconstruct several popular standard covariances within our framework.},
	language = {en},
	urldate = {2023-05-17},
	booktitle = {Proceedings of the 30th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Wilson, Andrew and Adams, Ryan},
	month = may,
	year = {2013},
	note = {ISSN: 1938-7228},
	pages = {1067--1075},
}

@misc{li_implicit_2019,
	title = {Implicit {Kernel} {Learning}},
	url = {http://arxiv.org/abs/1902.10214},
	doi = {10.48550/arXiv.1902.10214},
	abstract = {Kernels are powerful and versatile tools in machine learning and statistics. Although the notion of universal kernels and characteristic kernels has been studied, kernel selection still greatly influences the empirical performance. While learning the kernel in a data driven way has been investigated, in this paper we explore learning the spectral distribution of kernel via implicit generative models parametrized by deep neural networks. We called our method Implicit Kernel Learning (IKL). The proposed framework is simple to train and inference is performed via sampling random Fourier features. We investigate two applications of the proposed IKL as examples, including generative adversarial networks with MMD (MMD GAN) and standard supervised learning. Empirically, MMD GAN with IKL outperforms vanilla predefined kernels on both image and text generation benchmarks; using IKL with Random Kitchen Sinks also leads to substantial improvement over existing state-of-the-art kernel learning algorithms on popular supervised learning benchmarks. Theory and conditions for using IKL in both applications are also studied as well as connections to previous state-of-the-art methods.},
	urldate = {2023-05-09},
	publisher = {arXiv},
	author = {Li, Chun-Liang and Chang, Wei-Cheng and Mroueh, Youssef and Yang, Yiming and Póczos, Barnabás},
	month = feb,
	year = {2019},
	note = {arXiv:1902.10214 [cs, stat]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Statistics - Machine Learning},
}

@inproceedings{sinha_learning_2016,
	title = {Learning {Kernels} with {Random} {Features}},
	volume = {29},
	url = {https://papers.nips.cc/paper_files/paper/2016/hash/e70611883d2760c8bbafb4acb29e3446-Abstract.html},
	abstract = {Randomized features provide a computationally efficient way to approximate kernel machines in machine learning tasks. However, such methods require a user-defined kernel as input. We extend the randomized-feature approach to the task of learning a kernel (via its associated random features). Specifically, we present an efficient optimization problem that learns a kernel in a supervised manner. We prove the consistency of the estimated kernel as well as generalization bounds for the class of estimators induced by the optimized kernel, and we experimentally evaluate our technique on several datasets. Our approach is efficient and highly scalable, and we attain competitive results with a fraction of the training cost of other techniques.},
	urldate = {2023-05-09},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Sinha, Aman and Duchi, John C},
	year = {2016},
}

@article{cockayne_bayesian_2019,
	title = {Bayesian {Probabilistic} {Numerical} {Methods}},
	volume = {61},
	issn = {0036-1445},
	url = {https://epubs.siam.org/doi/10.1137/17M1139357},
	doi = {10.1137/17M1139357},
	abstract = {There are deep and important philosophical differences between Bayesian and frequentist approaches to quantifying uncertainty. However, some practitioners choose between these approaches primarily on the basis of convenience. For instance, the ability to incorporate parameter constraints is sometimes cited as a reason to use Bayesian methods. This reflects two misunderstandings: First, frequentist methods can indeed incorporate constraints on parameter values. Second, it ignores the crucial question of what the result of the analysis will mean. Bayesian and frequentist measures of uncertainty have similar sounding names but quite different meanings. For instance, Bayesian uncertainties typically involve expectations with respect to the posterior distribution of the parameter, holding the data fixed; frequentist uncertainties typically involve expectations with respect to the distribution of the data, holding the parameter fixed. Bayesian methods, including methods incorporating parameter constraints, require supplementing the constraints with a prior probability distribution for parameter values. This can cause frequentist and Bayesian estimates and their nominal uncertainties to differ substantially, even when the prior is “uninformative.” This paper gives simple examples where “uninformative” priors are, in fact, extremely informative, and sketches how to measure how much information the prior adds to the constraint. Bayesian methods can have good frequentist behavior, and a frequentist can use Bayesian methods and quantify the uncertainty by frequentist means---but absent a meaningful prior, Bayesian uncertainty measures lack meaning. The paper ends with brief reflections on practice.},
	number = {4},
	urldate = {2023-05-09},
	journal = {SIAM Review},
	author = {Cockayne, Jon and Oates, Chris J. and Sullivan, T. J. and Girolami, Mark},
	month = jan,
	year = {2019},
	note = {Publisher: Society for Industrial and Applied Mathematics},
	pages = {756--789},
}

@inproceedings{karin_photovoltaic_2019,
	title = {Photovoltaic {Degradation} {Climate} {Zones}},
	doi = {10.1109/PVSC40753.2019.8980831},
	abstract = {A large body of previous research indicates that climate affects photovoltaic (PV) degradation both in terms of steady power loss and hazardous failures. However, the geographic distribution of climate stressors has not yet been characterized in a systematic way. Most typically the Köppen-Geiger classification scheme is used for comparing PV degradation across different climates. However, Köppen-Geiger uses temperature and rainfall to develop zones relevant for botany and lacks the ability to distinguish locations based on climate stressors more relevant to PV degradation. Prior work has shown that specific stressors (e.g. high temperature, temperature cycling, damp heat, wind stress and UV exposure) induce multiple PV degradation modes such as solder bond degradation, corrosion by moisture intrusion, wind-induced cell cracking, encapsulant discoloration and others. We introduce a climate zone classification system specific to PV, PhotoVoltaic Climate Zones (PVCZ-2019 or PVCZ) that defines zones based on the geographic distribution in PV stressor intensity. This climate zone scheme provides quantitative thresholds on the climate stress experienced in each zone which can provide a basis for future work on the impact of climate on PV degradation and failure.},
	booktitle = {2019 {IEEE} 46th {Photovoltaic} {Specialists} {Conference} ({PVSC})},
	author = {Karin, Todd and Jones, C. Birk and Jain, Anubhav},
	month = jun,
	year = {2019},
	note = {ISSN: 0160-8371},
	keywords = {Köppen Geiger, PVCZ, climate zone map, climate zones, degradation, photovoltaic},
	pages = {0687--0694},
}

@article{jordan_photovoltaic_2022,
	title = {Photovoltaic fleet degradation insights},
	volume = {30},
	issn = {1099-159X},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/pip.3566},
	doi = {10.1002/pip.3566},
	abstract = {In the PV Fleet Performance Data Initiative, high-frequency data from commercial and utility-scale photovoltaic (PV) systems have been collected to examine performance loss rates (PLRs) at a fleet scale. To date, performance data from more than 7.2-gigawatt (GW) capacity, 1700 sites and 19,000 inverters—approximately equivalent to 6\% to 7\% of the entire US PV market—have been collected. An overall PLR of −0.75\%/year was found, which is in line with historical and recent findings. Tracked silicon (Si) and cadmium telluride (CdTe) performed comparably with all fixed-tilt systems. Higher PLRs were found for hotter temperature zones; cooler climates exhibit a median −0.48\%/year loss, which increases to −0.88\%/year in hotter climates. High-efficiency module technologies showed median PLRs in line with conventional Si technologies but demonstrated markedly different PLR behavior when filtered only for low-light conditions {\textless}600 W/m2. Causes for this technology-dependent behavior are under investigation.},
	language = {en},
	number = {10},
	urldate = {2023-05-07},
	journal = {Progress in Photovoltaics: Research and Applications},
	author = {Jordan, Dirk C. and Anderson, Kevin and Perry, Kirsten and Muller, Matthew and Deceglie, Michael and White, Robert and Deline, Chris},
	year = {2022},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/pip.3566},
	keywords = {degradation rates, durability, outdoor testing, performance, photovoltaic modules, photovoltaic systems, reliability},
	pages = {1166--1175},
}

@article{kaaya_photovoltaic_2020,
	title = {Photovoltaic lifetime forecast model based on degradation patterns},
	volume = {28},
	issn = {1099-159X},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/pip.3280},
	doi = {10.1002/pip.3280},
	abstract = {The ever-growing secondary market of photovoltaic (PV) systems (i.e., the transaction of solar plants ownership) calls for reliable and high-quality long-term PV degradation forecasts to mitigate the financial risks. However, when long-term PV performance degradation forecasts are required after a short time with limited degradation history, the existing physical and data-driven methods often provide unrealistic degradation scenarios. Therefore, we present a new data-driven method to forecast PV lifetime after a small performance degradation of only 3\%. To achieve an accurate and reliable forecast, the developed method addresses the fundamental challenges that usually affect long-term degradation evaluation such as data treatment, choosing a good degradation model, and understanding the different degradation patterns. In the paper, we propose and describe an algorithm for degradation trend evaluation, a new concept of multiple “time- and degradation pattern-dependent” degradation factors. The proposed method has been calibrated and validated using different PV modules and systems data of 5 to 35 years of field exposure. The model has been benchmarked against existing statistical models evaluating 11 experimental PV systems with different technologies. The key advantage of our model over statistical ones is the ability to perform more reliable forecasts with limited degradation history. With an average relative uncertainty of 7.0\%, our model is outstanding in consistency for different forecasting time horizons. Moreover, the model is applicable to all PV technologies. The proposed method will aid in making reliable financial decisions and also in adequately planning operation and maintenance activities.},
	language = {en},
	number = {10},
	urldate = {2023-05-07},
	journal = {Progress in Photovoltaics: Research and Applications},
	author = {Kaaya, Ismail and Lindig, Sascha and Weiss, Karl-Anders and Virtuani, Alessandro and Sidrach de Cardona Ortin, Mariano and Moser, David},
	year = {2020},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/pip.3280},
	keywords = {PV modules, PV systems, degradation, degradation patterns, forecast, lifetime},
	pages = {979--992},
}

@article{phinikarides_review_2014,
	title = {Review of photovoltaic degradation rate methodologies},
	volume = {40},
	issn = {1364-0321},
	url = {https://www.sciencedirect.com/science/article/pii/S1364032114006078},
	doi = {10.1016/j.rser.2014.07.155},
	abstract = {This paper provides a review of methodologies for measuring the degradation rate, RD, of photovoltaic (PV) technologies, as reported in the literature. As presented in this paper, each method yields different results with varying uncertainty depending on the measuring equipment, the data qualification and filtering criteria, the performance metric and the statistical method of estimation of the trend. This imposes the risk of overestimating or underestimating the true degradation rate and, subsequently, the effective lifetime of a PV module/array/system and proves the need for defining a standardized methodology. Through a literature search, four major statistical analysis methods were recognized for calculating degradation rates: (1) Linear Regression (LR), (2) Classical Seasonal Decomposition (CSD), (3) AutoRegressive Integrated Moving Average (ARIMA) and, (4) LOcally wEighted Scatterplot Smoothing (LOESS), with LR being the most common. These analyses were applied on the following performance metrics: (1) electrical parameters from IV curves recorded under outdoor or simulated indoor conditions and corrected to STC, (2) regression models such as the Photovoltaics for Utility Scale Applications (PVUSA) and Sandia models, (3) normalized ratings such as Performance Ratio, RP, and PMPP/GI and, (4) scaled ratings such as PMPP/Pmax, PAC/Pmax and kWh/kWp. The degradation rate results have shown that the IV method produced the lowest RD and LR produced results with large variation and the largest uncertainty. The ARIMA and LOESS methods, albeit less popular, produced results with low variation and uncertainty and with good agreement between them. Most importantly, this review showed that the RD is not only technology and site dependent, but also methodology dependent.},
	language = {en},
	urldate = {2023-05-07},
	journal = {Renewable and Sustainable Energy Reviews},
	author = {Phinikarides, Alexander and Kindyni, Nitsa and Makrides, George and Georghiou, George E.},
	month = dec,
	year = {2014},
	keywords = {Degradation, Durability, Photovoltaic, Statistical analysis, Weathering},
	pages = {143--152},
}

@article{lindig_review_2018,
	title = {Review of {Statistical} and {Analytical} {Degradation} {Models} for {Photovoltaic} {Modules} and {Systems} as {Well} as {Related} {Improvements}},
	volume = {8},
	issn = {2156-3403},
	doi = {10.1109/JPHOTOV.2018.2870532},
	abstract = {In this work, we investigate practical approaches of available degradation models and their usage in photovoltaic (PV) modules and systems. On the one hand, degradation prediction of models is described for the calculation of degradation at system level where the degradation mode is unknown and hence the physics cannot be included by the use of analytical models. Several statistical models are thus described and applied for the calculation of the performance loss using as case study two PV systems, installed in Bolzano/Italy. Namely, simple linear regression (SLR), classical seasonal-decomposition, seasonal- and trend-decomposition using Loess (STL), Holt-Winters exponential smoothing and autoregressive integrated moving average (ARIMA) are discussed. The performance loss results show that SLR produces results with highest uncertainties. In comparison, STL and ARIMA perform with the highest accuracy, whereby STL is favored because of its easier implementation. On the other hand, if monitoring data at PV module level are available in controlled conditions, analytical models can be applied. Several analytical models depending on different degradations modes are thus discussed. A comparison study is carried out for models proposed for corrosion. Although the results of the models in question agree in explanation of experimental observations, a big difference in degradation prediction was observed. Finally, a model proposed for potential induced degradation was applied to simulate the degradation of PV systems maximum power in three climatic zones: alpine (Zugspitze, Germany), maritime (Gran Canaria, Spain), and arid (Negev, Israel). As expected, a more severe degradation is predicted for arid climates.},
	number = {6},
	journal = {IEEE Journal of Photovoltaics},
	author = {Lindig, Sascha and Kaaya, Ismail and Weiss, Karl-Anders and Moser, David and Topic, Marko},
	month = nov,
	year = {2018},
	note = {Conference Name: IEEE Journal of Photovoltaics},
	keywords = {Analytical models, Data models, Degradation, Degradation models, Measurement, PV systems, Photovoltaic systems, performance loss, photovoltaic (PV) modules, service life prediction},
	pages = {1773--1786},
}

@inproceedings{brault_random_2016,
	title = {Random {Fourier} {Features} {For} {Operator}-{Valued} {Kernels}},
	url = {https://proceedings.mlr.press/v63/Brault39.html},
	abstract = {Devoted to multi-task learning and structured output learning, operator-valued kernels provide a flexible tool to build vector-valued functions in the context of Reproducing Kernel Hilbert Spaces. To scale up these methods, we extend the celebrated Random Fourier Feature methodology to get an approximation of operator-valued kernels. We propose a general principle for Operator-valued Random Fourier Feature construction relying on a generalization of Bochner’s theorem for translation-invariant operator-valued Mercer kernels. We prove the uniform convergence of the kernel approximation for bounded and unbounded operator random Fourier features using appropriate Bernstein matrix concentration inequality. An experimental proof-of-concept shows the quality of the approximation and the efficiency of the corresponding linear models on example datasets.},
	language = {en},
	urldate = {2023-05-06},
	booktitle = {Proceedings of {The} 8th {Asian} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Brault, Romain and Heinonen, Markus and Buc, Florence},
	month = nov,
	year = {2016},
	note = {ISSN: 1938-7228},
	pages = {110--125},
}

@inproceedings{xi_bayesian_2018,
	title = {Bayesian {Quadrature} for {Multiple} {Related} {Integrals}},
	url = {https://proceedings.mlr.press/v80/xi18a.html},
	abstract = {Bayesian probabilistic numerical methods are a set of tools providing posterior distributions on the output of numerical methods. The use of these methods is usually motivated by the fact that they can represent our uncertainty due to incomplete/finite information about the continuous mathematical problem being approximated. In this paper, we demonstrate that this paradigm can provide additional advantages, such as the possibility of transferring information between several numerical methods. This allows users to represent uncertainty in a more faithful manner and, as a by-product, provide increased numerical efficiency. We propose the first such numerical method by extending the well-known Bayesian quadrature algorithm to the case where we are interested in computing the integral of several related functions. We then prove convergence rates for the method in the well-specified and misspecified cases, and demonstrate its efficiency in the context of multi-fidelity models for complex engineering systems and a problem of global illumination in computer graphics.},
	language = {en},
	urldate = {2023-05-06},
	booktitle = {Proceedings of the 35th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Xi, Xiaoyue and Briol, Francois-Xavier and Girolami, Mark},
	month = jul,
	year = {2018},
	note = {ISSN: 2640-3498},
	pages = {5373--5382},
}

@article{jordan_photovoltaic_2013,
	title = {Photovoltaic {Degradation} {Rates}—an {Analytical} {Review}},
	volume = {21},
	issn = {1099-159X},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/pip.1182},
	doi = {10.1002/pip.1182},
	abstract = {As photovoltaic penetration of the power grid increases, accurate predictions of return on investment require accurate prediction of decreased power output over time. Degradation rates must be known in order to predict power delivery. This article reviews degradation rates of flat-plate terrestrial modules and systems reported in published literature from field testing throughout the last 40 years. Nearly 2000 degradation rates, measured on individual modules or entire systems, have been assembled from the literature, showing a median value of 0·5\%/year. The review consists of three parts: a brief historical outline, an analytical summary of degradation rates, and a detailed bibliography partitioned by technology. Copyright © 2011 John Wiley \& Sons, Ltd.},
	language = {en},
	number = {1},
	urldate = {2023-05-05},
	journal = {Progress in Photovoltaics: Research and Applications},
	author = {Jordan, D. C. and Kurtz, S. R.},
	year = {2013},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/pip.1182},
	keywords = {degradation rates, field testing, outdoor testing, performance, photovoltaic modules, photovoltaic systems},
	pages = {12--29},
}

@article{de_soto_improvement_2006,
	title = {Improvement and validation of a model for photovoltaic array performance},
	volume = {80},
	issn = {0038-092X},
	url = {https://www.sciencedirect.com/science/article/pii/S0038092X05002410},
	doi = {10.1016/j.solener.2005.06.010},
	abstract = {Manufacturers of photovoltaic panels typically provide electrical parameters at only one operating condition. Photovoltaic panels operate over a large range of conditions so the manufacturer’s information is not sufficient to determine their overall performance. Designers need a reliable tool to predict energy production from a photovoltaic panel under all conditions in order to make a sound decision on whether or not to incorporate this technology. A model to predict energy production has been developed by Sandia National Laboratory, but it requires input data that are normally not available from the manufacturer. The five-parameter model described in this paper uses data provided by the manufacturer, absorbed solar radiation and cell temperature together with semi-empirical equations, to predict the current–voltage curve. This paper indicates how the parameters of the five-parameter model are determined and compares predicted current–voltage curves with experimental data from a building integrated photovoltaic facility at the National Institute of Standards and Technology (NIST) for four different cell technologies (single crystalline, poly crystalline, silicon thin film, and triple-junction amorphous). The results obtained with the Sandia model are also shown. The predictions from the five-parameter model are shown to agree well with both the Sandia model results and the NIST measurements for all four cell types over a range of operating conditions. The five-parameter model is of interest because it requires only a small amount of input data available from the manufacturer and therefore it provides a valuable tool for energy prediction. The predictive capability could be improved if manufacturer’s data included information at two radiation levels.},
	language = {en},
	number = {1},
	urldate = {2023-04-19},
	journal = {Solar Energy},
	author = {De Soto, W. and Klein, S. A. and Beckman, W. A.},
	month = jan,
	year = {2006},
	keywords = {PV cells, Performance, Photovoltaic cells, Prediction, Solar energy, – curves},
	pages = {78--88},
}

@inproceedings{qian_pv_2017,
	title = {A {PV} {Module} {Current} {Mismatch} {Simulation} {Model} and {Application} to {Bifacial} {Modules}},
	url = {https://www.semanticscholar.org/paper/A-PV-Module-Current-Mismatch-Simulation-Model-and-Qian-Thomson/b951e73e11e52736f91a6801f5d2194db9509186},
	abstract = {Modelling the impact of mismatch and shading on PV module output is computationally difficult. The one diode equation for a single cell including series resistances is numerically solved. Then considering a string of solar cells, or modules, if there is any variation in the short circuit current, the impact of reverse bias and break down diodes must be considered. Hence to model such setups one must employ a software package such as LTspice to simulate the equivalent circuit. We have built a fast and robust photovoltaic (PV) module mismatch/shading simulation model which incorporates PV cell’s forward and reverse bias behavior and involves the avalanche breakdown behavior and bypass diodes. To achieve timely execution of our simulations, we employ the explicit solutions to cell’s voltage as a function of current that are derived for forward bias model using Lambert W function; and for reverse bias model using the General Solution for quartic equation. An accurate approximation of Lambert W function is used to improve computational practicality and enable implementation in software without Lambert W function capability. An example of simulation results presented with both module I-V curves and decomposed power dissipation from weak cell. A Microsoft Excel spreadsheet with simulation model implemented is also provided. We also provide simulations results of the impact of non-uniform irradiance for bi-facial modules on energy yield.},
	urldate = {2023-04-03},
	author = {Qian, Jiadong and Thomson, A. and Blakers, A. and Tokusato, Elissa and Ernst, M. and Haedrich, I.},
	year = {2017},
}

@article{jordan_photovoltaic_2017,
	title = {Photovoltaic failure and degradation modes},
	volume = {25},
	issn = {1099-159X},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/pip.2866},
	doi = {10.1002/pip.2866},
	abstract = {The extensive photovoltaic field reliability literature was analyzed and reviewed. Future work is prioritized based upon information assembled from recent installations, and inconsistencies in degradation mode identification are discussed to help guide future publication on this subject. Reported failure rates of photovoltaic modules fall mostly in the range of other consumer products; however, the long expected useful life of modules may not allow for direct comparison. In general, degradation percentages are reported to decrease appreciably in newer installations that are deployed after the year 2000. However, these trends may be convoluted with varying manufacturing and installation quality world-wide. Modules in hot and humid climates show considerably higher degradation modes than those in desert and moderate climates, which warrants further investigation. Delamination and diode/j-box issues are also more frequent in hot and humid climates than in other climates. The highest concerns of systems installed in the last 10 years appear to be hot spots followed by internal circuitry discoloration. Encapsulant discoloration was the most common degradation mode, particularly in older systems. In newer systems, encapsulant discoloration appears in hotter climates, but to a lesser degree. Thin-film degradation modes are dominated by glass breakage and absorber corrosion, although the breadth of information for thin-film modules is much smaller than for x-Si. Copyright © 2017 John Wiley \& Sons, Ltd.},
	language = {en},
	number = {4},
	urldate = {2023-03-01},
	journal = {Progress in Photovoltaics: Research and Applications},
	author = {Jordan, Dirk C. and Silverman, Timothy J. and Wohlgemuth, John H. and Kurtz, Sarah R. and VanSant, Kaitlyn T.},
	year = {2017},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/pip.2866},
	keywords = {degradation modes, durability, failure, failure rate, photovoltaic modules, reliability},
	pages = {318--326},
}

@article{jordan_compendium_2016,
	title = {Compendium of photovoltaic degradation rates},
	volume = {24},
	issn = {1099-159X},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/pip.2744},
	doi = {10.1002/pip.2744},
	abstract = {Published data on photovoltaic (PV) degradation measurements were aggregated and re-examined. The subject has seen an increased interest in recent years resulting in more than 11 000 degradation rates in almost 200 studies from 40 different countries. As studies have grown in number and size, we found an impact from sampling bias attributable to size and accuracy. Because of the correlational nature of this study we examined the data in several ways to minimize this bias. We found median degradation for x-Si technologies in the 0.5–0.6\%/year range with the mean in the 0.8–0.9\%/year range. Hetero-interface technology (HIT) and microcrystalline silicon (µc-Si) technologies, although not as plentiful, exhibit degradation around 1\%/year and resemble thin-film products more closely than x-Si. Several studies showing low degradation for copper indium gallium selenide (CIGS) have emerged. Higher degradation for cadmium telluride (CdTe) has been reported, but these findings could reflect a convolution of less accurate studies and longer stabilization periods for some products. Significant deviations for beginning-of-life measurements with respect to nameplate rating have been documented over the last 35 years. Therefore, degradation rates that use nameplate rating as reference may be significantly impacted. Studies that used nameplate rating as reference but used solar simulators showed less variation than similar studies using outdoor measurements, even when accounting for different climates. This could be associated with confounding effects of measurement uncertainty and soiling that take place outdoors. Hotter climates and mounting configurations that lead to sustained higher temperatures may lead to higher degradation in some, but not all, products. Wear-out non-linearities for the worst performing modules have been documented in a few select studies that took multiple measurements of an ensemble of modules during the lifetime of the system. However, the majority of these modules exhibit a fairly linear decline. Modeling these non-linearities, whether they occur at the beginning-of-life or end-of-life in the PV life cycle, has an important impact on the levelized cost of energy. Copyright © 2016 John Wiley \& Sons, Ltd.},
	language = {en},
	number = {7},
	urldate = {2023-02-28},
	journal = {Progress in Photovoltaics: Research and Applications},
	author = {Jordan, Dirk C. and Kurtz, Sarah R. and VanSant, Kaitlyn and Newmiller, Jeff},
	year = {2016},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/pip.2744},
	keywords = {degradation rates, non-linearity, outdoor testing, performance, photovoltaic ageing, photovoltaic modules, photovoltaic systems},
	pages = {978--989},
}

@article{onken_ot-flow_2021,
	title = {{OT}-{Flow}: {Fast} and {Accurate} {Continuous} {Normalizing} {Flows} via {Optimal} {Transport}},
	volume = {35},
	copyright = {Copyright (c) 2021 Association for the Advancement of Artificial Intelligence},
	issn = {2374-3468},
	shorttitle = {{OT}-{Flow}},
	url = {https://ojs.aaai.org/index.php/AAAI/article/view/17113},
	doi = {10.1609/aaai.v35i10.17113},
	abstract = {A normalizing flow is an invertible mapping between an arbitrary probability distribution and a standard normal distribution; it can be used for density estimation and statistical inference. Computing the flow follows the change of variables formula and thus requires invertibility of the mapping and an efficient way to compute the determinant of its Jacobian. To satisfy these requirements, normalizing flows typically consist of carefully chosen components. Continuous normalizing flows (CNFs) are mappings obtained by solving a neural ordinary differential equation (ODE). The neural ODE's dynamics can be chosen almost arbitrarily while ensuring invertibility. Moreover, the log-determinant of the flow's Jacobian can be obtained by integrating the trace of the dynamics' Jacobian along the flow. Our proposed OT-Flow approach tackles two critical computational challenges that limit a more widespread use of CNFs. First, OT-Flow leverages optimal transport (OT) theory to regularize the CNF and enforce straight trajectories that are easier to integrate. Second, OT-Flow features exact trace computation with time complexity equal to trace estimators used in existing CNFs. On five high-dimensional density estimation and generative modeling tasks, OT-Flow performs competitively to state-of-the-art CNFs while on average requiring one-fourth of the number of weights with an 8x speedup in training time and 24x speedup in inference.},
	language = {en},
	number = {10},
	urldate = {2023-02-24},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
	author = {Onken, Derek and Fung, Samy Wu and Li, Xingjian and Ruthotto, Lars},
	month = may,
	year = {2021},
	note = {Number: 10},
	keywords = {Neural Generative Models \& Autoencoders},
	pages = {9223--9232},
}

@article{gao_event_2020,
	title = {Event generation with normalizing flows},
	volume = {101},
	url = {https://link.aps.org/doi/10.1103/PhysRevD.101.076002},
	doi = {10.1103/PhysRevD.101.076002},
	abstract = {We present a novel integrator based on normalizing flows which can be used to improve the unweighting efficiency of Monte Carlo event generators for collider physics simulations. In contrast to machine learning approaches based on surrogate models, our method generates the correct result even if the underlying neural networks are not optimally trained. We exemplify the new strategy using the example of Drell-Yan type processes at the LHC, both at leading and partially at next-to-leading order QCD.},
	number = {7},
	urldate = {2023-02-24},
	journal = {Physical Review D},
	author = {Gao, Christina and Höche, Stefan and Isaacson, Joshua and Krause, Claudius and Schulz, Holger},
	month = apr,
	year = {2020},
	note = {Publisher: American Physical Society},
	pages = {076002},
}

@article{gao_i-_2020,
	title = {i- flow: {High}-dimensional integration and sampling with normalizing flows},
	volume = {1},
	issn = {2632-2153},
	shorttitle = {i- flow},
	url = {https://dx.doi.org/10.1088/2632-2153/abab62},
	doi = {10.1088/2632-2153/abab62},
	abstract = {In many fields of science, high-dimensional integration is required. Numerical methods have been developed to evaluate these complex integrals. We introduce the code i-flow, a Python package that performs high-dimensional numerical integration utilizing normalizing flows. Normalizing flows are machine-learned, bijective mappings between two distributions. i-flow can also be used to sample random points according to complicated distributions in high dimensions. We compare i-flow to other algorithms for high-dimensional numerical integration and show that i-flow outperforms them for high dimensional correlated integrals. The i-flow code is publicly available on gitlab at https://gitlab.com/i-flow/i-flow.},
	language = {en},
	number = {4},
	urldate = {2023-02-24},
	journal = {Machine Learning: Science and Technology},
	author = {Gao, Christina and Isaacson, Joshua and Krause, Claudius},
	month = oct,
	year = {2020},
	note = {Publisher: IOP Publishing},
	pages = {045023},
}

@misc{du_metakernel_2021,
	title = {{MetaKernel}: {Learning} {Variational} {Random} {Features} with {Limited} {Labels}},
	shorttitle = {{MetaKernel}},
	url = {http://arxiv.org/abs/2105.03781},
	doi = {10.48550/arXiv.2105.03781},
	abstract = {Few-shot learning deals with the fundamental and challenging problem of learning from a few annotated samples, while being able to generalize well on new tasks. The crux of few-shot learning is to extract prior knowledge from related tasks to enable fast adaptation to a new task with a limited amount of data. In this paper, we propose meta-learning kernels with random Fourier features for few-shot learning, we call MetaKernel. Specifically, we propose learning variational random features in a data-driven manner to obtain task-specific kernels by leveraging the shared knowledge provided by related tasks in a meta-learning setting. We treat the random feature basis as the latent variable, which is estimated by variational inference. The shared knowledge from related tasks is incorporated into a context inference of the posterior, which we achieve via a long-short term memory module. To establish more expressive kernels, we deploy conditional normalizing flows based on coupling layers to achieve a richer posterior distribution over random Fourier bases. The resultant kernels are more informative and discriminative, which further improves the few-shot learning. To evaluate our method, we conduct extensive experiments on both few-shot image classification and regression tasks. A thorough ablation study demonstrates that the effectiveness of each introduced component in our method. The benchmark results on fourteen datasets demonstrate MetaKernel consistently delivers at least comparable and often better performance than state-of-the-art alternatives.},
	urldate = {2023-02-24},
	publisher = {arXiv},
	author = {Du, Yingjun and Sun, Haoliang and Zhen, Xiantong and Xu, Jun and Yin, Yilong and Shao, Ling and Snoek, Cees G. M.},
	month = may,
	year = {2021},
	note = {arXiv:2105.03781 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
}

@article{kim_review_2021,
	title = {A {Review} of the {Degradation} of {Photovoltaic} {Modules} for {Life} {Expectancy}},
	volume = {14},
	copyright = {http://creativecommons.org/licenses/by/3.0/},
	issn = {1996-1073},
	url = {https://www.mdpi.com/1996-1073/14/14/4278},
	doi = {10.3390/en14144278},
	abstract = {Photovoltaic (PV) modules are generally considered to be the most reliable components of PV systems. The PV module has a high probability of being able to perform adequately for 30 years under typical operating conditions. In order to evaluate the long-term performance of a PV module under diversified terrestrial conditions, outdoor-performance data should be used. However, this requires a wait of 25 years to determine the module reliability, which is highly undesirable. Thus, accelerated-stress tests performed in the laboratory by mimicking different field conditions are important for understanding the performance of a PV module. In this paper, we discuss PV-module degradation types and different accelerated-stress types that are used to evaluate the PV-module reliability and durability for life expectancy before using them in the real field. Finally, prevention and correction measures are described to minimize economic losses.},
	language = {en},
	number = {14},
	urldate = {2023-02-20},
	journal = {Energies},
	author = {Kim, Jaeun and Rabelo, Matheus and Padi, Siva Parvathi and Yousuf, Hasnain and Cho, Eun-Chel and Yi, Junsin},
	month = jan,
	year = {2021},
	note = {Number: 14
Publisher: Multidisciplinary Digital Publishing Institute},
	keywords = {PV module, degradation, life expectancy, test methods},
	pages = {4278},
}

@article{jordan_photovoltaic_2013-1,
	title = {Photovoltaic {Degradation} {Rates}—an {Analytical} {Review}},
	volume = {21},
	issn = {1099-159X},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/pip.1182},
	doi = {10.1002/pip.1182},
	abstract = {As photovoltaic penetration of the power grid increases, accurate predictions of return on investment require accurate prediction of decreased power output over time. Degradation rates must be known in order to predict power delivery. This article reviews degradation rates of flat-plate terrestrial modules and systems reported in published literature from field testing throughout the last 40 years. Nearly 2000 degradation rates, measured on individual modules or entire systems, have been assembled from the literature, showing a median value of 0·5\%/year. The review consists of three parts: a brief historical outline, an analytical summary of degradation rates, and a detailed bibliography partitioned by technology. Copyright © 2011 John Wiley \& Sons, Ltd.},
	language = {en},
	number = {1},
	urldate = {2023-02-20},
	journal = {Progress in Photovoltaics: Research and Applications},
	author = {Jordan, D. C. and Kurtz, S. R.},
	year = {2013},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/pip.1182},
	keywords = {degradation rates, field testing, outdoor testing, performance, photovoltaic modules, photovoltaic systems},
	pages = {12--29},
}

@article{aghaei_review_2022,
	title = {Review of degradation and failure phenomena in photovoltaic modules},
	volume = {159},
	issn = {1364-0321},
	url = {https://www.sciencedirect.com/science/article/pii/S1364032122000880},
	doi = {10.1016/j.rser.2022.112160},
	abstract = {The degradation of photovoltaic (PV) systems is one of the key factors to address in order to reduce the cost of the electricity produced by increasing the operational lifetime of PV systems. To reduce the degradation, it is imperative to know the degradation and failure phenomena. This review article has been prepared to present an overview of the state-of-the-art knowledge on the reliability of PV modules. Whilst the most common technology today is mono- and multi-crystalline silicon, this article aims to give a generic summary which is relevant for a wider range of photovoltaic technologies including cadmium telluride, copper indium gallium selenide and emerging low-cost high-efficiency technologies. The review consists of three parts: firstly, a brief contextual summary about reliability metrics and how reliability is measured. Secondly, a summary of the main stress factors and how they influence module degradation. Finally, a detailed review of degradation and failure modes, which has been partitioned by the individual component within a PV module. This section connects the degradation phenomena and failure modes to the module component, and its effects on the PV system. Building on this knowledge, strategies to improve the operational lifetime of PV systems and thus, to reduce the electricity cost can be devised. Through extensive testing and failure analysis, researchers now have a much better overview of stressors and their impact on long term stability.},
	language = {en},
	urldate = {2023-02-20},
	journal = {Renewable and Sustainable Energy Reviews},
	author = {Aghaei, M. and Fairbrother, A. and Gok, A. and Ahmad, S. and Kazim, S. and Lobato, K. and Oreski, G. and Reinders, A. and Schmitz, J. and Theelen, M. and Yilmaz, P. and Kettle, J.},
	month = may,
	year = {2022},
	keywords = {Climate, Degradation, Energy payback time, Photovoltaics, Reliability, Solar cells, Stress, Wear-out, solar Modules},
	pages = {112160},
}

@inproceedings{karami_invertible_2019,
	title = {Invertible {Convolutional} {Flow}},
	volume = {32},
	url = {https://papers.nips.cc/paper/2019/hash/b1f62fa99de9f27a048344d55c5ef7a6-Abstract.html},
	abstract = {Normalizing flows can be used to construct high quality generative probabilistic
models, but training and sample generation require repeated evaluation of Jacobian determinants and function inverses. To make such computations feasible, current approaches employ highly constrained architectures that produce diagonal, triangular, or low rank Jacobian matrices.  As an alternative, we investigate a set of novel normalizing flows based on the circular and symmetric convolutions.  We show that these transforms admit efficient Jacobian determinant computation and inverse mapping (deconvolution) in O(N log N) time. Additionally, element-wise multiplication, widely used in normalizing flow architectures, can be combined with these transforms to increase modeling flexibility.   We further propose an analytic approach to designing nonlinear elementwise bijectors that induce special properties in the intermediate layers, by implicitly introducing specific regularizers in the loss. We show that these transforms allow more effective normalizing flow models to be developed for generative image models.},
	urldate = {2023-02-17},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Karami, Mahdi and Schuurmans, Dale and Sohl-Dickstein, Jascha and Dinh, Laurent and Duckworth, Daniel},
	year = {2019},
}

@inproceedings{alaa_generative_2021,
	title = {Generative {Time}-series {Modeling} with {Fourier} {Flows}},
	url = {https://openreview.net/forum?id=PpshD0AXfA},
	abstract = {Generating synthetic time-series data is crucial in various application domains, such as medical prognosis, wherein research is hamstrung by the lack of access to data due to concerns over privacy. Most of the recently proposed methods for generating synthetic time-series rely on implicit likelihood modeling using generative adversarial networks (GANs)—but such models can be difficult to train, and may jeopardize privacy by “memorizing” temporal patterns in training data. In this paper, we propose an explicit likelihood model based on a novel class of normalizing flows that view time-series data in the frequency-domain rather than the time-domain. The proposed flow, dubbed a Fourier flow, uses a discrete Fourier transform (DFT) to convert variable-length time-series with arbitrary sampling periods into fixed-length spectral representations, then applies a (data-dependent) spectral filter to the frequency-transformed time-series. We show that, by virtue of the DFT analytic properties, the Jacobian determinants and inverse mapping for the Fourier flow can be computed efficiently in linearithmic time, without imposing explicit structural constraints as in existing flows such as NICE (Dinh et al. (2014)), RealNVP (Dinh et al. (2016)) and GLOW (Kingma \& Dhariwal (2018)). Experiments show that Fourier flows perform competitively compared to state-of-the-art baselines.},
	language = {en},
	urldate = {2023-02-17},
	author = {Alaa, Ahmed and Chan, Alex James and Schaar, Mihaela van der},
	month = mar,
	year = {2021},
}

@article{cunningham_normalizing_2020,
	title = {Normalizing {Flows} {Across} {Dimensions}},
	url = {https://arxiv.org/abs/2006.13070v1},
	doi = {10.48550/arXiv.2006.13070},
	abstract = {Real-world data with underlying structure, such as pictures of faces, are hypothesized to lie on a low-dimensional manifold. This manifold hypothesis has motivated state-of-the-art generative algorithms that learn low-dimensional data representations. Unfortunately, a popular generative model, normalizing flows, cannot take advantage of this. Normalizing flows are based on successive variable transformations that are, by design, incapable of learning lower-dimensional representations. In this paper we introduce noisy injective flows (NIF), a generalization of normalizing flows that can go across dimensions. NIF explicitly map the latent space to a learnable manifold in a high-dimensional data space using injective transformations. We further employ an additive noise model to account for deviations from the manifold and identify a stochastic inverse of the generative process. Empirically, we demonstrate that a simple application of our method to existing flow architectures can significantly improve sample quality and yield separable data embeddings.},
	language = {en},
	urldate = {2023-02-15},
	author = {Cunningham, Edmond and Zabounidis, Renos and Agrawal, Abhinav and Fiterau, Madalina and Sheldon, Daniel},
	month = jun,
	year = {2020},
}

@misc{vadeboncoeur_random_2023,
	title = {Random {Grid} {Neural} {Processes} for {Parametric} {Partial} {Differential} {Equations}},
	url = {http://arxiv.org/abs/2301.11040},
	doi = {10.48550/arXiv.2301.11040},
	abstract = {We introduce a new class of spatially stochastic physics and data informed deep latent models for parametric partial differential equations (PDEs) which operate through scalable variational neural processes. We achieve this by assigning probability measures to the spatial domain, which allows us to treat collocation grids probabilistically as random variables to be marginalised out. Adapting this spatial statistics view, we solve forward and inverse problems for parametric PDEs in a way that leads to the construction of Gaussian process models of solution fields. The implementation of these random grids poses a unique set of challenges for inverse physics informed deep learning frameworks and we propose a new architecture called Grid Invariant Convolutional Networks (GICNets) to overcome these challenges. We further show how to incorporate noisy data in a principled manner into our physics informed model to improve predictions for problems where data may be available but whose measurement location does not coincide with any fixed mesh or grid. The proposed method is tested on a nonlinear Poisson problem, Burgers equation, and Navier-Stokes equations, and we provide extensive numerical comparisons. We demonstrate significant computational advantages over current physics informed neural learning methods for parametric PDEs while improving the predictive capabilities and flexibility of these models.},
	urldate = {2023-02-10},
	publisher = {arXiv},
	author = {Vadeboncoeur, Arnaud and Kazlauskaite, Ieva and Papandreou, Yanni and Cirak, Fehmi and Girolami, Mark and Akyildiz, Ömer Deniz},
	month = jan,
	year = {2023},
	note = {arXiv:2301.11040 [cs, math, stat]},
	keywords = {Computer Science - Machine Learning, Mathematics - Numerical Analysis, Statistics - Computation, Statistics - Machine Learning},
}

@article{duffin_low-rank_2022,
	title = {Low-rank statistical finite elements for scalable model-data synthesis},
	volume = {463},
	issn = {0021-9991},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999122003230},
	doi = {10.1016/j.jcp.2022.111261},
	abstract = {Statistical learning additions to physically derived mathematical models are gaining traction in the literature. A recent approach has been to augment the underlying physics of the governing equations with data driven Bayesian statistical methodology. Coined statFEM, the method acknowledges a priori model misspecification, by embedding stochastic forcing within the governing equations. Upon receipt of additional data, the posterior distribution of the discretised finite element solution is updated using classical Bayesian filtering techniques. The resultant posterior jointly quantifies uncertainty associated with the ubiquitous problem of model misspecification and the data intended to represent the true process of interest. Despite this appeal, computational scalability is a challenge to statFEM's application to high-dimensional problems typically experienced in physical and industrial contexts. This article overcomes this hurdle by embedding a low-rank approximation of the underlying dense covariance matrix, obtained from the leading order modes of the full-rank alternative. Demonstrated on a series of reaction-diffusion problems of increasing dimension, using experimental and simulated data, the method reconstructs the sparsely observed data-generating processes with minimal loss of information, in both the posterior mean and variance, paving the way for further integration of physical and probabilistic approaches to complex systems.},
	language = {en},
	urldate = {2023-02-10},
	journal = {Journal of Computational Physics},
	author = {Duffin, Connor and Cripps, Edward and Stemler, Thomas and Girolami, Mark},
	month = aug,
	year = {2022},
	keywords = {Bayesian filtering, Bayesian inverse problems, Finite element methods, Reaction-diffusion},
	pages = {111261},
}

@article{mishra_pi_2022,
	title = {\$\${\textbackslash}pi \$\${VAE}: a stochastic process prior for {Bayesian} deep learning with {MCMC}},
	volume = {32},
	issn = {1573-1375},
	shorttitle = {\$\${\textbackslash}pi \$\${VAE}},
	url = {https://doi.org/10.1007/s11222-022-10151-w},
	doi = {10.1007/s11222-022-10151-w},
	abstract = {Stochastic processes provide a mathematically elegant way to model complex data. In theory, they provide flexible priors over function classes that can encode a wide range of interesting assumptions. However, in practice efficient inference by optimisation or marginalisation is difficult, a problem further exacerbated with big data and high dimensional input spaces. We propose a novel variational autoencoder (VAE) called the prior encoding variational autoencoder (\$\${\textbackslash}pi \$\$VAE). \$\${\textbackslash}pi \$\$VAE is a new continuous stochastic process. We use \$\${\textbackslash}pi \$\$VAE to learn low dimensional embeddings of function classes by combining a trainable feature mapping with generative model using a VAE. We show that our framework can accurately learn expressive function classes such as Gaussian processes, but also properties of functions such as their integrals. For popular tasks, such as spatial interpolation, \$\${\textbackslash}pi \$\$VAE achieves state-of-the-art performance both in terms of accuracy and computational efficiency. Perhaps most usefully, we demonstrate an elegant and scalable means of performing fully Bayesian inference for stochastic processes within probabilistic programming languages such as Stan.},
	language = {en},
	number = {6},
	urldate = {2023-02-10},
	journal = {Statistics and Computing},
	author = {Mishra, Swapnil and Flaxman, Seth and Berah, Tresnia and Zhu, Harrison and Pakkanen, Mikko and Bhatt, Samir},
	month = oct,
	year = {2022},
	keywords = {Bayesian inference, MCMC, Spatio-temporal, VAE},
	pages = {96},
}

@article{guo_normalizing_2022,
	title = {Normalizing field flows: {Solving} forward and inverse stochastic differential equations using physics-informed flow models},
	volume = {461},
	issn = {0021-9991},
	shorttitle = {Normalizing field flows},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999122002649},
	doi = {10.1016/j.jcp.2022.111202},
	abstract = {We introduce in this work the normalizing field flows (NFF) for learning random fields from scattered measurements. More precisely, we construct a bijective transformation (a normalizing flow characterizing by neural networks) between a Gaussian random field with the Karhunen-Loève (KL) expansion structure and the target stochastic field, where the KL expansion coefficients and the invertible networks are trained by maximizing the sum of the log-likelihood on scattered measurements. This NFF model can be used to solve data-driven forward, inverse, and mixed forward/inverse stochastic partial differential equations in a unified framework. We demonstrate the capability of the proposed NFF model for learning non-Gaussian processes and different types of stochastic partial differential equations.},
	language = {en},
	urldate = {2023-02-10},
	journal = {Journal of Computational Physics},
	author = {Guo, Ling and Wu, Hao and Zhou, Tao},
	month = jul,
	year = {2022},
	keywords = {Data-driven modeling, Normalizing flows, Random fields, Uncertainty quantification},
	pages = {111202},
}

@inproceedings{maronas_transforming_2021,
	title = {Transforming {Gaussian} {Processes} {With} {Normalizing} {Flows}},
	url = {http://proceedings.mlr.press/v130/maronas21a.html},
	abstract = {Gaussian Processes (GP) can be used as flexible, non-parametric function priors. Inspired by the growing body of work on Normalizing Flows, we enlarge this class of priors through a parametric inv...},
	language = {en},
	urldate = {2021-04-30},
	booktitle = {International {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Maroñas, Juan and Hamelijnck, Oliver and Knoblauch, Jeremias and Damoulas, Theodoros},
	month = mar,
	year = {2021},
	note = {ISSN: 2640-3498},
	keywords = {🔥},
	pages = {1081--1089},
}

@inproceedings{hegde_variational_2022,
	title = {Variational multiple shooting for {Bayesian} {ODEs} with {Gaussian} processes},
	url = {https://openreview.net/forum?id=r2NuhIUoceq},
	abstract = {We introduce a method for efficiently performing Bayesian inference on unknown ODEs using Gaussian processes},
	language = {en},
	urldate = {2022-08-04},
	author = {Hegde, Pashupati Rajaram and Yildiz, Cagatay and Lähdesmäki, Harri and Kaski, Samuel and Heinonen, Markus},
	month = jun,
	year = {2022},
	keywords = {🔥},
}

@inproceedings{bao_physics_2022,
	title = {Physics {Guided} {Neural} {Networks} for {Spatio}-temporal {Super}-resolution of {Turbulent} {Flows}},
	url = {https://openreview.net/forum?id=S98VJL8jcxq},
	abstract = {Direct numerical simulation (DNS) of turbulent flows is computationally expensive and cannot be applied to flows with large Reynolds numbers.  Low-resolution large eddy simulation (LES) is a...},
	language = {en},
	urldate = {2022-08-04},
	author = {Bao, Tianshu and Chen, Shengyu and Johnson, Taylor T. and Givi, Peyman and Sammak, Shervin and Jia, Xiaowei},
	month = jun,
	year = {2022},
	keywords = {🔥},
}

@inproceedings{ouderaa_learning_2022,
	title = {Learning {Invariant} {Weights} in {Neural} {Networks}},
	url = {https://openreview.net/forum?id=BVxfSPUoqeq},
	abstract = {Assumptions about invariances or symmetries in data can significantly increase the predictive power of statistical models. Many commonly used machine learning models are constraint to respect...},
	language = {en},
	urldate = {2022-08-15},
	author = {Ouderaa, Tycho F. A. van der and Wilk, Mark van der},
	month = jun,
	year = {2022},
	keywords = {🔥},
}

@inproceedings{falk_implicit_2022,
	title = {Implicit kernel meta-learning using kernel integral forms},
	url = {https://openreview.net/forum?id=rNgqwPUsqgq},
	abstract = {Meta-learning algorithms have made significant progress in the context of meta-learning for image classification but less attention has been given to the regression setting. In this paper we...},
	language = {en},
	urldate = {2022-08-15},
	author = {Falk, John Isak Texas and Ciliberto, Carlo and Pontil, Massimiliano},
	month = jun,
	year = {2022},
	keywords = {🔥},
}

@techreport{noauthor_solar_2022,
	title = {Solar {Risk} {Assessment}: 2022},
	institution = {kWh Analytics},
	year = {2022},
}

@inproceedings{kimber_effect_2006,
	title = {The {Effect} of {Soiling} on {Large} {Grid}-{Connected} {Photovoltaic} {Systems} in {California} and the {Southwest} {Region} of the {United} {States}},
	volume = {2},
	doi = {10.1109/WCPEC.2006.279690},
	abstract = {The accumulation of dirt on solar panels ("soiling") can have a significant impact on the performance of PV systems in regions where rainfall is limited for a dry season of several months. This effect is magnified where rainfall is absent in the peak-solar summer months, such as in California and the Southwest region of the United States. This paper describes the effects of soiling on energy production for large grid-connected systems in the US and presents a model for predicting soiling losses},
	booktitle = {2006 {IEEE} 4th {World} {Conference} on {Photovoltaic} {Energy} {Conference}},
	author = {Kimber, A. and Mitchell, L. and Nogradi, S. and Wenger, H.},
	month = may,
	year = {2006},
	note = {ISSN: 0160-8371},
	keywords = {soiling},
	pages = {2391--2395},
}

@article{coello_simple_2019,
	title = {Simple {Model} for {Predicting} {Time} {Series} {Soiling} of {Photovoltaic} {Panels}},
	volume = {9},
	issn = {2156-3403},
	doi = {10.1109/JPHOTOV.2019.2919628},
	abstract = {Results from a model to predict soiling losses of photovoltaic (PV) arrays are presented. The model uses ambient airborne particulate matter concentrations, both PM10 and PM2.5, the tilt of the PV array including if the array is tracking, and rain data to estimate soiling losses over time. The model uses relationships between average airborne PM concentrations and dust accumulation, rain removal of accumulated dust, and the accumulated dust and transmission loss that have been developed in the literature to estimate soiling losses in a time series. Multiple model runs were performed and were compared with measured soiling data at seven locations. The model was run with both typical Meteorological Year rain data and with recorded rain data from Oregon State University's PRISM database. Each model was run with variable deposition velocities, static deposition velocities, and static settling velocities. When the model was run with recorded rain data and static settling velocities the results matched the general slopes, frequencies, and magnitudes of soiling losses when compared with measured soiling data. Overall, the simple model demonstrates the ability at accurately simulating soiling.},
	number = {5},
	journal = {IEEE Journal of Photovoltaics},
	author = {Coello, Merissa and Boyle, Liza},
	month = sep,
	year = {2019},
	note = {Conference Name: IEEE Journal of Photovoltaics},
	keywords = {soiling},
	pages = {1382--1387},
}

@article{tanesab_seasonal_2017,
	title = {Seasonal effect of dust on the degradation of {PV} modules performance deployed in different climate areas},
	volume = {111},
	issn = {0960-1481},
	url = {https://www.sciencedirect.com/science/article/pii/S0960148117302823},
	doi = {10.1016/j.renene.2017.03.091},
	abstract = {The aim of this study is to investigate the seasonal effect of dust on the degradation of PV modules deployed in two different climate areas, Perth, Western Australia, a temperate climate region and Nusa Tenggara Timur (NTT), Indonesia, a tropical climate region. Results revealed that PV performance varied with season. In Perth, the performance of PV modules which was maximal in the beginning of summer decreased significantly at the end of the season. The performance then increased back approaching the initial position at the end of autumn and reached a peak at the end of winter. Similar reduction to the summer’s performance was accounted by the modules at the end of spring. Meanwhile, in NTT, the performance of PV modules was maximal in the beginning of wet season, dropped slightly at the end of the season and decreased significantly at the end of dry season. Degradation of all modules in the two sites was more affected by dust compared to the non-dust related factors. The degradation is important information for future PV design in both areas, especially in NTT which accounted greater values than the typical dust de-rating factors.},
	language = {en},
	urldate = {2022-10-21},
	journal = {Renewable Energy},
	author = {Tanesab, Julius and Parlevliet, David and Whale, Jonathan and Urmee, Tania},
	month = oct,
	year = {2017},
	keywords = {soiling, solar energy},
	pages = {105--115},
}

@article{toth_predicting_2020,
	title = {Predicting {Photovoltaic} {Soiling} {From} {Air} {Quality} {Measurements}},
	volume = {10},
	issn = {2156-3403},
	doi = {10.1109/JPHOTOV.2020.2983990},
	abstract = {Ambient particulate matter in urban environments is dynamic and heterogeneous; therefore, understanding photovoltaic energy loss due to soiling is challenging. Silicon reference cells were deployed in an urban-industrial area in Colorado colocated with measurements of ambient particulate matter concentrations. Regressing measured soiling ratios against cumulative sums of ambient coarse particulate matter since the last precipitation event and ambient fine particulate matter since the first day of deployment produces a root-mean-square error of approximately 0.013. This model partially addresses the challenge of quantifying the relationships between ambient air quality and photovoltaic soiling.},
	number = {4},
	journal = {IEEE Journal of Photovoltaics},
	author = {Toth, Sarah and Hannigan, Michael and Vance, Marina and Deceglie, Michael},
	month = jul,
	year = {2020},
	note = {Conference Name: IEEE Journal of Photovoltaics},
	keywords = {soiling, solar energy, solar forecasting},
	pages = {1142--1147},
}

@article{younis_modeling_2021,
	title = {Modeling of dust soiling effects on solar photovoltaic performance: {A} review},
	volume = {220},
	issn = {0038-092X},
	shorttitle = {Modeling of dust soiling effects on solar photovoltaic performance},
	url = {https://www.sciencedirect.com/science/article/pii/S0038092X21002929},
	doi = {10.1016/j.solener.2021.04.011},
	abstract = {The known standard equations that describe the physics of light converting into electricity inside a photovoltaic (PV) cell do not necessarily comprise the different operating environments' effects. The scientists investigated several environmental factors that would influence the performance of solar sheets. Sometimes, some of them clarified these factors' significance by using numerical simulations, analytical modeling (i.e., deriving correlations), or even changing the existing PV equations by including a parameter or term that defines the designated factor's role. The accumulation of dust on the PV surfaces is one of the operational environmental factors leading to significantly reduced performance. Technically, the dust gathering results in shading, which must change, at least in theory, the current-voltage (I-V) and power-voltage (P-V) curves of the solar PV cell or module concerned. In the current review, we summarized the authors' work that models the impact of buildup of dust particles and otherwise on the PV installations, each from a distinct perspective. We also ensured that the review holds the mathematical add-ons and alterations made to the underlying PV model to characterize the effect of interest.},
	language = {en},
	urldate = {2022-09-08},
	journal = {Solar Energy},
	author = {Younis, A. and Alhorr, Y.},
	month = may,
	year = {2021},
	keywords = {climate modeling, physics, renewable energy, review, solar energy},
	pages = {1074--1088},
}

@article{figgis_condensation_2022,
	title = {Condensation as a predictor of {PV} soiling},
	volume = {238},
	issn = {0038-092X},
	url = {https://www.sciencedirect.com/science/article/pii/S0038092X22002730},
	doi = {10.1016/j.solener.2022.04.025},
	abstract = {Condensation (dew) strongly affects PV soiling: it causes dust particles to stick to PV modules, yet conversely it can also clean modules as the dew droplets run off. These mechanisms have been well studied qualitatively, but because weather stations at PV sites rarely include condensation sensors, the quantitative link between dew and PV soiling rate is not well known. The goal of this study was to investigate this relationship numerically, and especially whether condensation data improves the accuracy of PV soiling models. We performed a field study in Doha, Qatar, in which condensation on a PV module, “standard” meteorological parameters, and the daily PV soiling rate were measured for 2.5 years. We found low correlation between all variables and the soiling rate, however condensation was among the more beneficial variables to include in soiling models (depending on the statistical analysis method used). Also, it is necessary to measure condensation directly, because it was not accurately predicted from dew-point and module surface temperatures.},
	language = {en},
	urldate = {2022-10-21},
	journal = {Solar Energy},
	author = {Figgis, Benjamin and Scabbia, Giovanni and Aissa, Brahim},
	month = may,
	year = {2022},
	keywords = {soiling, solar energy},
	pages = {30--38},
}

@article{al_siyabi_effect_2021,
	title = {Effect of {Soiling} on {Solar} {Photovoltaic} {Performance} under {Desert} {Climatic} {Conditions}},
	volume = {14},
	copyright = {http://creativecommons.org/licenses/by/3.0/},
	issn = {1996-1073},
	url = {https://www.mdpi.com/1996-1073/14/3/659},
	doi = {10.3390/en14030659},
	abstract = {The solar irradiation at the gulf Arabia is considered one of the highest in the world. However, this region is classified as a desert with high dust accumulation. Thus, the objective of this study is to analyze the effect of soiling and the photovoltaic (PV) tilt angle on the performance of 2.0 MWp of car park PV plant in Oman. Experimental measurements were taken and a model was developed for simulation. The power generation by the cleaned PV system was measured as 1460 kW around noon. After one week of operation, the power production (at the same irradiance level) reduced to 1390 kW due to soiling. It further reduced to 1196 kW and 904 kW after three and five weeks of operation, respectively. The results also show that a soiling-percentage of 7.5\% reduced the monthly electricity generation (307 MWh) by 5.6\% and a soiling-percentage of 12.5\% reduced the generation by 10.8\%. Furthermore, the increase in tilt is not recommended due to the duo-pitch canopy effect of the car park where the panels with 180° azimuth generate lower electricity than the panels with 0° azimuth. In addition, the part of the car park with 180° azimuth caused shading to the other part for high tilt angles.},
	language = {en},
	number = {3},
	urldate = {2022-10-26},
	journal = {Energies},
	author = {Al Siyabi, Idris and Al Mayasi, Arwa and Al Shukaili, Aiman and Khanna, Sourav},
	month = jan,
	year = {2021},
	note = {Number: 3
Publisher: Multidisciplinary Digital Publishing Institute},
	keywords = {soiling, solar energy},
	pages = {659},
}

@article{smestad_modelling_2020,
	title = {Modelling photovoltaic soiling losses through optical characterization},
	volume = {10},
	copyright = {2020 The Author(s)},
	issn = {2045-2322},
	url = {https://www.nature.com/articles/s41598-019-56868-z},
	doi = {10.1038/s41598-019-56868-z},
	abstract = {The accumulation of soiling on photovoltaic (PV) modules affects PV systems worldwide. Soiling consists of mineral dust, soot particles, aerosols, pollen, fungi and/or other contaminants that deposit on the surface of PV modules. Soiling absorbs, scatters, and reflects a fraction of the incoming sunlight, reducing the intensity that reaches the active part of the solar cell. Here, we report on the comparison of naturally accumulated soiling on coupons of PV glass soiled at seven locations worldwide. The spectral hemispherical transmittance was measured. It was found that natural soiling disproportionately impacts the blue and ultraviolet (UV) portions of the spectrum compared to the visible and infrared (IR). Also, the general shape of the transmittance spectra was similar at all the studied sites and could adequately be described by a modified form of the Ångström turbidity equation. In addition, the distribution of particles sizes was found to follow the IEST-STD-CC 1246E cleanliness standard. The fractional coverage of the glass surface by particles could be determined directly or indirectly and, as expected, has a linear correlation with the transmittance. It thus becomes feasible to estimate the optical consequences of the soiling of PV modules from the particle size distribution and the cleanliness value.},
	language = {en},
	number = {1},
	urldate = {2022-09-08},
	journal = {Scientific Reports},
	author = {Smestad, Greg P. and Germer, Thomas A. and Alrashidi, Hameed and Fernández, Eduardo F. and Dey, Sumon and Brahma, Honey and Sarmah, Nabin and Ghosh, Aritra and Sellami, Nazmi and Hassan, Ibrahim A. I. and Desouky, Mai and Kasry, Amal and Pesala, Bala and Sundaram, Senthilarasu and Almonacid, Florencia and Reddy, K. S. and Mallick, Tapas K. and Micheli, Leonardo},
	month = jan,
	year = {2020},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {climate modeling, physics, renewable energy, soiling},
	pages = {58},
}

@article{maghami_power_2016,
	title = {Power loss due to soiling on solar panel: {A} review},
	volume = {59},
	issn = {1364-0321},
	shorttitle = {Power loss due to soiling on solar panel},
	url = {https://www.sciencedirect.com/science/article/pii/S1364032116000745},
	doi = {10.1016/j.rser.2016.01.044},
	abstract = {The power output delivered from a photovoltaic module highly depends on the amount of irradiance, which reaches the solar cells. Many factors determine the ideal output or optimum yield in a photovoltaic module. However, the environment is one of the contributing parameters which directly affect the photovoltaic performance. The authors review and evaluate key contributions to the understanding, performance effects, and mitigation of power loss due to soiling on a solar panel. Electrical characteristics of PV (Voltage and current) are discussed with respect to shading due to soiling. Shading due to soiling is divided in two categories, namely, soft shading such as air pollution, and hard shading which occurs when a solid such as accumulated dust blocks the sunlight. The result shows that soft shading affects the current provided by the PV module, but the voltage remains the same. In hard shading, the performance of the PV module depends on whether some cells are shaded or all cells of the PV module are shaded. If some cells are shaded, then as long as the unshaded cells receive solar irradiance, there will be some output although there will be a decrease in the voltage output of the PV module. This study also present a few cleaning method to prevent from dust accumulation on the surface of solar arrays.},
	language = {en},
	urldate = {2022-10-21},
	journal = {Renewable and Sustainable Energy Reviews},
	author = {Maghami, Mohammad Reza and Hizam, Hashim and Gomes, Chandima and Radzi, Mohd Amran and Rezadad, Mohammad Ismael and Hajighorbani, Shahrooz},
	month = jun,
	year = {2016},
	keywords = {soiling, solar energy},
	pages = {1307--1316},
}

@techreport{both_trp_2022,
	title = {{TRP} {Site} {Preparation} and {Vegetation} {Management} {Trial}},
	institution = {Succession Ecology},
	author = {Both, Sabine},
	year = {2022},
}

@misc{samo_generalized_2015,
	title = {Generalized {Spectral} {Kernels}},
	url = {http://arxiv.org/abs/1506.02236},
	doi = {10.48550/arXiv.1506.02236},
	abstract = {In this paper we propose a family of tractable kernels that is dense in the family of bounded positive semi-definite functions (i.e. can approximate any bounded kernel with arbitrary precision). We start by discussing the case of stationary kernels, and propose a family of spectral kernels that extends existing approaches such as spectral mixture kernels and sparse spectrum kernels. Our extension has two primary advantages. Firstly, unlike existing spectral approaches that yield infinite differentiability, the kernels we introduce allow learning the degree of differentiability of the latent function in Gaussian process (GP) models and functions in the reproducing kernel Hilbert space (RKHS) in other kernel methods. Secondly, we show that some of the kernels we propose require fewer parameters than existing spectral kernels for the same accuracy, thereby leading to faster and more robust inference. Finally, we generalize our approach and propose a flexible and tractable family of spectral kernels that we prove can approximate any continuous bounded nonstationary kernel.},
	urldate = {2022-11-17},
	publisher = {arXiv},
	author = {Samo, Yves-Laurent Kom and Roberts, Stephen},
	month = oct,
	year = {2015},
	note = {arXiv:1506.02236 [stat]},
	keywords = {Statistics - Machine Learning, 🔥},
}

@misc{jiang_survey_2020,
	title = {A {Survey} on {Spatial} and {Spatiotemporal} {Prediction} {Methods}},
	url = {http://arxiv.org/abs/2012.13384},
	doi = {10.48550/arXiv.2012.13384},
	abstract = {With the advancement of GPS and remote sensing technologies, large amounts of geospatial and spatiotemporal data are being collected from various domains, driving the need for effective and efficient prediction methods. Given spatial data samples with explanatory features and targeted responses (categorical or continuous) at a set of locations, the problem aims to learn a model that can predict the response variable based on explanatory features. The problem is important with broad applications in earth science, urban informatics, geosocial media analytics and public health, but is challenging due to the unique characteristics of spatiotemporal data, including spatial and temporal autocorrelation, spatial heterogeneity, temporal non-stationarity, limited ground truth, and multiple scales and resolutions. This paper provides a systematic review on principles and methods in spatial and spatiotemporal prediction. We provide a taxonomy of methods categorized by the key challenge they address. For each method, we introduce its underlying assumption, theoretical foundation, and discuss its advantages and disadvantages. Our goal is to help interdisciplinary domain scientists choose techniques to solve their problems, and more importantly, to help data mining researchers to understand the main principles and methods in spatial and spatiotemporal prediction and identify future research opportunities.},
	urldate = {2022-10-18},
	publisher = {arXiv},
	author = {Jiang, Zhe},
	month = dec,
	year = {2020},
	note = {arXiv:2012.13384 [cs]},
	keywords = {gaussian processes, spatio-temporal, 🔥},
}

@article{benavides_cesar_review_2022,
	title = {Review on {Spatio}-{Temporal} {Solar} {Forecasting} {Methods} {Driven} by {In} {Situ} {Measurements} or {Their} {Combination} with {Satellite} and {Numerical} {Weather} {Prediction} ({NWP}) {Estimates}},
	volume = {15},
	copyright = {http://creativecommons.org/licenses/by/3.0/},
	issn = {1996-1073},
	url = {https://www.mdpi.com/1996-1073/15/12/4341},
	doi = {10.3390/en15124341},
	abstract = {To better forecast solar variability, spatio-temporal methods exploit spatially distributed solar time series, seeking to improve forecasting accuracy by including neighboring solar information. This review work is, to the authors’ understanding, the first to offer a compendium of references published since 2011 on such approaches for global horizontal irradiance and photovoltaic generation. The identified bibliography was categorized according to different parameters (method, data sources, baselines, performance metrics, forecasting horizon), and associated statistics were explored. Lastly, general findings are outlined, and suggestions for future research are provided based on the identification of less explored methods and data sources.},
	language = {en},
	number = {12},
	urldate = {2022-10-18},
	journal = {Energies},
	author = {Benavides Cesar, Llinet and Amaro e Silva, Rodrigo and Manso Callejo, Miguel Ángel and Cira, Calimanut-Ionut},
	month = jan,
	year = {2022},
	note = {Number: 12
Publisher: Multidisciplinary Digital Publishing Institute},
	keywords = {review, solar forecasting, spatio-temporal},
	pages = {4341},
}

@article{dagher_computational_2022,
	title = {Computational prediction of dust deposition on solar panels},
	issn = {1614-7499},
	url = {https://doi.org/10.1007/s11356-022-22993-y},
	doi = {10.1007/s11356-022-22993-y},
	abstract = {This research is concerned with performing computational fluid dynamics (CFD) simulations to investigate the air flow and dust deposition behavior around a ground-mounted solar PV panel. The discrete phase model (DPM) is adopted to model the gas-solid flow. The influence of the wind speed, the dust particle size, and the dust material on the dust deposition rate was investigated based on the environment of Cairo, Egypt. The wind speeds range between 1 and 11.5 m/s with an average of 3.7 m/s. It is found that increasing the wind speed decreases the dust deposition rate. For wind speeds higher than 2 m/s, it is found that increasing the dust particle diameter or the dust density increases the dust deposition rate. For wind speeds lower than 2 m/s, it is found that there is a critical particle size before which increasing the dust density causes dust deposition rate to increase and after which increasing the dust density decreases the dust deposition. The maximum percentage of deposition rate equals 10.8\% and occurs for the dolomite dust material at a wind speed of 2 m/s and particles diameter of 150 μm.},
	language = {en},
	urldate = {2022-10-18},
	journal = {Environmental Science and Pollution Research},
	author = {Dagher, M. Mekawy and Kandil, Hamdy A.},
	month = sep,
	year = {2022},
	keywords = {soiling, solar energy, 🔥},
}

@article{chiteka_cfd_2021,
	title = {{CFD} {Prediction} of dust deposition and installation parametric optimisation for soiling mitigation in non-tracking solar {PV} modules},
	volume = {42},
	issn = {0143-0750},
	url = {https://doi.org/10.1080/01430750.2019.1594373},
	doi = {10.1080/01430750.2019.1594373},
	abstract = {The influence of installation parameters on dust accumulation behaviour for ground mounted solar photovoltaic collectors were investigated by computational fluid dynamics simulation. The effect of wind direction relative to the direction of the incident solar radiation on the photovoltaic module was also studied. The shear stress transport k − ω turbulence model and the discrete phase model were respectively adopted for wind flow simulation and dust deposition prediction. Installation parameter optimisation for minimum soiling and maximum energy generation was carried out using response surface methodology. Tilt angle was found to have a greater effect on dust deposition compared to orientation. A predictive model was developed for dust deposition and had a coefficient of determination, mean absolute percentage error and residual mean square error values of 0.983, 5.4 particles and 1.29 MWh/year respectively. The optimal installation configuration for a selected location was determined and it was found to produce 3.5\% more energy than a system installed using tilt as the latitude angle and facing north in the same location. When wind flows from behind the solar photovoltaic module, there is 16\% more energy generated than when wind flow is directly in front of the solar photovoltaic module at the optimal installation configuration.},
	number = {11},
	urldate = {2022-10-18},
	journal = {International Journal of Ambient Energy},
	author = {Chiteka, Kudzanayi and Arora, Rajesh and Jain, Vineet},
	month = aug,
	year = {2021},
	note = {Publisher: Taylor \& Francis
\_eprint: https://doi.org/10.1080/01430750.2019.1594373},
	keywords = {soiling, solar energy},
	pages = {1307--1320},
}

@misc{hamelijnck_spatio-temporal_2021,
	title = {Spatio-{Temporal} {Variational} {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/2111.01732},
	doi = {10.48550/arXiv.2111.01732},
	abstract = {We introduce a scalable approach to Gaussian process inference that combines spatio-temporal filtering with natural gradient variational inference, resulting in a non-conjugate GP method for multivariate data that scales linearly with respect to time. Our natural gradient approach enables application of parallel filtering and smoothing, further reducing the temporal span complexity to be logarithmic in the number of time steps. We derive a sparse approximation that constructs a state-space model over a reduced set of spatial inducing points, and show that for separable Markov kernels the full and sparse cases exactly recover the standard variational GP, whilst exhibiting favourable computational properties. To further improve the spatial scaling we propose a mean-field assumption of independence between spatial locations which, when coupled with sparsity and parallelisation, leads to an efficient and accurate method for large spatio-temporal problems.},
	urldate = {2022-10-18},
	publisher = {arXiv},
	author = {Hamelijnck, Oliver and Wilkinson, William J. and Loppi, Niki A. and Solin, Arno and Damoulas, Theodoros},
	month = nov,
	year = {2021},
	note = {arXiv:2111.01732 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@book{absil_optimization_2008,
	address = {Princeton, N.J. ; Woodstock},
	title = {Optimization algorithms on matrix manifolds},
	isbn = {978-0-691-13298-3},
	publisher = {Princeton University Press},
	author = {Absil, P.-A. and Mahony, R. and Sepulchre, R.},
	year = {2008},
	note = {OCLC: ocn174129993},
	keywords = {manifolds},
}

@article{klus_eigendecompositions_2017,
	title = {Eigendecompositions of {Transfer} {Operators} in {Reproducing} {Kernel} {Hilbert} {Spaces}},
	url = {https://arxiv.org/abs/1712.01572v3},
	doi = {10.1007/s00332-019-09574-z},
	abstract = {Transfer operators such as the Perron--Frobenius or Koopman operator play an important role in the global analysis of complex dynamical systems. The eigenfunctions of these operators can be used to detect metastable sets, to project the dynamics onto the dominant slow processes, or to separate superimposed signals. We extend transfer operator theory to reproducing kernel Hilbert spaces and show that these operators are related to Hilbert space representations of conditional distributions, known as conditional mean embeddings in the machine learning community. Moreover, numerical methods to compute empirical estimates of these embeddings are akin to data-driven methods for the approximation of transfer operators such as extended dynamic mode decomposition and its variants. One main benefit of the presented kernel-based approaches is that these methods can be applied to any domain where a similarity measure given by a kernel is available. We illustrate the results with the aid of guiding examples and highlight potential applications in molecular dynamics as well as video and text data analysis.},
	language = {en},
	urldate = {2022-09-27},
	author = {Klus, Stefan and Schuster, Ingmar and Muandet, Krikamol},
	month = dec,
	year = {2017},
	keywords = {hilbert spaces, kernel means, kernel methods, koopman operators},
}

@misc{borovitskiy_matern_2022,
	title = {Mat{\textbackslash}'ern {Gaussian} processes on {Riemannian} manifolds},
	url = {http://arxiv.org/abs/2006.10160},
	doi = {10.48550/arXiv.2006.10160},
	abstract = {Gaussian processes are an effective model class for learning unknown functions, particularly in settings where accurately representing predictive uncertainty is of key importance. Motivated by applications in the physical sciences, the widely-used Mat{\textbackslash}'ern class of Gaussian processes has recently been generalized to model functions whose domains are Riemannian manifolds, by re-expressing said processes as solutions of stochastic partial differential equations. In this work, we propose techniques for computing the kernels of these processes on compact Riemannian manifolds via spectral theory of the Laplace-Beltrami operator in a fully constructive manner, thereby allowing them to be trained via standard scalable techniques such as inducing point methods. We also extend the generalization from the Mat{\textbackslash}'ern to the widely-used squared exponential Gaussian process. By allowing Riemannian Mat{\textbackslash}'ern Gaussian processes to be trained using well-understood techniques, our work enables their use in mini-batch, online, and non-conjugate settings, and makes them more accessible to machine learning practitioners.},
	urldate = {2022-09-19},
	publisher = {arXiv},
	author = {Borovitskiy, Viacheslav and Terenin, Alexander and Mostowsky, Peter and Deisenroth, Marc Peter},
	month = mar,
	year = {2022},
	note = {arXiv:2006.10160 [cs, stat]},
	keywords = {gaussian processes, kernel methods, manifolds, physics-informed kernels, 🔥},
}

@article{raissi_machine_2017,
	title = {Machine {Learning} of {Linear} {Differential} {Equations} using {Gaussian} {Processes}},
	volume = {348},
	issn = {00219991},
	url = {http://arxiv.org/abs/1701.02440},
	doi = {10.1016/j.jcp.2017.07.050},
	abstract = {This work leverages recent advances in probabilistic machine learning to discover conservation laws expressed by parametric linear equations. Such equations involve, but are not limited to, ordinary and partial differential, integro-differential, and fractional order operators. Here, Gaussian process priors are modified according to the particular form of such operators and are employed to infer parameters of the linear equations from scarce and possibly noisy observations. Such observations may come from experiments or "black-box" computer simulations.},
	urldate = {2022-06-27},
	journal = {Journal of Computational Physics},
	author = {Raissi, Maziar and Karniadakis, George Em},
	month = nov,
	year = {2017},
	note = {arXiv:1701.02440 [cs, math, stat]},
	keywords = {gaussian processes, machine learning, physics-informed learning, ✅},
	pages = {683--693},
}

@article{tartakovsky_physics-informed_2022,
	title = {Physics-informed {Gaussian} process regression for states estimation and forecasting in power grids},
	issn = {0169-2070},
	url = {https://www.sciencedirect.com/science/article/pii/S0169207022000474},
	doi = {10.1016/j.ijforecast.2022.03.007},
	abstract = {Real-time state estimation and forecasting are critical for the efficient operation of power grids. In this paper, a physics-informed Gaussian process regression (PhI-GPR) method is presented and used for forecasting and estimating the phase angle, angular speed, and wind mechanical power of a three-generator power grid system using sparse measurements. In standard data-driven Gaussian process regression (GPR), parameterized models for the prior statistics are fit by maximizing the marginal likelihood of observed data. In the PhI-GPR method, we propose to compute the prior statistics offline by solving stochastic differential equations (SDEs) governing the power grid dynamics. The short-term forecast of a power grid system dominated by wind generation is complicated by the stochastic nature of the wind and the resulting uncertainty in wind mechanical power. Here, we assume that the power grid dynamics are governed by swing equations, with the wind mechanical power fluctuating randomly in time. We solve these equations for the mean and covariances of the power grid states using the Monte Carlo simulation method. We demonstrate that the proposed PhI-GPR method can accurately forecast and estimate observed and unobserved states. For the considered problem, PhI-GPR has computational advantages over the ensemble Kalman filter (EnKF) method: In PhI-GPR, ensembles are computed offline and independently of the data acquisition process, whereas for EnFK, ensembles are computed online with data acquisition, rendering real-time forecast more challenging. We also demonstrate that the PhI-GPR forecast is more accurate than the EnKF forecast when the random mechanical wind power is non-Markovian. In contrast, the two methods produce similar forecasts for the Markovian mechanical wind power. For observed states, we show that PhI-GPR provides a forecast comparable to the standard data-driven GPR; both forecasts are significantly more accurate than the autoregressive integrated moving average (ARIMA) forecast. We also show that the ARIMA forecast is more sensitive to observation frequency and measurement errors than the PhI-GPR forecast.},
	language = {en},
	urldate = {2022-09-09},
	journal = {International Journal of Forecasting},
	author = {Tartakovsky, Alexandre M. and Ma, Tong and Barajas-Solano, David A. and Tipireddy, Ramakrishna},
	month = apr,
	year = {2022},
}

@incollection{pang_physics-informed_2020,
	address = {Cham},
	series = {Nonlinear {Systems} and {Complexity}},
	title = {Physics-{Informed} {Learning} {Machines} for {Partial} {Differential} {Equations}: {Gaussian} {Processes} {Versus} {Neural} {Networks}},
	isbn = {978-3-030-44992-6},
	shorttitle = {Physics-{Informed} {Learning} {Machines} for {Partial} {Differential} {Equations}},
	url = {https://doi.org/10.1007/978-3-030-44992-6_14},
	abstract = {We review and compare physics-informed learning models built upon Gaussian processes and deep neural networks for solving forward and inverse problems governed by linear and nonlinear partial differential equations. We define a unified data model on which Gaussian processes, physics-informed Gaussian processes, neural networks, and physics-informed neural networks are based. We develop continuous-time and discrete-time models to facilitate different application scenarios. We present a connection between a Gaussian process and an infinitely wide neural network, which enables us to obtain a “best” kernel, which is determined directly by the data. We demonstrate the implementation of physics-informed Gaussian processes and physics-informed neural networks using a pedagogical example. Additionally, we compare physics-informed Gaussian processes and physics-informed neural networks for two nonlinear partial differential equations, i.e. the 1D Burgers’ equation and the 2D Navier–Stokes, and provide guidance in choosing the proper machine learning model according to the problem type, i.e. forward or inverse problem, and the availability of data. These new methods for solving partial differential equations governing multi-physics problems do not require any grid, and they are simple to implement, and agnostic to specific application. Hence, we expect that variants and proper extensions of these methods will find broad applicability in the near future across different scientific disciplines but also in industrial applications.},
	language = {en},
	urldate = {2022-09-09},
	booktitle = {Emerging {Frontiers} in {Nonlinear} {Science}},
	publisher = {Springer International Publishing},
	author = {Pang, Guofei and Karniadakis, George Em},
	editor = {Kevrekidis, Panayotis G. and Cuevas-Maraver, Jesús and Saxena, Avadh},
	year = {2020},
	doi = {10.1007/978-3-030-44992-6_14},
	pages = {323--343},
}

@misc{xie_deep_2019,
	title = {Deep {Kernel} {Learning} via {Random} {Fourier} {Features}},
	url = {http://arxiv.org/abs/1910.02660},
	doi = {10.48550/arXiv.1910.02660},
	abstract = {Kernel learning methods are among the most effective learning methods and have been vigorously studied in the past decades. However, when tackling with complicated tasks, classical kernel methods are not flexible or "rich" enough to describe the data and hence could not yield satisfactory performance. In this paper, via Random Fourier Features (RFF), we successfully incorporate the deep architecture into kernel learning, which significantly boosts the flexibility and richness of kernel machines while keeps kernels' advantage of pairwise handling small data. With RFF, we could establish a deep structure and make every kernel in RFF layers could be trained end-to-end. Since RFF with different distributions could represent different kernels, our model has the capability of finding suitable kernels for each layer, which is much more flexible than traditional kernel-based methods where the kernel is pre-selected. This fact also helps yield a more sophisticated kernel cascade connection in the architecture. On small datasets (less than 1000 samples), for which deep learning is generally not suitable due to overfitting, our method achieves superior performance compared to advanced kernel methods. On large-scale datasets, including non-image and image classification tasks, our method also has competitive performance.},
	urldate = {2022-07-06},
	publisher = {arXiv},
	author = {Xie, Jiaxuan and Liu, Fanghui and Wang, Kaijie and Huang, Xiaolin},
	month = oct,
	year = {2019},
	note = {arXiv:1910.02660 [cs, stat]},
	keywords = {deep kernels, kernel learning, kernel methods, random fourier features},
}

@article{brehmer_flows_2020,
	title = {Flows for simultaneous manifold learning and density estimation},
	volume = {33},
	url = {https://proceedings.neurips.cc/paper/2020/hash/051928341be67dcba03f0e04104d9047-Abstract.html},
	language = {en},
	urldate = {2021-04-15},
	journal = {Advances in Neural Information Processing Systems},
	author = {Brehmer, Johann and Cranmer, Kyle},
	year = {2020},
	keywords = {generative modeling, manifolds, normalizing flows},
	pages = {442--453},
}

@article{bhouri_gaussian_2021,
	title = {Gaussian processes meet {NeuralODEs}: {A} {Bayesian} framework for learning the dynamics of partially observed systems from scarce and noisy data},
	shorttitle = {Gaussian processes meet {NeuralODEs}},
	url = {http://arxiv.org/abs/2103.03385},
	abstract = {This paper presents a machine learning framework (GP-NODE) for Bayesian systems identification from partial, noisy and irregular observations of nonlinear dynamical systems. The proposed method takes advantage of recent developments in differentiable programming to propagate gradient information through ordinary differential equation solvers and perform Bayesian inference with respect to unknown model parameters using Hamiltonian Monte Carlo sampling and Gaussian Process priors over the observed system states. This allows us to exploit temporal correlations in the observed data, and efficiently infer posterior distributions over plausible models with quantified uncertainty. Moreover, the use of sparsity-promoting priors such as the Finnish Horseshoe for free model parameters enables the discovery of interpretable and parsimonious representations for the underlying latent dynamics. A series of numerical studies is presented to demonstrate the effectiveness of the proposed GP-NODE method including predator-prey systems, systems biology, and a 50-dimensional human motion dynamical system. Taken together, our findings put forth a novel, flexible and robust workflow for data-driven model discovery under uncertainty. All code and data accompanying this manuscript are available online at {\textbackslash}url\{https://github.com/PredictiveIntelligenceLab/GP-NODEs\}.},
	urldate = {2021-05-26},
	journal = {arXiv:2103.03385 [physics, stat]},
	author = {Bhouri, Mohamed Aziz and Perdikaris, Paris},
	month = mar,
	year = {2021},
	note = {arXiv: 2103.03385},
	keywords = {gaussian processes, machine learning, neural ode, physics-informed learning},
}

@article{moriconi_high-dimensional_2020,
	title = {High-dimensional {Bayesian} optimization using low-dimensional feature spaces},
	volume = {109},
	issn = {1573-0565},
	url = {https://doi.org/10.1007/s10994-020-05899-z},
	doi = {10.1007/s10994-020-05899-z},
	abstract = {Bayesian optimization (BO) is a powerful approach for seeking the global optimum of expensive black-box functions and has proven successful for fine tuning hyper-parameters of machine learning models. However, BO is practically limited to optimizing 10–20 parameters. To scale BO to high dimensions, we usually make structural assumptions on the decomposition of the objective and/or exploit the intrinsic lower dimensionality of the problem, e.g. by using linear projections. We could achieve a higher compression rate with nonlinear projections, but learning these nonlinear embeddings typically requires much data. This contradicts the BO objective of a relatively small evaluation budget. To address this challenge, we propose to learn a low-dimensional feature space jointly with (a) the response surface and (b) a reconstruction mapping. Our approach allows for optimization of BO’s acquisition function in the lower-dimensional subspace, which significantly simplifies the optimization problem. We reconstruct the original parameter space from the lower-dimensional subspace for evaluating the black-box function. For meaningful exploration, we solve a constrained optimization problem.},
	language = {en},
	number = {9},
	urldate = {2022-08-16},
	journal = {Machine Learning},
	author = {Moriconi, Riccardo and Deisenroth, Marc Peter and Sesh Kumar, K. S.},
	month = sep,
	year = {2020},
	keywords = {bayesian optimization, dimensionality reduction, manifolds},
	pages = {1925--1943},
}

@misc{calandra_manifold_2016,
	title = {Manifold {Gaussian} {Processes} for {Regression}},
	url = {http://arxiv.org/abs/1402.5876},
	doi = {10.48550/arXiv.1402.5876},
	abstract = {Off-the-shelf Gaussian Process (GP) covariance functions encode smoothness assumptions on the structure of the function to be modeled. To model complex and non-differentiable functions, these smoothness assumptions are often too restrictive. One way to alleviate this limitation is to find a different representation of the data by introducing a feature space. This feature space is often learned in an unsupervised way, which might lead to data representations that are not useful for the overall regression task. In this paper, we propose Manifold Gaussian Processes, a novel supervised method that jointly learns a transformation of the data into a feature space and a GP regression from the feature space to observed space. The Manifold GP is a full GP and allows to learn data representations, which are useful for the overall regression task. As a proof-of-concept, we evaluate our approach on complex non-smooth functions where standard GPs perform poorly, such as step functions and robotics tasks with contacts.},
	urldate = {2022-08-16},
	publisher = {arXiv},
	author = {Calandra, Roberto and Peters, Jan and Rasmussen, Carl Edward and Deisenroth, Marc Peter},
	month = apr,
	year = {2016},
	note = {arXiv:1402.5876 [cs, stat]},
	keywords = {gaussian processes, manifolds},
}

@article{wang_mfpc-net_2020,
	title = {{MFPC}-{Net}: {Multi}-fidelity {Physics}-{Constrained} {Neural} {Process}},
	shorttitle = {{MFPC}-{Net}},
	url = {http://arxiv.org/abs/2010.01378},
	abstract = {In this work, we propose a network which can utilize computational cheap low-fidelity data together with limited high-fidelity data to train surrogate models, where the multi-fidelity data are generated from multiple underlying models. The network takes a context set as input (physical observation points, low fidelity solution at observed points) and output (high fidelity solution at observed points) pairs. It uses the neural process to learn a distribution over functions conditioned on context sets and provide the mean and standard deviation at target sets. Moreover, the proposed framework also takes into account the available physical laws that govern the data and imposes them as constraints in the loss function. The multi-fidelity physical constraint network (MFPC-Net) (1) takes datasets obtained from multiple models at the same time in the training, (2) takes advantage of available physical information, (3) learns a stochastic process which can encode prior beliefs about the correlation between two fidelity with a few observations, and (4) produces predictions with uncertainty. The ability of representing a class of functions is ensured by the property of neural process and is achieved by the global latent variables in the neural network. Physical constraints are added to the loss using Lagrange multipliers. An algorithm to optimize the loss function is proposed to effectively train the parameters in the network on an ad hoc basis. Once trained, one can obtain fast evaluations at the entire domain of interest given a few observation points from a new low-and high-fidelity model pair. Particularly, one can further identify the unknown parameters such as permeability fields in elliptic PDEs with a simple modification of the network. Several numerical examples for both forward and inverse problems are presented to demonstrate the performance of the proposed method.},
	urldate = {2021-05-25},
	journal = {arXiv:2010.01378 [physics]},
	author = {Wang, Yating and Lin, Guang},
	month = oct,
	year = {2020},
	note = {arXiv: 2010.01378},
	keywords = {computational physics, multi-fidelity, neural processes, numerical modelling},
}

@article{wang_eigenvector_2021,
	title = {On the eigenvector bias of {Fourier} feature networks: {From} regression to solving multi-scale {PDEs} with physics-informed neural networks},
	volume = {384},
	issn = {0045-7825},
	shorttitle = {On the eigenvector bias of {Fourier} feature networks},
	url = {https://www.sciencedirect.com/science/article/pii/S0045782521002759},
	doi = {10.1016/j.cma.2021.113938},
	abstract = {Physics-informed neural networks (PINNs) are demonstrating remarkable promise in integrating physical models with gappy and noisy observational data, but they still struggle in cases where the target functions to be approximated exhibit high-frequency or multi-scale features. In this work we investigate this limitation through the lens of Neural Tangent Kernel (NTK) theory and elucidate how PINNs are biased towards learning functions along the dominant eigen-directions of their limiting NTK. Using this observation, we construct novel architectures that employ spatio-temporal and multi-scale random Fourier features, and justify how such coordinate embedding layers can lead to robust and accurate PINN models. Numerical examples are presented for several challenging cases where conventional PINN models fail, including wave propagation and reaction–diffusion dynamics, illustrating how the proposed methods can be used to effectively tackle both forward and inverse problems involving partial differential equations with multi-scale behavior. All code an data accompanying this manuscript will be made publicly available at https://github.com/PredictiveIntelligenceLab/MultiscalePINNs.},
	language = {en},
	urldate = {2022-06-27},
	journal = {Computer Methods in Applied Mechanics and Engineering},
	author = {Wang, Sifan and Wang, Hanwen and Perdikaris, Paris},
	month = oct,
	year = {2021},
	keywords = {neural networks, neural tangent kernel, partial differential equations, physics-informed learning, random fourier features, scientific machine learning},
	pages = {113938},
}

@article{pan_physics-informed_2020,
	title = {Physics-{Informed} {Probabilistic} {Learning} of {Linear} {Embeddings} of {Non}-linear {Dynamics} {With} {Guaranteed} {Stability}},
	volume = {19},
	issn = {1536-0040},
	url = {http://arxiv.org/abs/1906.03663},
	doi = {10.1137/19M1267246},
	abstract = {The Koopman operator has emerged as a powerful tool for the analysis of nonlinear dynamical systems as it provides coordinate transformations to globally linearize the dynamics. While recent deep learning approaches have been useful in extracting the Koopman operator from a data-driven perspective, several challenges remain. In this work, we formalize the problem of learning the continuous-time Koopman operator with deep neural networks in a measure-theoretic framework. Our approach induces two types of models: differential and recurrent form, the choice of which depends on the availability of the governing equations and data. We then enforce a structural parameterization that renders the realization of the Koopman operator provably stable. A new autoencoder architecture is constructed, such that only the residual of the dynamic mode decomposition is learned. Finally, we employ mean-field variational inference (MFVI) on the aforementioned framework in a hierarchical Bayesian setting to quantify uncertainties in the characterization and prediction of the dynamics of observables. The framework is evaluated on a simple polynomial system, the Duffing oscillator, and an unstable cylinder wake flow with noisy measurements.},
	number = {1},
	urldate = {2022-08-22},
	journal = {SIAM Journal on Applied Dynamical Systems},
	author = {Pan, Shaowu and Duraisamy, Karthik},
	month = jan,
	year = {2020},
	note = {arXiv:1906.03663 [math, stat]},
	keywords = {dynamical systems, koopman operators, physics-informed learning},
	pages = {480--509},
}

@article{sun_surrogate_2020,
	title = {Surrogate modeling for fluid flows based on physics-constrained deep learning without simulation data},
	volume = {361},
	issn = {0045-7825},
	url = {http://www.sciencedirect.com/science/article/pii/S004578251930622X},
	doi = {10.1016/j.cma.2019.112732},
	abstract = {Numerical simulations on fluid dynamics problems primarily rely on spatially or/and temporally discretization of the governing equation using polynomials into a finite-dimensional algebraic system. Due to the multi-scale nature of the physics and sensitivity from meshing a complicated geometry, such process can be computational prohibitive for most real-time applications (e.g., clinical diagnosis and surgery planning) and many-query analyses (e.g., optimization design and uncertainty quantification). Therefore, developing a cost-effective surrogate model is of great practical significance. Deep learning (DL) has shown new promises for surrogate modeling due to its capability of handling strong nonlinearity and high dimensionality. However, the off-the-shelf DL architectures, success of which heavily relies on the large amount of training data and interpolatory nature of the problem, fail to operate when the data becomes sparse. Unfortunately, data is often insufficient in most parametric fluid dynamics problems since each data point in the parameter space requires an expensive numerical simulation based on the first principle, e.g., Navier–Stokes equations. In this paper, we provide a physics-constrained DL approach for surrogate modeling of fluid flows without relying on any simulation data. Specifically, a structured deep neural network (DNN) architecture is devised to enforce the initial and boundary conditions, and the governing partial differential equations (i.e., Navier–Stokes equations) are incorporated into the loss of the DNN to drive the training. Numerical experiments are conducted on a number of internal flows relevant to hemodynamics applications, and the forward propagation of uncertainties in fluid properties and domain geometry is studied as well. The results show excellent agreement on the flow field and forward-propagated uncertainties between the DL surrogate approximations and the first-principle numerical simulations.},
	language = {en},
	urldate = {2020-11-10},
	journal = {Computer Methods in Applied Mechanics and Engineering},
	author = {Sun, Luning and Gao, Han and Pan, Shaowu and Wang, Jian-Xun},
	month = apr,
	year = {2020},
	keywords = {navier-stokes, neural networks, physics-informed learning, uncertainty estimation},
	pages = {112732},
}

@misc{dellaporta_robust_2022,
	title = {Robust {Bayesian} {Inference} for {Simulator}-based {Models} via the {MMD} {Posterior} {Bootstrap}},
	url = {http://arxiv.org/abs/2202.04744},
	doi = {10.48550/arXiv.2202.04744},
	abstract = {Simulator-based models are models for which the likelihood is intractable but simulation of synthetic data is possible. They are often used to describe complex real-world phenomena, and as such can often be misspecified in practice. Unfortunately, existing Bayesian approaches for simulators are known to perform poorly in those cases. In this paper, we propose a novel algorithm based on the posterior bootstrap and maximum mean discrepancy estimators. This leads to a highly-parallelisable Bayesian inference algorithm with strong robustness properties. This is demonstrated through an in-depth theoretical study which includes generalisation bounds and proofs of frequentist consistency and robustness of our posterior. The approach is then assessed on a range of examples including a g-and-k distribution and a toggle-switch model.},
	urldate = {2022-08-25},
	publisher = {arXiv},
	author = {Dellaporta, Charita and Knoblauch, Jeremias and Damoulas, Theodoros and Briol, François-Xavier},
	month = feb,
	year = {2022},
	note = {arXiv:2202.04744 [cs, stat]},
}

@misc{baddoo_physics-informed_2021,
	title = {Physics-informed dynamic mode decomposition ({piDMD})},
	url = {http://arxiv.org/abs/2112.04307},
	doi = {10.48550/arXiv.2112.04307},
	abstract = {In this work, we demonstrate how physical principles -- such as symmetries, invariances, and conservation laws -- can be integrated into the dynamic mode decomposition (DMD). DMD is a widely-used data analysis technique that extracts low-rank modal structures and dynamics from high-dimensional measurements. However, DMD frequently produces models that are sensitive to noise, fail to generalize outside the training data, and violate basic physical laws. Our physics-informed DMD (piDMD) optimization, which may be formulated as a Procrustes problem, restricts the family of admissible models to a matrix manifold that respects the physical structure of the system. We focus on five fundamental physical principles -- conservation, self-adjointness, localization, causality, and shift-invariance -- and derive several closed-form solutions and efficient algorithms for the corresponding piDMD optimizations. With fewer degrees of freedom, piDMD models are less prone to overfitting, require less training data, and are often less computationally expensive to build than standard DMD models. We demonstrate piDMD on a range of challenging problems in the physical sciences, including energy-preserving fluid flow, travelling-wave systems, the Schr{\textbackslash}"odinger equation, solute advection-diffusion, a system with causal dynamics, and three-dimensional transitional channel flow. In each case, piDMD significantly outperforms standard DMD in metrics such as spectral identification, state prediction, and estimation of optimal forcings and responses.},
	urldate = {2022-08-22},
	publisher = {arXiv},
	author = {Baddoo, Peter J. and Herrmann, Benjamin and McKeon, Beverley J. and Kutz, J. Nathan and Brunton, Steven L.},
	month = dec,
	year = {2021},
	note = {arXiv:2112.04307 [physics]},
	keywords = {dynamic mode decomposition, koopman operators, physics-informed learning, ✅},
}

@article{qian_lift_2020,
	title = {Lift \& {Learn}: {Physics}-informed machine learning for large-scale nonlinear dynamical systems},
	volume = {406},
	issn = {0167-2789},
	shorttitle = {Lift \& {Learn}},
	url = {https://www.sciencedirect.com/science/article/pii/S0167278919307651},
	doi = {10.1016/j.physd.2020.132401},
	abstract = {We present Lift \& Learn, a physics-informed method for learning low-dimensional models for large-scale dynamical systems. The method exploits knowledge of a system’s governing equations to identify a coordinate transformation in which the system dynamics have quadratic structure. This transformation is called a lifting map because it often adds auxiliary variables to the system state. The lifting map is applied to data obtained by evaluating a model for the original nonlinear system. This lifted data is projected onto its leading principal components, and low-dimensional linear and quadratic matrix operators are fit to the lifted reduced data using a least-squares operator inference procedure. Analysis of our method shows that the Lift \& Learn models are able to capture the system physics in the lifted coordinates at least as accurately as traditional intrusive model reduction approaches. This preservation of system physics makes the Lift \& Learn models robust to changes in inputs. Numerical experiments on the FitzHugh–Nagumo neuron activation model and the compressible Euler equations demonstrate the generalizability of our model.},
	language = {en},
	urldate = {2022-08-22},
	journal = {Physica D: Nonlinear Phenomena},
	author = {Qian, Elizabeth and Kramer, Boris and Peherstorfer, Benjamin and Willcox, Karen},
	month = may,
	year = {2020},
	keywords = {koopman operators, physics-informed learning, 🔥},
	pages = {132401},
}

@misc{kostic_learning_2022,
	title = {Learning {Dynamical} {Systems} via {Koopman} {Operator} {Regression} in {Reproducing} {Kernel} {Hilbert} {Spaces}},
	url = {http://arxiv.org/abs/2205.14027},
	doi = {10.48550/arXiv.2205.14027},
	abstract = {We study a class of dynamical systems modelled as Markov chains that admit an invariant distribution via the corresponding transfer, or Koopman, operator. While data-driven algorithms to reconstruct such operators are well known, their relationship with statistical learning is largely unexplored. We formalize a framework to learn the Koopman operator from finite data trajectories of the dynamical system. We consider the restriction of this operator to a reproducing kernel Hilbert space and introduce a notion of risk, from which different estimators naturally arise. We link the risk with the estimation of the spectral decomposition of the Koopman operator. These observations motivate a reduced-rank operator regression (RRR) estimator. We derive learning bounds for the proposed estimator, holding both in i.i.d. and non i.i.d. settings, the latter in terms of mixing coefficients. Our results suggest RRR might be beneficial over other widely used estimators as confirmed in numerical experiments both for forecasting and mode decomposition.},
	urldate = {2022-09-06},
	publisher = {arXiv},
	author = {Kostic, Vladimir and Novelli, Pietro and Maurer, Andreas and Ciliberto, Carlo and Rosasco, Lorenzo and Pontil, Massimiliano},
	month = may,
	year = {2022},
	note = {arXiv:2205.14027 [cs, math]},
	keywords = {kernel learning, koopman operators, physics-informed kernels, physics-informed learning, 🔥},
}

@misc{solin_know_2019,
	title = {Know {Your} {Boundaries}: {Constraining} {Gaussian} {Processes} by {Variational} {Harmonic} {Features}},
	shorttitle = {Know {Your} {Boundaries}},
	url = {http://arxiv.org/abs/1904.05207},
	doi = {10.48550/arXiv.1904.05207},
	abstract = {Gaussian processes (GPs) provide a powerful framework for extrapolation, interpolation, and noise removal in regression and classification. This paper considers constraining GPs to arbitrarily-shaped domains with boundary conditions. We solve a Fourier-like generalised harmonic feature representation of the GP prior in the domain of interest, which both constrains the GP and attains a low-rank representation that is used for speeding up inference. The method scales as \${\textbackslash}mathcal\{O\}(nm{\textasciicircum}2)\$ in prediction and \${\textbackslash}mathcal\{O\}(m{\textasciicircum}3)\$ in hyperparameter learning for regression, where \$n\$ is the number of data points and \$m\$ the number of features. Furthermore, we make use of the variational approach to allow the method to deal with non-Gaussian likelihoods. The experiments cover both simulated and empirical data in which the boundary conditions allow for inclusion of additional physical information.},
	urldate = {2022-08-16},
	publisher = {arXiv},
	author = {Solin, Arno and Kok, Manon},
	month = apr,
	year = {2019},
	note = {arXiv:1904.05207 [cs, stat]},
	keywords = {gaussian processes, physics-informed learning},
}

@inproceedings{takeishi_kernel_2019,
	title = {Kernel {Learning} for {Data}-{Driven} {Spectral} {Analysis} of {Koopman} {Operators}},
	url = {https://proceedings.mlr.press/v101/takeishi19a.html},
	abstract = {Spectral analysis of the Koopman operators is a useful tool for studying nonlinear dynamical systems and has been utilized in various branches of science and engineering for purposes such as understanding complex phenomena and designing a controller. Several methods to compute the Koopman spectral analysis have been studied, among which data-driven methods are attracting attention. We focus on one of the popular data-driven methods, which is based on the Galerkin approximation of the operator using a basis estimated in a data-driven manner via the diffusion maps algorithm. The performance of this method with a finite amount of data depends on the choice of the kernel function used in diffusion maps, which creates a need for kernel selection. In this paper, we propose a method to learn the kernel function adaptively to obtain better performance in approximating spectra of the Koopman operator using the Galerkin approximation with diffusion maps. The proposed method depends on the multiple kernel learning scheme, and our objective function is based on the idea that a diffusion operator should commute with the Koopman operator. We also show the effectiveness of the proposed method empirically with numerical examples.},
	language = {en},
	urldate = {2022-09-06},
	booktitle = {Proceedings of {The} {Eleventh} {Asian} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Takeishi, Naoya},
	month = oct,
	year = {2019},
	note = {ISSN: 2640-3498},
	keywords = {kernel learning, koopman operators, physics-informed kernels, physics-informed learning, simulation, 🔥},
	pages = {956--971},
}

@article{baddoo_kernel_2022,
	title = {Kernel learning for robust dynamic mode decomposition: linear and nonlinear disambiguation optimization},
	volume = {478},
	shorttitle = {Kernel learning for robust dynamic mode decomposition},
	url = {https://royalsocietypublishing.org/doi/10.1098/rspa.2021.0830},
	doi = {10.1098/rspa.2021.0830},
	abstract = {Research in modern data-driven dynamical systems is typically focused on the three key challenges of high dimensionality, unknown dynamics and nonlinearity. The dynamic mode decomposition (DMD) has emerged as a cornerstone for modelling high-dimensional systems from data. However, the quality of the linear DMD model is known to be fragile with respect to strong nonlinearity, which contaminates the model estimate. By contrast, sparse identification of nonlinear dynamics learns fully nonlinear models, disambiguating the linear and nonlinear effects, but is restricted to low-dimensional systems. In this work, we present a kernel method that learns interpretable data-driven models for high-dimensional, nonlinear systems. Our method performs kernel regression on a sparse dictionary of samples that appreciably contribute to the dynamics. We show that this kernel method efficiently handles high-dimensional data and is flexible enough to incorporate partial knowledge of system physics. It is possible to recover the linear model contribution with this approach, thus separating the effects of the implicitly defined nonlinear terms. We demonstrate our approach on data from a range of nonlinear ordinary and partial differential equations. This framework can be used for many practical engineering tasks such as model order reduction, diagnostics, prediction, control and discovery of governing laws.},
	number = {2260},
	urldate = {2022-09-08},
	journal = {Proceedings of the Royal Society A: Mathematical, Physical and Engineering Sciences},
	author = {Baddoo, Peter J. and Herrmann, Benjamin and McKeon, Beverley J. and Brunton, Steven L.},
	month = apr,
	year = {2022},
	note = {Publisher: Royal Society},
	keywords = {dynamic mode decomposition, kernel learning, koopman operators, physics-informed learning, 🔥},
	pages = {20210830},
}

@article{zanini_estimating_2021,
	series = {19th {IFAC} {Symposium} on {System} {Identification} {SYSID} 2021},
	title = {Estimating {Koopman} operators for nonlinear dynamical systems: a nonparametric approach},
	volume = {54},
	issn = {2405-8963},
	shorttitle = {Estimating {Koopman} operators for nonlinear dynamical systems},
	url = {https://www.sciencedirect.com/science/article/pii/S2405896321012155},
	doi = {10.1016/j.ifacol.2021.08.441},
	abstract = {The Koopman operator provides a linear description of non-linear systems exploiting an embedding into an infinite dimensional space. Dynamic Mode Decomposition and Extended Dynamic Mode Decomposition are amongst the most popular finite dimensional approximations of the Koopman Operator. In this paper we capture their core essence as a dual version of the same problem, embedding them into the Kernel framework. To do so, we leverage the RKHS as a suitable space for learning the Koopman dynamics. Learning from finite length data automatically provides a finite dimensional approximation induced by data. Simulations and comparison with standard procedures are included.},
	language = {en},
	number = {7},
	urldate = {2022-09-02},
	journal = {IFAC-PapersOnLine},
	author = {Zanini, Francesco and Chiuso, Alessandro},
	month = jan,
	year = {2021},
	keywords = {gaussian processes, kernel methods, koopman operators, physics-informed learning, 🔥},
	pages = {691--696},
}

@misc{williams_kernel-based_2015,
	title = {A {Kernel}-{Based} {Approach} to {Data}-{Driven} {Koopman} {Spectral} {Analysis}},
	url = {http://arxiv.org/abs/1411.2260},
	doi = {10.48550/arXiv.1411.2260},
	abstract = {A data driven, kernel-based method for approximating the leading Koopman eigenvalues, eigenfunctions, and modes in problems with high dimensional state spaces is presented. This approach approximates the Koopman operator using a set of scalar observables, which are functions defined on state space, that is determined \{{\textbackslash}em implicitly\} by the choice of a kernel. This circumvents the computational issues that arise due to the number of basis functions required to span a "sufficiently rich" subspace of the space of scalar observables in these problems. We illustrate this method on the FitzHugh-Nagumo PDE, a prototypical example of a one-dimensional reaction diffusion system, and compare our results with related methods such as Dynamic Mode Decomposition (DMD) that have the same computational cost as our approach. In this example, the resulting approximations of the leading Koopman eigenvalues, eigenfunctions, and modes are both more accurate and less sensitive to the distribution of the data used in the computation than those produced by DMD.},
	urldate = {2022-09-02},
	publisher = {arXiv},
	author = {Williams, Matthew O. and Rowley, Clarence W. and Kevrekidis, Ioannis G.},
	month = jul,
	year = {2015},
	note = {arXiv:1411.2260 [math]},
	keywords = {kernel methods, koopman operators, physics-informed learning, 🔥},
}

@article{barron-gafford_photovoltaic_2016,
	title = {The {Photovoltaic} {Heat} {Island} {Effect}: {Larger} solar power plants increase local temperatures},
	volume = {6},
	copyright = {2016 The Author(s)},
	issn = {2045-2322},
	shorttitle = {The {Photovoltaic} {Heat} {Island} {Effect}},
	url = {https://www.nature.com/articles/srep35070},
	doi = {10.1038/srep35070},
	abstract = {While photovoltaic (PV) renewable energy production has surged, concerns remain about whether or not PV power plants induce a “heat island” (PVHI) effect, much like the increase in ambient temperatures relative to wildlands generates an Urban Heat Island effect in cities. Transitions to PV plants alter the way that incoming energy is reflected back to the atmosphere or absorbed, stored, and reradiated because PV plants change the albedo, vegetation, and structure of the terrain. Prior work on the PVHI has been mostly theoretical or based upon simulated models. Furthermore, past empirical work has been limited in scope to a single biome. Because there are still large uncertainties surrounding the potential for a PHVI effect, we examined the PVHI empirically with experiments that spanned three biomes. We found temperatures over a PV plant were regularly 3–4 °C warmer than wildlands at night, which is in direct contrast to other studies based on models that suggested that PV systems should decrease ambient temperatures. Deducing the underlying cause and scale of the PVHI effect and identifying mitigation strategies are key in supporting decision-making regarding PV development, particularly in semiarid landscapes, which are among the most likely for large-scale PV installations.},
	language = {en},
	number = {1},
	urldate = {2022-08-29},
	journal = {Scientific Reports},
	author = {Barron-Gafford, Greg A. and Minor, Rebecca L. and Allen, Nathan A. and Cronin, Alex D. and Brooks, Adria E. and Pavao-Zuckerman, Mitchell A.},
	month = oct,
	year = {2016},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {earth sciences, geophysics, heat islands, renewable energy, solar energy, 🔥},
	pages = {35070},
}

@misc{hutchinson_vector-valued_2021,
	title = {Vector-valued {Gaussian} {Processes} on {Riemannian} {Manifolds} via {Gauge} {Independent} {Projected} {Kernels}},
	url = {http://arxiv.org/abs/2110.14423},
	doi = {10.48550/arXiv.2110.14423},
	abstract = {Gaussian processes are machine learning models capable of learning unknown functions in a way that represents uncertainty, thereby facilitating construction of optimal decision-making systems. Motivated by a desire to deploy Gaussian processes in novel areas of science, a rapidly-growing line of research has focused on constructively extending these models to handle non-Euclidean domains, including Riemannian manifolds, such as spheres and tori. We propose techniques that generalize this class to model vector fields on Riemannian manifolds, which are important in a number of application areas in the physical sciences. To do so, we present a general recipe for constructing gauge independent kernels, which induce Gaussian vector fields, i.e. vector-valued Gaussian processes coherent with geometry, from scalar-valued Riemannian kernels. We extend standard Gaussian process training methods, such as variational inference, to this setting. This enables vector-valued Gaussian processes on Riemannian manifolds to be trained using standard methods and makes them accessible to machine learning practitioners.},
	urldate = {2022-08-16},
	publisher = {arXiv},
	author = {Hutchinson, Michael and Terenin, Alexander and Borovitskiy, Viacheslav and Takao, So and Teh, Yee Whye and Deisenroth, Marc Peter},
	month = nov,
	year = {2021},
	note = {arXiv:2110.14423 [cs, stat]},
	keywords = {gaussian processes, manifolds, physics-informed learning, 🔥},
}

@article{sillmann_understanding_2017,
	title = {Understanding, modeling and predicting weather and climate extremes: {Challenges} and opportunities},
	volume = {18},
	issn = {2212-0947},
	shorttitle = {Understanding, modeling and predicting weather and climate extremes},
	url = {https://www.sciencedirect.com/science/article/pii/S2212094717300440},
	doi = {10.1016/j.wace.2017.10.003},
	abstract = {Weather and climate extremes are identified as major areas necessitating further progress in climate research and have thus been selected as one of the World Climate Research Programme (WCRP) Grand Challenges. Here, we provide an overview of current challenges and opportunities for scientific progress and cross-community collaboration on the topic of understanding, modeling and predicting extreme events based on an expert workshop organized as part of the implementation of the WCRP Grand Challenge on Weather and Climate Extremes. In general, the development of an extreme event depends on a favorable initial state, the presence of large-scale drivers, and positive local feedbacks, as well as stochastic processes. We, therefore, elaborate on the scientific challenges related to large-scale drivers and local-to-regional feedback processes leading to extreme events. A better understanding of the drivers and processes will improve the prediction of extremes and will support process-based evaluation of the representation of weather and climate extremes in climate model simulations. Further, we discuss how to address these challenges by focusing on short-duration (less than three days) and long-duration (weeks to months) extreme events, their underlying mechanisms and approaches for their evaluation and prediction.},
	language = {en},
	urldate = {2022-08-15},
	journal = {Weather and Climate Extremes},
	author = {Sillmann, Jana and Thorarinsdottir, Thordis and Keenlyside, Noel and Schaller, Nathalie and Alexander, Lisa V. and Hegerl, Gabriele and Seneviratne, Sonia I. and Vautard, Robert and Zhang, Xuebin and Zwiers, Francis W.},
	month = dec,
	year = {2017},
	keywords = {climate modeling, review, weather},
	pages = {65--74},
}

@misc{wang_physics_2022,
	title = {Physics {Informed} {Deep} {Kernel} {Learning}},
	url = {http://arxiv.org/abs/2006.04976},
	doi = {10.48550/arXiv.2006.04976},
	abstract = {Deep kernel learning is a promising combination of deep neural networks and nonparametric function learning. However, as a data driven approach, the performance of deep kernel learning can still be restricted by scarce or insufficient data, especially in extrapolation tasks. To address these limitations, we propose Physics Informed Deep Kernel Learning (PI-DKL) that exploits physics knowledge represented by differential equations with latent sources. Specifically, we use the posterior function sample of the Gaussian process as the surrogate for the solution of the differential equation, and construct a generative component to integrate the equation in a principled Bayesian hybrid framework. For efficient and effective inference, we marginalize out the latent variables in the joint probability and derive a collapsed model evidence lower bound (ELBO), based on which we develop a stochastic model estimation algorithm. Our ELBO can be viewed as a nice, interpretable posterior regularization objective. On synthetic datasets and real-world applications, we show the advantage of our approach in both prediction accuracy and uncertainty quantification.},
	urldate = {2022-08-15},
	publisher = {arXiv},
	author = {Wang, Zheng and Xing, Wei and Kirby, Robert and Zhe, Shandian},
	month = jan,
	year = {2022},
	note = {arXiv:2006.04976 [cs, stat]},
	keywords = {deep kernels, gaussian processes, physics-informed learning},
}

@misc{liu_random_2021,
	title = {Random {Features} for {Kernel} {Approximation}: {A} {Survey} on {Algorithms}, {Theory}, and {Beyond}},
	shorttitle = {Random {Features} for {Kernel} {Approximation}},
	url = {http://arxiv.org/abs/2004.11154},
	doi = {10.48550/arXiv.2004.11154},
	abstract = {Random features is one of the most popular techniques to speed up kernel methods in large-scale problems. Related works have been recognized by the NeurIPS Test-of-Time award in 2017 and the ICML Best Paper Finalist in 2019. The body of work on random features has grown rapidly, and hence it is desirable to have a comprehensive overview on this topic explaining the connections among various algorithms and theoretical results. In this survey, we systematically review the work on random features from the past ten years. First, the motivations, characteristics and contributions of representative random features based algorithms are summarized according to their sampling schemes, learning procedures, variance reduction properties and how they exploit training data. Second, we review theoretical results that center around the following key question: how many random features are needed to ensure a high approximation quality or no loss in the empirical/expected risks of the learned estimator. Third, we provide a comprehensive evaluation of popular random features based algorithms on several large-scale benchmark datasets and discuss their approximation quality and prediction performance for classification. Last, we discuss the relationship between random features and modern over-parameterized deep neural networks (DNNs), including the use of high dimensional random features in the analysis of DNNs as well as the gaps between current theoretical and empirical results. This survey may serve as a gentle introduction to this topic, and as a users' guide for practitioners interested in applying the representative algorithms and understanding theoretical results under various technical assumptions. We hope that this survey will facilitate discussion on the open problems in this topic, and more importantly, shed light on future research directions.},
	urldate = {2022-07-06},
	publisher = {arXiv},
	author = {Liu, Fanghui and Huang, Xiaolin and Chen, Yudong and Suykens, Johan A. K.},
	month = jul,
	year = {2021},
	note = {arXiv:2004.11154 [cs, stat]},
	keywords = {kernel learning, kernel methods, random features, random fourier features, review},
}

@article{pang_neural-net-induced_2019,
	title = {Neural-net-induced {Gaussian} process regression for function approximation and {PDE} solution},
	volume = {384},
	issn = {0021-9991},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999119301032},
	doi = {10.1016/j.jcp.2019.01.045},
	abstract = {Neural-net-induced Gaussian process (NNGP) regression inherits both the high expressivity of deep neural networks (deep NNs) as well as the uncertainty quantification property of Gaussian processes (GPs). We generalize the current NNGP to first include a larger number of hyperparameters and subsequently train the model by maximum likelihood estimation. Unlike previous works on NNGP that targeted classification, here we apply the generalized NNGP to function approximation and to solving partial differential equations (PDEs). Specifically, we develop an analytical iteration formula to compute the covariance function of GP induced by deep NN with an error-function nonlinearity. We compare the performance of the generalized NNGP for function approximations and PDE solutions with those of GPs and fully-connected NNs. We observe that for smooth functions the generalized NNGP can yield the same order of accuracy with GP, while both NNGP and GP outperform deep NN. For non-smooth functions, the generalized NNGP is superior to GP and comparable or superior to deep NN.},
	language = {en},
	urldate = {2021-04-30},
	journal = {Journal of Computational Physics},
	author = {Pang, Guofei and Yang, Liu and Karniadakis, George Em},
	month = may,
	year = {2019},
	keywords = {gaussian processes, machine learning, neural networks, partial differential equations, uncertainty estimation},
	pages = {270--288},
}

@inproceedings{boyle_dependent_2004,
	title = {Dependent {Gaussian} {Processes}},
	volume = {17},
	url = {https://papers.nips.cc/paper/2004/hash/59eb5dd36914c29b299c84b7ddaf08ec-Abstract.html},
	abstract = {Gaussian processes are usually parameterised in terms of their covari-          ance functions. However, this makes it difficult to deal with multiple          outputs, because ensuring that the covariance matrix is positive definite          is problematic. An alternative formulation is to treat Gaussian processes          as white noise sources convolved with smoothing kernels, and to param-          eterise the kernel instead. Using this, we extend Gaussian processes to          handle multiple, coupled outputs.},
	urldate = {2022-06-27},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {MIT Press},
	author = {Boyle, Phillip and Frean, Marcus},
	year = {2004},
	keywords = {bayesian statistics, gaussian processes, kernel convolution, kernel means, kernel methods, uncertainty estimation},
}

@inproceedings{saemundsson_variational_2020,
	title = {Variational {Integrator} {Networks} for {Physically} {Structured} {Embeddings}},
	url = {http://proceedings.mlr.press/v108/saemundsson20a.html},
	abstract = {Learning workable representations of dynamical systems is becoming an increasingly important problem in a number of application areas. By leveraging recent work connecting deep neural networks to s...},
	language = {en},
	urldate = {2020-12-11},
	booktitle = {International {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Saemundsson, Steindor and Terenin, Alexander and Hofmann, Katja and Deisenroth, Marc},
	month = jun,
	year = {2020},
	note = {ISSN: 2640-3498},
	keywords = {discriminative autoencoders, interpretability, manifolds, physics-informed learning, variational autoencoders, 🔥},
	pages = {3078--3087},
}

@article{veers_grand_2019,
	title = {Grand challenges in the science of wind energy},
	copyright = {Copyright © 2019, American Association for the Advancement of Science},
	issn = {0036-8075, 1095-9203},
	url = {https://science.sciencemag.org/content/early/2019/10/09/science.aau2027},
	doi = {10.1126/science.aau2027},
	abstract = {Harvested by advanced technical systems honed over decades of research and development, wind energy has become a mainstream energy resource. However, continued innovation is needed to realize the potential of wind to serve global demand for clean energy. Here, we outline three interdependent, cross-disciplinary grand challenges underpinning this research endeavor. The first is the need for a deeper understanding of the physics of atmospheric flow in the critical zone of plant operation. The second involves science and engineering of the largest dynamic, rotating machines in the world. The third encompasses optimization and control of fleets of wind plants working synergistically within the electricity grid. Addressing these challenges could enable wind power to provide as much as half of our global electricity needs.},
	language = {en},
	urldate = {2020-11-12},
	journal = {Science},
	author = {Veers, Paul and Dykes, Katherine and Lantz, Eric and Barth, Stephan and Bottasso, Carlo L. and Carlson, Ola and Clifton, Andrew and Green, Johney and Green, Peter and Holttinen, Hannele and Laird, Daniel and Lehtomäki, Ville and Lundquist, Julie K. and Manwell, James and Marquis, Melinda and Meneveau, Charles and Moriarty, Patrick and Munduate, Xabier and Muskulus, Michael and Naughton, Jonathan and Pao, Lucy and Paquette, Joshua and Peinke, Joachim and Robertson, Amy and Rodrigo, Javier Sanz and Sempreviva, Anna Maria and Smith, J. Charles and Tuohy, Aidan and Wiser, Ryan},
	month = oct,
	year = {2019},
	pmid = {31601706},
	note = {Publisher: American Association for the Advancement of Science
Section: Review},
	keywords = {climate, renewable energy, review, wind energy, wind power forecasting},
}

@techreport{stoll_hydropower_2017,
	title = {Hydropower {Modeling} {Challenges}},
	url = {http://www.osti.gov/servlets/purl/1353003/},
	language = {en},
	number = {NREL/TP--5D00-68231, 1353003},
	urldate = {2020-11-13},
	author = {Stoll, Brady and Andrade, Juan and Cohen, Stuart and Brinkman, Greg and Brancucci Martinez-Anido, Carlo},
	month = apr,
	year = {2017},
	doi = {10.2172/1353003},
	keywords = {climate, fluid dynamics, hydropower, review, weather},
	pages = {NREL/TP--5D00--68231, 1353003},
}

@inproceedings{perera_machine_2014,
	address = {Cham},
	series = {Lecture {Notes} in {Computer} {Science}},
	title = {Machine {Learning} {Techniques} for {Supporting} {Renewable} {Energy} {Generation} and {Integration}: {A} {Survey}},
	isbn = {978-3-319-13290-7},
	shorttitle = {Machine {Learning} {Techniques} for {Supporting} {Renewable} {Energy} {Generation} and {Integration}},
	doi = {10.1007/978-3-319-13290-7_7},
	abstract = {The extraction of energy from renewable sources is rapidly growing. The current pace of technological development makes it commercially viable to harness energy from sun, wind, geothermal and many other renewable sources. Because of the negative effects on the environment and the economy, conventional energy sources like natural gas, crude oil and coal are coming under political and economic pressure. Thus, they require a better mix of energy sources with a higher percentage of renewable energy sources. Harnessing energy from renewable sources range from small scale (e.g., a single household) to large scale (e.g., power plants producing several MWs to a few GWs providing energy to an entire city). An inherent characteristic common to all renewable power plants is that power generation is dependent on environmental parameters and thus cannot be fully controlled or planned for in advance. In a power grid, it is necessary to predict the amount of power that will be generated in the future, including those from the renewable sources, as fluctuations in capacity and/or quality can have negative impacts on the physical health of the entire grid as well as the quality of life of its users. As renewable power plants continue to expand, it will also be necessary to determine their optimal sizes, locations and configurations. In addition, management of the smart grid, in which the renewable energy plants are integrated, is also a challenging problem. In this paper we provide a survey on different machine learning techniques used to address the above issues related to renewable energy generation and integration.},
	language = {en},
	booktitle = {Data {Analytics} for {Renewable} {Energy} {Integration}},
	publisher = {Springer International Publishing},
	author = {Perera, Kasun S. and Aung, Zeyar and Woon, Wei Lee},
	editor = {Woon, Wei Lee and Aung, Zeyar and Madnick, Stuart},
	year = {2014},
	keywords = {renewable energy, review, solar energy},
	pages = {81--96},
}

@article{joseph_neural_2020,
	title = {Neural hierarchical models of ecological populations},
	volume = {23},
	copyright = {© 2020 John Wiley \& Sons Ltd/CNRS},
	issn = {1461-0248},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/ele.13462},
	doi = {https://doi.org/10.1111/ele.13462},
	abstract = {Neural networks are increasingly being used in science to infer hidden dynamics of natural systems from noisy observations, a task typically handled by hierarchical models in ecology. This article describes a class of hierarchical models parameterised by neural networks – neural hierarchical models. The derivation of such models analogises the relationship between regression and neural networks. A case study is developed for a neural dynamic occupancy model of North American bird populations, trained on millions of detection/non-detection time series for hundreds of species, providing insights into colonisation and extinction at a continental scale. Flexible models are increasingly needed that scale to large data and represent ecological processes. Neural hierarchical models satisfy this need, providing a bridge between deep learning and ecological modelling that combines the function representation power of neural networks with the inferential capacity of hierarchical models.},
	language = {en},
	number = {4},
	urldate = {2020-12-11},
	journal = {Ecology Letters},
	author = {Joseph, Maxwell B.},
	year = {2020},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/ele.13462},
	keywords = {ecological modeling, hierarchical model, neural networks},
	pages = {734--747},
}

@article{jones_toward_2017,
	title = {Toward a new generation of agricultural system data, models, and knowledge products: {State} of agricultural systems science},
	volume = {155},
	issn = {0308521X},
	shorttitle = {Toward a new generation of agricultural system data, models, and knowledge products},
	url = {https://linkinghub.elsevier.com/retrieve/pii/S0308521X1630590X},
	doi = {10.1016/j.agsy.2016.09.021},
	language = {en},
	urldate = {2020-11-13},
	journal = {Agricultural Systems},
	author = {Jones, James W. and Antle, John M. and Basso, Bruno and Boote, Kenneth J. and Conant, Richard T. and Foster, Ian and Godfray, H. Charles J. and Herrero, Mario and Howitt, Richard E. and Janssen, Sander and Keating, Brian A. and Munoz-Carpena, Rafael and Porter, Cheryl H. and Rosenzweig, Cynthia and Wheeler, Tim R.},
	month = jul,
	year = {2017},
	keywords = {agriculture, review},
	pages = {269--288},
}

@incollection{guo_toward_2015,
	title = {Toward a new generation of ecological modelling techniques},
	volume = {27},
	isbn = {978-0-444-63536-5},
	url = {https://linkinghub.elsevier.com/retrieve/pii/B9780444635365000028},
	language = {en},
	urldate = {2020-11-13},
	booktitle = {Developments in {Environmental} {Modelling}},
	publisher = {Elsevier},
	author = {Guo, Chunanbo and Park, Young-Seuk and Liu, Yang and Lek, Sovan},
	year = {2015},
	doi = {10.1016/B978-0-444-63536-5.00002-8},
	keywords = {ecological modeling, review},
	pages = {11--44},
}

@inproceedings{zhi_learning_2022,
	title = {Learning {Efficient} and {Robust} {Ordinary} {Differential} {Equations} via {Invertible} {Neural} {Networks}},
	url = {https://proceedings.mlr.press/v162/zhi22a.html},
	abstract = {Advances in differentiable numerical integrators have enabled the use of gradient descent techniques to learn ordinary differential equations (ODEs), where a flexible function approximator (often a neural network) is used to estimate the system dynamics, given as a time derivative. However, these integrators can be unsatisfactorily slow and unstable when learning systems of ODEs from long sequences. We propose to learn an ODE of interest from data by viewing its dynamics as a vector field related to another base vector field via a diffeomorphism (i.e., a differentiable bijection), represented by an invertible neural network (INN). By learning both the INN and the dynamics of the base ODE, we provide an avenue to offload some of the complexity in modelling the dynamics directly on to the INN. Consequently, by restricting the base ODE to be amenable to integration, we can speed up and improve the robustness of integrating trajectories from the learned system. We demonstrate the efficacy of our method in training and evaluating benchmark ODE systems, as well as within continuous-depth neural networks models. We show that our approach attains speed-ups of up to two orders of magnitude when integrating learned ODEs.},
	language = {en},
	urldate = {2022-07-19},
	booktitle = {Proceedings of the 39th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Zhi, Weiming and Lai, Tin and Ott, Lionel and Bonilla, Edwin V. and Ramos, Fabio},
	month = jun,
	year = {2022},
	note = {ISSN: 2640-3498},
	keywords = {bijections, invertible neural networks, neural ode, ✅},
	pages = {27060--27074},
}

@article{yang_inference_2021,
	title = {Inference of dynamic systems from noisy and sparse data via manifold-constrained {Gaussian} processes},
	volume = {118},
	url = {https://www.pnas.org/doi/10.1073/pnas.2020397118},
	doi = {10.1073/pnas.2020397118},
	number = {15},
	urldate = {2022-08-16},
	journal = {Proceedings of the National Academy of Sciences},
	author = {Yang, Shihao and Wong, Samuel W. K. and Kou, S. C.},
	month = apr,
	year = {2021},
	note = {Publisher: Proceedings of the National Academy of Sciences},
	keywords = {dynamical systems, gaussian processes, manifolds, physics-informed learning, 🔥},
	pages = {e2020397118},
}

@misc{raissi_numerical_2017,
	title = {Numerical {Gaussian} {Processes} for {Time}-dependent and {Non}-linear {Partial} {Differential} {Equations}},
	url = {http://arxiv.org/abs/1703.10230},
	doi = {10.48550/arXiv.1703.10230},
	abstract = {We introduce the concept of numerical Gaussian processes, which we define as Gaussian processes with covariance functions resulting from temporal discretization of time-dependent partial differential equations. Numerical Gaussian processes, by construction, are designed to deal with cases where: (1) all we observe are noisy data on black-box initial conditions, and (2) we are interested in quantifying the uncertainty associated with such noisy data in our solutions to time-dependent partial differential equations. Our method circumvents the need for spatial discretization of the differential operators by proper placement of Gaussian process priors. This is an attempt to construct structured and data-efficient learning machines, which are explicitly informed by the underlying physics that possibly generated the observed data. The effectiveness of the proposed approach is demonstrated through several benchmark problems involving linear and nonlinear time-dependent operators. In all examples, we are able to recover accurate approximations of the latent solutions, and consistently propagate uncertainty, even in cases involving very long time integration.},
	urldate = {2022-08-15},
	publisher = {arXiv},
	author = {Raissi, Maziar and Perdikaris, Paris and Karniadakis, George Em},
	month = mar,
	year = {2017},
	note = {arXiv:1703.10230 [cs, math, stat]},
	keywords = {gaussian processes, partial differential equations, physics-informed learning, 🔥},
}

@article{mooij_joint_2020,
	title = {Joint {Causal} {Inference} from {Multiple} {Contexts}},
	volume = {21},
	issn = {1533-7928},
	url = {http://jmlr.org/papers/v21/17-123.html},
	abstract = {The gold standard for discovering causal relations is by means of experimentation. Over the last decades, alternative methods have been proposed that can infer causal relations between variables from certain statistical patterns in purely observational data. We introduce Joint Causal Inference (JCI), a novel approach to causal discovery from multiple data sets from different contexts that elegantly unifies both approaches. JCI is a causal modeling framework rather than a specific algorithm, and it can be implemented using any causal discovery algorithm that can take into account certain background knowledge. JCI can deal with different types of interventions (e.g., perfect, imperfect, stochastic, etc.) in a unified fashion, and does not require knowledge of intervention targets or types in case of interventional data. We explain how several well-known causal discovery algorithms can be seen as addressing special cases of the JCI framework, and we also propose novel implementations that extend existing causal discovery methods for purely observational data to the JCI setting. We evaluate different JCI implementations on synthetic data and on flow cytometry protein expression data and conclude that JCI implementations can considerably outperform state-of-the-art causal discovery algorithms.},
	number = {99},
	urldate = {2022-08-04},
	journal = {Journal of Machine Learning Research},
	author = {Mooij, Joris M. and Magliacane, Sara and Claassen, Tom},
	year = {2020},
	pages = {1--108},
}

@inproceedings{schwobel_probabilistic_2022,
	title = {Probabilistic {Spatial} {Transformer} {Networks}},
	url = {https://openreview.net/forum?id=HFUxb_Uiqec},
	abstract = {A Bayesian extension to spatial transformer networks},
	language = {en},
	urldate = {2022-08-04},
	author = {Schwöbel, Pola and Warburg, Frederik Rahbæk and Jørgensen, Martin and Madsen, Kristoffer Hougaard and Hauberg, Søren},
	month = jun,
	year = {2022},
}

@inproceedings{shi_conditional_2022,
	title = {Conditional {Simulation} {Using} {Diffusion} {Schrödinger} {Bridges}},
	url = {https://openreview.net/forum?id=H9Lu6P8sqec},
	abstract = {We provide a novel diffusion-based algorithm to sample complex posterior distributions using Schrödinger bridges.},
	language = {en},
	urldate = {2022-08-04},
	author = {Shi, Yuyang and Bortoli, Valentin De and Deligiannidis, George and Doucet, Arnaud},
	month = jun,
	year = {2022},
}

@inproceedings{besserve_learning_2022,
	title = {Learning soft interventions in complex equilibrium systems},
	url = {https://openreview.net/forum?id=HrBgldLo9ec},
	abstract = {Motivated by environmental economics, we develop a theoretical foundations and algorithmic tools to design soft interventions in cyclic causal models.},
	language = {en},
	urldate = {2022-08-04},
	author = {Besserve, Michel and Schölkopf, Bernhard},
	month = jun,
	year = {2022},
}

@inproceedings{qian_d-code_2022,
	title = {D-{CODE}: {Discovering} {Closed}-form {ODEs} from {Observed} {Trajectories}},
	shorttitle = {D-{CODE}},
	url = {https://openreview.net/forum?id=wENMvIsxNN},
	abstract = {For centuries, scientists have manually designed closed-form ordinary differential equations (ODEs) to model dynamical systems. An automated tool to distill closed-form ODEs from observed...},
	language = {en},
	urldate = {2022-08-04},
	author = {Qian, Zhaozhi and Kacprzyk, Krzysztof and Schaar, Mihaela van der},
	month = feb,
	year = {2022},
}

@misc{scholkopf_towards_2021,
	title = {Towards {Causal} {Representation} {Learning}},
	url = {http://arxiv.org/abs/2102.11107},
	doi = {10.48550/arXiv.2102.11107},
	abstract = {The two fields of machine learning and graphical causality arose and developed separately. However, there is now cross-pollination and increasing interest in both fields to benefit from the advances of the other. In the present paper, we review fundamental concepts of causal inference and relate them to crucial open problems of machine learning, including transfer and generalization, thereby assaying how causality can contribute to modern machine learning research. This also applies in the opposite direction: we note that most work in causality starts from the premise that the causal variables are given. A central problem for AI and causality is, thus, causal representation learning, the discovery of high-level causal variables from low-level observations. Finally, we delineate some implications of causality for machine learning and propose key research areas at the intersection of both communities.},
	urldate = {2022-08-04},
	publisher = {arXiv},
	author = {Schölkopf, Bernhard and Locatello, Francesco and Bauer, Stefan and Ke, Nan Rosemary and Kalchbrenner, Nal and Goyal, Anirudh and Bengio, Yoshua},
	month = feb,
	year = {2021},
	note = {arXiv:2102.11107 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
}

@misc{deleu_bayesian_2022,
	title = {Bayesian {Structure} {Learning} with {Generative} {Flow} {Networks}},
	url = {http://arxiv.org/abs/2202.13903},
	doi = {10.48550/arXiv.2202.13903},
	abstract = {In Bayesian structure learning, we are interested in inferring a distribution over the directed acyclic graph (DAG) structure of Bayesian networks, from data. Defining such a distribution is very challenging, due to the combinatorially large sample space, and approximations based on MCMC are often required. Recently, a novel class of probabilistic models, called Generative Flow Networks (GFlowNets), have been introduced as a general framework for generative modeling of discrete and composite objects, such as graphs. In this work, we propose to use a GFlowNet as an alternative to MCMC for approximating the posterior distribution over the structure of Bayesian networks, given a dataset of observations. Generating a sample DAG from this approximate distribution is viewed as a sequential decision problem, where the graph is constructed one edge at a time, based on learned transition probabilities. Through evaluation on both simulated and real data, we show that our approach, called DAG-GFlowNet, provides an accurate approximation of the posterior over DAGs, and it compares favorably against other methods based on MCMC or variational inference.},
	urldate = {2022-08-04},
	publisher = {arXiv},
	author = {Deleu, Tristan and Góis, António and Emezue, Chris and Rankawat, Mansi and Lacoste-Julien, Simon and Bauer, Stefan and Bengio, Yoshua},
	month = jun,
	year = {2022},
	note = {arXiv:2202.13903 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@inproceedings{wang_meta-learning_2022,
	title = {Meta-{Learning} without {Data} via {Wasserstein} {Distributionally}-{Robust} {Model} {Fusion}},
	url = {https://openreview.net/forum?id=Hq8Ug88j5ec},
	abstract = {Existing meta-learning works assume that each task has available training and testing data. However, we can only use many available pre-trained models without accessing their training data in...},
	language = {en},
	urldate = {2022-08-04},
	author = {Wang, Zhenyi and Wang, Xiaoyang and Shen, Li and Suo, Qiuling and Song, Kaiqiang and Yu, Dong and Shen, Yan and Gao, Mingchen},
	month = jun,
	year = {2022},
}

@article{antle_towards_2017,
	title = {Towards a new generation of agricultural system data, models and knowledge products: {Design} and improvement},
	volume = {155},
	issn = {0308521X},
	shorttitle = {Towards a new generation of agricultural system data, models and knowledge products},
	url = {https://linkinghub.elsevier.com/retrieve/pii/S0308521X16306096},
	doi = {10.1016/j.agsy.2016.10.002},
	language = {en},
	urldate = {2020-11-13},
	journal = {Agricultural Systems},
	author = {Antle, John M and Basso, Bruno and Conant, Richard T and Godfray, H Charles J and Jones, James W and Herrero, Mario and Howitt, Richard E and Keating, Brian A and Munoz-Carpena, Rafael and Rosenzweig, Cynthia and Tittonell, Pablo and Wheeler, Tim R},
	month = jul,
	year = {2017},
	keywords = {agriculture, design, ecology, land use planning, machine learning, review},
	pages = {255--268},
}

@article{bakhshaii_review_2019,
	title = {A review of a new generation of wildfire–atmosphere modeling},
	copyright = {Published by NRC Research Press},
	url = {https://cdnsciencepub.com/doi/abs/10.1139/cjfr-2018-0138},
	doi = {10.1139/cjfr-2018-0138},
	abstract = {One of the first significant developments in wildfire modeling research was to introduce heat flux as wildfire line intensity (kW·m–1). This idea could be adapted to using weather station measureme...},
	language = {en},
	urldate = {2021-02-23},
	journal = {Canadian Journal of Forest Research},
	author = {Bakhshaii, A. and Johnson, E. A.},
	month = jan,
	year = {2019},
	note = {Publisher: NRC Research Press},
	keywords = {climate modeling, dynamical systems, fire, review, weather},
}

@misc{xu_infinitely_2022,
	title = {Infinitely {Deep} {Bayesian} {Neural} {Networks} with {Stochastic} {Differential} {Equations}},
	url = {http://arxiv.org/abs/2102.06559},
	doi = {10.48550/arXiv.2102.06559},
	abstract = {We perform scalable approximate inference in continuous-depth Bayesian neural networks. In this model class, uncertainty about separate weights in each layer gives hidden units that follow a stochastic differential equation. We demonstrate gradient-based stochastic variational inference in this infinite-parameter setting, producing arbitrarily-flexible approximate posteriors. We also derive a novel gradient estimator that approaches zero variance as the approximate posterior over weights approaches the true posterior. This approach brings continuous-depth Bayesian neural nets to a competitive comparison against discrete-depth alternatives, while inheriting the memory-efficient training and tunable precision of Neural ODEs.},
	urldate = {2022-06-27},
	publisher = {arXiv},
	author = {Xu, Winnie and Chen, Ricky T. Q. and Li, Xuechen and Duvenaud, David},
	month = jan,
	year = {2022},
	note = {Number: arXiv:2102.06559
arXiv:2102.06559 [cs, stat]},
	keywords = {bayesian neural networks, machine learning, physics-informed learning, physics-informed neural networks, stochastics differential equations, 🔥},
}

@article{raissi_hidden_2018,
	title = {Hidden physics models: {Machine} learning of nonlinear partial differential equations},
	volume = {357},
	issn = {0021-9991},
	shorttitle = {Hidden physics models},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999117309014},
	doi = {10.1016/j.jcp.2017.11.039},
	abstract = {While there is currently a lot of enthusiasm about “big data”, useful data is usually “small” and expensive to acquire. In this paper, we present a new paradigm of learning partial differential equations from small data. In particular, we introduce hidden physics models, which are essentially data-efficient learning machines capable of leveraging the underlying laws of physics, expressed by time dependent and nonlinear partial differential equations, to extract patterns from high-dimensional data generated from experiments. The proposed methodology may be applied to the problem of learning, system identification, or data-driven discovery of partial differential equations. Our framework relies on Gaussian processes, a powerful tool for probabilistic inference over functions, that enables us to strike a balance between model complexity and data fitting. The effectiveness of the proposed approach is demonstrated through a variety of canonical problems, spanning a number of scientific domains, including the Navier–Stokes, Schrödinger, Kuramoto–Sivashinsky, and time dependent linear fractional equations. The methodology provides a promising new direction for harnessing the long-standing developments of classical methods in applied mathematics and mathematical physics to design learning machines with the ability to operate in complex domains without requiring large quantities of data.},
	language = {en},
	urldate = {2022-06-27},
	journal = {Journal of Computational Physics},
	author = {Raissi, Maziar and Karniadakis, George Em},
	month = mar,
	year = {2018},
	keywords = {bayesian statistics, inverse problems, machine learning, nonlinear dynamical systems, partial differential equations, physics-informed learning, small data, uncertainty estimation, 🔥},
	pages = {125--141},
}

@article{mitrovic_representation_2020,
	title = {Representation {Learning} via {Invariant} {Causal} {Mechanisms}},
	url = {http://arxiv.org/abs/2010.07922},
	abstract = {Self-supervised learning has emerged as a strategy to reduce the reliance on costly supervised signal by pretraining representations only using unlabeled data. These methods combine heuristic proxy classification tasks with data augmentations and have achieved significant success, but our theoretical understanding of this success remains limited. In this paper we analyze self-supervised representation learning using a causal framework. We show how data augmentations can be more effectively utilized through explicit invariance constraints on the proxy classifiers employed during pretraining. Based on this, we propose a novel self-supervised objective, Representation Learning via Invariant Causal Mechanisms (ReLIC), that enforces invariant prediction of proxy targets across augmentations through an invariance regularizer which yields improved generalization guarantees. Further, using causality we generalize contrastive learning, a particular kind of self-supervised method, and provide an alternative theoretical explanation for the success of these methods. Empirically, ReLIC significantly outperforms competing methods in terms of robustness and out-of-distribution generalization on ImageNet, while also significantly outperforming these methods on Atari achieving above human-level performance on \$51\$ out of \$57\$ games.},
	urldate = {2020-12-11},
	journal = {arXiv:2010.07922 [cs, stat]},
	author = {Mitrovic, Jovana and McWilliams, Brian and Walker, Jacob and Buesing, Lars and Blundell, Charles},
	month = oct,
	year = {2020},
	note = {arXiv: 2010.07922},
	keywords = {causality, computer vision, discriminative autoencoders, interpretability, machine learning, variational autoencoders},
}

@article{lee_high-fidelity_2020,
	title = {High-{Fidelity} {Synthesis} with {Disentangled} {Representation}},
	url = {http://arxiv.org/abs/2001.04296},
	abstract = {Learning disentangled representation of data without supervision is an important step towards improving the interpretability of generative models. Despite recent advances in disentangled representation learning, existing approaches often suffer from the trade-off between representation learning and generation performance i.e. improving generation quality sacrifices disentanglement performance). We propose an Information-Distillation Generative Adversarial Network (ID-GAN), a simple yet generic framework that easily incorporates the existing state-of-the-art models for both disentanglement learning and high-fidelity synthesis. Our method learns disentangled representation using VAE-based models, and distills the learned representation with an additional nuisance variable to the separate GAN-based generator for high-fidelity synthesis. To ensure that both generative models are aligned to render the same generative factors, we further constrain the GAN generator to maximize the mutual information between the learned latent code and the output. Despite the simplicity, we show that the proposed method is highly effective, achieving comparable image generation quality to the state-of-the-art methods using the disentangled representation. We also show that the proposed decomposition leads to an efficient and stable model design, and we demonstrate photo-realistic high-resolution image synthesis results (1024x1024 pixels) for the first time using the disentangled representations.},
	urldate = {2020-12-11},
	journal = {arXiv:2001.04296 [cs, eess]},
	author = {Lee, Wonkwang and Kim, Donggyun and Hong, Seunghoon and Lee, Honglak},
	month = jan,
	year = {2020},
	note = {arXiv: 2001.04296},
	keywords = {computer vision, discriminative autoencoders, interpretability, variational autoencoders},
}

@inproceedings{gao_learning_2020,
	title = {Learning {Energy}-{Based} {Models} by {Diffusion} {Recovery} {Likelihood}},
	url = {https://openreview.net/forum?id=v_1Soh8QUNc},
	abstract = {While energy-based models (EBMs) exhibit a number of desirable properties, training and sampling on high-dimensional datasets remains challenging. Inspired by recent progress on diffusion...},
	language = {en},
	urldate = {2021-04-14},
	author = {Gao, Ruiqi and Song, Yang and Poole, Ben and Wu, Ying Nian and Kingma, Diederik P.},
	month = sep,
	year = {2020},
	keywords = {energy-based modelling, training},
}

@article{song_how_2021,
	title = {How to {Train} {Your} {Energy}-{Based} {Models}},
	url = {http://arxiv.org/abs/2101.03288},
	abstract = {Energy-Based Models (EBMs), also known as non-normalized probabilistic models, specify probability density or mass functions up to an unknown normalizing constant. Unlike most other probabilistic models, EBMs do not place a restriction on the tractability of the normalizing constant, thus are more flexible to parameterize and can model a more expressive family of probability distributions. However, the unknown normalizing constant of EBMs makes training particularly difficult. Our goal is to provide a friendly introduction to modern approaches for EBM training. We start by explaining maximum likelihood training with Markov chain Monte Carlo (MCMC), and proceed to elaborate on MCMC-free approaches, including Score Matching (SM) and Noise Constrastive Estimation (NCE). We highlight theoretical connections among these three approaches, and end with a brief survey on alternative training methods, which are still under active research. Our tutorial is targeted at an audience with basic understanding of generative models who want to apply EBMs or start a research project in this direction.},
	urldate = {2021-04-14},
	journal = {arXiv:2101.03288 [cs, stat]},
	author = {Song, Yang and Kingma, Diederik P.},
	month = feb,
	year = {2021},
	note = {arXiv: 2101.03288},
	keywords = {energy-based modelling, machine learning, training},
}

@article{matsubara_deep_nodate,
	title = {Deep {Energy}-{Based} {Modeling} of {Discrete}-{Time} {Physics}},
	abstract = {Physical phenomena in the real world are often described by energy-based modeling theories, such as Hamiltonian mechanics or the Landau theory, which yield various physical laws. Recent developments in neural networks have enabled the mimicking of the energy conservation law by learning the underlying continuoustime differential equations. However, this may not be possible in discrete time, which is often the case in practical learning and computation. Moreover, other physical laws have been overlooked in the previous neural network models. In this study, we propose a deep energy-based physical model that admits a speciﬁc differential geometric structure. From this structure, the conservation or dissipation law of energy and the mass conservation law follow naturally. To ensure the energetic behavior in discrete time, we also propose an automatic discrete differentiation algorithm that enables neural networks to employ the discrete gradient method.},
	language = {en},
	author = {Matsubara, Takashi and Ishikawa, Ai and Yaguchi, Takaharu},
	keywords = {energy-based modelling, machine learning, physics-informed learning},
	pages = {12},
}

@article{che_your_nodate,
	title = {Your {GAN} is {Secretly} an {Energy}-based {Model} and {You} {Should} {Use} {Discriminator} {Driven} {Latent} {Sampling}},
	abstract = {The sum of the implicit generator log-density log pg of a GAN with the logit score of the discriminator deﬁnes an energy function which yields the true data density when the generator is imperfect but the discriminator is optimal. This makes it possible to improve on the typical generator (with implicit density pg). We show that samples can be generated from this modiﬁed density by sampling in latent space according to an energy-based model induced by the sum of the latent prior log-density and the discriminator output score. We call this process of running Markov Chain Monte Carlo in the latent space, and then applying the generator function, Discriminator Driven Latent Sampling (DDLS). We show that DDLS is highly efﬁcient compared to previous methods which work in the high-dimensional pixel space, and can be applied to improve on previously trained GANs of many types. We evaluate DDLS on both synthetic and real-world datasets qualitatively and quantitatively. On CIFAR-10, DDLS substantially improves the Inception Score of an off-the-shelf pre-trained SN-GAN [1] from 8.22 to 9.09 which is comparable to the class-conditional BigGAN [2] model. This achieves a new state-of-the-art in the unconditional image synthesis setting without introducing extra parameters or additional training.},
	language = {en},
	author = {Che, Tong and Zhang, Ruixiang and Sohl-Dickstein, Jascha and Larochelle, Hugo and Paull, Liam and Cao, Yuan and Bengio, Yoshua},
	keywords = {energy-based modelling, generative adversarial networks, generative modeling, machine learning},
	pages = {13},
}

@article{grathwohl_your_2020,
	title = {Your {Classifier} is {Secretly} an {Energy} {Based} {Model} and {You} {Should} {Treat} it {Like} {One}},
	url = {http://arxiv.org/abs/1912.03263},
	abstract = {We propose to reinterpret a standard discriminative classiﬁer of p(y{\textbar}x) as an energy based model for the joint distribution p(x, y). In this setting, the standard class probabilities can be easily computed as well as unnormalized values of p(x) and p(x{\textbar}y). Within this framework, standard discriminative architectures may be used and the model can also be trained on unlabeled data. We demonstrate that energy based training of the joint distribution improves calibration, robustness, and out-of-distribution detection while also enabling our models to generate samples rivaling the quality of recent GAN approaches. We improve upon recently proposed techniques for scaling up the training of energy based models and present an approach which adds little overhead compared to standard classiﬁcation training. Our approach is able to achieve performance rivaling the state-of-the-art in both generative and discriminative learning within one hybrid model.},
	language = {en},
	urldate = {2021-01-13},
	journal = {arXiv:1912.03263 [cs, stat]},
	author = {Grathwohl, Will and Wang, Kuan-Chieh and Jacobsen, Jörn-Henrik and Duvenaud, David and Norouzi, Mohammad and Swersky, Kevin},
	month = sep,
	year = {2020},
	note = {arXiv: 1912.03263},
	keywords = {computer vision, energy-based modelling, machine learning},
}

@article{fletcher_learning_2019,
	title = {Learning about climate change uncertainty enables flexible water infrastructure planning},
	volume = {10},
	copyright = {2019 The Author(s)},
	issn = {2041-1723},
	url = {https://www.nature.com/articles/s41467-019-09677-x},
	doi = {10.1038/s41467-019-09677-x},
	abstract = {Water resources planning requires decision-making about infrastructure development under uncertainty in future regional climate conditions. However, uncertainty in climate change projections will evolve over the 100-year lifetime of a dam as new climate observations become available. Flexible strategies in which infrastructure is proactively designed to be changed in the future have the potential to meet water supply needs without expensive over-building. Evaluating tradeoffs between flexible and traditional static planning approaches requires extension of current paradigms for planning under climate change uncertainty which do not assess opportunities to reduce uncertainty in the future. We develop a new planning framework that assesses the potential to learn about regional climate change over time and therefore evaluates the appropriateness of flexible approaches today. We demonstrate it on a reservoir planning problem in Mombasa, Kenya. This approach identifies opportunities to reliably use incremental approaches, enabling adaptation investments to reach more vulnerable communities with fewer resources.},
	language = {en},
	number = {1},
	urldate = {2020-12-11},
	journal = {Nature Communications},
	author = {Fletcher, Sarah and Lickley, Megan and Strzepek, Kenneth},
	month = apr,
	year = {2019},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {civil engineering, climate modeling, computational design, computational fluid dynamics, computational physics, water modelling},
	pages = {1782},
}

@misc{lu_artificial_2012,
	type = {Review {Article}},
	title = {Artificial {Intelligence} in {Civil} {Engineering}},
	url = {https://www.hindawi.com/journals/mpe/2012/145974/},
	abstract = {Artificial intelligence is a branch of computer science, involved in the research, design, and application of intelligent computer. Traditional methods for modeling and optimizing complex structure systems require huge amounts of computing resources, and artificial-intelligence-based solutions can often provide valuable alternatives for efficiently solving problems in the civil engineering. This paper summarizes recently developed methods and theories in the developing direction for applications of artificial intelligence in civil engineering, including evolutionary computation, neural networks, fuzzy systems, expert system, reasoning, classification, and learning, as well as others like chaos theory, cuckoo search, firefly algorithm, knowledge-based engineering, and simulated annealing. The main research trends are also pointed out in the end. The paper provides an overview of the advances of artificial intelligence applied in civil engineering.},
	language = {en},
	urldate = {2020-12-11},
	journal = {Mathematical Problems in Engineering},
	author = {Lu, Pengzhen and Chen, Shengyong and Zheng, Yujun},
	month = dec,
	year = {2012},
	doi = {https://doi.org/10.1155/2012/145974},
	doi = {https://doi.org/10.1155/2012/145974},
	note = {ISSN: 1024-123X
Pages: e145974
Publisher: Hindawi
Volume: 2012},
	keywords = {civil engineering, computational design, land use planning, machine learning, urban planning},
}

@inproceedings{odaudu_machine_2019,
	title = {Machine {Learning} for {Strategic} {Urban} {Planning}},
	doi = {10.1109/NigeriaComputConf45974.2019.8949665},
	abstract = {Data mining is an important part of strategic planning for the development of modern urban settlement with capacities to accommodate population explosion. Developing countries are fast becoming urbanized giving the developments and opportunities that are lacking in rural areas. Data regarding urban development such as satellite image need to be analysed to ascertain the possibilities for further development or opening up of new settlements. This work presents a binary sub-pixel and feature based method of classification to detect water bodies and vegetation in earth observatory images. In this work, the images were subjected data pre-processing, feature extraction, and analysed the data using machine learning classification method to detection regions that support urban expansion or development of new settlement. The proposed method achieved 88.93\% accuracy and 0.06\% RMSE.},
	booktitle = {2019 2nd {International} {Conference} of the {IEEE} {Nigeria} {Computer} {Chapter} ({NigeriaComputConf})},
	author = {Odaudu, S. N. and Umoh, I. J. and Mu’azu, M. B. and Adedokun, E. A.},
	month = oct,
	year = {2019},
	keywords = {computational design, feature engineering, feature extraction, image classification, land use planning, machine learning, object detection, remote sensing, urban planning},
	pages = {1--7},
}

@article{shaygan_spatial_2014,
	title = {Spatial {Multi}-{Objective} {Optimization} {Approach} for {Land} {Use} {Allocation} {Using} {NSGA}-{II}},
	volume = {7},
	issn = {2151-1535},
	doi = {10.1109/JSTARS.2013.2280697},
	abstract = {Analysis and evaluation of land use patterns are of prime importance for natural resources management. Recent studies on land use allocation have been mainly based on linear programming optimization. Although these methods have the ability to solve multi-objective problems, spatial aspects of optimization are not considered when they are used for land use management. This study applied the non-dominated sorting genetic algorithm II (NSGA-II) to optimize land-use allocation in the Taleghan watershed, northwest of Karaj, Iran. The four land use classes of irrigated farming, dry farming, rangeland, and other uses were extracted from the ETM+ image. The objective functions of the proposed model were erosion, economic return, suitability, and compactness-compatibility. A novel crossover operator called exchange randomly block (ERB) was used to exchange information between individuals. Results showed that the optimization model can find a set of optimal land use combinations in accordance with the proposed conditions. For comparison purposes, land use allocation was also done using the combined goal attainment-multi-objective land allocation (GoA-MOLA) approach. The results showed that NSGA-II performance acceptably when compared to GoA-MOLA.},
	number = {3},
	journal = {IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing},
	author = {Shaygan, M. and Alimohammadi, A. and Mansourian, A. and Govara, Z. S. and Kalami, S. M.},
	month = mar,
	year = {2014},
	note = {Conference Name: IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing},
	keywords = {economics, geophysics, land use planning, linear programming, multi-objective optimization, optimization, remote sensing, resource management, soil, spatio-temporal modelling, urban planning},
	pages = {906--916},
}

@article{wong_grand_2015,
	title = {Grand {Challenges} in {Sustainable} {Design} and {Construction}},
	volume = {1},
	issn = {2297-3362},
	url = {https://www.frontiersin.org/articles/10.3389/fbuil.2015.00022/full},
	doi = {10.3389/fbuil.2015.00022},
	abstract = {Grand Challenges in Sustainable Design and Construction},
	language = {English},
	urldate = {2020-12-11},
	journal = {Frontiers in Built Environment},
	author = {Wong, Nyuk Hien},
	year = {2015},
	note = {Publisher: Frontiers},
	keywords = {computational design, construction, land use planning, sustainable design, urban planning},
}

@article{liu_machine_2017,
	title = {A machine learning-based method for the large-scale evaluation of the qualities of the urban environment},
	volume = {65},
	issn = {0198-9715},
	url = {http://www.sciencedirect.com/science/article/pii/S0198971516301831},
	doi = {10.1016/j.compenvurbsys.2017.06.003},
	abstract = {Given the present size of modern cities, it is beyond the perceptual capacity of most people to develop a good knowledge about the qualities of the urban space at every street corner. Correspondingly, for planners, it is also difficult to accurately answer questions such as ‘where the quality of the physical environment is the most dilapidated in the city that regeneration should be given first consideration’ and ‘in fast urbanising cities, how is the city appearance changing’. To address this issue, in the present study, we present a computer vision method that contains three machine learning models for the large-scale and automatic evaluation on the qualities of the urban environment by leveraging state-of-the-art machine learning techniques and wide-coverage street view images. From various physical qualities that have been identified by previous research to be important for the urban visual experience, we choose two key qualities, the construction and maintenance quality of building facade and the continuity of street wall, to be measured in this research. To test the validity of the proposed method, we compare the machine scores with public rating scores collected on-site from 752 passers-by at 56 locations in the city. We show that the machine learning models can produce a medium-to-good estimation of people's real experience, and the modelling results can be applied in many ways by researchers, planners and local residents.},
	language = {en},
	urldate = {2020-12-11},
	journal = {Computers, Environment and Urban Systems},
	author = {Liu, Lun and Silva, Elisabete A. and Wu, Chunyang and Wang, Hui},
	month = sep,
	year = {2017},
	keywords = {architecture, computational design, land use planning, machine learning, urban planning},
	pages = {113--125},
}

@inproceedings{gilpin_explaining_2018,
	title = {Explaining {Explanations}: {An} {Overview} of {Interpretability} of {Machine} {Learning}},
	shorttitle = {Explaining {Explanations}},
	doi = {10.1109/DSAA.2018.00018},
	abstract = {There has recently been a surge of work in explanatory artificial intelligence (XAI). This research area tackles the important problem that complex machines and algorithms often cannot provide insights into their behavior and thought processes. XAI allows users and parts of the internal system to be more transparent, providing explanations of their decisions in some level of detail. These explanations are important to ensure algorithmic fairness, identify potential bias/problems in the training data, and to ensure that the algorithms perform as expected. However, explanations produced by these systems is neither standardized nor systematically assessed. In an effort to create best practices and identify open challenges, we describe foundational concepts of explainability and show how they can be used to classify existing literature. We discuss why current approaches to explanatory methods especially for deep neural networks are insufficient. Finally, based on our survey, we conclude with suggested future research directions for explanatory artificial intelligence.},
	booktitle = {2018 {IEEE} 5th {International} {Conference} on {Data} {Science} and {Advanced} {Analytics} ({DSAA})},
	author = {Gilpin, Leilani H. and Bau, David and Yuan, Ben Z. and Bajwa, Ayesha and Specter, Michael and Kagal, Lalana},
	month = oct,
	year = {2018},
	keywords = {artificial intelligence, complexity, decision trees, interpretability, neural networks},
	pages = {80--89},
}

@article{barredo_arrieta_explainable_2020,
	title = {Explainable {Artificial} {Intelligence} ({XAI}): {Concepts}, taxonomies, opportunities and challenges toward responsible {AI}},
	volume = {58},
	issn = {1566-2535},
	shorttitle = {Explainable {Artificial} {Intelligence} ({XAI})},
	url = {https://www.sciencedirect.com/science/article/pii/S1566253519308103},
	doi = {10.1016/j.inffus.2019.12.012},
	abstract = {In the last few years, Artificial Intelligence (AI) has achieved a notable momentum that, if harnessed appropriately, may deliver the best of expectations over many application sectors across the field. For this to occur shortly in Machine Learning, the entire community stands in front of the barrier of explainability, an inherent problem of the latest techniques brought by sub-symbolism (e.g. ensembles or Deep Neural Networks) that were not present in the last hype of AI (namely, expert systems and rule based models). Paradigms underlying this problem fall within the so-called eXplainable AI (XAI) field, which is widely acknowledged as a crucial feature for the practical deployment of AI models. The overview presented in this article examines the existing literature and contributions already done in the field of XAI, including a prospect toward what is yet to be reached. For this purpose we summarize previous efforts made to define explainability in Machine Learning, establishing a novel definition of explainable Machine Learning that covers such prior conceptual propositions with a major focus on the audience for which the explainability is sought. Departing from this definition, we propose and discuss about a taxonomy of recent contributions related to the explainability of different Machine Learning models, including those aimed at explaining Deep Learning methods for which a second dedicated taxonomy is built and examined in detail. This critical literature analysis serves as the motivating background for a series of challenges faced by XAI, such as the interesting crossroads of data fusion and explainability. Our prospects lead toward the concept of Responsible Artificial Intelligence, namely, a methodology for the large-scale implementation of AI methods in real organizations with fairness, model explainability and accountability at its core. Our ultimate goal is to provide newcomers to the field of XAI with a thorough taxonomy that can serve as reference material in order to stimulate future research advances, but also to encourage experts and professionals from other disciplines to embrace the benefits of AI in their activity sectors, without any prior bias for its lack of interpretability.},
	language = {en},
	urldate = {2021-11-24},
	journal = {Information Fusion},
	author = {Barredo Arrieta, Alejandro and Díaz-Rodríguez, Natalia and Del Ser, Javier and Bennetot, Adrien and Tabik, Siham and Barbado, Alberto and Garcia, Salvador and Gil-Lopez, Sergio and Molina, Daniel and Benjamins, Richard and Chatila, Raja and Herrera, Francisco},
	month = jun,
	year = {2020},
	keywords = {data fusion, fairness, interpretability, machine learning, neural networks},
	pages = {82--115},
}

@inproceedings{lin_deep_2014,
	title = {Deep learning of spatio-temporal features with geometric-based moving point detection for motion segmentation},
	doi = {10.1109/ICRA.2014.6907299},
	abstract = {This paper introduces an approach to accomplish motion segmentation from a moving stereo camera based on deep learning. Previous work on moving object detection mostly use point features based on 3D geometric constraints. However, point features require good features, and are hard to detect or to be matched correctly in situations where objects have smooth textures. To alleviate this problem, learning high-level spatio-temporal features unsupervisedly from raw image data based on Reconstruction Independent Component Analysis (RICA) autoencoders is proposed. Despite the power of the new spatio-temporal features, these features cannot not learn and be used to interpret 3D geometry of dynamic scenes, which is critical for moving object detection from moving cameras. As detected moving points based on 3D geometric constraints still contain valuable information of 3D scene as well as the camera egomotion, we propose a framework that incorporates both the detected moving point results and the learned spatio-temporal features as inputs to Recursive Neural Networks (RNN) that performs motion segmentation. Both features effectively complement each other. The proposed approach is demonstrated with real-world stereo video data that contains multiple moving objects, and has achieved 26\% better detection rate over the existing 3D geometric-based moving points detector.},
	booktitle = {2014 {IEEE} {International} {Conference} on {Robotics} and {Automation} ({ICRA})},
	author = {Lin, Tsung-Han and Wang, Chieh-Chih},
	month = may,
	year = {2014},
	note = {ISSN: 1050-4729},
	keywords = {computer vision, feature extraction, image segmentation, soccer, spatio-temporal modelling, sports analytics},
	pages = {3058--3065},
}

@inproceedings{danks_basisdevae_2021,
	title = {{BasisDeVAE}: {Interpretable} {Simultaneous} {Dimensionality} {Reduction} and {Feature}-{Level} {Clustering} with {Derivative}-{Based} {Variational} {Autoencoders}},
	shorttitle = {{BasisDeVAE}},
	url = {https://proceedings.mlr.press/v139/danks21a.html},
	abstract = {The Variational Autoencoder (VAE) performs effective nonlinear dimensionality reduction in a variety of problem settings. However, the black-box neural network decoder function typically employed limits the ability of the decoder function to be constrained and interpreted, making the use of VAEs problematic in settings where prior knowledge should be embedded within the decoder. We present DeVAE, a novel VAE-based model with a derivative-based forward mapping, allowing for greater control over decoder behaviour via specification of the decoder function in derivative space. Additionally, we show how DeVAE can be paired with a sparse clustering prior to create BasisDeVAE and perform interpretable simultaneous dimensionality reduction and feature-level clustering. We demonstrate the performance and scalability of the DeVAE and BasisDeVAE models on synthetic and real-world data and present how the derivative-based approach allows for expressive yet interpretable forward models which respect prior knowledge.},
	language = {en},
	urldate = {2021-11-26},
	booktitle = {Proceedings of the 38th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Danks, Dominic and Yau, Christopher},
	month = jul,
	year = {2021},
	note = {ISSN: 2640-3498},
	keywords = {bayesian optimization, discriminative autoencoders, interpretability, soccer, sports analytics, variational autoencoders},
	pages = {2410--2420},
}

@article{sapienza_deep_2019,
	title = {Deep {Neural} {Networks} for {Optimal} {Team} {Composition}},
	volume = {2},
	issn = {2624-909X},
	url = {https://www.frontiersin.org/article/10.3389/fdata.2019.00014},
	doi = {10.3389/fdata.2019.00014},
	abstract = {Cooperation is a fundamental social mechanism, whose effects on human performance have been investigated in several environments. Online games are modern-days natural settings in which cooperation strongly affects human behavior. Every day, millions of players connect and play together in team-based games: the patterns of cooperation can either foster or hinder individual skill learning and performance. This work has three goals: (i) identifying teammates' influence on players' performance in the short and long term, (ii) designing a computational framework to recommend teammates to improve players' performance, and (iii) setting to demonstrate that such improvements can be predicted via deep learning. We leverage a large dataset from Dota 2, a popular Multiplayer Online Battle Arena game. We generate a directed co-play network, whose links' weights depict the effect of teammates on players' performance. Specifically, we propose a measure of network influence that captures skill transfer from player to player over time. We then use such framing to design a recommendation system to suggest new teammates based on a modified deep neural autoencoder and we demonstrate its state-of-the-art recommendation performance. We finally provide insights into skill transfer effects: our experimental results demonstrate that such dynamics can be predicted using deep neural networks.},
	urldate = {2021-11-26},
	journal = {Frontiers in Big Data},
	author = {Sapienza, Anna and Goyal, Palash and Ferrara, Emilio},
	year = {2019},
	keywords = {machine learning, optimization, soccer, sports analytics},
	pages = {14},
}

@article{grosnit_high-dimensional_2021,
	title = {High-{Dimensional} {Bayesian} {Optimisation} with {Variational} {Autoencoders} and {Deep} {Metric} {Learning}},
	url = {http://arxiv.org/abs/2106.03609},
	abstract = {We introduce a method combining variational autoencoders (VAEs) and deep metric learning to perform Bayesian optimisation (BO) over high-dimensional and structured input spaces. By adapting ideas from deep metric learning, we use label guidance from the blackbox function to structure the VAE latent space, facilitating the Gaussian process fit and yielding improved BO performance. Importantly for BO problem settings, our method operates in semi-supervised regimes where only few labelled data points are available. We run experiments on three real-world tasks, achieving state-of-the-art results on the penalised logP molecule generation benchmark using just 3\% of the labelled data required by previous approaches. As a theoretical contribution, we present a proof of vanishing regret for VAE BO.},
	urldate = {2021-11-26},
	journal = {arXiv:2106.03609 [cs]},
	author = {Grosnit, Antoine and Tutunov, Rasul and Maraval, Alexandre Max and Griffiths, Ryan-Rhys and Cowen-Rivers, Alexander I. and Yang, Lin and Zhu, Lin and Lyu, Wenlong and Chen, Zhitang and Wang, Jun and Peters, Jan and Bou-Ammar, Haitham},
	month = nov,
	year = {2021},
	note = {arXiv: 2106.03609},
	keywords = {bayesian optimization, machine learning, soccer, sports analytics, variational autoencoders},
}

@article{vahdat_nvae_2021,
	title = {{NVAE}: {A} {Deep} {Hierarchical} {Variational} {Autoencoder}},
	shorttitle = {{NVAE}},
	url = {http://arxiv.org/abs/2007.03898},
	abstract = {Normalizing flows, autoregressive models, variational autoencoders (VAEs), and deep energy-based models are among competing likelihood-based frameworks for deep generative learning. Among them, VAEs have the advantage of fast and tractable sampling and easy-to-access encoding networks. However, they are currently outperformed by other models such as normalizing flows and autoregressive models. While the majority of the research in VAEs is focused on the statistical challenges, we explore the orthogonal direction of carefully designing neural architectures for hierarchical VAEs. We propose Nouveau VAE (NVAE), a deep hierarchical VAE built for image generation using depth-wise separable convolutions and batch normalization. NVAE is equipped with a residual parameterization of Normal distributions and its training is stabilized by spectral regularization. We show that NVAE achieves state-of-the-art results among non-autoregressive likelihood-based models on the MNIST, CIFAR-10, CelebA 64, and CelebA HQ datasets and it provides a strong baseline on FFHQ. For example, on CIFAR-10, NVAE pushes the state-of-the-art from 2.98 to 2.91 bits per dimension, and it produces high-quality images on CelebA HQ. To the best of our knowledge, NVAE is the first successful VAE applied to natural images as large as 256\${\textbackslash}times\$256 pixels. The source code is available at https://github.com/NVlabs/NVAE .},
	urldate = {2021-03-17},
	journal = {arXiv:2007.03898 [cs, stat]},
	author = {Vahdat, Arash and Kautz, Jan},
	month = jan,
	year = {2021},
	note = {arXiv: 2007.03898},
	keywords = {computer vision, deep learning, machine learning, soccer, sports analytics, variational autoencoders},
}

@article{raissi_physics_2017,
	title = {Physics {Informed} {Deep} {Learning} ({Part} {II}): {Data}-driven {Discovery} of {Nonlinear} {Partial} {Differential} {Equations}},
	shorttitle = {Physics {Informed} {Deep} {Learning} ({Part} {II})},
	url = {http://arxiv.org/abs/1711.10566},
	abstract = {We introduce physics informed neural networks -- neural networks that are trained to solve supervised learning tasks while respecting any given law of physics described by general nonlinear partial differential equations. In this second part of our two-part treatise, we focus on the problem of data-driven discovery of partial differential equations. Depending on whether the available data is scattered in space-time or arranged in fixed temporal snapshots, we introduce two main classes of algorithms, namely continuous time and discrete time models. The effectiveness of our approach is demonstrated using a wide range of benchmark problems in mathematical physics, including conservation laws, incompressible fluid flow, and the propagation of nonlinear shallow-water waves.},
	urldate = {2020-12-11},
	journal = {arXiv:1711.10566 [cs, math, stat]},
	author = {Raissi, Maziar and Perdikaris, Paris and Karniadakis, George Em},
	month = nov,
	year = {2017},
	note = {arXiv: 1711.10566},
	keywords = {machine learning, numerical modelling, partial differential equations, physics-informed learning, physics-informed neural networks},
}

@article{raissi_physics_2017-1,
	title = {Physics {Informed} {Deep} {Learning} ({Part} {I}): {Data}-driven {Solutions} of {Nonlinear} {Partial} {Differential} {Equations}},
	shorttitle = {Physics {Informed} {Deep} {Learning} ({Part} {I})},
	url = {http://arxiv.org/abs/1711.10561},
	abstract = {We introduce physics informed neural networks -- neural networks that are trained to solve supervised learning tasks while respecting any given law of physics described by general nonlinear partial differential equations. In this two part treatise, we present our developments in the context of solving two main classes of problems: data-driven solution and data-driven discovery of partial differential equations. Depending on the nature and arrangement of the available data, we devise two distinct classes of algorithms, namely continuous time and discrete time models. The resulting neural networks form a new class of data-efficient universal function approximators that naturally encode any underlying physical laws as prior information. In this first part, we demonstrate how these networks can be used to infer solutions to partial differential equations, and obtain physics-informed surrogate models that are fully differentiable with respect to all input coordinates and free parameters.},
	urldate = {2020-12-11},
	journal = {arXiv:1711.10561 [cs, math, stat]},
	author = {Raissi, Maziar and Perdikaris, Paris and Karniadakis, George Em},
	month = nov,
	year = {2017},
	note = {arXiv: 1711.10561},
	keywords = {dynamical systems, machine learning, numerical modelling, physics-informed learning, physics-informed neural networks},
}

@article{greydanus_hamiltonian_2019,
	title = {Hamiltonian {Neural} {Networks}},
	url = {http://arxiv.org/abs/1906.01563},
	abstract = {Even though neural networks enjoy widespread use, they still struggle to learn the basic laws of physics. How might we endow them with better inductive biases? In this paper, we draw inspiration from Hamiltonian mechanics to train models that learn and respect exact conservation laws in an unsupervised manner. We evaluate our models on problems where conservation of energy is important, including the two-body problem and pixel observations of a pendulum. Our model trains faster and generalizes better than a regular neural network. An interesting side effect is that our model is perfectly reversible in time.},
	urldate = {2020-12-11},
	journal = {arXiv:1906.01563 [cs]},
	author = {Greydanus, Sam and Dzamba, Misko and Yosinski, Jason},
	month = sep,
	year = {2019},
	note = {arXiv: 1906.01563},
	keywords = {hamiltonian monte carlo, neural networks, physics-informed learning, physics-informed neural networks},
}

@article{barbier_geothermal_2002,
	title = {Geothermal energy technology and current status: an overview},
	volume = {6},
	issn = {1364-0321},
	shorttitle = {Geothermal energy technology and current status},
	url = {http://www.sciencedirect.com/science/article/pii/S1364032102000023},
	doi = {10.1016/S1364-0321(02)00002-3},
	abstract = {Geothermal energy is the energy contained as heat in the Earth’s interior. This overview describes the internal structure of the Earth together with the heat transfer mechanisms inside mantle and crust. It also shows the location of geothermal fields on specific areas of the Earth. The Earth’s heat flow and geothermal gradient are defined, as well as the types of geothermal fields, the geologic environment of geothermal energy, and the methods of exploration for geothermal resources including drilling and resource assessment. Geothermal energy, as natural steam and hot water, has been exploited for decades to generate electricity, and both in space heating and industrial processes. The geothermal electrical installed capacity in the world is 7974 MWe (year 2000), and the electrical energy generated is 49.3 billion kWh/year, representing 0.3 \% of the world total electrical energy which was 15,342 billion kWh in 2000. In developing countries, where total installed electrical power is still low, geothermal energy can play a significant role: in the Philippines 21\% of electricity comes from geothermal steam, 20\% in El Salvador, 17\% in Nicaragua, 10\% in Costa Rica and 8\% in Kenya. Electricity is produced with an efficiency of 10–17\%. The geothermal kWh is generally cost-competitive with conventional sources of energy, in the range 2–10 UScents/kWh, and the geothermal electrical capacity installed in the world (1998) was 1/5 of that from biomass, but comparable with that from wind sources. The thermal capacity in non-electrical uses (greenhouses, aquaculture, district heating, industrial processes) is 15,14 MWt (year 2000). Financial investments in geothermal electrical and non-electrical uses world-wide in the period 1973–1992 were estimated at about US\$22,000 million. Present technology makes it possible to control the environmental impact of geothermal exploitation, and an effective and easily implemented policy to encourage geothermal energy development, and the abatement of carbon dioxide emissions would take advantage from the imposition of a carbon tax. The future use of geothermal energy from advanced technologies such as the exploitation of hot dry rock/hot wet rock systems, magma bodies and geopressured reservoirs, is briefly discussed. While the viability of hot dry rock technology has been proven, research and development are still necessary for the other two sources. A brief discussion on training of specialists, geothermal literature, on-line information, and geothermal associations concludes the review.},
	language = {en},
	number = {1},
	urldate = {2020-11-12},
	journal = {Renewable and Sustainable Energy Reviews},
	author = {Barbier, Enrico},
	month = jan,
	year = {2002},
	keywords = {geothermal, review},
	pages = {3--65},
}

@article{antonanzas-torres_clear_2019,
	title = {Clear sky solar irradiance models: {A} review of seventy models},
	volume = {107},
	issn = {1364-0321},
	shorttitle = {Clear sky solar irradiance models},
	url = {http://www.sciencedirect.com/science/article/pii/S1364032119301261},
	doi = {10.1016/j.rser.2019.02.032},
	abstract = {Clear sky solar irradiance parametric models seek to simplify the atmospheric attenuation with relatively simple parameterizations in order to estimate solar irradiance under clear sky conditions, avoiding the use of computationally expensive radiative transfer models. These models are particularly useful when estimating solar irradiation with satellite retrievals. Due to the great number of clear sky parametric models, it is somehow complicated to decide the choice of model to be selected. This article continues the work of previous reviews of clear sky models adding new models up to seventy described models for diffuse, beam and global components. A model benchmark is performed with ancillary solar irradiance data from two meteorological stations belonging to the Baseline Surface Radiation Network (BSRN) and also ancillary aerosol data from the Aerosol Robotic Network (AERONET). Results show great differences in performance between models, leading to priorize the use of only a few of them. Besides the review and evaluation of models, one of the most valuable deliverables of this paper is the R code of each clear sky model provided in a public GitHub repository (http://www.github.com/EDMANSolar/pcsol).},
	language = {en},
	urldate = {2020-11-12},
	journal = {Renewable and Sustainable Energy Reviews},
	author = {Antonanzas-Torres, F. and Urraca, R. and Polo, J. and Perpiñán-Lamigueiro, O. and Escobar, R.},
	month = jun,
	year = {2019},
	keywords = {review, solar energy},
	pages = {374--387},
}

@article{willard_integrating_2020,
	title = {Integrating {Physics}-{Based} {Modeling} with {Machine} {Learning}: {A} {Survey}},
	shorttitle = {Integrating {Physics}-{Based} {Modeling} with {Machine} {Learning}},
	url = {http://arxiv.org/abs/2003.04919},
	abstract = {In this manuscript, we provide a structured and comprehensive overview of techniques to integrate machine learning with physics-based modeling. First, we provide a summary of application areas for which these approaches have been applied. Then, we describe classes of methodologies used to construct physics-guided machine learning models and hybrid physics-machine learning frameworks from a machine learning standpoint. With this foundation, we then provide a systematic organization of these existing techniques and discuss ideas for future research.},
	language = {en},
	urldate = {2021-01-08},
	journal = {arXiv:2003.04919 [physics, stat]},
	author = {Willard, Jared and Jia, Xiaowei and Xu, Shaoming and Steinbach, Michael and Kumar, Vipin},
	month = jul,
	year = {2020},
	note = {arXiv: 2003.04919},
	keywords = {computational physics, machine learning, review},
}

@article{sobri_solar_2018,
	title = {Solar photovoltaic generation forecasting methods: {A} review},
	volume = {156},
	issn = {0196-8904},
	shorttitle = {Solar photovoltaic generation forecasting methods},
	url = {http://www.sciencedirect.com/science/article/pii/S0196890417310622},
	doi = {10.1016/j.enconman.2017.11.019},
	abstract = {Solar photovoltaic plants are widely integrated into most countries worldwide. Due to the ever-growing utilization of solar photovoltaic plants, either via grid-connection or stand-alone networks, dramatic changes can be anticipated in both power system planning and operating stages. Solar photovoltaic integration requires the capability of handling the uncertainty and fluctuations of power output. In this case, solar photovoltaic power forecasting is a crucial aspect to ensure optimum planning and modelling of the solar photovoltaic plants. Accurate forecasting provides the grid operators and power system designers with significant information to design an optimal solar photovoltaic plant as well as managing the power of demand and supply. This paper presents an extensive review on recent advancements in the field of solar photovoltaic power forecasting. This paper aims to analyze and compare various methods of solar photovoltaic power forecasting in terms of characteristics and performance. This work classifies solar photovoltaic power forecasting methods into three major categories i.e., time-series statistical methods, physical methods, and ensemble methods. To date, Artificial Intelligence approaches are widely used due to their capability in solving the non-linear and complex structure of data. The performance analysis shows that these methods outperform the traditional methods. Recently, the ensemble methods were also developed by researchers to extract the unique features of single models to enhance the forecast model performances. This combination produces accurate results compared to individual models. This paper also elaborates on the metrics assessment which was implemented to evaluate the forecast model performances. This work provides information which is beneficial for researchers and engineers who are involved in the modelling and planning of the solar photovoltaic plant.},
	language = {en},
	urldate = {2020-11-13},
	journal = {Energy Conversion and Management},
	author = {Sobri, Sobrina and Koohi-Kamali, Sam and Rahim, Nasrudin Abd.},
	month = jan,
	year = {2018},
	keywords = {energy modeling, land use planning, renewable energy, review, solar energy, spatio-temporal modelling},
	pages = {459--497},
}

@article{kauffeldt_technical_2016,
	title = {Technical review of large-scale hydrological models for implementation in operational flood forecasting schemes on continental level},
	volume = {75},
	issn = {1364-8152},
	url = {http://www.sciencedirect.com/science/article/pii/S1364815215300529},
	doi = {10.1016/j.envsoft.2015.09.009},
	abstract = {Uncertainty in operational hydrological forecast systems forced with numerical weather predictions is often assessed by quantifying the uncertainty from the inputs only. However, part of the uncertainty in modelled discharge stems from the hydrological model. A multi-model system can account for some of this uncertainty, but there exists a plethora of hydrological models and it is not trivial to select those that fit specific needs and collectively capture a representative spread of model uncertainty. This paper provides a technical review of 24 large-scale models to provide guidance for model selection. Suitability for the European Flood Awareness System (EFAS), as example of an operational continental flood forecasting system, is discussed based on process descriptions, flexibility in resolution, input data requirements, availability of code and more. The model choice is in the end subjective, but this review intends to objectively assist in selecting the most appropriate model for the intended purpose.},
	language = {en},
	urldate = {2020-11-12},
	journal = {Environmental Modelling \& Software},
	author = {Kauffeldt, A. and Wetterhall, F. and Pappenberger, F. and Salamon, P. and Thielen, J.},
	month = jan,
	year = {2016},
	keywords = {ensemble methods, flood forecasting, review, water modelling},
	pages = {68--76},
}

@article{noszczyk_review_2019,
	title = {A review of approaches to land use changes modeling},
	volume = {25},
	issn = {1080-7039, 1549-7860},
	url = {https://www.tandfonline.com/doi/full/10.1080/10807039.2018.1468994},
	doi = {10.1080/10807039.2018.1468994},
	language = {en},
	number = {6},
	urldate = {2020-11-13},
	journal = {Human and Ecological Risk Assessment: An International Journal},
	author = {Noszczyk, Tomasz},
	month = aug,
	year = {2019},
	keywords = {land use planning, review},
	pages = {1377--1405},
}

@article{whitaker_bayesian_2021,
	title = {A {Bayesian} approach for determining player abilities in football},
	volume = {70},
	copyright = {© 2020 Royal Statistical Society},
	issn = {1467-9876},
	url = {https://rss.onlinelibrary.wiley.com/doi/abs/10.1111/rssc.12454},
	doi = {https://doi.org/10.1111/rssc.12454},
	abstract = {We consider the task of determining a football player’s ability for a given event type, for example, scoring a goal. We propose an interpretable Bayesian model which is fit using variational inference methods. We implement a Poisson model to capture occurrences of event types, from which we infer player abilities. Our approach also allows the visualisation of differences between players, for a specific ability, through the marginal posterior variational densities. We then use these inferred player abilities to extend the Bayesian hierarchical model of Baio and Blangiardo (2010, Journal of Applied Statistics, 37(2), 253–264) which captures a team’s scoring rate (the rate at which they score goals). We apply the resulting scheme to the English Premier League, capturing player abilities over the 2013/2014 season, before using output from the hierarchical model to predict whether over or under 2.5 goals will be scored in a given game in the 2014/2015 season. This validates our model as a way of providing insights into team formation and the individual success of sports teams.},
	language = {en},
	number = {1},
	urldate = {2021-04-09},
	journal = {Journal of the Royal Statistical Society: Series C (Applied Statistics)},
	author = {Whitaker, Gavin A. and Silva, Ricardo and Edwards, Daniel and Kosmidis, Ioannis},
	year = {2021},
	note = {\_eprint: https://rss.onlinelibrary.wiley.com/doi/pdf/10.1111/rssc.12454},
	keywords = {bayesian hierarchical modelling, bayesian statistics, soccer, sports analytics, variational inference},
	pages = {174--201},
}

@article{le_data-driven_2017,
	title = {Data-{Driven} {Ghosting} using {Deep} {Imitation} {Learning}},
	language = {en},
	author = {Le, Hoang M and Carr, Peter and Yue, Yisong and Lucey, Patrick},
	year = {2017},
	keywords = {soccer, sports analytics},
	pages = {15},
}

@article{chong_abnormal_2017,
	title = {Abnormal {Event} {Detection} in {Videos} using {Spatiotemporal} {Autoencoder}},
	url = {http://arxiv.org/abs/1701.01546},
	abstract = {We present an efficient method for detecting anomalies in videos. Recent applications of convolutional neural networks have shown promises of convolutional layers for object detection and recognition, especially in images. However, convolutional neural networks are supervised and require labels as learning signals. We propose a spatiotemporal architecture for anomaly detection in videos including crowded scenes. Our architecture includes two main components, one for spatial feature representation, and one for learning the temporal evolution of the spatial features. Experimental results on Avenue, Subway and UCSD benchmarks confirm that the detection accuracy of our method is comparable to state-of-the-art methods at a considerable speed of up to 140 fps.},
	urldate = {2021-11-24},
	journal = {arXiv:1701.01546 [cs]},
	author = {Chong, Yong Shean and Tay, Yong Haur},
	month = jan,
	year = {2017},
	note = {arXiv: 1701.01546},
	keywords = {computer vision, soccer, sports analytics},
}

@article{cwiklinski_who_2021,
	title = {Who {Will} {Score}? {A} {Machine} {Learning} {Approach} to {Supporting} {Football} {Team} {Building} and {Transfers}},
	volume = {23},
	shorttitle = {Who {Will} {Score}?},
	doi = {10.3390/e23010090},
	abstract = {Background: 
the machine learning (ML) techniques have been implemented in numerous applications, including health-care, security, entertainment, and sports. In this article, we present how the ML can be used for building a professional football team and planning player transfers.

Methods:
in this research, we defined numerous parameters for player assessment, and three definitions of a successful transfer. We used the Random Forest, Naive Bayes, and AdaBoost algorithms in order to predict the player transfer success. We used realistic, publicly available data in order to train and test the classifiers.

Results:
in the article, we present numerous experiments; they differ in the weights of parameters, the successful transfer definitions, and other factors. We report promising results (accuracy = 0.82, precision = 0.84, recall = 0.82, and F1-score = 0.83).

Conclusion:
the presented research proves that machine learning can be helpful in professional football team building. The proposed algorithm will be developed in the future and it may be implemented as a professional tool for football talent scouts.},
	journal = {Entropy},
	author = {Ćwiklinski, Bartosz and Giełczyk, Agata and Choraś, Michał},
	month = jan,
	year = {2021},
	keywords = {soccer, sports analytics},
	pages = {90},
}

@article{ding_interpretable_2020,
	title = {Interpretable spatio-temporal attention {LSTM} model for flood forecasting},
	volume = {403},
	issn = {0925-2312},
	url = {https://www.sciencedirect.com/science/article/pii/S0925231220307530},
	doi = {10.1016/j.neucom.2020.04.110},
	abstract = {Modeling interpretable artificial intelligence (AI) for flood forecasting represents a serious challenge: both accuracy and interpretability are indispensable. Because of the uncertainty and nonlinearity of flood, existing hydrological solutions always achieve low prediction robustness while machine learning (ML) approaches neglect the physical interpretability of models. In this paper, we focus on the need for flood forecasting and propose an interpretable Spatio-Temporal Attention Long Short Term Memory model (STA-LSTM) based on LSTM and attention mechanism. We use dynamic attention mechanism and LSTM to build model, Max-Min method to normalize data, variable control method to select hyperparameters, and Adam algorithm to train the model. Emphasis is placed on the visualization and interpretation of attention weights. Experiment results on three small and medium basins in China suggest that the proposed STA-LSTM model outperforms Historical Average (HA), Fully Connected Network (FCN), Convolutional Neural Networks (CNN), Graph Convolutional Networks (GCN), original LSTM (LSTM), spatial attention LSTM (SA-LSTM), and temporal attention LSTM (TA-LSTM) in most cases. Visualization and interpretation of spatial and temporal attention weights reflect the reasonability of the proposed attention-based model.},
	language = {en},
	urldate = {2021-11-24},
	journal = {Neurocomputing},
	author = {Ding, Yukai and Zhu, Yuelong and Feng, Jun and Zhang, Pengcheng and Cheng, Zirun},
	month = aug,
	year = {2020},
	keywords = {attention, interpretability, lstms, soccer, sports analytics, water modelling},
	pages = {348--359},
}

@article{kiran_overview_2018,
	title = {An overview of deep learning based methods for unsupervised and semi-supervised anomaly detection in videos},
	url = {http://arxiv.org/abs/1801.03149},
	abstract = {Videos represent the primary source of information for surveillance applications and are available in large amounts but in most cases contain little or no annotation for supervised learning. This article reviews the state-of-the-art deep learning based methods for video anomaly detection and categorizes them based on the type of model and criteria of detection. We also perform simple studies to understand the different approaches and provide the criteria of evaluation for spatio-temporal anomaly detection.},
	urldate = {2021-11-24},
	journal = {arXiv:1801.03149 [cs]},
	author = {Kiran, B. Ravi and Thomas, Dilip Mathew and Parakkal, Ranjith},
	month = jan,
	year = {2018},
	note = {arXiv: 1801.03149},
	keywords = {computer vision, soccer, sports analytics},
}

@article{fassmeyer_toward_2021,
	title = {Toward {Automatically} {Labeling} {Situations} in {Soccer}},
	volume = {3},
	issn = {2624-9367},
	url = {https://www.frontiersin.org/article/10.3389/fspor.2021.725431},
	doi = {10.3389/fspor.2021.725431},
	abstract = {We study the automatic annotation of situations in soccer games. At first sight, this translates nicely into a standard supervised learning problem. However, in a fully supervised setting, predictive accuracies are supposed to correlate positively with the amount of labeled situations: more labeled training data simply promise better performance. Unfortunately, non-trivially annotated situations in soccer games are scarce, expensive and almost always require human experts; a fully supervised approach appears infeasible. Hence, we split the problem into two parts and learn (i) a meaningful feature representation using variational autoencoders on unlabeled data at large scales and (ii) a large-margin classifier acting in this feature space but utilize only a few (manually) annotated examples of the situation of interest. We propose four different architectures of the variational autoencoder and empirically study the detection of corner kicks, crosses and counterattacks. We observe high predictive accuracies above 90\% AUC irrespectively of the task.},
	urldate = {2021-11-24},
	journal = {Frontiers in Sports and Active Living},
	author = {Fassmeyer, Dennis and Anzer, Gabriel and Bauer, Pascal and Brefeld, Ulf},
	year = {2021},
	keywords = {soccer, sports analytics},
	pages = {313},
}

@article{shah_review_2020,
	title = {A review of approaches for water depth estimation with multispectral data},
	volume = {6},
	copyright = {© 2020 Policy Studies Organization},
	issn = {2639-541X},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/wwp2.12029},
	doi = {https://doi.org/10.1002/wwp2.12029},
	abstract = {The purpose of this paper is to give an exhaustive assessment of water depth estimation in shallow inland and coastal water environments with the assistance of optical remote detecting procedures, utilizing satellite/airborne multispectral information. Remote sensing has been used to map bathymetry for several decades. Inaccessibility, large-scale depth mapping, very shallow areas, and cost constraint are some of the reasons why remote sensing technique is a boon for bathymetry. Researchers have come a long way since the remote sensing-based bathymetry has been used. Fourfold increase in spatial, spectral, radiometric and temporal resolution of sensors have also increased the accuracy. There are two types of remote sensing techniques; Active and Passive. Active remote sensing includes the use of ground penetrating radar and bathymetric lidar, which are expensive but precise surveys successful up to 70 m depth. On the other hand, passive remote sensing deals with water photogrammetry and radiometric methods done by capturing imagery with space borne/air borne platforms having depth penetration up to 25 m. The latter technique includes finding of a logarithmic relationship between river depth and image values. This technique is most widely used because of its cost effectiveness. In this paper, the emphasis has been given to derive bathymetric techniques using multispectral data as these methods have proved reasonably successful when water quality, bottom reflectance, and atmospheric effects are all invariant. At a specific area, the detection of possible changes in water flow and levels indicates flow path changes which can be used to predict potential surface level flooding for better water management and policy decisions.},
	language = {en},
	number = {1},
	urldate = {2021-01-15},
	journal = {World Water Policy},
	author = {Shah, Ankit and Deshmukh, Benidhar and Sinha, L. K.},
	year = {2020},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/wwp2.12029},
	keywords = {bathymetry, oceanography, remote sensing},
	pages = {152--167},
}

@article{meng_multi-fidelity_2021,
	title = {Multi-fidelity {Bayesian} neural networks: {Algorithms} and applications},
	volume = {438},
	issn = {0021-9991},
	shorttitle = {Multi-fidelity {Bayesian} neural networks},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999121002564},
	doi = {10.1016/j.jcp.2021.110361},
	abstract = {We propose a new class of Bayesian neural networks (BNNs) that can be trained using noisy data of variable fidelity, and we apply them to learn function approximations as well as to solve inverse problems based on partial differential equations (PDEs). These multi-fidelity BNNs consist of three neural networks: The first is a fully connected neural network, which is trained following the maximum a posteriori probability (MAP) method to fit the low-fidelity data; the second is a Bayesian neural network employed to capture the cross-correlation with uncertainty quantification between the low- and high-fidelity data; and the last one is the physics-informed neural network, which encodes the physical laws described by PDEs. For the training of the last two neural networks, we first employ the mean-field variational inference (VI) to maximize the evidence lower bound (ELBO) to obtain informative prior distributions for the hyperparameters in the BNNs, and subsequently we use the Hamiltonian Monte Carlo (HMC) method to estimate accurately the posterior distributions for the corresponding hyperparameters. We demonstrate the accuracy of the present method using synthetic data as well as real measurements. Specifically, we first approximate a one- and four-dimensional function, and then infer the reaction rates in one- and two-dimensional diffusion-reaction systems. Moreover, we infer the sea surface temperature (SST) in the Massachusetts and Cape Cod Bays using satellite images and in-situ measurements. Taken together, our results demonstrate that the present method can capture both linear and nonlinear correlation between the low- and high-fidelity data adaptively, identify unknown parameters in PDEs, and quantify uncertainties in predictions, given a few scattered noisy high-fidelity data. Finally, we demonstrate that we can effectively and efficiently reduce the uncertainties and hence enhance the prediction accuracy with an active learning approach, using as examples a specific one-dimensional function approximation and an inverse PDE problem.},
	language = {en},
	urldate = {2021-05-26},
	journal = {Journal of Computational Physics},
	author = {Meng, Xuhui and Babaee, Hessam and Karniadakis, George Em},
	month = aug,
	year = {2021},
	keywords = {active learning, bayesian neural networks, hamiltonian monte carlo, physics-informed learning, remote sensing, uncertainty estimation},
	pages = {110361},
}

@article{perdikaris_multi-fidelity_2015,
	title = {Multi-fidelity modelling via recursive co-kriging and {Gaussian}–{Markov} random fields},
	volume = {471},
	url = {https://royalsocietypublishing.org/doi/10.1098/rspa.2015.0018},
	doi = {10.1098/rspa.2015.0018},
	abstract = {We propose a new framework for design under uncertainty based on stochastic computer simulations and multi-level recursive co-kriging. The proposed methodology simultaneously takes into account multi-fidelity in models, such as direct numerical simulations versus empirical formulae, as well as multi-fidelity in the probability space (e.g. sparse grids versus tensor product multi-element probabilistic collocation). We are able to construct response surfaces of complex dynamical systems by blending multiple information sources via auto-regressive stochastic modelling. A computationally efficient machine learning framework is developed based on multi-level recursive co-kriging with sparse precision matrices of Gaussian–Markov random fields. The effectiveness of the new algorithms is demonstrated in numerical examples involving a prototype problem in risk-averse design, regression of random functions, as well as uncertainty quantification in fluid mechanics involving the evolution of a Burgers equation from a random initial state, and random laminar wakes behind circular cylinders.},
	number = {2179},
	urldate = {2021-09-10},
	journal = {Proceedings of the Royal Society A: Mathematical, Physical and Engineering Sciences},
	author = {Perdikaris, P. and Venturi, D. and Royset, J. O. and Karniadakis, G. E.},
	month = jul,
	year = {2015},
	note = {Publisher: Royal Society},
	keywords = {big data, gaussian processes, machine learning, multi-fidelity, risk-averse design, surrogate modelling, uncertainty estimation},
	pages = {20150018},
}

@article{hamelijnck_multi-resolution_2019,
	title = {Multi-resolution {Multi}-task {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/1906.08344},
	abstract = {We consider evidence integration from potentially dependent observation processes under varying spatio-temporal sampling resolutions and noise levels. We develop a multi-resolution multi-task (MRGP) framework while allowing for both inter-task and intra-task multi-resolution and multi-fidelity. We develop shallow Gaussian Process (GP) mixtures that approximate the difficult to estimate joint likelihood with a composite one and deep GP constructions that naturally handle biases in the mean. By doing so, we generalize and outperform state of the art GP compositions and offer information-theoretic corrections and efficient variational approximations. We demonstrate the competitiveness of MRGPs on synthetic settings and on the challenging problem of hyper-local estimation of air pollution levels across London from multiple sensing modalities operating at disparate spatio-temporal resolutions.},
	urldate = {2021-04-30},
	journal = {arXiv:1906.08344 [cs, stat]},
	author = {Hamelijnck, Oliver and Damoulas, Theodoros and Wang, Kangrui and Girolami, Mark},
	month = nov,
	year = {2019},
	note = {arXiv: 1906.08344},
	keywords = {gaussian processes, machine learning, multi-fidelity, multi-task},
}

@article{perdikaris_nonlinear_2017,
	title = {Nonlinear information fusion algorithms for data-efficient multi-fidelity modelling},
	volume = {473},
	url = {https://royalsocietypublishing.org/doi/10.1098/rspa.2016.0751},
	doi = {10.1098/rspa.2016.0751},
	abstract = {Multi-fidelity modelling enables accurate inference of quantities of interest by synergistically combining realizations of low-cost/low-fidelity models with a small set of high-fidelity observations. This is particularly effective when the low- and high-fidelity models exhibit strong correlations, and can lead to significant computational gains over approaches that solely rely on high-fidelity models. However, in many cases of practical interest, low-fidelity models can only be well correlated to their high-fidelity counterparts for a specific range of input parameters, and potentially return wrong trends and erroneous predictions if probed outside of their validity regime. Here we put forth a probabilistic framework based on Gaussian process regression and nonlinear autoregressive schemes that is capable of learning complex nonlinear and space-dependent cross-correlations between models of variable fidelity, and can effectively safeguard against low-fidelity models that provide wrong trends. This introduces a new class of multi-fidelity information fusion algorithms that provide a fundamental extension to the existing linear autoregressive methodologies, while still maintaining the same algorithmic complexity and overall computational cost. The performance of the proposed methods is tested in several benchmark problems involving both synthetic and real multi-fidelity datasets from computational fluid dynamics simulations.},
	number = {2198},
	urldate = {2021-05-26},
	journal = {Proceedings of the Royal Society A: Mathematical, Physical and Engineering Sciences},
	author = {Perdikaris, P. and Raissi, M. and Damianou, A. and Lawrence, N. D. and Karniadakis, G. E.},
	month = feb,
	year = {2017},
	note = {Publisher: Royal Society},
	keywords = {model fusion, multi-fidelity},
	pages = {20160751},
}

@article{wang_np-ode_2020,
	title = {{NP}-{ODE}: {Neural} {Process} {Aided} {Ordinary} {Differential} {Equations} for {Uncertainty} {Quantification} of {Finite} {Element} {Analysis}},
	shorttitle = {{NP}-{ODE}},
	url = {http://arxiv.org/abs/2012.06914},
	abstract = {Finite element analysis (FEA) has been widely used to generate simulations of complex and nonlinear systems. Despite its strength and accuracy, the limitations of FEA can be summarized into two aspects: a) running high-fidelity FEA often requires significant computational cost and consumes a large amount of time; b) FEA is a deterministic method that is insufficient for uncertainty quantification (UQ) when modeling complex systems with various types of uncertainties. In this paper, a physics-informed data-driven surrogate model, named Neural Process Aided Ordinary Differential Equation (NP-ODE), is proposed to model the FEA simulations and capture both input and output uncertainties. To validate the advantages of the proposed NP-ODE, we conduct experiments on both the simulation data generated from a given ordinary differential equation and the data collected from a real FEA platform for tribocorrosion. The performances of the proposed NP-ODE and several benchmark methods are compared. The results show that the proposed NP-ODE outperforms benchmark methods. The NP-ODE method realizes the smallest predictive error as well as generates the most reasonable confidence interval having the best coverage on testing data points.},
	urldate = {2021-05-11},
	journal = {arXiv:2012.06914 [cs, stat]},
	author = {Wang, Yinan and Wang, Kaiwen and Cai, Wenjun and Yue, Xiaowei},
	month = dec,
	year = {2020},
	note = {arXiv: 2012.06914},
	keywords = {differential equations, finite-element methods, machine learning, neural ode, neural processes, 🔥},
}

@article{fernandez-godino_review_2019,
	title = {Review of multi-fidelity models},
	volume = {57},
	issn = {0001-1452, 1533-385X},
	url = {http://arxiv.org/abs/1609.07196},
	doi = {10.2514/1.J057750},
	abstract = {Simulations are often computationally expensive and the need for multiple realizations, as in uncertainty quantification or optimization, makes surrogate models an attractive option. For expensive high-fidelity models (HFMs), however, even performing the number of simulations needed for fitting a surrogate may be too expensive. Inexpensive but less accurate low-fidelity models (LFMs) are often also available. Multi-fidelity models (MFMs) combine HFMs and LFMs in order to achieve accuracy at a reasonable cost. With the increasing popularity of MFMs in mind, the aim of this paper is to summarize the state-of-the-art of MFM trends. For this purpose, publications in this field are classified based on application, surrogate selection if any, the difference between fidelities, the method used to combine these fidelities, the field of application and the year published. Available methods of combining fidelities are also reviewed, focusing our attention especially on multi-fidelity surrogate models in which fidelities are combined inside a surrogate model. Computation time savings are usually the reason for using MFMs, hence it is important to properly report the achieved savings. Unfortunately, we find that many papers do not present sufficient information to determine these savings. Therefore, the paper also includes guidelines for authors to present their MFM savings in a way that is useful to future MFM users. Based on papers that provided enough information, we find that time savings are highly problem dependent and that MFM methods we surveyed provided time savings up to 90\%. Keywords: Multi-fidelity, Variable-complexity, Variable-fidelity, Surrogate models, Optimization, Uncertainty quantification, Review, Survey},
	number = {5},
	urldate = {2021-05-27},
	journal = {AIAA Journal},
	author = {Fernández-Godino, M. Giselle and Park, Chanyoung and Kim, Nam-Ho and Haftka, Raphael T.},
	month = may,
	year = {2019},
	note = {arXiv: 1609.07196},
	keywords = {multi-fidelity, review},
	pages = {2039--2054},
}

@article{meng_composite_2020,
	title = {A composite neural network that learns from multi-fidelity data: {Application} to function approximation and inverse {PDE} problems},
	volume = {401},
	issn = {00219991},
	shorttitle = {A composite neural network that learns from multi-fidelity data},
	url = {http://arxiv.org/abs/1903.00104},
	doi = {10.1016/j.jcp.2019.109020},
	abstract = {We propose a new composite neural network (NN) that can be trained based on multi-fidelity data. It is comprised of three NNs, with the first NN trained using the low-fidelity data and coupled to two high-fidelity NNs, one with activation functions and another one without, in order to discover and exploit nonlinear and linear correlations, respectively, between the low-fidelity and the high-fidelity data. We first demonstrate the accuracy of the new multi-fidelity NN for approximating some standard benchmark functions but also a 20-dimensional function. Subsequently, we extend the recently developed physics-informed neural networks (PINNs) to be trained with multi-fidelity data sets (MPINNs). MPINNs contain four fully-connected neural networks, where the first one approximates the low-fidelity data, while the second and third construct the correlation between the low- and high-fidelity data and produce the multi-fidelity approximation, which is then used in the last NN that encodes the partial differential equations (PDEs). Specifically, in the two high-fidelity NNs a relaxation parameter is introduced, which can be optimized to combine the linear and nonlinear sub-networks. By optimizing this parameter, the present model is capable of learning both the linear and complex nonlinear correlations between the low- and high-fidelity data adaptively. By training the MPINNs, we can:(1) obtain the correlation between the low- and high-fidelity data, (2) infer the quantities of interest based on a few scattered data, and (3) identify the unknown parameters in the PDEs. In particular, we employ the MPINNs to learn the hydraulic conductivity field for unsaturated flows as well as the reactive models for reactive transport. The results demonstrate that MPINNs can achieve relatively high accuracy based on a very small set of high-fidelity data.},
	urldate = {2021-05-25},
	journal = {Journal of Computational Physics},
	author = {Meng, Xuhui and Karniadakis, George Em},
	month = jan,
	year = {2020},
	note = {arXiv: 1903.00104},
	keywords = {computational physics, physics-informed learning},
	pages = {109020},
}

@article{kim_attentive_2019,
	title = {Attentive {Neural} {Processes}},
	url = {http://arxiv.org/abs/1901.05761},
	abstract = {Neural Processes (NPs) (Garnelo et al 2018a;b) approach regression by learning to map a context set of observed input-output pairs to a distribution over regression functions. Each function models the distribution of the output given an input, conditioned on the context. NPs have the benefit of fitting observed data efficiently with linear complexity in the number of context input-output pairs, and can learn a wide family of conditional distributions; they learn predictive distributions conditioned on context sets of arbitrary size. Nonetheless, we show that NPs suffer a fundamental drawback of underfitting, giving inaccurate predictions at the inputs of the observed data they condition on. We address this issue by incorporating attention into NPs, allowing each input location to attend to the relevant context points for the prediction. We show that this greatly improves the accuracy of predictions, results in noticeably faster training, and expands the range of functions that can be modelled.},
	urldate = {2021-03-16},
	journal = {arXiv:1901.05761 [cs, stat]},
	author = {Kim, Hyunjik and Mnih, Andriy and Schwarz, Jonathan and Garnelo, Marta and Eslami, Ali and Rosenbaum, Dan and Vinyals, Oriol and Teh, Yee Whye},
	month = jul,
	year = {2019},
	note = {arXiv: 1901.05761},
}

@article{perdikaris_multifidelity_2016,
	title = {Multifidelity {Information} {Fusion} {Algorithms} for {High}-{Dimensional} {Systems} and {Massive} {Data} sets},
	volume = {38},
	issn = {1064-8275},
	url = {https://epubs.siam.org/doi/abs/10.1137/15M1055164},
	doi = {10.1137/15M1055164},
	abstract = {We develop a framework for multifidelity information fusion and predictive inference in high-dimensional input spaces and in the presence of massive data sets. Hence, we tackle simultaneously the “big N" problem for big data and the curse of dimensionality in multivariate parametric problems. The proposed methodology establishes a new paradigm for constructing response surfaces of high-dimensional stochastic dynamical systems, simultaneously accounting for multifidelity in physical models as well as multifidelity in probability space. Scaling to high dimensions is achieved by data-driven dimensionality reduction techniques based on hierarchical functional decompositions and a graph-theoretic approach for encoding custom autocorrelation structure in Gaussian process priors. Multifidelity information fusion is facilitated through stochastic autoregressive schemes and frequency-domain machine learning algorithms that scale linearly with the data. Taking together these new developments leads to linear complexity algorithms as demonstrated in benchmark problems involving deterministic and stochastic fields in up to \$10{\textasciicircum}\{5\}\$ input dimensions and \$10{\textasciicircum}\{5\}\$ training points on a standard desktop computer.},
	number = {4},
	urldate = {2021-10-02},
	journal = {SIAM Journal on Scientific Computing},
	author = {Perdikaris, Paris and Venturi, Daniele and Karniadakis, George Em},
	month = jan,
	year = {2016},
	note = {Publisher: Society for Industrial and Applied Mathematics},
	keywords = {big data, gaussian processes, machine learning, multi-fidelity, uncertainty estimation},
	pages = {B521--B538},
}

@misc{hamid_marginalising_2022,
	title = {Marginalising over {Stationary} {Kernels} with {Bayesian} {Quadrature}},
	url = {http://arxiv.org/abs/2106.07452},
	abstract = {Marginalising over families of Gaussian Process kernels produces flexible model classes with well-calibrated uncertainty estimates. Existing approaches require likelihood evaluations of many kernels, rendering them prohibitively expensive for larger datasets. We propose a Bayesian Quadrature scheme to make this marginalisation more efficient and thereby more practical. Through use of the maximum mean discrepancies between distributions, we define a kernel over kernels that captures invariances between Spectral Mixture (SM) Kernels. Kernel samples are selected by generalising an information-theoretic acquisition function for warped Bayesian Quadrature. We show that our framework achieves more accurate predictions with better calibrated uncertainty than state-of-the-art baselines, especially when given limited (wall-clock) time budgets.},
	urldate = {2022-06-15},
	publisher = {arXiv},
	author = {Hamid, Saad and Schulze, Sebastian and Osborne, Michael A. and Roberts, Stephen J.},
	month = mar,
	year = {2022},
	note = {Number: arXiv:2106.07452
arXiv:2106.07452 [cs, stat]},
	keywords = {machine learning},
}

@article{klus_eigendecompositions_2020,
	title = {Eigendecompositions of {Transfer} {Operators} in {Reproducing} {Kernel} {Hilbert} {Spaces}},
	volume = {30},
	issn = {0938-8974, 1432-1467},
	url = {http://arxiv.org/abs/1712.01572},
	doi = {10.1007/s00332-019-09574-z},
	abstract = {Transfer operators such as the Perron--Frobenius or Koopman operator play an important role in the global analysis of complex dynamical systems. The eigenfunctions of these operators can be used to detect metastable sets, to project the dynamics onto the dominant slow processes, or to separate superimposed signals. We extend transfer operator theory to reproducing kernel Hilbert spaces and show that these operators are related to Hilbert space representations of conditional distributions, known as conditional mean embeddings in the machine learning community. Moreover, numerical methods to compute empirical estimates of these embeddings are akin to data-driven methods for the approximation of transfer operators such as extended dynamic mode decomposition and its variants. One main benefit of the presented kernel-based approaches is that these methods can be applied to any domain where a similarity measure given by a kernel is available. We illustrate the results with the aid of guiding examples and highlight potential applications in molecular dynamics as well as video and text data analysis.},
	number = {1},
	urldate = {2022-06-01},
	journal = {Journal of Nonlinear Science},
	author = {Klus, Stefan and Schuster, Ingmar and Muandet, Krikamol},
	month = feb,
	year = {2020},
	note = {arXiv:1712.01572 [cs, math, stat]},
	keywords = {dynamical systems, machine learning},
	pages = {283--315},
}

@article{teichert_machine_2019,
	title = {Machine learning materials physics: {Integrable} deep neural networks enable scale bridging by learning free energy functions},
	volume = {353},
	issn = {00457825},
	shorttitle = {Machine learning materials physics},
	url = {http://arxiv.org/abs/1901.00081},
	doi = {10.1016/j.cma.2019.05.019},
	abstract = {The free energy of a system is central to many material models. Although free energy data is not generally found directly, its derivatives can be observed or calculated. In this work, we present an Integrable Deep Neural Network (IDNN) that can be trained to derivative data, then analytically integrated to recover an accurate representation of the free energy. The IDNN is demonstrated by training to the chemical potential values of a binary alloy with B2 ordering. The resulting DNN representation of the free energy is used in a phase field simulation and found to predict the appropriate formation of antiphase boundaries in the material. In contrast, a B-spline representation of the same data failed to represent the physics of the system with sufficient fidelity to resolve the antiphase boundaries.},
	urldate = {2022-05-31},
	journal = {Computer Methods in Applied Mechanics and Engineering},
	author = {Teichert, G. H. and Natarajan, A. R. and Van der Ven, A. and Garikipati, K.},
	month = aug,
	year = {2019},
	note = {arXiv:1901.00081 [cond-mat, physics:physics]},
	keywords = {computational physics, materials science},
	pages = {201--216},
}

@techreport{yang_energy-based_2020,
	title = {Energy-{Based} {Processes} for {Exchangeable} {Data}},
	url = {http://arxiv.org/abs/2003.07521},
	abstract = {Recently there has been growing interest in modeling sets with exchangeability such as point clouds. A shortcoming of current approaches is that they restrict the cardinality of the sets considered or can only express limited forms of distribution over unobserved data. To overcome these limitations, we introduce Energy-Based Processes (EBPs), which extend energy based models to exchangeable data while allowing neural network parameterizations of the energy function. A key advantage of these models is the ability to express more flexible distributions over sets without restricting their cardinality. We develop an efficient training procedure for EBPs that demonstrates state-of-the-art performance on a variety of tasks such as point cloud generation, classification, denoising, and image completion.},
	number = {arXiv:2003.07521},
	urldate = {2022-05-31},
	institution = {arXiv},
	author = {Yang, Mengjiao and Dai, Bo and Dai, Hanjun and Schuurmans, Dale},
	month = jul,
	year = {2020},
	doi = {10.48550/arXiv.2003.07521},
	note = {arXiv:2003.07521 [cs, stat]
type: article},
	keywords = {machine learning},
}

@article{greenhill_bayesian_2020,
	title = {Bayesian {Optimization} for {Adaptive} {Experimental} {Design}: {A} {Review}},
	volume = {8},
	issn = {2169-3536},
	shorttitle = {Bayesian {Optimization} for {Adaptive} {Experimental} {Design}},
	doi = {10.1109/ACCESS.2020.2966228},
	abstract = {Bayesian optimisation is a statistical method that efficiently models and optimises expensive “black-box” functions. This review considers the application of Bayesian optimisation to experimental design, in comparison to existing Design of Experiments (DOE) methods. Solutions are surveyed for a range of core issues in experimental design including: the incorporation of prior knowledge, high dimensional optimisation, constraints, batch evaluation, multiple objectives, multi-fidelity data, and mixed variable types.},
	journal = {IEEE Access},
	author = {Greenhill, Stewart and Rana, Santu and Gupta, Sunil and Vellanki, Pratibha and Venkatesh, Svetha},
	year = {2020},
	note = {Conference Name: IEEE Access},
	keywords = {bayesian statistics, computational design, experimental design, gaussian processes, machine learning, optimization, uncertainty estimation},
	pages = {13937--13948},
}

@article{song_implicit_2021,
	title = {Implicit {Kernel} {Attention}},
	url = {http://arxiv.org/abs/2006.06147},
	abstract = {{\textbackslash}textit\{Attention\} computes the dependency between representations, and it encourages the model to focus on the important selective features. Attention-based models, such as Transformer and graph attention network (GAT), are widely utilized for sequential data and graph-structured data. This paper suggests a new interpretation and generalized structure of the attention in Transformer and GAT. For the attention in Transformer and GAT, we derive that the attention is a product of two parts: 1) the RBF kernel to measure the similarity of two instances and 2) the exponential of \$L{\textasciicircum}\{2\}\$ norm to compute the importance of individual instances. From this decomposition, we generalize the attention in three ways. First, we propose implicit kernel attention with an implicit kernel function instead of manual kernel selection. Second, we generalize \$L{\textasciicircum}\{2\}\$ norm as the \$L{\textasciicircum}\{p\}\$ norm. Third, we extend our attention to structured multi-head attention. Our generalized attention shows better performance on classification, translation, and regression tasks.},
	urldate = {2022-05-04},
	journal = {arXiv:2006.06147 [cs, stat]},
	author = {Song, Kyungwoo and Jung, Yohan and Kim, Dongjun and Moon, Il-Chul},
	month = mar,
	year = {2021},
	note = {arXiv: 2006.06147},
	keywords = {machine learning},
}

@misc{noauthor_morokoff_nodate,
	title = {Morokoff \& {Caflisch} (1995) {Function} 2},
	url = {https://www.sfu.ca/~ssurjano/morcaf95b.html},
}

@article{morokoff_quasi-monte_1995,
	title = {Quasi-{Monte} {Carlo} {Integration}},
	volume = {122},
	issn = {0021-9991},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999185712090},
	doi = {10.1006/jcph.1995.1209},
	abstract = {The standard Monte Carlo approach to evaluating multidimensional integrals using (pseudo)-random integration nodes is frequently used when quadrature methods are too difficult or expensive to implement. As an alternative to the random methods, it has been suggested that lower error and improved convergence may be obtained by replacing the pseudo-random sequences with more uniformly distributed sequences known as quasi-random. In this paper quasi-random (Halton, Sobol', and Faure) and pseudo-random sequences are compared in computational experiments designed to determine the effects on convergence of certain properties of the integrand, including variance, variation, smoothness, and dimension. The results show that variation, which plays an important role in the theoretical upper bound given by the Koksma-Hlawka inequality, does not affect convergence, while variance, the determining factor in random Monte Carlo, is shown to provide a rough upper bound, but does not accurately predict performance. In general, quasi-Monte Carlo methods are superior to random Monte Carlo, but the advantage may be slight, particularly in high dimensions or for integrands that are not smooth. For discontinuous integrands, we derive a bound which shows that the exponent for algebraic decay of the integration error from quasi-Monte Carlo is only slightly larger than 12 in high dimensions.},
	language = {en},
	number = {2},
	urldate = {2022-04-28},
	journal = {Journal of Computational Physics},
	author = {Morokoff, William J. and Caflisch, Russel E.},
	month = dec,
	year = {1995},
	pages = {218--230},
}

@article{tsai_transformer_2019,
	title = {Transformer {Dissection}: {A} {Unified} {Understanding} of {Transformer}'s {Attention} via the {Lens} of {Kernel}},
	shorttitle = {Transformer {Dissection}},
	url = {http://arxiv.org/abs/1908.11775},
	abstract = {Transformer is a powerful architecture that achieves superior performance on various sequence learning tasks, including neural machine translation, language understanding, and sequence prediction. At the core of the Transformer is the attention mechanism, which concurrently processes all inputs in the streams. In this paper, we present a new formulation of attention via the lens of the kernel. To be more precise, we realize that the attention can be seen as applying kernel smoother over the inputs with the kernel scores being the similarities between inputs. This new formulation gives us a better way to understand individual components of the Transformer's attention, such as the better way to integrate the positional embedding. Another important advantage of our kernel-based formulation is that it paves the way to a larger space of composing Transformer's attention. As an example, we propose a new variant of Transformer's attention which models the input as a product of symmetric kernels. This approach achieves competitive performance to the current state of the art model with less computation. In our experiments, we empirically study different kernel construction strategies on two widely used tasks: neural machine translation and sequence prediction.},
	urldate = {2022-03-30},
	journal = {arXiv:1908.11775 [cs, stat]},
	author = {Tsai, Yao-Hung Hubert and Bai, Shaojie and Yamada, Makoto and Morency, Louis-Philippe and Salakhutdinov, Ruslan},
	month = nov,
	year = {2019},
	note = {arXiv: 1908.11775},
	keywords = {machine learning},
}

@article{janner_offline_2021,
	title = {Offline {Reinforcement} {Learning} as {One} {Big} {Sequence} {Modeling} {Problem}},
	url = {http://arxiv.org/abs/2106.02039},
	abstract = {Reinforcement learning (RL) is typically concerned with estimating stationary policies or single-step models, leveraging the Markov property to factorize problems in time. However, we can also view RL as a generic sequence modeling problem, with the goal being to produce a sequence of actions that leads to a sequence of high rewards. Viewed in this way, it is tempting to consider whether high-capacity sequence prediction models that work well in other domains, such as natural-language processing, can also provide effective solutions to the RL problem. To this end, we explore how RL can be tackled with the tools of sequence modeling, using a Transformer architecture to model distributions over trajectories and repurposing beam search as a planning algorithm. Framing RL as sequence modeling problem simplifies a range of design decisions, allowing us to dispense with many of the components common in offline RL algorithms. We demonstrate the flexibility of this approach across long-horizon dynamics prediction, imitation learning, goal-conditioned RL, and offline RL. Further, we show that this approach can be combined with existing model-free algorithms to yield a state-of-the-art planner in sparse-reward, long-horizon tasks.},
	urldate = {2022-03-23},
	journal = {arXiv:2106.02039 [cs]},
	author = {Janner, Michael and Li, Qiyang and Levine, Sergey},
	month = nov,
	year = {2021},
	note = {arXiv: 2106.02039},
	keywords = {machine learning},
}

@article{halton_efficiency_1960,
	title = {On the efficiency of certain quasi-random sequences of points in evaluating multi-dimensional integrals},
	volume = {2},
	issn = {0945-3245},
	url = {https://doi.org/10.1007/BF01386213},
	doi = {10.1007/BF01386213},
	language = {en},
	number = {1},
	urldate = {2022-02-25},
	journal = {Numerische Mathematik},
	author = {Halton, J. H.},
	month = dec,
	year = {1960},
	pages = {84--90},
}

@article{morokoff_quasi-monte_1995-1,
	title = {Quasi-{Monte} {Carlo} {Integration}},
	volume = {122},
	issn = {0021-9991},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999185712090},
	doi = {10.1006/jcph.1995.1209},
	abstract = {The standard Monte Carlo approach to evaluating multidimensional integrals using (pseudo)-random integration nodes is frequently used when quadrature methods are too difficult or expensive to implement. As an alternative to the random methods, it has been suggested that lower error and improved convergence may be obtained by replacing the pseudo-random sequences with more uniformly distributed sequences known as quasi-random. In this paper quasi-random (Halton, Sobol', and Faure) and pseudo-random sequences are compared in computational experiments designed to determine the effects on convergence of certain properties of the integrand, including variance, variation, smoothness, and dimension. The results show that variation, which plays an important role in the theoretical upper bound given by the Koksma-Hlawka inequality, does not affect convergence, while variance, the determining factor in random Monte Carlo, is shown to provide a rough upper bound, but does not accurately predict performance. In general, quasi-Monte Carlo methods are superior to random Monte Carlo, but the advantage may be slight, particularly in high dimensions or for integrands that are not smooth. For discontinuous integrands, we derive a bound which shows that the exponent for algebraic decay of the integration error from quasi-Monte Carlo is only slightly larger than 12 in high dimensions.},
	language = {en},
	number = {2},
	urldate = {2022-02-25},
	journal = {Journal of Computational Physics},
	author = {Morokoff, William J. and Caflisch, Russel E.},
	month = dec,
	year = {1995},
	pages = {218--230},
}

@inproceedings{briol_sampling_2017,
	address = {Sydney, NSW, Australia},
	series = {{ICML}'17},
	title = {On the sampling problem for {Kernel} {Quadrature}},
	abstract = {The standard Kernel Quadrature method for numerical integration with random point sets (also called Bayesian Monte Carlo) is known to converge in root mean square error at a rate determined by the ratio s/d, where s and d encode the smoothness and dimension of the integrand. However, an empirical investigation reveals that the rate constant C is highly sensitive to the distribution of the random points. In contrast to standard Monte Carlo integration, for which optimal importance sampling is well-understood, the sampling distribution that minimises C for Kernel Quadrature does not admit a closed form. This paper argues that the practical choice of sampling distribution is an important open problem. One solution is considered; a novel automatic approach based on adaptive tempering and sequential Monte Carlo. Empirical results demonstrate a dramatic reduction in integration error of up to 4 orders of magnitude can be achieved with the proposed method.},
	urldate = {2022-02-25},
	booktitle = {Proceedings of the 34th {International} {Conference} on {Machine} {Learning} - {Volume} 70},
	publisher = {JMLR.org},
	author = {Briol, François-Xavier and Oates, Chris J. and Cockayne, Jon and Chen, Wilson Ye and Girolami, Mark},
	month = aug,
	year = {2017},
	pages = {586--595},
}

@article{bezanson_julia_2015,
	title = {Julia: {A} {Fresh} {Approach} to {Numerical} {Computing}},
	shorttitle = {Julia},
	url = {http://arxiv.org/abs/1411.1607},
	abstract = {Bridging cultures that have often been distant, Julia combines expertise from the diverse fields of computer science and computational science to create a new approach to numerical computing. Julia is designed to be easy and fast. Julia questions notions generally held as "laws of nature" by practitioners of numerical computing: 1. High-level dynamic programs have to be slow. 2. One must prototype in one language and then rewrite in another language for speed or deployment, and 3. There are parts of a system for the programmer, and other parts best left untouched as they are built by the experts. We introduce the Julia programming language and its design --- a dance between specialization and abstraction. Specialization allows for custom treatment. Multiple dispatch, a technique from computer science, picks the right algorithm for the right circumstance. Abstraction, what good computation is really about, recognizes what remains the same after differences are stripped away. Abstractions in mathematics are captured as code through another technique from computer science, generic programming. Julia shows that one can have machine performance without sacrificing human convenience.},
	urldate = {2022-02-25},
	journal = {arXiv:1411.1607 [cs]},
	author = {Bezanson, Jeff and Edelman, Alan and Karpinski, Stefan and Shah, Viral B.},
	month = jul,
	year = {2015},
	note = {arXiv: 1411.1607},
	keywords = {simulation},
}

@article{kennedy_bayesian_1998,
	title = {Bayesian quadrature with non-normal approximating functions},
	volume = {8},
	issn = {1573-1375},
	url = {https://doi.org/10.1023/A:1008832824006},
	doi = {10.1023/A:1008832824006},
	abstract = {We consider an efficient Bayesian approach to estimating integration-based posterior summaries from a separate Bayesian application. In Bayesian quadrature we model an intractable posterior density function f(·) as a Gaussian process, using an approximating function g(·), and find a posterior distribution for the integral of f(·), conditional on a few evaluations of f (·) at selected design points. Bayesian quadrature using normal g (·) is called Bayes-Hermite quadrature. We extend this theory by allowing g(·) to be chosen from two wider classes of functions. One is a family of skew densities and the other is the family of finite mixtures of normal densities. For the family of skew densities we describe an iterative updating procedure to select the most suitable approximation and apply the method to two simulated posterior density functions.},
	language = {en},
	number = {4},
	urldate = {2021-12-13},
	journal = {Statistics and Computing},
	author = {Kennedy, Marc},
	month = dec,
	year = {1998},
	pages = {365--375},
}

@article{oates_modern_2019,
	title = {A modern retrospective on probabilistic numerics},
	volume = {29},
	issn = {1573-1375},
	url = {https://doi.org/10.1007/s11222-019-09902-z},
	doi = {10.1007/s11222-019-09902-z},
	abstract = {This article attempts to place the emergence of probabilistic numerics as a mathematical–statistical research field within its historical context and to explore how its gradual development can be related both to applications and to a modern formal treatment. We highlight in particular the parallel contributions of Sul\$\$'\$\$din and Larkin in the 1960s and how their pioneering early ideas have reached a degree of maturity in the intervening period, mediated by paradigms such as average-case analysis and information-based complexity. We provide a subjective assessment of the state of research in probabilistic numerics and highlight some difficulties to be addressed by future works.},
	language = {en},
	number = {6},
	urldate = {2022-02-25},
	journal = {Statistics and Computing},
	author = {Oates, C. J. and Sullivan, T. J.},
	month = nov,
	year = {2019},
	pages = {1335--1351},
}

@book{davis_methods_2014,
	address = {Kent},
	title = {Methods of {Numerical} {Integration}.},
	isbn = {978-1-4832-6428-8},
	url = {http://qut.eblib.com.au/patron/FullRecord.aspx?p=1901551},
	abstract = {Methods of Numerical Integration, Second Edition describes the theoretical and practical aspects of major methods of numerical integration. Numerical integration is the study of how the numerical value of an integral can be found. This book contains six chapters and begins with a discussion of the basic principles and limitations of numerical integration. The succeeding chapters present the approximate integration rules and formulas over finite and infinite intervals. These topics are followed by a review of error analysis and estimation, as well as the application of functional analys.},
	language = {English},
	urldate = {2022-02-25},
	publisher = {Elsevier Science},
	author = {Davis, Philip J and Rabinowitz, Philip and Rheinbolt, Werner},
	year = {2014},
	note = {OCLC: 1041863307},
}

@book{noauthor_methods_1984,
	title = {Methods of {Numerical} {Integration}},
	isbn = {978-0-12-206360-2},
	url = {https://linkinghub.elsevier.com/retrieve/pii/C20130105661},
	language = {en},
	urldate = {2022-02-25},
	publisher = {Elsevier},
	year = {1984},
	doi = {10.1016/C2013-0-10566-1},
}

@inproceedings{kandasamy_bayesian_2015,
	address = {Buenos Aires, Argentina},
	series = {{IJCAI}'15},
	title = {Bayesian active learning for posterior estimation},
	isbn = {978-1-57735-738-4},
	abstract = {This paper studies active posterior estimation in a Bayesian setting when the likelihood is expensive to evaluate. Existing techniques for posterior estimation are based on generating samples representative of the posterior. Such methods do not consider efficiency in terms of likelihood evaluations. In order to be query efficient we treat posterior estimation in an active regression framework. We propose two myopic query strategies to choose where to evaluate the likelihood and implement them using Gaussian processes. Via experiments on a series of synthetic and real examples we demonstrate that our approach is significantly more query efficient than existing techniques and other heuristics for posterior estimation.},
	urldate = {2022-02-24},
	booktitle = {Proceedings of the 24th {International} {Conference} on {Artificial} {Intelligence}},
	publisher = {AAAI Press},
	author = {Kandasamy, Kirthevasan and Schneider, Jeff and Póczos, Barnabás},
	month = jul,
	year = {2015},
	pages = {3605--3611},
}

@article{rossberg_positive_1995,
	title = {Positive definite probability densities and probability distributions},
	volume = {76},
	issn = {1573-8795},
	url = {https://doi.org/10.1007/BF02363232},
	doi = {10.1007/BF02363232},
	abstract = {In the first short survey article [H.-J. Rossberg,Wiss. Z. Karl Marx Univ.,37, 366–374 (1988)] the author indicated that positive definite probability densities deserve attention and that the investigation of this class leads to serious new mathematical problems. Later [R. Riedel and H.-J. Rossberg,Metrika, (submitted for publication)] it turned out that it is worthwhile to consider an even larger class of distribution functions, namely those having characteristic function f≥0; they were first briefly considered in [P. Lévy,C. R. Acad. Sci.,265, 249–252 (1967)]. We survey the basic ideas and the present state of the developing new theory; moreover, we formulate open problems. Some new results are proved. For the sake of brevity we do not enter into a discussion of continuation problems and limit theorems; for these subjects see [H.-J. Rossberg,Appendix to Gnedenko B. W., Einführung in die Wahrscheinlichkeitstheorie, 9th ed., Akademie-Verlag, Berlin (1991)], [H.-J. Rossberg,Math. Nachr.,141, 227–232 (1989)], and [H.-J. Rossberg,Lect. Notes Math.,1412, 296–308 (1989)].},
	language = {en},
	number = {1},
	urldate = {2022-02-25},
	journal = {Journal of Mathematical Sciences},
	author = {Rossberg, H. -J.},
	month = aug,
	year = {1995},
	pages = {2181--2197},
}

@article{durand_streaming_2017,
	title = {Streaming kernel regression with provably adaptive mean, variance, and regularization},
	url = {http://arxiv.org/abs/1708.00768},
	abstract = {We consider the problem of streaming kernel regression, when the observations arrive sequentially and the goal is to recover the underlying mean function, assumed to belong to an RKHS. The variance of the noise is not assumed to be known. In this context, we tackle the problem of tuning the regularization parameter adaptively at each time step, while maintaining tight confidence bounds estimates on the value of the mean function at each point. To this end, we first generalize existing results for finite-dimensional linear regression with fixed regularization and known variance to the kernel setup with a regularization parameter allowed to be a measurable function of past observations. Then, using appropriate self-normalized inequalities we build upper and lower bound estimates for the variance, leading to Bersntein-like concentration bounds. The later is used in order to define the adaptive regularization. The bounds resulting from our technique are valid uniformly over all observation points and all time steps, and are compared against the literature with numerical experiments. Finally, the potential of these tools is illustrated by an application to kernelized bandits, where we revisit the Kernel UCB and Kernel Thompson Sampling procedures, and show the benefits of the novel adaptive kernel tuning strategy.},
	urldate = {2022-02-25},
	journal = {arXiv:1708.00768 [cs, stat]},
	author = {Durand, Audrey and Maillard, Odalric-Ambrym and Pineau, Joelle},
	month = aug,
	year = {2017},
	note = {arXiv: 1708.00768},
	keywords = {machine learning},
}

@inproceedings{mutny_efficient_2018,
	title = {Efficient {High} {Dimensional} {Bayesian} {Optimization} with {Additivity} and {Quadrature} {Fourier} {Features}},
	volume = {31},
	url = {https://proceedings.neurips.cc/paper/2018/hash/4e5046fc8d6a97d18a5f54beaed54dea-Abstract.html},
	urldate = {2022-02-25},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Mutny, Mojmir and Krause, Andreas},
	year = {2018},
}

@incollection{gupta_bayesian_1988,
	address = {New York, NY},
	title = {Bayesian {Numerical} {Analysis}},
	isbn = {978-1-4613-8770-1 978-1-4613-8768-8},
	url = {http://link.springer.com/10.1007/978-1-4613-8768-8_20},
	language = {en},
	urldate = {2022-02-25},
	booktitle = {Statistical {Decision} {Theory} and {Related} {Topics} {IV}},
	publisher = {Springer New York},
	author = {Diaconis, Persi},
	editor = {Gupta, Shanti S. and Berger, James O.},
	year = {1988},
	doi = {10.1007/978-1-4613-8768-8_20},
	pages = {163--175},
}

@inproceedings{diaconis_bayesian_1988,
	title = {Bayesian {Numerical} {Analysis}},
	doi = {10.1007/978-1-4613-8768-8_20},
	abstract = {Semantic Scholar extracted view of "Bayesian Numerical Analysis" by P. Diaconis},
	author = {Diaconis, P.},
	year = {1988},
}

@article{sriperumbudur_hilbert_2010,
	title = {Hilbert space embeddings and metrics on probability measures},
	url = {http://arxiv.org/abs/0907.5309},
	abstract = {A Hilbert space embedding for probability measures has recently been proposed, with applications including dimensionality reduction, homogeneity testing, and independence testing. This embedding represents any probability measure as a mean element in a reproducing kernel Hilbert space (RKHS). A pseudometric on the space of probability measures can be defined as the distance between distribution embeddings: we denote this as \${\textbackslash}gamma\_k\$, indexed by the kernel function \$k\$ that defines the inner product in the RKHS. We present three theoretical properties of \${\textbackslash}gamma\_k\$. First, we consider the question of determining the conditions on the kernel \$k\$ for which \${\textbackslash}gamma\_k\$ is a metric: such \$k\$ are denoted \{{\textbackslash}em characteristic kernels\}. Unlike pseudometrics, a metric is zero only when two distributions coincide, thus ensuring the RKHS embedding maps all distributions uniquely (i.e., the embedding is injective). While previously published conditions may apply only in restricted circumstances (e.g. on compact domains), and are difficult to check, our conditions are straightforward and intuitive: bounded continuous strictly positive definite kernels are characteristic. Alternatively, if a bounded continuous kernel is translation-invariant on \${\textbackslash}bb\{R\}{\textasciicircum}d\$, then it is characteristic if and only if the support of its Fourier transform is the entire \${\textbackslash}bb\{R\}{\textasciicircum}d\$. Second, we show that there exist distinct distributions that are arbitrarily close in \${\textbackslash}gamma\_k\$. Third, to understand the nature of the topology induced by \${\textbackslash}gamma\_k\$, we relate \${\textbackslash}gamma\_k\$ to other popular metrics on probability measures, and present conditions on the kernel \$k\$ under which \${\textbackslash}gamma\_k\$ metrizes the weak topology.},
	urldate = {2022-01-28},
	journal = {arXiv:0907.5309 [math, stat]},
	author = {Sriperumbudur, Bharath K. and Gretton, Arthur and Fukumizu, Kenji and Schölkopf, Bernhard and Lanckriet, Gert R. G.},
	month = jan,
	year = {2010},
	note = {arXiv: 0907.5309},
	keywords = {machine learning, statistics},
}

@article{kovachki_universal_2021,
	title = {On universal approximation and error bounds for {Fourier} {Neural} {Operators}},
	url = {http://arxiv.org/abs/2107.07562},
	abstract = {Fourier neural operators (FNOs) have recently been proposed as an effective framework for learning operators that map between infinite-dimensional spaces. We prove that FNOs are universal, in the sense that they can approximate any continuous operator to desired accuracy. Moreover, we suggest a mechanism by which FNOs can approximate operators associated with PDEs efficiently. Explicit error bounds are derived to show that the size of the FNO, approximating operators associated with a Darcy type elliptic PDE and with the incompressible Navier-Stokes equations of fluid dynamics, only increases sub (log)-linearly in terms of the reciprocal of the error. Thus, FNOs are shown to efficiently approximate operators arising in a large class of PDEs.},
	urldate = {2022-01-11},
	journal = {arXiv:2107.07562 [cs, math]},
	author = {Kovachki, Nikola and Lanthaler, Samuel and Mishra, Siddhartha},
	month = jul,
	year = {2021},
	note = {arXiv: 2107.07562},
	keywords = {numerical modelling},
}

@article{nelsen_random_2021,
	title = {The {Random} {Feature} {Model} for {Input}-{Output} {Maps} between {Banach} {Spaces}},
	url = {http://arxiv.org/abs/2005.10224},
	abstract = {Well known to the machine learning community, the random feature model is a parametric approximation to kernel interpolation or regression methods. It is typically used to approximate functions mapping a finite-dimensional input space to the real line. In this paper, we instead propose a methodology for use of the random feature model as a data-driven surrogate for operators that map an input Banach space to an output Banach space. Although the methodology is quite general, we consider operators defined by partial differential equations (PDEs); here, the inputs and outputs are themselves functions, with the input parameters being functions required to specify the problem, such as initial data or coefficients, and the outputs being solutions of the problem. Upon discretization, the model inherits several desirable attributes from this infinite-dimensional viewpoint, including mesh-invariant approximation error with respect to the true PDE solution map and the capability to be trained at one mesh resolution and then deployed at different mesh resolutions. We view the random feature model as a non-intrusive data-driven emulator, provide a mathematical framework for its interpretation, and demonstrate its ability to efficiently and accurately approximate the nonlinear parameter-to-solution maps of two prototypical PDEs arising in physical science and engineering applications: viscous Burgers' equation and a variable coefficient elliptic equation.},
	urldate = {2022-01-11},
	journal = {arXiv:2005.10224 [physics, stat]},
	author = {Nelsen, Nicholas H. and Stuart, Andrew M.},
	month = jun,
	year = {2021},
	note = {arXiv: 2005.10224},
	keywords = {computational physics, machine learning, numerical modelling},
}

@article{kovachki_neural_2021,
	title = {Neural {Operator}: {Learning} {Maps} {Between} {Function} {Spaces}},
	shorttitle = {Neural {Operator}},
	url = {http://arxiv.org/abs/2108.08481},
	abstract = {The classical development of neural networks has primarily focused on learning mappings between finite dimensional Euclidean spaces or finite sets. We propose a generalization of neural networks to learn operators that maps between infinite dimensional function spaces. We formulate the approximation of operators by composition of a class of linear integral operators and nonlinear activation functions, so that the composed operator can approximate complex nonlinear operators. We prove a universal approximation theorem for our construction. Furthermore, we introduce four classes of operator parameterizations: graph-based operators, low-rank operators, multipole graph-based operators, and Fourier operators and describe efficient algorithms for computing with each one. The proposed neural operators are resolution-invariant: they share the same network parameters between different discretizations of the underlying function spaces and can be used for zero-shot super-resolutions. Numerically, the proposed models show superior performance compared to existing machine learning based methodologies on Burgers' equation, Darcy flow, and the Navier-Stokes equation, while being several order of magnitude faster compared to conventional PDE solvers.},
	urldate = {2022-01-11},
	journal = {arXiv:2108.08481 [cs, math]},
	author = {Kovachki, Nikola and Li, Zongyi and Liu, Burigede and Azizzadenesheli, Kamyar and Bhattacharya, Kaushik and Stuart, Andrew and Anandkumar, Anima},
	month = dec,
	year = {2021},
	note = {arXiv: 2108.08481},
	keywords = {machine learning, numerical modelling},
}

@article{mildenhall_nerf_2020,
	title = {{NeRF}: {Representing} {Scenes} as {Neural} {Radiance} {Fields} for {View} {Synthesis}},
	shorttitle = {{NeRF}},
	url = {http://arxiv.org/abs/2003.08934},
	abstract = {We present a method that achieves state-of-the-art results for synthesizing novel views of complex scenes by optimizing an underlying continuous volumetric scene function using a sparse set of input views. Our algorithm represents a scene using a fully-connected (non-convolutional) deep network, whose input is a single continuous 5D coordinate (spatial location \$(x,y,z)\$ and viewing direction \$({\textbackslash}theta, {\textbackslash}phi)\$) and whose output is the volume density and view-dependent emitted radiance at that spatial location. We synthesize views by querying 5D coordinates along camera rays and use classic volume rendering techniques to project the output colors and densities into an image. Because volume rendering is naturally differentiable, the only input required to optimize our representation is a set of images with known camera poses. We describe how to effectively optimize neural radiance fields to render photorealistic novel views of scenes with complicated geometry and appearance, and demonstrate results that outperform prior work on neural rendering and view synthesis. View synthesis results are best viewed as videos, so we urge readers to view our supplementary video for convincing comparisons.},
	urldate = {2021-12-14},
	journal = {arXiv:2003.08934 [cs]},
	author = {Mildenhall, Ben and Srinivasan, Pratul P. and Tancik, Matthew and Barron, Jonathan T. and Ramamoorthi, Ravi and Ng, Ren},
	month = aug,
	year = {2020},
	note = {arXiv: 2003.08934},
	keywords = {computer vision, graphics},
}

@article{sarkka_relation_2015,
	title = {On the relation between {Gaussian} process quadratures and sigma-point methods},
	url = {http://arxiv.org/abs/1504.05994},
	abstract = {This article is concerned with Gaussian process quadratures, which are numerical integration methods based on Gaussian process regression methods, and sigma-point methods, which are used in advanced non-linear Kalman filtering and smoothing algorithms. We show that many sigma-point methods can be interpreted as Gaussian quadrature based methods with suitably selected covariance functions. We show that this interpretation also extends to more general multivariate Gauss--Hermite integration methods and related spherical cubature rules. Additionally, we discuss different criteria for selecting the sigma-point locations: exactness for multivariate polynomials up to a given order, minimum average error, and quasi-random point sets. The performance of the different methods is tested in numerical experiments.},
	urldate = {2021-12-14},
	journal = {arXiv:1504.05994 [math, stat]},
	author = {Särkkä, Simo and Hartikainen, Jouni and Svensson, Lennart and Sandblom, Fredrik},
	month = apr,
	year = {2015},
	note = {arXiv: 1504.05994},
	keywords = {dynamical systems, machine learning, statistics},
}

@article{hayakawa_positively_2021,
	title = {Positively {Weighted} {Kernel} {Quadrature} via {Subsampling}},
	url = {http://arxiv.org/abs/2107.09597},
	abstract = {We study kernel quadrature rules with positive weights for probability measures on general domains. Our theoretical analysis combines the spectral properties of the kernel with random sampling of points. This results in effective algorithms to construct kernel quadrature rules with positive weights and small worst-case error. Besides the additional benefits resulting from positive weights, our numerical experiments indicate that this can achieve fast convergence rates that compete with the optimal bounds in well-known examples.},
	urldate = {2021-12-14},
	journal = {arXiv:2107.09597 [cs, math, stat]},
	author = {Hayakawa, Satoshi and Oberhauser, Harald and Lyons, Terry},
	month = oct,
	year = {2021},
	note = {arXiv: 2107.09597},
	keywords = {machine learning, numerical modelling},
}

@article{liu_towards_2021,
	title = {Towards a {Unified} {Quadrature} {Framework} for {Large}-{Scale} {Kernel} {Machines}},
	issn = {1939-3539},
	doi = {10.1109/TPAMI.2021.3120183},
	abstract = {In this paper, we develop a quadrature framework for large-scale kernel machines via a numerical integration representation. Considering that the integration domain and measure of typical kernels, e.g., Gaussian kernels, arc-cosine kernels, are fully symmetric, we leverage deterministic fully symmetric interpolatory rules to efficiently compute quadrature nodes and associated weights for kernel approximation. The developed interpolatory rules are able to reduce the number of needed nodes while retaining a high approximation accuracy. Further, we randomize the above deterministic rules by the classical Monte-Carlo sampling and control variates techniques with two merits: 1) The proposed stochastic rules make the dimension of the feature mapping flexibly varying, such that we can control the discrepancy between the original and approximate kernels by tuning the dimnension. 2) Our stochastic rules have nice statistical properties of unbiasedness and variance reduction with fast convergence rate. In addition, we elucidate the relationship between our deterministic/stochastic interpolatory rules and current quadrature rules for kernel approximation, including the sparse grids quadrature and stochastic spherical-radial rules, thereby unifying these methods under our framework. Experimental results on several benchmark datasets show that our methods compare favorably with other representative kernel approximation based methods.},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	author = {Liu, Fanghui and Huang, Xiaolin and Chen, Yudong and Suykens, Johan A. K.},
	year = {2021},
	note = {Conference Name: IEEE Transactions on Pattern Analysis and Machine Intelligence},
	keywords = {kernel learning, kernel methods, monte carlo, quadrature, random features},
	pages = {1--1},
}

@inproceedings{li_towards_2019,
	title = {Towards a {Unified} {Analysis} of {Random} {Fourier} {Features}},
	url = {https://proceedings.mlr.press/v97/li19k.html},
	abstract = {Random Fourier features is a widely used, simple, and effective technique for scaling up kernel methods. The existing theoretical analysis of the approach, however, remains focused on specific learning tasks and typically gives pessimistic bounds which are at odds with the empirical results. We tackle these problems and provide the first unified risk analysis of learning with random Fourier features using the squared error and Lipschitz continuous loss functions. In our bounds, the trade-off between the computational cost and the expected risk convergence rate is problem specific and expressed in terms of the regularization parameter and the number of effective degrees of freedom. We study both the standard random Fourier features method for which we improve the existing bounds on the number of features required to guarantee the corresponding minimax risk convergence rate of kernel ridge regression, as well as a data-dependent modification which samples features proportional to ridge leverage scores and further reduces the required number of features. As ridge leverage scores are expensive to compute, we devise a simple approximation scheme which provably reduces the computational cost without loss of statistical efficiency.},
	language = {en},
	urldate = {2021-12-14},
	booktitle = {Proceedings of the 36th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Li, Zhu and Ton, Jean-Francois and Oglic, Dino and Sejdinovic, Dino},
	month = may,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {3905--3914},
}

@article{bietti_inductive_2019,
	title = {On the {Inductive} {Bias} of {Neural} {Tangent} {Kernels}},
	url = {http://arxiv.org/abs/1905.12173},
	abstract = {State-of-the-art neural networks are heavily over-parameterized, making the optimization algorithm a crucial ingredient for learning predictive models with good generalization properties. A recent line of work has shown that in a certain over-parameterized regime, the learning dynamics of gradient descent are governed by a certain kernel obtained at initialization, called the neural tangent kernel. We study the inductive bias of learning in such a regime by analyzing this kernel and the corresponding function space (RKHS). In particular, we study smoothness, approximation, and stability properties of functions with finite norm, including stability to image deformations in the case of convolutional networks, and compare to other known kernels for similar architectures.},
	urldate = {2021-12-14},
	journal = {arXiv:1905.12173 [cs, stat]},
	author = {Bietti, Alberto and Mairal, Julien},
	month = oct,
	year = {2019},
	note = {arXiv: 1905.12173},
	keywords = {machine learning},
}

@article{bach_equivalence_2017,
	title = {On the equivalence between kernel quadrature rules and random feature expansions},
	volume = {18},
	issn = {1532-4435},
	abstract = {We show that kernel-based quadrature rules for computing integrals can be seen as a special case of random feature expansions for positive definite kernels, for a particular decomposition that always exists for such kernels. We provide a theoretical analysis of the number of required samples for a given approximation error, leading to both upper and lower bounds that are based solely on the eigenvalues of the associated integral operator and match up to logarithmic terms. In particular, we show that the upper bound may be obtained from independent and identically distributed samples from a specific non-uniform distribution, while the lower bound if valid for any set of points. Applying our results to kernel-based quadrature, while our results are fairly general, we recover known upper and lower bounds for the special cases of Sobolev spaces. Moreover, our results extend to the more general problem of full function approximations (beyond simply computing an integral), with results in L2- and L∞-norm that match known results for special cases. Applying our results to random features, we show an improvement of the number of random features needed to preserve the generalization guarantees for learning with Lipshitz-continuous losses.},
	number = {1},
	journal = {The Journal of Machine Learning Research},
	author = {Bach, Francis},
	month = jan,
	year = {2017},
	keywords = {quadrature},
	pages = {714--751},
}

@article{belhadji_kernel_2019,
	title = {Kernel quadrature with {DPPs}},
	url = {http://arxiv.org/abs/1906.07832},
	abstract = {We study quadrature rules for functions from an RKHS, using nodes sampled from a determinantal point process (DPP). DPPs are parametrized by a kernel, and we use a truncated and saturated version of the RKHS kernel. This link between the two kernels, along with DPP machinery, leads to relatively tight bounds on the quadrature error, that depends on the spectrum of the RKHS kernel. Finally, we experimentally compare DPPs to existing kernel-based quadratures such as herding, Bayesian quadrature, or leverage score sampling. Numerical results confirm the interest of DPPs, and even suggest faster rates than our bounds in particular cases.},
	urldate = {2021-12-14},
	journal = {arXiv:1906.07832 [cs, stat]},
	author = {Belhadji, Ayoub and Bardenet, Rémi and Chainais, Pierre},
	month = dec,
	year = {2019},
	note = {arXiv: 1906.07832},
	keywords = {machine learning},
}

@article{gunter_sampling_2014,
	title = {Sampling for {Inference} in {Probabilistic} {Models} with {Fast} {Bayesian} {Quadrature}},
	url = {http://arxiv.org/abs/1411.0439},
	abstract = {We propose a novel sampling framework for inference in probabilistic models: an active learning approach that converges more quickly (in wall-clock time) than Markov chain Monte Carlo (MCMC) benchmarks. The central challenge in probabilistic inference is numerical integration, to average over ensembles of models or unknown (hyper-)parameters (for example to compute the marginal likelihood or a partition function). MCMC has provided approaches to numerical integration that deliver state-of-the-art inference, but can suffer from sample inefficiency and poor convergence diagnostics. Bayesian quadrature techniques offer a model-based solution to such problems, but their uptake has been hindered by prohibitive computation costs. We introduce a warped model for probabilistic integrands (likelihoods) that are known to be non-negative, permitting a cheap active learning scheme to optimally select sample locations. Our algorithm is demonstrated to offer faster convergence (in seconds) relative to simple Monte Carlo and annealed importance sampling on both synthetic and real-world examples.},
	urldate = {2021-12-13},
	journal = {arXiv:1411.0439 [stat]},
	author = {Gunter, Tom and Osborne, Michael A. and Garnett, Roman and Hennig, Philipp and Roberts, Stephen J.},
	month = nov,
	year = {2014},
	note = {arXiv: 1411.0439},
	keywords = {machine learning},
}

@inproceedings{ma_active_2014,
	title = {Active {Area} {Search} via {Bayesian} {Quadrature}},
	url = {https://proceedings.mlr.press/v33/ma14.html},
	abstract = {The selection of data collection locations is a problem that has received significant research attention from classical design of experiments to various recent active learning algorithms.  Typical objectives are to map an unknown function, optimize it, or find level sets in it.  Each of these objectives focuses on an assessment of individual points.  The introduction of set kernels has led to algorithms that instead consider labels assigned to sets of data points.  In this paper we combine these two concepts and consider the problem of choosing data collection locations when the goal is to identify regions whose set of collected data would be labeled positively by a set classifier.  We present an algorithm for the case where the positive class is defined in terms of a region’s average function value being above some threshold with high probability, a problem we call active area search. To this end, we model the latent function using a Gaussian process and use Bayesian quadrature to estimate its integral on predefined regions.  Our method is the first which directly solves the active area search problem.  In experiments it outperforms previous algorithms that were developed for other active search goals.},
	language = {en},
	urldate = {2021-12-02},
	booktitle = {Proceedings of the {Seventeenth} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Ma, Yifei and Garnett, Roman and Schneider, Jeff},
	month = apr,
	year = {2014},
	note = {ISSN: 1938-7228},
	pages = {595--603},
}

@inproceedings{acerbi_variational_2018,
	title = {Variational {Bayesian} {Monte} {Carlo}},
	volume = {31},
	url = {https://papers.nips.cc/paper/2018/hash/747c1bcceb6109a4ef936bc70cfe67de-Abstract.html},
	urldate = {2021-12-01},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Acerbi, Luigi},
	year = {2018},
}

@article{briol_frank-wolfe_2015,
	title = {Frank-{Wolfe} {Bayesian} {Quadrature}: {Probabilistic} {Integration} with {Theoretical} {Guarantees}},
	shorttitle = {Frank-{Wolfe} {Bayesian} {Quadrature}},
	url = {http://arxiv.org/abs/1506.02681},
	abstract = {There is renewed interest in formulating integration as an inference problem, motivated by obtaining a full distribution over numerical error that can be propagated through subsequent computation. Current methods, such as Bayesian Quadrature, demonstrate impressive empirical performance but lack theoretical analysis. An important challenge is to reconcile these probabilistic integrators with rigorous convergence guarantees. In this paper, we present the first probabilistic integrator that admits such theoretical treatment, called Frank-Wolfe Bayesian Quadrature (FWBQ). Under FWBQ, convergence to the true value of the integral is shown to be exponential and posterior contraction rates are proven to be superexponential. In simulations, FWBQ is competitive with state-of-the-art methods and out-performs alternatives based on Frank-Wolfe optimisation. Our approach is applied to successfully quantify numerical error in the solution to a challenging model choice problem in cellular biology.},
	urldate = {2021-11-30},
	journal = {arXiv:1506.02681 [stat]},
	author = {Briol, François-Xavier and Oates, Chris J. and Girolami, Mark and Osborne, Michael A.},
	month = dec,
	year = {2015},
	note = {arXiv: 1506.02681},
}

@article{ohagan_bayeshermite_1991,
	title = {Bayes–{Hermite} quadrature},
	volume = {29},
	issn = {0378-3758},
	url = {https://www.sciencedirect.com/science/article/pii/037837589190002V},
	doi = {10.1016/0378-3758(91)90002-V},
	abstract = {Bayesian quadrature treats the problem of numerical integration as one of statistical inference. A prior Gaussian process distribution is assumed for the integrand, observations arise from evaluating the integrand at selected points, and a posterior distribution is derived for the integrand and the integral. Methods are developed for quadrature in Rp. A particular application is integrating the posterior density arising from some other Bayesian analysis. Simulation results are presented, to show that the resulting Bayes–Hermite quadrature rules may perform better than the conventional Gauss–Hermite rules for this application. A key result is derived for product designs, which makes Bayesian quadrature practically useful for integrating in several dimensions. Although the method does not at present provide a solution to the more difficult problem of quadrature in high dimensions, it does seem to offer real improvements over existing methods in relatively low dimensions.},
	language = {en},
	number = {3},
	urldate = {2021-12-01},
	journal = {Journal of Statistical Planning and Inference},
	author = {O'Hagan, A.},
	month = nov,
	year = {1991},
	keywords = {bayesian quadrature, gaussian processes, numerical integration},
	pages = {245--260},
}

@article{zhou_adaptive_2020,
	title = {Adaptive {Bayesian} quadrature based statistical moments estimation for structural reliability analysis},
	volume = {198},
	issn = {0951-8320},
	url = {https://www.sciencedirect.com/science/article/pii/S0951832019307306},
	doi = {10.1016/j.ress.2020.106902},
	abstract = {An efficient method for structural reliability analysis is presented, which combines the statistical moments estimation and a versatile probability distribution model, i.e., the shifted generalized lognormal distribution (SGLD) model. In this method, a complete adaptive Bayesian quadrature (ABQ)-based procedure is developed to evaluate the first-four central moments of the equivalent extreme value (EEV) of structural responses, where an adaptive sampling scheme is formulated to iteratively add new integration points so as to satisfy the prescribed accuracy requirement with as fewer deterministic response analyses as possible. The probability density function (PDF) of the EEV is then recovered by fitting the SGLD model using the estimated statistical moments. Finally, the structural reliability is obtained by a simple one-dimensional integral of the PDF of the EEV over the safe domain. To demonstrate the efficacy of the proposed methodology, two numerical examples are carried out, involving the statistical moments estimation of analytical functions with different mathematical characteristics and the dynamic reliability assessment of nonlinear stochastic structures subjected to seismic excitations. It is revealed that the proposed method is capable of attaining fairly desired reliability measures on engineering structures with satisfactory accuracy and efficiency.},
	language = {en},
	urldate = {2021-11-30},
	journal = {Reliability Engineering \& System Safety},
	author = {Zhou, Tong and Peng, Yongbo},
	month = jun,
	year = {2020},
	keywords = {bayesian quadrature, structural reliability},
	pages = {106902},
}

@inproceedings{nguyen_distributionally_2020,
	title = {Distributionally {Robust} {Bayesian} {Quadrature} {Optimization}},
	url = {https://proceedings.mlr.press/v108/nguyen20a.html},
	abstract = {Bayesian quadrature optimization (BQO) maximizes the expectation of an expensive black-box integrand taken over a known probability distribution. In this work, we study BQO under distributional uncertainty in which the underlying probability distribution is unknown except for a limited set of its i.i.d samples. A standard BQO approach maximizes the Monte Carlo estimate of the true expected objective given the fixed sample set. Though Monte Carlo estimate is unbiased, it has high variance given a small set of samples; thus can result in a spurious objective function. We adopt the distributionally robust optimization perspective to this problem by maximizing the expected objective under the most adversarial distribution. In particular, we propose a novel posterior sampling based algorithm, namely distributionally robust BQO (DRBQO) for this purpose. We demonstrate the empirical effectiveness of our proposed framework in synthetic and real-world problems, and characterize its theoretical convergence via Bayesian regret.},
	language = {en},
	urldate = {2021-11-30},
	booktitle = {Proceedings of the {Twenty} {Third} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Nguyen, Thanh and Gupta, Sunil and Ha, Huong and Rana, Santu and Venkatesh, Svetha},
	month = jun,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {1921--1931},
}

@article{pronzato_bayesian_2020,
	title = {Bayesian {Quadrature}, {Energy} {Minimization}, and {Space}-{Filling} {Design}},
	volume = {8},
	url = {https://epubs.siam.org/doi/abs/10.1137/18M1210332},
	doi = {10.1137/18M1210332},
	abstract = {A standard objective in computer experiments is to approximate the behavior of an unknown function on a compact domain from a few evaluations inside the domain. When little is known about the function, space-filling design is advisable: typically, points of evaluation spread out across the available space are obtained by minimizing a geometrical (for instance, covering radius) or a discrepancy criterion measuring distance to uniformity. The paper investigates connections between design for integration (quadrature design), construction of the (continuous) best linear unbiased estimator (BLUE) for the location model, space-filling design, and minimization of energy (kernel discrepancy) for signed measures. Integrally strictly positive definite kernels define strictly convex energy functionals, with an equivalence between the notions of potential and directional derivative, showing the strong relation between discrepancy minimization and more traditional design of optimal experiments. In particular, kernel herding algorithms, which are special instances of vertex-direction methods used in optimal design, can be applied to the construction of point sequences with suitable space-filling properties.},
	number = {3},
	urldate = {2021-11-30},
	journal = {SIAM/ASA Journal on Uncertainty Quantification},
	author = {Pronzato, Luc and Zhigljavsky, Anatoly},
	month = jan,
	year = {2020},
	note = {Publisher: Society for Industrial and Applied Mathematics},
	keywords = {bayesian quadrature},
	pages = {959--1011},
}

@article{wu_exploiting_2018,
	title = {Exploiting gradients and {Hessians} in {Bayesian} optimization and {Bayesian} quadrature},
	url = {http://arxiv.org/abs/1704.00060},
	abstract = {An exciting branch of machine learning research focuses on methods for learning, optimizing, and integrating unknown functions that are difficult or costly to evaluate. A popular Bayesian approach to this problem uses a Gaussian process (GP) to construct a posterior distribution over the function of interest given a set of observed measurements, and selects new points to evaluate using the statistics of this posterior. Here we extend these methods to exploit derivative information from the unknown function. We describe methods for Bayesian optimization (BO) and Bayesian quadrature (BQ) in settings where first and second derivatives may be evaluated along with the function itself. We perform sampling-based inference in order to incorporate uncertainty over hyperparameters, and show that both hyperparameter and function uncertainty decrease much more rapidly when using derivative information. Moreover, we introduce techniques for overcoming ill-conditioning issues that have plagued earlier methods for gradient-enhanced Gaussian processes and kriging. We illustrate the efficacy of these methods using applications to real and simulated Bayesian optimization and quadrature problems, and show that exploting derivatives can provide substantial gains over standard methods.},
	urldate = {2021-11-30},
	journal = {arXiv:1704.00060 [stat]},
	author = {Wu, Anqi and Aoi, Mikio C. and Pillow, Jonathan W.},
	month = mar,
	year = {2018},
	note = {arXiv: 1704.00060},
	keywords = {machine learning},
}

@article{kanagawa_convergence_2019,
	title = {Convergence {Guarantees} for {Adaptive} {Bayesian} {Quadrature} {Methods}},
	url = {http://arxiv.org/abs/1905.10271},
	abstract = {Adaptive Bayesian quadrature (ABQ) is a powerful approach to numerical integration that empirically compares favorably with Monte Carlo integration on problems of medium dimensionality (where non-adaptive quadrature is not competitive). Its key ingredient is an acquisition function that changes as a function of previously collected values of the integrand. While this adaptivity appears to be empirically powerful, it complicates analysis. Consequently, there are no theoretical guarantees so far for this class of methods. In this work, for a broad class of adaptive Bayesian quadrature methods, we prove consistency, deriving non-tight but informative convergence rates. To do so we introduce a new concept we call weak adaptivity. Our results identify a large and flexible class of adaptive Bayesian quadrature rules as consistent, within which practitioners can develop empirically efficient methods.},
	urldate = {2021-11-30},
	journal = {arXiv:1905.10271 [cs, math, stat]},
	author = {Kanagawa, Motonobu and Hennig, Philipp},
	month = oct,
	year = {2019},
	note = {arXiv: 1905.10271},
	keywords = {machine learning, numerical modelling, statistics},
}

@inproceedings{chai_automated_2019,
	title = {Automated {Model} {Selection} with {Bayesian} {Quadrature}},
	url = {https://proceedings.mlr.press/v97/chai19a.html},
	abstract = {We present a novel technique for tailoring Bayesian quadrature (BQ) to model selection. The state-of-the-art for comparing the evidence of multiple models relies on Monte Carlo methods, which converge slowly and are unreliable for computationally expensive models. Although previous research has shown that BQ offers sample efficiency superior to Monte Carlo in computing the evidence of an individual model, applying BQ directly to model comparison may waste computation producing an overly-accurate estimate for the evidence of a clearly poor model. We propose an automated and efficient algorithm for computing the most-relevant quantity for model selection: the posterior model probability. Our technique maximizes the mutual information between this quantity and observations of the models’ likelihoods, yielding efficient sample acquisition across disparate model spaces when likelihood observations are limited. Our method produces more-accurate posterior estimates using fewer likelihood evaluations than standard Bayesian quadrature and Monte Carlo estimators, as we demonstrate on synthetic and real-world examples.},
	language = {en},
	urldate = {2021-11-30},
	booktitle = {Proceedings of the 36th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Chai, Henry and Ton, Jean-Francois and Osborne, Michael A. and Garnett, Roman},
	month = may,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {931--940},
}

@inproceedings{gessner_active_2020,
	title = {Active {Multi}-{Information} {Source} {Bayesian} {Quadrature}},
	url = {https://proceedings.mlr.press/v115/gessner20a.html},
	abstract = {Bayesian quadrature (BQ) is a sample-efficient probabilistic numerical method to solve integrals of expensive-to-evaluate black-box functions, yet so far, active BQ learning schemes focus merely on the integrand itself as information source, and do not allow for information transfer from cheaper, related functions. Here, we set the scene for active learning in BQ when multiple related information sources of variable cost (in input and source) are accessible. This setting arises for example when evaluating the integrand requires a complex simulation to be run that can be approximated by simulating at lower levels of sophistication and at lesser expense. We construct meaningful cost-sensitive multi-source acquisition-rates as an extension to common utility functions from vanilla BQ (VBQ), and discuss pitfalls that arise from blindly generalizing. In proof-of-concept experiments we scrutinize the behavior of our generalized acquisition functions. On an epidemiological model, we demonstrate that active multi-source BQ (AMS-BQ) is more cost-efficient than VBQ in learning the integral to a good accuracy.},
	language = {en},
	urldate = {2021-11-30},
	booktitle = {Proceedings of {The} 35th {Uncertainty} in {Artificial} {Intelligence} {Conference}},
	publisher = {PMLR},
	author = {Gessner, Alexandra and Gonzalez, Javier and Mahsereci, Maren},
	month = aug,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {712--721},
}

@inproceedings{osborne_active_2012,
	title = {Active {Learning} of {Model} {Evidence} {Using} {Bayesian} {Quadrature}},
	volume = {25},
	url = {https://papers.nips.cc/paper/2012/hash/6364d3f0f495b6ab9dcf8d3b5c6e0b01-Abstract.html},
	urldate = {2021-11-30},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Osborne, Michael and Garnett, Roman and Ghahramani, Zoubin and Duvenaud, David K and Roberts, Stephen J and Rasmussen, Carl},
	year = {2012},
}

@inproceedings{gonzalez_preferential_2017,
	title = {Preferential {Bayesian} {Optimization}},
	url = {https://proceedings.mlr.press/v70/gonzalez17a.html},
	abstract = {Bayesian optimization (BO) has emerged during the last few years as an effective approach to optimize black-box functions where direct queries of the objective are expensive. We consider the case where direct access to the function is not possible, but information about user preferences is. Such scenarios arise in problems where human preferences are modeled, such as A/B tests or recommender systems. We present a new framework for this scenario that we call Preferential Bayesian Optimization (PBO) and that allows to find the optimum of a latent function that can only be queried through pairwise comparisons, so-called duels. PBO extend the applicability of standard BO ideas and generalizes previous discrete dueling approaches by modeling the probability of the the winner of each duel by means of Gaussian process model with a Bernoulli likelihood. The latent preference function is used to define a family of acquisition functions that extend usual policies used in BO. We illustrate the benefits of PBO in a variety of experiments in which we show how the way correlations are modeled is the key ingredient to drastically reduce the number of comparisons to find the optimum of the latent function of interest.},
	language = {en},
	urldate = {2021-11-30},
	booktitle = {Proceedings of the 34th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {González, Javier and Dai, Zhenwen and Damianou, Andreas and Lawrence, Neil D.},
	month = jul,
	year = {2017},
	note = {ISSN: 2640-3498},
	pages = {1282--1291},
}

@inproceedings{osborne_bayesian_2012,
	title = {Bayesian {Quadrature} for {Ratios}},
	url = {https://proceedings.mlr.press/v22/osborne12.html},
	abstract = {We describe a novel approach to quadrature for ratios of probabilistic integrals, such as are used to compute posterior probabilities. It offers performance superior to Monte Carlo methods by exploiting a Bayesian quadrature framework. We improve upon previous Bayesian quadrature techniques by explicitly modelling the non-negativity of our integrands, and the correlations that exist between them. It offers most where the integrand is multi-modal and expensive to evaluate, as is commonplace in exoplanets research; we demonstrate the efficacy of our method on data from the Kepler spacecraft.},
	language = {en},
	urldate = {2021-11-30},
	booktitle = {Proceedings of the {Fifteenth} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Osborne, Michael and Garnett, Roman and Roberts, Stephen and Hart, Christopher and Aigrain, Suzanne and Gibson, Neale},
	month = mar,
	year = {2012},
	note = {ISSN: 1938-7228},
	pages = {832--840},
}

@inproceedings{lin_deep_2014-1,
	title = {Deep learning of spatio-temporal features with geometric-based moving point detection for motion segmentation},
	doi = {10.1109/ICRA.2014.6907299},
	abstract = {This paper introduces an approach to accomplish motion segmentation from a moving stereo camera based on deep learning. Previous work on moving object detection mostly use point features based on 3D geometric constraints. However, point features require good features, and are hard to detect or to be matched correctly in situations where objects have smooth textures. To alleviate this problem, learning high-level spatio-temporal features unsupervisedly from raw image data based on Reconstruction Independent Component Analysis (RICA) autoencoders is proposed. Despite the power of the new spatio-temporal features, these features cannot not learn and be used to interpret 3D geometry of dynamic scenes, which is critical for moving object detection from moving cameras. As detected moving points based on 3D geometric constraints still contain valuable information of 3D scene as well as the camera egomotion, we propose a framework that incorporates both the detected moving point results and the learned spatio-temporal features as inputs to Recursive Neural Networks (RNN) that performs motion segmentation. Both features effectively complement each other. The proposed approach is demonstrated with real-world stereo video data that contains multiple moving objects, and has achieved 26\% better detection rate over the existing 3D geometric-based moving points detector.},
	booktitle = {2014 {IEEE} {International} {Conference} on {Robotics} and {Automation} ({ICRA})},
	author = {Lin, Tsung-Han and Wang, Chieh-Chih},
	month = may,
	year = {2014},
	note = {ISSN: 1050-4729},
	keywords = {computer vision, feature extraction, image segmentation},
	pages = {3058--3065},
}

@article{fassmeyer_toward_2021-1,
	title = {Toward {Automatically} {Labeling} {Situations} in {Soccer}},
	volume = {3},
	issn = {2624-9367},
	url = {https://www.frontiersin.org/article/10.3389/fspor.2021.725431},
	doi = {10.3389/fspor.2021.725431},
	abstract = {We study the automatic annotation of situations in soccer games. At first sight, this translates nicely into a standard supervised learning problem. However, in a fully supervised setting, predictive accuracies are supposed to correlate positively with the amount of labeled situations: more labeled training data simply promise better performance. Unfortunately, non-trivially annotated situations in soccer games are scarce, expensive and almost always require human experts; a fully supervised approach appears infeasible. Hence, we split the problem into two parts and learn (i) a meaningful feature representation using variational autoencoders on unlabeled data at large scales and (ii) a large-margin classifier acting in this feature space but utilize only a few (manually) annotated examples of the situation of interest. We propose four different architectures of the variational autoencoder and empirically study the detection of corner kicks, crosses and counterattacks. We observe high predictive accuracies above 90\% AUC irrespectively of the task.},
	urldate = {2021-11-24},
	journal = {Frontiers in Sports and Active Living},
	author = {Fassmeyer, Dennis and Anzer, Gabriel and Bauer, Pascal and Brefeld, Ulf},
	year = {2021},
	pages = {313},
}

@incollection{alexandrov_optimization_2000,
	series = {Aerospace {Sciences} {Meetings}},
	title = {Optimization with variable-fidelity models applied to wing design},
	url = {https://arc.aiaa.org/doi/10.2514/6.2000-841},
	urldate = {2021-10-22},
	booktitle = {38th {Aerospace} {Sciences} {Meeting} and {Exhibit}},
	publisher = {American Institute of Aeronautics and Astronautics},
	author = {Alexandrov, N. and Lewis, R. and Gumbert, C. and Green, L. and Newman, P.},
	month = jan,
	year = {2000},
	doi = {10.2514/6.2000-841},
	keywords = {computational fluid dynamics, nonlinear programming},
}

@article{wang_fusion_2016,
	title = {Fusion of {Sentinel}-2 images},
	volume = {187},
	issn = {0034-4257},
	url = {https://www.sciencedirect.com/science/article/pii/S0034425716304023},
	doi = {10.1016/j.rse.2016.10.030},
	abstract = {Sentinel-2 is a very new programme of the European Space Agency (ESA) that is designed for fine spatial resolution global monitoring. Sentinel-2 images provide four 10m bands and six 20m bands. To provide more explicit spatial information, this paper aims to downscale the six 20m bands to 10m spatial resolution using the four directly observed 10m bands. The outcome of this fusion task is the production of 10 Sentinel-2 bands with 10m spatial resolution. This new fusion problem involves four fine spatial resolution bands, which is different to, and more complex than, the common pan-sharpening fusion problem which involves only one fine band. To address this, we extend the existing two main families of image fusion approaches (i.e., component substitution, CS, and multiresolution analysis, MRA) with two different schemes, a band synthesis scheme and a band selection scheme. Moreover, the recently developed area-to-point regression kriging (ATPRK) approach was also developed and applied for the Sentinel-2 fusion task. Using two Sentinel-2 datasets released online, the three types of approaches (eight CS and MRA-based approaches, and ATPRK) were compared comprehensively in terms of their accuracies to provide recommendations for the task of fusion of Sentinel-2 images. The downscaled ten-band 10m Sentinel-2 datasets represent important and promising products for a wide range of applications in remote sensing. They also have potential for blending with the upcoming Sentinel-3 data for fine spatio-temporal resolution monitoring at the global scale.},
	language = {en},
	urldate = {2021-10-22},
	journal = {Remote Sensing of Environment},
	author = {Wang, Qunming and Shi, Wenzhong and Li, Zhongbin and Atkinson, Peter M.},
	month = dec,
	year = {2016},
	keywords = {image fusion, multi-fidelity kernels},
	pages = {241--252},
}

@article{wang_fusion_2016-1,
	title = {Fusion of {Sentinel}-2 images},
	volume = {187},
	issn = {0034-4257},
	url = {https://www.sciencedirect.com/science/article/pii/S0034425716304023},
	doi = {10.1016/j.rse.2016.10.030},
	abstract = {Sentinel-2 is a very new programme of the European Space Agency (ESA) that is designed for fine spatial resolution global monitoring. Sentinel-2 images provide four 10m bands and six 20m bands. To provide more explicit spatial information, this paper aims to downscale the six 20m bands to 10m spatial resolution using the four directly observed 10m bands. The outcome of this fusion task is the production of 10 Sentinel-2 bands with 10m spatial resolution. This new fusion problem involves four fine spatial resolution bands, which is different to, and more complex than, the common pan-sharpening fusion problem which involves only one fine band. To address this, we extend the existing two main families of image fusion approaches (i.e., component substitution, CS, and multiresolution analysis, MRA) with two different schemes, a band synthesis scheme and a band selection scheme. Moreover, the recently developed area-to-point regression kriging (ATPRK) approach was also developed and applied for the Sentinel-2 fusion task. Using two Sentinel-2 datasets released online, the three types of approaches (eight CS and MRA-based approaches, and ATPRK) were compared comprehensively in terms of their accuracies to provide recommendations for the task of fusion of Sentinel-2 images. The downscaled ten-band 10m Sentinel-2 datasets represent important and promising products for a wide range of applications in remote sensing. They also have potential for blending with the upcoming Sentinel-3 data for fine spatio-temporal resolution monitoring at the global scale.},
	language = {en},
	urldate = {2021-10-22},
	journal = {Remote Sensing of Environment},
	author = {Wang, Qunming and Shi, Wenzhong and Li, Zhongbin and Atkinson, Peter M.},
	month = dec,
	year = {2016},
	keywords = {image fusion, multi-fidelity kernels},
	pages = {241--252},
}

@article{kyriakidis_geostatistical_2005,
	title = {Geostatistical {Prediction} and {Simulation} of {Point} {Values} from {Areal} {Data}},
	volume = {37},
	issn = {1538-4632},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/j.1538-4632.2005.00633.x},
	doi = {10.1111/j.1538-4632.2005.00633.x},
	abstract = {The spatial prediction and simulation of point values from areal data are addressed within the general geostatistical framework of change of support (the term support referring to the domain informed by each measurement or unknown value). It is shown that the geostatistical framework (i) can explicitly and consistently account for the support differences between the available areal data and the sought-after point predictions, (ii) yields coherent (mass-preserving or pycnophylactic) predictions, and (iii) provides a measure of reliability (standard error) associated with each prediction. In the case of stochastic simulation, alternative point-support simulated realizations of a spatial attribute reproduce (i) a point-support histogram (Gaussian in this work), (ii) a point-support semivariogram model (possibly including anisotropic nested structures), and (iii) when upscaled, the available areal data. Such point-support-simulated realizations can be used in a Monte Carlo framework to assess the uncertainty in spatially distributed model outputs operating at a fine spatial resolution because of uncertain input parameters inferred from coarser spatial resolution data. Alternatively, such simulated realizations can be used in a model-based hypothesis-testing context to approximate the sampling distribution of, say, the correlation coefficient between two spatial data sets, when one is available at a point support and the other at an areal support. A case study using synthetic data illustrates the application of the proposed methodology in a remote sensing context, whereby areal data are available on a regular pixel support. It is demonstrated that point-support (sub-pixel scale) predictions and simulated realizations can be readily obtained, and that such predictions and realizations are consistent with the available information at the coarser (pixel-level) spatial resolution.},
	language = {en},
	number = {2},
	urldate = {2021-10-21},
	journal = {Geographical Analysis},
	author = {Kyriakidis, Phaedon C. and Yoo, Eun-Hye},
	year = {2005},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/j.1538-4632.2005.00633.x},
	pages = {124--151},
}

@article{goovaerts_geostatistical_2006,
	title = {Geostatistical analysis of disease data: accounting for spatial support and population density in the isopleth mapping of cancer mortality risk using area-to-point {Poisson} kriging},
	volume = {5},
	issn = {1476-072X},
	shorttitle = {Geostatistical analysis of disease data},
	url = {https://doi.org/10.1186/1476-072X-5-52},
	doi = {10.1186/1476-072X-5-52},
	abstract = {Geostatistical techniques that account for spatially varying population sizes and spatial patterns in the filtering of choropleth maps of cancer mortality were recently developed. Their implementation was facilitated by the initial assumption that all geographical units are the same size and shape, which allowed the use of geographic centroids in semivariogram estimation and kriging. Another implicit assumption was that the population at risk is uniformly distributed within each unit. This paper presents a generalization of Poisson kriging whereby the size and shape of administrative units, as well as the population density, is incorporated into the filtering of noisy mortality rates and the creation of isopleth risk maps. An innovative procedure to infer the point-support semivariogram of the risk from aggregated rates (i.e. areal data) is also proposed.},
	number = {1},
	urldate = {2021-10-21},
	journal = {International Journal of Health Geographics},
	author = {Goovaerts, Pierre},
	month = nov,
	year = {2006},
	keywords = {remote sensing},
	pages = {52},
}

@article{goovaerts_combining_2010,
	title = {Combining {Areal} and {Point} {Data} in {Geostatistical} {Interpolation}: {Applications} to {Soil} {Science} and {Medical} {Geography}},
	volume = {42},
	issn = {1874-8953},
	shorttitle = {Combining {Areal} and {Point} {Data} in {Geostatistical} {Interpolation}},
	url = {https://doi.org/10.1007/s11004-010-9286-5},
	doi = {10.1007/s11004-010-9286-5},
	abstract = {A common issue in spatial interpolation is the combination of data measured over different spatial supports. For example, information available for mapping disease risk typically includes point data (e.g. patients’ and controls’ residence) and aggregated data (e.g. socio-demographic and economic attributes recorded at the census track level). Similarly, soil measurements at discrete locations in the field are often supplemented with choropleth maps (e.g. soil or geological maps) that model the spatial distribution of soil attributes as the juxtaposition of polygons (areas) with constant values. This paper presents a general formulation of kriging that allows the combination of both point and areal data through the use of area-to-area, area-to-point, and point-to-point covariances in the kriging system. The procedure is illustrated using two data sets: (1) geological map and heavy metal concentrations recorded in the topsoil of the Swiss Jura, and (2) incidence rates of late-stage breast cancer diagnosis per census tract and location of patient residences for three counties in Michigan. In the second case, the kriging system includes an error variance term derived according to the binomial distribution to account for varying degree of reliability of incidence rates depending on the total number of cases recorded in those tracts. Except under the binomial kriging framework, area-and-point (AAP) kriging ensures the coherence of the prediction so that the average of interpolated values within each mapping unit is equal to the original areal datum. The relationships between binomial kriging, Poisson kriging, and indicator kriging are discussed under different scenarios for the population size and spatial support. Sensitivity analysis demonstrates the smaller smoothing and greater prediction accuracy of the new procedure over ordinary and traditional residual kriging based on the assumption that the local mean is constant within each mapping unit.},
	language = {en},
	number = {5},
	urldate = {2021-10-21},
	journal = {Mathematical Geosciences},
	author = {Goovaerts, Pierre},
	month = jul,
	year = {2010},
	pages = {535--554},
}

@article{liu_populationdensity_2008,
	title = {Population‐density estimation using regression and area‐to‐point residual kriging},
	volume = {22},
	issn = {1365-8816},
	url = {https://doi.org/10.1080/13658810701492225},
	doi = {10.1080/13658810701492225},
	abstract = {Census population data are associated with several analytical and cartographic problems. Regression models using remote‐sensing covariates have been examined to estimate urban population density, but the performance may not be satisfactory. This paper describes a kriging‐based areal interpolation method, namely area‐to‐point residual kriging, which can be used to disaggregate the residuals remaining from regression. Compared with conventional cokriging, the area‐to‐point residual kriging is much simpler in that only a semivariogram model for the point residuals is required, as opposed to a set of auto‐ and cross‐semivariogram models involving the dependent variable and all the covariates. In addition, area‐to‐point residual kriging explicitly accounts for any scale differences between source data and target values. The method is illustrated by disaggregating population from census units to the land‐use zones within them. Comparative results for regression with and without area‐to‐point residual kriging show that area‐to‐point residual kriging can substantially improve interpolation accuracy.},
	number = {4},
	urldate = {2021-10-21},
	journal = {International Journal of Geographical Information Science},
	author = {Liu, X. H. and Kyriakidis, P. C. and Goodchild, M. F.},
	month = apr,
	year = {2008},
	note = {Publisher: Taylor \& Francis
\_eprint: https://doi.org/10.1080/13658810701492225},
	keywords = {gaussian processes, geostatistics},
	pages = {431--447},
}

@article{forrester_multi-fidelity_2007,
	title = {Multi-fidelity optimization via surrogate modelling},
	volume = {463},
	url = {https://royalsocietypublishing.org/doi/abs/10.1098/rspa.2007.1900},
	doi = {10.1098/rspa.2007.1900},
	abstract = {This paper demonstrates the application of correlated Gaussian process based approximations to optimization where multiple levels of analysis are available, using an extension to the geostatistical method of co-kriging. An exchange algorithm is used to choose which points of the search space to sample within each level of analysis. The derivation of the co-kriging equations is presented in an intuitive manner, along with a new variance estimator to account for varying degrees of computational ‘noise’ in the multiple levels of analysis. A multi-fidelity wing optimization is used to demonstrate the methodology.},
	number = {2088},
	urldate = {2021-10-21},
	journal = {Proceedings of the Royal Society A: Mathematical, Physical and Engineering Sciences},
	author = {Forrester, Alexander I.J and Sóbester, András and Keane, Andy J},
	month = dec,
	year = {2007},
	note = {Publisher: Royal Society},
	keywords = {gaussian processes, multi-output gp},
	pages = {3251--3269},
}

@article{gratiet_recursive_2014,
	title = {Recursive co-kriging model for design of computer experiments with multiple levels of fidelity},
	volume = {4},
	issn = {2152-5080, 2152-5099},
	url = {http://www.dl.begellhouse.com/journals/52034eb04b657aea,2f7b99cc281f2702,4c83626c5e64207a.html},
	doi = {10.1615/Int.J.UncertaintyQuantification.2014006914},
	language = {English},
	number = {5},
	urldate = {2021-05-17},
	journal = {International Journal for Uncertainty Quantification},
	author = {Gratiet, Loic Le and Garnier, Josselin},
	year = {2014},
	note = {Publisher: Begel House Inc.},
}

@misc{noauthor_two-dimensional_nodate,
	title = {Two-dimensional turbulence · {Oceananigans}.jl},
	url = {https://clima.github.io/OceananigansDocumentation/stable/generated/two_dimensional_turbulence/},
	urldate = {2021-10-14},
}

@article{ramadhan_oceananigansjl_2020,
	title = {Oceananigans.jl: {Fast} and friendly geophysical fluid dynamics on {GPUs}},
	volume = {5},
	issn = {2475-9066},
	shorttitle = {Oceananigans.jl},
	url = {https://joss.theoj.org/papers/10.21105/joss.02018},
	doi = {10.21105/joss.02018},
	number = {53},
	urldate = {2021-10-14},
	journal = {Journal of Open Source Software},
	author = {Ramadhan, Ali and Wagner, Gregory and Hill, Chris and Campin, Jean-Michel and Churavy, Valentin and Besard, Tim and Souza, Andre and Edelman, Alan and Ferrari, Raffaele and Marshall, John},
	month = sep,
	year = {2020},
	pages = {2018},
}

@article{avron_quasi-monte_2016,
	title = {Quasi-{Monte} {Carlo} {Feature} {Maps} for {Shift}-{Invariant} {Kernels}},
	volume = {17},
	issn = {1533-7928},
	url = {http://jmlr.org/papers/v17/14-538.html},
	abstract = {We consider the problem of improving the efficiency of randomized Fourier feature maps to accelerate training and testing speed of kernel methods on large data sets. These approximate feature maps arise as Monte Carlo approximations to integral representations of shift-invariant kernel functions (e.g., Gaussian kernel). In this paper, we propose to use Quasi-Monte Carlo (QMC) approximations instead, where the relevant integrands are evaluated on a low-discrepancy sequence of points as opposed to random point sets as in the Monte Carlo approach. We derive a new discrepancy measure called box discrepancy based on theoretical characterizations of the integration error with respect to a given sequence. We then propose to learn QMC sequences adapted to our setting based on explicit box discrepancy minimization. Our theoretical analyses are complemented with empirical results that demonstrate the effectiveness of classical and adaptive QMC techniques for this problem.},
	number = {120},
	urldate = {2021-10-14},
	journal = {Journal of Machine Learning Research},
	author = {Avron, Haim and Sindhwani, Vikas and Yang, Jiyan and Mahoney, Michael W.},
	year = {2016},
	pages = {1--38},
}

@inproceedings{hensman_gaussian_2013,
	address = {Arlington, Virginia, USA},
	series = {{UAI}'13},
	title = {Gaussian processes for {Big} data},
	abstract = {We introduce stochastic variational inference for Gaussian process models. This enables the application of Gaussian process (GP) models to data sets containing millions of data points. We show how GPs can be variationally decomposed to depend on a set of globally relevant inducing variables which factorize the model in the necessary manner to perform variational inference. Our approach is readily extended to models with non-Gaussian likelihoods and latent variable models based around Gaussian processes. We demonstrate the approach on a simple toy problem and two real world data sets.},
	urldate = {2021-10-14},
	booktitle = {Proceedings of the {Twenty}-{Ninth} {Conference} on {Uncertainty} in {Artificial} {Intelligence}},
	publisher = {AUAI Press},
	author = {Hensman, James and Fusi, Nicolò and Lawrence, Neil D.},
	month = aug,
	year = {2013},
	pages = {282--290},
}

@article{skoien_top-kriging_2006,
	title = {Top-kriging - geostatistics on stream networks},
	volume = {10},
	issn = {1027-5606},
	url = {https://hess.copernicus.org/articles/10/277/2006/},
	doi = {10.5194/hess-10-277-2006},
	abstract = {{\textless}p{\textgreater}{\textless}strong class="journal-contentHeaderColor"{\textgreater}Abstract.{\textless}/strong{\textgreater} We present Top-kriging, or topological kriging, as a method for estimating streamflow-related variables in ungauged catchments. It takes both the area and the nested nature of catchments into account. The main appeal of the method is that it is a best linear unbiased estimator (BLUE) adapted for the case of stream networks without any additional assumptions. The concept is built on the work of Sauquet et al.\&nbsp;(2000) and extends it in a number of ways. We test the method for the case of the specific 100-year flood for two Austrian regions. The method provides more plausible and, indeed, more accurate estimates than Ordinary Kriging. For the variable of interest, Top-kriging also provides estimates of the uncertainty. On the main stream the estimated uncertainties are smallest and they gradually increase as one moves towards the headwaters. The method as presented here is able to exploit the information contained in short records by accounting for the uncertainty of each gauge. We suggest that Top-kriging can be used for spatially interpolating a range of streamflow-related variables including mean annual discharge, flood characteristics, low flow characteristics, concentrations, turbidity and stream temperature.{\textless}/p{\textgreater}},
	language = {English},
	number = {2},
	urldate = {2021-10-14},
	journal = {Hydrology and Earth System Sciences},
	author = {Skøien, J. O. and Merz, R. and Blöschl, G.},
	month = apr,
	year = {2006},
	note = {Publisher: Copernicus GmbH},
	pages = {277--287},
}

@book{rudin_fourier_2011,
	address = {Hoboken},
	title = {Fourier {Analysis} on {Groups}.},
	isbn = {978-1-118-16564-5},
	url = {http://www.123library.org/book_details/?id=30165},
	abstract = {In the late 1950s, many of the more refined aspects of Fourier analysis were transferred from their original settings (the unit circle, the integers, the real line) to arbitrary locally compact abelian (LCA) groups. Rudin's book, published in 1962, was the first to give a systematic account of these developments and has come to be regarded as a classic in the field. The basic facts concerning Fourier analysis and the structure of LCA groups are proved in the opening chapters, in order to make the treatment relatively self-contained.},
	language = {English},
	urldate = {2021-10-14},
	publisher = {John Wiley \& Sons},
	author = {Rudin, Walter},
	year = {2011},
	note = {OCLC: 772844593},
}

@misc{roberts_mohc_2018,
	title = {{MOHC} {HadGEM3}-{GC31}-{HM} model output prepared for {CMIP6} {HighResMIP} hist-1950},
	copyright = {Creative Commons Attribution Share Alike 4.0 International},
	url = {http://cera-www.dkrz.de/WDCC/meta/CMIP6/CMIP6.HighResMIP.MOHC.HadGEM3-GC31-HM.hist-1950},
	doi = {10.22033/ESGF/CMIP6.6040},
	abstract = {Coupled Model Intercomparison Project Phase 6 (CMIP6) data sets: These data includes all datasets published for 'CMIP6.HighResMIP.MOHC.HadGEM3-GC31-HM.hist-1950' according to the Data Reference Syntax defined as 'mip\_era.activity\_id.institution\_id.source\_id.experiment\_id.member\_id.table\_id.variable\_id.grid\_label.version'. The model used in climate research named HadGEM3-GC3.1-N512ORCA025, released in 2016, includes the components: aerosol: UKCA-GLOMAP-mode, atmos: MetUM-HadGEM3-GA7.1 (N512; 1024 x 768 longitude/latitude; 85 levels; top level 85 km), land: JULES-HadGEM3-GL7.1, ocean: NEMO-HadGEM3-GO6.0 (eORCA025 tripolar primarily 0.25 deg; 1440 x 1205 longitude/latitude; 75 levels; top grid cell 0-1 m), seaIce: CICE-HadGEM3-GSI8 (eORCA025 tripolar primarily 0.25 deg; 1440 x 1205 longitude/latitude). The model was run by the Met Office Hadley Centre, Fitzroy Road, Exeter, Devon, EX1 3PB, UK (MOHC) in native nominal resolutions: aerosol: 50 km, atmos: 50 km, land: 50 km, ocean: 25 km, seaIce: 25 km. Project: These data have been generated as part of the internationally-coordinated Coupled Model Intercomparison Project Phase 6 (CMIP6; see also GMD Special Issue: http://www.geosci-model-dev.net/special\_issue590.html). The simulation data provides a basis for climate research designed to answer fundamental science questions, and the results will undoubtedly be relied on by authors of the Sixth Assessment Report of the Intergovernmental Panel on Climate Change (IPCC-AR6). CMIP6 is a project coordinated by the Working Group on Coupled Modelling (WGCM) as part of the World Climate Research Programme (WCRP). Phase 6 builds on previous phases executed under the leadership of the Program for Climate Model Diagnosis and Intercomparison (PCMDI) and relies on the Earth System Grid Federation (ESGF) and the Centre for Environmental Data Analysis (CEDA) along with numerous related activities for implementation. The original data is hosted and partially replicated at a federated collection of data nodes, and most of the data relied on by the IPCC is being archived for long-term preservation at the IPCC Data Distribution Centre (IPCC DDC) hosted by the German Climate Computing Center (DKRZ). The project includes simulations from about 120 global climate models and around 45 institutions and organizations worldwide. - Project website: https://pcmdi.llnl.gov/CMIP6.},
	language = {en},
	urldate = {2021-10-14},
	publisher = {Earth System Grid Federation},
	author = {Roberts, Malcolm},
	collaborator = {Roberts, Malcolm and Walters, David and Willett, Martin and Brooks, Malcolm and Earnshaw, Paul and Milton, Sean and Williams, Keith and Best, Martin and Edwards, John and Gilham, Richard and Essery, Richard and Wood, Richard and Hewitt, Helene and Storkey, David and Graham, Tim and Calvert, Daley and Hyder, Pat and Blockley, Ed and West, Alex and Keen, Ann and Ridley, Jeff and Rae, Jamie and Hill, Richard and Copsey, Dan and Roberts, Malcolm and Mizielinski, Matthew and Mulcahy, Jane and Sellar, Alistair and Boutle, Ian and Lock, Adrian and Johnson, Colin and Johnson, Ben and Jones, Colin and O'Connor, Fiona and Hardiman, Steven and Dalvi, Mohit and Zerroukat, Mohamed and Rioual, Jean-Christophe and Neininger, Erica and Shepherd, Harry and Andrejczuk, Miroslaw and Linova-Pavlova, Irina and Baran, Anthony and Furtado, Kalli and Hill, Peter and Manners, James and Morcrette, Cyril and Sanchez, Claudio and Smith, Chris and Stratton, Rachel and Tennant, Warren and Tomassini, Lorenzo and van Weverberg, Kwinten and Vosper, Simon and Browse, Jo and Bushell, Andrew and Gedney, Nicola and Jones, Andy and Mann, Graham and Rumbold, Heather and Ujiie, Masashi and Blaker, Adam T. and Mathiot, Pierre and Megann, Alex and Aksenov, Yevgeny and Kuhlbrodt, Till and Sinha, Bablu and Schroeder, David and Whitall, Michael and McCoy, Renata B. and Grosvenor, Daniel and Andrews, Tim and Rumbold, Steve and Mollard, James and Bellouin, Nicolas},
	year = {2018},
	note = {Medium: application/x-netcdf},
	keywords = {climate},
}

@misc{roberts_ecmwf_2018,
	title = {{ECMWF} {ECMWF}-{IFS}-{MR} model output prepared for {CMIP6} {HighResMIP} hist-1950},
	copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International},
	url = {http://cera-www.dkrz.de/WDCC/meta/CMIP6/CMIP6.HighResMIP.ECMWF.ECMWF-IFS-MR.hist-1950},
	doi = {10.22033/ESGF/CMIP6.4983},
	abstract = {Coupled Model Intercomparison Project Phase 6 (CMIP6) data sets: These data includes all datasets published for 'CMIP6.HighResMIP.ECMWF.ECMWF-IFS-MR.hist-1950' according to the Data Reference Syntax defined as 'mip\_era.activity\_id.institution\_id.source\_id.experiment\_id.member\_id.table\_id.variable\_id.grid\_label.version'. The model used in climate research named ECMWF-IFS-MR (50 km atmosphere and 25 km ocean), released in 2017, includes the components: atmos: IFS (IFS CY43R1, Tco199, cubic octahedral reduced Gaussian grid equivalent to 800 x 400 longitude/latitude; 91 levels; top level 0.01 hPa), land: HTESSEL (as implemented in IFS CY43R1), ocean: NEMO3.4 (NEMO v3.4; ORCA025 tripolar grid; 1442 x 1021 longitude/latitude; 75 levels; top grid cell 0-1 m), seaIce: LIM2 (LIM v2; ORCA025 tripolar grid; 1442 x 1021 longitude/latitude). The model was run by the European Centre for Medium-Range Weather Forecasts, Reading RG2 9AX, UK (ECMWF) in native nominal resolutions: atmos: 50 km, land: 50 km, ocean: 25 km, seaIce: 25 km. Project: These data have been generated as part of the internationally-coordinated Coupled Model Intercomparison Project Phase 6 (CMIP6; see also GMD Special Issue: http://www.geosci-model-dev.net/special\_issue590.html). The simulation data provides a basis for climate research designed to answer fundamental science questions, and the results will undoubtedly be relied on by authors of the Sixth Assessment Report of the Intergovernmental Panel on Climate Change (IPCC-AR6). CMIP6 is a project coordinated by the Working Group on Coupled Modelling (WGCM) as part of the World Climate Research Programme (WCRP). Phase 6 builds on previous phases executed under the leadership of the Program for Climate Model Diagnosis and Intercomparison (PCMDI) and relies on the Earth System Grid Federation (ESGF) and the Centre for Environmental Data Analysis (CEDA) along with numerous related activities for implementation. The original data is hosted and partially replicated at a federated collection of data nodes, and most of the data relied on by the IPCC is being archived for long-term preservation at the IPCC Data Distribution Centre (IPCC DDC) hosted by the German Climate Computing Center (DKRZ). The project includes simulations from about 120 global climate models and around 45 institutions and organizations worldwide. - Project website: https://pcmdi.llnl.gov/CMIP6.},
	language = {en},
	urldate = {2021-10-14},
	publisher = {Earth System Grid Federation},
	author = {Roberts, Christopher David and Senan, Retish and Molteni, Franco and Boussetta, Souhail and Keeley, Sarah},
	collaborator = {Roberts, Christopher David},
	year = {2018},
	note = {Medium: application/x-netcdf},
	keywords = {climate},
}

@misc{voldoire_cnrm-cerfacs_2019,
	title = {{CNRM}-{CERFACS} {CNRM}-{CM6}-1-{HR} model output prepared for {CMIP6} {HighResMIP}},
	copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International},
	url = {http://cera-www.dkrz.de/WDCC/meta/CMIP6/CMIP6.HighResMIP.CNRM-CERFACS.CNRM-CM6-1-HR},
	doi = {10.22033/ESGF/CMIP6.1387},
	abstract = {Coupled Model Intercomparison Project Phase 6 (CMIP6) data sets: These data includes all datasets published for 'CMIP6.HighResMIP.CNRM-CERFACS.CNRM-CM6-1-HR' according to the Data Reference Syntax defined as 'mip\_era.activity\_id.institution\_id.source\_id.experiment\_id.member\_id.table\_id.variable\_id.grid\_label.version'. The model used in climate research named CNRM-CM6-1-HR, released in 2017, includes the components: aerosol: prescribed monthly fields computed by TACTIC\_v2 scheme, atmos: Arpege 6.3 (T359; Gaussian Reduced with 181724 grid points in total distributed over 360 latitude circles (with 720 grid points per latitude circle between 32.2degN and 32.2degS reducing to 18 grid points per latitude circle at 89.6degN and 89.6degS); 91 levels; top level 78.4 km), atmosChem: OZL\_v2, land: Surfex 8.0c, ocean: Nemo 3.6 (eORCA025, tripolar primarily 1/4deg; 1442 x 1050 longitude/latitude; 75 levels; top grid cell 0-1 m), seaIce: Gelato 6.1. The model was run by the CNRM (Centre National de Recherches Meteorologiques, Toulouse 31057, France), CERFACS (Centre Europeen de Recherche et de Formation Avancee en Calcul Scientifique, Toulouse 31057, France) (CNRM-CERFACS) in native nominal resolutions: aerosol: 100 km, atmos: 100 km, atmosChem: 100 km, land: 100 km, ocean: 25 km, seaIce: 25 km. Project: These data have been generated as part of the internationally-coordinated Coupled Model Intercomparison Project Phase 6 (CMIP6; see also GMD Special Issue: http://www.geosci-model-dev.net/special\_issue590.html). The simulation data provides a basis for climate research designed to answer fundamental science questions, and the results will undoubtedly be relied on by authors of the Sixth Assessment Report of the Intergovernmental Panel on Climate Change (IPCC-AR6). CMIP6 is a project coordinated by the Working Group on Coupled Modelling (WGCM) as part of the World Climate Research Programme (WCRP). Phase 6 builds on previous phases executed under the leadership of the Program for Climate Model Diagnosis and Intercomparison (PCMDI) and relies on the Earth System Grid Federation (ESGF) and the Centre for Environmental Data Analysis (CEDA) along with numerous related activities for implementation. The original data is hosted and partially replicated at a federated collection of data nodes, and most of the data relied on by the IPCC is being archived for long-term preservation at the IPCC Data Distribution Centre (IPCC DDC) hosted by the German Climate Computing Center (DKRZ). The project includes simulations from about 120 global climate models and around 45 institutions and organizations worldwide. - Project website: https://pcmdi.llnl.gov/CMIP6.},
	language = {en},
	urldate = {2021-10-14},
	publisher = {Earth System Grid Federation},
	author = {Voldoire, Aurore},
	collaborator = {Voldoire, Aurore},
	year = {2019},
	note = {Medium: application/x-netcdf},
	keywords = {climate},
}

@book{scholkopf_learning_2002,
	address = {Cambridge, Mass},
	series = {Adaptive computation and machine learning},
	title = {Learning with kernels: support vector machines, regularization, optimization, and beyond},
	isbn = {978-0-262-19475-4},
	shorttitle = {Learning with kernels},
	publisher = {MIT Press},
	author = {Schölkopf, Bernhard and Smola, Alexander J.},
	year = {2002},
	keywords = {kernel methods, support vector machines},
}

@inproceedings{reid_bayesian_2013,
	title = {Bayesian {Fusion} for {Multi}-{Modal} {Aerial} {Images}},
	volume = {09},
	isbn = {978-981-07-3937-9},
	url = {http://www.roboticsproceedings.org/rss09/p25.html},
	urldate = {2021-10-14},
	author = {Reid, Alistair and Ramos, Fabio and Sukkarieh, Salah},
	month = jun,
	year = {2013},
}

@article{goovaerts_kriging_2008,
	title = {Kriging and {Semivariogram} {Deconvolution} in the {Presence} of {Irregular} {Geographical} {Units}},
	volume = {40},
	issn = {1874-8953},
	url = {https://doi.org/10.1007/s11004-007-9129-1},
	doi = {10.1007/s11004-007-9129-1},
	abstract = {This paper presents a methodology to conduct geostatistical variography and interpolation on areal data measured over geographical units (or blocks) with different sizes and shapes, while accounting for heterogeneous weight or kernel functions within those units. The deconvolution method is iterative and seeks the point-support model that minimizes the difference between the theoretically regularized semivariogram model and the model fitted to areal data. This model is then used in area-to-point (ATP) kriging to map the spatial distribution of the attribute of interest within each geographical unit. The coherence constraint ensures that the weighted average of kriged estimates equals the areal datum.},
	language = {en},
	number = {1},
	urldate = {2021-10-14},
	journal = {Mathematical Geosciences},
	author = {Goovaerts, Pierre},
	month = jan,
	year = {2008},
	pages = {101--128},
}

@article{wang_downscaling_2015,
	title = {Downscaling {MODIS} images with area-to-point regression kriging},
	volume = {166},
	issn = {0034-4257},
	url = {https://www.sciencedirect.com/science/article/pii/S003442571530033X},
	doi = {10.1016/j.rse.2015.06.003},
	abstract = {The first seven bands of the Moderate Resolution Imaging Spectroradiometer (MODIS) data have been used widely for global land-cover/land-use (LCLU) monitoring (e.g., deforestation over the Amazon basin). However, the spatial resolution of MODIS bands 3–7 (i.e., 500m) is coarser than that of bands 1 and 2 (i.e., 250m), and may be too coarse for a large number of applications. In this paper, a new geostatistical approach based on area-to-point regression kriging (ATPRK) is proposed for downscaling coarse spatial resolution bands 3–7 such as to produce a complete set of MODIS images at 250m. ATPRK takes advantages of the fine spatial resolution information in bands 1 and 2 by regression modeling, and uses area-to-point kriging to downscale the coarse residuals from the regression. ATPRK was compared to four existing methods, including the principal component analysis, wavelets, high-pass filter and kriging with external drift (KED) methods for downscaling in two experiments on MODIS data from the Brazilian Amazon. Both visual and quantitative evaluations (in terms of the root mean square error, correlation coefficient, relative global-dimensional synthesis error, universal image quality index, spectral angle mapper and spectral information divergence) showed that ATPRK produced sharpened images with the greatest quality. In addition, ATPRK perfectly preserved the spectral properties of the original coarse data and was faster than KED. The results reveal the great potential of ATPRK applied to MODIS data for a wide variety of applications, including global monitoring of deforestation. The ATPRK proposed in this paper is an entirely new image fusion approach based on a new conceptualization.},
	language = {en},
	urldate = {2021-10-14},
	journal = {Remote Sensing of Environment},
	author = {Wang, Qunming and Shi, Wenzhong and Atkinson, Peter M. and Zhao, Yuanling},
	month = sep,
	year = {2015},
	keywords = {geostatistics, modis, multi-fidelity kernels},
	pages = {191--204},
}

@article{neumann_assessing_2019,
	title = {Assessing the scales in numerical weather and climate predictions: will exascale be the rescue?},
	volume = {377},
	shorttitle = {Assessing the scales in numerical weather and climate predictions},
	url = {https://royalsocietypublishing.org/doi/10.1098/rsta.2018.0148},
	doi = {10.1098/rsta.2018.0148},
	abstract = {We discuss scientific features and computational performance of kilometre-scale global weather and climate simulations, considering the Icosahedral Non-hydrostatic (ICON) model and the Integrated Forecast System (IFS). Scalability measurements and a performance modelling approach are used to derive performance estimates for these models on upcoming exascale supercomputers. This is complemented by preliminary analyses of the model data that illustrate the importance of high-resolution models to gain improvements in the accuracy of convective processes, a better understanding of physics dynamics interactions and poorly resolved or parametrized processes, such as gravity waves, convection and boundary layer.

This article is part of the theme issue ‘Multiscale modelling, simulation and computing: from the desktop to the exascale’.},
	number = {2142},
	urldate = {2021-10-12},
	journal = {Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences},
	author = {Neumann, Philipp and Düben, Peter and Adamidis, Panagiotis and Bauer, Peter and Brück, Matthias and Kornblueh, Luis and Klocke, Daniel and Stevens, Bjorn and Wedi, Nils and Biercamp, Joachim},
	month = apr,
	year = {2019},
	note = {Publisher: Royal Society},
	keywords = {scalability},
	pages = {20180148},
}

@article{kyriakidis_geostatistical_2004,
	title = {A {Geostatistical} {Framework} for {Area}-to-{Point} {Spatial} {Interpolation}},
	volume = {36},
	issn = {1538-4632},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/j.1538-4632.2004.tb01135.x},
	doi = {10.1111/j.1538-4632.2004.tb01135.x},
	abstract = {The spatial prediction of point values from areal data of the same attribute is addressed within the general geostatistical framework of change of support; the term support refers to the domain informed by each datum or unknown value. It is demonstrated that the proposed geostatistical framework can explicitly and consistently account for the support differences between the available areal data and the sought-after point predictions. In particular, it is proved that appropriate modeling of all area-to-area and area-to-point covariances required by the geostatistical frame-work yields coherent (mass-preserving or pycnophylactic) predictions. In other words, the areal average (or areal total) of point predictions within any arbitrary area informed by an areal-average (or areal-total) datum is equal to that particular datum. In addition, the proposed geostatistical framework offers the unique advantage of providing a measure of the reliability (standard error) of each point prediction. It is also demonstrated that several existing approaches for area-to-point interpolation can be viewed within this geostatistical framework. More precisely, it is shown that (i) the choropleth map case corresponds to the geostatistical solution under the assumption of spatial independence at the point support level; (ii) several forms of kernel smoothing can be regarded as alternative (albeit sometimes incoherent) implementations of the geostatistical approach; and (iii) Tobler's smooth pycnophylactic interpolation, on a quasi-infinite domain without non-negativity constraints, corresponds to the geostatistical solution when the semivariogram model adopted at the point support level is identified to the free-space Green's functions (linear in 1-D or logarithmic in 2-D) of Poisson's partial differential equation. In lieu of a formal case study, several 1-D examples are given to illustrate pertinent concepts.},
	language = {en},
	number = {3},
	urldate = {2021-10-08},
	journal = {Geographical Analysis},
	author = {Kyriakidis, Phaedon C.},
	year = {2004},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/j.1538-4632.2004.tb01135.x},
	pages = {259--289},
}

@article{maolin_multi-fidelity_2020,
	title = {A multi-fidelity surrogate model based on support vector regression},
	volume = {61},
	copyright = {© Springer-Verlag GmbH Germany, part of Springer Nature 2020.},
	issn = {1615147X},
	url = {http://www.proquest.com/docview/2419210101/abstract/C1D6076CA22E4FDEPQ/1},
	doi = {http://dx.doi.org.ezproxy.library.sydney.edu.au/10.1007/s00158-020-02522-6},
	abstract = {Computational simulations with different fidelities have been widely used in engineering design and optimization. A high-fidelity (HF) model is generally more accurate but also more time-consuming than the corresponding low-fidelity (LF) model. To take advantage of both HF and LF models, a number of multi-fidelity surrogate (MFS) models based on different surrogate models (e.g., Kriging, response surface, and radial basis function) have been developed, but MFS models based on support vector regression are rarely reported. In this paper, a new MFS model based on support vector regression, which is named Co\_SVR, is developed. In the proposed method, the HF and LF samples are mapped into a high-dimensional feature space through a kernel function, and then, a linear model is utilized to evaluate the relationship between inputs and outputs. The root mean square error (RMSE) of HF responses of interest is used to express the training error of Co\_SVR, and a heuristic algorithm, grey wolf optimizer, is used to obtain the optimal parameters. For verification, the Co\_SVR model is compared with four popular multi-fidelity surrogate models and four single-fidelity surrogate models through a number of numerical cases and a pressure relief valve design problem. The results show that Co\_SVR provides competitive performance in both numerical cases and the practical case. Moreover, the effects of key factors (i.e., the correlation between HF and LF models, the cost ratio of HF to LF models, and the combination of HF and LF samples) on the performance of Co\_SVR are also explored.},
	language = {English},
	number = {6},
	urldate = {2021-10-01},
	journal = {Structural and Multidisciplinary Optimization},
	author = {Maolin, Shi and Liye, Lv and Sun, Wei and Xueguan, Song},
	month = jun,
	year = {2020},
	note = {Num Pages: 2363-2375
Place: Heidelberg, Netherlands
Publisher: Springer Nature B.V.},
	keywords = {multi-fidelity, simulation, support vector machines},
	pages = {2363--2375},
}

@article{zhu_multi-resolution_2021,
	title = {Multi-resolution {Spatial} {Regression} for {Aggregated} {Data} with an {Application} to {Crop} {Yield} {Prediction}},
	url = {http://arxiv.org/abs/2105.01460},
	abstract = {We develop a new methodology for spatial regression of aggregated outputs on multi-resolution covariates. Such problems often occur with spatial data, for example in crop yield prediction, where the output is spatially-aggregated over an area and the covariates may be observed at multiple resolutions. Building upon previous work on aggregated output regression, we propose a regression framework to synthesise the effects of the covariates at different resolutions on the output and provide uncertainty estimation. We show that, for a crop yield prediction problem, our approach is more scalable, via variational inference, than existing multi-resolution regression models. We also show that our framework yields good predictive performance, compared to existing multi-resolution crop yield models, whilst being able to provide estimation of the underlying spatial effects.},
	urldate = {2021-09-28},
	journal = {arXiv:2105.01460 [stat]},
	author = {Zhu, Harrison and Howes, Adam and van Eer, Owen and Rischard, Maxime and Sejdinovic, Dino and Flaxman, Seth},
	month = may,
	year = {2021},
	note = {arXiv: 2105.01460},
	keywords = {statistics},
}

@inproceedings{tanskanen_non-linearities_2020,
	title = {Non-{Linearities} in {Gaussian} {Processes} with {Integral} {Observations}},
	doi = {10.1109/MLSP49062.2020.9231553},
	abstract = {Gaussian processes (GP) can be used for inferring latent continuous functions also based on aggregate observations corresponding to integrals of the function, for example to learn daily rate of new infections in a population based on cumulative observations collected only weekly. We extend these approaches to cases where the observations correspond to aggregates of arbitrary non-linear transformations of a GP. Such models are needed, for example, when the latent function of interest is known to be non-negative or bounded. We present a solution based on Markov chain Monte Carlo with numerical integration for aggregation, and demonstrate it in binned Poisson regression and in non-invasive detection of fouling using ultrasound waves.},
	booktitle = {2020 {IEEE} 30th {International} {Workshop} on {Machine} {Learning} for {Signal} {Processing} ({MLSP})},
	author = {Tanskanen, Ville and Longi, Krista and Klami, Arto},
	month = sep,
	year = {2020},
	note = {ISSN: 1551-2541},
	keywords = {aggregated data, gaussian processes},
	pages = {1--6},
}

@article{katzfuss_multi-resolution_2017,
	title = {A {Multi}-{Resolution} {Approximation} for {Massive} {Spatial} {Datasets}},
	volume = {112},
	issn = {0162-1459},
	url = {https://doi.org/10.1080/01621459.2015.1123632},
	doi = {10.1080/01621459.2015.1123632},
	abstract = {Automated sensing instruments on satellites and aircraft have enabled the collection of massive amounts of high-resolution observations of spatial fields over large spatial regions. If these datasets can be efficiently exploited, they can provide new insights on a wide variety of issues. However, traditional spatial-statistical techniques such as kriging are not computationally feasible for big datasets. We propose a multi-resolution approximation (M-RA) of Gaussian processes observed at irregular locations in space. The M-RA process is specified as a linear combination of basis functions at multiple levels of spatial resolution, which can capture spatial structure from very fine to very large scales. The basis functions are automatically chosen to approximate a given covariance function, which can be nonstationary. All computations involving the M-RA, including parameter inference and prediction, are highly scalable for massive datasets. Crucially, the inference algorithms can also be parallelized to take full advantage of large distributed-memory computing environments. In comparisons using simulated data and a large satellite dataset, the M-RA outperforms a related state-of-the-art method. Supplementary materials for this article are available online.},
	number = {517},
	urldate = {2021-09-28},
	journal = {Journal of the American Statistical Association},
	author = {Katzfuss, Matthias},
	month = jan,
	year = {2017},
	note = {Publisher: Taylor \& Francis
\_eprint: https://doi.org/10.1080/01621459.2015.1123632},
	keywords = {gaussian processes, remote sensing},
	pages = {201--214},
}

@article{ton_spatial_2018,
	series = {One world, one health},
	title = {Spatial mapping with {Gaussian} processes and nonstationary {Fourier} features},
	volume = {28},
	issn = {2211-6753},
	url = {https://www.sciencedirect.com/science/article/pii/S2211675317302890},
	doi = {10.1016/j.spasta.2018.02.002},
	abstract = {The use of covariance kernels is ubiquitous in the field of spatial statistics. Kernels allow data to be mapped into high-dimensional feature spaces and can thus extend simple linear additive methods to nonlinear methods with higher order interactions. However, until recently, there has been a strong reliance on a limited class of stationary kernels such as the Matérn or squared exponential, limiting the expressiveness of these modelling approaches. Recent machine learning research has focused on spectral representations to model arbitrary stationary kernels and introduced more general representations that include classes of nonstationary kernels. In this paper, we exploit the connections between Fourier feature representations, Gaussian processes and neural networks to generalise previous approaches and develop a simple and efficient framework to learn arbitrarily complex nonstationary kernel functions directly from the data, while taking care to avoid overfitting using state-of-the-art methods from deep learning. We highlight the very broad array of kernel classes that could be created within this framework. We apply this to a time series dataset and a remote sensing problem involving land surface temperature in Eastern Africa. We show that without increasing the computational or storage complexity, nonstationary kernels can be used to improve generalisation performance and provide more interpretable results.},
	language = {en},
	urldate = {2021-09-28},
	journal = {Spatial Statistics},
	author = {Ton, Jean-Francois and Flaxman, Seth and Sejdinovic, Dino and Bhatt, Samir},
	month = dec,
	year = {2018},
	keywords = {gaussian processes, random fourier features, spatio-temporal modelling},
	pages = {59--78},
}

@inproceedings{sriperumbudur_optimal_2015,
	address = {Cambridge, MA, USA},
	series = {{NIPS}'15},
	title = {Optimal rates for {Random} {Fourier} features},
	abstract = {Kernel methods represent one of the most powerful tools in machine learning to tackle problems expressed in terms of function values and derivatives due to their capability to represent and model complex relations. While these methods show good versatility, they are computationally intensive and have poor scalability to large data as they require operations on Gram matrices. In order to mitigate this serious computational limitation, recently randomized constructions have been proposed in the literature, which allow the application of fast linear algorithms. Random Fourier features (RFF) are among the most popular and widely applied constructions: they provide an easily computable, low-dimensional feature representation for shift-invariant kernels. Despite the popularity of RFFs, very little is understood theoretically about their approximation quality. In this paper, we provide a detailed finite-sample theoretical analysis about the approximation quality of RFFs by (i) establishing optimal (in terms of the RFF dimension, and growing set size) performance guarantees in uniform norm, and (ii) presenting guarantees in Lr (1 ≤ r {\textless} ∞) norms. We also propose an RFF approximation to derivatives of a kernel with a theoretical study on its approximation quality.},
	urldate = {2021-09-28},
	booktitle = {Proceedings of the 28th {International} {Conference} on {Neural} {Information} {Processing} {Systems} - {Volume} 1},
	publisher = {MIT Press},
	author = {Sriperumbudur, Bharath K. and Szabó, Zoltán},
	month = dec,
	year = {2015},
	pages = {1144--1152},
}

@inproceedings{longi_sensor_2020,
	title = {Sensor {Placement} for {Spatial} {Gaussian} {Processes} with {Integral} {Observations}},
	url = {https://proceedings.mlr.press/v124/longi20a.html},
	abstract = {Gaussian processes (GP) are a natural tool for estimating unknown functions, typically based on a collection of point-wise observations. Interestingly, the GP formalism can be used also with observations that are integrals of the unknown function along some known trajectories, which makes GPs a promising technique for inverse problems in a wide range of physical sensing problems. However, in many real world applications collecting data is laborious and time consuming. We provide tools for optimizing sensor locations for GPs using integral observations, extending both model-based and geometric strategies for GP sensor placement.We demonstrate the techniques in ultrasonic detection of fouling in closed pipes.},
	language = {en},
	urldate = {2021-09-23},
	booktitle = {Proceedings of the 36th {Conference} on {Uncertainty} in {Artificial} {Intelligence} ({UAI})},
	publisher = {PMLR},
	author = {Longi, Krista and Rajani, Chang and Sillanpää, Tom and Mäkinen, Joni and Rauhala, Timo and Salmi, Ari and Haeggström, Edward and Klami, Arto},
	month = aug,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {1009--1018},
}

@article{chau_deconditional_2021,
	title = {Deconditional {Downscaling} with {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/2105.12909},
	abstract = {Refining low-resolution (LR) spatial fields with high-resolution (HR) information is challenging as the diversity of spatial datasets often prevents direct matching of observations. Yet, when LR samples are modeled as aggregate conditional means of HR samples with respect to a mediating variable that is globally observed, the recovery of the underlying fine-grained field can be framed as taking an "inverse" of the conditional expectation, namely a deconditioning problem. In this work, we introduce conditional mean processes (CMP), a new class of Gaussian Processes describing conditional means. By treating CMPs as inter-domain features of the underlying field, a posterior for the latent field can be established as a solution to the deconditioning problem. Furthermore, we show that this solution can be viewed as a two-staged vector-valued kernel ridge regressor and show that it has a minimax optimal convergence rate under mild assumptions. Lastly, we demonstrate its proficiency in a synthetic and a real-world atmospheric field downscaling problem, showing substantial improvements over existing methods.},
	urldate = {2021-09-23},
	journal = {arXiv:2105.12909 [cs, stat]},
	author = {Chau, Siu Lun and Bouabid, Shahine and Sejdinovic, Dino},
	month = jun,
	year = {2021},
	note = {arXiv: 2105.12909},
	keywords = {machine learning},
}

@inproceedings{osborne_active_2012,
	title = {Active {Learning} of {Model} {Evidence} {Using} {Bayesian} {Quadrature}},
	volume = {25},
	url = {https://papers.nips.cc/paper/2012/hash/6364d3f0f495b6ab9dcf8d3b5c6e0b01-Abstract.html},
	urldate = {2021-09-22},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Osborne, Michael and Garnett, Roman and Ghahramani, Zoubin and Duvenaud, David K and Roberts, Stephen J and Rasmussen, Carl},
	year = {2012},
}

@article{zhang_learning_2021,
	title = {Learning from {Aggregate} {Observations}},
	url = {http://arxiv.org/abs/2004.06316},
	abstract = {We study the problem of learning from aggregate observations where supervision signals are given to sets of instances instead of individual instances, while the goal is still to predict labels of unseen individuals. A well-known example is multiple instance learning (MIL). In this paper, we extend MIL beyond binary classification to other problems such as multiclass classification and regression. We present a general probabilistic framework that accommodates a variety of aggregate observations, e.g., pairwise similarity/triplet comparison for classification and mean/difference/rank observation for regression. Simple maximum likelihood solutions can be applied to various differentiable models such as deep neural networks and gradient boosting machines. Moreover, we develop the concept of consistency up to an equivalence relation to characterize our estimator and show that it has nice convergence properties under mild assumptions. Experiments on three problem settings -- classification via triplet comparison and regression via mean/rank observation indicate the effectiveness of the proposed method.},
	urldate = {2021-09-21},
	journal = {arXiv:2004.06316 [cs, stat]},
	author = {Zhang, Yivan and Charoenphakdee, Nontawat and Wu, Zhenguo and Sugiyama, Masashi},
	month = jan,
	year = {2021},
	note = {arXiv: 2004.06316},
	keywords = {machine learning},
}

@article{mcdonald_compositional_2021,
	title = {Compositional {Modeling} of {Nonlinear} {Dynamical} {Systems} with {ODE}-based {Random} {Features}},
	url = {http://arxiv.org/abs/2106.05960},
	abstract = {Effectively modeling phenomena present in highly nonlinear dynamical systems whilst also accurately quantifying uncertainty is a challenging task, which often requires problem-specific techniques. We present a novel, domain-agnostic approach to tackling this problem, using compositions of physics-informed random features, derived from ordinary differential equations. The architecture of our model leverages recent advances in approximate inference for deep Gaussian processes, such as layer-wise weight-space approximations which allow us to incorporate random Fourier features, and stochastic variational inference for approximate Bayesian inference. We provide evidence that our model is capable of capturing highly nonlinear behaviour in real-world multivariate time series data. In addition, we find that our approach achieves comparable performance to a number of other probabilistic models on benchmark regression tasks.},
	urldate = {2021-09-21},
	journal = {arXiv:2106.05960 [cs, stat]},
	author = {McDonald, Thomas M. and Álvarez, Mauricio A.},
	month = jun,
	year = {2021},
	note = {arXiv: 2106.05960},
	keywords = {machine learning},
}

@article{yousefi_multi-task_2020,
	title = {Multi-task {Learning} for {Aggregated} {Data} using {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/1906.09412},
	abstract = {Aggregated data is commonplace in areas such as epidemiology and demography. For example, census data for a population is usually given as averages defined over time periods or spatial resolutions (cities, regions or countries). In this paper, we present a novel multi-task learning model based on Gaussian processes for joint learning of variables that have been aggregated at different input scales. Our model represents each task as the linear combination of the realizations of latent processes that are integrated at a different scale per task. We are then able to compute the cross-covariance between the different tasks either analytically or numerically. We also allow each task to have a potentially different likelihood model and provide a variational lower bound that can be optimised in a stochastic fashion making our model suitable for larger datasets. We show examples of the model in a synthetic example, a fertility dataset, and an air pollution prediction application.},
	urldate = {2021-09-21},
	journal = {arXiv:1906.09412 [cs, stat]},
	author = {Yousefi, Fariba and Smith, Michael Thomas and Álvarez, Mauricio A.},
	month = feb,
	year = {2020},
	note = {arXiv: 1906.09412},
	keywords = {machine learning},
}

@article{pu_vae_2017,
	title = {{VAE} {Learning} via {Stein} {Variational} {Gradient} {Descent}},
	url = {http://arxiv.org/abs/1704.05155},
	abstract = {A new method for learning variational autoencoders (VAEs) is developed, based on Stein variational gradient descent. A key advantage of this approach is that one need not make parametric assumptions about the form of the encoder distribution. Performance is further enhanced by integrating the proposed encoder with importance sampling. Excellent performance is demonstrated across multiple unsupervised and semi-supervised problems, including semi-supervised analysis of the ImageNet data, demonstrating the scalability of the model to large datasets.},
	urldate = {2021-09-08},
	journal = {arXiv:1704.05155 [cs]},
	author = {Pu, Yunchen and Gan, Zhe and Henao, Ricardo and Li, Chunyuan and Han, Shaobo and Carin, Lawrence},
	month = nov,
	year = {2017},
	note = {arXiv: 1704.05155},
	keywords = {machine learning},
}

@article{gelfand_change_2001,
	title = {On the change of support problem for spatio-temporal data},
	volume = {2},
	issn = {1465-4644},
	url = {https://doi.org/10.1093/biostatistics/2.1.31},
	doi = {10.1093/biostatistics/2.1.31},
	abstract = {In practice, spatial data are sometimes collected at points (i.e. point-referenced data) and at other times are associated with
areal units (i.e. block data). The change of support problem is concerned with inference about the values of a variable at points
or blocks different from those at which it has been observed. In the
context of block data which can be sensibly viewed as averaging over
point data, we propose a unifying approach for prediction from points
to points, points to blocks, blocks to points, and blocks to blocks.
The approach includes fully Bayesian kriging. We also extend our
approach to the the case of spatio-temporal data, wherein a judicious
specification of spatio-temporal association enables manageable
computation. Exemplification of the static spatial case is provided
using a dataset of point-level ozone measurements in the Atlanta,
Georgia metropolitan area. The dynamic spatial case is illustrated
using a temporally extended version of this dataset, enabling
comparison at the common time point.*To whom correspondence should be
addressed.},
	number = {1},
	urldate = {2021-08-09},
	journal = {Biostatistics},
	author = {Gelfand, Alan E. and Zhu, Li and Carlin, Bradley P.},
	month = mar,
	year = {2001},
	pages = {31--45},
}

@inproceedings{higdon_space_2002,
	address = {London},
	title = {Space and {Space}-{Time} {Modeling} using {Process} {Convolutions}},
	isbn = {978-1-4471-0657-9},
	doi = {10.1007/978-1-4471-0657-9_2},
	abstract = {A continuous spatial model can be constructed by convolving a very simple, perhaps independent, process with a kernel or point spread function. This approach for constructing a spatial process offers a number of advantages over specification through a spatial covariogram. In particular, this process convolution specification leads to computational simplifications and easily extends beyond simple stationary models. This paper uses process convolution models to build space and space-time models that are flexible and able to accommodate large amounts of data. Data from environmental monitoring is considered.},
	language = {en},
	booktitle = {Quantitative {Methods} for {Current} {Environmental} {Issues}},
	publisher = {Springer},
	author = {Higdon, Dave},
	editor = {Anderson, Clive W. and Barnett, Vic and Chatwin, Philip C. and El-Shaarawi, Abdel H.},
	year = {2002},
	keywords = {gaussian processes, spatio-temporal modelling},
	pages = {37--56},
}

@article{tanaka_spatially_2020,
	title = {Spatially {Aggregated} {Gaussian} {Processes} with {Multivariate} {Areal} {Outputs}},
	url = {http://arxiv.org/abs/1907.08350},
	abstract = {We propose a probabilistic model for inferring the multivariate function from multiple areal data sets with various granularities. Here, the areal data are observed not at location points but at regions. Existing regression-based models can only utilize the sufficiently fine-grained auxiliary data sets on the same domain (e.g., a city). With the proposed model, the functions for respective areal data sets are assumed to be a multivariate dependent Gaussian process (GP) that is modeled as a linear mixing of independent latent GPs. Sharing of latent GPs across multiple areal data sets allows us to effectively estimate the spatial correlation for each areal data set; moreover it can easily be extended to transfer learning across multiple domains. To handle the multivariate areal data, we design an observation model with a spatial aggregation process for each areal data set, which is an integral of the mixed GP over the corresponding region. By deriving the posterior GP, we can predict the data value at any location point by considering the spatial correlations and the dependences between areal data sets, simultaneously. Our experiments on real-world data sets demonstrate that our model can 1) accurately refine coarse-grained areal data, and 2) offer performance improvements by using the areal data sets from multiple domains.},
	urldate = {2021-08-09},
	journal = {arXiv:1907.08350 [cs, stat]},
	author = {Tanaka, Yusuke and Tanaka, Toshiyuki and Iwata, Tomoharu and Kurashima, Takeshi and Okawa, Maya and Akagi, Yasunori and Toda, Hiroyuki},
	month = jan,
	year = {2020},
	note = {arXiv: 1907.08350},
	keywords = {machine learning},
}

@article{ng_generative_2021,
	title = {Generative {Archimedean} {Copulas}},
	url = {http://arxiv.org/abs/2102.11351},
	abstract = {We propose a new generative modeling technique for learning multidimensional cumulative distribution functions (CDFs) in the form of copulas. Specifically, we consider certain classes of copulas known as Archimedean and hierarchical Archimedean copulas, popular for their parsimonious representation and ability to model different tail dependencies. We consider their representation as mixture models with Laplace transforms of latent random variables from generative neural networks. This alternative representation allows for computational efficiencies and easy sampling, especially in high dimensions. We describe multiple methods for optimizing the network parameters. Finally, we present empirical results that demonstrate the efficacy of our proposed method in learning multidimensional CDFs and its computational efficiency compared to existing methods.},
	urldate = {2021-08-03},
	journal = {arXiv:2102.11351 [cs]},
	author = {Ng, Yuting and Hasan, Ali and Elkhalil, Khalil and Tarokh, Vahid},
	month = jun,
	year = {2021},
	note = {arXiv: 2102.11351},
	keywords = {machine learning},
}

@article{reid_gaussian_2011,
	title = {Gaussian {Process} {Models} for {Analysis} of {Remotely} {Sensed} {Geo}-{Spatial} {Data}},
	url = {https://www.semanticscholar.org/paper/Gaussian-Process-Models-for-Analysis-of-Remotely-Reid/5f25955a9e2b6e3ba816d1028e69787c5d4f4ca6},
	abstract = {Alistair S. Reid Doctor of Philosophy The University of Sydney March 2011 Gaussian Process Models for Analysis of Remotely Sensed Geo-Spatial Data Remote sensing technologies provide an effective source of data to study geo-spatial problems in fields such as ecology, meteorology or geology. In the current literature, the study of a geo-spatial process is typically approached by classifying the pixels of hyper-spectral images acquired by satellites and high altitude aircraft. This thesis is motivated by the need to add new modalities of geospatial data acquired by a low flying robotic aircraft into the spatial analysis, in order to maximise the scientific value of its high resolution, three channel image data. This has led to the exploration of two underlying problems. Firstly, a principled fusion method was sought to exploit the complementary spatial resolution of the UAV data and the spectral resolution of other available data modalities in order to synthesise an accurate, high quality reconstructed image of attributes that the UAV was unable to observe. This has been approached as a general multimodal image fusion problem, which requires a fusion framework to analyse the spatial relationships between observations from different sensors, as well as the dependence between spectral bands of different wavelengths. Secondly, the problem of classifying different features of interest directly from the new UAV data has been explored. With only colour spectral bands, a vision based classification approach must rely on other descriptors such as texture to discriminate between features. The recognition of natural features based on these features can pose a difficult perception problem, motivating the search for an efficient Bayesian multi-class classifier to model the data. Gaussian process (GP) models are a popular framework for Bayesian inference using both spatial point-data, and supervised classification data. GP models are fully statistical, have a principled marginal likelihood learning criterion, and are able to use flexible covariance models to determine the relevance of input data dimensions, rejecting irrelevant information and accommodating noisy signals. The thesis postulates that the GP estimation framework is an appropriate basis for addressing the two},
	language = {en},
	urldate = {2021-07-13},
	journal = {undefined},
	author = {Reid, Alistair},
	year = {2011},
}

@inproceedings{vido_grids_2016,
	address = {Stockholm, Sweden},
	title = {From grids to continuous occupancy maps through area kernels},
	isbn = {978-1-4673-8026-3},
	url = {http://ieeexplore.ieee.org/document/7487235/},
	doi = {10.1109/ICRA.2016.7487235},
	urldate = {2021-07-06},
	booktitle = {2016 {IEEE} {International} {Conference} on {Robotics} and {Automation} ({ICRA})},
	publisher = {IEEE},
	author = {Vido, Carlos E.O. and Ramos, Fabio},
	month = may,
	year = {2016},
	pages = {1043--1048},
}

@article{smith_gaussian_2019,
	title = {Gaussian {Process} {Regression} for {Binned} {Data}},
	url = {http://arxiv.org/abs/1809.02010},
	abstract = {Many datasets are in the form of tables of binned data. Performing regression on these data usually involves either reading off bin heights, ignoring data from neighbouring bins or interpolating between bins thus over or underestimating the true bin integrals. In this paper we propose an elegant method for performing Gaussian Process (GP) regression given such binned data, allowing one to make probabilistic predictions of the latent function which produced the binned data. We look at several applications. First, for differentially private regression; second, to make predictions over other integrals; and third when the input regions are irregularly shaped collections of polytopes. In summary, our method provides an effective way of analysing binned data such that one can use more information from the histogram representation, and thus reconstruct a more useful and precise density for making predictions.},
	urldate = {2021-06-28},
	journal = {arXiv:1809.02010 [cs, stat]},
	author = {Smith, Michael Thomas and Alvarez, Mauricio A. and Lawrence, Neil D.},
	month = may,
	year = {2019},
	note = {arXiv: 1809.02010},
	keywords = {machine learning},
}

@article{law_variational_2018,
	title = {Variational {Learning} on {Aggregate} {Outputs} with {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/1805.08463},
	abstract = {While a typical supervised learning framework assumes that the inputs and the outputs are measured at the same levels of granularity, many applications, including global mapping of disease, only have access to outputs at a much coarser level than that of the inputs. Aggregation of outputs makes generalization to new inputs much more difficult. We consider an approach to this problem based on variational learning with a model of output aggregation and Gaussian processes, where aggregation leads to intractability of the standard evidence lower bounds. We propose new bounds and tractable approximations, leading to improved prediction accuracy and scalability to large datasets, while explicitly taking uncertainty into account. We develop a framework which extends to several types of likelihoods, including the Poisson model for aggregated count data. We apply our framework to a challenging and important problem, the fine-scale spatial modelling of malaria incidence, with over 1 million observations.},
	urldate = {2021-06-28},
	journal = {arXiv:1805.08463 [cs, stat]},
	author = {Law, Ho Chung Leon and Sejdinovic, Dino and Cameron, Ewan and Lucas, Tim CD and Flaxman, Seth and Battle, Katherine and Fukumizu, Kenji},
	month = may,
	year = {2018},
	note = {arXiv: 1805.08463},
	keywords = {machine learning, statistics},
}

@article{brevault_overview_2020,
	title = {Overview of {Gaussian} process based multi-fidelity techniques with variable relationship between fidelities},
	url = {http://arxiv.org/abs/2006.16728},
	abstract = {The design process of complex systems such as new configurations of aircraft or launch vehicles is usually decomposed in different phases which are characterized for instance by the depth of the analyses in terms of number of design variables and fidelity of the physical models. At each phase, the designers have to compose with accurate but computationally intensive models as well as cheap but inaccurate models. Multi-fidelity modeling is a way to merge different fidelity models to provide engineers with accurate results with a limited computational cost. Within the context of multi-fidelity modeling, approaches relying on Gaussian Processes emerge as popular techniques to fuse information between the different fidelity models. The relationship between the fidelity models is a key aspect in multi-fidelity modeling. This paper provides an overview of Gaussian process-based multi-fidelity modeling techniques for variable relationship between the fidelity models (e.g., linearity, non-linearity, variable correlation). Each technique is described within a unified framework and the links between the different techniques are highlighted. All the approaches are numerically compared on a series of analytical test cases and four aerospace related engineering problems in order to assess their benefits and disadvantages with respect to the problem characteristics.},
	urldate = {2021-06-28},
	journal = {arXiv:2006.16728 [cs, stat]},
	author = {Brevault, Loïc and Balesdent, Mathieu and Hebbal, Ali},
	month = jun,
	year = {2020},
	note = {arXiv: 2006.16728},
	keywords = {machine learning},
}

@book{reid_bayesian_nodate,
	title = {Bayesian {Fusion} for {Multi}-{Modal} {Aerial} {Images}},
	abstract = {Abstract—This paper presents a fusion method to combine aerial images from a low flying Unmanned Aerial Vehicle (UAV) with images of other spectral bands from sources such as satellites or commercial hyperspectral imagers. The proposed method propagates information from high-resolution images into other low-resolution modalities while allowing the images to have different spectral channels. This means the relationship between the high-resolution and low-resolution channels is expected to be non-deterministic, non-linear and non-stationary. A novel Gaussian Process (GP) framework was developed to define a stochastic prior over the estimated images. Its covariance function is computed to replicate the local structure of the high-resolution image, and allows the model to infer a high-resolution estimate from a low-resolution channel. Results are presented for natural images acquired by a UAV in a farmland mapping application. I.},
	author = {Reid, Alistair and Ramos, Fabio and Sukkarieh, Salah},
}

@article{girard_gaussian_2002,
	title = {Gaussian {Process} {Priors} with {Uncertain} {Inputs} {Application} to {Multiple}-{Step} {Ahead} {Time} {Series} {Forecasting}},
	volume = {15},
	url = {https://proceedings.neurips.cc/paper/2002/hash/f3ac63c91272f19ce97c7397825cc15f-Abstract.html},
	language = {en},
	urldate = {2021-06-21},
	journal = {Advances in Neural Information Processing Systems},
	author = {Girard, Agathe and Rasmussen, Carl and Candela, Joaquin Q. and Murray-Smith, Roderick},
	year = {2002},
}

@article{oliva_bayesian_2015,
	title = {Bayesian {Nonparametric} {Kernel}-{Learning}},
	url = {https://arxiv.org/abs/1506.08776v2},
	abstract = {Kernel methods are ubiquitous tools in machine learning. However, there is often little reason for the common practice of selecting a kernel a priori. Even if a universal approximating kernel is selected, the quality of the finite sample estimator may be greatly affected by the choice of kernel. Furthermore, when directly applying kernel methods, one typically needs to compute a \$N {\textbackslash}times N\$ Gram matrix of pairwise kernel evaluations to work with a dataset of \$N\$ instances. The computation of this Gram matrix precludes the direct application of kernel methods on large datasets, and makes kernel learning especially difficult. In this paper we introduce Bayesian nonparmetric kernel-learning (BaNK), a generic, data-driven framework for scalable learning of kernels. BaNK places a nonparametric prior on the spectral distribution of random frequencies allowing it to both learn kernels and scale to large datasets. We show that this framework can be used for large scale regression and classification tasks. Furthermore, we show that BaNK outperforms several other scalable approaches for kernel learning on a variety of real world datasets.},
	language = {en},
	urldate = {2021-06-08},
	author = {Oliva, Junier and Dubey, Avinava and Wilson, Andrew G. and Poczos, Barnabas and Schneider, Jeff and Xing, Eric P.},
	month = jun,
	year = {2015},
}

@article{sutherland_linear-time_2015,
	title = {Linear-time {Learning} on {Distributions} with {Approximate} {Kernel} {Embeddings}},
	url = {https://arxiv.org/abs/1509.07553v2},
	abstract = {Many interesting machine learning problems are best posed by considering instances that are distributions, or sample sets drawn from distributions. Previous work devoted to machine learning tasks with distributional inputs has done so through pairwise kernel evaluations between pdfs (or sample sets). While such an approach is fine for smaller datasets, the computation of an \$N {\textbackslash}times N\$ Gram matrix is prohibitive in large datasets. Recent scalable estimators that work over pdfs have done so only with kernels that use Euclidean metrics, like the \$L\_2\$ distance. However, there are a myriad of other useful metrics available, such as total variation, Hellinger distance, and the Jensen-Shannon divergence. This work develops the first random features for pdfs whose dot product approximates kernels using these non-Euclidean metrics, allowing estimators using such kernels to scale to large datasets by working in a primal space, without computing large Gram matrices. We provide an analysis of the approximation error in using our proposed random features and show empirically the quality of our approximation both in estimating a Gram matrix and in solving learning tasks in real-world and synthetic data.},
	language = {en},
	urldate = {2021-06-08},
	author = {Sutherland, Danica J. and Oliva, Junier B. and Póczos, Barnabás and Schneider, Jeff},
	month = sep,
	year = {2015},
}

@article{sutherland_linear-time_2015-1,
	title = {Linear-time {Learning} on {Distributions} with {Approximate} {Kernel} {Embeddings}},
	url = {https://arxiv.org/abs/1509.07553v2},
	abstract = {Many interesting machine learning problems are best posed by considering instances that are distributions, or sample sets drawn from distributions. Previous work devoted to machine learning tasks with distributional inputs has done so through pairwise kernel evaluations between pdfs (or sample sets). While such an approach is fine for smaller datasets, the computation of an \$N {\textbackslash}times N\$ Gram matrix is prohibitive in large datasets. Recent scalable estimators that work over pdfs have done so only with kernels that use Euclidean metrics, like the \$L\_2\$ distance. However, there are a myriad of other useful metrics available, such as total variation, Hellinger distance, and the Jensen-Shannon divergence. This work develops the first random features for pdfs whose dot product approximates kernels using these non-Euclidean metrics, allowing estimators using such kernels to scale to large datasets by working in a primal space, without computing large Gram matrices. We provide an analysis of the approximation error in using our proposed random features and show empirically the quality of our approximation both in estimating a Gram matrix and in solving learning tasks in real-world and synthetic data.},
	language = {en},
	urldate = {2021-06-08},
	author = {Sutherland, Danica J. and Oliva, Junier B. and Póczos, Barnabás and Schneider, Jeff},
	month = sep,
	year = {2015},
}

@article{law_testing_2017,
	title = {Testing and {Learning} on {Distributions} with {Symmetric} {Noise} {Invariance}},
	url = {http://arxiv.org/abs/1703.07596},
	abstract = {Kernel embeddings of distributions and the Maximum Mean Discrepancy (MMD), the resulting distance between distributions, are useful tools for fully nonparametric two-sample testing and learning on distributions. However, it is rarely that all possible differences between samples are of interest -- discovered differences can be due to different types of measurement noise, data collection artefacts or other irrelevant sources of variability. We propose distances between distributions which encode invariance to additive symmetric noise, aimed at testing whether the assumed true underlying processes differ. Moreover, we construct invariant features of distributions, leading to learning algorithms robust to the impairment of the input distributions with symmetric additive noise.},
	urldate = {2021-06-07},
	journal = {arXiv:1703.07596 [stat]},
	author = {Law, Ho Chung Leon and Yau, Christopher and Sejdinovic, Dino},
	month = nov,
	year = {2017},
	note = {arXiv: 1703.07596},
	keywords = {machine learning},
}

@article{gribonval_compressive_2020,
	title = {Compressive {Statistical} {Learning} with {Random} {Feature} {Moments}},
	url = {http://arxiv.org/abs/1706.07180},
	abstract = {We describe a general framework --compressive statistical learning-- for resource-efficient large-scale learning: the training collection is compressed in one pass into a low-dimensional sketch (a vector of random empirical generalized moments) that captures the information relevant to the considered learning task. A near-minimizer of the risk is computed from the sketch through the solution of a nonlinear least squares problem. We investigate sufficient sketch sizes to control the generalization error of this procedure. The framework is illustrated on compressive PCA, compressive clustering, and compressive Gaussian mixture Modeling with fixed known variance. The latter two are further developed in a companion paper.},
	urldate = {2021-06-07},
	journal = {arXiv:1706.07180 [cs, math, stat]},
	author = {Gribonval, Rémi and Blanchard, Gilles and Keriven, Nicolas and Traonmilin, Yann},
	month = apr,
	year = {2020},
	note = {arXiv: 1706.07180},
	keywords = {information theory, machine learning, statistics},
}

@inproceedings{law_bayesian_2018,
	title = {Bayesian {Approaches} to {Distribution} {Regression}},
	url = {http://proceedings.mlr.press/v84/law18a.html},
	abstract = {Distribution regression has recently attracted much interest as a generic solution to the problem of supervised learning where labels are available at the group level, rather than at the individual...},
	language = {en},
	urldate = {2021-06-07},
	booktitle = {International {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Law, Ho Chung Leon and Sutherland, Danica J. and Sejdinovic, Dino and Flaxman, Seth},
	month = mar,
	year = {2018},
	note = {ISSN: 2640-3498},
	pages = {1167--1176},
}

@article{liu_deep_2021,
	title = {Deep {Latent}-{Variable} {Kernel} {Learning}},
	issn = {2168-2275},
	doi = {10.1109/TCYB.2021.3062140},
	abstract = {Deep kernel learning (DKL) leverages the connection between the Gaussian process (GP) and neural networks (NNs) to build an end-to-end hybrid model. It combines the capability of NN to learn rich representations under massive data and the nonparametric property of GP to achieve automatic regularization that incorporates a tradeoff between model fit and model complexity. However, the deterministic NN encoder may weaken the model regularization of the following GP part, especially on small datasets, due to the free latent representation. We, therefore, present a complete deep latent-variable kernel learning (DLVKL) model wherein the latent variables perform stochastic encoding for regularized representation. We further enhance the DLVKL from two aspects: 1) the expressive variational posterior through neural stochastic differential equation (NSDE) to improve the approximation quality and 2) the hybrid prior taking knowledge from both the SDE prior and the posterior to arrive at a flexible tradeoff. Extensive experiments imply that DLVKL-NSDE performs similar to the well-calibrated GP on small datasets, and shows superiority on large datasets.},
	journal = {IEEE Transactions on Cybernetics},
	author = {Liu, Haitao and Ong, Yew-Soon and Jiang, Xiaomo and Wang, Xiaofang},
	year = {2021},
	note = {Conference Name: IEEE Transactions on Cybernetics},
	keywords = {complexity, gaussian processes, kernel methods, neural networks, neural sde, regularization},
	pages = {1--14},
}

@inproceedings{zhen_learning_2020,
	title = {Learning to {Learn} {Kernels} with {Variational} {Random} {Features}},
	url = {http://proceedings.mlr.press/v119/zhen20a.html},
	abstract = {We introduce kernels with random Fourier features in the meta-learning framework for few-shot learning. We propose meta variational random features (MetaVRF) to learn adaptive kernels for the base-...},
	language = {en},
	urldate = {2021-06-07},
	booktitle = {International {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Zhen, Xiantong and Sun, Haoliang and Du, Yingjun and Xu, Jun and Yin, Yilong and Shao, Ling and Snoek, Cees},
	month = nov,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {11409--11419},
}

@article{wang_kernel_nodate,
	title = {Kernel {Learning} via {Random} {Fourier} {Representations}},
	abstract = {Random Fourier Features (RFF) is a kernel approximation method that works well for large datasets, however It makes certain assumptions about the choice of the kernel and so here in this report, we present and validate an alternative method based on ﬁtting neural networks. Our alternative method consists of training the neural network with one hidden layer- approximating the frequencies from Fourier expansion of some translation invariant kernel in a data dependent fashion. The predictive accuracy is comparable to the one from the RFF algorithm, achieving however a lower variance. We then applied a similar idea of ﬁtting neural networks using this time two hidden layers and implementing mean pooling to the two-stage sampled learning. We achieved again similar predictive accuracy, but with reduced variance. However, both methods involve signiﬁcant computational cost. Furthermore, we have explored various optimisation methods and tested them in the neural networks.},
	language = {en},
	author = {Wang, Andi and Law, Leon},
	pages = {10},
}

@article{li_fourier_2021,
	title = {Fourier {Neural} {Operator} for {Parametric} {Partial} {Differential} {Equations}},
	url = {http://arxiv.org/abs/2010.08895},
	abstract = {The classical development of neural networks has primarily focused on learning mappings between finite-dimensional Euclidean spaces. Recently, this has been generalized to neural operators that learn mappings between function spaces. For partial differential equations (PDEs), neural operators directly learn the mapping from any functional parametric dependence to the solution. Thus, they learn an entire family of PDEs, in contrast to classical methods which solve one instance of the equation. In this work, we formulate a new neural operator by parameterizing the integral kernel directly in Fourier space, allowing for an expressive and efficient architecture. We perform experiments on Burgers' equation, Darcy flow, and Navier-Stokes equation. The Fourier neural operator is the first ML-based method to successfully model turbulent flows with zero-shot super-resolution. It is up to three orders of magnitude faster compared to traditional PDE solvers. Additionally, it achieves superior accuracy compared to previous learning-based solvers under fixed resolution.},
	urldate = {2021-06-04},
	journal = {arXiv:2010.08895 [cs, math]},
	author = {Li, Zongyi and Kovachki, Nikola and Azizzadenesheli, Kamyar and Liu, Burigede and Bhattacharya, Kaushik and Stuart, Andrew and Anandkumar, Anima},
	month = may,
	year = {2021},
	note = {arXiv: 2010.08895},
	keywords = {machine learning, numerical modelling},
}

@inproceedings{grunewalder_conditional_2012,
	address = {Madison, WI, USA},
	series = {{ICML}'12},
	title = {Conditional mean embeddings as regressors},
	isbn = {978-1-4503-1285-1},
	abstract = {We demonstrate an equivalence between reproducing kernel Hilbert space (RKHS) embeddings of conditional distributions and vector-valued regressors. This connection introduces a natural regularized loss function which the RKHS embeddings minimise, providing an intuitive understanding of the embeddings and a justification for their use. Furthermore, the equivalence allows the application of vector-valued regression methods and results to the problem of learning conditional distributions. Using this link we derive a sparse version of the embedding by considering alternative formulations. Further, by applying convergence results for vector-valued regression to the embedding problem we derive minimax convergence rates which are O(log(n)/n) - compared to current state of the art rates of O(n-1/4) - and are valid under milder and more intuitive assumptions. These minimax upper rates coincide with lower rates up to a logarithmic factor, showing that the embedding method achieves nearly optimal rates. We study our sparse embedding algorithm in a reinforcement learning task where the algorithm shows significant improvement in sparsity over an incomplete Cholesky decomposition.},
	urldate = {2021-06-04},
	booktitle = {Proceedings of the 29th {International} {Coference} on {International} {Conference} on {Machine} {Learning}},
	publisher = {Omnipress},
	author = {Grünewälder, Steffen and Lever, Guy and Baldassarre, Luca and Patterson, Sam and Gretton, Arthur and Pontil, Massimilano},
	month = jun,
	year = {2012},
	pages = {1803--1810},
}

@article{zhu_new_2020,
	title = {A {New} {Distribution}-{Free} {Concept} for {Representing}, {Comparing}, and {Propagating} {Uncertainty} in {Dynamical} {Systems} with {Kernel} {Probabilistic} {Programming}},
	url = {http://arxiv.org/abs/1911.11082},
	abstract = {This work presents the concept of kernel mean embedding and kernel probabilistic programming in the context of stochastic systems. We propose formulations to represent, compare, and propagate uncertainties for fairly general stochastic dynamics in a distribution-free manner. The new tools enjoy sound theory rooted in functional analysis and wide applicability as demonstrated in distinct numerical examples. The implication of this new concept is a new mode of thinking about the statistical nature of uncertainty in dynamical systems.},
	urldate = {2021-06-04},
	journal = {arXiv:1911.11082 [cs, eess, math, stat]},
	author = {Zhu, Jia-Jie and Muandet, Krikamol and Diehl, Moritz and Schölkopf, Bernhard},
	month = may,
	year = {2020},
	note = {arXiv: 1911.11082},
	keywords = {control, machine learning},
}

@article{lee_linking_2019,
	title = {Linking {Gaussian} process regression with data-driven manifold embeddings for nonlinear data fusion},
	volume = {9},
	url = {https://royalsocietypublishing.org/doi/10.1098/rsfs.2018.0083},
	doi = {10.1098/rsfs.2018.0083},
	abstract = {In statistical modelling with Gaussian process regression, it has been shown that combining (few) high-fidelity data with (many) low-fidelity data can enhance prediction accuracy, compared to prediction based on the few high-fidelity data only. Such information fusion techniques for multi-fidelity data commonly approach the high-fidelity model fh(t) as a function of two variables (t, s), and then use fl(t) as the s data. More generally, the high-fidelity model can be written as a function of several variables (t, s1, s2….); the low-fidelity model fl and, say, some of its derivatives can then be substituted for these variables. In this paper, we will explore mathematical algorithms for multi-fidelity information fusion that use such an approach towards improving the representation of the high-fidelity function with only a few training data points. Given that fh may not be a simple function—and sometimes not even a function—of fl, we demonstrate that using additional functions of t, such as derivatives or shifts of fl, can drastically improve the approximation of fh through Gaussian processes. We also point out a connection with ‘embedology’ techniques from topology and dynamical systems. Our illustrative examples range from instructive caricatures to computational biology models, such as Hodgkin–Huxley neural oscillations.},
	number = {3},
	urldate = {2021-06-04},
	journal = {Interface Focus},
	author = {Lee, Seungjoon and Dietrich, Felix and Karniadakis, George E. and Kevrekidis, Ioannis G.},
	month = jun,
	year = {2019},
	note = {Publisher: Royal Society},
	pages = {20180083},
}

@inproceedings{pearce_bayesian_2017,
	title = {Bayesian simulation optimization with input uncertainty},
	doi = {10.1109/WSC.2017.8247958},
	abstract = {We consider simulation optimization in the presence of input uncertainty. In particular, we assume that the input distribution can be described by some continuous parameters, and that we have some prior knowledge defining the probability distribution for these parameters. We then seek the simulation design that has the best expected performance over the possible parameters of the input distributions. Assuming correlation of performance between solutions and also between input distributions, we propose modifications of two well-known simulation optimization algorithms, Efficient Global Optimization and Knowledge Gradient with Continuous Parameters, so that they work efficiently under input uncertainty.},
	booktitle = {2017 {Winter} {Simulation} {Conference} ({WSC})},
	author = {Pearce, Michael and Branke, Juergen},
	month = dec,
	year = {2017},
	note = {ISSN: 1558-4305},
	keywords = {gaussian processes, optimization, uncertainty estimation},
	pages = {2268--2278},
}

@article{sklar_fonctions_1959,
	title = {Fonctions de repartition a n dimensions et leurs marges},
	language = {en},
	number = {8},
	urldate = {2021-05-30},
	journal = {Publications de l’Institut Statistique de l’Université de Paris},
	author = {Sklar, M.},
	year = {1959},
	pages = {229--231},
}

@article{liu_when_2019,
	title = {When {Gaussian} {Process} {Meets} {Big} {Data}: {A} {Review} of {Scalable} {GPs}},
	shorttitle = {When {Gaussian} {Process} {Meets} {Big} {Data}},
	url = {http://arxiv.org/abs/1807.01065},
	abstract = {The vast quantity of information brought by big data as well as the evolving computer hardware encourages success stories in the machine learning community. In the meanwhile, it poses challenges for the Gaussian process (GP) regression, a well-known non-parametric and interpretable Bayesian model, which suffers from cubic complexity to data size. To improve the scalability while retaining desirable prediction quality, a variety of scalable GPs have been presented. But they have not yet been comprehensively reviewed and analyzed in order to be well understood by both academia and industry. The review of scalable GPs in the GP community is timely and important due to the explosion of data size. To this end, this paper is devoted to the review on state-of-the-art scalable GPs involving two main categories: global approximations which distillate the entire data and local approximations which divide the data for subspace learning. Particularly, for global approximations, we mainly focus on sparse approximations comprising prior approximations which modify the prior but perform exact inference, posterior approximations which retain exact prior but perform approximate inference, and structured sparse approximations which exploit specific structures in kernel matrix; for local approximations, we highlight the mixture/product of experts that conducts model averaging from multiple local experts to boost predictions. To present a complete review, recent advances for improving the scalability and capability of scalable GPs are reviewed. Finally, the extensions and open issues regarding the implementation of scalable GPs in various scenarios are reviewed and discussed to inspire novel ideas for future research avenues.},
	urldate = {2021-05-30},
	journal = {arXiv:1807.01065 [cs, stat]},
	author = {Liu, Haitao and Ong, Yew-Soon and Shen, Xiaobo and Cai, Jianfei},
	month = apr,
	year = {2019},
	note = {arXiv: 1807.01065},
	keywords = {machine learning},
}

@article{karniadakis_physics-informed_2021,
	title = {Physics-informed machine learning},
	copyright = {2021 Springer Nature Limited},
	issn = {2522-5820},
	url = {https://www.nature.com/articles/s42254-021-00314-5},
	doi = {10.1038/s42254-021-00314-5},
	abstract = {Despite great progress in simulating multiphysics problems using the numerical discretization of partial differential equations (PDEs), one still cannot seamlessly incorporate noisy data into existing algorithms, mesh generation remains complex, and high-dimensional problems governed by parameterized PDEs cannot be tackled. Moreover, solving inverse problems with hidden physics is often prohibitively expensive and requires different formulations and elaborate computer codes. Machine learning has emerged as a promising alternative, but training deep neural networks requires big data, not always available for scientific problems. Instead, such networks can be trained from additional information obtained by enforcing the physical laws (for example, at random points in the continuous space-time domain). Such physics-informed learning integrates (noisy) data and mathematical models, and implements them through neural networks or other kernel-based regression networks. Moreover, it may be possible to design specialized network architectures that automatically satisfy some of the physical invariants for better accuracy, faster training and improved generalization. Here, we review some of the prevailing trends in embedding physics into machine learning, present some of the current capabilities and limitations and discuss diverse applications of physics-informed learning both for forward and inverse problems, including discovering hidden physics and tackling high-dimensional problems.},
	language = {en},
	urldate = {2021-05-30},
	journal = {Nature Reviews Physics},
	author = {Karniadakis, George Em and Kevrekidis, Ioannis G. and Lu, Lu and Perdikaris, Paris and Wang, Sifan and Yang, Liu},
	month = may,
	year = {2021},
	note = {Publisher: Nature Publishing Group},
	pages = {1--19},
}

@inproceedings{wan_unscented_2000,
	title = {The unscented {Kalman} filter for nonlinear estimation},
	doi = {10.1109/ASSPCC.2000.882463},
	abstract = {This paper points out the flaws in using the extended Kalman filter (EKE) and introduces an improvement, the unscented Kalman filter (UKF), proposed by Julier and Uhlman (1997). A central and vital operation performed in the Kalman filter is the propagation of a Gaussian random variable (GRV) through the system dynamics. In the EKF the state distribution is approximated by a GRV, which is then propagated analytically through the first-order linearization of the nonlinear system. This can introduce large errors in the true posterior mean and covariance of the transformed GRV, which may lead to sub-optimal performance and sometimes divergence of the filter. The UKF addresses this problem by using a deterministic sampling approach. The state distribution is again approximated by a GRV, but is now represented using a minimal set of carefully chosen sample points. These sample points completely capture the true mean and covariance of the GRV, and when propagated through the true nonlinear system, captures the posterior mean and covariance accurately to the 3rd order (Taylor series expansion) for any nonlinearity. The EKF in contrast, only achieves first-order accuracy. Remarkably, the computational complexity of the UKF is the same order as that of the EKF. Julier and Uhlman demonstrated the substantial performance gains of the UKF in the context of state-estimation for nonlinear control. Machine learning problems were not considered. We extend the use of the UKF to a broader class of nonlinear estimation problems, including nonlinear system identification, training of neural networks, and dual estimation problems. In this paper, the algorithms are further developed and illustrated with a number of additional examples.},
	booktitle = {Proceedings of the {IEEE} 2000 {Adaptive} {Systems} for {Signal} {Processing}, {Communications}, and {Control} {Symposium} ({Cat}. {No}.{00EX373})},
	author = {Wan, E.A. and Van Der Merwe, R.},
	month = oct,
	year = {2000},
	keywords = {complexity, machine learning, neural networks, nonlinear dynamical systems, sampling methods},
	pages = {153--158},
}

@article{kalman_new_1960,
	title = {A {New} {Approach} to {Linear} {Filtering} and {Prediction} {Problems}},
	volume = {82},
	issn = {0021-9223},
	url = {https://doi.org/10.1115/1.3662552},
	doi = {10.1115/1.3662552},
	abstract = {The classical filtering and prediction problem is re-examined using the Bode-Shannon representation of random processes and the “state-transition” method of analysis of dynamic systems. New results are: (1) The formulation and methods of solution of the problem apply without modification to stationary and nonstationary statistics and to growing-memory and infinite-memory filters. (2) A nonlinear difference (or differential) equation is derived for the covariance matrix of the optimal estimation error. From the solution of this equation the co-efficients of the difference (or differential) equation of the optimal linear filter are obtained without further calculations. (3) The filtering problem is shown to be the dual of the noise-free regulator problem. The new method developed here is applied to two well-known problems, confirming and extending earlier results. The discussion is largely self-contained and proceeds from first principles; basic concepts of the theory of random processes are reviewed in the Appendix.},
	number = {1},
	urldate = {2021-05-30},
	journal = {Journal of Basic Engineering},
	author = {Kalman, R. E.},
	month = mar,
	year = {1960},
	pages = {35--45},
}

@inproceedings{garnelo_conditional_2018,
	title = {Conditional {Neural} {Processes}},
	url = {http://proceedings.mlr.press/v80/garnelo18a.html},
	abstract = {Deep neural networks excel at function approximation, yet they are typically trained from scratch for each new function. On the other hand, Bayesian methods, such as Gaussian Processes (GPs), explo...},
	language = {en},
	urldate = {2021-05-30},
	booktitle = {International {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Garnelo, Marta and Rosenbaum, Dan and Maddison, Christopher and Ramalho, Tiago and Saxton, David and Shanahan, Murray and Teh, Yee Whye and Rezende, Danilo and Eslami, S. M. Ali},
	month = jul,
	year = {2018},
	note = {ISSN: 2640-3498},
	pages = {1704--1713},
}

@article{garnelo_neural_2018,
	title = {Neural {Processes}},
	url = {http://arxiv.org/abs/1807.01622},
	abstract = {A neural network (NN) is a parameterised function that can be tuned via gradient descent to approximate a labelled collection of data with high precision. A Gaussian process (GP), on the other hand, is a probabilistic model that defines a distribution over possible functions, and is updated in light of data via the rules of probabilistic inference. GPs are probabilistic, data-efficient and flexible, however they are also computationally intensive and thus limited in their applicability. We introduce a class of neural latent variable models which we call Neural Processes (NPs), combining the best of both worlds. Like GPs, NPs define distributions over functions, are capable of rapid adaptation to new observations, and can estimate the uncertainty in their predictions. Like NNs, NPs are computationally efficient during training and evaluation but also learn to adapt their priors to data. We demonstrate the performance of NPs on a range of learning tasks, including regression and optimisation, and compare and contrast with related models in the literature.},
	urldate = {2021-05-30},
	journal = {arXiv:1807.01622 [cs, stat]},
	author = {Garnelo, Marta and Schwarz, Jonathan and Rosenbaum, Dan and Viola, Fabio and Rezende, Danilo J. and Eslami, S. M. Ali and Teh, Yee Whye},
	month = jul,
	year = {2018},
	note = {arXiv: 1807.01622},
	keywords = {machine learning},
}

@article{chen_neural_2019,
	title = {Neural {Ordinary} {Differential} {Equations}},
	url = {http://arxiv.org/abs/1806.07366},
	abstract = {We introduce a new family of deep neural network models. Instead of specifying a discrete sequence of hidden layers, we parameterize the derivative of the hidden state using a neural network. The output of the network is computed using a black-box differential equation solver. These continuous-depth models have constant memory cost, adapt their evaluation strategy to each input, and can explicitly trade numerical precision for speed. We demonstrate these properties in continuous-depth residual networks and continuous-time latent variable models. We also construct continuous normalizing flows, a generative model that can train by maximum likelihood, without partitioning or ordering the data dimensions. For training, we show how to scalably backpropagate through any ODE solver, without access to its internal operations. This allows end-to-end training of ODEs within larger models.},
	urldate = {2021-05-30},
	journal = {arXiv:1806.07366 [cs, stat]},
	author = {Chen, Ricky T. Q. and Rubanova, Yulia and Bettencourt, Jesse and Duvenaud, David},
	month = dec,
	year = {2019},
	note = {arXiv: 1806.07366},
	keywords = {machine learning},
}

@article{liu_neural_2019,
	title = {Neural {SDE}: {Stabilizing} {Neural} {ODE} {Networks} with {Stochastic} {Noise}},
	shorttitle = {Neural {SDE}},
	url = {http://arxiv.org/abs/1906.02355},
	abstract = {Neural Ordinary Differential Equation (Neural ODE) has been proposed as a continuous approximation to the ResNet architecture. Some commonly used regularization mechanisms in discrete neural networks (e.g. dropout, Gaussian noise) are missing in current Neural ODE networks. In this paper, we propose a new continuous neural network framework called Neural Stochastic Differential Equation (Neural SDE) network, which naturally incorporates various commonly used regularization mechanisms based on random noise injection. Our framework can model various types of noise injection frequently used in discrete networks for regularization purpose, such as dropout and additive/multiplicative noise in each block. We provide theoretical analysis explaining the improved robustness of Neural SDE models against input perturbations/adversarial attacks. Furthermore, we demonstrate that the Neural SDE network can achieve better generalization than the Neural ODE and is more resistant to adversarial and non-adversarial input perturbations.},
	urldate = {2021-05-30},
	journal = {arXiv:1906.02355 [cs, stat]},
	author = {Liu, Xuanqing and Xiao, Tesi and Si, Si and Cao, Qin and Kumar, Sanjiv and Hsieh, Cho-Jui},
	month = jun,
	year = {2019},
	note = {arXiv: 1906.02355},
	keywords = {computer vision, machine learning},
}

@article{kingma_auto-encoding_2014,
	title = {Auto-{Encoding} {Variational} {Bayes}},
	url = {http://arxiv.org/abs/1312.6114},
	abstract = {How can we perform efficient inference and learning in directed probabilistic models, in the presence of continuous latent variables with intractable posterior distributions, and large datasets? We introduce a stochastic variational inference and learning algorithm that scales to large datasets and, under some mild differentiability conditions, even works in the intractable case. Our contributions is two-fold. First, we show that a reparameterization of the variational lower bound yields a lower bound estimator that can be straightforwardly optimized using standard stochastic gradient methods. Second, we show that for i.i.d. datasets with continuous latent variables per datapoint, posterior inference can be made especially efficient by fitting an approximate inference model (also called a recognition model) to the intractable posterior using the proposed lower bound estimator. Theoretical advantages are reflected in experimental results.},
	urldate = {2021-05-30},
	journal = {arXiv:1312.6114 [cs, stat]},
	author = {Kingma, Diederik P. and Welling, Max},
	month = may,
	year = {2014},
	note = {arXiv: 1312.6114},
	keywords = {machine learning},
}

@inproceedings{damianou_deep_2013,
	title = {Deep {Gaussian} {Processes}},
	url = {http://proceedings.mlr.press/v31/damianou13a.html},
	abstract = {In this paper we introduce deep Gaussian process (GP) models. Deep GPs are a deep belief network based on Gaussian process mappings. The data is modeled as the output of a multivariate GP. The inpu...},
	language = {en},
	urldate = {2021-05-30},
	booktitle = {Artificial {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Damianou, Andreas and Lawrence, Neil D.},
	month = apr,
	year = {2013},
	note = {ISSN: 1938-7228},
	pages = {207--215},
}

@article{raissi_machine_2017,
	title = {Machine learning of linear differential equations using {Gaussian} processes},
	volume = {348},
	issn = {0021-9991},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999117305582},
	doi = {10.1016/j.jcp.2017.07.050},
	abstract = {This work leverages recent advances in probabilistic machine learning to discover governing equations expressed by parametric linear operators. Such equations involve, but are not limited to, ordinary and partial differential, integro-differential, and fractional order operators. Here, Gaussian process priors are modified according to the particular form of such operators and are employed to infer parameters of the linear equations from scarce and possibly noisy observations. Such observations may come from experiments or “black-box” computer simulations, as demonstrated in several synthetic examples and a realistic application in functional genomics.},
	language = {en},
	urldate = {2021-05-30},
	journal = {Journal of Computational Physics},
	author = {Raissi, Maziar and Perdikaris, Paris and Karniadakis, George Em},
	month = nov,
	year = {2017},
	keywords = {inverse problems, machine learning, uncertainty estimation},
	pages = {683--693},
}

@book{leamer_specification_1978,
	address = {New York},
	series = {Wiley series in probability and mathematical statistics},
	title = {Specification searches: ad hoc inference with nonexperimental data},
	isbn = {978-0-471-01520-8},
	shorttitle = {Specification searches},
	publisher = {Wiley},
	author = {Leamer, Edward E.},
	year = {1978},
	keywords = {statistics},
}

@article{popov_multifidelity_2020,
	title = {A {Multifidelity} {Ensemble} {Kalman} {Filter} with {Reduced} {Order} {Control} {Variates}},
	url = {http://arxiv.org/abs/2007.00793},
	abstract = {This work develops a new multifidelity ensemble Kalman filter (MFEnKF) algorithm based on linear control variate framework. The approach allows for rigorous multifidelity extensions of the EnKF, where the uncertainty in coarser fidelities in the hierarchy of models represent control variates for the uncertainty in finer fidelities. Small ensembles of high fidelity model runs are complemented by larger ensembles of cheaper, lower fidelity runs, to obtain much improved analyses at only small additional computational costs. We investigate the use of reduced order models as coarse fidelity control variates in the MFEnKF, and provide analyses to quantify the improvements over the traditional ensemble Kalman filters. We apply these ideas to perform data assimilation with a quasi-geostrophic test problem, using direct numerical simulation and a corresponding POD-Galerkin reduced order model. Numerical results show that the two-fidelity MFEnKF provides better analyses than existing EnKF algorithms at comparable or reduced computational costs.},
	urldate = {2021-05-29},
	journal = {arXiv:2007.00793 [cs, math]},
	author = {Popov, Andrey A. and Mou, Changhong and Iliescu, Traian and Sandu, Adrian},
	month = jul,
	year = {2020},
	note = {arXiv: 2007.00793},
	keywords = {control, numerical modelling},
}

@book{welch_introduction_1995,
	title = {An {Introduction} to the {Kalman} {Filter}},
	abstract = {In 1960, R.E. Kalman published his famous paper describing a recursive solution to the discrete-data linear filtering problem. Since that time, due in large part to advances in digital computing, the Kalman filter has been the subject of extensive research and application, particularly in the area of autonomous or assisted navigation.

The Kalman filter is a set of mathematical equations that provides an efficient computational (recursive) means to estimate the state of a process, in a way that minimizes the mean of the squared error. The filter is very powerful in several aspects: it supports estimations of past, present, and even future states, and it can do so even when the precise nature of the modeled system is unknown.

The purpose of this paper is to provide a practical introduction to the discrete Kalman filter. This introduction includes a description and some discussion of the basic discrete Kalman filter, a derivation, description and some discussion of the extended Kalman filter, and a relatively simple (tangible) example with real numbers \& results.},
	author = {Welch, Greg and Bishop, Gary},
	year = {1995},
}

@inproceedings{hebbal_multi-fidelity_2019,
	title = {Multi-fidelity modeling using {DGPs}: {Improvements} and a generalization to varying input space dimensions},
	shorttitle = {Multi-fidelity modeling using {DGPs}},
	url = {https://hal.inria.fr/hal-02919781},
	abstract = {Multi-fidelity approaches improve the inference of a high-fidelity model which is constructed using a small set of accurate observations, by taking advantage of its correlations with a low-fidelity model built using a larger set of approximated data. Most existing multi-fidelity methods consider the inputs of the low and high fidelity models defined identically over the same input space. However, it happens that the low fidelity model variables are defined over a different space than the variables of the high fidelity model due to different modeling approaches i.e. input spaces with different dimensionality and different nature of the variables. Recently, Deep Gaussian Processes have been used to exhibit the correlations between the low and high fidelity models. In this paper, Deep Gaussian Processes for multi-fidelity (MF-DGP) are extended to the case where the input spaces of the low and high fidelity models are different. Moreover, the learning capacity of MF-DGP is improved by proposing an optimization approach for the inducing inputs and by using natural gradients for the variational distributions of the inducing variables which also allows time reduction in the training.},
	language = {en},
	urldate = {2021-05-28},
	author = {Hebbal, Ali and Brevault, Loïc and Balesdent, Mathieu and Talbi, El-Ghazali and Melab, Nouredine},
	month = dec,
	year = {2019},
}

@article{kroese_why_2014,
	title = {Why the {Monte} {Carlo} method is so important today},
	volume = {6},
	copyright = {© 2014 Wiley Periodicals, Inc.},
	issn = {1939-0068},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/wics.1314},
	doi = {https://doi.org/10.1002/wics.1314},
	abstract = {Since the beginning of electronic computing, people have been interested in carrying out random experiments on a computer. Such Monte Carlo techniques are now an essential ingredient in many quantitative investigations. Why is the Monte Carlo method (MCM) so important today? This article explores the reasons why the MCM has evolved from a ‘last resort’ solution to a leading methodology that permeates much of contemporary science, finance, and engineering. WIREs Comput Stat 2014, 6:386–392. doi: 10.1002/wics.1314 This article is categorized under: Statistical and Graphical Methods of Data Analysis {\textgreater} Markov Chain Monte Carlo (MCMC) Statistical Models {\textgreater} Simulation Models},
	language = {en},
	number = {6},
	urldate = {2021-05-28},
	journal = {WIREs Computational Statistics},
	author = {Kroese, Dirk P. and Brereton, Tim and Taimre, Thomas and Botev, Zdravko I.},
	year = {2014},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/wics.1314},
	keywords = {mcmc, monte carlo, simulation},
	pages = {386--392},
}

@article{zhang_modern_2020,
	title = {Modern {Monte} {Carlo} {Methods} for {Efficient} {Uncertainty} {Quantification} and {Propagation}: {A} {Survey}},
	shorttitle = {Modern {Monte} {Carlo} {Methods} for {Efficient} {Uncertainty} {Quantification} and {Propagation}},
	url = {http://arxiv.org/abs/2011.00680},
	abstract = {Uncertainty quantification (UQ) includes the characterization, integration, and propagation of uncertainties that result from stochastic variations and a lack of knowledge or data in the natural world. Monte Carlo (MC) method is a sampling-based approach that has widely used for quantification and propagation of uncertainties. However, the standard MC method is often time-consuming if the simulation-based model is computationally intensive. This article gives an overview of modern MC methods to address the existing challenges of the standard MC in the context of UQ. Specifically, multilevel Monte Carlo (MLMC) extending the concept of control variates achieves a significant reduction of the computational cost by performing most evaluations with low accuracy and corresponding low cost, and relatively few evaluations at high accuracy and correspondingly high cost. Multifidelity Monte Carlo (MFMC) accelerates the convergence of standard Monte Carlo by generalizing the control variates with different models having varying fidelities and varying computational costs. Multimodel Monte Carlo method (MMMC), having a different setting of MLMC and MFMC, aims to address the issue of uncertainty quantification and propagation when data for characterizing probability distributions are limited. Multimodel inference combined with importance sampling is proposed for quantifying and efficiently propagating the uncertainties resulting from small datasets. All of these three modern MC methods achieve a significant improvement of computational efficiency for probabilistic UQ, particularly uncertainty propagation. An algorithm summary and the corresponding code implementation are provided for each of the modern Monte Carlo methods. The extension and application of these methods are discussed in detail.},
	urldate = {2021-05-28},
	journal = {arXiv:2011.00680 [stat]},
	author = {Zhang, Jiaxin},
	month = nov,
	year = {2020},
	note = {arXiv: 2011.00680},
	keywords = {statistics},
}

@article{gander_multigrid_2018,
	title = {Multigrid interpretations of the parareal algorithm leading to an overlapping variant and {MGRIT}},
	volume = {19},
	issn = {1433-0369},
	url = {https://doi.org/10.1007/s00791-018-0297-y},
	doi = {10.1007/s00791-018-0297-y},
	abstract = {The parareal algorithm is by construction a two level method, and there are several ways to interpret the parareal algorithm to obtain multilevel versions. We first review the three main interpretations of the parareal algorithm as a two-level method, a direct one, one based on geometric multigrid and one based on algebraic multigrid. The algebraic multigrid interpretation leads to the MGRIT algorithm, when using instead of only an F-smoother, a so called \$\${\textbackslash}textit\{FCF\}\$\$-smoother. We show that this can be interpreted at the level of the parareal algorithm as generous overlap in time. This allows us to generalize the method to even larger overlap, corresponding in MGRIT to \$\$F({\textbackslash}textit\{CF\}){\textasciicircum}\{{\textbackslash}nu \}\$\$-smoothing, \$\${\textbackslash}nu {\textbackslash}ge 1\$\$, and we prove two new convergence results for such algorithms in the fully non-linear setting: convergence in a finite number of steps, becoming smaller when \$\${\textbackslash}nu \$\$increases, and a general superlinear convergence estimate for this generalized version of MGRIT. We illustrate our results with numerical experiments, both for linear and non-linear systems of ordinary and partial differential equations. Our results show that overlap only sometimes leads to faster algorithms.},
	language = {en},
	number = {3},
	urldate = {2021-05-26},
	journal = {Computing and Visualization in Science},
	author = {Gander, Martin J. and Kwok, Felix and Zhang, Hui},
	month = jul,
	year = {2018},
	pages = {59--74},
}

@article{romor_multi-fidelity_2020,
	title = {Multi-fidelity data fusion for the approximation of scalar functions with low intrinsic dimensionality through active subspaces},
	url = {http://arxiv.org/abs/2010.08349},
	abstract = {Gaussian processes are employed for non-parametric regression in a Bayesian setting. They generalize linear regression, embedding the inputs in a latent manifold inside an infinite-dimensional reproducing kernel Hilbert space. We can augment the inputs with the observations of low-fidelity models in order to learn a more expressive latent manifold and thus increment the model's accuracy. This can be realized recursively with a chain of Gaussian processes with incrementally higher fidelity. We would like to extend these multi-fidelity model realizations to case studies affected by a high-dimensional input space but with low intrinsic dimensionality. In this cases physical supported or purely numerical low-order models are still affected by the curse of dimensionality when queried for responses. When the model's gradient information is provided, the presence of an active subspace can be exploited to design low-fidelity response surfaces and thus enable Gaussian process multi-fidelity regression, without the need to perform new simulations. This is particularly useful in the case of data scarcity. In this work we present a multi-fidelity approach involving active subspaces and we test it on two different high-dimensional benchmarks.},
	urldate = {2021-05-26},
	journal = {arXiv:2010.08349 [cs, math, stat]},
	author = {Romor, Francesco and Tezzele, Marco and Rozza, Gianluigi},
	month = oct,
	year = {2020},
	note = {arXiv: 2010.08349},
	keywords = {machine learning, numerical modelling},
}

@article{castanedo_review_2013,
	title = {A {Review} of {Data} {Fusion} {Techniques}},
	volume = {2013},
	issn = {2356-6140},
	url = {https://www.hindawi.com/journals/tswj/2013/704504/},
	doi = {10.1155/2013/704504},
	abstract = {The integration of data and knowledge from several sources is known as data fusion. This paper summarizes the state of the data fusion field and describes the most relevant studies. We first enumerate and explain different classification schemes for data fusion. Then, the most common algorithms are reviewed. These methods and algorithms are presented using three different categories: (i) data association, (ii) state estimation, and (iii) decision fusion.},
	language = {en},
	urldate = {2021-05-26},
	journal = {The Scientific World Journal},
	author = {Castanedo, Federico},
	month = oct,
	year = {2013},
	note = {Publisher: Hindawi},
	pages = {e704504},
}

@article{acheson_machine_2020,
	title = {Machine learning for cross-gazetteer matching of natural features},
	volume = {34},
	issn = {1365-8816},
	url = {https://doi.org/10.1080/13658816.2019.1599123},
	doi = {10.1080/13658816.2019.1599123},
	abstract = {Defining and identifying duplicate records in a dataset is a challenging task which grows more complex when the modeled entities themselves are hard to delineate. In the geospatial domain, it may not be clear where a mountain, stream, or valley ends and begins, a problem carried over when such entities are catalogued in gazetteers. In this paper, we take two gazetteers, GeoNames and SwissNames3D, and perform matching – identifying records in each that are about the same entity – across a sample of natural feature records. We first perform rule-based matching, establishing competitive results, then apply machine learning using Random Forests, a method well-suited to the matching task. We report on the performance of a wider array of matching features than has been previously studied, including domain-specific ones such as feature type, land cover class, and elevation. Our results show an increase in performance using machine learning over rules, with a notable performance gain from considering feature types, but negligible gains from other specialized matching features. We argue that future work in this area should strive to be more reproducible and report results on a realistic testing pipeline including candidate selection, feature extraction, and classification.},
	number = {4},
	urldate = {2021-05-26},
	journal = {International Journal of Geographical Information Science},
	author = {Acheson, Elise and Volpi, Michele and Purves, Ross S.},
	month = apr,
	year = {2020},
	note = {Publisher: Taylor \& Francis
\_eprint: https://doi.org/10.1080/13658816.2019.1599123},
	keywords = {random forests},
	pages = {708--734},
}

@article{perdikaris_data-driven_nodate,
	title = {Data-{Driven} {Parallel} {Scientiﬁc} {Computing}: {Multi}-{Fidelity} {Information} {Fusion} {Algorithms} and {Applications} to {Physical} and {Biological} {Systems}},
	language = {en},
	author = {Perdikaris, Paris},
	pages = {257},
}

@article{saalfeld_conflation_2007,
	title = {Conflation {Automated} map compilation},
	copyright = {Copyright Taylor and Francis Group, LLC},
	url = {https://www.tandfonline.com/doi/abs/10.1080/02693798808927897},
	doi = {10.1080/02693798808927897},
	abstract = {Map compilation, or conflation, is now being accomplished by computer. Interactive routines manipulate the graphic images of two different digital maps of the same region in order to permit map sim...},
	language = {en},
	urldate = {2021-05-26},
	journal = {International Journal of Geographical Information System},
	author = {SAALFELD, ALAN},
	month = apr,
	year = {2007},
	note = {Publisher: Taylor \& Francis Group},
}

@article{ruiz_digital_2011,
	title = {Digital map conflation: a review of the process and a proposal for classification},
	volume = {25},
	issn = {1365-8816},
	shorttitle = {Digital map conflation},
	url = {https://doi.org/10.1080/13658816.2010.519707},
	doi = {10.1080/13658816.2010.519707},
	abstract = {This article is centred on analysing the state of the art of the conflation processes applied to geospatial databases (GDBs) from heterogeneous sources. The term conflation is used to describe the procedure for the integration of these different data, and conflation methods play an important role in systems for updating GDBs, derivation of new cartographic products, densification of digital elevation models, automatic features extraction and so on. In this article we define extensively each conflation process, its evaluation measures and its main application problems and present a classification of all conflation processes. Finally, we introduce a bibliography which the reader may find useful to further explore the field. It tries to serve as a starting point and direct the reader to characteristic research in this area.},
	number = {9},
	urldate = {2021-05-26},
	journal = {International Journal of Geographical Information Science},
	author = {Ruiz, Juan J. and Ariza, F. Javier and Ureña, Manuel A. and Blázquez, Elidia B.},
	month = sep,
	year = {2011},
	note = {Publisher: Taylor \& Francis
\_eprint: https://doi.org/10.1080/13658816.2010.519707},
	keywords = {data fusion},
	pages = {1439--1466},
}

@article{perdikaris_multi-fidelity_2015,
	title = {Multi-fidelity modelling via recursive co-kriging and {Gaussian}–{Markov} random fields},
	volume = {471},
	url = {https://royalsocietypublishing.org/doi/full/10.1098/rspa.2015.0018},
	doi = {10.1098/rspa.2015.0018},
	abstract = {We propose a new framework for design under uncertainty based on stochastic computer simulations and multi-level recursive co-kriging. The proposed methodology simultaneously takes into account multi-fidelity in models, such as direct numerical simulations versus empirical formulae, as well as multi-fidelity in the probability space (e.g. sparse grids versus tensor product multi-element probabilistic collocation). We are able to construct response surfaces of complex dynamical systems by blending multiple information sources via auto-regressive stochastic modelling. A computationally efficient machine learning framework is developed based on multi-level recursive co-kriging with sparse precision matrices of Gaussian–Markov random fields. The effectiveness of the new algorithms is demonstrated in numerical examples involving a prototype problem in risk-averse design, regression of random functions, as well as uncertainty quantification in fluid mechanics involving the evolution of a Burgers equation from a random initial state, and random laminar wakes behind circular cylinders.},
	number = {2179},
	urldate = {2021-05-26},
	journal = {Proceedings of the Royal Society A: Mathematical, Physical and Engineering Sciences},
	author = {Perdikaris, P. and Venturi, D. and Royset, J. O. and Karniadakis, G. E.},
	month = jul,
	year = {2015},
	note = {Publisher: Royal Society},
	pages = {20150018},
}

@article{romor_multi-fidelity_2021,
	title = {Multi-fidelity data fusion for the approximation of scalar functions with low intrinsic dimensionality through active subspaces},
	volume = {20},
	copyright = {© 2021 Wiley-VCH GmbH},
	issn = {1617-7061},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/pamm.202000349},
	doi = {https://doi.org/10.1002/pamm.202000349},
	abstract = {Gaussian processes are employed for non-parametric regression in a Bayesian setting. They generalize linear regression embedding the inputs in a latent manifold inside an infinite-dimensional reproducing kernel Hilbert space. We can augment the inputs with the observations of low-fidelity models in order to learn a more expressive latent manifold and thus increment the model's accuracy. This can be realized recursively with a chain of Gaussian processes with incrementally higher fidelity. We would like to extend these multi-fidelity model realizations to case studies affected by an high-dimensional input space but with a low intrinsic dimensionality. In this cases physical supported or purely numerical low-order models are still affected by the curse of dimensionality when queried for responses. When the model's gradients information is provided, the presence of an active subspace can be exploited to design low-fidelity response surfaces and thus enable Gaussian process multi-fidelity regression, without the need to perform new simulations. This is particularly useful in the case of data scarcity. In this work we present a multi-fidelity approach involving active subspaces and we test it on two different high-dimensional benchmarks.},
	language = {en},
	number = {S1},
	urldate = {2021-05-26},
	journal = {PAMM},
	author = {Romor, Francesco and Tezzele, Marco and Rozza, Gianluigi},
	year = {2021},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/pamm.202000349},
	pages = {e202000349},
}

@article{embrechts_copulas_2009,
	title = {Copulas: {A} {Personal} {View}},
	volume = {76},
	copyright = {© The Journal of Risk and Insurance, 2009},
	issn = {1539-6975},
	shorttitle = {Copulas},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/j.1539-6975.2009.01310.x},
	doi = {https://doi.org/10.1111/j.1539-6975.2009.01310.x},
	abstract = {Copula modeling has taken the world of finance and insurance, and well beyond, by storm. Why is this? In this article, I review the early start of this development, discuss some important current research, mainly from an applications point of view, and comment on potential future developments. An alternative title of the article would be “Demystifying the copula craze.” The article also contains what I would like to call the copula must-reads.},
	language = {en},
	number = {3},
	urldate = {2021-05-25},
	journal = {Journal of Risk and Insurance},
	author = {Embrechts, Paul},
	year = {2009},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/j.1539-6975.2009.01310.x},
	pages = {639--650},
}

@article{genest_everything_2007,
	title = {Everything {You} {Always} {Wanted} to {Know} about {Copula} {Modeling} but {Were} {Afraid} to {Ask}},
	volume = {12},
	issn = {1084-0699},
	url = {https://ascelibrary.org/doi/abs/10.1061/%28ASCE%291084-0699%282007%2912%3A4%28347%29},
	doi = {10.1061/(ASCE)1084-0699(2007)12:4(347)},
	abstract = {This paper presents an introduction to inference for copula models, based on rank methods. By working out in detail a small, fictitious numerical example, the writers exhibit the various steps involved in investigating the dependence between two random variables and in modeling it using copulas. Simple graphical tools and numerical techniques are presented for selecting an appropriate model, estimating its parameters, and checking its goodness-of-fit. A larger, realistic application of the methodology to hydrological data is then presented.},
	language = {EN},
	number = {4},
	urldate = {2021-05-25},
	journal = {Journal of Hydrologic Engineering},
	author = {Genest, Christian and Favre, Anne-Catherine},
	month = jul,
	year = {2007},
	note = {Publisher: American Society of Civil Engineers},
	keywords = {statistics},
	pages = {347--368},
}

@inproceedings{rezende_variational_2015,
	title = {Variational {Inference} with {Normalizing} {Flows}},
	url = {http://proceedings.mlr.press/v37/rezende15.html},
	abstract = {The choice of the approximate posterior distribution is one of the core problems in variational inference. Most applications of variational inference employ simple families of posterior approximati...},
	language = {en},
	urldate = {2021-05-25},
	booktitle = {International {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Rezende, Danilo and Mohamed, Shakir},
	month = jun,
	year = {2015},
	note = {ISSN: 1938-7228},
	pages = {1530--1538},
}

@article{ng_multifidelity_2014,
	title = {Multifidelity approaches for optimization under uncertainty},
	volume = {100},
	copyright = {Copyright © 2014 John Wiley \& Sons, Ltd.},
	issn = {1097-0207},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/nme.4761},
	doi = {https://doi.org/10.1002/nme.4761},
	abstract = {SUMMARYIt is important to design robust and reliable systems by accounting for uncertainty and variability in the design process. However, performing optimization in this setting can be computationally expensive, requiring many evaluations of the numerical model to compute statistics of the system performance at every optimization iteration. This paper proposes a multifidelity approach to optimization under uncertainty that makes use of inexpensive, low-fidelity models to provide approximate information about the expensive, high-fidelity model. The multifidelity estimator is developed based on the control variate method to reduce the computational cost of achieving a specified mean square error in the statistic estimate. The method optimally allocates the computational load between the two models based on their relative evaluation cost and the strength of the correlation between them. This paper also develops an information reuse estimator that exploits the autocorrelation structure of the high-fidelity model in the design space to reduce the cost of repeatedly estimating statistics during the course of optimization. Finally, a combined estimator incorporates the features of both the multifidelity estimator and the information reuse estimator. The methods demonstrate 90\% computational savings in an acoustic horn robust optimization example and practical design turnaround time in a robust wing optimization problem. Copyright © 2014 John Wiley \& Sons, Ltd.},
	language = {en},
	number = {10},
	urldate = {2021-05-19},
	journal = {International Journal for Numerical Methods in Engineering},
	author = {Ng, Leo W. T. and Willcox, Karen E.},
	year = {2014},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/nme.4761},
	keywords = {computational design, model reduction, multi-fidelity, optimization},
	pages = {746--772},
}

@article{metropolis_monte_1949,
	title = {The {Monte} {Carlo} {Method}},
	volume = {44},
	issn = {0162-1459},
	url = {https://www.tandfonline.com/doi/abs/10.1080/01621459.1949.10483310},
	doi = {10.1080/01621459.1949.10483310},
	abstract = {We shall present here the motivation and a general description of a method dealing with a class of problems in mathematical physics. The method is, essentially, a statistical approach to the study of differential equations, or more generally, of integro-differential equations that occur in various branches of the natural sciences.},
	number = {247},
	urldate = {2021-05-19},
	journal = {Journal of the American Statistical Association},
	author = {Metropolis, Nicholas and Ulam, S.},
	month = sep,
	year = {1949},
	pmid = {18139350},
	note = {Publisher: Taylor \& Francis
\_eprint: https://www.tandfonline.com/doi/pdf/10.1080/01621459.1949.10483310},
	pages = {335--341},
}

@article{wolpert_stacked_1992,
	title = {Stacked generalization},
	volume = {5},
	issn = {0893-6080},
	url = {https://www.sciencedirect.com/science/article/pii/S0893608005800231},
	doi = {10.1016/S0893-6080(05)80023-1},
	abstract = {This paper introduces stacked generalization, a scheme for minimizing the generalization error rate of one or more generalizers. Stacked generalization works by deducing the biases of the generalizer(s) with respect to a provided learning set. This deduction proceeds by generalizing in a second space whose inputs are (for example) the guesses of the original generalizers when taught with part of the learning set and trying to guess the rest of it, and whose output is (for example) the correct guess. When used with multiple generalizers, stacked generalization can be seen as a more sophisticated version of cross-validation, exploiting a strategy more sophisticated than cross-validation's crude winner-takes-all for combining the individual generalizers. When used with a single generalizer, stacked generalization is a scheme for estimating (and then correcting for) the error of a generalizer which has been trained on a particular learning set and then asked a particular question. After introducing stacked generalization and justifying its use, this paper presents two numerical experiments. The first demonstrates how stacked generalization improves upon a set of separate generalizers for the NETtalk task of translating text to phonemes. The second demonstrates how stacked generalization improves the performance of a single surface-fitter. With the other experimental evidence in the literature, the usual arguments supporting cross-validation, and the abstract justifications presented in this paper, the conclusion is that for almost any real-world generalization problem one should use some version of stacked generalization to minimize the generalization error rate. This paper ends by discussing some of the variations of stacked generalization, and how it touches on other fields like chaos theory.},
	language = {en},
	number = {2},
	urldate = {2021-05-18},
	journal = {Neural Networks},
	author = {Wolpert, David H.},
	month = jan,
	year = {1992},
	pages = {241--259},
}

@article{breiman_random_2001,
	title = {Random {Forests}},
	volume = {45},
	issn = {1573-0565},
	url = {https://doi.org/10.1023/A:1010933404324},
	doi = {10.1023/A:1010933404324},
	abstract = {Random forests are a combination of tree predictors such that each tree depends on the values of a random vector sampled independently and with the same distribution for all trees in the forest. The generalization error for forests converges a.s. to a limit as the number of trees in the forest becomes large. The generalization error of a forest of tree classifiers depends on the strength of the individual trees in the forest and the correlation between them. Using a random selection of features to split each node yields error rates that compare favorably to Adaboost (Y. Freund \& R. Schapire, Machine Learning: Proceedings of the Thirteenth International conference, ***, 148–156), but are more robust with respect to noise. Internal estimates monitor error, strength, and correlation and these are used to show the response to increasing the number of features used in the splitting. Internal estimates are also used to measure variable importance. These ideas are also applicable to regression.},
	language = {en},
	number = {1},
	urldate = {2021-05-18},
	journal = {Machine Learning},
	author = {Breiman, Leo},
	month = oct,
	year = {2001},
	pages = {5--32},
}

@inproceedings{dietterich_ensemble_2000,
	address = {Berlin, Heidelberg},
	series = {Lecture {Notes} in {Computer} {Science}},
	title = {Ensemble {Methods} in {Machine} {Learning}},
	isbn = {978-3-540-45014-6},
	doi = {10.1007/3-540-45014-9_1},
	abstract = {Ensemble methods are learning algorithms that construct a set of classifiers and then classify new data points by taking a (weighted) vote of their predictions. The original ensemble method is Bayesian averaging, but more recent algorithms include error-correcting output coding, Bagging, and boosting. This paper reviews these methods and explains why ensembles can often perform better than any single classifier. Some previous studies comparing ensemble methods are reviewed, and some new experiments are presented to uncover the reasons that Adaboost does not overfit rapidly.},
	language = {en},
	booktitle = {Multiple {Classifier} {Systems}},
	publisher = {Springer},
	author = {Dietterich, Thomas G.},
	year = {2000},
	keywords = {decision trees, ensemble methods},
	pages = {1--15},
}

@article{kennedy_predicting_2000,
	title = {Predicting the output from a complex computer code when fast approximations are available},
	volume = {87},
	issn = {0006-3444},
	url = {https://doi.org/10.1093/biomet/87.1.1},
	doi = {10.1093/biomet/87.1.1},
	abstract = {We consider prediction and uncertainty analysis for complex computer codes which can be run at different levels of sophistication. In particular, we wish to improve efficiency by combining expensive runs of the most complex versions of the code with relatively cheap runs from one or more simpler approximations. A Bayesian approach is described in which prior beliefs about the codes are represented in terms of Gaussian processes. An example is presented using two versions of an oil reservoir simulator.},
	number = {1},
	urldate = {2021-05-17},
	journal = {Biometrika},
	author = {Kennedy, MC and O'Hagan, A},
	month = mar,
	year = {2000},
	pages = {1--13},
}

@article{wang_gaussian_2020,
	title = {Gaussian process based optimization algorithms with input uncertainty},
	volume = {52},
	issn = {2472-5854},
	url = {https://doi.org/10.1080/24725854.2019.1639859},
	doi = {10.1080/24725854.2019.1639859},
	abstract = {Metamodels as cheap approximation models for expensive to evaluate functions have been commonly used in simulation optimization problems. Among various types of metamodels, the Gaussian Process (GP) model is popular for both deterministic and stochastic simulation optimization problems. However, input uncertainty is usually ignored in simulation optimization problems, and thus current GP-based optimization algorithms do not incorporate input uncertainty. This article aims to refine the current GP-based optimization algorithms to solve the stochastic simulation optimization problems when input uncertainty is considered. The comprehensive numerical results indicate that our refined algorithms with input uncertainty can find optimal designs more efficiently than the existing algorithms when input uncertainty is present.},
	number = {4},
	urldate = {2021-05-11},
	journal = {IISE Transactions},
	author = {Wang, Haowei and Yuan, Jun and Ng, Szu Hui},
	month = apr,
	year = {2020},
	note = {Publisher: Taylor \& Francis
\_eprint: https://doi.org/10.1080/24725854.2019.1639859},
	keywords = {gaussian processes, input uncertainty},
	pages = {377--393},
}

@article{wang_nonparametric_2021,
	title = {A nonparametric {Bayesian} approach for simulation optimization with input uncertainty},
	url = {http://arxiv.org/abs/2008.02154},
	abstract = {Stochastic simulation models are increasingly popular for analyzing complex stochastic systems. However, the input distributions required to drive the simulation are typically unknown in practice and are usually estimated from real world data. Since the amount of real world data tends to be limited, the resulting estimation of the input distribution will contain errors. This estimation error is commonly known as input uncertainty. In this paper, we consider the stochastic simulation optimization problem when input uncertainty is present and assume that both the input distribution family and the distribution parameters are unknown. We adopt a nonparametric Bayesian approach to model the input uncertainty and propose a nonparametric Bayesian risk optimization (NBRO) framework to study simulation optimization under input distribution uncertainty. We establish the consistency and asymptotic normality of the value of the objective function, the optimal solution, and the optimal value for the NBRO problem. We further propose a simulation optimization algorithm based on Efficient Global Optimization (EGO), which explicitly takes into account the input uncertainty so as to efficiently solve the NBRO problem. We study the consistency property of the modified EGO algorithm and empirically test its efficiency with an inventory example and a critical care facility problem.},
	urldate = {2021-05-11},
	journal = {arXiv:2008.02154 [math]},
	author = {Wang, Haowei and Zhang, Xun and Ng, Szu Hui},
	month = jan,
	year = {2021},
	note = {arXiv: 2008.02154},
	keywords = {control},
}

@article{song_general_2018,
	title = {A {General} {Framework} for {Multi}-fidelity {Bayesian} {Optimization} with {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/1811.00755},
	abstract = {How can we efficiently gather information to optimize an unknown function, when presented with multiple, mutually dependent information sources with different costs? For example, when optimizing a robotic system, intelligently trading off computer simulations and real robot testings can lead to significant savings. Existing methods, such as multi-fidelity GP-UCB or Entropy Search-based approaches, either make simplistic assumptions on the interaction among different fidelities or use simple heuristics that lack theoretical guarantees. In this paper, we study multi-fidelity Bayesian optimization with complex structural dependencies among multiple outputs, and propose MF-MI-Greedy, a principled algorithmic framework for addressing this problem. In particular, we model different fidelities using additive Gaussian processes based on shared latent structures with the target function. Then we use cost-sensitive mutual information gain for efficient Bayesian global optimization. We propose a simple notion of regret which incorporates the cost of different fidelities, and prove that MF-MI-Greedy achieves low regret. We demonstrate the strong empirical performance of our algorithm on both synthetic and real-world datasets.},
	urldate = {2021-05-10},
	journal = {arXiv:1811.00755 [cs, stat]},
	author = {Song, Jialin and Chen, Yuxin and Yue, Yisong},
	month = nov,
	year = {2018},
	note = {arXiv: 1811.00755},
	keywords = {machine learning},
}

@article{motamed_multi-fidelity_2020,
	title = {A {MULTI}-{FIDELITY} {NEURAL} {NETWORK} {SURROGATE} {SAMPLING} {METHOD} {FOR} {UNCERTAINTY} {QUANTIFICATION}},
	volume = {10},
	issn = {2152-5080, 2152-5099},
	url = {http://www.dl.begellhouse.com/journals/52034eb04b657aea,148a754a61aebe77,6378f84e53896868.html},
	doi = {10.1615/Int.J.UncertaintyQuantification.2020031957},
	language = {English},
	number = {4},
	urldate = {2021-05-05},
	journal = {International Journal for Uncertainty Quantification},
	author = {Motamed, Mohammad},
	year = {2020},
	note = {Publisher: Begel House Inc.},
}

@inproceedings{hegde_deep_2019,
	title = {Deep learning with differential {Gaussian} process flows},
	url = {http://proceedings.mlr.press/v89/hegde19a.html},
	abstract = {We propose a novel deep learning paradigm of differential flows that learn a stochastic differential equation transformations of inputs prior to a standard classification or regression function. Th...},
	language = {en},
	urldate = {2021-05-03},
	booktitle = {The 22nd {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Hegde, Pashupati and Heinonen, Markus and Lähdesmäki, Harri and Kaski, Samuel},
	month = apr,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {1812--1821},
}

@article{cutajar_deep_2019,
	title = {Deep {Gaussian} {Processes} for {Multi}-fidelity {Modeling}},
	url = {http://arxiv.org/abs/1903.07320},
	abstract = {Multi-fidelity methods are prominently used when cheaply-obtained, but possibly biased and noisy, observations must be effectively combined with limited or expensive true data in order to construct reliable models. This arises in both fundamental machine learning procedures such as Bayesian optimization, as well as more practical science and engineering applications. In this paper we develop a novel multi-fidelity model which treats layers of a deep Gaussian process as fidelity levels, and uses a variational inference scheme to propagate uncertainty across them. This allows for capturing nonlinear correlations between fidelities with lower risk of overfitting than existing methods exploiting compositional structure, which are conversely burdened by structural assumptions and constraints. We show that the proposed approach makes substantial improvements in quantifying and propagating uncertainty in multi-fidelity set-ups, which in turn improves their effectiveness in decision making pipelines.},
	urldate = {2021-04-30},
	journal = {arXiv:1903.07320 [cs, stat]},
	author = {Cutajar, Kurt and Pullin, Mark and Damianou, Andreas and Lawrence, Neil and González, Javier},
	month = mar,
	year = {2019},
	note = {arXiv: 1903.07320},
	keywords = {machine learning},
}

@article{dwivedi_distributed_2019,
	title = {Distributed physics informed neural network for data-efficient solution to partial differential equations},
	url = {http://arxiv.org/abs/1907.08967},
	abstract = {The physics informed neural network (PINN) is evolving as a viable method to solve partial differential equations. In the recent past PINNs have been successfully tested and validated to find solutions to both linear and non-linear partial differential equations (PDEs). However, the literature lacks detailed investigation of PINNs in terms of their representation capability. In this work, we first test the original PINN method in terms of its capability to represent a complicated function. Further, to address the shortcomings of the PINN architecture, we propose a novel distributed PINN, named DPINN. We first perform a direct comparison of the proposed DPINN approach against PINN to solve a non-linear PDE (Burgers' equation). We show that DPINN not only yields a more accurate solution to the Burgers' equation, but it is found to be more data-efficient as well. At last, we employ our novel DPINN to two-dimensional steady-state Navier-Stokes equation, which is a system of non-linear PDEs. To the best of the authors' knowledge, this is the first such attempt to directly solve the Navier-Stokes equation using a physics informed neural network.},
	urldate = {2021-04-30},
	journal = {arXiv:1907.08967 [physics, stat]},
	author = {Dwivedi, Vikas and Parashar, Nishant and Srinivasan, Balaji},
	month = jul,
	year = {2019},
	note = {arXiv: 1907.08967},
	keywords = {computational physics, machine learning},
}

@article{raissi_inferring_2017,
	title = {Inferring solutions of differential equations using noisy multi-fidelity data},
	volume = {335},
	issn = {0021-9991},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999117300761},
	doi = {10.1016/j.jcp.2017.01.060},
	abstract = {For more than two centuries, solutions of differential equations have been obtained either analytically or numerically based on typically well-behaved forcing and boundary conditions for well-posed problems. We are changing this paradigm in a fundamental way by establishing an interface between probabilistic machine learning and differential equations. We develop data-driven algorithms for general linear equations using Gaussian process priors tailored to the corresponding integro-differential operators. The only observables are scarce noisy multi-fidelity data for the forcing and solution that are not required to reside on the domain boundary. The resulting predictive posterior distributions quantify uncertainty and naturally lead to adaptive solution refinement via active learning. This general framework circumvents the tyranny of numerical discretization as well as the consistency and stability issues of time-integration, and is scalable to high-dimensions.},
	language = {en},
	urldate = {2021-04-30},
	journal = {Journal of Computational Physics},
	author = {Raissi, Maziar and Perdikaris, Paris and Karniadakis, George Em},
	month = apr,
	year = {2017},
	keywords = {machine learning, multi-fidelity, uncertainty estimation},
	pages = {736--746},
}

@article{li_multi-fidelity_2020,
	title = {Multi-{Fidelity} {Bayesian} {Optimization} via {Deep} {Neural} {Networks}},
	volume = {33},
	url = {https://papers.nips.cc/paper/2020/hash/60e1deb043af37db5ea4ce9ae8d2c9ea-Abstract.html},
	language = {en},
	urldate = {2021-04-30},
	journal = {Advances in Neural Information Processing Systems},
	author = {Li, Shibo and Xing, Wei and Kirby, Robert and Zhe, Shandian},
	year = {2020},
	pages = {8521--8531},
}

@article{meng_multi-fidelity_2021,
	title = {Multi-fidelity {Bayesian} {Neural} {Networks}: {Algorithms} and {Applications}},
	issn = {0021-9991},
	shorttitle = {Multi-fidelity {Bayesian} {Neural} {Networks}},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999121002564},
	doi = {10.1016/j.jcp.2021.110361},
	abstract = {We propose a new class of Bayesian neural networks (BNNs) that can be trained using noisy data of variable fidelity, and we apply them to learn function approximations as well as to solve inverse problems based on partial differential equations (PDEs). These multi-fidelity BNNs consist of three neural networks: The first is a fully connected neural network, which is trained following the maximum a posteriori probability (MAP) method to fit the low-fidelity data; the second is a Bayesian neural network employed to capture the cross-correlation with uncertainty quantification between the low- and high-fidelity data; and the last one is the physics-informed neural network, which encodes the physical laws described by PDEs. For the training of the last two neural networks, we first employ the mean-field variational inference (VI) to maximize the evidence lower bound (ELBO) to obtain informative prior distributions for the hyperparameters in the BNNs, and subsequently we use the Hamiltonian Monte Carlo (HMC) method to estimate accurately the posterior distributions for the corresponding hyperparameters. We demonstrate the accuracy of the present method using synthetic data as well as real measurements. Specifically, we first approximate a one- and four-dimensional function, and then infer the reaction rates in one- and two-dimensional diffusion-reaction systems. Moreover, we infer the sea surface temperature (SST) in the Massachusetts and Cape Cod Bays using satellite images and in-situ measurements. Taken together, our results demonstrate that the present method can capture both linear and nonlinear correlation between the low- and high-fidelity data adaptively, identify unknown parameters in PDEs, and quantify uncertainties in predictions, given a few scattered noisy high-fidelity data. Finally, we demonstrate that we can effectively and efficiently reduce the uncertainties and hence enhance the prediction accuracy with an active learning approach, using as examples a specific one-dimensional function approximation and an inverse PDE problem.},
	language = {en},
	urldate = {2021-04-30},
	journal = {Journal of Computational Physics},
	author = {Meng, Xuhui and Babaee, Hessam and Karniadakis, George Em},
	month = apr,
	year = {2021},
	keywords = {active learning, hamiltonian monte carlo, physics-informed neural networks, remote sensing, uncertainty estimation},
	pages = {110361},
}

@article{chakraborty_transfer_2021,
	title = {Transfer learning based multi-fidelity physics informed deep neural network},
	volume = {426},
	issn = {0021-9991},
	url = {https://www.sciencedirect.com/science/article/pii/S0021999120307166},
	doi = {10.1016/j.jcp.2020.109942},
	abstract = {For many systems in science and engineering, the governing differential equation is either not known or known in an approximate sense. Analyses and design of such systems are governed by data collected from the field and/or laboratory experiments. This challenging scenario is further worsened when data-collection is expensive and time-consuming. To address this issue, this paper presents a novel multi-fidelity physics informed deep neural network (MF-PIDNN). The framework proposed is particularly suitable when the physics of the problem is known in an approximate sense (low-fidelity physics) and only a few high-fidelity data are available. MF-PIDNN blends physics informed and data-driven deep learning techniques by using the concept of transfer learning. The approximate governing equation is first used to train a low-fidelity physics informed deep neural network. This is followed by transfer learning where the low-fidelity model is updated by using the available high-fidelity data. MF-PIDNN is able to encode useful information on the physics of the problem from the approximate governing differential equation and hence, provides accurate prediction even in zones with no data. Additionally, no low-fidelity data is required for training this model. Two examples involving function approximations with linear and nonlinear correlation are presented to illustrate the effectiveness of transfer learning in solving multi-fidelity problems. Applicability and utility of MF-PIDNN are illustrated in solving four benchmark reliability analysis problems. Case studies presented illustrate interesting features of the proposed approach.},
	language = {en},
	urldate = {2021-04-30},
	journal = {Journal of Computational Physics},
	author = {Chakraborty, Souvik},
	month = feb,
	year = {2021},
	keywords = {multi-fidelity, neural networks, physics-informed learning, transfer learning},
	pages = {109942},
}

@article{hebbal_multi-fidelity_2021,
	title = {Multi-fidelity modeling with different input domain definitions using deep {Gaussian} processes},
	volume = {63},
	issn = {1615-1488},
	url = {https://doi.org/10.1007/s00158-020-02802-1},
	doi = {10.1007/s00158-020-02802-1},
	abstract = {Multi-fidelity approaches combine different models built on a scarce but accurate dataset (high-fidelity dataset), and a large but approximate one (low-fidelity dataset) in order to improve the prediction accuracy. Gaussian processes (GPs) are one of the popular approaches to exhibit the correlations between these different fidelity levels. Deep Gaussian processes (DGPs) that are functional compositions of GPs have also been adapted to multi-fidelity using the multi-fidelity deep Gaussian process (MF-DGP) model. This model increases the expressive power compared to GPs by considering non-linear correlations between fidelities within a Bayesian framework. However, these multi-fidelity methods consider only the case where the inputs of the different fidelity models are defined over the same domain of definition (e.g., same variables, same dimensions). However, due to simplification in the modeling of the low fidelity, some variables may be omitted or a different parametrization may be used compared to the high-fidelity model. In this paper, deep Gaussian processes for multi-fidelity (MF-DGP) are extended to the case where a different parametrization is used for each fidelity. The performance of the proposed multi-fidelity modeling technique is assessed on analytical test cases and on structural and aerodynamic real physical problems.},
	language = {en},
	number = {5},
	urldate = {2021-04-30},
	journal = {Structural and Multidisciplinary Optimization},
	author = {Hebbal, Ali and Brevault, Loïc and Balesdent, Mathieu and Talbi, El-Ghazali and Melab, Nouredine},
	month = may,
	year = {2021},
	pages = {2267--2288},
}

@article{sauer_active_2020,
	title = {Active {Learning} for {Deep} {Gaussian} {Process} {Surrogates}},
	url = {http://arxiv.org/abs/2012.08015},
	abstract = {Deep Gaussian processes (DGPs) are increasingly popular as predictive models in machine learning (ML) for their non-stationary flexibility and ability to cope with abrupt regime changes in training data. Here we explore DGPs as surrogates for computer simulation experiments whose response surfaces exhibit similar characteristics. In particular, we transport a DGP's automatic warping of the input space and full uncertainty quantification (UQ), via a novel elliptical slice sampling (ESS) Bayesian posterior inferential scheme, through to active learning (AL) strategies that distribute runs non-uniformly in the input space -- something an ordinary (stationary) GP could not do. Building up the design sequentially in this way allows smaller training sets, limiting both expensive evaluation of the simulator code and mitigating cubic costs of DGP inference. When training data sizes are kept small through careful acquisition, and with parsimonious layout of latent layers, the framework can be both effective and computationally tractable. Our methods are illustrated on simulation data and two real computer experiments of varying input dimensionality. We provide an open source implementation in the "deepgp" package on CRAN.},
	urldate = {2021-04-30},
	journal = {arXiv:2012.08015 [stat]},
	author = {Sauer, Annie and Gramacy, Robert B. and Higdon, David},
	month = dec,
	year = {2020},
	note = {arXiv: 2012.08015},
	keywords = {machine learning, statistics},
}

@inproceedings{wang_multi-fidelity_2021,
	title = {Multi-{Fidelity} {High}-{Order} {Gaussian} {Processes} for {Physical} {Simulation}},
	url = {http://proceedings.mlr.press/v130/wang21c.html},
	abstract = {The key task of physical simulation is to solve partial differential equations (PDEs) on discretized domains, which is known to be costly. In particular, high-fidelity solutions are much more expe...},
	language = {en},
	urldate = {2021-04-30},
	booktitle = {International {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Wang, Zheng and Xing, Wei and Kirby, Robert and Zhe, Shandian},
	month = mar,
	year = {2021},
	note = {ISSN: 2640-3498},
	pages = {847--855},
}

@article{peng_multiscale_2021,
	title = {Multiscale {Modeling} {Meets} {Machine} {Learning}: {What} {Can} {We} {Learn}?},
	volume = {28},
	issn = {1886-1784},
	shorttitle = {Multiscale {Modeling} {Meets} {Machine} {Learning}},
	url = {https://doi.org/10.1007/s11831-020-09405-5},
	doi = {10.1007/s11831-020-09405-5},
	abstract = {Machine learning is increasingly recognized as a promising technology in the biological, biomedical, and behavioral sciences. There can be no argument that this technique is incredibly successful in image recognition with immediate applications in diagnostics including electrophysiology, radiology, or pathology, where we have access to massive amounts of annotated data. However, machine learning often performs poorly in prognosis, especially when dealing with sparse data. This is a field where classical physics-based simulation seems to remain irreplaceable. In this review, we identify areas in the biomedical sciences where machine learning and multiscale modeling can mutually benefit from one another: Machine learning can integrate physics-based knowledge in the form of governing equations, boundary conditions, or constraints to manage ill-posted problems and robustly handle sparse and noisy data; multiscale modeling can integrate machine learning to create surrogate models, identify system dynamics and parameters, analyze sensitivities, and quantify uncertainty to bridge the scales and understand the emergence of function. With a view towards applications in the life sciences, we discuss the state of the art of combining machine learning and multiscale modeling, identify applications and opportunities, raise open questions, and address potential challenges and limitations. We anticipate that it will stimulate discussion within the community of computational mechanics and reach out to other disciplines including mathematics, statistics, computer science, artificial intelligence, biomedicine, systems biology, and precision medicine to join forces towards creating robust and efficient models for biological systems.},
	language = {en},
	number = {3},
	urldate = {2021-04-30},
	journal = {Archives of Computational Methods in Engineering},
	author = {Peng, Grace C. Y. and Alber, Mark and Buganza Tepole, Adrian and Cannon, William R. and De, Suvranu and Dura-Bernal, Savador and Garikipati, Krishna and Karniadakis, George and Lytton, William W. and Perdikaris, Paris and Petzold, Linda and Kuhl, Ellen},
	month = may,
	year = {2021},
	pages = {1017--1037},
}

@article{pfaff_learning_2021,
	title = {Learning {Mesh}-{Based} {Simulation} with {Graph} {Networks}},
	url = {http://arxiv.org/abs/2010.03409},
	abstract = {Mesh-based simulations are central to modeling complex physical systems in many disciplines across science and engineering. Mesh representations support powerful numerical integration methods and their resolution can be adapted to strike favorable trade-offs between accuracy and efficiency. However, high-dimensional scientific simulations are very expensive to run, and solvers and parameters must often be tuned individually to each system studied. Here we introduce MeshGraphNets, a framework for learning mesh-based simulations using graph neural networks. Our model can be trained to pass messages on a mesh graph and to adapt the mesh discretization during forward simulation. Our results show it can accurately predict the dynamics of a wide range of physical systems, including aerodynamics, structural mechanics, and cloth. The model's adaptivity supports learning resolution-independent dynamics and can scale to more complex state spaces at test time. Our method is also highly efficient, running 1-2 orders of magnitude faster than the simulation on which it is trained. Our approach broadens the range of problems on which neural network simulators can operate and promises to improve the efficiency of complex, scientific modeling tasks.},
	urldate = {2021-04-29},
	journal = {arXiv:2010.03409 [cs]},
	author = {Pfaff, Tobias and Fortunato, Meire and Sanchez-Gonzalez, Alvaro and Battaglia, Peter W.},
	month = jan,
	year = {2021},
	note = {arXiv: 2010.03409},
	keywords = {machine learning},
}

@article{fer_linking_2018,
	title = {Linking big models to big data: efficient ecosystem model calibration through {Bayesian} model emulation},
	volume = {15},
	issn = {1726-4170},
	shorttitle = {Linking big models to big data},
	url = {https://bg.copernicus.org/articles/15/5801/2018/},
	doi = {10.5194/bg-15-5801-2018},
	abstract = {{\textless}p{\textgreater}{\textless}strong class="journal-contentHeaderColor"{\textgreater}Abstract.{\textless}/strong{\textgreater} Data-model integration plays a critical role in assessing and improving our capacity to predict ecosystem dynamics. Similarly, the ability to attach quantitative statements of uncertainty around model forecasts is crucial for model assessment and interpretation and for setting field research priorities. Bayesian methods provide a rigorous data assimilation framework for these applications, especially for problems with multiple data constraints. However, the Markov chain Monte Carlo (MCMC) techniques underlying most Bayesian calibration can be prohibitive for computationally demanding models and large datasets. We employ an alternative method, Bayesian model emulation of sufficient statistics, that can approximate the full joint posterior density, is more amenable to parallelization, and provides an estimate of parameter sensitivity. Analysis involved informative priors constructed from a meta-analysis of the primary literature and specification of both model and data uncertainties, and it introduced novel approaches to autocorrelation corrections on multiple data streams and emulating the sufficient statistics surface. We report the integration of this method within an ecological workflow management software, Predictive Ecosystem Analyzer (PEcAn), and its application and validation with two process-based terrestrial ecosystem models: SIPNET and ED2. In a test against a synthetic dataset, the emulator was able to retrieve the true parameter values. A comparison of the emulator approach to standard {\textless}q{\textgreater}brute-force{\textless}/q{\textgreater} MCMC involving multiple data constraints showed that the emulator method was able to constrain the faster and simpler SIPNET model's parameters with comparable performance to the brute-force approach but reduced computation time by more than 2 orders of magnitude. The emulator was then applied to calibration of the ED2 model, whose complexity precludes standard (brute-force) Bayesian data assimilation techniques. Both models are constrained after assimilation of the observational data with the emulator method, reducing the uncertainty around their predictions. Performance metrics showed increased agreement between model predictions and data. Our study furthers efforts toward reducing model uncertainties, showing that the emulator method makes it possible to efficiently calibrate complex models.{\textless}/p{\textgreater}},
	language = {English},
	number = {19},
	urldate = {2021-04-28},
	journal = {Biogeosciences},
	author = {Fer, Istem and Kelly, Ryan and Moorcroft, Paul R. and Richardson, Andrew D. and Cowdery, Elizabeth M. and Dietze, Michael C.},
	month = oct,
	year = {2018},
	note = {Publisher: Copernicus GmbH},
	pages = {5801--5830},
}

@article{reichstein_deep_2019,
	title = {Deep learning and process understanding for data-driven {Earth} system science},
	volume = {566},
	copyright = {2019 Springer Nature Limited},
	issn = {1476-4687},
	url = {https://www.nature.com/articles/s41586-019-0912-1},
	doi = {10.1038/s41586-019-0912-1},
	abstract = {Machine learning approaches are increasingly used to extract patterns and insights from the ever-increasing stream of geospatial data, but current approaches may not be optimal when system behaviour is dominated by spatial or temporal context. Here, rather than amending classical machine learning, we argue that these contextual cues should be used as part of deep learning (an approach that is able to extract spatio-temporal features automatically) to gain further process understanding of Earth system science problems, improving the predictive ability of seasonal forecasting and modelling of long-range spatial connections across multiple timescales, for example. The next step will be a hybrid modelling approach, coupling physical process models with the versatility of data-driven machine learning.},
	language = {en},
	number = {7743},
	urldate = {2021-04-26},
	journal = {Nature},
	author = {Reichstein, Markus and Camps-Valls, Gustau and Stevens, Bjorn and Jung, Martin and Denzler, Joachim and Carvalhais, Nuno and Prabhat},
	month = feb,
	year = {2019},
	note = {Number: 7743
Publisher: Nature Publishing Group},
	pages = {195--204},
}

@article{koutsourelakis_accurate_2009,
	title = {Accurate {Uncertainty} {Quantification} {Using} {Inaccurate} {Computational} {Models}},
	volume = {31},
	issn = {1064-8275},
	url = {https://epubs.siam.org/doi/abs/10.1137/080733565},
	doi = {10.1137/080733565},
	abstract = {This paper proposes a novel uncertainty quantification framework for computationally demanding systems characterized by a large vector of non-Gaussian uncertainties. It combines state-of-the-art techniques in advanced Monte Carlo sampling with Bayesian formulations. The key departure from existing works is the use of inexpensive, approximate computational models in a rigorous manner. Such models can readily be derived by coarsening the discretization size in the solution of the governing PDEs, increasing the time step when integration of ODEs is performed, using fewer iterations if a nonlinear solver is employed or making use of lower-order models. It is shown that even in cases where the inaccurate models provide very poor approximations of the high-fidelity response, statistics of the latter can be quantified accurately with significant reductions in the computational effort. Multiple approximate models can be used, and rigorous confidence bounds of the estimates produced are provided at all stages.},
	number = {5},
	urldate = {2021-04-26},
	journal = {SIAM Journal on Scientific Computing},
	author = {Koutsourelakis, Phaedon-Stelios},
	month = jan,
	year = {2009},
	note = {Publisher: Society for Industrial and Applied Mathematics},
	pages = {3274--3300},
}

@article{peherstorfer_optimal_2016,
	title = {Optimal {Model} {Management} for {Multifidelity} {Monte} {Carlo} {Estimation}},
	volume = {38},
	issn = {1064-8275},
	url = {https://epubs.siam.org/doi/abs/10.1137/15M1046472},
	doi = {10.1137/15M1046472},
	abstract = {This work presents an optimal model management strategy that exploits multifidelity surrogate models to accelerate the estimation of statistics of outputs of computationally expensive high-fidelity models. Existing acceleration methods typically exploit a multilevel hierarchy of surrogate models that follow a known rate of error decay and computational costs; however, a general collection of surrogate models, which may include projection-based reduced models, data-fit models, support vector machines, and simplified-physics models, does not necessarily give rise to such a hierarchy. Our multifidelity approach provides a framework to combine an arbitrary number of surrogate models of any type. Instead of relying on error and cost rates, an optimization problem balances the number of model evaluations across the high-fidelity and surrogate models with respect to error and costs. We show that a unique analytic solution of the model management optimization problem exists under mild conditions on the models. Our multifidelity method makes occasional recourse to the high-fidelity model; in doing so it provides an unbiased estimator of the statistics of the high-fidelity model, even in the absence of error bounds and error estimators for the surrogate models. Numerical experiments with linear and nonlinear examples show that speedups by orders of magnitude are obtained compared to Monte Carlo estimation that invokes a single model only.},
	number = {5},
	urldate = {2021-04-26},
	journal = {SIAM Journal on Scientific Computing},
	author = {Peherstorfer, Benjamin and Willcox, Karen and Gunzburger, Max},
	month = jan,
	year = {2016},
	note = {Publisher: Society for Industrial and Applied Mathematics},
	pages = {A3163--A3194},
}

@article{liu_accurate_2019,
	title = {Accurate {Uncertainty} {Estimation} and {Decomposition} in {Ensemble} {Learning}},
	volume = {32},
	url = {https://papers.nips.cc/paper/2019/hash/1cc8a8ea51cd0adddf5dab504a285915-Abstract.html},
	language = {en},
	urldate = {2021-04-23},
	journal = {Advances in Neural Information Processing Systems},
	author = {Liu, Jeremiah and Paisley, John and Kioumourtzoglou, Marianthi-Anna and Coull, Brent},
	year = {2019},
}

@article{geffner_using_2018,
	title = {Using {Large} {Ensembles} of {Control} {Variates} for {Variational} {Inference}},
	volume = {31},
	url = {https://papers.nips.cc/paper/2018/hash/dead35fa1512ad67301d09326177c42f-Abstract.html},
	language = {en},
	urldate = {2021-04-23},
	journal = {Advances in Neural Information Processing Systems},
	author = {Geffner, Tomas and Domke, Justin},
	year = {2018},
}

@inproceedings{lyu_batch_2018,
	title = {Batch {Bayesian} {Optimization} via {Multi}-objective {Acquisition} {Ensemble} for {Automated} {Analog} {Circuit} {Design}},
	url = {http://proceedings.mlr.press/v80/lyu18a.html},
	abstract = {Bayesian optimization methods are promising for the optimization of black-box functions that are expensive to evaluate. In this paper, a novel batch Bayesian optimization approach is proposed. The ...},
	language = {en},
	urldate = {2021-04-23},
	booktitle = {International {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Lyu, Wenlong and Yang, Fan and Yan, Changhao and Zhou, Dian and Zeng, Xuan},
	month = jul,
	year = {2018},
	note = {ISSN: 2640-3498},
	pages = {3306--3314},
}

@article{lin_ensemble_2021,
	title = {Ensemble {Distillation} for {Robust} {Model} {Fusion} in {Federated} {Learning}},
	url = {http://arxiv.org/abs/2006.07242},
	abstract = {Federated Learning (FL) is a machine learning setting where many devices collaboratively train a machine learning model while keeping the training data decentralized. In most of the current training schemes the central model is refined by averaging the parameters of the server model and the updated parameters from the client side. However, directly averaging model parameters is only possible if all models have the same structure and size, which could be a restrictive constraint in many scenarios. In this work we investigate more powerful and more flexible aggregation schemes for FL. Specifically, we propose ensemble distillation for model fusion, i.e. training the central classifier through unlabeled data on the outputs of the models from the clients. This knowledge distillation technique mitigates privacy risk and cost to the same extent as the baseline FL algorithms, but allows flexible aggregation over heterogeneous client models that can differ e.g. in size, numerical precision or structure. We show in extensive empirical experiments on various CV/NLP datasets (CIFAR-10/100, ImageNet, AG News, SST2) and settings (heterogeneous models/data) that the server model can be trained much faster, requiring fewer communication rounds than any existing FL technique so far.},
	urldate = {2021-04-23},
	journal = {arXiv:2006.07242 [cs, stat]},
	author = {Lin, Tao and Kong, Lingjing and Stich, Sebastian U. and Jaggi, Martin},
	month = mar,
	year = {2021},
	note = {arXiv: 2006.07242},
	keywords = {machine learning},
}

@inproceedings{atiya_learning_1988,
	title = {Learning on a {General} {Network}},
	url = {https://proceedings.neurips.cc/paper/1987/file/45c48cce2e2d7fbdea1afc51c7c6ad26-Paper.pdf},
	urldate = {2021-04-23},
	booktitle = {Neural {Information} {Processing} {Systems}},
	publisher = {American Institute of Physics},
	author = {Atiya, Amir},
	editor = {Anderson, D.},
	year = {1988},
}

@article{zhang_diversified_2020,
	title = {The {Diversified} {Ensemble} {Neural} {Network}},
	volume = {33},
	url = {https://papers.nips.cc/paper/2020/hash/b86e8d03fe992d1b0e19656875ee557c-Abstract.html},
	language = {en},
	urldate = {2021-04-23},
	journal = {Advances in Neural Information Processing Systems},
	author = {Zhang, Shaofeng and Liu, Meng and Yan, Junchi},
	year = {2020},
	pages = {16001--16011},
}

@inproceedings{finlay_how_2020,
	title = {How to {Train} {Your} {Neural} {ODE}: the {World} of {Jacobian} and {Kinetic} {Regularization}},
	shorttitle = {How to {Train} {Your} {Neural} {ODE}},
	url = {http://proceedings.mlr.press/v119/finlay20a.html},
	abstract = {Training neural ODEs on large datasets has not been tractable due to the necessity of allowing the adaptive numerical ODE solver to refine its step size to very small values. In practice this leads...},
	language = {en},
	urldate = {2021-04-22},
	booktitle = {International {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Finlay, Chris and Jacobsen, Joern-Henrik and Nurbekyan, Levon and Oberman, Adam},
	month = nov,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {3154--3164},
}

@phdthesis{ng_kernel-based_nodate,
	title = {Kernel-based {Copula} {Processes}},
	language = {en},
	author = {Ng, Eddie K H},
}

@article{loaizamaya_time_2018,
	title = {Time series copulas for heteroskedastic data},
	volume = {33},
	copyright = {Copyright © 2017 John Wiley \& Sons, Ltd.},
	issn = {1099-1255},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/jae.2610},
	doi = {https://doi.org/10.1002/jae.2610},
	abstract = {We propose parametric copulas that capture serial dependence in stationary heteroskedastic time series. We suggest copulas for first-order Markov series, and then extend them to higher orders and multivariate series. We derive the copula of a volatility proxy, based on which we propose new measures of volatility dependence, including co-movement and spillover in multivariate series. In general, these depend upon the marginal distributions of the series. Using exchange rate returns, we show that the resulting copula models can capture their marginal distributions more accurately than univariate and multivariate generalized autoregressive conditional heteroskedasticity models, and produce more accurate value-at-risk forecasts.},
	language = {en},
	number = {3},
	urldate = {2021-04-21},
	journal = {Journal of Applied Econometrics},
	author = {Loaiza‐Maya, Rubén and Smith, Michael S. and Maneesoonthorn, Worapree},
	year = {2018},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/jae.2610},
	pages = {332--354},
}

@inproceedings{oliveira_bayesian_2019,
	title = {Bayesian optimisation under uncertain inputs},
	url = {http://proceedings.mlr.press/v89/oliveira19a.html},
	abstract = {Bayesian optimisation (BO) has been a successful approach to optimise functions which are expensive to evaluate and whose observations are noisy. Classical BO algorithms, however, do not account fo...},
	language = {en},
	urldate = {2021-04-21},
	booktitle = {The 22nd {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Oliveira, Rafael and Ott, Lionel and Ramos, Fabio},
	month = apr,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {1177--1184},
}

@article{chan_neural_2019,
	title = {Neural {Entropic} {Estimation}: {A} faster path to mutual information estimation},
	shorttitle = {Neural {Entropic} {Estimation}},
	url = {http://arxiv.org/abs/1905.12957},
	abstract = {We point out a limitation of the mutual information neural estimation (MINE) where the network fails to learn at the initial training phase, leading to slow convergence in the number of training iterations. To solve this problem, we propose a faster method called the mutual information neural entropic estimation (MI-NEE). Our solution first generalizes MINE to estimate the entropy using a custom reference distribution. The entropy estimate can then be used to estimate the mutual information. We argue that the seemingly redundant intermediate step of entropy estimation allows one to improve the convergence by an appropriate reference distribution. In particular, we show that MI-NEE reduces to MINE in the special case when the reference distribution is the product of marginal distributions, but faster convergence is possible by choosing the uniform distribution as the reference distribution instead. Compared to the product of marginals, the uniform distribution introduces more samples in low-density regions and fewer samples in high-density regions, which appear to lead to an overall larger gradient for faster convergence.},
	urldate = {2021-04-16},
	journal = {arXiv:1905.12957 [cs, math]},
	author = {Chan, Chung and Al-Bashabsheh, Ali and Huang, Hing Pang and Lim, Michael and Tam, Da Sun Handason and Zhao, Chao},
	month = may,
	year = {2019},
	note = {arXiv: 1905.12957},
	keywords = {information theory, machine learning},
}

@article{senarathne_bayesian_2020,
	title = {Bayesian sequential design for {Copula} models},
	volume = {29},
	issn = {1863-8260},
	url = {https://doi.org/10.1007/s11749-019-00661-7},
	doi = {10.1007/s11749-019-00661-7},
	abstract = {Bayesian design requires determining the value of controllable variables in an experiment to maximise the information that will be obtained for subsequently collected data, with the majority of research in this field being focused on experiments that yield a univariate response. In this paper, a robust and computationally efficient Bayesian design approach is proposed to derive designs for experiments which yield bivariate discrete and mixed responses. To construct the joint distribution of responses, Copula models are considered, and a sequential Monte Carlo algorithm is adopted to reduce the computational effort required in deriving sequential designs. The total entropy utility function is considered to derive designs for the dual experimental goals of parameter estimation and model discrimination for Copula models. The results show that designs constructed within our framework are able to precisely estimate model parameters and that it is possible to discriminate between different competing Copula models. However, for experiments which yield binary and continuous data, it appears as though discriminating between Copula models can require a large number of data points, which may limit the general applicability of our methods and/or the range of experimental objectives that can be considered in experiments that yield multiple responses.},
	language = {en},
	number = {2},
	urldate = {2021-04-16},
	journal = {TEST},
	author = {Senarathne, S. G. J. and Drovandi, C. C. and McGree, J. M.},
	month = jun,
	year = {2020},
	pages = {454--478},
}

@article{loaiza-ganem_maximum_2017,
	title = {Maximum {Entropy} {Flow} {Networks}},
	url = {http://arxiv.org/abs/1701.03504},
	abstract = {Maximum entropy modeling is a flexible and popular framework for formulating statistical models given partial knowledge. In this paper, rather than the traditional method of optimizing over the continuous density directly, we learn a smooth and invertible transformation that maps a simple distribution to the desired maximum entropy distribution. Doing so is nontrivial in that the objective being maximized (entropy) is a function of the density itself. By exploiting recent developments in normalizing flow networks, we cast the maximum entropy problem into a finite-dimensional constrained optimization, and solve the problem by combining stochastic optimization with the augmented Lagrangian method. Simulation results demonstrate the effectiveness of our method, and applications to finance and computer vision show the flexibility and accuracy of using maximum entropy flow networks.},
	urldate = {2021-04-16},
	journal = {arXiv:1701.03504 [stat]},
	author = {Loaiza-Ganem, Gabriel and Gao, Yuanjun and Cunningham, John P.},
	month = apr,
	year = {2017},
	note = {arXiv: 1701.03504},
	keywords = {machine learning, statistics},
}

@article{foster_deep_2021,
	title = {Deep {Adaptive} {Design}: {Amortizing} {Sequential} {Bayesian} {Experimental} {Design}},
	shorttitle = {Deep {Adaptive} {Design}},
	url = {http://arxiv.org/abs/2103.02438},
	abstract = {We introduce Deep Adaptive Design (DAD), a general method for amortizing the cost of performing sequential adaptive experiments using the framework of Bayesian optimal experimental design (BOED). Traditional sequential BOED approaches require substantial computational time at each stage of the experiment. This makes them unsuitable for most real-world applications, where decisions must typically be made quickly. DAD addresses this restriction by learning an amortized design network upfront and then using this to rapidly run (multiple) adaptive experiments at deployment time. This network takes as input the data from previous steps, and outputs the next design using a single forward pass; these design decisions can be made in milliseconds during the live experiment. To train the network, we introduce contrastive information bounds that are suitable objectives for the sequential setting, and propose a customized network architecture that exploits key symmetries. We demonstrate that DAD successfully amortizes the process of experimental design, outperforming alternative strategies on a number of problems.},
	urldate = {2021-04-16},
	journal = {arXiv:2103.02438 [cs, stat]},
	author = {Foster, Adam and Ivanova, Desi R. and Malik, Ilyas and Rainforth, Tom},
	month = mar,
	year = {2021},
	note = {arXiv: 2103.02438},
	keywords = {machine learning, statistics},
}

@article{kleinegesse_sequential_2020,
	title = {Sequential {Bayesian} {Experimental} {Design} for {Implicit} {Models} via {Mutual} {Information}},
	url = {http://arxiv.org/abs/2003.09379},
	abstract = {Bayesian experimental design (BED) is a framework that uses statistical models and decision making under uncertainty to optimise the cost and performance of a scientific experiment. Sequential BED, as opposed to static BED, considers the scenario where we can sequentially update our beliefs about the model parameters through data gathered in the experiment. A class of models of particular interest for the natural and medical sciences are implicit models, where the data generating distribution is intractable, but sampling from it is possible. Even though there has been a lot of work on static BED for implicit models in the past few years, the notoriously difficult problem of sequential BED for implicit models has barely been touched upon. We address this gap in the literature by devising a novel sequential design framework for parameter estimation that uses the Mutual Information (MI) between model parameters and simulated data as a utility function to find optimal experimental designs, which has not been done before for implicit models. Our approach uses likelihood-free inference by ratio estimation to simultaneously estimate posterior distributions and the MI. During the sequential BED procedure we utilise Bayesian optimisation to help us optimise the MI utility. We find that our framework is efficient for the various implicit models tested, yielding accurate parameter estimates after only a few iterations.},
	urldate = {2021-04-16},
	journal = {arXiv:2003.09379 [cs, stat]},
	author = {Kleinegesse, Steven and Drovandi, Christopher and Gutmann, Michael U.},
	month = mar,
	year = {2020},
	note = {arXiv: 2003.09379},
	keywords = {machine learning, statistics},
}

@inproceedings{NEURIPS2020_2adee881,
	title = {Sequential bayesian experimental design with variable cost structure},
	volume = {33},
	url = {https://proceedings.neurips.cc/paper/2020/file/2adee8815dd939548ee6b2772524b6f2-Paper.pdf},
	booktitle = {Advances in neural information processing systems},
	publisher = {Curran Associates, Inc.},
	author = {Zheng, Sue and Hayden, David and Pacheco, Jason and Fisher III, John W},
	editor = {Larochelle, H. and Ranzato, M. and Hadsell, R. and Balcan, M. F. and Lin, H.},
	year = {2020},
	pages = {4127--4137},
}

@article{song_implicit_2021,
	title = {Implicit {Kernel} {Attention}},
	url = {http://arxiv.org/abs/2006.06147},
	abstract = {{\textbackslash}textit\{Attention\} computes the dependency between representations, and it encourages the model to focus on the important selective features. Attention-based models, such as Transformer and graph attention network (GAT), are widely utilized for sequential data and graph-structured data. This paper suggests a new interpretation and generalized structure of the attention in Transformer and GAT. For the attention in Transformer and GAT, we derive that the attention is a product of two parts: 1) the RBF kernel to measure the similarity of two instances and 2) the exponential of \$L{\textasciicircum}\{2\}\$ norm to compute the importance of individual instances. From this decomposition, we generalize the attention in three ways. First, we propose implicit kernel attention with an implicit kernel function instead of manual kernel selection. Second, we generalize \$L{\textasciicircum}\{2\}\$ norm as the \$L{\textasciicircum}\{p\}\$ norm. Third, we extend our attention to structured multi-head attention. Our generalized attention shows better performance on classification, translation, and regression tasks.},
	urldate = {2021-04-16},
	journal = {arXiv:2006.06147 [cs, stat]},
	author = {Song, Kyungwoo and Jung, Yohan and Kim, Dongjun and Moon, Il-Chul},
	month = mar,
	year = {2021},
	note = {arXiv: 2006.06147},
	keywords = {machine learning},
}

@article{rad_theory-training_2021,
	title = {On {Theory}-training {Neural} {Networks} to {Infer} the {Solution} of {Highly} {Coupled} {Differential} {Equations}},
	url = {http://arxiv.org/abs/2102.04890},
	abstract = {Deep neural networks are transforming fields ranging from computer vision to computational medicine, and we recently extended their application to the field of phase-change heat transfer by introducing theory-trained neural networks (TTNs) for a solidification problem {\textbackslash}cite\{TTN\}. Here, we present general, in-depth, and empirical insights into theory-training networks for learning the solution of highly coupled differential equations. We analyze the deteriorating effects of the oscillating loss on the ability of a network to satisfy the equations at the training data points, measured by the final training loss, and on the accuracy of the inferred solution. We introduce a theory-training technique that, by leveraging regularization, eliminates those oscillations, decreases the final training loss, and improves the accuracy of the inferred solution, with no additional computational cost. Then, we present guidelines that allow a systematic search for the network that has the optimal training time and inference accuracy for a given set of equations; following these guidelines can reduce the number of tedious training iterations in that search. Finally, a comparison between theory-training and the rival, conventional method of solving differential equations using discretization attests to the advantages of theory-training not being necessarily limited to high-dimensional sets of equations. The comparison also reveals a limitation of the current theory-training framework that may limit its application in domains where extreme accuracies are necessary.},
	urldate = {2021-04-16},
	journal = {arXiv:2102.04890 [physics]},
	author = {Rad, M. Torabi and Viardin, A. and Apel, M.},
	month = feb,
	year = {2021},
	note = {arXiv: 2102.04890},
	keywords = {machine learning, physics},
}

@article{ma_mutual_2011,
	title = {Mutual {Information} {Is} {Copula} {Entropy}},
	volume = {16},
	issn = {1007-0214},
	url = {https://www.sciencedirect.com/science/article/pii/S1007021411700086},
	doi = {10.1016/S1007-0214(11)70008-6},
	abstract = {Mutual information (MI) is a basic concept in information theory. Therefore, estimates of the MI are fundamentally important in most information theory applications. This paper provides a new way of understanding and estimating the MI using the copula function. First, the entropy of the copula, named the copula entropy, is defined as a measure of the dependence uncertainty represented by the copula function and then the MI is shown to be equivalent to the negative copula entropy. With this equivalence, the MI can be estimated by first estimating the empirical copula and then estimating the entropy of the empirical copula. Thus, the MI estimate is an estimation of the entropy, which reduces the complexity and computational requirements. Tests show that the method is more effective than the traditional method.},
	language = {en},
	number = {1},
	urldate = {2021-04-16},
	journal = {Tsinghua Science \& Technology},
	author = {Ma, Jian and Sun, Zengqi},
	month = feb,
	year = {2011},
	keywords = {copulas, mutual information},
	pages = {51--54},
}

@article{chen_copula_2014,
	title = {Copula entropy coupled with artificial neural network for rainfall–runoff simulation},
	volume = {28},
	issn = {1436-3259},
	url = {https://doi.org/10.1007/s00477-013-0838-3},
	doi = {10.1007/s00477-013-0838-3},
	abstract = {The rainfall–runoff relationship is not only nonlinear and complex but also difficult to model. Artificial neural network (ANN), as a data-driven technique, has gained significant attention in recent years and has been shown to be an efficient alternative to traditional methods for hydrological modeling. However, for different input combinations, ANN models can yield different results. Therefore, input variables and ANN types need to be carefully considered, when using an ANN model for stream flow forecasting. This study proposes the copula-entropy (CE) theory to identify the inputs of an ANN model. The CE theory permits to calculate mutual information (MI) and partial MI directly which avoids calculating the marginal and joint probability distributions. Three different ANN models, namely multi-layer feed (MLF) forward networks, radial basis function networks and general regression neural network, were applied to predict stream flow of Jinsha River, China. Results showed that the inputs selected by the CE method were better than those by the traditional linear correlation analysis, and the MLF ANN model with the inputs selected by CE method obtained the best predicted results for the Jinsha River at Pingshan gauging station.},
	language = {en},
	number = {7},
	urldate = {2021-04-16},
	journal = {Stochastic Environmental Research and Risk Assessment},
	author = {Chen, Lu and Singh, Vijay P. and Guo, Shenglian and Zhou, Jianzhong and Ye, Lei},
	month = oct,
	year = {2014},
	pages = {1755--1767},
}

@inproceedings{rasul_multivariate_2020,
	title = {Multivariate {Probabilistic} {Time} {Series} {Forecasting} via {Conditioned} {Normalizing} {Flows}},
	url = {https://openreview.net/forum?id=WiGQBFuVRv},
	abstract = {Time series forecasting is often fundamental to scientific and engineering problems and enables decision making. With ever increasing data set sizes, a trivial solution to scale up predictions is...},
	language = {en},
	urldate = {2021-04-15},
	author = {Rasul, Kashif and Sheikh, Abdul-Saboor and Schuster, Ingmar and Bergmann, Urs M. and Vollgraf, Roland},
	month = sep,
	year = {2020},
}

@article{deng_modeling_2020,
	title = {Modeling {Continuous} {Stochastic} {Processes} with {Dynamic} {Normalizing} {Flows}},
	url = {http://arxiv.org/abs/2002.10516},
	abstract = {Normalizing flows transform a simple base distribution into a complex target distribution and have proved to be powerful models for data generation and density estimation. In this work, we propose a novel type of normalizing flow driven by a differential deformation of the Wiener process. As a result, we obtain a rich time series model whose observable process inherits many of the appealing properties of its base process, such as efficient computation of likelihoods and marginals. Furthermore, our continuous treatment provides a natural framework for irregular time series with an independent arrival process, including straightforward interpolation. We illustrate the desirable properties of the proposed model on popular stochastic processes and demonstrate its superior flexibility to variational RNN and latent ODE baselines in a series of experiments on synthetic and real-world data.},
	urldate = {2021-04-15},
	journal = {arXiv:2002.10516 [cs, stat]},
	author = {Deng, Ruizhi and Chang, Bo and Brubaker, Marcus A. and Mori, Greg and Lehrmann, Andreas},
	month = oct,
	year = {2020},
	note = {arXiv: 2002.10516},
	keywords = {machine learning},
}

@article{kirichenko_why_2020,
	title = {Why {Normalizing} {Flows} {Fail} to {Detect} {Out}-of-{Distribution} {Data}},
	volume = {33},
	url = {https://proceedings.neurips.cc//paper/2020/hash/ecb9fe2fbb99c31f567e9823e884dbec-Abstract.html},
	language = {en},
	urldate = {2021-04-15},
	journal = {Advances in Neural Information Processing Systems},
	author = {Kirichenko, Polina and Izmailov, Pavel and Wilson, Andrew G.},
	year = {2020},
	pages = {20578--20589},
}

@article{bhattacharyya_haar_2020,
	title = {Haar {Wavelet} based {Block} {Autoregressive} {Flows} for {Trajectories}},
	url = {http://arxiv.org/abs/2009.09878},
	abstract = {Prediction of trajectories such as that of pedestrians is crucial to the performance of autonomous agents. While previous works have leveraged conditional generative models like GANs and VAEs for learning the likely future trajectories, accurately modeling the dependency structure of these multimodal distributions, particularly over long time horizons remains challenging. Normalizing flow based generative models can model complex distributions admitting exact inference. These include variants with split coupling invertible transformations that are easier to parallelize compared to their autoregressive counterparts. To this end, we introduce a novel Haar wavelet based block autoregressive model leveraging split couplings, conditioned on coarse trajectories obtained from Haar wavelet based transformations at different levels of granularity. This yields an exact inference method that models trajectories at different spatio-temporal resolutions in a hierarchical manner. We illustrate the advantages of our approach for generating diverse and accurate trajectories on two real-world datasets - Stanford Drone and Intersection Drone.},
	urldate = {2021-04-15},
	journal = {arXiv:2009.09878 [cs]},
	author = {Bhattacharyya, Apratim and Straehle, Christoph-Nikolas and Fritz, Mario and Schiele, Bernt},
	month = sep,
	year = {2020},
	note = {arXiv: 2009.09878},
	keywords = {computer vision, machine learning},
}

@inproceedings{wehenkel_graphical_2021,
	title = {Graphical {Normalizing} {Flows}},
	url = {http://proceedings.mlr.press/v130/wehenkel21a.html},
	abstract = {Normalizing flows model complex probability distributions by combining a base distribution with a series of bijective neural networks. State-of-the-art architectures rely on coupling and autoregre...},
	language = {en},
	urldate = {2021-04-15},
	booktitle = {International {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Wehenkel, Antoine and Louppe, Gilles},
	month = mar,
	year = {2021},
	note = {ISSN: 2640-3498},
	pages = {37--45},
}

@article{pathak_hybrid_2018,
	title = {Hybrid {Forecasting} of {Chaotic} {Processes}: {Using} {Machine} {Learning} in {Conjunction} with a {Knowledge}-{Based} {Model}},
	volume = {28},
	issn = {1054-1500, 1089-7682},
	shorttitle = {Hybrid {Forecasting} of {Chaotic} {Processes}},
	url = {http://arxiv.org/abs/1803.04779},
	doi = {10.1063/1.5028373},
	abstract = {A model-based approach to forecasting chaotic dynamical systems utilizes knowledge of the physical processes governing the dynamics to build an approximate mathematical model of the system. In contrast, machine learning techniques have demonstrated promising results for forecasting chaotic systems purely from past time series measurements of system state variables (training data), without prior knowledge of the system dynamics. The motivation for this paper is the potential of machine learning for filling in the gaps in our underlying mechanistic knowledge that cause widely-used knowledge-based models to be inaccurate. Thus we here propose a general method that leverages the advantages of these two approaches by combining a knowledge-based model and a machine learning technique to build a hybrid forecasting scheme. Potential applications for such an approach are numerous (e.g., improving weather forecasting). We demonstrate and test the utility of this approach using a particular illustrative version of a machine learning known as reservoir computing, and we apply the resulting hybrid forecaster to a low-dimensional chaotic system, as well as to a high-dimensional spatiotemporal chaotic system. These tests yield extremely promising results in that our hybrid technique is able to accurately predict for a much longer period of time than either its machine-learning component or its model-based component alone.},
	number = {4},
	urldate = {2021-04-14},
	journal = {Chaos: An Interdisciplinary Journal of Nonlinear Science},
	author = {Pathak, Jaideep and Wikner, Alexander and Fussell, Rebeckah and Chandra, Sarthak and Hunt, Brian and Girvan, Michelle and Ott, Edward},
	month = apr,
	year = {2018},
	note = {arXiv: 1803.04779},
	keywords = {chaos, machine learning},
	pages = {041101},
}

@inproceedings{kleinegesse_bayesian_2020,
	title = {Bayesian {Experimental} {Design} for {Implicit} {Models} by {Mutual} {Information} {Neural} {Estimation}},
	url = {http://proceedings.mlr.press/v119/kleinegesse20a.html},
	abstract = {Implicit stochastic models, where the data-generation distribution is intractable but sampling is possible, are ubiquitous in the natural sciences. The models typically have free parameters that ne...},
	language = {en},
	urldate = {2021-04-14},
	booktitle = {International {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Kleinegesse, Steven and Gutmann, Michael U.},
	month = nov,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {5316--5326},
}

@article{yang_b-pinns_2020,
	title = {B-{PINNs}: {Bayesian} {Physics}-{Informed} {Neural} {Networks} for {Forward} and {Inverse} {PDE} {Problems} with {Noisy} {Data}},
	shorttitle = {B-{PINNs}},
	url = {http://arxiv.org/abs/2003.06097},
	doi = {10.1016/j.jcp.2020.109913},
	abstract = {We propose a Bayesian physics-informed neural network (B-PINN) to solve both forward and inverse nonlinear problems described by partial differential equations (PDEs) and noisy data. In this Bayesian framework, the Bayesian neural network (BNN) combined with a PINN for PDEs serves as the prior while the Hamiltonian Monte Carlo (HMC) or the variational inference (VI) could serve as an estimator of the posterior. B-PINNs make use of both physical laws and scattered noisy measurements to provide predictions and quantify the aleatoric uncertainty arising from the noisy data in the Bayesian framework. Compared with PINNs, in addition to uncertainty quantification, B-PINNs obtain more accurate predictions in scenarios with large noise due to their capability of avoiding overfitting. We conduct a systematic comparison between the two different approaches for the B-PINN posterior estimation (i.e., HMC or VI), along with dropout used for quantifying uncertainty in deep neural networks. Our experiments show that HMC is more suitable than VI for the B-PINNs posterior estimation, while dropout employed in PINNs can hardly provide accurate predictions with reasonable uncertainty. Finally, we replace the BNN in the prior with a truncated Karhunen-Lo{\textbackslash}`eve (KL) expansion combined with HMC or a deep normalizing flow (DNF) model as posterior estimators. The KL is as accurate as BNN and much faster but this framework cannot be easily extended to high-dimensional problems unlike the BNN based framework.},
	urldate = {2021-04-14},
	journal = {arXiv:2003.06097 [cs, stat]},
	author = {Yang, Liu and Meng, Xuhui and Karniadakis, George Em},
	month = mar,
	year = {2020},
	note = {arXiv: 2003.06097},
	keywords = {machine learning},
}

@article{smith_bayesian_2013,
	title = {Bayesian {Approaches} to {Copula} {Modelling}},
	url = {http://arxiv.org/abs/1112.4204},
	doi = {10.1093/acprof:oso/9780199695607.001.0001},
	abstract = {Copula models have become one of the most widely used tools in the applied modelling of multivariate data. Similarly, Bayesian methods are increasingly used to obtain efficient likelihood-based inference. However, to date, there has been only limited use of Bayesian approaches in the formulation and estimation of copula models. This article aims to address this shortcoming in two ways. First, to introduce copula models and aspects of copula theory that are especially relevant for a Bayesian analysis. Second, to outline Bayesian approaches to formulating and estimating copula models, and their advantages over alternative methods. Copulas covered include Archimedean, copulas constructed by inversion, and vine copulas; along with their interpretation as transformations. A number of parameterisations of a correlation matrix of a Gaussian copula are considered, along with hierarchical priors that allow for Bayesian selection and model averaging for each parameterisation. Markov chain Monte Carlo sampling schemes for fitting Gaussian and D-vine copulas, with and without selection, are given in detail. The relationship between the prior for the parameters of a D-vine, and the prior for a correlation matrix of a Gaussian copula, is discussed. Last, it is shown how to compute Bayesian inference when the data are discrete-valued using data augmentation. This approach generalises popular Bayesian methods for the estimation of models for multivariate binary and other ordinal data to more general copula models. Bayesian data augmentation has substantial advantages over other methods of estimation for this class of models.},
	urldate = {2021-04-14},
	journal = {arXiv:1112.4204 [stat]},
	author = {Smith, Michael Stanley},
	month = jan,
	year = {2013},
	note = {arXiv: 1112.4204},
	keywords = {statistics},
}

@article{trippe_conditional_2018,
	title = {Conditional {Density} {Estimation} with {Bayesian} {Normalising} {Flows}},
	url = {http://arxiv.org/abs/1802.04908},
	abstract = {Modeling complex conditional distributions is critical in a variety of settings. Despite a long tradition of research into conditional density estimation, current methods employ either simple parametric forms or are difficult to learn in practice. This paper employs normalising flows as a flexible likelihood model and presents an efficient method for fitting them to complex densities. These estimators must trade-off between modeling distributional complexity, functional complexity and heteroscedasticity without overfitting. We recognize these trade-offs as modeling decisions and develop a Bayesian framework for placing priors over these conditional density estimators using variational Bayesian neural networks. We evaluate this method on several small benchmark regression datasets, on some of which it obtains state of the art performance. Finally, we apply the method to two spatial density modeling tasks with over 1 million datapoints using the New York City yellow taxi dataset and the Chicago crime dataset.},
	urldate = {2021-04-14},
	journal = {arXiv:1802.04908 [stat]},
	author = {Trippe, Brian L. and Turner, Richard E.},
	month = feb,
	year = {2018},
	note = {arXiv: 1802.04908},
	keywords = {machine learning},
}

@article{karpatne_theory-guided_2017,
	title = {Theory-guided {Data} {Science}: {A} {New} {Paradigm} for {Scientific} {Discovery} from {Data}},
	volume = {29},
	issn = {1041-4347},
	shorttitle = {Theory-guided {Data} {Science}},
	url = {http://arxiv.org/abs/1612.08544},
	doi = {10.1109/TKDE.2017.2720168},
	abstract = {Data science models, although successful in a number of commercial domains, have had limited applicability in scientific problems involving complex physical phenomena. Theory-guided data science (TGDS) is an emerging paradigm that aims to leverage the wealth of scientific knowledge for improving the effectiveness of data science models in enabling scientific discovery. The overarching vision of TGDS is to introduce scientific consistency as an essential component for learning generalizable models. Further, by producing scientifically interpretable models, TGDS aims to advance our scientific understanding by discovering novel domain insights. Indeed, the paradigm of TGDS has started to gain prominence in a number of scientific disciplines such as turbulence modeling, material discovery, quantum chemistry, bio-medical science, bio-marker discovery, climate science, and hydrology. In this paper, we formally conceptualize the paradigm of TGDS and present a taxonomy of research themes in TGDS. We describe several approaches for integrating domain knowledge in different research themes using illustrative examples from different disciplines. We also highlight some of the promising avenues of novel research for realizing the full potential of theory-guided data science.},
	number = {10},
	urldate = {2021-04-13},
	journal = {IEEE Transactions on Knowledge and Data Engineering},
	author = {Karpatne, Anuj and Atluri, Gowtham and Faghmous, James and Steinbach, Michael and Banerjee, Arindam and Ganguly, Auroop and Shekhar, Shashi and Samatova, Nagiza and Kumar, Vipin},
	month = oct,
	year = {2017},
	note = {arXiv: 1612.08544},
	keywords = {machine learning},
	pages = {2318--2331},
}

@article{grathwohl_your_2020,
	title = {Your {Classifier} is {Secretly} an {Energy} {Based} {Model} and {You} {Should} {Treat} it {Like} {One}},
	url = {http://arxiv.org/abs/1912.03263},
	abstract = {We propose to reinterpret a standard discriminative classifier of p(y{\textbar}x) as an energy based model for the joint distribution p(x,y). In this setting, the standard class probabilities can be easily computed as well as unnormalized values of p(x) and p(x{\textbar}y). Within this framework, standard discriminative architectures may beused and the model can also be trained on unlabeled data. We demonstrate that energy based training of the joint distribution improves calibration, robustness, andout-of-distribution detection while also enabling our models to generate samplesrivaling the quality of recent GAN approaches. We improve upon recently proposed techniques for scaling up the training of energy based models and presentan approach which adds little overhead compared to standard classification training. Our approach is the first to achieve performance rivaling the state-of-the-artin both generative and discriminative learning within one hybrid model.},
	urldate = {2021-04-13},
	journal = {arXiv:1912.03263 [cs, stat]},
	author = {Grathwohl, Will and Wang, Kuan-Chieh and Jacobsen, Jörn-Henrik and Duvenaud, David and Norouzi, Mohammad and Swersky, Kevin},
	month = sep,
	year = {2020},
	note = {arXiv: 1912.03263},
	keywords = {computer vision, machine learning},
}

@article{raissi_machine_2017,
	title = {Machine {Learning} of {Linear} {Differential} {Equations} using {Gaussian} {Processes}},
	volume = {348},
	issn = {00219991},
	url = {http://arxiv.org/abs/1701.02440},
	doi = {10.1016/j.jcp.2017.07.050},
	abstract = {This work leverages recent advances in probabilistic machine learning to discover conservation laws expressed by parametric linear equations. Such equations involve, but are not limited to, ordinary and partial differential, integro-differential, and fractional order operators. Here, Gaussian process priors are modified according to the particular form of such operators and are employed to infer parameters of the linear equations from scarce and possibly noisy observations. Such observations may come from experiments or "black-box" computer simulations.},
	urldate = {2021-04-13},
	journal = {Journal of Computational Physics},
	author = {Raissi, Maziar and Karniadakis, George Em},
	month = nov,
	year = {2017},
	note = {arXiv: 1701.02440},
	keywords = {machine learning, numerical modelling},
	pages = {683--693},
}

@article{duffin_statistical_2021,
	title = {Statistical finite elements for misspecified models},
	volume = {118},
	copyright = {© 2021 . https://creativecommons.org/licenses/by/4.0/This open access article is distributed under Creative Commons Attribution License 4.0 (CC BY).},
	issn = {0027-8424, 1091-6490},
	url = {https://www.pnas.org/content/118/2/e2015006118},
	doi = {10.1073/pnas.2015006118},
	abstract = {We present a statistical finite element method for nonlinear, time-dependent phenomena, illustrated in the context of nonlinear internal waves (solitons). We take a Bayesian approach and leverage the finite element method to cast the statistical problem as a nonlinear Gaussian state–space model, updating the solution, in receipt of data, in a filtering framework. The method is applicable to problems across science and engineering for which finite element methods are appropriate. The Korteweg–de Vries equation for solitons is presented because it reflects the necessary complexity while being suitably familiar and succinct for pedagogical purposes. We present two algorithms to implement this method, based on the extended and ensemble Kalman filters, and demonstrate effectiveness with a simulation study and a case study with experimental data. The generality of our approach is demonstrated in SI Appendix, where we present examples from additional nonlinear, time-dependent partial differential equations (Burgers equation, Kuramoto–Sivashinsky equation).},
	language = {en},
	number = {2},
	urldate = {2021-04-13},
	journal = {Proceedings of the National Academy of Sciences},
	author = {Duffin, Connor and Cripps, Edward and Stemler, Thomas and Girolami, Mark},
	month = jan,
	year = {2021},
	pmid = {33372139},
	note = {Publisher: National Academy of Sciences
Section: Physical Sciences},
	keywords = {finite-element methods},
}

@article{kashinath_physics-informed_2021,
	title = {Physics-informed machine learning: case studies for weather and climate modelling},
	volume = {379},
	shorttitle = {Physics-informed machine learning},
	url = {https://royalsocietypublishing.org/doi/10.1098/rsta.2020.0093},
	doi = {10.1098/rsta.2020.0093},
	abstract = {Machine learning (ML) provides novel and powerful ways of accurately and efficiently recognizing complex patterns, emulating nonlinear dynamics, and predicting the spatio-temporal evolution of weather and climate processes. Off-the-shelf ML models, however, do not necessarily obey the fundamental governing laws of physical systems, nor do they generalize well to scenarios on which they have not been trained. We survey systematic approaches to incorporating physics and domain knowledge into ML models and distill these approaches into broad categories. Through 10 case studies, we show how these approaches have been used successfully for emulating, downscaling, and forecasting weather and climate processes. The accomplishments of these studies include greater physical consistency, reduced training time, improved data efficiency, and better generalization. Finally, we synthesize the lessons learned and identify scientific, diagnostic, computational, and resource challenges for developing truly robust and reliable physics-informed ML models for weather and climate processes.This article is part of the theme issue ‘Machine learning for weather and climate modelling’.},
	number = {2194},
	urldate = {2021-04-13},
	journal = {Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences},
	author = {Kashinath, K. and Mustafa, M. and Albert, A. and Wu, J-L. and Jiang, C. and Esmaeilzadeh, S. and Azizzadenesheli, K. and Wang, R. and Chattopadhyay, A. and Singh, A. and Manepalli, A. and Chirila, D. and Yu, R. and Walters, R. and White, B. and Xiao, H. and Tchelepi, H. A. and Marcus, P. and Anandkumar, A. and Hassanzadeh, P. and Prabhat, null},
	month = apr,
	year = {2021},
	note = {Publisher: Royal Society},
	pages = {20200093},
}

@inproceedings{chilinski_neural_2020,
	title = {Neural {Likelihoods} via {Cumulative} {Distribution} {Functions}},
	url = {http://proceedings.mlr.press/v124/chilinski20a.html},
	abstract = {We leverage neural networks as universal approximators of monotonic functions to build a parameterization of conditional cumulative distribution functions (CDFs). By the application of automatic di...},
	language = {en},
	urldate = {2021-04-09},
	booktitle = {Conference on {Uncertainty} in {Artificial} {Intelligence}},
	publisher = {PMLR},
	author = {Chilinski, Pawel and Silva, Ricardo},
	month = aug,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {420--429},
}

@inproceedings{kleinegesse_bayesian_2020-1,
	title = {Bayesian {Experimental} {Design} for {Implicit} {Models} by {Mutual} {Information} {Neural} {Estimation}},
	url = {http://proceedings.mlr.press/v119/kleinegesse20a.html},
	abstract = {Implicit stochastic models, where the data-generation distribution is intractable but sampling is possible, are ubiquitous in the natural sciences. The models typically have free parameters that ne...},
	language = {en},
	urldate = {2021-04-07},
	booktitle = {International {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Kleinegesse, Steven and Gutmann, Michael U.},
	month = nov,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {5316--5326},
}

@article{grathwohl_ffjord_2018,
	title = {{FFJORD}: {Free}-form {Continuous} {Dynamics} for {Scalable} {Reversible} {Generative} {Models}},
	shorttitle = {{FFJORD}},
	url = {http://arxiv.org/abs/1810.01367},
	abstract = {A promising class of generative models maps points from a simple distribution to a complex distribution through an invertible neural network. Likelihood-based training of these models requires restricting their architectures to allow cheap computation of Jacobian determinants. Alternatively, the Jacobian trace can be used if the transformation is specified by an ordinary differential equation. In this paper, we use Hutchinson's trace estimator to give a scalable unbiased estimate of the log-density. The result is a continuous-time invertible generative model with unbiased density estimation and one-pass sampling, while allowing unrestricted neural network architectures. We demonstrate our approach on high-dimensional density estimation, image generation, and variational inference, achieving the state-of-the-art among exact likelihood methods with efficient sampling.},
	urldate = {2021-04-07},
	journal = {arXiv:1810.01367 [cs, stat]},
	author = {Grathwohl, Will and Chen, Ricky T. Q. and Bettencourt, Jesse and Sutskever, Ilya and Duvenaud, David},
	month = oct,
	year = {2018},
	note = {arXiv: 1810.01367},
	keywords = {computer vision, machine learning},
}

@article{kobyzev_normalizing_2020,
	title = {Normalizing {Flows}: {An} {Introduction} and {Review} of {Current} {Methods}},
	issn = {0162-8828, 2160-9292, 1939-3539},
	shorttitle = {Normalizing {Flows}},
	url = {http://arxiv.org/abs/1908.09257},
	doi = {10.1109/TPAMI.2020.2992934},
	abstract = {Normalizing Flows are generative models which produce tractable distributions where both sampling and density evaluation can be efficient and exact. The goal of this survey article is to give a coherent and comprehensive review of the literature around the construction and use of Normalizing Flows for distribution learning. We aim to provide context and explanation of the models, review current state-of-the-art literature, and identify open questions and promising future directions.},
	urldate = {2021-04-07},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	author = {Kobyzev, Ivan and Prince, Simon J. D. and Brubaker, Marcus A.},
	year = {2020},
	note = {arXiv: 1908.09257},
	keywords = {machine learning},
	pages = {1--1},
}

@inproceedings{samo_inductive_2021,
	title = {Inductive {Mutual} {Information} {Estimation}: {A} {Convex} {Maximum}-{Entropy} {Copula} {Approach}},
	shorttitle = {Inductive {Mutual} {Information} {Estimation}},
	url = {http://proceedings.mlr.press/v130/kom-samo21a.html},
	abstract = {We propose a novel estimator of the mutual information between two ordinal vectors \$x\$ and \$y\$. Our approach is inductive (as opposed to deductive) in that it depends on the data generating distri...},
	language = {en},
	urldate = {2021-04-07},
	booktitle = {International {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Samo, Yves-Laurent Kom},
	month = mar,
	year = {2021},
	note = {ISSN: 2640-3498},
	pages = {2242--2250},
}

@article{ma_copulagnn_2021,
	title = {{CopulaGNN}: {Towards} {Integrating} {Representational} and {Correlational} {Roles} of {Graphs} in {Graph} {Neural} {Networks}},
	shorttitle = {{CopulaGNN}},
	url = {http://arxiv.org/abs/2010.02089},
	abstract = {Graph-structured data are ubiquitous. However, graphs encode diverse types of information and thus play different roles in data representation. In this paper, we distinguish the {\textbackslash}textit\{representational\} and the {\textbackslash}textit\{correlational\} roles played by the graphs in node-level prediction tasks, and we investigate how Graph Neural Network (GNN) models can effectively leverage both types of information. Conceptually, the representational information provides guidance for the model to construct better node features; while the correlational information indicates the correlation between node outcomes conditional on node features. Through a simulation study, we find that many popular GNN models are incapable of effectively utilizing the correlational information. By leveraging the idea of the copula, a principled way to describe the dependence among multivariate random variables, we offer a general solution. The proposed Copula Graph Neural Network (CopulaGNN) can take a wide range of GNN models as base models and utilize both representational and correlational information stored in the graphs. Experimental results on two types of regression tasks verify the effectiveness of the proposed method.},
	urldate = {2021-04-07},
	journal = {arXiv:2010.02089 [cs, stat]},
	author = {Ma, Jiaqi and Chang, Bo and Zhang, Xuefei and Mei, Qiaozhu},
	month = mar,
	year = {2021},
	note = {arXiv: 2010.02089},
	keywords = {machine learning},
}

@inproceedings{chen_adaptive_2019,
	title = {Adaptive {Gaussian} {Copula} {ABC}},
	url = {http://proceedings.mlr.press/v89/chen19d.html},
	abstract = {Approximate Bayesian computation (ABC) is a set of techniques for Bayesian inference when the likelihood is intractable but sampling from the model is possible. This work presents a simple yet effe...},
	language = {en},
	urldate = {2021-04-07},
	booktitle = {The 22nd {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Chen, Yanzhi and Gutmann, Michael U.},
	month = apr,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {1584--1592},
}

@article{wen_deep_2019,
	title = {Deep {Generative} {Quantile}-{Copula} {Models} for {Probabilistic} {Forecasting}},
	url = {http://arxiv.org/abs/1907.10697},
	abstract = {We introduce a new category of multivariate conditional generative models and demonstrate its performance and versatility in probabilistic time series forecasting and simulation. Specifically, the output of quantile regression networks is expanded from a set of fixed quantiles to the whole Quantile Function by a univariate mapping from a latent uniform distribution to the target distribution. Then the multivariate case is solved by learning such quantile functions for each dimension's marginal distribution, followed by estimating a conditional Copula to associate these latent uniform random variables. The quantile functions and copula, together defining the joint predictive distribution, can be parameterized by a single implicit generative Deep Neural Network.},
	urldate = {2021-04-06},
	journal = {arXiv:1907.10697 [cs, stat]},
	author = {Wen, Ruofeng and Torkkola, Kari},
	month = jul,
	year = {2019},
	note = {arXiv: 1907.10697},
	keywords = {machine learning},
}

@inproceedings{gasthaus_probabilistic_2019,
	title = {Probabilistic {Forecasting} with {Spline} {Quantile} {Function} {RNNs}},
	url = {http://proceedings.mlr.press/v89/gasthaus19a.html},
	abstract = {In this paper, we propose a flexible method for probabilistic modeling with conditional quantile functions using monotonic regression splines. The shape of the spline is parameterized by a neural n...},
	language = {en},
	urldate = {2021-04-06},
	booktitle = {The 22nd {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Gasthaus, Jan and Benidis, Konstantinos and Wang, Yuyang and Rangapuram, Syama Sundar and Salinas, David and Flunkert, Valentin and Januschowski, Tim},
	month = apr,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {1901--1910},
}

@inproceedings{song_score-based_2020,
	title = {Score-{Based} {Generative} {Modeling} through {Stochastic} {Differential} {Equations}},
	url = {https://openreview.net/forum?id=PxTIG12RRHS},
	abstract = {Creating noise from data is easy; creating data from noise is generative modeling. We present a stochastic differential equation (SDE) that smoothly transforms a complex data distribution to a...},
	language = {en},
	urldate = {2021-04-05},
	author = {Song, Yang and Sohl-Dickstein, Jascha and Kingma, Diederik P. and Kumar, Abhishek and Ermon, Stefano and Poole, Ben},
	month = sep,
	year = {2020},
}

@article{papamakarios_normalizing_2019,
	title = {Normalizing {Flows} for {Probabilistic} {Modeling} and {Inference}},
	url = {http://arxiv.org/abs/1912.02762},
	abstract = {Normalizing flows provide a general mechanism for defining expressive probability distributions, only requiring the specification of a (usually simple) base distribution and a series of bijective transformations. There has been much recent work on normalizing flows, ranging from improving their expressive power to expanding their application. We believe the field has now matured and is in need of a unified perspective. In this review, we attempt to provide such a perspective by describing flows through the lens of probabilistic modeling and inference. We place special emphasis on the fundamental principles of flow design, and discuss foundational topics such as expressive power and computational trade-offs. We also broaden the conceptual framing of flows by relating them to more general probability transformations. Lastly, we summarize the use of flows for tasks such as generative modeling, approximate inference, and supervised learning.},
	urldate = {2021-04-05},
	journal = {arXiv:1912.02762 [cs, stat]},
	author = {Papamakarios, George and Nalisnick, Eric and Rezende, Danilo Jimenez and Mohamed, Shakir and Lakshminarayanan, Balaji},
	month = dec,
	year = {2019},
	note = {arXiv: 1912.02762},
	keywords = {machine learning},
}

@article{toubeau_deep_2019,
	title = {Deep {Learning}-{Based} {Multivariate} {Probabilistic} {Forecasting} for {Short}-{Term} {Scheduling} in {Power} {Markets}},
	volume = {34},
	issn = {1558-0679},
	doi = {10.1109/TPWRS.2018.2870041},
	abstract = {In the current competition framework governing the electricity sector, complex dependencies exist between electrical and market data, which complicates the decision-making procedure of energy actors. These must indeed operate within a complex, uncertain environment, and consequently need to rely on accurate multivariate, multi-step ahead probabilistic predictions. This paper aims to take advantage of recent breakthroughs in deep learning, while exploiting the structure of the problem to design prediction tools with tailored architectural alterations that improve their performance. The method can provide prediction intervals and densities, but is here extended with the objective to generate predictive scenarios. It is achieved by sampling the predicted multivariate distribution with a copula-based strategy so as to embody both temporal information and cross-variable dependencies. The effectiveness of the proposed methodology is emphasized and compared with several other architectures in terms of both statistical performance and impact on the quality of decisions optimized within a dedicated stochastic optimization tool of an electricity retailer participating in short-term electricity markets.},
	number = {2},
	journal = {IEEE Transactions on Power Systems},
	author = {Toubeau, J. and Bottieau, J. and Vallée, F. and Grève, Z. De},
	month = mar,
	year = {2019},
	note = {Conference Name: IEEE Transactions on Power Systems},
	keywords = {copulas, lstms, optimization, scenario generation, uncertainty estimation},
	pages = {1203--1215},
}

@article{wiese_copula_2019,
	title = {Copula \& {Marginal} {Flows}: {Disentangling} the {Marginal} from its {Joint}},
	shorttitle = {Copula \& {Marginal} {Flows}},
	url = {http://arxiv.org/abs/1907.03361},
	abstract = {Deep generative networks such as GANs and normalizing flows flourish in the context of high-dimensional tasks such as image generation. However, so far exact modeling or extrapolation of distributional properties such as the tail asymptotics generated by a generative network is not available. In this paper, we address this issue for the first time in the deep learning literature by making two novel contributions. First, we derive upper bounds for the tails that can be expressed by a generative network and demonstrate Lp-space related properties. There we show specifically that in various situations an optimal generative network does not exist. Second, we introduce and propose copula and marginal generative flows (CM flows) which allow for an exact modeling of the tail and any prior assumption on the CDF up to an approximation of the uniform distribution. Our numerical results support the use of CM flows.},
	urldate = {2021-04-01},
	journal = {arXiv:1907.03361 [cs, stat]},
	author = {Wiese, Magnus and Knobloch, Robert and Korn, Ralf},
	month = jul,
	year = {2019},
	note = {arXiv: 1907.03361},
	keywords = {machine learning},
}

@article{liu_copula_2019,
	title = {Copula {Multi}-label {Learning}},
	volume = {32},
	url = {https://papers.nips.cc/paper/2019/hash/5d2c2cee8ab0b9a36bd1ed7196bd6c4a-Abstract.html},
	language = {en},
	urldate = {2021-04-01},
	journal = {Advances in Neural Information Processing Systems},
	author = {Liu, Weiwei},
	year = {2019},
}

@article{zhao_matrix_2020,
	title = {Matrix {Completion} with {Quantified} {Uncertainty} through {Low} {Rank} {Gaussian} {Copula}},
	volume = {33},
	url = {https://papers.nips.cc/paper/2020/hash/f076073b2082f8741a9cd07b789c77a0-Abstract.html},
	language = {en},
	urldate = {2021-04-01},
	journal = {Advances in Neural Information Processing Systems},
	author = {Zhao, Yuxuan and Udell, Madeleine},
	year = {2020},
	pages = {20977--20988},
}

@article{salinas_high-dimensional_2019,
	title = {High-dimensional multivariate forecasting with low-rank {Gaussian} {Copula} {Processes}},
	volume = {32},
	url = {https://papers.nips.cc/paper/2019/hash/0b105cf1504c4e241fcc6d519ea962fb-Abstract.html},
	language = {en},
	urldate = {2021-04-01},
	journal = {Advances in Neural Information Processing Systems},
	author = {Salinas, David and Bohlke-Schneider, Michael and Callot, Laurent and Medico, Roberto and Gasthaus, Jan},
	year = {2019},
}

@article{ling_deep_2020,
	title = {Deep {Archimedean} {Copulas}},
	url = {http://arxiv.org/abs/2012.03137},
	abstract = {A central problem in machine learning and statistics is to model joint densities of random variables from data. Copulas are joint cumulative distribution functions with uniform marginal distributions and are used to capture interdependencies in isolation from marginals. Copulas are widely used within statistics, but have not gained traction in the context of modern deep learning. In this paper, we introduce ACNet, a novel differentiable neural network architecture that enforces structural properties and enables one to learn an important class of copulas--Archimedean Copulas. Unlike Generative Adversarial Networks, Variational Autoencoders, or Normalizing Flow methods, which learn either densities or the generative process directly, ACNet learns a generator of the copula, which implicitly defines the cumulative distribution function of a joint distribution. We give a probabilistic interpretation of the network parameters of ACNet and use this to derive a simple but efficient sampling algorithm for the learned copula. Our experiments show that ACNet is able to both approximate common Archimedean Copulas and generate new copulas which may provide better fits to data.},
	urldate = {2021-04-01},
	journal = {arXiv:2012.03137 [cs]},
	author = {Ling, Chun Kai and Fang, Fei and Kolter, J. Zico},
	month = dec,
	year = {2020},
	note = {arXiv: 2012.03137},
	keywords = {machine learning},
}

@inproceedings{lin_copula_2020,
	address = {New York, NY, USA},
	series = {{SIGIR} '20},
	title = {Copula {Guided} {Neural} {Topic} {Modelling} for {Short} {Texts}},
	isbn = {978-1-4503-8016-4},
	url = {https://doi.org/10.1145/3397271.3401245},
	doi = {10.1145/3397271.3401245},
	abstract = {Extracting the topical information from documents is important for public opinion analysis, text classification, and information retrieval tasks. Compared with identifying a wide variety of topics from long documents, it is challenging to generate a concentrated topic distribution for each short message. Although this problem can be tackled by adjusting the hyper-parameters in traditional topic models such as Latent Dirichlet Allocation, it remains an open problem in neural topic modelling. In this paper, we focus on adapting the popular Auto-Encoding Variational Bayes based neural topic models to short texts, by exploring the Archimedean copulas to guide the estimated topic distributions derived from linear projected samples of re-parameterized posterior distributions. Experimental results show the superiority of our method when compared with existing neural topic models in terms of perplexity, topic coherence, and classification accuracy.},
	urldate = {2021-04-01},
	booktitle = {Proceedings of the 43rd {International} {ACM} {SIGIR} {Conference} on {Research} and {Development} in {Information} {Retrieval}},
	publisher = {Association for Computing Machinery},
	author = {Lin, Lihui and Jiang, Hongyu and Rao, Yanghui},
	month = jul,
	year = {2020},
	keywords = {copulas, variational autoencoders},
	pages = {1773--1776},
}

@article{tagasovska_copulas_2019,
	title = {Copulas as {High}-{Dimensional} {Generative} {Models}: {Vine} {Copula} {Autoencoders}},
	shorttitle = {Copulas as {High}-{Dimensional} {Generative} {Models}},
	url = {http://arxiv.org/abs/1906.05423},
	abstract = {We introduce the vine copula autoencoder (VCAE), a flexible generative model for high-dimensional distributions built in a straightforward three-step procedure. First, an autoencoder (AE) compresses the data into a lower dimensional representation. Second, the multivariate distribution of the encoded data is estimated with vine copulas. Third, a generative model is obtained by combining the estimated distribution with the decoder part of the AE. As such, the proposed approach can transform any already trained AE into a flexible generative model at a low computational cost. This is an advantage over existing generative models such as adversarial networks and variational AEs which can be difficult to train and can impose strong assumptions on the latent space. Experiments on MNIST, Street View House Numbers and Large-Scale CelebFaces Attributes datasets show that VCAEs can achieve competitive results to standard baselines.},
	urldate = {2021-04-01},
	journal = {arXiv:1906.05423 [cs, stat]},
	author = {Tagasovska, Natasa and Ackerer, Damien and Vatter, Thibault},
	month = nov,
	year = {2019},
	note = {arXiv: 1906.05423},
	keywords = {computer vision, machine learning},
}

@article{tran_copula_2018,
	title = {Copula {Variational} {Bayes} inference via information geometry},
	url = {http://arxiv.org/abs/1803.10998},
	abstract = {Variational Bayes (VB), also known as independent mean-field approximation, has become a popular method for Bayesian network inference in recent years. Its application is vast, e.g. in neural network, compressed sensing, clustering, etc. to name just a few. In this paper, the independence constraint in VB will be relaxed to a conditional constraint class, called copula in statistics. Since a joint probability distribution always belongs to a copula class, the novel copula VB (CVB) approximation is a generalized form of VB. Via information geometry, we will see that CVB algorithm iteratively projects the original joint distribution to a copula constraint space until it reaches a local minimum Kullback-Leibler (KL) divergence. By this way, all mean-field approximations, e.g. iterative VB, Expectation-Maximization (EM), Iterated Conditional Mode (ICM) and k-means algorithms, are special cases of CVB approximation. For a generic Bayesian network, an augmented hierarchy form of CVB will also be designed. While mean-field algorithms can only return a locally optimal approximation for a correlated network, the augmented CVB network, which is an optimally weighted average of a mixture of simpler network structures, can potentially achieve the globally optimal approximation for the first time. Via simulations of Gaussian mixture clustering, the classification's accuracy of CVB will be shown to be far superior to that of state-of-the-art VB, EM and k-means algorithms.},
	urldate = {2021-04-01},
	journal = {arXiv:1803.10998 [cs, math, stat]},
	author = {Tran, Viet Hung},
	month = mar,
	year = {2018},
	note = {arXiv: 1803.10998},
	keywords = {information theory, machine learning},
}

@article{philippe_mixed_2020,
	title = {Mixed {Copula}-{Based} {Uncertainty} {Modeling} of {Hourly} {Wind} {Farm} {Production} for {Power} {System} {Operational} {Planning} {Studies}},
	volume = {8},
	issn = {2169-3536},
	doi = {10.1109/ACCESS.2020.3012437},
	abstract = {An effective model to represent real-world wind power production scenarios is essential for an accurate assessment of the impact of wind power generation on power systems. Such a model should capture the spatial and temporal correlations between different wind turbines that are part of a wind farm. Accurate modeling of these correlations will ensure that the wind power uncertainty can be properly quantified. This is especially critical when analyzing systems with high penetration of wind power productions. This paper addresses this critical need by developing the Hourly Mixed Copula Model, a new probabilistic model, based on Gaussian and Archimedean copulas, to model the uncertainty and variability of wind power generation. This model enables the simulation of realistic scenarios of a wind farm power production. Metrics such as the Euclidean distance and Kullback-Leibler divergence are used to demonstrate the effectiveness of the developed model in incorporating the spatio-temporal dependence among the turbine power productions. The impact of accurate characterization of uncertainty in wind power modeling on power system steady-state operational planning is also demonstrated by case studies. These studies indicate that, in order to correctly evaluate the effect of wind power variation on steady-state operational planning, the dependence among wind power resources should not be neglected for power flow studies.},
	journal = {IEEE Access},
	author = {Philippe, W. P. J. and Zhang, S. and Eftekharnejad, S. and Ghosh, P. K. and Varshney, P. K.},
	year = {2020},
	note = {Conference Name: IEEE Access},
	keywords = {copulas, spatio-temporal modelling, uncertainty estimation, wind energy},
	pages = {138569--138583},
}

@article{liu_multivariate_2015,
	title = {A multivariate conditional model for streamflow prediction and spatial precipitation refinement},
	volume = {120},
	copyright = {©2015. American Geophysical Union. All Rights Reserved.},
	issn = {2169-8996},
	url = {https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1002/2015JD023787},
	doi = {https://doi.org/10.1002/2015JD023787},
	abstract = {The effective prediction and estimation of hydrometeorological variables are important for water resources planning and management. In this study, we propose a multivariate conditional model for streamflow prediction and the refinement of spatial precipitation estimates. This model consists of high dimensional vine copulas, conditional bivariate copula simulations, and a quantile-copula function. The vine copula is employed because of its flexibility in modeling the high dimensional joint distribution of multivariate data by building a hierarchy of conditional bivariate copulas. We investigate two cases to evaluate the performance and applicability of the proposed approach. In the first case, we generate one month ahead streamflow forecasts that incorporate multiple predictors including antecedent precipitation and streamflow records in a basin located in South China. The prediction accuracy of the vine-based model is compared with that of traditional data-driven models such as the support vector regression (SVR) and the adaptive neuro-fuzzy inference system (ANFIS). The results indicate that the proposed model produces more skillful forecasts than SVR and ANFIS. Moreover, this probabilistic model yields additional information concerning the predictive uncertainty. The second case involves refining spatial precipitation estimates derived from the tropical rainfall measuring mission precipitationproduct for the Yangtze River basin by incorporating remotely sensed soil moisture data and the observed precipitation from meteorological gauges over the basin. The validation results indicate that the proposed model successfully refines the spatial precipitation estimates. Although this model is tested for specific cases, it can be extended to other hydrometeorological variables for predictions and spatial estimations.},
	language = {en},
	number = {19},
	urldate = {2021-03-29},
	journal = {Journal of Geophysical Research: Atmospheres},
	author = {Liu, Zhiyong and Zhou, Ping and Chen, Xiuzhi and Guan, Yinghui},
	year = {2015},
	note = {\_eprint: https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1002/2015JD023787},
	keywords = {spatio-temporal modelling, uncertainty estimation, water modelling},
	pages = {10,116--10,129},
}

@article{durkan_neural_2019,
	title = {Neural {Spline} {Flows}},
	url = {http://arxiv.org/abs/1906.04032},
	abstract = {A normalizing flow models a complex probability density as an invertible transformation of a simple base density. Flows based on either coupling or autoregressive transforms both offer exact density evaluation and sampling, but rely on the parameterization of an easily invertible elementwise transformation, whose choice determines the flexibility of these models. Building upon recent work, we propose a fully-differentiable module based on monotonic rational-quadratic splines, which enhances the flexibility of both coupling and autoregressive transforms while retaining analytic invertibility. We demonstrate that neural spline flows improve density estimation, variational inference, and generative modeling of images.},
	urldate = {2021-03-29},
	journal = {arXiv:1906.04032 [cs, stat]},
	author = {Durkan, Conor and Bekasov, Artur and Murray, Iain and Papamakarios, George},
	month = dec,
	year = {2019},
	note = {arXiv: 1906.04032},
	keywords = {machine learning},
}

@phdthesis{zhang_copula-based_nodate,
	title = {Copula-based {Multimodal} {Data} {Fusion} for {Inference} with {Dependent} {Observations}},
	language = {en},
	author = {Zhang, Shan},
}

@inproceedings{lopez-paz_gaussian_2013,
	title = {Gaussian {Process} {Vine} {Copulas} for {Multivariate} {Dependence}},
	url = {http://proceedings.mlr.press/v28/lopez-paz13.html},
	abstract = {Copulas allow to learn marginal distributions separately from the multivariate dependence structure (copula) that links them together into a density function.  Vine factorizations ease the learning...},
	language = {en},
	urldate = {2021-03-29},
	booktitle = {International {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Lopez-Paz, David and Hernández-Lobato, Jose Miguel and Zoubin, Ghahramani},
	month = may,
	year = {2013},
	note = {ISSN: 1938-7228},
	pages = {10--18},
}

@article{hirt_copula-like_2019,
	title = {Copula-like {Variational} {Inference}},
	url = {http://arxiv.org/abs/1904.07153},
	abstract = {This paper considers a new family of variational distributions motivated by Sklar's theorem. This family is based on new copula-like densities on the hypercube with non-uniform marginals which can be sampled efficiently, i.e. with a complexity linear in the dimension of state space. Then, the proposed variational densities that we suggest can be seen as arising from these copula-like densities used as base distributions on the hypercube with Gaussian quantile functions and sparse rotation matrices as normalizing flows. The latter correspond to a rotation of the marginals with complexity \${\textbackslash}mathcal\{O\}(d {\textbackslash}log d)\$. We provide some empirical evidence that such a variational family can also approximate non-Gaussian posteriors and can be beneficial compared to Gaussian approximations. Our method performs largely comparably to state-of-the-art variational approximations on standard regression and classification benchmarks for Bayesian Neural Networks.},
	urldate = {2021-03-29},
	journal = {arXiv:1904.07153 [cs, stat]},
	author = {Hirt, Marcel and Dellaportas, Petros and Durmus, Alain},
	month = dec,
	year = {2019},
	note = {arXiv: 1904.07153},
	keywords = {machine learning},
}

@article{wang_neural_2019,
	title = {Neural {Gaussian} {Copula} for {Variational} {Autoencoder}},
	url = {http://arxiv.org/abs/1909.03569},
	abstract = {Variational language models seek to estimate the posterior of latent variables with an approximated variational posterior. The model often assumes the variational posterior to be factorized even when the true posterior is not. The learned variational posterior under this assumption does not capture the dependency relationships over latent variables. We argue that this would cause a typical training problem called posterior collapse observed in all other variational language models. We propose Gaussian Copula Variational Autoencoder (VAE) to avert this problem. Copula is widely used to model correlation and dependencies of high-dimensional random variables, and therefore it is helpful to maintain the dependency relationships that are lost in VAE. The empirical results show that by modeling the correlation of latent variables explicitly using a neural parametric copula, we can avert this training difficulty while getting competitive results among all other VAE approaches.},
	urldate = {2021-03-29},
	journal = {arXiv:1909.03569 [cs, stat]},
	author = {Wang, Prince Zizhuang and Wang, William Yang},
	month = sep,
	year = {2019},
	note = {arXiv: 1909.03569},
	keywords = {machine learning, neural networks, nlp},
}

@article{zhang_fusion_2019,
	title = {Fusion of {Deep} {Neural} {Networks} for {Activity} {Recognition}: {A} {Regular} {Vine} {Copula} {Based} {Approach}},
	shorttitle = {Fusion of {Deep} {Neural} {Networks} for {Activity} {Recognition}},
	url = {http://arxiv.org/abs/1905.02703},
	abstract = {In this paper, we propose regular vine copula based fusion of multiple deep neural network classifiers for the problem of multi-sensor based human activity recognition. We take the cross-modal dependence into account by employing regular vine copulas that are extremely flexible and powerful graphical models to characterize complex dependence among multiple modalities. Multiple deep neural networks are used to extract high-level features from multi-sensing modalities, with each deep neural network processing the data collected from a single sensor. The extracted high-level features are then combined using a regular vine copula model. Numerical experiments are conducted to demonstrate the effectiveness of our approach.},
	urldate = {2021-03-29},
	journal = {arXiv:1905.02703 [eess]},
	author = {Zhang, Shan and Geng, Baocheng and Varshney, Pramod K. and Rangaswamy, Muralidhar},
	month = nov,
	year = {2019},
	note = {arXiv: 1905.02703},
}

@article{zhang_fusion_2019-1,
	title = {Fusion of {Correlated} {Decisions} {Using} {Regular} {Vine} {Copulas}},
	volume = {67},
	issn = {1053-587X, 1941-0476},
	url = {http://arxiv.org/abs/1803.09350},
	doi = {10.1109/TSP.2019.2901379},
	abstract = {In this paper, we propose a regular vine copula based methodology for the fusion of correlated decisions. Regular vine copula is an extremely flexible and powerful graphical model to characterize complex dependence among multiple modalities. It can express a multivariate copula by using a cascade of bivariate copulas, the so-called pair copulas. Assuming that local detectors are single threshold binary quantizers and taking complex dependence among sensor decisions into account, we design an optimal fusion rule using a regular vine copula under the Neyman-Pearson framework. In order to reduce the computational complexity resulting from the complex dependence, we propose an efficient and computationally light regular vine copula based optimal fusion algorithm. Numerical experiments are conducted to demonstrate the effectiveness of our approach.},
	number = {8},
	urldate = {2021-03-29},
	journal = {IEEE Transactions on Signal Processing},
	author = {Zhang, Shan and Theagarajan, Lakshmi Narasimhan and Choi, Sora and Varshney, Pramod K.},
	month = apr,
	year = {2019},
	note = {arXiv: 1803.09350},
	pages = {2066--2079},
}

@article{ozdemir_copula_2018,
	title = {Copula {Based} {Classifier} {Fusion} {Under} {Statistical} {Dependence}},
	volume = {40},
	issn = {1939-3539},
	doi = {10.1109/TPAMI.2017.2774300},
	abstract = {We consider the problem of fusing probability scores from a set of classifiers to estimate a final fused probability score. Our interest is in scenarios where the classifiers are statistically dependent. To that end, we propose a new classifier fusion approach that is data driven and founded on the statistical theory of copulas. Numerical results with both simulated and real data show that our copula based classifier fusion approach produces better probability scores than individual classifiers and outperforms existing probability score fusion approaches.},
	number = {11},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	author = {Ozdemir, O. and Allen, T. G. and Choi, S. and Wimalajeewa, T. and Varshney, P. K.},
	month = nov,
	year = {2018},
	note = {Conference Name: IEEE Transactions on Pattern Analysis and Machine Intelligence},
	keywords = {classification, copulas, probability, probability score fusion, sensor fusion},
	pages = {2740--2748},
}

@article{tran_copula_2015,
	title = {Copula variational inference},
	url = {http://arxiv.org/abs/1506.03159},
	abstract = {We develop a general variational inference method that preserves dependency among the latent variables. Our method uses copulas to augment the families of distributions used in mean-field and structured approximations. Copulas model the dependency that is not captured by the original variational distribution, and thus the augmented variational family guarantees better approximations to the posterior. With stochastic optimization, inference on the augmented distribution is scalable. Furthermore, our strategy is generic: it can be applied to any inference procedure that currently uses the mean-field or structured approach. Copula variational inference has many advantages: it reduces bias; it is less sensitive to local optima; it is less sensitive to hyperparameters; and it helps characterize and interpret the dependency among the latent variables.},
	urldate = {2021-03-29},
	journal = {arXiv:1506.03159 [cs, stat]},
	author = {Tran, Dustin and Blei, David M. and Airoldi, Edoardo M.},
	month = oct,
	year = {2015},
	note = {arXiv: 1506.03159},
	keywords = {machine learning, statistics},
}

@article{yang_physics-informed_2019,
	title = {Physics-{Informed} {CoKriging}: {A} {Gaussian}-{Process}-{Regression}-{Based} {Multifidelity} {Method} for {Data}-{Model} {Convergence}},
	volume = {395},
	issn = {00219991},
	shorttitle = {Physics-{Informed} {CoKriging}},
	url = {http://arxiv.org/abs/1811.09757},
	doi = {10.1016/j.jcp.2019.06.041},
	abstract = {In this work, we propose a new Gaussian process regression (GPR)-based multifidelity method: physics-informed CoKriging (CoPhIK). In CoKriging-based multifidelity methods, the quantities of interest are modeled as linear combinations of multiple parameterized stationary Gaussian processes (GPs), and the hyperparameters of these GPs are estimated from data via optimization. In CoPhIK, we construct a GP representing low-fidelity data using physics-informed Kriging (PhIK), and model the discrepancy between low- and high-fidelity data using a parameterized GP with hyperparameters identified via optimization. Our approach reduces the cost of optimization for inferring hyperparameters by incorporating partial physical knowledge. We prove that the physical constraints in the form of deterministic linear operators are satisfied up to an error bound. Furthermore, we combine CoPhIK with a greedy active learning algorithm for guiding the selection of additional observation locations. The efficiency and accuracy of CoPhIK are demonstrated for reconstructing the partially observed modified Branin function, reconstructing the sparsely observed state of a steady state heat transport problem, and learning a conservative tracer distribution from sparse tracer concentration measurements.},
	urldate = {2021-03-26},
	journal = {Journal of Computational Physics},
	author = {Yang, Xiu and Barajas-Solano, David and Tartakovsky, Guzel and Tartakovsky, Alexandre},
	month = oct,
	year = {2019},
	note = {arXiv: 1811.09757},
	keywords = {machine learning},
	pages = {410--431},
}

@techreport{seeger_semiparametric_2004,
	title = {Semiparametric latent factor models},
	abstract = {We propose a semiparametric model for regression and classification problems involving multiple response variables. The model makes use of a set of Gaussian processes to model the relationship to the inputs in a nonparametric fashion. Conditional dependencies between the responses can be captured through a linear mixture of the driving processes. This feature becomes important if some of the responses of predictive interest are less densely supplied by observed data than related auxiliary ones. We propose an efficient approximate inference scheme for this semiparametric model whose complexity is linear in the number of training data points. 1 Semiparametric Latent Factor Models We are interested in predicting multiple responses yc ∈ Yc, c = 1,..., C from covariates x ∈ X, and we would like to model the responses as conditionally dependent. In statistical terminology, we would like to “share statistical strength ” between the yc; in machine learning parlance this is often referred to as “transfer of learning. ” As we demonstrate empirically, such sharing can be especially powerful if the data for the responses is partially missing.},
	institution = {Workshop on Artificial Intelligence and Statistics 10},
	author = {Seeger, Matthias and Teh, Yee-whye and Jordan, Michael I.},
	year = {2004},
}

@inproceedings{elidan_copulas_2013,
	address = {Berlin, Heidelberg},
	series = {Lecture {Notes} in {Statistics}},
	title = {Copulas in {Machine} {Learning}},
	isbn = {978-3-642-35407-6},
	doi = {10.1007/978-3-642-35407-6_3},
	abstract = {Despite overlapping goals of multivariate modeling and dependence identification, until recently the fields of machine learning in general and probabilistic graphical models in particular have been ignorant of the framework of copulas. At the same time, the complementing strengths of the two fields suggests the great fruitfulness of a synergy. The purpose of this paper is to survey recent copula-based constructions in the field of machine learning, so as to provide a stepping stone for those interested in further exploring this emerging symbiotic research.},
	language = {en},
	booktitle = {Copulae in {Mathematical} and {Quantitative} {Finance}},
	publisher = {Springer},
	author = {Elidan, Gal},
	editor = {Jaworski, Piotr and Durante, Fabrizio and Härdle, Wolfgang Karl},
	year = {2013},
	keywords = {bayesian neural networks, copulas, rkhs},
	pages = {39--60},
}

@inproceedings{NIPS2016_3147da8a,
	title = {Interaction networks for learning about objects, relations and physics},
	volume = {29},
	url = {https://proceedings.neurips.cc/paper/2016/file/3147da8ab4a0437c15ef51a5cc7f2dc4-Paper.pdf},
	booktitle = {Advances in neural information processing systems},
	publisher = {Curran Associates, Inc.},
	author = {Battaglia, Peter and Pascanu, Razvan and Lai, Matthew and Jimenez Rezende, Danilo and kavukcuoglu, koray},
	editor = {Lee, D. and Sugiyama, M. and Luxburg, U. and Guyon, I. and Garnett, R.},
	year = {2016},
}

@inproceedings{NEURIPS2020_91c77393,
	title = {Agree to disagree: {Adaptive} ensemble knowledge distillation in gradient space},
	volume = {33},
	url = {https://proceedings.neurips.cc/paper/2020/file/91c77393975889bd08f301c9e13a44b7-Paper.pdf},
	booktitle = {Advances in neural information processing systems},
	publisher = {Curran Associates, Inc.},
	author = {Du, Shangchen and You, Shan and Li, Xiaojie and Wu, Jianlong and Wang, Fei and Qian, Chen and Zhang, Changshui},
	editor = {Larochelle, H. and Ranzato, M. and Hadsell, R. and Balcan, M. F. and Lin, H.},
	year = {2020},
	pages = {12345--12355},
}

@article{wilson_copula_2010,
	title = {Copula {Processes}},
	url = {https://arxiv.org/abs/1006.1350v2},
	abstract = {We define a copula process which describes the dependencies between
arbitrarily many random variables independently of their marginal
distributions. As an example, we develop a stochastic volatility model,
Gaussian Copula Process Volatility (GCPV), to predict the latent standard
deviations of a sequence of random variables. To make predictions we use
Bayesian inference, with the Laplace approximation, and with Markov chain Monte
Carlo as an alternative. We find both methods comparable. We also find our
model can outperform GARCH on simulated and financial data. And unlike GARCH,
GCPV can easily handle missing data, incorporate covariates other than time,
and model a rich class of covariance structures.},
	language = {en},
	urldate = {2021-03-17},
	author = {Wilson, Andrew Gordon and Ghahramani, Zoubin},
	month = jun,
	year = {2010},
}

@article{yang_physics-informed_2018,
	title = {Physics-informed deep generative models},
	url = {http://arxiv.org/abs/1812.03511},
	abstract = {We consider the application of deep generative models in propagating uncertainty through complex physical systems. Specifically, we put forth an implicit variational inference formulation that constrains the generative model output to satisfy given physical laws expressed by partial differential equations. Such physics-informed constraints provide a regularization mechanism for effectively training deep probabilistic models for modeling physical systems in which the cost of data acquisition is high and training data-sets are typically small. This provides a scalable framework for characterizing uncertainty in the outputs of physical systems due to randomness in their inputs or noise in their observations. We demonstrate the effectiveness of our approach through a canonical example in transport dynamics.},
	urldate = {2021-03-16},
	journal = {arXiv:1812.03511 [cs, stat]},
	author = {Yang, Yibo and Perdikaris, Paris},
	month = dec,
	year = {2018},
	note = {arXiv: 1812.03511},
	keywords = {machine learning},
}

@article{yu_implicit_2019,
	title = {Implicit {Posterior} {Variational} {Inference} for {Deep} {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/1910.11998},
	abstract = {A multi-layer deep Gaussian process (DGP) model is a hierarchical composition of GP models with a greater expressive power. Exact DGP inference is intractable, which has motivated the recent development of deterministic and stochastic approximation methods. Unfortunately, the deterministic approximation methods yield a biased posterior belief while the stochastic one is computationally costly. This paper presents an implicit posterior variational inference (IPVI) framework for DGPs that can ideally recover an unbiased posterior belief and still preserve time efficiency. Inspired by generative adversarial networks, our IPVI framework achieves this by casting the DGP inference problem as a two-player game in which a Nash equilibrium, interestingly, coincides with an unbiased posterior belief. This consequently inspires us to devise a best-response dynamics algorithm to search for a Nash equilibrium (i.e., an unbiased posterior belief). Empirical evaluation shows that IPVI outperforms the state-of-the-art approximation methods for DGPs.},
	urldate = {2021-03-16},
	journal = {arXiv:1910.11998 [cs, stat]},
	author = {Yu, Haibin and Chen, Yizhou and Dai, Zhongxiang and Low, Kian Hsiang and Jaillet, Patrick},
	month = oct,
	year = {2019},
	note = {arXiv: 1910.11998},
	keywords = {machine learning},
}

@article{tran_hierarchical_2017,
	title = {Hierarchical {Implicit} {Models} and {Likelihood}-{Free} {Variational} {Inference}},
	url = {http://arxiv.org/abs/1702.08896},
	abstract = {Implicit probabilistic models are a flexible class of models defined by a simulation process for data. They form the basis for theories which encompass our understanding of the physical world. Despite this fundamental nature, the use of implicit models remains limited due to challenges in specifying complex latent structure in them, and in performing inferences in such models with large data sets. In this paper, we first introduce hierarchical implicit models (HIMs). HIMs combine the idea of implicit densities with hierarchical Bayesian modeling, thereby defining models via simulators of data with rich hidden structure. Next, we develop likelihood-free variational inference (LFVI), a scalable variational inference algorithm for HIMs. Key to LFVI is specifying a variational family that is also implicit. This matches the model's flexibility and allows for accurate approximation of the posterior. We demonstrate diverse applications: a large-scale physical simulator for predator-prey populations in ecology; a Bayesian generative adversarial network for discrete data; and a deep implicit model for text generation.},
	urldate = {2021-03-16},
	journal = {arXiv:1702.08896 [cs, stat]},
	author = {Tran, Dustin and Ranganath, Rajesh and Blei, David M.},
	month = nov,
	year = {2017},
	note = {arXiv: 1702.08896},
	keywords = {machine learning, statistics},
}

@article{battaglia_relational_2018,
	title = {Relational inductive biases, deep learning, and graph networks},
	url = {http://arxiv.org/abs/1806.01261},
	abstract = {Artificial intelligence (AI) has undergone a renaissance recently, making major progress in key domains such as vision, language, control, and decision-making. This has been due, in part, to cheap data and cheap compute resources, which have fit the natural strengths of deep learning. However, many defining characteristics of human intelligence, which developed under much different pressures, remain out of reach for current approaches. In particular, generalizing beyond one's experiences--a hallmark of human intelligence from infancy--remains a formidable challenge for modern AI. The following is part position paper, part review, and part unification. We argue that combinatorial generalization must be a top priority for AI to achieve human-like abilities, and that structured representations and computations are key to realizing this objective. Just as biology uses nature and nurture cooperatively, we reject the false choice between "hand-engineering" and "end-to-end" learning, and instead advocate for an approach which benefits from their complementary strengths. We explore how using relational inductive biases within deep learning architectures can facilitate learning about entities, relations, and rules for composing them. We present a new building block for the AI toolkit with a strong relational inductive bias--the graph network--which generalizes and extends various approaches for neural networks that operate on graphs, and provides a straightforward interface for manipulating structured knowledge and producing structured behaviors. We discuss how graph networks can support relational reasoning and combinatorial generalization, laying the foundation for more sophisticated, interpretable, and flexible patterns of reasoning. As a companion to this paper, we have released an open-source software library for building graph networks, with demonstrations of how to use them in practice.},
	urldate = {2021-03-10},
	journal = {arXiv:1806.01261 [cs, stat]},
	author = {Battaglia, Peter W. and Hamrick, Jessica B. and Bapst, Victor and Sanchez-Gonzalez, Alvaro and Zambaldi, Vinicius and Malinowski, Mateusz and Tacchetti, Andrea and Raposo, David and Santoro, Adam and Faulkner, Ryan and Gulcehre, Caglar and Song, Francis and Ballard, Andrew and Gilmer, Justin and Dahl, George and Vaswani, Ashish and Allen, Kelsey and Nash, Charles and Langston, Victoria and Dyer, Chris and Heess, Nicolas and Wierstra, Daan and Kohli, Pushmeet and Botvinick, Matt and Vinyals, Oriol and Li, Yujia and Pascanu, Razvan},
	month = oct,
	year = {2018},
	note = {arXiv: 1806.01261},
	keywords = {machine learning},
}

@article{fiasconaro_asymptotic_2006,
	title = {Asymptotic regime in {N} random interacting species},
	volume = {50},
	issn = {1434-6028, 1434-6036},
	url = {http://arxiv.org/abs/physics/0510066},
	doi = {10.1140/epjb/e2006-00058-4},
	abstract = {The asymptotic regime of a complex ecosystem with {\textbackslash}emph\{N\}random interacting species and in the presence of an external multiplicative noise is analyzed. We find the role of the external noise on the long time probability distribution of the i-th density species, the extinction of species and the local field acting on the i-th population. We analyze in detail the transient dynamics of this field and the cavity field, which is the field acting on the \$i{\textasciicircum}\{th\}\$ species when this is absent. We find that the presence or the absence of some population give different asymptotic distributions of these fields.},
	number = {1-2},
	urldate = {2021-03-04},
	journal = {The European Physical Journal B},
	author = {Fiasconaro, A. and Valenti, D. and Spagnolo, B.},
	month = mar,
	year = {2006},
	note = {arXiv: physics/0510066},
	keywords = {physics},
	pages = {189--194},
}

@article{kondoh_foraging_2003,
	title = {Foraging {Adaptation} and the {Relationship} {Between} {Food}-{Web} {Complexity} and {Stability}},
	volume = {299},
	issn = {00368075, 10959203},
	url = {https://www.sciencemag.org/lookup/doi/10.1126/science.1079154},
	doi = {10.1126/science.1079154},
	number = {5611},
	urldate = {2021-03-03},
	journal = {Science},
	author = {Kondoh, M.},
	month = feb,
	year = {2003},
	pages = {1388--1391},
}

@article{ackland_stabilization_2004,
	title = {Stabilization of {Large} {Generalized} {Lotka}-{Volterra} {Foodwebs} {By} {Evolutionary} {Feedback}},
	volume = {93},
	issn = {0031-9007, 1079-7114},
	url = {https://link.aps.org/doi/10.1103/PhysRevLett.93.158701},
	doi = {10.1103/PhysRevLett.93.158701},
	language = {en},
	number = {15},
	urldate = {2021-03-03},
	journal = {Physical Review Letters},
	author = {Ackland, G. J. and Gallagher, I. D.},
	month = oct,
	year = {2004},
	pages = {158701},
}

@article{pchelkina_control_2012,
	title = {Control of oscillatory behavior of multispecies populations},
	volume = {227},
	issn = {0304-3800},
	url = {https://www.sciencedirect.com/science/article/pii/S030438001100514X},
	doi = {10.1016/j.ecolmodel.2011.10.022},
	abstract = {Control problem for the model of the multispecies Lotka–Volterra ecosystem is considered. The algorithm for control of oscillatory behavior of the ecosystem based on the speed-gradient method is proposed. The conditions of convergence of the algorithm are obtained. The conditions of convergence of the algorithm are obtained. The results of numerical experiments are presented demonstrating that the speed-gradient method can achieve control goal with a small control signal, which is important in controlling real ecosystems where control action should be sufficiently small. In turn, it allows to increase stability and robustness of the oscillation behavior by means of increase of the distance from boundaries of population survival area.},
	language = {en},
	urldate = {2021-03-03},
	journal = {Ecological Modelling},
	author = {Pchelkina, Irina and Fradkov, Alexander L.},
	month = feb,
	year = {2012},
	keywords = {ecology, lotka-volterra},
	pages = {1--6},
}

@article{ladde_stability_1992,
	title = {Stability of {Lotka}–{Volterra} model},
	volume = {16},
	issn = {0895-7177},
	url = {https://www.sciencedirect.com/science/article/pii/089571779290051L},
	doi = {10.1016/0895-7177(92)90051-L},
	abstract = {By assuming the positivity of the equilibrium state of Lotka–Volterra model of n-species in the community in the absence of random fluctuations, sufficient conditions are given for the stability of the equilibrium state of such a multispecies model. Using this information, stability analysis of Lotka-Volterra model of n-species in the community under random environmental perturbations is investigated. This analysis provides a partial solution to one of the fundamental problems in modelling of dynamic systems, namely to what extent randomness in the system causes the change in behavior of the system relative to its deterministic version. The presented analysis incorporates the stochastic population model discussed by Montroll as a special case. Furthermore, our study provides direct stability analysis of the equilibrium state of the populations without the knowledge of the probability densities of the species in the community.},
	language = {en},
	number = {3},
	urldate = {2021-03-03},
	journal = {Mathematical and Computer Modelling},
	author = {Ladde, G. S. and Sathananthan, S.},
	month = mar,
	year = {1992},
	pages = {99--107},
}

@article{jankowiak_neural_2019,
	title = {Neural {Likelihoods} for {Multi}-{Output} {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/1905.13697},
	abstract = {We construct flexible likelihoods for multi-output Gaussian process models that leverage neural networks as components. We make use of sparse variational inference methods to enable scalable approximate inference for the resulting class of models. An attractive feature of these models is that they can admit analytic predictive means even when the likelihood is non-linear and the predictive distributions are non-Gaussian. We validate the modeling potential of these models in a variety of experiments in both the supervised and unsupervised setting. We demonstrate that the flexibility of these `neural' likelihoods can improve prediction quality as compared to simpler Gaussian process models and that neural likelihoods can be readily combined with a variety of underlying Gaussian process models, including deep Gaussian processes.},
	urldate = {2021-03-03},
	journal = {arXiv:1905.13697 [cs, stat]},
	author = {Jankowiak, Martin and Gardner, Jacob},
	month = may,
	year = {2019},
	note = {arXiv: 1905.13697},
	keywords = {machine learning},
}

@article{xu_stochastic_2020,
	title = {Stochastic {Generalized} {Lotka}-{Volterra} {Model} with {An} {Application} to {Learning} {Microbial} {Community} {Structures}},
	url = {http://arxiv.org/abs/2009.10922},
	abstract = {Inferring microbial community structure based on temporal metagenomics data is an important goal in microbiome studies. The deterministic generalized Lotka-Volterra differential (GLV) equations have been used to model the dynamics of microbial data. However, these approaches fail to take random environmental fluctuations into account, which may negatively impact the estimates. We propose a new stochastic GLV (SGLV) differential equation model, where the random perturbations of Brownian motion in the model can naturally account for the external environmental effects on the microbial community. We establish new conditions and show various mathematical properties of the solutions including general existence and uniqueness, stationary distribution, and ergodicity. We further develop approximate maximum likelihood estimators based on discrete observations and systematically investigate the consistency and asymptotic normality of the proposed estimators. Our method is demonstrated through simulation studies and an application to the well-known "moving picture" temporal microbial dataset.},
	urldate = {2021-03-02},
	journal = {arXiv:2009.10922 [stat]},
	author = {Xu, Libai and Xu, Ximing and Kong, Dehan and Gu, Hong and Kenney, Toby},
	month = sep,
	year = {2020},
	note = {arXiv: 2009.10922},
	keywords = {statistics},
}

@book{fort_ecological_2020,
	title = {Ecological {Modelling} and {Ecophysics}: {Agricultural} and environmental applications},
	isbn = {978-0-7503-2432-8},
	shorttitle = {Ecological {Modelling} and {Ecophysics}},
	url = {https://iopscience.iop.org/book/978-0-7503-2432-8},
	abstract = {The linear Lotka–Volterra equations are considered by many ecologists as exploratory models with limited direct relationship to real ecosystems because of their many simplifying assumptions, their popularity being partly attributable to sentimental attachment (Keddy 2001).},
	language = {en},
	urldate = {2021-03-02},
	publisher = {IOP Publishing},
	author = {Fort, Hugo},
	month = jun,
	year = {2020},
	doi = {10.1088/978-0-7503-2432-8},
}

@article{rees_quantifying_1996,
	title = {Quantifying the {Impact} of {Competition} and {Spatial} {Heterogeneity} on the {Structure} and {Dynamics} of a {Four}-{Species} {Guild} of {Winter} {Annuals}},
	volume = {147},
	issn = {0003-0147},
	url = {http://www.jstor.org/stable/2463221},
	abstract = {We develop statistical methods appropriate for the analysis of spatially structured population data. The methods are used to study the structure and dynamics of a four-species annual plant guild recorded in 1,000 permanent squares over a 10-yr period. We parameterize models that predict population density from one year to the next. In agreement with theoretical expectation all the models have locally stable equilibria, and overcompensation is rare. We demonstrate that interspecific interactions are extremely weak, relative to intraspecific ones, and that the spatial arrangement of species and individuals within them is critical to the observed dynamics. The impact of spatial density-dependent population growth on observed densities was calculated. In 52\% of the cases population size would have been increased by at least a factor of 1.5 had there been no interactions between individuals, and in 9\% of these it would have increased by a factor of four or more. This effect is shown to be largely a result of intraspecific interactions. We discuss possible explanations for the weakness of interspecific interactions.},
	number = {1},
	urldate = {2021-03-02},
	journal = {The American Naturalist},
	author = {Rees, Mark and Grubb, Peter J. and Kelly, Dave},
	year = {1996},
	note = {Publisher: [University of Chicago Press, American Society of Naturalists]},
	pages = {1--32},
}

@article{gard_persistence_1984,
	title = {Persistence in stochastic food web models},
	volume = {46},
	issn = {0092-8240},
	url = {https://www.sciencedirect.com/science/article/pii/S0092824084800440},
	doi = {10.1016/S0092-8240(84)80044-0},
	abstract = {A sufficient condition is given for stochastic boundedness persistence of a top predator in generalized Lotka-Volterra-type stochastic food web models in arbitrary bounded regions of state space. The main result indicates that persistence in the corresponding deterministic system is preserved in the stochastic system if the intensities of the random fluctuations are not too large.},
	language = {en},
	number = {3},
	urldate = {2021-03-01},
	journal = {Bulletin of Mathematical Biology},
	author = {Gard, Thomas C.},
	month = jan,
	year = {1984},
	pages = {357--370},
}

@article{gard_stability_1986,
	title = {Stability for multispecies population models in random environments},
	volume = {10},
	issn = {0362-546X},
	url = {https://www.sciencedirect.com/science/article/pii/0362546X86901112},
	doi = {10.1016/0362-546X(86)90111-2},
	language = {en},
	number = {12},
	urldate = {2021-03-01},
	journal = {Nonlinear Analysis: Theory, Methods \& Applications},
	author = {Gard, Thomas C.},
	month = dec,
	year = {1986},
	keywords = {population models},
	pages = {1411--1419},
}

@article{ma_variational_2019,
	title = {Variational {Implicit} {Processes}},
	url = {http://arxiv.org/abs/1806.02390},
	abstract = {We introduce the implicit processes (IPs), a stochastic process that places implicitly defined multivariate distributions over any finite collections of random variables. IPs are therefore highly flexible implicit priors over functions, with examples including data simulators, Bayesian neural networks and non-linear transformations of stochastic processes. A novel and efficient approximate inference algorithm for IPs, namely the variational implicit processes (VIPs), is derived using generalised wake-sleep updates. This method returns simple update equations and allows scalable hyper-parameter learning with stochastic optimization. Experiments show that VIPs return better uncertainty estimates and lower errors over existing inference methods for challenging models such as Bayesian neural networks, and Gaussian processes.},
	urldate = {2021-03-01},
	journal = {arXiv:1806.02390 [cs, stat]},
	author = {Ma, Chao and Li, Yingzhen and Hernández-Lobato, José Miguel},
	month = may,
	year = {2019},
	note = {arXiv: 1806.02390},
	keywords = {machine learning},
}

@article{tahara_asymptotic_2018,
	title = {Asymptotic stability of a modified {Lotka}-{Volterra} model with small immigrations},
	volume = {8},
	copyright = {2018 The Author(s)},
	issn = {2045-2322},
	url = {https://www.nature.com/articles/s41598-018-25436-2},
	doi = {10.1038/s41598-018-25436-2},
	abstract = {Predator-prey systems have been studied intensively for over a hundred years. These studies have demonstrated that the dynamics of Lotka-Volterra (LV) systems are not stable, that is, exhibiting either cyclic oscillation or divergent extinction of one species. Stochastic versions of the deterministic cyclic oscillations also exhibit divergent extinction. Thus, we have no solution for asymptotic stability in predator-prey systems, unlike most natural predator-prey interactions that sometimes exhibit stable and persistent coexistence. Here, we demonstrate that adding a small immigration into the prey or predator population can stabilize the LV system. Although LV systems have been studied intensively, there is no study on the non-linear modifications that we have tested. We also checked the effect of the inclusion of non-linear interaction term to the stability of the LV system. Our results show that small immigrations invoke stable convergence in the LV system with three types of functional responses. This means that natural predator-prey populations can be stabilized by a small number of sporadic immigrants.},
	language = {en},
	number = {1},
	urldate = {2021-03-01},
	journal = {Scientific Reports},
	author = {Tahara, Takeru and Gavina, Maica Krizna Areja and Kawano, Takenori and Tubay, Jerrold M. and Rabajante, Jomar F. and Ito, Hiromu and Morita, Satoru and Ichinose, Genki and Okabe, Takuya and Togashi, Tatsuya and Tainaka, Kei-ichi and Shimizu, Akira and Nagatani, Takashi and Yoshimura, Jin},
	month = may,
	year = {2018},
	note = {Number: 1
Publisher: Nature Publishing Group},
	pages = {7029},
}

@article{papamakarios_fast_2018,
	title = {Fast \${\textbackslash}epsilon\$-free {Inference} of {Simulation} {Models} with {Bayesian} {Conditional} {Density} {Estimation}},
	url = {http://arxiv.org/abs/1605.06376},
	abstract = {Many statistical models can be simulated forwards but have intractable likelihoods. Approximate Bayesian Computation (ABC) methods are used to infer properties of these models from data. Traditionally these methods approximate the posterior over parameters by conditioning on data being inside an \${\textbackslash}epsilon\$-ball around the observed data, which is only correct in the limit \${\textbackslash}epsilon{\textbackslash}!{\textbackslash}rightarrow{\textbackslash}!0\$. Monte Carlo methods can then draw samples from the approximate posterior to approximate predictions or error bars on parameters. These algorithms critically slow down as \${\textbackslash}epsilon{\textbackslash}!{\textbackslash}rightarrow{\textbackslash}!0\$, and in practice draw samples from a broader distribution than the posterior. We propose a new approach to likelihood-free inference based on Bayesian conditional density estimation. Preliminary inferences based on limited simulation data are used to guide later simulations. In some cases, learning an accurate parametric representation of the entire true posterior distribution requires fewer model simulations than Monte Carlo ABC methods need to produce a single sample from an approximate posterior.},
	urldate = {2021-03-01},
	journal = {arXiv:1605.06376 [cs, stat]},
	author = {Papamakarios, George and Murray, Iain},
	month = apr,
	year = {2018},
	note = {arXiv: 1605.06376},
	keywords = {machine learning, statistics},
}

@article{wood_statistical_2010,
	title = {Statistical inference for noisy nonlinear ecological dynamic systems},
	volume = {466},
	copyright = {2010 Nature Publishing Group, a division of Macmillan Publishers Limited. All Rights Reserved.},
	issn = {1476-4687},
	url = {https://www.nature.com/articles/nature09319},
	doi = {10.1038/nature09319},
	abstract = {Many ecological systems have chaotic or near-chaotic dynamics. In such cases, it is difficult to test whether the data fit particular models, because the noise makes statistical comparison with the model impossible. Simon Wood has now devised a statistical method for making such inferences, based on extracting phase-insensitive summary statistics from the raw data and comparing to data that are simulated from the model. The method is demonstrated by an application to a well-known problem: the nature of the cycles in John Nicholson's classic ecological experiments on population size in the sheep blowfly Lucilia cuprina.},
	language = {en},
	number = {7310},
	urldate = {2021-03-01},
	journal = {Nature},
	author = {Wood, Simon N.},
	month = aug,
	year = {2010},
	note = {Number: 7310
Publisher: Nature Publishing Group},
	pages = {1102--1104},
}

@article{hsu_bayesian_2019,
	title = {Bayesian {Learning} of {Conditional} {Kernel} {Mean} {Embeddings} for {Automatic} {Likelihood}-{Free} {Inference}},
	url = {http://arxiv.org/abs/1903.00863},
	abstract = {In likelihood-free settings where likelihood evaluations are intractable, approximate Bayesian computation (ABC) addresses the formidable inference task to discover plausible parameters of simulation programs that explain the observations. However, they demand large quantities of simulation calls. Critically, hyperparameters that determine measures of simulation discrepancy crucially balance inference accuracy and sample efficiency, yet are difficult to tune. In this paper, we present kernel embedding likelihood-free inference (KELFI), a holistic framework that automatically learns model hyperparameters to improve inference accuracy given limited simulation budget. By leveraging likelihood smoothness with conditional mean embeddings, we nonparametrically approximate likelihoods and posteriors as surrogate densities and sample from closed-form posterior mean embeddings, whose hyperparameters are learned under its approximate marginal likelihood. Our modular framework demonstrates improved accuracy and efficiency on challenging inference problems in ecology.},
	urldate = {2021-03-01},
	journal = {arXiv:1903.00863 [cs, stat]},
	author = {Hsu, Kelvin and Ramos, Fabio},
	month = mar,
	year = {2019},
	note = {arXiv: 1903.00863},
	keywords = {machine learning},
}

@article{hening_stochastic_2017,
	title = {Stochastic {Lotka}-{Volterra} food chains},
	url = {http://arxiv.org/abs/1703.04809},
	abstract = {We study the persistence and extinction of species in a simple food chain that is modelled by a Lotka-Volterra system with environmental stochasticity. There exist sharp results for deterministic Lotka-Volterra systems in the literature but few for their stochastic counterparts. The food chain we analyze consists of one prey and \$n-1\$ predators. The \$j\$th predator eats the \$j-1\$th species and is eaten by the \$j+1\$th predator; this way each species only interacts with at most two other species - the ones that are immediately above or below it in the trophic chain. We show that one can classify, based on an explicit quantity depending on the interaction coefficients of the system, which species go extinct and which converge to their unique invariant probability measure. Our work can be seen as a natural extension of the deterministic results of Gard and Hallam '79 to a stochastic setting. As one consequence we show that environmental stochasticity makes species more likely to go extinct. However, if the environmental fluctuations are small, persistence in the deterministic setting is preserved in the stochastic system. Our analysis also shows that the addition of a new apex predator makes, as expected, the different species more prone to extinction. Another novelty of our analysis is the fact that we can describe the behavior the system when the noise is degenerate. This is relevant because of the possibility of strong correlations between the effects of the environment on the different species.},
	urldate = {2021-02-26},
	journal = {arXiv:1703.04809 [math, q-bio]},
	author = {Hening, Alexandru and Nguyen, Dang H.},
	month = mar,
	year = {2017},
	note = {arXiv: 1703.04809},
	keywords = {ecological modeling, probability},
}

@article{yao_quality_2019,
	title = {Quality of {Uncertainty} {Quantification} for {Bayesian} {Neural} {Network} {Inference}},
	url = {http://arxiv.org/abs/1906.09686},
	abstract = {Bayesian Neural Networks (BNNs) place priors over the parameters in a neural network. Inference in BNNs, however, is difficult; all inference methods for BNNs are approximate. In this work, we empirically compare the quality of predictive uncertainty estimates for 10 common inference methods on both regression and classification tasks. Our experiments demonstrate that commonly used metrics (e.g. test log-likelihood) can be misleading. Our experiments also indicate that inference innovations designed to capture structure in the posterior do not necessarily produce high quality posterior approximations.},
	urldate = {2021-02-24},
	journal = {arXiv:1906.09686 [cs, stat]},
	author = {Yao, Jiayu and Pan, Weiwei and Ghosh, Soumya and Doshi-Velez, Finale},
	month = jun,
	year = {2019},
	note = {arXiv: 1906.09686},
	keywords = {machine learning},
}

@article{chen_stochastic_2014,
	title = {Stochastic {Gradient} {Hamiltonian} {Monte} {Carlo}},
	url = {http://arxiv.org/abs/1402.4102},
	abstract = {Hamiltonian Monte Carlo (HMC) sampling methods provide a mechanism for defining distant proposals with high acceptance probabilities in a Metropolis-Hastings framework, enabling more efficient exploration of the state space than standard random-walk proposals. The popularity of such methods has grown significantly in recent years. However, a limitation of HMC methods is the required gradient computation for simulation of the Hamiltonian dynamical system-such computation is infeasible in problems involving a large sample size or streaming data. Instead, we must rely on a noisy gradient estimate computed from a subset of the data. In this paper, we explore the properties of such a stochastic gradient HMC approach. Surprisingly, the natural implementation of the stochastic approximation can be arbitrarily bad. To address this problem we introduce a variant that uses second-order Langevin dynamics with a friction term that counteracts the effects of the noisy gradient, maintaining the desired target distribution as the invariant distribution. Results on simulated data validate our theory. We also provide an application of our methods to a classification task using neural networks and to online Bayesian matrix factorization.},
	urldate = {2021-02-22},
	journal = {arXiv:1402.4102 [cs, stat]},
	author = {Chen, Tianqi and Fox, Emily B. and Guestrin, Carlos},
	month = may,
	year = {2014},
	note = {arXiv: 1402.4102},
	keywords = {machine learning, statistics},
}

@article{bhardwaj_adaptively_2019,
	title = {Adaptively {Preconditioned} {Stochastic} {Gradient} {Langevin} {Dynamics}},
	url = {http://arxiv.org/abs/1906.04324},
	abstract = {Stochastic Gradient Langevin Dynamics infuses isotropic gradient noise to SGD to help navigate pathological curvature in the loss landscape for deep networks. Isotropic nature of the noise leads to poor scaling, and adaptive methods based on higher order curvature information such as Fisher Scoring have been proposed to precondition the noise in order to achieve better convergence. In this paper, we describe an adaptive method to estimate the parameters of the noise and conduct experiments on well-known model architectures to show that the adaptively preconditioned SGLD method achieves convergence with the speed of adaptive first order methods such as Adam, AdaGrad etc. and achieves generalization equivalent of SGD in the test set.},
	urldate = {2021-02-22},
	journal = {arXiv:1906.04324 [cs, stat]},
	author = {Bhardwaj, Chandrasekaran Anirudh},
	month = jun,
	year = {2019},
	note = {arXiv: 1906.04324},
	keywords = {computer vision, machine learning},
}

@article{li_preconditioned_2015,
	title = {Preconditioned {Stochastic} {Gradient} {Langevin} {Dynamics} for {Deep} {Neural} {Networks}},
	url = {http://arxiv.org/abs/1512.07666},
	abstract = {Effective training of deep neural networks suffers from two main issues. The first is that the parameter spaces of these models exhibit pathological curvature. Recent methods address this problem by using adaptive preconditioning for Stochastic Gradient Descent (SGD). These methods improve convergence by adapting to the local geometry of parameter space. A second issue is overfitting, which is typically addressed by early stopping. However, recent work has demonstrated that Bayesian model averaging mitigates this problem. The posterior can be sampled by using Stochastic Gradient Langevin Dynamics (SGLD). However, the rapidly changing curvature renders default SGLD methods inefficient. Here, we propose combining adaptive preconditioners with SGLD. In support of this idea, we give theoretical properties on asymptotic convergence and predictive risk. We also provide empirical results for Logistic Regression, Feedforward Neural Nets, and Convolutional Neural Nets, demonstrating that our preconditioned SGLD method gives state-of-the-art performance on these models.},
	urldate = {2021-02-22},
	journal = {arXiv:1512.07666 [stat]},
	author = {Li, Chunyuan and Chen, Changyou and Carlson, David and Carin, Lawrence},
	month = dec,
	year = {2015},
	note = {arXiv: 1512.07666},
	keywords = {machine learning},
}

@inproceedings{welling_bayesian_2011,
	address = {Madison, WI, USA},
	series = {{ICML}'11},
	title = {Bayesian learning via stochastic gradient langevin dynamics},
	isbn = {978-1-4503-0619-5},
	abstract = {In this paper we propose a new framework for learning from large scale datasets based on iterative learning from small mini-batches. By adding the right amount of noise to a standard stochastic gradient optimization algorithm we show that the iterates will converge to samples from the true posterior distribution as we anneal the stepsize. This seamless transition between optimization and Bayesian posterior sampling provides an inbuilt protection against overfitting. We also propose a practical method for Monte Carlo estimates of posterior statistics which monitors a "sampling threshold" and collects samples after it has been surpassed. We apply the method to three models: a mixture of Gaussians, logistic regression and ICA with natural gradients.},
	urldate = {2021-02-22},
	booktitle = {Proceedings of the 28th {International} {Conference} on {International} {Conference} on {Machine} {Learning}},
	publisher = {Omnipress},
	author = {Welling, Max and Teh, Yee Whye},
	month = jun,
	year = {2011},
	pages = {681--688},
}

@article{kuntal_web-glv_2019,
	title = {Web-{gLV}: {A} {Web} {Based} {Platform} for {Lotka}-{Volterra} {Based} {Modeling} and {Simulation} of {Microbial} {Populations}},
	volume = {10},
	issn = {1664-302X},
	shorttitle = {Web-{gLV}},
	url = {https://www.frontiersin.org/articles/10.3389/fmicb.2019.00288/full},
	doi = {10.3389/fmicb.2019.00288},
	abstract = {The affordability of high throughput DNA sequencing has allowed us to explore the dynamics of microbial populations in various ecosystems. Mathematical modeling and simulation of such microbiome time series data can help in getting better understanding of bacterial communities. In this paper, we present Web-gLV - a GUI based interactive platform for generalized Lotka-Volterra (gLV) based modeling and simulation of microbial populations. The tool can be used to generate the mathematical models with automatic estimation of parameters and use them to predict future trajectories using numerical simulations. We also demonstrate the utility of our tool on few publicly available datasets. The case studies demonstrate the ease with which the current tool can be used by biologists to model bacterial populations and simulate their dynamics to get biological insights. We expect Web-gLV to be a valuable contribution in the field of ecological modeling and metagenomic systems biology. Availability: http://web.rniapps.net/webglv},
	language = {English},
	urldate = {2021-02-19},
	journal = {Frontiers in Microbiology},
	author = {Kuntal, Bhusan K. and Gadgil, Chetan and Mande, Sharmila S.},
	year = {2019},
	note = {Publisher: Frontiers},
	keywords = {lotka-volterra, microbiome, modeling, simulation, time-series modelling, visualization},
}

@article{wang_when_2020,
	title = {When and why {PINNs} fail to train: {A} neural tangent kernel perspective},
	shorttitle = {When and why {PINNs} fail to train},
	url = {http://arxiv.org/abs/2007.14527},
	abstract = {Physics-informed neural networks (PINNs) have lately received great attention thanks to their flexibility in tackling a wide range of forward and inverse problems involving partial differential equations. However, despite their noticeable empirical success, little is known about how such constrained neural networks behave during their training via gradient descent. More importantly, even less is known about why such models sometimes fail to train at all. In this work, we aim to investigate these questions through the lens of the Neural Tangent Kernel (NTK); a kernel that captures the behavior of fully-connected neural networks in the infinite width limit during training via gradient descent. Specifically, we derive the NTK of PINNs and prove that, under appropriate conditions, it converges to a deterministic kernel that stays constant during training in the infinite-width limit. This allows us to analyze the training dynamics of PINNs through the lens of their limiting NTK and find a remarkable discrepancy in the convergence rate of the different loss components contributing to the total training error. To address this fundamental pathology, we propose a novel gradient descent algorithm that utilizes the eigenvalues of the NTK to adaptively calibrate the convergence rate of the total training error. Finally, we perform a series of numerical experiments to verify the correctness of our theory and the practical effectiveness of the proposed algorithms. The data and code accompanying this manuscript are publicly available at {\textbackslash}url\{https://github.com/PredictiveIntelligenceLab/PINNsNTK\}.},
	urldate = {2021-02-18},
	journal = {arXiv:2007.14527 [cs, math, stat]},
	author = {Wang, Sifan and Yu, Xinling and Perdikaris, Paris},
	month = jul,
	year = {2020},
	note = {arXiv: 2007.14527},
	keywords = {machine learning, numerical modelling},
}

@article{bunin_ecological_2017,
	title = {Ecological communities with {Lotka}-{Volterra} dynamics},
	volume = {95},
	url = {https://link.aps.org/doi/10.1103/PhysRevE.95.042414},
	doi = {10.1103/PhysRevE.95.042414},
	abstract = {Ecological communities in heterogeneous environments assemble through the combined effect of species interaction and migration. Understanding the effect of these processes on the community properties is central to ecology. Here we study these processes for a single community subject to migration from a pool of species, with population dynamics described by the generalized Lotka-Volterra equations. We derive exact results for the phase diagram describing the dynamical behaviors, and for the diversity and species abundance distributions. A phase transition is found from a phase where a unique globally attractive fixed point exists to a phase where multiple dynamical attractors exist, leading to history-dependent community properties. The model is shown to possess a symmetry that also establishes a connection with other well-known models.},
	number = {4},
	urldate = {2021-02-18},
	journal = {Physical Review E},
	author = {Bunin, Guy},
	month = apr,
	year = {2017},
	note = {Publisher: American Physical Society},
	pages = {042414},
}

@article{yeakel_generalized_2011,
	title = {Generalized modeling of ecological population dynamics},
	volume = {4},
	issn = {1874-1746},
	url = {https://doi.org/10.1007/s12080-011-0112-6},
	doi = {10.1007/s12080-011-0112-6},
	abstract = {Over the past 7 years, several authors have used the approach of generalized modeling to study the dynamics of food chains and food webs. Generalized models come close to the efficiency of random matrix models, while being as directly interpretable as conventional differential-equation-based models. Here, we present a pedagogical introduction to the approach of generalized modeling. This introduction places more emphasis on the underlying concepts of generalized modeling than previous publications. Moreover, we propose a shortcut that can significantly accelerate the formulation of generalized models and introduce an iterative procedure that can be used to refine existing generalized models by integrating new biological insights.},
	language = {en},
	number = {2},
	urldate = {2021-02-18},
	journal = {Theoretical Ecology},
	author = {Yeakel, Justin D. and Stiefs, Dirk and Novak, Mark and Gross, Thilo},
	month = may,
	year = {2011},
	pages = {179--194},
}

@article{jackson_introduction_2000,
	title = {An {Introduction} to the {Practice} of {Ecological} {Modeling}},
	volume = {50},
	issn = {0006-3568},
	url = {https://doi.org/10.1641/0006-3568(2000)050[0694:AITTPO]2.0.CO;2},
	doi = {10.1641/0006-3568(2000)050[0694:AITTPO]2.0.CO;2},
	number = {8},
	urldate = {2021-02-18},
	journal = {BioScience},
	author = {Jackson, Leland J. and Trebitz, Anett S. and Cottingham, Kathryn L.},
	month = aug,
	year = {2000},
	pages = {694--706},
}

@article{cranmer_frontier_2020,
	title = {The frontier of simulation-based inference},
	volume = {117},
	copyright = {© 2020 . https://www.pnas.org/site/aboutpnas/licenses.xhtmlPublished under the PNAS license.},
	issn = {0027-8424, 1091-6490},
	url = {https://www.pnas.org/content/117/48/30055},
	doi = {10.1073/pnas.1912789117},
	abstract = {Many domains of science have developed complex simulations to describe phenomena of interest. While these simulations provide high-fidelity models, they are poorly suited for inference and lead to challenging inverse problems. We review the rapidly developing field of simulation-based inference and identify the forces giving additional momentum to the field. Finally, we describe how the frontier is expanding so that a broad audience can appreciate the profound influence these developments may have on science.},
	language = {en},
	number = {48},
	urldate = {2021-02-18},
	journal = {Proceedings of the National Academy of Sciences},
	author = {Cranmer, Kyle and Brehmer, Johann and Louppe, Gilles},
	month = dec,
	year = {2020},
	pmid = {32471948},
	note = {Publisher: National Academy of Sciences
Section: Colloquium Paper},
	keywords = {approximate bayesian computation, implicit models, likelihood-free inference},
	pages = {30055--30062},
}

@book{bishop_pattern_2006,
	address = {New York},
	series = {Information science and statistics},
	title = {Pattern recognition and machine learning},
	isbn = {978-0-387-31073-2},
	publisher = {Springer},
	author = {Bishop, Christopher M.},
	year = {2006},
	keywords = {machine learning},
}

@inproceedings{hoang_learning_2020,
	title = {Learning {Task}-{Agnostic} {Embedding} of {Multiple} {Black}-{Box} {Experts} for {Multi}-{Task} {Model} {Fusion}},
	url = {http://proceedings.mlr.press/v119/hoang20b.html},
	abstract = {Model fusion is an emerging study in collective learning where heterogeneous experts with private data and learning architectures need to combine their black-box knowledge for better performance. E...},
	language = {en},
	urldate = {2021-02-08},
	booktitle = {International {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Hoang, Nghia and Lam, Thanh and Low, Bryan Kian Hsiang and Jaillet, Patrick},
	month = nov,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {4282--4292},
}

@article{peherstorfer_survey_2018,
	title = {Survey of {Multifidelity} {Methods} in {Uncertainty} {Propagation}, {Inference}, and {Optimization}},
	volume = {60},
	issn = {0036-1445, 1095-7200},
	url = {https://epubs.siam.org/doi/10.1137/16M1082469},
	doi = {10.1137/16M1082469},
	language = {en},
	number = {3},
	urldate = {2021-02-16},
	journal = {SIAM Review},
	author = {Peherstorfer, Benjamin and Willcox, Karen and Gunzburger, Max},
	month = jan,
	year = {2018},
	pages = {550--591},
}

@article{pearce_uncertainty_2020,
	title = {Uncertainty in {Neural} {Networks}: {Approximately} {Bayesian} {Ensembling}},
	shorttitle = {Uncertainty in {Neural} {Networks}},
	url = {http://arxiv.org/abs/1810.05546},
	abstract = {Understanding the uncertainty of a neural network's (NN) predictions is essential for many purposes. The Bayesian framework provides a principled approach to this, however applying it to NNs is challenging due to large numbers of parameters and data. Ensembling NNs provides an easily implementable, scalable method for uncertainty quantification, however, it has been criticised for not being Bayesian. This work proposes one modification to the usual process that we argue does result in approximate Bayesian inference; regularising parameters about values drawn from a distribution which can be set equal to the prior. A theoretical analysis of the procedure in a simplified setting suggests the recovered posterior is centred correctly but tends to have an underestimated marginal variance, and overestimated correlation. However, two conditions can lead to exact recovery. We argue that these conditions are partially present in NNs. Empirical evaluations demonstrate it has an advantage over standard ensembling, and is competitive with variational methods.},
	urldate = {2021-02-15},
	journal = {arXiv:1810.05546 [cs, stat]},
	author = {Pearce, Tim and Leibfried, Felix and Brintrup, Alexandra and Zaki, Mohamed and Neely, Andy},
	month = feb,
	year = {2020},
	note = {arXiv: 1810.05546},
	keywords = {machine learning},
}

@article{mandt_stochastic_2018,
	title = {Stochastic {Gradient} {Descent} as {Approximate} {Bayesian} {Inference}},
	url = {http://arxiv.org/abs/1704.04289},
	abstract = {Stochastic Gradient Descent with a constant learning rate (constant SGD) simulates a Markov chain with a stationary distribution. With this perspective, we derive several new results. (1) We show that constant SGD can be used as an approximate Bayesian posterior inference algorithm. Specifically, we show how to adjust the tuning parameters of constant SGD to best match the stationary distribution to a posterior, minimizing the Kullback-Leibler divergence between these two distributions. (2) We demonstrate that constant SGD gives rise to a new variational EM algorithm that optimizes hyperparameters in complex probabilistic models. (3) We also propose SGD with momentum for sampling and show how to adjust the damping coefficient accordingly. (4) We analyze MCMC algorithms. For Langevin Dynamics and Stochastic Gradient Fisher Scoring, we quantify the approximation errors due to finite learning rates. Finally (5), we use the stochastic process perspective to give a short proof of why Polyak averaging is optimal. Based on this idea, we propose a scalable approximate MCMC algorithm, the Averaged Stochastic Gradient Sampler.},
	urldate = {2021-02-12},
	journal = {arXiv:1704.04289 [cs, stat]},
	author = {Mandt, Stephan and Hoffman, Matthew D. and Blei, David M.},
	month = jan,
	year = {2018},
	note = {arXiv: 1704.04289},
	keywords = {machine learning},
}

@inproceedings{hoang_collective_2019,
	title = {Collective {Model} {Fusion} for {Multiple} {Black}-{Box} {Experts}},
	url = {http://proceedings.mlr.press/v97/hoang19a.html},
	abstract = {Model fusion is a fundamental problem in collec-tive machine learning (ML) where independentexperts with heterogeneous learning architecturesare required to combine expertise to improve pre-dictive...},
	language = {en},
	urldate = {2021-02-08},
	booktitle = {International {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Hoang, Minh and Hoang, Nghia and Low, Bryan Kian Hsiang and Kingsford, Carleton},
	month = may,
	year = {2019},
	note = {ISSN: 2640-3498},
	pages = {2742--2750},
}

@article{liu_splitting_2019,
	title = {Splitting {Steepest} {Descent} for {Growing} {Neural} {Architectures}},
	url = {http://arxiv.org/abs/1910.02366},
	abstract = {We develop a progressive training approach for neural networks which adaptively grows the network structure by splitting existing neurons to multiple off-springs. By leveraging a functional steepest descent idea, we derive a simple criterion for deciding the best subset of neurons to split and a splitting gradient for optimally updating the off-springs. Theoretically, our splitting strategy is a second-order functional steepest descent for escaping saddle points in an \${\textbackslash}infty\$-Wasserstein metric space, on which the standard parametric gradient descent is a first-order steepest descent. Our method provides a new computationally efficient approach for optimizing neural network structures, especially for learning lightweight neural architectures in resource-constrained settings.},
	urldate = {2021-02-08},
	journal = {arXiv:1910.02366 [cs, stat]},
	author = {Liu, Qiang and Wu, Lemeng and Wang, Dilin},
	month = nov,
	year = {2019},
	note = {arXiv: 1910.02366},
	keywords = {machine learning, neural networks},
}

@article{pierrot_learning_2019,
	title = {Learning {Compositional} {Neural} {Programs} with {Recursive} {Tree} {Search} and {Planning}},
	url = {http://arxiv.org/abs/1905.12941},
	abstract = {We propose a novel reinforcement learning algorithm, AlphaNPI, that incorporates the strengths of Neural Programmer-Interpreters (NPI) and AlphaZero. NPI contributes structural biases in the form of modularity, hierarchy and recursion, which are helpful to reduce sample complexity, improve generalization and increase interpretability. AlphaZero contributes powerful neural network guided search algorithms, which we augment with recursion. AlphaNPI only assumes a hierarchical program specification with sparse rewards: 1 when the program execution satisfies the specification, and 0 otherwise. Using this specification, AlphaNPI is able to train NPI models effectively with RL for the first time, completely eliminating the need for strong supervision in the form of execution traces. The experiments show that AlphaNPI can sort as well as previous strongly supervised NPI variants. The AlphaNPI agent is also trained on a Tower of Hanoi puzzle with two disks and is shown to generalize to puzzles with an arbitrary number of disk},
	urldate = {2021-02-08},
	journal = {arXiv:1905.12941 [cs]},
	author = {Pierrot, Thomas and Ligner, Guillaume and Reed, Scott and Sigaud, Olivier and Perrin, Nicolas and Laterre, Alexandre and Kas, David and Beguir, Karim and de Freitas, Nando},
	month = may,
	year = {2019},
	note = {arXiv: 1905.12941},
	keywords = {machine learning},
}

@article{yu_implicit_2019,
	title = {Implicit {Posterior} {Variational} {Inference} for {Deep} {Gaussian} {Processes}},
	url = {http://arxiv.org/abs/1910.11998},
	abstract = {A multi-layer deep Gaussian process (DGP) model is a hierarchical composition of GP models with a greater expressive power. Exact DGP inference is intractable, which has motivated the recent development of deterministic and stochastic approximation methods. Unfortunately, the deterministic approximation methods yield a biased posterior belief while the stochastic one is computationally costly. This paper presents an implicit posterior variational inference (IPVI) framework for DGPs that can ideally recover an unbiased posterior belief and still preserve time efficiency. Inspired by generative adversarial networks, our IPVI framework achieves this by casting the DGP inference problem as a two-player game in which a Nash equilibrium, interestingly, coincides with an unbiased posterior belief. This consequently inspires us to devise a best-response dynamics algorithm to search for a Nash equilibrium (i.e., an unbiased posterior belief). Empirical evaluation shows that IPVI outperforms the state-of-the-art approximation methods for DGPs.},
	urldate = {2021-02-08},
	journal = {arXiv:1910.11998 [cs, stat]},
	author = {Yu, Haibin and Chen, Yizhou and Dai, Zhongxiang and Low, Kian Hsiang and Jaillet, Patrick},
	month = oct,
	year = {2019},
	note = {arXiv: 1910.11998},
	keywords = {machine learning},
}

@article{yurochkin_statistical_2019,
	title = {Statistical {Model} {Aggregation} via {Parameter} {Matching}},
	url = {http://arxiv.org/abs/1911.00218},
	abstract = {We consider the problem of aggregating models learned from sequestered, possibly heterogeneous datasets. Exploiting tools from Bayesian nonparametrics, we develop a general meta-modeling framework that learns shared global latent structures by identifying correspondences among local model parameterizations. Our proposed framework is model-independent and is applicable to a wide range of model types. After verifying our approach on simulated data, we demonstrate its utility in aggregating Gaussian topic models, hierarchical Dirichlet process based hidden Markov models, and sparse Gaussian processes with applications spanning text summarization, motion capture analysis, and temperature forecasting.},
	urldate = {2021-02-08},
	journal = {arXiv:1911.00218 [cs, stat]},
	author = {Yurochkin, Mikhail and Agarwal, Mayank and Ghosh, Soumya and Greenewald, Kristjan and Hoang, Trong Nghia},
	month = nov,
	year = {2019},
	note = {arXiv: 1911.00218},
	keywords = {machine learning},
}

@article{osawa_practical_2019,
	title = {Practical {Deep} {Learning} with {Bayesian} {Principles}},
	url = {http://arxiv.org/abs/1906.02506},
	abstract = {Bayesian methods promise to fix many shortcomings of deep learning, but they are impractical and rarely match the performance of standard methods, let alone improve them. In this paper, we demonstrate practical training of deep networks with natural-gradient variational inference. By applying techniques such as batch normalisation, data augmentation, and distributed training, we achieve similar performance in about the same number of epochs as the Adam optimiser, even on large datasets such as ImageNet. Importantly, the benefits of Bayesian principles are preserved: predictive probabilities are well-calibrated, uncertainties on out-of-distribution data are improved, and continual-learning performance is boosted. This work enables practical deep learning while preserving benefits of Bayesian principles. A PyTorch implementation is available as a plug-and-play optimiser.},
	urldate = {2021-02-12},
	journal = {arXiv:1906.02506 [cs, stat]},
	author = {Osawa, Kazuki and Swaroop, Siddharth and Jain, Anirudh and Eschenhagen, Runa and Turner, Richard E. and Yokota, Rio and Khan, Mohammad Emtiyaz},
	month = oct,
	year = {2019},
	note = {arXiv: 1906.02506},
	keywords = {machine learning},
}

@article{bardsley_randomize-then-optimize_2014,
	title = {Randomize-{Then}-{Optimize}: {A} {Method} for {Sampling} from {Posterior} {Distributions} in {Nonlinear} {Inverse} {Problems}},
	volume = {36},
	issn = {1064-8275},
	shorttitle = {Randomize-{Then}-{Optimize}},
	url = {https://epubs.siam.org/doi/abs/10.1137/140964023},
	doi = {10.1137/140964023},
	abstract = {High-dimensional inverse problems present a challenge for  Markov chain Monte Carlo (MCMC)-type sampling schemes. Typically, they rely on finding an efficient proposal distribution, which can be difficult for large-scale problems, even with adaptive approaches. Moreover, the autocorrelations of the samples typically increase with dimension, which leads to the need for long sample chains. We present an alternative method for sampling from posterior distributions in nonlinear inverse problems, when the measurement error and prior are both Gaussian. The approach computes a candidate sample by solving a stochastic optimization problem. In the linear case, these samples are directly from the posterior density, but this is not so in the nonlinear case. We derive the form of the sample density in the nonlinear case, and then show how to use it within both a Metropolis--Hastings and importance sampling framework to obtain samples from the posterior distribution of the parameters. We demonstrate, with various small- and medium-scale problems, that randomize-then-optimize can be efficient compared to standard adaptive MCMC algorithms.},
	number = {4},
	urldate = {2021-02-12},
	journal = {SIAM Journal on Scientific Computing},
	author = {Bardsley, Johnathan M. and Solonen, Antti and Haario, Heikki and Laine, Marko},
	month = jan,
	year = {2014},
	note = {Publisher: Society for Industrial and Applied Mathematics},
	pages = {A1895--A1910},
}

@article{zhang_light_2019,
	title = {Light scattering by pure water and seawater: the depolarization ratio and its variation with salinity},
	volume = {58},
	copyright = {\&\#169; 2019 Optical Society of America},
	issn = {2155-3165},
	shorttitle = {Light scattering by pure water and seawater},
	url = {https://www.osapublishing.org/ao/abstract.cfm?uri=ao-58-4-991},
	doi = {10.1364/AO.58.000991},
	abstract = {We measured the linearly polarized light scattering of pure water and seawater at various salinities and estimated the depolarization ratio using five different methods of data analysis after removing the scattering due to contamination by residual nanoparticles. The depolarization ratio values (\&\#x03B4;) estimated for pure water using these different methods are largely consistent with each other and result in a mean value of 0.039\&\#x00B1;0.001. For seawater, our results reveal a trend of a slight linear increase of \&\#x03B4; with salinity (S), \&\#x03B4;=0.039+a1\&\#x00D7;S, where a1 varies in the range of 1\&\#x00D7;10\&\#x2212;4 to 2\&\#x00D7;10\&\#x2212;4 between the methods.},
	language = {EN},
	number = {4},
	urldate = {2021-02-10},
	journal = {Applied Optics},
	author = {Zhang, Xiaodong and Stramski, Dariusz and Reynolds, Rick A. and Blocker, E. Riley},
	month = feb,
	year = {2019},
	note = {Publisher: Optical Society of America},
	pages = {991--1004},
}

@article{geyman_simple_2019,
	title = {A {Simple} {Method} for {Extracting} {Water} {Depth} {From} {Multispectral} {Satellite} {Imagery} in {Regions} of {Variable} {Bottom} {Type}},
	volume = {6},
	issn = {2333-5084},
	url = {https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1029/2018EA000539},
	doi = {https://doi.org/10.1029/2018EA000539},
	abstract = {Satellite imagery offers an efficient and cost-effective means of estimating water depth in shallow environments. However, traditional empirical algorithms for calculating water depth often are unable to account for varying bottom reflectance, and therefore yield biased estimates for certain benthic environments. We present a simple method that is grounded in the physics of radiative transfer in seawater, but made more robust through the calibration of individual color-to-depth relationships for separate spectral classes. Our cluster-based regression (CBR) algorithm, applied to a portion of the Great Bahama Bank, drastically reduces the geographic structure in the residual and has a mean absolute error of 0.19 m with quantified uncertainties. Our CBR bathymetry is 3–5 times more accurate than existing models and outperforms machine learning protocols at extrapolating beyond the calibration data. Finally, we demonstrate how comparison of CBR with traditional models sensitive to bottom type reveals the characteristic length scales of biosedimentary facies belts.},
	language = {en},
	number = {3},
	urldate = {2021-02-10},
	journal = {Earth and Space Science},
	author = {Geyman, Emily C. and Maloof, Adam C.},
	year = {2019},
	note = {\_eprint: https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1029/2018EA000539},
	keywords = {bathymetry, classification, machine learning, remote sensing},
	pages = {527--537},
}

@article{ghamisi_multisource_2018,
	title = {Multisource and {Multitemporal} {Data} {Fusion} in {Remote} {Sensing}},
	url = {http://arxiv.org/abs/1812.08287},
	abstract = {The sharp and recent increase in the availability of data captured by different sensors combined with their considerably heterogeneous natures poses a serious challenge for the effective and efficient processing of remotely sensed data. Such an increase in remote sensing and ancillary datasets, however, opens up the possibility of utilizing multimodal datasets in a joint manner to further improve the performance of the processing approaches with respect to the application at hand. Multisource data fusion has, therefore, received enormous attention from researchers worldwide for a wide variety of applications. Moreover, thanks to the revisit capability of several spaceborne sensors, the integration of the temporal information with the spatial and/or spectral/backscattering information of the remotely sensed data is possible and helps to move from a representation of 2D/3D data to 4D data structures, where the time variable adds new information as well as challenges for the information extraction algorithms. There are a huge number of research works dedicated to multisource and multitemporal data fusion, but the methods for the fusion of different modalities have expanded in different paths according to each research community. This paper brings together the advances of multisource and multitemporal data fusion approaches with respect to different research communities and provides a thorough and discipline-specific starting point for researchers at different levels (i.e., students, researchers, and senior researchers) willing to conduct novel investigations on this challenging topic by supplying sufficient detail and references.},
	urldate = {2021-02-10},
	journal = {arXiv:1812.08287 [cs, eess, stat]},
	author = {Ghamisi, Pedram and Rasti, Behnood and Yokoya, Naoto and Wang, Qunming and Hofle, Bernhard and Bruzzone, Lorenzo and Bovolo, Francesca and Chi, Mingmin and Anders, Katharina and Gloaguen, Richard and Atkinson, Peter M. and Benediktsson, Jon Atli},
	month = dec,
	year = {2018},
	note = {arXiv: 1812.08287},
	keywords = {machine learning},
}

@article{kochkov_machine_2021,
	title = {Machine learning accelerated computational fluid dynamics},
	url = {http://arxiv.org/abs/2102.01010},
	abstract = {Numerical simulation of fluids plays an essential role in modeling many physical phenomena, such as weather, climate, aerodynamics and plasma physics. Fluids are well described by the Navier-Stokes equations, but solving these equations at scale remains daunting, limited by the computational cost of resolving the smallest spatiotemporal features. This leads to unfavorable trade-offs between accuracy and tractability. Here we use end-to-end deep learning to improve approximations inside computational fluid dynamics for modeling two-dimensional turbulent flows. For both direct numerical simulation of turbulence and large eddy simulation, our results are as accurate as baseline solvers with 8-10x finer resolution in each spatial dimension, resulting in 40-80x fold computational speedups. Our method remains stable during long simulations, and generalizes to forcing functions and Reynolds numbers outside of the flows where it is trained, in contrast to black box machine learning approaches. Our approach exemplifies how scientific computing can leverage machine learning and hardware accelerators to improve simulations without sacrificing accuracy or generalization.},
	urldate = {2021-02-05},
	journal = {arXiv:2102.01010 [physics]},
	author = {Kochkov, Dmitrii and Smith, Jamie A. and Alieva, Ayya and Wang, Qing and Brenner, Michael P. and Hoyer, Stephan},
	month = jan,
	year = {2021},
	note = {arXiv: 2102.01010},
	keywords = {fluid dynamics, machine learning},
}

@article{mandt_stochastic_2018-1,
	title = {Stochastic {Gradient} {Descent} as {Approximate} {Bayesian} {Inference}},
	url = {http://arxiv.org/abs/1704.04289},
	abstract = {Stochastic Gradient Descent with a constant learning rate (constant SGD) simulates a Markov chain with a stationary distribution. With this perspective, we derive several new results. (1) We show that constant SGD can be used as an approximate Bayesian posterior inference algorithm. Specifically, we show how to adjust the tuning parameters of constant SGD to best match the stationary distribution to a posterior, minimizing the Kullback-Leibler divergence between these two distributions. (2) We demonstrate that constant SGD gives rise to a new variational EM algorithm that optimizes hyperparameters in complex probabilistic models. (3) We also propose SGD with momentum for sampling and show how to adjust the damping coefficient accordingly. (4) We analyze MCMC algorithms. For Langevin Dynamics and Stochastic Gradient Fisher Scoring, we quantify the approximation errors due to finite learning rates. Finally (5), we use the stochastic process perspective to give a short proof of why Polyak averaging is optimal. Based on this idea, we propose a scalable approximate MCMC algorithm, the Averaged Stochastic Gradient Sampler.},
	urldate = {2021-02-05},
	journal = {arXiv:1704.04289 [cs, stat]},
	author = {Mandt, Stephan and Hoffman, Matthew D. and Blei, David M.},
	month = jan,
	year = {2018},
	note = {arXiv: 1704.04289},
	keywords = {machine learning},
}

@article{guichard_short_2017,
	title = {A short review of numerical cloud-resolving models},
	volume = {69},
	issn = {null},
	url = {https://doi.org/10.1080/16000870.2017.1373578},
	doi = {10.1080/16000870.2017.1373578},
	abstract = {A cloud-resolving model (CRM) allows performing numerical simulations of convective clouds, such as shallow cumulus and stratocumulus, or storms and squall-lines with a resolution on the order of a few tens of metres to a few kilometres over a limited-area 4D (time and space) domain. The development of such models over the past decades is reviewed and their specific features are presented. The latter include a non-hydrostatic dynamic and parameterizations of sub-grid turbulence, microphysical and radiative processes. The capabilities of such models are discussed based on comparisons with observations and model-intercomparison studies. CRMs are used in a variety of ways, from the exploration of cloud phenomenology and process-understanding studies to the development of algorithms for satellite products, as well as to address climate issues and to develop convective and cloud parametrizations for large-scale weather and climate models. A few results illustrating this wide utilization are presented. The continuous increase of computer power induces rapid changes in modelling perspectives and therefore, influences the developments and applications of CRMs. This is discussed together with emerging scientific questions which will further benefit from CRM simulations.},
	number = {1},
	urldate = {2021-01-17},
	journal = {Tellus A: Dynamic Meteorology and Oceanography},
	author = {Guichard, Françoise and Couvreux, Fleur},
	month = jan,
	year = {2017},
	note = {Publisher: Taylor \& Francis
\_eprint: https://doi.org/10.1080/16000870.2017.1373578},
	keywords = {convection, large-eddy simulation, numerical modelling, physical processes, turbulence},
	pages = {1373578},
}

@article{stern_flow_nodate,
	title = {Flow networks as learning machines},
	abstract = {Materials and machines are often designed with particular goals in mind, so that they exhibit desired responses to given forces. Here we explore an alternative approach, physical coupled learning. In this paradigm, a physical network such as a ﬂow network consisting of pipes conveying ﬂuid, adapts the pipe conductances to obtain a desired pressure response at target nodes in response to pressures applied at source nodes, similar to supervised machine learning. Instead of obtaining the desired pressure response by minimizing some loss function, coupled learning introduces physically plausible learning rules. By physically plausible we mean (1) that the conductance of each pipe responds only to properties of the ﬂow through that pipe, such as the pressure drop, and that (2) the learning rule itself should not contain explicit information about the desired response. We demonstrate how disordered ﬂow networks can learn to distinguish handwritten digits. These results suggest the feasibility of a new class of smart metamaterials that can adapt in-situ to users’ needs.},
	language = {en},
	author = {Stern, Menachem and Hexner, Daniel},
	pages = {6},
}

@article{che_your_2020,
	title = {Your {GAN} is {Secretly} an {Energy}-based {Model} and {You} {Should} use {Discriminator} {Driven} {Latent} {Sampling}},
	url = {http://arxiv.org/abs/2003.06060},
	abstract = {We show that the sum of the implicit generator log-density \${\textbackslash}log p\_g\$ of a GAN with the logit score of the discriminator defines an energy function which yields the true data density when the generator is imperfect but the discriminator is optimal, thus making it possible to improve on the typical generator (with implicit density \$p\_g\$). To make that practical, we show that sampling from this modified density can be achieved by sampling in latent space according to an energy-based model induced by the sum of the latent prior log-density and the discriminator output score. This can be achieved by running a Langevin MCMC in latent space and then applying the generator function, which we call Discriminator Driven Latent Sampling{\textasciitilde}(DDLS). We show that DDLS is highly efficient compared to previous methods which work in the high-dimensional pixel space and can be applied to improve on previously trained GANs of many types. We evaluate DDLS on both synthetic and real-world datasets qualitatively and quantitatively. On CIFAR-10, DDLS substantially improves the Inception Score of an off-the-shelf pre-trained SN-GAN{\textasciitilde}{\textbackslash}citep\{sngan\} from \$8.22\$ to \$9.09\$ which is even comparable to the class-conditional BigGAN{\textasciitilde}{\textbackslash}citep\{biggan\} model. This achieves a new state-of-the-art in unconditional image synthesis setting without introducing extra parameters or additional training.},
	urldate = {2021-01-15},
	journal = {arXiv:2003.06060 [cs, stat]},
	author = {Che, Tong and Zhang, Ruixiang and Sohl-Dickstein, Jascha and Larochelle, Hugo and Paull, Liam and Cao, Yuan and Bengio, Yoshua},
	month = mar,
	year = {2020},
	note = {arXiv: 2003.06060},
	keywords = {machine learning},
}

@article{muandet_kernel_2017,
	title = {Kernel {Mean} {Embedding} of {Distributions}: {A} {Review} and {Beyond}},
	volume = {10},
	issn = {1935-8237, 1935-8245},
	shorttitle = {Kernel {Mean} {Embedding} of {Distributions}},
	url = {http://arxiv.org/abs/1605.09522},
	doi = {10.1561/2200000060},
	abstract = {A Hilbert space embedding of a distribution---in short, a kernel mean embedding---has recently emerged as a powerful tool for machine learning and inference. The basic idea behind this framework is to map distributions into a reproducing kernel Hilbert space (RKHS) in which the whole arsenal of kernel methods can be extended to probability measures. It can be viewed as a generalization of the original "feature map" common to support vector machines (SVMs) and other kernel methods. While initially closely associated with the latter, it has meanwhile found application in fields ranging from kernel machines and probabilistic modeling to statistical inference, causal discovery, and deep learning. The goal of this survey is to give a comprehensive review of existing work and recent advances in this research area, and to discuss the most challenging issues and open problems that could lead to new research directions. The survey begins with a brief introduction to the RKHS and positive definite kernels which forms the backbone of this survey, followed by a thorough discussion of the Hilbert space embedding of marginal distributions, theoretical guarantees, and a review of its applications. The embedding of distributions enables us to apply RKHS methods to probability measures which prompts a wide range of applications such as kernel two-sample testing, independent testing, and learning on distributional data. Next, we discuss the Hilbert space embedding for conditional distributions, give theoretical insights, and review some applications. The conditional mean embedding enables us to perform sum, product, and Bayes' rules---which are ubiquitous in graphical model, probabilistic inference, and reinforcement learning---in a non-parametric way. We then discuss relationships between this framework and other related areas. Lastly, we give some suggestions on future research directions.},
	number = {1-2},
	urldate = {2021-01-15},
	journal = {Foundations and Trends® in Machine Learning},
	author = {Muandet, Krikamol and Fukumizu, Kenji and Sriperumbudur, Bharath and Schölkopf, Bernhard},
	year = {2017},
	note = {arXiv: 1605.09522},
	keywords = {machine learning},
	pages = {1--141},
}

@article{satoh_global_2019,
	title = {Global {Cloud}-{Resolving} {Models}},
	volume = {5},
	issn = {2198-6061},
	url = {https://doi.org/10.1007/s40641-019-00131-0},
	doi = {10.1007/s40641-019-00131-0},
	abstract = {Global cloud-resolving models (GCRMs) are a new type of atmospheric model which resolve nonhydrostatic accelerations globally with kilometer-scale resolution. This review explains what distinguishes GCRMs from other types of models, the problems they solve, and the questions their more commonplace use is raising.},
	language = {en},
	number = {3},
	urldate = {2021-01-15},
	journal = {Current Climate Change Reports},
	author = {Satoh, Masaki and Stevens, Bjorn and Judt, Falko and Khairoutdinov, Marat and Lin, Shian-Jiann and Putman, William M. and Düben, Peter},
	month = sep,
	year = {2019},
	pages = {172--184},
}

@article{hanifi_critical_2020,
	title = {A {Critical} {Review} of {Wind} {Power} {Forecasting} {Methods}—{Past}, {Present} and {Future}},
	volume = {13},
	copyright = {http://creativecommons.org/licenses/by/3.0/},
	url = {https://www.mdpi.com/1996-1073/13/15/3764},
	doi = {10.3390/en13153764},
	abstract = {The largest obstacle that suppresses the increase of wind power penetration within the power grid is uncertainties and fluctuations in wind speeds. Therefore, accurate wind power forecasting is a challenging task, which can significantly impact the effective operation of power systems. Wind power forecasting is also vital for planning unit commitment, maintenance scheduling and profit maximisation of power traders. The current development of cost-effective operation and maintenance methods for modern wind turbines benefits from the advancement of effective and accurate wind power forecasting approaches. This paper systematically reviewed the state-of-the-art approaches of wind power forecasting with regard to physical, statistical (time series and artificial neural networks) and hybrid methods, including factors that affect accuracy and computational time in the predictive modelling efforts. Besides, this study provided a guideline for wind power forecasting process screening, allowing the wind turbine/farm operators to identify the most appropriate predictive methods based on time horizons, input features, computational time, error measurements, etc. More specifically, further recommendations for the research community of wind power forecasting were proposed based on reviewed literature.},
	language = {en},
	number = {15},
	urldate = {2021-01-15},
	journal = {Energies},
	author = {Hanifi, Shahram and Liu, Xiaolei and Lin, Zi and Lotfian, Saeid},
	month = jan,
	year = {2020},
	note = {Number: 15
Publisher: Multidisciplinary Digital Publishing Institute},
	keywords = {neural networks, physics-informed learning, wind power forecasting},
	pages = {3764},
}

@article{aggarwal_comprehensive_2013,
	title = {A {Comprehensive} {Review} of {Numerical} {Weather} {Prediction} {Models}},
	volume = {74},
	issn = {09758887},
	url = {http://research.ijcaonline.org/volume74/number18/pxc3890246.pdf},
	doi = {10.5120/12989-0246},
	abstract = {Weather forecasting has been an area of considerable interest among researchers since long. In particular, precipitation has been found to be interesting because of its chaotic nature and also because of the direct impact it has on the society. Even after the invention of complex Coupled Numerical Weather Prediction Models, the errors in prediction have been found to be of significant magnitude. The present study aims at investigating all the aspects of error dynamics in dynamic and statistical predictions, and reviews these two prediction models on the basis of errors arising due to initial conditions and understanding of physical processes generating with time series.},
	language = {en},
	number = {18},
	urldate = {2021-01-15},
	journal = {International Journal of Computer Applications},
	author = {Aggarwal, Rashi and Kumar, Rajendra},
	month = jul,
	year = {2013},
	pages = {44--48},
}

@book{noauthor_computational_2020,
	title = {Computational {Methods} for {Fluid} {Dynamics}},
	isbn = {978-3-319-99693-6},
	url = {https://doi.org/10.1007/978-3-319-99693-6},
	language = {English},
	urldate = {2021-01-15},
	year = {2020},
	note = {OCLC: 1190114630},
}

@article{shah_review_2020,
	title = {A review of approaches for water depth estimation with multispectral data},
	volume = {6},
	copyright = {© 2020 Policy Studies Organization},
	issn = {2639-541X},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/wwp2.12029},
	doi = {https://doi.org/10.1002/wwp2.12029},
	abstract = {The purpose of this paper is to give an exhaustive assessment of water depth estimation in shallow inland and coastal water environments with the assistance of optical remote detecting procedures, utilizing satellite/airborne multispectral information. Remote sensing has been used to map bathymetry for several decades. Inaccessibility, large-scale depth mapping, very shallow areas, and cost constraint are some of the reasons why remote sensing technique is a boon for bathymetry. Researchers have come a long way since the remote sensing-based bathymetry has been used. Fourfold increase in spatial, spectral, radiometric and temporal resolution of sensors have also increased the accuracy. There are two types of remote sensing techniques; Active and Passive. Active remote sensing includes the use of ground penetrating radar and bathymetric lidar, which are expensive but precise surveys successful up to 70 m depth. On the other hand, passive remote sensing deals with water photogrammetry and radiometric methods done by capturing imagery with space borne/air borne platforms having depth penetration up to 25 m. The latter technique includes finding of a logarithmic relationship between river depth and image values. This technique is most widely used because of its cost effectiveness. In this paper, the emphasis has been given to derive bathymetric techniques using multispectral data as these methods have proved reasonably successful when water quality, bottom reflectance, and atmospheric effects are all invariant. At a specific area, the detection of possible changes in water flow and levels indicates flow path changes which can be used to predict potential surface level flooding for better water management and policy decisions.},
	language = {en},
	number = {1},
	urldate = {2021-01-14},
	journal = {World Water Policy},
	author = {Shah, Ankit and Deshmukh, Benidhar and Sinha, L. K.},
	year = {2020},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/wwp2.12029},
	keywords = {bathymetry, remote sensing},
	pages = {152--167},
}

@book{world_meteorological_organization_guide_1988,
	address = {Geneva, Switzerland},
	series = {{WMO}},
	title = {Guide to wave analysis and forecasting},
	isbn = {978-92-63-10702-2},
	language = {en},
	number = {no. 702},
	publisher = {Secretariat of the World Meteorological Organization},
	editor = {World Meteorological Organization and World Meteorological Organization},
	year = {1988},
	keywords = {oceanography},
}

@article{brunton_machine_2020,
	title = {Machine {Learning} for {Fluid} {Mechanics}},
	volume = {52},
	issn = {0066-4189, 1545-4479},
	url = {https://www.annualreviews.org/doi/10.1146/annurev-fluid-010719-060214},
	doi = {10.1146/annurev-fluid-010719-060214},
	abstract = {The field of fluid mechanics is rapidly advancing, driven by unprecedented volumes of data from experiments, field measurements, and large-scale simulations at multiple spatiotemporal scales. Machine learning (ML) offers a wealth of techniques to extract information from data that can be translated into knowledge about the underlying fluid mechanics. Moreover, ML algorithms can augment domain knowledge and automate tasks related to flow control and optimization. This article presents an overview of past history, current developments, and emerging opportunities of ML for fluid mechanics. We outline fundamental ML methodologies and discuss their uses for understanding, modeling, optimizing, and controlling fluid flows. The strengths and limitations of these methods are addressed from the perspective of scientific inquiry that considers data as an inherent part of modeling, experiments, and simulations. ML provides a powerful informationprocessing framework that can augment, and possibly even transform, current lines of fluid mechanics research and industrial applications.},
	language = {en},
	number = {1},
	urldate = {2021-01-14},
	journal = {Annual Review of Fluid Mechanics},
	author = {Brunton, Steven L. and Noack, Bernd R. and Koumoutsakos, Petros},
	month = jan,
	year = {2020},
	pages = {477--508},
}

@article{filoche_completing_nodate,
	title = {Completing physics-based model by learning hidden dynamics through data assimilation},
	abstract = {Data Assimilation remains the operational choice when it comes to forecast and estimate Earth’s dynamical systems. The analogy with Machine Learning has already been shown and is still being investigated to address the problem of improving physics-based models. Even though both techniques learn from data, machine learning focuses on inferring models while data assimilation concentrates on hidden system state estimation with the help of a dynamical model. In this work, we exploit the complementarity of these methods in a twin experiment where the system is partially observed and the known dynamics are incomplete. Finally, we succeed in partially retrieving the dynamics of a fully-unobserved variable by training a hybrid model through variational data assimilation.},
	language = {en},
	author = {Filoche, Arthur and Brajard, Julien and Charantonis, Anastase and Béréziat, Dominique},
	pages = {8},
}

@article{ehrhardt_relate_nodate,
	title = {{RELATE}: {Physically} {Plausible} {Multi}-{Object} {Scene} {Synthesis} {Using} {Structured} {Latent} {Spaces}},
	abstract = {We present RELATE, a model that learns to generate physically plausible scenes and videos of multiple interacting objects. Similar to other generative approaches, RELATE is trained end-to-end on raw, unlabeled data. RELATE combines an object-centric GAN formulation with a model that explicitly accounts for correlations between individual objects. This allows the model to generate realistic scenes and videos from a physically-interpretable parameterization. Furthermore, we show that modeling the object correlation is necessary to learn to disentangle object positions and identity. We ﬁnd that RELATE is also amenable to physically realistic scene editing and that it signiﬁcantly outperforms prior art in object-centric scene generation in both synthetic (CLEVR, ShapeStacks) and real-world data (cars). In addition, in contrast to state-of-the-art methods in object-centric generative modeling, RELATE also extends naturally to dynamic scenes and generates videos of high visual ﬁdelity. Source code, datasets and more results are available at http://geometry.cs.ucl.ac.uk/projects/2020/relate/.},
	language = {en},
	author = {Ehrhardt, Sébastien and Groth, Oliver and Monszpart, Áron and Engelcke, Martin and Posner, Ingmar and Mitra, Niloy J and Vedaldi, Andrea},
	pages = {12},
}

@article{li_causal_nodate,
	title = {Causal {Discovery} in {Physical} {Systems} from {Videos}},
	abstract = {Causal discovery is at the core of human cognition. It enables us to reason about the environment and make counterfactual predictions about unseen scenarios that can vastly differ from our previous experiences. We consider the task of causal discovery from videos in an end-to-end fashion without supervision on the ground-truth graph structure. In particular, our goal is to discover the structural dependencies among environmental and object variables: inferring the type and strength of interactions that have a causal effect on the behavior of the dynamical system. Our model consists of (a) a perception module that extracts a semantically meaningful and temporally consistent keypoint representation from images, (b) an inference module for determining the graph distribution induced by the detected keypoints, and (c) a dynamics module that can predict the future by conditioning on the inferred graph. We assume access to different conﬁgurations and environmental conditions, i.e., data from unknown interventions on the underlying system; thus, we can hope to discover the correct underlying causal graph without explicit interventions. We evaluate our method in a planar multi-body interaction environment and scenarios involving fabrics of different shapes like shirts and pants. Experiments demonstrate that our model can correctly identify the interactions from a short sequence of images and make long-term future predictions. The causal structure assumed by the model also allows it to make counterfactual predictions and extrapolate to systems of unseen interaction graphs or graphs of various sizes. Please refer to our project page for additional results: https://yunzhuli.github.io/V-CDN/.},
	language = {en},
	author = {Li, Yunzhu and Torralba, Antonio and Anandkumar, Animashree and Fox, Dieter and Garg, Animesh},
	pages = {13},
}

@article{um_solver---loop_nodate,
	title = {Solver-in-the-{Loop}: {Learning} from {Differentiable} {Physics} to {Interact} with {Iterative} {PDE}-{Solvers}},
	abstract = {Finding accurate solutions to partial differential equations (PDEs) is a crucial task in all scientiﬁc and engineering disciplines. It has recently been shown that machine learning methods can improve the solution accuracy by correcting for effects not captured by the discretized PDE. We target the problem of reducing numerical errors of iterative PDE solvers and compare different learning approaches for ﬁnding complex correction functions. We ﬁnd that previously used learning approaches are signiﬁcantly outperformed by methods that integrate the solver into the training loop and thereby allow the model to interact with the PDE during training. This provides the model with realistic input distributions that take previous corrections into account, yielding improvements in accuracy with stable rollouts of several hundred recurrent evaluation steps and surpassing even tailored supervised variants. We highlight the performance of the differentiable physics networks for a wide variety of PDEs, from non-linear advection-diffusion systems to threedimensional Navier-Stokes ﬂows.},
	language = {en},
	author = {Um, Kiwon and Brand, Robert},
	pages = {12},
}

@article{sengupta_ensembling_nodate,
	title = {Ensembling geophysical models with {Bayesian} {Neural} {Networks}},
	abstract = {Ensembles of geophysical models improve prediction accuracy and express uncertainties. We develop a novel data-driven ensembling strategy for combining geophysical models using Bayesian Neural Networks, which infers spatiotemporally varying model weights and bias, while accounting for heteroscedastic uncertainties in the observations. This produces more accurate and uncertaintyaware predictions without sacriﬁcing interpretability. Applied to the prediction of total column ozone from an ensemble of 15 chemistry-climate models, we ﬁnd that the Bayesian neural network ensemble (BayNNE) outperforms existing methods for ensembling physical models, achieving a 49.4\% reduction in RMSE for temporal extrapolation, and a 67.4\% reduction in RMSE for polar data voids, compared to a weighted mean. Uncertainty is also well-characterized, with 91.9\% of the data points in our extrapolation validation dataset lying within 2 standard deviations and 98.9\% within 3 standard deviations.},
	language = {en},
	author = {Sengupta, Ushnish and Amos, Matt and Rasmussen, Carl Edward},
	pages = {13},
}

@article{ba_blending_2019,
	title = {Blending {Diverse} {Physical} {Priors} with {Neural} {Networks}},
	url = {http://arxiv.org/abs/1910.00201},
	abstract = {Machine learning in context of physical systems merits a re-examination of the learning strategy. In addition to data, one can leverage a vast library of physical prior models (e.g. kinematics, ﬂuid ﬂow, etc) to perform more robust inference. The nascent sub-ﬁeld of physics-based learning (PBL) studies the blending of neural networks with physical priors. While previous PBL algorithms have been applied successfully to speciﬁc tasks, it is hard to generalize existing PBL methods to a wide range of physics-based problems. Such generalization would require an architecture that can adapt to variations in the correctness of the physics, or in the quality of training data. No such architecture exists. In this paper, we aim to generalize PBL, by making a ﬁrst attempt to bring neural architecture search (NAS) to the realm of PBL. We introduce a new method known as physics-based neural architecture search (PhysicsNAS) that is a top-performer across a diverse range of quality in the physical model and the dataset. Code is available at: https://github.com/PhysicsNAS/PhysicsNAS.},
	language = {en},
	urldate = {2021-01-11},
	journal = {arXiv:1910.00201 [cs, stat]},
	author = {Ba, Yunhao and Zhao, Guangyuan and Kadambi, Achuta},
	month = oct,
	year = {2019},
	note = {arXiv: 1910.00201},
	keywords = {machine learning},
}

@article{meng_multi-fidelity_2020,
	title = {Multi-fidelity {Bayesian} {Neural} {Networks}: {Algorithms} and {Applications}},
	shorttitle = {Multi-fidelity {Bayesian} {Neural} {Networks}},
	url = {http://arxiv.org/abs/2012.13294},
	abstract = {We propose a new class of Bayesian neural networks (BNNs) that can be trained using noisy data of variable fidelity, and we apply them to learn function approximations as well as to solve inverse problems based on partial differential equations (PDEs). These multi-fidelity BNNs consist of three neural networks: The first is a fully connected neural network, which is trained following the maximum a posteriori probability (MAP) method to fit the low-fidelity data; the second is a Bayesian neural network employed to capture the cross-correlation with uncertainty quantification between the low- and high-fidelity data; and the last one is the physics-informed neural network, which encodes the physical laws described by PDEs. For the training of the last two neural networks, we use the Hamiltonian Monte Carlo method to estimate accurately the posterior distributions for the corresponding hyperparameters. We demonstrate the accuracy of the present method using synthetic data as well as real measurements. Specifically, we first approximate a one- and four-dimensional function, and then infer the reaction rates in one- and two-dimensional diffusion-reaction systems. Moreover, we infer the sea surface temperature (SST) in the Massachusetts and Cape Cod Bays using satellite images and in-situ measurements. Taken together, our results demonstrate that the present method can capture both linear and nonlinear correlation between the low- and high-fideilty data adaptively, identify unknown parameters in PDEs, and quantify uncertainties in predictions, given a few scattered noisy high-fidelity data. Finally, we demonstrate that we can effectively and efficiently reduce the uncertainties and hence enhance the prediction accuracy with an active learning approach, using as examples a specific one-dimensional function approximation and an inverse PDE problem.},
	urldate = {2021-01-08},
	journal = {arXiv:2012.13294 [physics]},
	author = {Meng, Xuhui and Babaee, Hessam and Karniadakis, George Em},
	month = dec,
	year = {2020},
	note = {arXiv: 2012.13294},
	keywords = {computational physics, machine learning},
}

@article{meng_fast_2020,
	title = {A fast multi-fidelity method with uncertainty quantification for complex data correlations: {Application} to vortex-induced vibrations of marine risers},
	shorttitle = {A fast multi-fidelity method with uncertainty quantification for complex data correlations},
	url = {http://arxiv.org/abs/2012.13481},
	abstract = {We develop a fast multi-fidelity modeling method for very complex correlations between high- and low-fidelity data by working in modal space to extract the proper correlation function. We apply this method to infer the amplitude of motion of a flexible marine riser in cross-flow, subject to vortex-induced vibrations (VIV). VIV are driven by an absolute instability in the flow, which imposes a frequency (Strouhal) law that requires a matching with the impedance of the structure; this matching is easily achieved because of the rapid parametric variation of the added mass force. As a result, the wavenumber of the riser spatial response is within narrow bands of uncertainty. Hence, an error in wavenumber prediction can cause significant phase-related errors in the shape of the amplitude of response along the riser, rendering correlation between low- and high-fidelity data very complex. Working in modal space as outlined herein, dense data from low-fidelity data, provided by the semi-empirical computer code VIVA, can correlate in modal space with few high-fidelity data, obtained from experiments or fully-resolved CFD simulations, to correct both phase and amplitude and provide predictions that agree very well overall with the correct shape of the amplitude response. We also quantify the uncertainty in the prediction using Bayesian modeling and exploit this uncertainty to formulate an active learning strategy for the best possible location of the sensors providing the high fidelity measurements.},
	urldate = {2021-01-08},
	journal = {arXiv:2012.13481 [physics]},
	author = {Meng, Xuhui and Wang, Zhicheng and Fan, Dixia and Triantafyllou, Michael and Karniadakis, George Em},
	month = dec,
	year = {2020},
	note = {arXiv: 2012.13481},
	keywords = {fluid dynamics},
}

@article{lin_ensemble_2020,
	title = {Ensemble {Distillation} for {Robust} {Model} {Fusion} in {Federated} {Learning}},
	url = {http://arxiv.org/abs/2006.07242},
	abstract = {Federated Learning (FL) is a machine learning setting where many devices collaboratively train a machine learning model while keeping the training data decentralized. In most of the current training schemes the central model is refined by averaging the parameters of the server model and the updated parameters from the client side. However, directly averaging model parameters is only possible if all models have the same structure and size, which could be a restrictive constraint in many scenarios. In this work we investigate more powerful and more flexible aggregation schemes for FL. Specifically, we propose ensemble distillation for model fusion, i.e. training the central classifier through unlabeled data on the outputs of the models from the clients. This knowledge distillation technique mitigates privacy risk and cost to the same extent as the baseline FL algorithms, but allows flexible aggregation over heterogeneous client models that can differ e.g. in size, numerical precision or structure. We show in extensive empirical experiments on various CV/NLP datasets (CIFAR-10/100, ImageNet, AG News, SST2) and settings (heterogeneous models/data) that the server model can be trained much faster, requiring fewer communication rounds than any existing FL technique so far.},
	urldate = {2020-12-16},
	journal = {arXiv:2006.07242 [cs, stat]},
	author = {Lin, Tao and Kong, Lingjing and Stich, Sebastian U. and Jaggi, Martin},
	month = oct,
	year = {2020},
	note = {arXiv: 2006.07242},
	keywords = {machine learning},
}

@article{dzeroski_learning_2003,
	title = {Learning population dynamics models from data and domain knowledge},
	volume = {170},
	issn = {03043800},
	url = {https://linkinghub.elsevier.com/retrieve/pii/S0304380003002217},
	doi = {10.1016/S0304-3800(03)00221-7},
	abstract = {This paper is concerned with integrating knowledge-based modeling or modeling from ﬁrst principles, with data-driven or automated modeling of dynamic systems. The approach presented here includes methods for equation discovery: unlike mainstream system identiﬁcation methods, which work under the assumption that the form of the equations is known, equation discovery systems explore a space of possible equation structures. We propose a formalism for representing knowledge about processes in population dynamics domains, and a method to transform such knowledge into an operational form that could be used by equation discovery systems. We also describe the extensions of the equation discovery system Lagramge necessary to incorporate this kind of knowledge in the process of equation discovery.},
	language = {en},
	number = {2-3},
	urldate = {2020-12-11},
	journal = {Ecological Modelling},
	author = {Džeroski, Sašo and Todorovski, Ljupčo},
	month = dec,
	year = {2003},
	pages = {129--140},
}

@incollection{bohan_chapter_2013,
	series = {Ecological {Networks} in an {Agricultural} {World}},
	title = {Chapter {One} - {Networking} {Agroecology}: {Integrating} the {Diversity} of {Agroecosystem} {Interactions}},
	volume = {49},
	shorttitle = {Chapter {One} - {Networking} {Agroecology}},
	url = {http://www.sciencedirect.com/science/article/pii/B9780124200029000019},
	abstract = {Worldwide demand for food will increase dramatically in the future as global human population grows. Increasing efficiency of crop production is unlikely to be sufficient to meet the demand, presenting a long-term threat to humanity’s ‘well-being’. Knowledge of the system-level behaviour of agroecosystems, however, remains surprisingly limited, reflecting the agricultural focus on particular species. This is starting to change towards an ecosystem and network-based approach, following the recent revolution in thinking about resource use and sustainability in our other global food production industry: commercial fisheries. Agroecosystems appear to retain plasticity of ecological processes that might be manipulated for productivity and sustainability. Network structure and dynamics have substantial impacts on ecosystem performance, but evidence from agroecosystems lags behind network theory. Here, we provide an introduction to network theory and application in agroecosystems, identify network metrics for management and environmental change, and, finally, we highlight gaps in our current knowledge and key research themes. These themes include: is the structure of agroecological networks affected by sampling; how do ecosystem services ‘emerge’ empirically from ecological organization, function and network properties; how do spatial and temporal scale and resolution influence system performance; and, can network agroecology be used to design systems that maximize ecosystem services?},
	language = {en},
	urldate = {2020-12-11},
	booktitle = {Advances in {Ecological} {Research}},
	publisher = {Academic Press},
	author = {Bohan, David A. and Raybould, Alan and Mulder, Christian and Woodward, Guy and Tamaddoni-Nezhad, Alireza and Bluthgen, Nico and Pocock, Michael J. O. and Muggleton, Stephen and Evans, Darren M. and Astegiano, Julia and Massol, François and Loeuille, Nicolas and Petit, Sandrine and Macfadyen, Sarina},
	editor = {Woodward, Guy and Bohan, David A.},
	month = jan,
	year = {2013},
	doi = {10.1016/B978-0-12-420002-9.00001-9},
	keywords = {ecology},
	pages = {1--67},
}

@article{oyetunde_leveraging_2018,
	title = {Leveraging knowledge engineering and machine learning for microbial bio-manufacturing},
	volume = {36},
	issn = {0734-9750},
	url = {http://www.sciencedirect.com/science/article/pii/S0734975018300806},
	doi = {10.1016/j.biotechadv.2018.04.008},
	abstract = {Genome scale modeling (GSM) predicts the performance of microbial workhorses and helps identify beneficial gene targets. GSM integrated with intracellular flux dynamics, omics, and thermodynamics have shown remarkable progress in both elucidating complex cellular phenomena and computational strain design (CSD). Nonetheless, these models still show high uncertainty due to a poor understanding of innate pathway regulations, metabolic burdens, and other factors (such as stress tolerance and metabolite channeling). Besides, the engineered hosts may have genetic mutations or non-genetic variations in bioreactor conditions and thus CSD rarely foresees fermentation rate and titer. Metabolic models play important role in design-build-test-learn cycles for strain improvement, and machine learning (ML) may provide a viable complementary approach for driving strain design and deciphering cellular processes. In order to develop quality ML models, knowledge engineering leverages and standardizes the wealth of information in literature (e.g., genomic/phenomic data, synthetic biology strategies, and bioprocess variables). Data driven frameworks can offer new constraints for mechanistic models to describe cellular regulations, to design pathways, to search gene targets, and to estimate fermentation titer/rate/yield under specified growth conditions (e.g., mixing, nutrients, and O2). This review highlights the scope of information collections, database constructions, and machine learning techniques (such as deep learning and transfer learning), which may facilitate “Learn and Design” for strain development.},
	language = {en},
	number = {4},
	urldate = {2020-12-11},
	journal = {Biotechnology Advances},
	author = {Oyetunde, Tolutola and Bao, Forrest Sheng and Chen, Jiung-Wen and Martin, Hector Garcia and Tang, Yinjie J.},
	month = jul,
	year = {2018},
	keywords = {neural networks},
	pages = {1308--1315},
}

@inproceedings{bialkowski_large-scale_2014,
	title = {Large-{Scale} {Analysis} of {Soccer} {Matches} {Using} {Spatiotemporal} {Tracking} {Data}},
	doi = {10.1109/ICDM.2014.133},
	abstract = {Although the collection of player and ball tracking data is fast becoming the norm in professional sports, large-scale mining of such spatiotemporal data has yet to surface. In this paper, given an entire season's worth of player and ball tracking data from a professional soccer league (≈400,000,000 data points), we present a method which can conduct both individual player and team analysis. Due to the dynamic, continuous and multi-player nature of team sports like soccer, a major issue is aligning player positions over time. We present a "role-based" representation that dynamically updates each player's relative role at each frame and demonstrate how this captures the short-term context to enable both individual player and team analysis. We discover role directly from data by utilizing a minimum entropy data partitioning method and show how this can be used to accurately detect and visualize formations, as well as analyze individual player behavior.},
	booktitle = {2014 {IEEE} {International} {Conference} on {Data} {Mining}},
	author = {Bialkowski, A. and Lucey, P. and Carr, P. and Yue, Y. and Sridharan, S. and Matthews, I.},
	month = dec,
	year = {2014},
	note = {ISSN: 2374-8486},
	keywords = {data analysis, entropy, soccer, spatio-temporal modelling, sports, sports analytics, visualization},
	pages = {725--730},
}

@article{noauthor_five-step_nodate,
	title = {The {Five}-{Step} {Process} {Framework} for {Project} {Development}},
	language = {en},
	pages = {87},
}

@article{karpatne_physics-guided_2018,
	title = {Physics-guided {Neural} {Networks} ({PGNN}): {An} {Application} in {Lake} {Temperature} {Modeling}},
	shorttitle = {Physics-guided {Neural} {Networks} ({PGNN})},
	url = {http://arxiv.org/abs/1710.11431},
	abstract = {This paper introduces a novel framework for combining scientific knowledge of physics-based models with neural networks to advance scientific discovery. This framework, termed as physics-guided neural network (PGNN), leverages the output of physics-based model simulations along with observational features to generate predictions using a neural network architecture. Further, this paper presents a novel framework for using physics-based loss functions in the learning objective of neural networks, to ensure that the model predictions not only show lower errors on the training set but are also scientifically consistent with the known physics on the unlabeled set. We illustrate the effectiveness of PGNN for the problem of lake temperature modeling, where physical relationships between the temperature, density, and depth of water are used to design a physics-based loss function. By using scientific knowledge to guide the construction and learning of neural networks, we are able to show that the proposed framework ensures better generalizability as well as scientific consistency of results.},
	urldate = {2020-12-07},
	journal = {arXiv:1710.11431 [physics, stat]},
	author = {Karpatne, Anuj and Watkins, William and Read, Jordan and Kumar, Vipin},
	month = feb,
	year = {2018},
	note = {arXiv: 1710.11431},
	keywords = {computer vision, machine learning, statistical physics},
}

@article{sengupta_ensembling_2020,
	title = {Ensembling geophysical models with {Bayesian} {Neural} {Networks}},
	url = {http://arxiv.org/abs/2010.03561},
	abstract = {Ensembles of geophysical models improve projection accuracy and express uncertainties. We develop a novel data-driven ensembling strategy for combining geophysical models using Bayesian Neural Networks, which infers spatiotemporally varying model weights and bias while accounting for heteroscedastic uncertainties in the observations. This produces more accurate and uncertainty-aware projections without sacrificing interpretability. Applied to the prediction of total column ozone from an ensemble of 15 chemistry-climate models, we find that the Bayesian neural network ensemble (BayNNE) outperforms existing ensembling methods, achieving a 49.4\% reduction in RMSE for temporal extrapolation, and a 67.4\% reduction in RMSE for polar data voids, compared to a weighted mean. Uncertainty is also well-characterized, with 90.6\% of the data points in our extrapolation validation dataset lying within 2 standard deviations and 98.5\% within 3 standard deviations.},
	urldate = {2020-12-07},
	journal = {arXiv:2010.03561 [physics, stat]},
	author = {Sengupta, Ushnish and Amos, Matt and Hosking, J. Scott and Rasmussen, Carl Edward and Juniper, Matthew and Young, Paul J.},
	month = oct,
	year = {2020},
	note = {arXiv: 2010.03561},
	keywords = {geophysics, machine learning},
}

@article{li_neural_2020,
	title = {Neural {Operator}: {Graph} {Kernel} {Network} for {Partial} {Differential} {Equations}},
	shorttitle = {Neural {Operator}},
	url = {http://arxiv.org/abs/2003.03485},
	abstract = {The classical development of neural networks has been primarily for mappings between a ﬁnite-dimensional Euclidean space and a set of classes, or between two ﬁnite-dimensional Euclidean spaces. The purpose of this work is to generalize neural networks so that they can learn mappings between inﬁnite-dimensional spaces (operators). The key innovation in our work is that a single set of network parameters, within a carefully designed network architecture, may be used to describe mappings between inﬁnite-dimensional spaces and between different ﬁnitedimensional approximations of those spaces. We formulate approximation of the inﬁnite-dimensional mapping by composing nonlinear activation functions and a class of integral operators. The kernel integration is computed by message passing on graph networks. This approach has substantial practical consequences which we will illustrate in the context of mappings between input data to partial differential equations (PDEs) and their solutions. In this context, such learned networks can generalize among different approximation methods for the PDE (such as ﬁnite difference or ﬁnite element methods) and among approximations corresponding to different underlying levels of resolution and discretization. Experiments conﬁrm that the proposed graph kernel network does have the desired properties and show competitive performance compared to the state of the art solvers.},
	language = {en},
	urldate = {2020-11-25},
	journal = {arXiv:2003.03485 [cs, math, stat]},
	author = {Li, Zongyi and Kovachki, Nikola and Azizzadenesheli, Kamyar and Liu, Burigede and Bhattacharya, Kaushik and Stuart, Andrew and Anandkumar, Anima},
	month = mar,
	year = {2020},
	note = {arXiv: 2003.03485},
	keywords = {machine learning, numerical modelling},
}

@article{gregory_synthesis_2018,
	title = {The synthesis of data from instrumented structures and physics-based models via {Gaussian} processes},
	url = {https://arxiv.org/abs/1811.10882v2},
	doi = {10.1016/j.jcp.2019.04.065},
	abstract = {A recent development which is poised to disrupt current structural
engineering practice is the use of data obtained from physical structures such
as bridges, viaducts and buildings. These data can represent how the structure
responds to various stimuli over time when in operation, providing engineers
with a unique insight into how their designs are performing. With the advent of
advanced sensing technologies and the Internet of Things, the efficient
interpretation of structural health monitoring data has become a big data
challenge. Many models have been proposed in literature to represent such data,
such as linear statistical models. Based upon these models, the health of the
structure is reasoned about, e.g. through damage indices, changes in likelihood
and statistical parameter estimates. On the other hand, physics-based models
are typically used when designing structures to predict how the structure will
respond to operational stimuli. What remains unclear in the literature is how
to combine the observed data with information from the idealised physics-based
model into a model that describes the responses of the operational structure.
This paper introduces a new approach which fuses together observed data from a
physical structure during operation and information from a mathematical model.
The observed data are combined with data simulated from the physics-based model
using a multi-output Gaussian process formulation. The novelty of this method
is how the information from observed data and the physics-based model is
balanced to obtain a representative model of the structures response to
stimuli. We present our method using data obtained from a fibre-optic sensor
network installed on experimental railway sleepers. We discuss how this
approach can be used to reason about changes in the structures behaviour over
time using simulations and experimental data.},
	language = {en},
	urldate = {2020-11-25},
	author = {Gregory, Alastair and Lau, Din-Houn and Girolami, Mark and Butler, Liam and Elshafie, Mohammed},
	month = nov,
	year = {2018},
}

@book{taylor_classical_2005,
	address = {Sausalito, Calif},
	title = {Classical mechanics},
	isbn = {978-1-891389-22-1},
	publisher = {University Science Books},
	author = {Taylor, John R.},
	year = {2005},
	keywords = {physics},
}

@article{rasp_deep_2018,
	title = {Deep learning to represent subgrid processes in climate models},
	volume = {115},
	copyright = {Copyright © 2018 the Author(s). Published by PNAS.. https://creativecommons.org/licenses/by-nc-nd/4.0/This open access article is distributed under Creative Commons Attribution-NonCommercial-NoDerivatives License 4.0 (CC BY-NC-ND).},
	issn = {0027-8424, 1091-6490},
	url = {https://www.pnas.org/content/115/39/9684},
	doi = {10.1073/pnas.1810286115},
	abstract = {The representation of nonlinear subgrid processes, especially clouds, has been a major source of uncertainty in climate models for decades. Cloud-resolving models better represent many of these processes and can now be run globally but only for short-term simulations of at most a few years because of computational limitations. Here we demonstrate that deep learning can be used to capture many advantages of cloud-resolving modeling at a fraction of the computational cost. We train a deep neural network to represent all atmospheric subgrid processes in a climate model by learning from a multiscale model in which convection is treated explicitly. The trained neural network then replaces the traditional subgrid parameterizations in a global general circulation model in which it freely interacts with the resolved dynamics and the surface-flux scheme. The prognostic multiyear simulations are stable and closely reproduce not only the mean climate of the cloud-resolving simulation but also key aspects of variability, including precipitation extremes and the equatorial wave spectrum. Furthermore, the neural network approximately conserves energy despite not being explicitly instructed to. Finally, we show that the neural network parameterization generalizes to new surface forcing patterns but struggles to cope with temperatures far outside its training manifold. Our results show the feasibility of using deep learning for climate model parameterization. In a broader context, we anticipate that data-driven Earth system model development could play a key role in reducing climate prediction uncertainty in the coming decade.},
	language = {en},
	number = {39},
	urldate = {2020-11-17},
	journal = {Proceedings of the National Academy of Sciences},
	author = {Rasp, Stephan and Pritchard, Michael S. and Gentine, Pierre},
	month = sep,
	year = {2018},
	pmid = {30190437},
	note = {ISBN: 9781810286112
Publisher: National Academy of Sciences
Section: Physical Sciences},
	keywords = {climate modeling, convection, deep learning},
	pages = {9684--9689},
}

@inproceedings{xue_deepfusion_2019,
	address = {Catania Italy},
	title = {{DeepFusion}: {A} {Deep} {Learning} {Framework} for the {Fusion} of {Heterogeneous} {Sensory} {Data}},
	isbn = {978-1-4503-6764-6},
	shorttitle = {{DeepFusion}},
	url = {https://dl.acm.org/doi/10.1145/3323679.3326513},
	doi = {10.1145/3323679.3326513},
	abstract = {In recent years, significant research efforts have been spent towards building intelligent and user-friendly IoT systems to enable a new generation of applications capable of performing complex sensing and recognition tasks. In many of such applications, there are usually multiple different sensors monitoring the same object. Each of these sensors can be regarded as an information source and provides us a unique “view” of the observed object. Intuitively, if we can combine the complementary information carried by multiple sensors, we will be able to improve the sensing performance. Towards this end, we propose DeepFusion, a unified multi-sensor deep learning framework, to learn informative representations of heterogeneous sensory data. DeepFusion can combine different sensors’ information weighted by the quality of their data and incorporate cross-sensor correlations, and thus can benefit a wide spectrum of IoT applications. To evaluate the proposed DeepFusion model, we set up two real-world human activity recognition testbeds using commercialized wearable and wireless sensing devices. Experiment results show that DeepFusion can outperform the state-of-the-art human activity recognition methods.},
	language = {en},
	urldate = {2020-11-17},
	booktitle = {Proceedings of the {Twentieth} {ACM} {International} {Symposium} on {Mobile} {Ad} {Hoc} {Networking} and {Computing}},
	publisher = {ACM},
	author = {Xue, Hongfei and Jiang, Wenjun and Miao, Chenglin and Yuan, Ye and Ma, Fenglong and Ma, Xin and Wang, Yijiang and Yao, Shuochao and Xu, Wenyao and Zhang, Aidong and Su, Lu},
	month = jul,
	year = {2019},
	pages = {151--160},
}

@article{schneider_earth_2017,
	title = {Earth {System} {Modeling} 2.0: {A} {Blueprint} for {Models} {That} {Learn} {From} {Observations} and {Targeted} {High}-{Resolution} {Simulations}},
	volume = {44},
	copyright = {©2017. American Geophysical Union. All Rights Reserved.},
	issn = {1944-8007},
	shorttitle = {Earth {System} {Modeling} 2.0},
	url = {https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1002/2017GL076101},
	doi = {https://doi.org/10.1002/2017GL076101},
	abstract = {Climate projections continue to be marred by large uncertainties, which originate in processes that need to be parameterized, such as clouds, convection, and ecosystems. But rapid progress is now within reach. New computational tools and methods from data assimilation and machine learning make it possible to integrate global observations and local high-resolution simulations in an Earth system model (ESM) that systematically learns from both and quantifies uncertainties. Here we propose a blueprint for such an ESM. We outline how parameterization schemes can learn from global observations and targeted high-resolution simulations, for example, of clouds and convection, through matching low-order statistics between ESMs, observations, and high-resolution simulations. We illustrate learning algorithms for ESMs with a simple dynamical system that shares characteristics of the climate system; and we discuss the opportunities the proposed framework presents and the challenges that remain to realize it.},
	language = {en},
	number = {24},
	urldate = {2020-11-17},
	journal = {Geophysical Research Letters},
	author = {Schneider, Tapio and Lan, Shiwei and Stuart, Andrew and Teixeira, João},
	year = {2017},
	note = {\_eprint: https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1002/2017GL076101},
	keywords = {climate modeling, data fusion, machine learning, mcmc},
	pages = {12,396--12,417},
}

@article{strobach_improvement_2015,
	title = {Improvement of climate predictions and reduction of their uncertainties using learning algorithms},
	volume = {15},
	issn = {1680-7316},
	url = {https://acp.copernicus.org/articles/15/8631/2015/},
	doi = {https://doi.org/10.5194/acp-15-8631-2015},
	abstract = {{\textless}p{\textgreater}{\textless}strong{\textgreater}Abstract.{\textless}/strong{\textgreater} Simulated climate dynamics, initialized with observed conditions, is expected to be synchronized, for several years, with the actual dynamics. However, the predictions of climate models are not sufficiently accurate. Moreover, there is a large variance between simulations initialized at different times and between different models. One way to improve climate predictions and to reduce the associated uncertainties is to use an ensemble of climate model predictions, weighted according to their past performances. Here, we show that skillful predictions, for a decadal time scale, of the 2 m temperature can be achieved by applying a sequential learning algorithm to an ensemble of decadal climate model simulations. The predictions generated by the learning algorithm are shown to be better than those of each of the models in the ensemble, the better performing simple average and a reference climatology. In addition, the uncertainties associated with the predictions are shown to be reduced relative to those derived from an equally weighted ensemble of bias-corrected predictions. The results show that learning algorithms can help to better assess future climate dynamics.{\textless}/p{\textgreater}},
	language = {English},
	number = {15},
	urldate = {2020-11-17},
	journal = {Atmospheric Chemistry and Physics},
	author = {Strobach, E. and Bel, G.},
	month = aug,
	year = {2015},
	note = {Publisher: Copernicus GmbH},
	pages = {8631--8641},
}

@inproceedings{quick_multifidelity_2019,
	address = {San Diego, California},
	title = {Multifidelity {Uncertainty} {Quantification} with {Applications} in {Wind} {Turbine} {Aerodynamics}},
	isbn = {978-1-62410-578-4},
	url = {https://arc.aiaa.org/doi/10.2514/6.2019-0542},
	doi = {10.2514/6.2019-0542},
	language = {en},
	urldate = {2020-11-17},
	booktitle = {{AIAA} {Scitech} 2019 {Forum}},
	publisher = {American Institute of Aeronautics and Astronautics},
	author = {Quick, Julian and Hamlington, Peter E. and King, Ryan and Sprague, Michael A.},
	month = jan,
	year = {2019},
}

@article{vaccaro_adaptive_2011,
	title = {An adaptive framework based on multi-model data fusion for one-day-ahead wind power forecasting},
	volume = {81},
	issn = {0378-7796},
	url = {http://www.sciencedirect.com/science/article/pii/S0378779610002816},
	doi = {10.1016/j.epsr.2010.11.009},
	abstract = {This paper proposes a novel framework for one-day-ahead wind power forecasting based on information amalgamation from multiple sources. The final objective is to provide a better solution than could otherwise be achieved from the use of single-source data alone. The proposed framework combines multiple forecasting models and adaptive machine learning techniques for information processing. The input data sources are the wind forecast profiles computed by synoptic physical models and measured data coming from meteorological stations which are amalgamated via an adaptive supervised learning system. The latter is based on a local learning algorithm, called the Lazy Learning (LL) algorithm. This algorithm is sequentially updated, in order to adapt the whole architecture to “new” operating conditions. Experimental results obtained on a one-year time scenario show the effectiveness of the proposed data fusion paradigm in addressing the problem of one-day-ahead wind power forecasting.},
	language = {en},
	number = {3},
	urldate = {2020-11-17},
	journal = {Electric Power Systems Research},
	author = {Vaccaro, A. and Mercogliano, P. and Schiano, P. and Villacci, D.},
	month = mar,
	year = {2011},
	keywords = {data analysis, wind power forecasting},
	pages = {775--782},
}

@article{dong_survey_2020,
	title = {A survey on ensemble learning},
	volume = {14},
	issn = {2095-2236},
	url = {https://doi.org/10.1007/s11704-019-8208-z},
	doi = {10.1007/s11704-019-8208-z},
	abstract = {Despite significant successes achieved in knowledge discovery, traditional machine learning methods may fail to obtain satisfactory performances when dealing with complex data, such as imbalanced, high-dimensional, noisy data, etc. The reason behind is that it is difficult for these methods to capture multiple characteristics and underlying structure of data. In this context, it becomes an important topic in the data mining field that how to effectively construct an efficient knowledge discovery and mining model. Ensemble learning, as one research hot spot, aims to integrate data fusion, data modeling, and data mining into a unified framework. Specifically, ensemble learning firstly extracts a set of features with a variety of transformations. Based on these learned features, multiple learning algorithms are utilized to produce weak predictive results. Finally, ensemble learning fuses the informative knowledge from the above results obtained to achieve knowledge discovery and better predictive performance via voting schemes in an adaptive way. In this paper, we review the research progress of the mainstream approaches of ensemble learning and classify them based on different characteristics. In addition, we present challenges and possible research directions for each mainstream approach of ensemble learning, and we also give an extra introduction for the combination of ensemble learning with other machine learning hot spots such as deep learning, reinforcement learning, etc.},
	language = {en},
	number = {2},
	urldate = {2020-11-12},
	journal = {Frontiers of Computer Science},
	author = {Dong, Xibin and Yu, Zhiwen and Cao, Wenming and Shi, Yifan and Ma, Qianli},
	month = apr,
	year = {2020},
	pages = {241--258},
}

@article{raissi_inferring_2017,
	title = {Inferring solutions of differential equations using noisy multi-fidelity data},
	volume = {335},
	issn = {00219991},
	url = {http://arxiv.org/abs/1607.04805},
	doi = {10.1016/j.jcp.2017.01.060},
	abstract = {For more than two centuries, solutions of differential equations have been obtained either analytically or numerically based on typically well-behaved forcing and boundary conditions for well-posed problems. We are changing this paradigm in a fundamental way by establishing an interface between probabilistic machine learning and differential equations. We develop data-driven algorithms for general linear equations using Gaussian process priors tailored to the corresponding integro-differential operators. The only observables are scarce noisy multi-fidelity data for the forcing and solution that are not required to reside on the domain boundary. The resulting predictive posterior distributions quantify uncertainty and naturally lead to adaptive solution refinement via active learning. This general framework circumvents the tyranny of numerical discretization as well as the consistency and stability issues of time-integration, and is scalable to high-dimensions.},
	urldate = {2020-11-12},
	journal = {Journal of Computational Physics},
	author = {Raissi, Maziar and Perdikaris, Paris and Karniadakis, George Em},
	month = apr,
	year = {2017},
	note = {arXiv: 1607.04805},
	keywords = {machine learning},
	pages = {736--746},
}

@article{singh_hydrologic_2018,
	title = {Hydrologic modeling: progress and future directions},
	volume = {5},
	issn = {2196-4092},
	shorttitle = {Hydrologic modeling},
	url = {https://doi.org/10.1186/s40562-018-0113-z},
	doi = {10.1186/s40562-018-0113-z},
	abstract = {Briefly tracing the history of hydrologic modeling, this paper discusses the progress that has been achieved in hydrologic modeling since the advent of computer and what the future may have in store for hydrologic modeling. Hydrologic progress can be described through the developments in data collection and processing, concepts and theories, integration with allied sciences, computational and analysis tools, and models and model results. It is argued that with the aid of new information gathering and computational tools, hydrology will witness greater integration with both technical and non-technical areas and increasing applications of information technology tools. Furthermore, hydrology will play an increasingly important role in meeting grand challenges of the twenty-first century, such as food security, water security, energy security, health security, ecosystem security, and sustainable development.},
	number = {1},
	urldate = {2020-11-12},
	journal = {Geoscience Letters},
	author = {Singh, Vijay P.},
	month = may,
	year = {2018},
	pages = {15},
}

@article{diaz-isaac_calibration_2019,
	title = {Calibration of a multi-physics ensemble for estimating the uncertainty of a greenhouse gas atmospheric transport model},
	volume = {19},
	issn = {1680-7316},
	url = {https://acp.copernicus.org/articles/19/5695/2019/},
	doi = {https://doi.org/10.5194/acp-19-5695-2019},
	abstract = {{\textless}p{\textgreater}{\textless}strong{\textgreater}Abstract.{\textless}/strong{\textgreater} Atmospheric inversions have been used to assess biosphere–atmosphere {\textless}span class="inline-formula"{\textgreater}CO$_{\textrm{2}}${\textless}/span{\textgreater} surface exchanges at various scales, but variability among inverse flux estimates remains significant, especially at continental scales. Atmospheric transport errors are one of the main contributors to this variability. To characterize transport errors and their spatiotemporal structures, we present an objective method to generate a calibrated ensemble adjusted with meteorological measurements collected across a region, here the upper US Midwest in midsummer. Using multiple model configurations of the Weather Research and Forecasting (WRF) model, we show that a reduced number of simulations (less than 10 members) reproduces the transport error characteristics of a 45-member ensemble while minimizing the size of the ensemble. The large ensemble of 45 members was constructed using different physics parameterization (i.e., land surface models (LSMs), planetary boundary layer (PBL) schemes, cumulus parameterizations and microphysics parameterizations) and meteorological initial/boundary conditions. All the different models were coupled to {\textless}span class="inline-formula"{\textgreater}CO$_{\textrm{2}}${\textless}/span{\textgreater} fluxes and lateral boundary conditions from CarbonTracker to simulate {\textless}span class="inline-formula"{\textgreater}CO$_{\textrm{2}}${\textless}/span{\textgreater} mole fractions. Observed meteorological variables critical to inverse flux estimates, PBL wind speed, PBL wind direction and PBL height are used to calibrate our ensemble over the region. Two optimization techniques (i.e., simulated annealing and a genetic algorithm) are used for the selection of the optimal ensemble using the flatness of the rank histograms as the main criterion. We also choose model configurations that minimize the systematic errors (i.e., monthly biases) in the ensemble. We evaluate the impact of transport errors on atmospheric {\textless}span class="inline-formula"{\textgreater}CO$_{\textrm{2}}${\textless}/span{\textgreater} mole fraction to represent up to 40\&thinsp;\% of the model–data mismatch (fraction of the total variance). We conclude that a carefully chosen subset of the physics ensemble can represent the uncertainties in the full ensemble, and that transport ensembles calibrated with relevant meteorological variables provide a promising path forward for improving the treatment of transport uncertainties in atmospheric inverse flux estimates.{\textless}/p{\textgreater}},
	language = {English},
	number = {8},
	urldate = {2020-11-11},
	journal = {Atmospheric Chemistry and Physics},
	author = {Díaz-Isaac, Liza I. and Lauvaux, Thomas and Bocquet, Marc and Davis, Kenneth J.},
	month = apr,
	year = {2019},
	note = {Publisher: Copernicus GmbH},
	pages = {5695--5718},
}

@article{alexiadis_deep_2019,
	title = {Deep multiphysics: {Coupling} discrete multiphysics with machine learning to attain self-learning in-silico models replicating human physiology},
	volume = {98},
	issn = {0933-3657},
	shorttitle = {Deep multiphysics},
	url = {http://www.sciencedirect.com/science/article/pii/S0933365718305852},
	doi = {10.1016/j.artmed.2019.06.005},
	abstract = {Objectives
The objective of this study is to devise a modelling strategy for attaining in-silico models replicating human physiology and, in particular, the activity of the autonomic nervous system.
Method
Discrete Multiphysics (a multiphysics modelling technique) and Reinforcement Learning (a Machine Learning algorithm) are combined to achieve an in-silico model with the ability of self-learning and replicating feedback loops occurring in human physiology. Computational particles, used in Discrete Multiphysics to model biological systems, are associated to (computational) neurons: Reinforcement Learning trains these neurons to behave like they would in real biological systems.
Results
As benchmark/validation, we use the case of peristalsis in the oesophagus. Results show that the in-silico model effectively learns by itself how to propel the bolus in the oesophagus.
Conclusions
The combination of first principles modelling (e.g. multiphysics) and machine learning (e.g. Reinforcement Learning) represents a new powerful tool for in-silico modelling of human physiology. Biological feedback loops occurring, for instance, in peristaltic or metachronal motion, which until now could not be accounted for in in-silico models, can be tackled by the proposed technique.},
	language = {en},
	urldate = {2020-11-11},
	journal = {Artificial Intelligence in Medicine},
	author = {Alexiadis, Alessio},
	month = jul,
	year = {2019},
	keywords = {multi-physics, physics-informed learning, reinforcement learning},
	pages = {27--34},
}

@article{ricchi_multi-physics_2019,
	title = {Multi-{Physics} {Ensemble} versus {Atmosphere}–{Ocean} {Coupled} {Model} {Simulations} for a {Tropical}-{Like} {Cyclone} in the {Mediterranean} {Sea}},
	volume = {10},
	copyright = {http://creativecommons.org/licenses/by/3.0/},
	url = {https://www.mdpi.com/2073-4433/10/4/202},
	doi = {10.3390/atmos10040202},
	abstract = {Between 19 and 22 January 2014, a baroclinic wave moving eastward from the Atlantic Ocean generated a cut-off low over the Strait of Gibraltar and was responsible for the subsequent intensification of an extra-tropical cyclone. This system exhibited tropical-like features in the following stages of its life cycle and remained active for approximately 80 h, moving along the Mediterranean Sea from west to east, eventually reaching the Adriatic Sea. Two different modeling approaches, which are comparable in terms of computational cost, are analyzed here to represent the cyclone evolution. First, a multi-physics ensemble using different microphysics and turbulence parameterization schemes available in the WRF (weather research and forecasting) model is employed. Second, the COAWST (coupled ocean\&ndash;atmosphere wave sediment transport modeling system) suite, including WRF as an atmospheric model, ROMS (regional ocean modeling system) as an ocean model, and SWAN (simulating waves in nearshore) as a wave model, is used. The advantage of using a coupled modeling system is evaluated taking into account air\&ndash;sea interaction processes at growing levels of complexity. First, a high-resolution sea surface temperature (SST) field, updated every 6 h, is used to force a WRF model stand-alone atmospheric simulation. Later, a two-way atmosphere\&ndash;ocean coupled configuration is employed using COAWST, where SST is updated using consistent sea surface fluxes in the atmospheric and ocean models. Results show that a 1D ocean model is able to reproduce the evolution of the cyclone rather well, given a high-resolution initial SST field produced by ROMS after a long spin-up time. Additionally, coupled simulations reproduce more accurate (less intense) sea surface heat fluxes and a cyclone track and intensity, compared with a multi-physics ensemble of standalone atmospheric simulations.},
	language = {en},
	number = {4},
	urldate = {2020-11-11},
	journal = {Atmosphere},
	author = {Ricchi, Antonio and Miglietta, Mario Marcello and Bonaldo, Davide and Cioni, Guido and Rizza, Umberto and Carniel, Sandro},
	month = apr,
	year = {2019},
	note = {Number: 4
Publisher: Multidisciplinary Digital Publishing Institute},
	keywords = {multi-physics},
	pages = {202},
}

@article{wu_steepest_2020,
	title = {Steepest {Descent} {Neural} {Architecture} {Optimization}: {Escaping} {Local} {Optimum} with {Signed} {Neural} {Splitting}},
	shorttitle = {Steepest {Descent} {Neural} {Architecture} {Optimization}},
	url = {http://arxiv.org/abs/2003.10392},
	abstract = {Developing efficient and principled neural architecture optimization methods is a critical challenge of modern deep learning. Recently, Liu et al.[19] proposed a splitting steepest descent (S2D) method that jointly optimizes the neural parameters and architectures based on progressively growing network structures by splitting neurons into multiple copies in a steepest descent fashion. However, S2D suffers from a local optimality issue when all the neurons become "splitting stable", a concept akin to local stability in parametric optimization. In this work, we develop a significant and surprising extension of the splitting descent framework that addresses the local optimality issue. The idea is to observe that the original S2D is unnecessarily restricted to splitting neurons into positive weighted copies. By simply allowing both positive and negative weights during splitting, we can eliminate the appearance of splitting stability in S2D and hence escape the local optima to obtain better performance. By incorporating signed splittings, we significantly extend the optimization power of splitting steepest descent both theoretically and empirically. We verify our method on various challenging benchmarks such as CIFAR-100, ImageNet and ModelNet40, on which we outperform S2D and other advanced methods on learning accurate and energy-efficient neural networks.},
	urldate = {2020-11-11},
	journal = {arXiv:2003.10392 [cs, stat]},
	author = {Wu, Lemeng and Ye, Mao and Lei, Qi and Lee, Jason D. and Liu, Qiang},
	month = sep,
	year = {2020},
	note = {arXiv: 2003.10392},
	keywords = {machine learning},
}

@article{jankov_performance_2017,
	title = {A {Performance} {Comparison} between {Multiphysics} and {Stochastic} {Approaches} within a {North} {American} {RAP} {Ensemble}},
	volume = {145},
	issn = {0027-0644},
	url = {https://journals.ametsoc.org/mwr/article/145/4/1161/343002/A-Performance-Comparison-between-Multiphysics-and},
	doi = {10.1175/MWR-D-16-0160.1},
	language = {en},
	number = {4},
	urldate = {2020-11-11},
	journal = {Monthly Weather Review},
	author = {Jankov, Isidora and Berner, Judith and Beck, Jeffrey and Jiang, Hongli and Olson, Joseph B. and Grell, Georg and Smirnova, Tatiana G. and Benjamin, Stanley G. and Brown, John M.},
	month = apr,
	year = {2017},
	note = {Publisher: American Meteorological Society},
	pages = {1161--1179},
}

@inproceedings{pearce_expressive_2020,
	title = {Expressive {Priors} in {Bayesian} {Neural} {Networks}: {Kernel} {Combinations} and {Periodic} {Functions}},
	shorttitle = {Expressive {Priors} in {Bayesian} {Neural} {Networks}},
	url = {http://proceedings.mlr.press/v115/pearce20a.html},
	abstract = {A simple, flexible approach to creating expressive priors in Gaussian process (GP) models makes new kernels from a combination of basic kernels, e.g. summing a periodic and linear kernel can captur...},
	language = {en},
	urldate = {2020-11-11},
	booktitle = {Uncertainty in {Artificial} {Intelligence}},
	publisher = {PMLR},
	author = {Pearce, Tim and Tsuchida, Russell and Zaki, Mohamed and Brintrup, Alexandra and Neely, Andy},
	month = aug,
	year = {2020},
	note = {ISSN: 2640-3498},
	pages = {134--144},
}

@article{paul_real-time_2019,
	title = {A real-time iterative machine learning approach for temperature profile prediction in additive manufacturing processes},
	url = {http://arxiv.org/abs/1907.12953},
	abstract = {Additive Manufacturing (AM) is a manufacturing paradigm that builds three-dimensional objects from a computeraided design model by successively adding material layer by layer. AM has become very popular in the past decade due to its utility for fast prototyping such as 3D printing as well as manufacturing functional parts with complex geometries using processes such as laser metal deposition that would be difﬁcult to create using traditional machining. As the process for creating an intricate part for an expensive metal such as Titanium is prohibitive with respect to cost, computational models are used to simulate the behavior of AM processes before the experimental run. However, as the simulations are computationally costly and timeconsuming for predicting multiscale multi-physics phenomena in AM, physics-informed data-driven machine-learning systems for predicting the behavior of AM processes are immensely beneﬁcial. Such models accelerate not only multiscale simulation tools but also empower real-time control systems using in-situ data. In this paper, we design and develop essential components of a scientiﬁc framework for developing a data-driven model-based real-time control system. Finite element methods are employed for solving time-dependent heat equations and developing the database. The proposed framework uses extremely randomized trees - an ensemble of bagged decision trees as the regression algorithm iteratively using temperatures of prior voxels and laser information as inputs to predict temperatures of subsequent voxels. The models achieve mean absolute percentage errors below 1\% for predicting temperature proﬁles for AM processes. The code is made available for the research community at https://github.com/paularindam/ml-iter-additive.},
	language = {en},
	urldate = {2020-11-11},
	journal = {arXiv:1907.12953 [cs, stat]},
	author = {Paul, Arindam and Mozaffar, Mojtaba and Yang, Zijiang and Liao, Wei-keng and Choudhary, Alok and Cao, Jian and Agrawal, Ankit},
	month = aug,
	year = {2019},
	note = {arXiv: 1907.12953},
	keywords = {machine learning},
}

@article{louizos_functional_2019,
	title = {The {Functional} {Neural} {Process}},
	volume = {32},
	url = {https://proceedings.neurips.cc/paper/2019/hash/db182d2552835bec774847e06406bfa2-Abstract.html},
	language = {en},
	urldate = {2020-11-11},
	journal = {Advances in Neural Information Processing Systems},
	author = {Louizos, Christos and Shi, Xiahan and Schutte, Klamer and Welling, Max},
	year = {2019},
	pages = {8746--8757},
}

@article{alber_integrating_2019,
	title = {Integrating machine learning and multiscale modeling—perspectives, challenges, and opportunities in the biological, biomedical, and behavioral sciences},
	volume = {2},
	copyright = {2019 The Author(s)},
	issn = {2398-6352},
	url = {https://www.nature.com/articles/s41746-019-0193-y},
	doi = {10.1038/s41746-019-0193-y},
	abstract = {Fueled by breakthrough technology developments, the biological, biomedical, and behavioral sciences are now collecting more data than ever before. There is a critical need for time- and cost-efficient strategies to analyze and interpret these data to advance human health. The recent rise of machine learning as a powerful technique to integrate multimodality, multifidelity data, and reveal correlations between intertwined phenomena presents a special opportunity in this regard. However, machine learning alone ignores the fundamental laws of physics and can result in ill-posed problems or non-physical solutions. Multiscale modeling is a successful strategy to integrate multiscale, multiphysics data and uncover mechanisms that explain the emergence of function. However, multiscale modeling alone often fails to efficiently combine large datasets from different sources and different levels of resolution. Here we demonstrate that machine learning and multiscale modeling can naturally complement each other to create robust predictive models that integrate the underlying physics to manage ill-posed problems and explore massive design spaces. We review the current literature, highlight applications and opportunities, address open questions, and discuss potential challenges and limitations in four overarching topical areas: ordinary differential equations, partial differential equations, data-driven approaches, and theory-driven approaches. Towards these goals, we leverage expertise in applied mathematics, computer science, computational biology, biophysics, biomechanics, engineering mechanics, experimentation, and medicine. Our multidisciplinary perspective suggests that integrating machine learning and multiscale modeling can provide new insights into disease mechanisms, help identify new targets and treatment strategies, and inform decision making for the benefit of human health.},
	language = {en},
	number = {1},
	urldate = {2020-11-11},
	journal = {npj Digital Medicine},
	author = {Alber, Mark and Buganza Tepole, Adrian and Cannon, William R. and De, Suvranu and Dura-Bernal, Salvador and Garikipati, Krishna and Karniadakis, George and Lytton, William W. and Perdikaris, Paris and Petzold, Linda and Kuhl, Ellen},
	month = nov,
	year = {2019},
	note = {Number: 1
Publisher: Nature Publishing Group},
	pages = {1--11},
}

@inproceedings{melkumyan_multi-kernel_2011,
	title = {Multi-{Kernel} {Gaussian} {Processes}.},
	doi = {10.5591/978-1-57735-516-8/IJCAI11-238},
	abstract = {Multi-task learning remains a difficult yet important problem in machine learning. In Gaussian processes the main challenge is the definition of valid kernels (covariance functions) able to capture the relationships between different tasks. This paper presents a novel methodology to construct valid multi-task covariance functions (Mercer kernels) for Gaussian processes allowing for a combination of kernels with different forms. The method is based on Fourier analysis and is general for arbitrary stationary covariance functions. Analytical solutions for cross covariance terms between popular forms are provided including Matérn, squared exponential and sparse covariance functions. Experiments are conducted with both artificial and real datasets demonstrating the benefits of the approach.},
	author = {Melkumyan, Arman and Ramos, Fabio},
	month = jul,
	year = {2011},
	pages = {1408--1413},
}

@article{hanachi_hybrid_2019,
	title = {Hybrid data-driven physics-based model fusion framework for tool wear prediction},
	volume = {101},
	issn = {1433-3015},
	url = {https://doi.org/10.1007/s00170-018-3157-5},
	doi = {10.1007/s00170-018-3157-5},
	abstract = {An integral part of modern manufacturing process management is to acquire useful information from machining processes to monitor machine and tool condition. Various models have been introduced to detect, classify, and predict tool wear, as a key parameter of the machining process. In more recent developments, sensor-based approaches have been attempted to infer the tool wear condition from real-time processing of the measurement data. Experiments show that the physics-based prediction models can include large uncertainties. Likewise, the measurement-based (or sensor-based) inference techniques are affected by sensor noise and measurement model uncertainties. To manage uncertainties and noise of both methods, a hybrid framework is proposed to fuse together the results of the prediction model and the measurement-based inference data in a stepwise manner. The fusion framework is an extension to the regularized particle filtering technique, used to facilitate updating the state prediction with a numerical inference model, when measurement models alone are not satisfactory. The results show significant improvement in tool wear state estimation, reducing the prediction errors by almost half, compared to the prediction model and sensor-based monitoring method used independently.},
	language = {en},
	number = {9},
	urldate = {2020-11-10},
	journal = {The International Journal of Advanced Manufacturing Technology},
	author = {Hanachi, Houman and Yu, Wennian and Kim, Il Yong and Liu, Jie and Mechefske, Chris K.},
	month = apr,
	year = {2019},
	pages = {2861--2872},
}

@article{lai_survey_2020,
	title = {A {Survey} of {Machine} {Learning} {Models} in {Renewable} {Energy} {Predictions}},
	volume = {10},
	doi = {10.3390/app10175975},
	abstract = {The use of renewable energy to reduce the effects of climate change and global warming has become an increasing trend. In order to improve the prediction ability of renewable energy, various prediction techniques have been developed. The aims of this review are illustrated as follows. First, this survey attempts to provide a review and analysis of machine-learning models in renewable-energy predictions. Secondly, this study depicts procedures, including data pre-processing techniques, parameter selection algorithms, and prediction performance measurements, used in machine-learning models for renewable-energy predictions. Thirdly, the analysis of sources of renewable energy, values of the mean absolute percentage error, and values of the coefficient of determination were conducted. Finally, some possible potential opportunities for future work were provided at end of this survey.},
	journal = {Applied Sciences},
	author = {Lai, Jung-Pin and Chang, Yu-Ming and Chen, Chieh-Huang and Pai, Ping-Feng},
	month = aug,
	year = {2020},
	pages = {5975},
}

@article{sun_functional_2019,
	title = {Functional {Variational} {Bayesian} {Neural} {Networks}},
	url = {http://arxiv.org/abs/1903.05779},
	abstract = {Variational Bayesian neural networks (BNNs) perform variational inference over weights, but it is difficult to specify meaningful priors and approximate posteriors in a high-dimensional weight space. We introduce functional variational Bayesian neural networks (fBNNs), which maximize an Evidence Lower BOund (ELBO) defined directly on stochastic processes, i.e. distributions over functions. We prove that the KL divergence between stochastic processes equals the supremum of marginal KL divergences over all finite sets of inputs. Based on this, we introduce a practical training objective which approximates the functional ELBO using finite measurement sets and the spectral Stein gradient estimator. With fBNNs, we can specify priors entailing rich structures, including Gaussian processes and implicit stochastic processes. Empirically, we find fBNNs extrapolate well using various structured priors, provide reliable uncertainty estimates, and scale to large datasets.},
	urldate = {2020-11-10},
	journal = {arXiv:1903.05779 [cs, stat]},
	author = {Sun, Shengyang and Zhang, Guodong and Shi, Jiaxin and Grosse, Roger},
	month = mar,
	year = {2019},
	note = {arXiv: 1903.05779},
	keywords = {machine learning},
}

@article{roeder_efficient_2019,
	title = {Efficient {Amortised} {Bayesian} {Inference} for {Hierarchical} and {Nonlinear} {Dynamical} {Systems}},
	url = {http://arxiv.org/abs/1905.12090},
	abstract = {We introduce a flexible, scalable Bayesian inference framework for nonlinear dynamical systems characterised by distinct and hierarchical variability at the individual, group, and population levels. Our model class is a generalisation of nonlinear mixed-effects (NLME) dynamical systems, the statistical workhorse for many experimental sciences. We cast parameter inference as stochastic optimisation of an end-to-end differentiable, block-conditional variational autoencoder. We specify the dynamics of the data-generating process as an ordinary differential equation (ODE) such that both the ODE and its solver are fully differentiable. This model class is highly flexible: the ODE right-hand sides can be a mixture of user-prescribed or "white-box" sub-components and neural network or "black-box" sub-components. Using stochastic optimisation, our amortised inference algorithm could seamlessly scale up to massive data collection pipelines (common in labs with robotic automation). Finally, our framework supports interpretability with respect to the underlying dynamics, as well as predictive generalization to unseen combinations of group components (also called "zero-shot" learning). We empirically validate our method by predicting the dynamic behaviour of bacteria that were genetically engineered to function as biosensors. Our implementation of the framework, the dataset, and all code to reproduce the experimental results is available at https://www.github.com/Microsoft/vi-hds .},
	urldate = {2020-11-10},
	journal = {arXiv:1905.12090 [cs, stat]},
	author = {Roeder, Geoffrey and Grant, Paul K. and Phillips, Andrew and Dalchau, Neil and Meeds, Edward},
	month = oct,
	year = {2019},
	note = {arXiv: 1905.12090},
	keywords = {machine learning},
}

@article{rackauckas_universal_2020,
	title = {Universal {Differential} {Equations} for {Scientific} {Machine} {Learning}},
	url = {http://arxiv.org/abs/2001.04385},
	abstract = {In the context of science, the well-known adage "a picture is worth a thousand words" might well be "a model is worth a thousand datasets." Scientific models, such as Newtonian physics or biological gene regulatory networks, are human-driven simplifications of complex phenomena that serve as surrogates for the countless experiments that validated the models. Recently, machine learning has been able to overcome the inaccuracies of approximate modeling by directly learning the entire set of nonlinear interactions from data. However, without any predetermined structure from the scientific basis behind the problem, machine learning approaches are flexible but data-expensive, requiring large databases of homogeneous labeled training data. A central challenge is reconciling data that is at odds with simplified models without requiring "big data". In this work we develop a new methodology, universal differential equations (UDEs), which augments scientific models with machine-learnable structures for scientifically-based learning. We show how UDEs can be utilized to discover previously unknown governing equations, accurately extrapolate beyond the original data, and accelerate model simulation, all in a time and data-efficient manner. This advance is coupled with open-source software that allows for training UDEs which incorporate physical constraints, delayed interactions, implicitly-defined events, and intrinsic stochasticity in the model. Our examples show how a diverse set of computationally-difficult modeling issues across scientific disciplines, from automatically discovering biological mechanisms to accelerating the training of physics-informed neural networks and large-eddy simulations, can all be transformed into UDE training problems that are efficiently solved by a single software methodology.},
	urldate = {2020-11-10},
	journal = {arXiv:2001.04385 [cs, math, q-bio, stat]},
	author = {Rackauckas, Christopher and Ma, Yingbo and Martensen, Julius and Warner, Collin and Zubov, Kirill and Supekar, Rohit and Skinner, Dominic and Ramadhan, Ali and Edelman, Alan},
	month = aug,
	year = {2020},
	note = {arXiv: 2001.04385},
	keywords = {dynamical systems, machine learning},
}

@article{kim_attentive_2019,
	title = {Attentive {Neural} {Processes}},
	url = {http://arxiv.org/abs/1901.05761},
	abstract = {Neural Processes (NPs) (Garnelo et al 2018a;b) approach regression by learning to map a context set of observed input-output pairs to a distribution over regression functions. Each function models the distribution of the output given an input, conditioned on the context. NPs have the benefit of fitting observed data efficiently with linear complexity in the number of context input-output pairs, and can learn a wide family of conditional distributions; they learn predictive distributions conditioned on context sets of arbitrary size. Nonetheless, we show that NPs suffer a fundamental drawback of underfitting, giving inaccurate predictions at the inputs of the observed data they condition on. We address this issue by incorporating attention into NPs, allowing each input location to attend to the relevant context points for the prediction. We show that this greatly improves the accuracy of predictions, results in noticeably faster training, and expands the range of functions that can be modelled.},
	urldate = {2020-11-10},
	journal = {arXiv:1901.05761 [cs, stat]},
	author = {Kim, Hyunjik and Mnih, Andriy and Schwarz, Jonathan and Garnelo, Marta and Eslami, Ali and Rosenbaum, Dan and Vinyals, Oriol and Teh, Yee Whye},
	month = jul,
	year = {2019},
	note = {arXiv: 1901.05761},
	keywords = {machine learning},
}

@article{jin_bayesian_2020,
	title = {Bayesian {Symbolic} {Regression}},
	url = {http://arxiv.org/abs/1910.08892},
	abstract = {Interpretability is crucial for machine learning in many scenarios such as quantitative finance, banking, healthcare, etc. Symbolic regression (SR) is a classic interpretable machine learning method by bridging X and Y using mathematical expressions composed of some basic functions. However, the search space of all possible expressions grows exponentially with the length of the expression, making it infeasible for enumeration. Genetic programming (GP) has been traditionally and commonly used in SR to search for the optimal solution, but it suffers from several limitations, e.g. the difficulty in incorporating prior knowledge; overly-complicated output expression and reduced interpretability etc. To address these issues, we propose a new method to fit SR under a Bayesian framework. Firstly, Bayesian model can naturally incorporate prior knowledge (e.g., preference of basis functions, operators and raw features) to improve the efficiency of fitting SR. Secondly, to improve interpretability of expressions in SR, we aim to capture concise but informative signals. To this end, we assume the expected signal has an additive structure, i.e., a linear combination of several concise expressions, whose complexity is controlled by a well-designed prior distribution. In our setup, each expression is characterized by a symbolic tree, and the proposed SR model could be solved by sampling symbolic trees from the posterior distribution using an efficient Markov chain Monte Carlo (MCMC) algorithm. Finally, compared with GP, the proposed BSR(Bayesian Symbolic Regression) method saves computer memory with no need to keep an updated 'genome pool'. Numerical experiments show that, compared with GP, the solutions of BSR are closer to the ground truth and the expressions are more concise. Meanwhile we find the solution of BSR is robust to hyper-parameter specifications such as the number of trees.},
	urldate = {2020-11-10},
	journal = {arXiv:1910.08892 [stat]},
	author = {Jin, Ying and Fu, Weilin and Kang, Jian and Guo, Jiadong and Guo, Jian},
	month = jan,
	year = {2020},
	note = {arXiv: 1910.08892},
	keywords = {statistics},
}

@article{joseph_neural_2020,
	title = {Neural hierarchical models of ecological populations},
	volume = {23},
	copyright = {© 2020 John Wiley \& Sons Ltd/CNRS},
	issn = {1461-0248},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/ele.13462},
	doi = {https://doi.org/10.1111/ele.13462},
	abstract = {Neural networks are increasingly being used in science to infer hidden dynamics of natural systems from noisy observations, a task typically handled by hierarchical models in ecology. This article describes a class of hierarchical models parameterised by neural networks – neural hierarchical models. The derivation of such models analogises the relationship between regression and neural networks. A case study is developed for a neural dynamic occupancy model of North American bird populations, trained on millions of detection/non-detection time series for hundreds of species, providing insights into colonisation and extinction at a continental scale. Flexible models are increasingly needed that scale to large data and represent ecological processes. Neural hierarchical models satisfy this need, providing a bridge between deep learning and ecological modelling that combines the function representation power of neural networks with the inferential capacity of hierarchical models.},
	language = {en},
	number = {4},
	urldate = {2020-11-10},
	journal = {Ecology Letters},
	author = {Joseph, Maxwell B.},
	year = {2020},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/ele.13462},
	keywords = {hierarchical model, neural networks},
	pages = {734--747},
}

@article{singh_model_2020,
	title = {Model {Fusion} via {Optimal} {Transport}},
	url = {http://arxiv.org/abs/1910.05653},
	abstract = {Combining different models is a widely used paradigm in machine learning applications. While the most common approach is to form an ensemble of models and average their individual predictions, this approach is often rendered infeasible by given resource constraints in terms of memory and computation, which grow linearly with the number of models. We present a layer-wise model fusion algorithm for neural networks that utilizes optimal transport to (soft-) align neurons across the models before averaging their associated parameters. We show that this can successfully yield "one-shot" knowledge transfer (i.e, without requiring any retraining) between neural networks trained on heterogeneous non-i.i.d. data. In both i.i.d. and non-i.i.d. settings, we illustrate that our approach significantly outperforms vanilla averaging, as well as how it can serve as an efficient replacement for the ensemble with moderate fine-tuning, for standard convolutional networks (like VGG11), residual networks (like ResNet18), and multi-layer perceptrons on CIFAR10 and MNIST. Finally, our approach also provides a principled way to combine the parameters of neural networks with different widths, and we explore its application for model compression. Code is available under the following link https://github.com/modelfusion.},
	urldate = {2020-11-10},
	journal = {arXiv:1910.05653 [cs, stat]},
	author = {Singh, Sidak Pal and Jaggi, Martin},
	month = jun,
	year = {2020},
	note = {arXiv: 1910.05653},
	keywords = {machine learning},
}

@article{li_heavy_2019,
	title = {Heavy {Rain} {Image} {Restoration}: {Integrating} {Physics} {Model} and {Conditional} {Adversarial} {Learning}},
	shorttitle = {Heavy {Rain} {Image} {Restoration}},
	url = {https://arxiv.org/abs/1904.05050v1},
	abstract = {Most deraining works focus on rain streaks removal but they cannot deal
adequately with heavy rain images. In heavy rain, streaks are strongly visible,
dense rain accumulation or rain veiling effect significantly washes out the
image, further scenes are relatively more blurry, etc. In this paper, we
propose a novel method to address these problems. We put forth a 2-stage
network: a physics-based backbone followed by a depth-guided GAN refinement.
The first stage estimates the rain streaks, the transmission, and the
atmospheric light governed by the underlying physics. To tease out these
components more reliably, a guided filtering framework is used to decompose the
image into its low- and high-frequency components. This filtering is guided by
a rain-free residue image --- its content is used to set the passbands for the
two channels in a spatially-variant manner so that the background details do
not get mixed up with the rain-streaks. For the second stage, the refinement
stage, we put forth a depth-guided GAN to recover the background details failed
to be retrieved by the first stage, as well as correcting artefacts introduced
by that stage. We have evaluated our method against the state of the art
methods. Extensive experiments show that our method outperforms them on real
rain image data, recovering visually clean images with good details.},
	language = {en},
	urldate = {2020-11-10},
	author = {Li, Ruotent and Cheong, Loong Fah and Tan, Robby T.},
	month = apr,
	year = {2019},
}

@article{yao_using_2017,
	title = {Using stacking to average {Bayesian} predictive distributions},
	url = {https://arxiv.org/abs/1704.02030v3},
	doi = {10.1214/17-BA1091},
	abstract = {The widely recommended procedure of Bayesian model averaging is flawed in the
M-open setting in which the true data-generating process is not one of the
candidate models being fit. We take the idea of stacking from the point
estimation literature and generalize to the combination of predictive
distributions, extending the utility function to any proper scoring rule, using
Pareto smoothed importance sampling to efficiently compute the required
leave-one-out posterior distributions and regularization to get more stability.
We compare stacking of predictive distributions to several alternatives:
stacking of means, Bayesian model averaging (BMA), pseudo-BMA using AIC-type
weighting, and a variant of pseudo-BMA that is stabilized using the Bayesian
bootstrap. Based on simulations and real-data applications, we recommend
stacking of predictive distributions, with BB-pseudo-BMA as an approximate
alternative when computation cost is an issue.},
	language = {en},
	urldate = {2020-11-10},
	author = {Yao, Yuling and Vehtari, Aki and Simpson, Daniel and Gelman, Andrew},
	month = apr,
	year = {2017},
}
