@article{chen2000mixture,
  title={Mixture kalman filters},
  author={Chen, Rong and Liu, Jun S},
  journal={Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
  volume={62},
  number={3},
  pages={493--508},
  year={2000},
  publisher={Wiley Online Library}
}
@article{ho2020denoising,
  title={Denoising diffusion probabilistic models},
  author={Ho, Jonathan and Jain, Ajay and Abbeel, Pieter},
  journal={arXiv preprint arXiv:2006.11239},
  year={2020}
}

@inproceedings{sohl2015deep,
  title={Deep unsupervised learning using nonequilibrium thermodynamics},
  author={Sohl-Dickstein, Jascha and Weiss, Eric and Maheswaranathan, Niru and Ganguli, Surya},
  booktitle={International Conference on Machine Learning},
  pages={2256--2265},
  year={2015},
  organization={PMLR}
}

@article{wolf2016variational,
  title={Variational inference with hamiltonian monte carlo},
  author={Wolf, Christopher and Karl, Maximilian and van der Smagt, Patrick},
  journal={arXiv preprint arXiv:1609.08203},
  year={2016}
}

@article{Kucukelbir2017,
abstract = {Probabilistic modeling is iterative. A scientist posits a simple model, fits it to her data, refines it according to her analysis, and repeats. However, fitting complex models to large data is a bottleneck in this process. Deriving algorithms for new models can be both mathematically and computationally challenging, which makes it difficult to efficiently cycle through the steps. To this end, we develop automatic differentiation variational inference (advi). Using our method, the scientist only provides a probabilistic model and a dataset, nothing else. advi automatically derives an efficient variational inference algorithm, freeing the scientist to refine and explore many models. advi supports a broad class of models-no conjugacy assumptions are required. We study advi across ten modern probabilistic models and apply it to a dataset with millions of observations. We deploy advi as part of Stan, a probabilistic programming system.},
archivePrefix = {arXiv},
arxivId = {1603.00788},
author = {Kucukelbir, Alp and Blei, David M. and Gelman, Andrew and Ranganath, Rajesh and Tran, Dustin},
eprint = {1603.00788},
file = {:Users/richard/Documents/papers/1603.00788.pdf:pdf},
issn = {15337928},
journal = {Journal of Machine Learning Research},
keywords = {Approximate inference,Bayesian inference,Probabilistic programming,parametric-sampling},
mendeley-tags = {parametric-sampling},
pages = {1--45},
title = {{Automatic Differentiation Variational Inference}},
volume = {18},
year = {2017}
}
@article{Gill1995,
author = {Gill, Richard D. and Levit, Boris Y.},
file = {:Users/richard/Documents/papers/3318681.pdf:pdf},
journal = {Bernoulli},
keywords = {Lower bounds,Nonparametric estimation,Parameter estimation,Quadratic risk,Semi-parametric models,parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {1},
pages = {59--79},
title = {{Applications of the van Trees inequality: A Bayesian Cram{\'{e}}r-Rao bound}},
url = {https://www.jstor.org/stable/3318681},
volume = {1},
year = {1995}
}
@article{Fauss2021,
abstract = {It is shown that both the classic and the Bayesian Cram{\'{e}}r–Rao bounds can be obtained by minimizing the mean square error of an estimator while constraining the underlying distribution to be within a Fisher information ball. The presented results allow for some nonstandard interpretations of the Cram{\'{e}}r–Rao bound and, more importantly, provide a template for novel bounds on the accuracy of estimators.},
author = {Fau{\ss}, Michael and Dytso, Alex and Poor, H. Vincent},
doi = {10.1016/j.sigpro.2020.107917},
file = {:Users/richard/Documents/papers/1-s2.0-S0165168420304618-main.pdf:pdf},
issn = {01651684},
journal = {Signal Processing},
keywords = {Cram{\'{e}}r–Rao bound,Fisher information,Variational techniques,parametric-sampling},
mendeley-tags = {parametric-sampling},
title = {{A variational interpretation of the Cram{\'{e}}r–Rao bound}},
volume = {182},
year = {2021}
}
@article{Gelman1998,
abstract = {Computing (ratios of) normalizing constants of probability models is a fundamental computational problem for many statistical and scientific studies. Monte Carlo simulation is an effective technique, especially with complex and high-dimensional models. This paper aims to bring to the attention of general statistical audiences of some effective methods originating from theoretical physics and at the same time to explore these methods from a more statistical perspective, through establishing theoretical connections and illustrating their uses with statistical problems. We show that the acceptance ratio method and thermodynamic integration are natural generalizations of importance sampling, which is most familiar to statistical audiences. The former generalizes importance sampling through the use of a single “bridge” density and is thus a case of bridge sampling in the sense of Meng and Wong. Thermodynamic integration, which is also known in the numerical analysis literature as Ogata's method for high-dimensional integration, corresponds to the use of infinitely many and continuously connected bridges (and thus a “path”). Our path sampling formulation offers more exibility and thus potential efficiency to thermodynamic integration, and the search of optimal paths turns out to have close connections with the Jeffreys prior density and the Rao and Hellinger distances between two densities. We provide an informative theoretical example as well as two empirical examples (involving 17- to 70-dimensional integrations) to illustrate the potential and implementation of path sampling. We also discuss some open problems. {\textcopyright} 1998 Rocky Mountain Mathematics Consortium.},
author = {Gelman, Andrew and Meng, Xiao Li},
doi = {10.1214/ss/1028905934},
file = {:Users/richard/Documents/papers/1028905934.pdf:pdf},
issn = {08834237},
journal = {Statistical Science},
keywords = {Acceptance ratio method,Hellinger distance,Jeffreys prior density,Markov chain Monte Carlo,Numerical integration,Rao distance,Thermodynamic integration,parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {2},
pages = {163--185},
title = {{Simulating normalizing constants: From importance sampling to bridge sampling to path sampling}},
volume = {13},
year = {1998}
}
@article{Betancourt2016,
abstract = {By leveraging the natural geometry of a smooth probabilistic system, Hamiltonian Monte Carlo yields computationally efficient Markov Chain Monte Carlo estimation. At least provided that the algorithm is sufficiently well-tuned. In this paper I show how the geometric foundations of Hamiltonian Monte Carlo implicitly identify the optimal choice of these parameters, especially the integration time. I then consider the practical consequences of these principles in both existing algorithms and a new implementation called \emph{Exhaustive Hamiltonian Monte Carlo} before demonstrating the utility of these ideas in some illustrative examples.},
archivePrefix = {arXiv},
arxivId = {1601.00225},
author = {Betancourt, Michael},
eprint = {1601.00225},
file = {:Users/richard/Documents/papers/1601.pdf:pdf},
keywords = {a function with respect,and phrases,carlo,distribution,hamiltonian monte,markov chain monte carlo,microcanonical systems,one of the most,parametric-sampling,statistics is the estima-,tion of expectations of,to a given target,ubiquitous computational challenges in,$\pi$},
mendeley-tags = {parametric-sampling},
title = {{Identifying the Optimal Integration Time in Hamiltonian Monte Carlo}},
url = {http://arxiv.org/abs/1601.00225},
year = {2016}
}
@article{Maclaurin2015,
archivePrefix = {arXiv},
arxivId = {1504.01344},
author = {Duvenaud, David and Maclaurin, Dougal and Adams, Ryan},
eprint = {1504.01344},
file = {:Users/richard/Documents/papers/1504.01344v1.pdf:pdf},
journal = {International Conference on Machine Learning},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
pages = {1070--1077},
title = {{Early Stopping is Nonparametric Variational Inference}},
url = {http://arxiv.org/abs/1504.01344},
volume = {51},
year = {2016}
}
@article{Salimans2013,
abstract = {We propose a general algorithm for approximating nonstandard Bayesian posterior distributions. The algorithm minimizes the Kullback-Leibler divergence of an approximating distribution to the intractable posterior distribu- tion. Our method can be used to approximate any posterior distribution, provided that it is given in closed form up to the proportionality constant. The approxi- mation can be any distribution in the exponential family or any mixture of such distributions, which means that it can be made arbitrarily precise. Several exam- ples illustrate the speed and accuracy of our approximation method in practice. {\textcopyright} 2013 International Society for Bayesian Analysis.},
archivePrefix = {arXiv},
arxivId = {1206.6679},
author = {Salimans, Tim and Knowles, David A.},
doi = {10.1214/13-BA858},
eprint = {1206.6679},
file = {:Users/richard/Documents/papers/13-BA858.pdf:pdf},
issn = {19360975},
journal = {Bayesian Analysis},
keywords = {Approximate inference,Stochastic approximation,Variational bayes,parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {4},
pages = {837--882},
title = {{Fixed-form variational posterior approximation through stochastic linear regression}},
volume = {8},
year = {2013}
}
@article{Barp2018,
abstract = {Markov chain Monte Carlo methods have revolutionized mathematical computation and enabled statistical inference within many previously intractable models. In this context, Hamiltonian dynamics have been proposed as an efficient way of building chains that can explore probability densities efficiently. The method emerges from physics and geometry, and these links have been extensively studied over the past thirty years. The aim of this review is to provide a comprehensive introduction to the geometric tools used in Hamiltonian Monte Carlo at a level accessible to statisticians, machine learners, and other users of the methodology with only a basic understanding of Monte Carlo methods. This will be complemented with some discussion of the most recent advances in the field, which we believe will become increasingly relevant to scientists.},
archivePrefix = {arXiv},
arxivId = {1705.02891},
author = {Barp, Alessandro and Briol, Fran{\c{c}}ois Xavier and Kennedy, Anthony D. and Girolami, Mark},
doi = {10.1146/annurev-statistics-031017-100141},
eprint = {1705.02891},
file = {:Users/richard/Documents/papers/1705.02891.pdf:pdf},
issn = {2326831X},
journal = {Annual Review of Statistics and Its Application},
keywords = {Hamiltonian mechanics,Information geometry,Markov chain Monte Carlo,Shadow Hamiltonians,Symplectic integrators,parametric-sampling},
mendeley-tags = {parametric-sampling},
pages = {451--471},
title = {{Geometry and Dynamics for Markov Chain Monte Carlo}},
volume = {5},
year = {2018}
}
@article{Zhang2019,
abstract = {Many modern unsupervised or semi-supervised machine learning algorithms rely on Bayesian probabilistic models. These models are usually intractable and thus require approximate inference. Variational inference (VI) lets us approximate a high-dimensional Bayesian posterior with a simpler variational distribution by solving an optimization problem. This approach has been successfully applied to various models and large-scale applications. In this review, we give an overview of recent trends in variational inference. We first introduce standard mean field variational inference, then review recent advances focusing on the following aspects: (a) scalable VI, which includes stochastic approximations, (b) generic VI, which extends the applicability of VI to a large class of otherwise intractable models, such as non-conjugate models, (c) accurate VI, which includes variational models beyond the mean field approximation or with atypical divergences, and (d) amortized VI, which implements the inference over local latent variables with inference networks. Finally, we provide a summary of promising future research directions.},
archivePrefix = {arXiv},
arxivId = {1711.05597},
author = {Zhang, Cheng and Butepage, Judith and Kjellstrom, Hedvig and Mandt, Stephan},
doi = {10.1109/TPAMI.2018.2889774},
eprint = {1711.05597},
file = {:Users/richard/Documents/papers/Advances_in_Variational_Inference.pdf:pdf},
issn = {19393539},
journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
keywords = {Variational inference,approximate Bayesian inference,inference networks,parametric-sampling,reparameterization gradients,scalable inference,structured variational approximations},
mendeley-tags = {parametric-sampling},
number = {8},
pages = {2008--2026},
pmid = {30596568},
title = {{Advances in Variational Inference}},
volume = {41},
year = {2019}
}
@article{Zhang2021,
abstract = {We introduce Pathfinder, a variational method for approximately sampling from differentiable log densities. Starting from a random initialization, Pathfinder locates normal approximations to the target density along a quasi-Newton optimization path, with local covariance estimated using the inverse Hessian estimates produced by the optimizer. Pathfinder returns draws from the approximation with the lowest estimated Kullback-Leibler (KL) divergence to the true posterior. We evaluate Pathfinder on a wide range of posterior distributions, demonstrating that its approximate draws are better than those from automatic differentiation variational inference (ADVI) and comparable to those produced by short chains of dynamic Hamiltonian Monte Carlo (HMC), as measured by 1-Wasserstein distance. Compared to ADVI and short dynamic HMC runs, Pathfinder requires one to two orders of magnitude fewer log density and gradient evaluations, with greater reductions for more challenging posteriors. Importance resampling over multiple runs of Pathfinder improves the diversity of approximate draws, reducing 1-Wasserstein distance further and providing a measure of robustness to optimization failures on plateaus, saddle points, or in minor modes. The Monte Carlo KL-divergence estimates are embarrassingly parallelizable in the core Pathfinder algorithm, as are multiple runs in the resampling version, further increasing Pathfinder's speed advantage with multiple cores.},
archivePrefix = {arXiv},
arxivId = {2108.03782},
author = {Zhang, Lu and Carpenter, Bob and Gelman, Andrew and Vehtari, Aki},
eprint = {2108.03782},
file = {:Users/richard/Documents/papers/2108.03782.pdf:pdf},
journal = {arXiv},
keywords = {hamiltonian monte carlo,imation,importance resampling,laplace approx-,parametric-sampling,quasi-newton optimization,variational inference},
mendeley-tags = {parametric-sampling},
title = {{Pathfinder: Parallel quasi-Newton variational inference}},
url = {http://arxiv.org/abs/2108.03782},
year = {2021}
}
@article{Poole2019,
abstract = {Estimating and optimizing Mutual Information (MI) is core to many problems in machine learning; however, bounding MI in high dimensions is challenging. To establish tractable and scalable objectives, recent work has turned to variational bounds parameterized by neural networks, but the relationships and tradeoffs between these bounds remains unclear. In this work, we unify these recent developments in a single framework. We find that the existing variational lower bounds degrade when the MI is large, exhibiting either high bias or high variance. To address this problem, we introduce a continuum of lower bounds that encompasses previous bounds and flexibly trades off bias and variance. On high-dimensional, controlled problems, we empirically characterize the bias and variance of the bounds and their gradients and demonstrate the effectiveness of our new bounds for estimation and representation learning.},
author = {Poole, Ben and Ozair, Sherjil and van den Oord, Aaron and Alemi, Alexander A. and Tucker, George},
file = {:Users/richard/Documents/papers/poole19a.pdf:pdf},
issn = {23318422},
journal = {arXiv},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
title = {{On variational bounds of mutual information}},
year = {2019}
}
@article{Dembo1991,
abstract = {The role of inequalities in information theory is reviewed and the relationship of these inequalities to inequalities in other branches of mathematics is developed. {\textcopyright} 1991 IEEE},
author = {Dembo, Amir and Cover, Thomas M. and Thomas, Joy A.},
doi = {10.1109/18.104312},
file = {:Users/richard/Documents/papers/dembo1991_files/getPDF_002.pdf:pdf},
issn = {15579654},
journal = {IEEE Transactions on Information Theory},
keywords = {Fisher,Information inequalities,entropy power,information,parametric-sampling,uncertainty principles},
mendeley-tags = {parametric-sampling},
number = {6},
pages = {1501--1518},
title = {{Information Theoretic Inequalities}},
volume = {37},
year = {1991}
}
@article{Brunel1998,
abstract = {In the context of parameter estimation and model selection, it is only quite recently that a direct link between the Fisher information and information-theoretic quantities has been exhibited. We give an interpretation of this link within the standard framework of information theory. We show that in the context of population coding, the mutual information between the activity of a large array of neurons and a stimulus to which the neurons are tuned is naturally related to the Fisher information. In the light of this result, we consider the optimization of the tuning curves parameters in the case of neurons responding to a stimulus represented by an angular variable.},
author = {Brunel, Nicolas and Nadal, Jean Pierre},
doi = {10.1162/089976698300017115},
file = {:Users/richard/Documents/papers/brunel-mutual.pdf:pdf},
issn = {08997667},
journal = {Neural Computation},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {7},
pages = {1731--1757},
pmid = {9744895},
title = {{Mutual Information, Fisher Information, and Population Coding}},
volume = {10},
year = {1998}
}
@article{Stam1959,
abstract = {A certain analogy is found to exist between a special case of Fisher's quantity of information I and the inverse of the "entropy power" of Shannon (1949, p. 60). This can be inferred from two facts: (1) Both quantities satisfy inequalities that bear a certain resemblance to each other. (2) There is an inequality connecting the two quantities. This last result constitutes a sharpening of the uncertainty relation of quantum mechanics for canonically conjugated variables. Two of these relations are used to give a direct proof of an inequality of Shannon (1949, p. 63, Theorem 15). Proofs are not elaborated fully. Details will be given in a doctoral thesis that is in preparation. {\textcopyright} 1959 Academic Press, Inc.},
author = {Stam, A. J.},
doi = {10.1016/S0019-9958(59)90348-1},
file = {:Users/richard/Documents/papers/1-s2.0-S0019995859903481-main.pdf:pdf},
issn = {00199958},
journal = {Information and Control},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {2},
pages = {101--112},
title = {{Some inequalities satisfied by the quantities of information of Fisher and Shannon}},
volume = {2},
year = {1959}
}
@article{Marriott2002,
abstract = {Despite the well-known difficulties of undertaking inference with mixture models, they are frequently used for modelling. These inferential problems arise because the underlying geometry of a mixture family is very complicated. This paper shows that by adding a simplifying assumption, which frequently is natural statistically, the geometric structure is reduced to a much more tractable form. This enables standard inferential techniques to be applied successfully. One result of studying the local geometry is that it unifies the convex and differential geometric theories of mixture models. The techniques proposed are applied to prediction, random effects and measurement error models. {\textcopyright} 2002 Biometrika Trust.},
author = {Marriott, Paul},
doi = {10.1093/biomet/89.1.77},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Marriott - 2002 - On the local geometry of mixture models.pdf:pdf},
issn = {00063444},
journal = {Biometrika},
keywords = {Convex geometry,Differential geometry,Measurement error,Mixture model,Prediction,Random effects model,Statistical manifold,parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {1},
pages = {77--93},
title = {{On the local geometry of mixture models}},
volume = {89},
year = {2002}
}
@article{Anaya-Izquierdo2007a,
abstract = {Exponential families are the workhorses of parametric modelling theory. One reason for their popularity is their associated inference theory, which is very clean, both from a theoretical and a computational point of view. One way in which this set of tools can be enriched in a natural and interpretable way is through mixing. This paper develops and applies the idea of local mixture modelling to exponential families. It shows that the highly interpretable and flexible models which result have enough structure to retain the attractive inferential properties of exponential families. In particular, results on identification, parameter orthogonality and log-concavity of the likelihood are proved. {\textcopyright} 2007 ISI/BS.},
author = {Anaya-Izquierdo, Karim and Marriott, Paul},
doi = {10.3150/07-BEJ6170},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Anaya-Izquierdo, Marriott - 2007 - Local mixture models of exponential families(2).pdf:pdf},
issn = {13507265},
journal = {Bernoulli},
keywords = {Affine geometry,Convex geometry,Differential geometry,Dispersion model,Exponential families,Mixture model,Statistical manifold,parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {3},
pages = {623--640},
title = {{Local mixture models of exponential families}},
volume = {13},
year = {2007}
}
@article{Kolchinsky2017,
abstract = {Mixture distributions arise in many parametric and non-parametric settings-for example, in Gaussian mixture models and in non-parametric estimation. It is often necessary to compute the entropy of a mixture, but, in most cases, this quantity has no closed-form expression, making some form of approximation necessary. We propose a family of estimators based on a pairwise distance function between mixture components, and show that this estimator class has many attractive properties. For many distributions of interest, the proposed estimators are efficient to compute, differentiable in the mixture parameters, and become exact when the mixture components are clustered. We prove this family includes lower and upper bounds on the mixture entropy. The Chernoff $\alpha$-divergence gives a lower bound when chosen as the distance function, with the Bhattacharyaa distance providing the tightest lower bound for components that are symmetric and members of a location family. The Kullback-Leibler divergence gives an upper bound when used as the distance function. We provide closed-form expressions of these bounds for mixtures of Gaussians, and discuss their applications to the estimation of mutual information. We then demonstrate that our bounds are significantly tighter than well-known existing bounds using numeric simulations. This estimator class is very useful in optimization problems involving maximization/minimization of entropy and mutual information, such as MaxEnt and rate distortion problems.},
archivePrefix = {arXiv},
arxivId = {1706.02419},
author = {Kolchinsky, Artemy and Tracey, Brendan D.},
doi = {10.3390/e19070361},
eprint = {1706.02419},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Kolchinsky, Tracey - 2017 - Estimating mixture entropy with pairwise distances.pdf:pdf},
issn = {10994300},
journal = {Entropy},
keywords = {Entropy estimation,MaxEnt,Mixture distribution,Mixture of Gaussians,Rate distortion,parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {7},
pages = {1--17},
title = {{Estimating mixture entropy with pairwise distances}},
volume = {19},
year = {2017}
}
@article{Zhang2017,
abstract = {Variational Bayesian neural nets combine the flexibility of deep learning with Bayesian uncertainty estimation. Unfortunately, there is a tradeoff between cheap but simple variational families (e.g. fully factorized) or expensive and complicated inference procedures. We show that natural gradient ascent with adaptive weight noise implicitly fits a variational posterior to maximize the evidence lower bound (ELBO). This insight allows us to train full-covariance, fully factorized, or matrix-variate Gaussian variational posteriors using noisy versions of natural gradient, Adam, and K-FAC, respectively, making it possible to scale up to modern ConvNets. On standard regression benchmarks, our noisy K-FAC algorithm makes better predictions and matches Hamiltonian Monte Carlo's predictive variances better than existing methods. Its improved uncertainty estimates lead to more efficient exploration in active learning, and intrinsic motivation for reinforcement learning.},
author = {Zhang, Guodong and Sun, Shengyang and Duvenaud, David and Grosse, Roger},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Zhang et al. - 2017 - Noisy natural gradient as variational inference.pdf:pdf},
journal = {International Conference on Machine Learning},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
title = {{Noisy natural gradient as variational inference}},
volume = {80},
year = {2018}
}
@article{Carpenter2017,
abstract = {Stan is a probabilistic programming language for specifying statistical models. A Stan program imperatively defines a log probability function over parameters conditioned on specified data and constants. As of version 2.14.0, Stan provides full Bayesian inference for continuous-variable models through Markov chain Monte Carlo methods such as the No-U-Turn sampler, an adaptive form of Hamiltonian Monte Carlo sampling. Penalized maximum likelihood estimates are calculated using optimization methods such as the limited memory Broyden-Fletcher-Goldfarb-Shanno algorithm. Stan is also a platform for computing log densities and their gradients and Hessians, which can be used in alternative algorithms such as variational Bayes, expectation propagation, and marginal inference using approximate integration. To this end, Stan is set up so that the densities, gradients, and Hessians, along with intermediate quantities of the algorithm such as acceptance probabilities, are easily accessible. Stan can be called from the command line using the cmdstan package, through R using the rstan package, and through Python using the pystan package. All three interfaces support sampling and optimization-based inference with diagnostics and posterior analysis. rstan and pystan also provide access to log probabilities, gradients, Hessians, parameter transforms, and specialized plotting.},
author = {Carpenter, Bob and Gelman, Andrew and Hoffman, Matthew D. and Lee, Daniel and Goodrich, Ben and Betancourt, Michael and Brubaker, Marcus A. and Guo, Jiqiang and Li, Peter and Riddell, Allen},
doi = {10.18637/jss.v076.i01},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Carpenter et al. - 2017 - Stan A probabilistic programming language.pdf:pdf},
issn = {15487660},
journal = {Journal of Statistical Software},
keywords = {Algorithmic differentiation,Bayesian inference,Probabilistic program,Stan,parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {1},
title = {{Stan: A probabilistic programming language}},
volume = {76},
year = {2017}
}
@article{Acerbi2020,
abstract = {Many probabilistic models of interest in scientific computing and machine learning have expensive, black-box likelihoods that prevent the application of standard techniques for Bayesian inference, such as MCMC, which would require access to the gradient or a large number of likelihood evaluations. We introduce here a novel sample-efficient inference framework, Variational Bayesian Monte Carlo (VBMC). VBMC combines variational inference with Gaussian-process based, active-sampling Bayesian quadrature, using the latter to efficiently approximate the intractable integral in the variational objective. Our method produces both a nonparametric approximation of the posterior distribution and an approximate lower bound of the model evidence, useful for model selection. We demonstrate VBMC both on several synthetic likelihoods and on a neuronal model with data from real neurons. Across all tested problems and dimensions (up to D = 10), VBMC performs consistently well in reconstructing the posterior and the model evidence with a limited budget of likelihood evaluations, unlike other methods that work only in very low dimensions. Our framework shows great promise as a novel tool for posterior and model inference with expensive, black-box likelihoods.},
archivePrefix = {arXiv},
arxivId = {1810.05558},
author = {Acerbi, Luigi},
eprint = {1810.05558},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Acerbi - 2020 - Variational Bayesian Monte Carlo with Noisy Likelihoods.pdf:pdf;:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Acerbi - 2020 - Variational Bayesian Monte Carlo with Noisy Likelihoods(2).pdf:pdf},
issn = {10495258},
journal = {arXiv},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
title = {{Variational Bayesian Monte Carlo with Noisy Likelihoods}},
year = {2020}
}
@article{Zhang2011,
abstract = {The performance of Markov chain Monte Carlo methods is often sensitive to the scaling and correlations between the random variables of interest. An important source of information about the local correlation and scale is given by the Hessian matrix of the target distribution, but this is often either computationally expensive or infeasible. In this paper we propose MCMC samplers that make use of quasi-Newton approximations, which approximate the Hessian of the target distribution from previous samples and gradients generated by the sampler. A key issue is that MCMC samplers that depend on the history of previous states are in general not valid. We address this problem by using limited memory quasi-Newton methods, which depend only on a fixed window of previous samples. On several real world datasets, we show that the quasi-Newton sampler is more effective than standard Hamiltonian Monte Carlo at a fraction of the cost of MCMC methods that require higher-order derivatives.},
author = {Zhang, Yichuan and Sutton, Charles},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Zhang, Sutton - 2011 - Quasi-Newton methods for Markov chain Monte Carlo.pdf:pdf},
isbn = {9781618395993},
journal = {Advances in Neural Information Processing Systems 24: 25th Annual Conference on Neural Information Processing Systems 2011, NIPS 2011},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
pages = {1--9},
title = {{Quasi-Newton methods for Markov chain Monte Carlo}},
year = {2011}
}
@article{Bishop2006b,
author = {Bishop, C.M. and Winn, J.},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Bishop, Winn - 2006 - Variational message passing.pdf:pdf},
journal = {Journal of Machine Learning Research},
keywords = {bayesian networks,message passing,parametric-sampling,variational inference},
mendeley-tags = {parametric-sampling},
number = {1},
pages = {661},
title = {{Variational message passing}},
url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.92.7892&rep=rep1&type=pdf},
volume = {6},
year = {2006}
}
@article{Kass1998,
author = {Kass, Robert E. and Carlin, Bradley P. and Gelman, Andrew and Neal, Radford M.},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Kass et al. - 1998 - Markov Chain Monte Carlo in Practice A Roundtable Discussion.pdf:pdf},
journal = {The American Statistician},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {2},
pages = {93----100},
title = {{Markov Chain Monte Carlo in Practice: A Roundtable Discussion}},
volume = {52},
year = {1998}
}
@article{Hoffman2014,
author = {Hoffman, Matthew D and Gelman, Andrew},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Hoffman, Gelman - 2014 - The No-U-Turn Sampler Adaptively Setting Path Lengths in Hamiltonian Monte Carlo.pdf:pdf},
journal = {Journal of Machine Learning Research},
keywords = {adaptive monte carlo,bayesian inference,dual averaging,hamiltonian monte carlo,markov chain monte carlo,parametric-sampling},
mendeley-tags = {parametric-sampling},
pages = {1351--1381},
title = {{The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo}},
volume = {15},
year = {2014}
}
@article{deFreitas2001,
author = {de Freitas, Nando and H{\o}jen-S{\o}rensen, Pedro and Jordan, Michael I. and Russel, Stuart},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/de Freitas et al. - 2001 - Variational MCMC.pdf:pdf},
journal = {Uncertainty in Artificial Intelligence},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
title = {{Variational MCMC}},
year = {2001}
}
@article{Wainwright2008,
author = {Wainwright, Martin J. and Jordan, Michael I.},
doi = {10.1561/2200000001},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Wainwright, Jordan - 2008 - Graphical Models, Exponential Families, and Variational Inference.pdf:pdf},
isbn = {9781601981844},
issn = {1935-8237},
journal = {Foundations and Trends{\textregistered} in Machine Learning},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {1–2},
pages = {1--305},
title = {{Graphical Models, Exponential Families, and Variational Inference}},
volume = {1},
year = {2008}
}
@incollection{Jaakkola1998,
author = {Jaakkola, Tommi S. and Jordan, Michael I.},
booktitle = {Learning in Graphical Models},
editor = {Jordan, Michael I.},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Jaakkola, Jordan - 1998 - Improving the Mean Field Approximation via the Use of Mixture Distributions.pdf:pdf},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
publisher = {Kluwer Academic Publishers},
title = {{Improving the Mean Field Approximation via the Use of Mixture Distributions}},
year = {1998}
}
@article{Gershman2012b,
abstract = {Variational methods are widely used for ap- proximate posterior inference. However, their use is typically limited to families of distributions that enjoy particular conjugacy properties. To circumvent this limitation, we propose a family of variational approx- imations inspired by nonparametric kernel density estimation. The locations of these kernels and their bandwidth are treated as variational parameters and optimized to im- prove an approximate lower bound on the marginal likelihood of the data. Unlike most other variational approximations, using mul- tiple kernels allows the approximation to cap- ture multiple modes of the posterior. We demonstrate the efficacy of the nonparamet- ric approximation with a hierarchical logistic regression model and a nonlinear matrix fac- torization model. We obtain predictive per- formance as good as or better than more spe- cialized variational methods and MCMC ap- proximations. The method is easy to apply to graphical models for which standard vari- ational methods are difficult to derive.},
archivePrefix = {arXiv},
arxivId = {1206.4665},
author = {Gershman, Samuel J. and Hoffman, Matthew D. and Blei, David M.},
doi = {10.1162/089976699300016331},
eprint = {1206.4665},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Gershman, Hoffman, Blei - 2012 - Nonparametric Variational Inference.pdf:pdf},
isbn = {978-1-4503-1285-1},
issn = {0899-7667},
journal = {Proceedings of the 29th International Conference on Machine Learning},
keywords = {Approximation Methods,Variational inference,nonparametric Bayesian statistics,parametric-sampling,variational methods},
mendeley-tags = {parametric-sampling},
pages = {235--242},
pmid = {80990000001},
title = {{Nonparametric Variational Inference}},
url = {https://icml.cc/Conferences/2012/papers/360.pdf},
year = {2012}
}
@article{Lindsay1983b,
abstract = {Geometric analysis of the mixture likelihood set for univariate exponential family densities yields results which tie the number and location of support points for the nonparametric maximum likelihood estimator of the mixing distribution to sign changes in certain integrated polynomials. One corollary is a very general uniqueness theorem for the estimator.},
author = {Lindsay, Bruce G.},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Lindsay - 1983 - The Geometry of Mixture Likelihoods, Part II The Exponential Family.pdf:pdf},
journal = {The Annals of Statistics},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {3},
pages = {783--792},
title = {{The Geometry of Mixture Likelihoods, Part II: The Exponential Family}},
volume = {11},
year = {1983}
}
@article{Lindsay1983a,
author = {Lindsay, Bruce G.},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Lindsay - 1983 - The Geometry of Mixture Likelihoods A General Theory.pdf:pdf},
journal = {The Annals of Statistics},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {1},
pages = {86--94},
title = {{The Geometry of Mixture Likelihoods: A General Theory}},
volume = {11},
year = {1983}
}
@article{Acerbi2018,
archivePrefix = {arXiv},
arxivId = {arXiv:1810.05558v1},
author = {Acerbi, Luigi},
eprint = {arXiv:1810.05558v1},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Acerbi - 2018 - Variational Bayesian Monte Carlo.pdf:pdf},
journal = {Advances in Neural Information Processing Systems},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
title = {{Variational Bayesian Monte Carlo}},
year = {2018}
}
@article{Ma2015,
abstract = {Many recent Markov chain Monte Carlo (MCMC) samplers leverage continuous dynamics to define a transition kernel that efficiently explores a target distribution. In tandem, a focus has been on devising scalable variants that subsample the data and use stochastic gradients in place of full-data gradients in the dynamic simulations. However, such stochastic gradient MCMC samplers have lagged behind their full-data counterparts in terms of the complexity of dynamics considered since proving convergence in the presence of the stochastic gradient noise is non-trivial. Even with simple dynamics, significant physical intuition is often required to modify the dynamical system to account for the stochastic gradient noise. In this paper, we provide a general recipe for constructing MCMC samplers--including stochastic gradient versions--based on continuous Markov processes specified via two matrices. We constructively prove that the framework is complete. That is, any continuous Markov process that provides samples from the target distribution can be written in our framework. We show how previous continuous-dynamic samplers can be trivially "reinvented" in our framework, avoiding the complicated sampler-specific proofs. We likewise use our recipe to straightforwardly propose a new state-adaptive sampler: stochastic gradient Riemann Hamiltonian Monte Carlo (SGRHMC). Our experiments on simulated data and a streaming Wikipedia analysis demonstrate that the proposed SGRHMC sampler inherits the benefits of Riemann HMC, with the scalability of stochastic gradient methods.},
archivePrefix = {arXiv},
arxivId = {1506.04696},
author = {Ma, Yi-An and Chen, Tianqi and Fox, Emily B.},
eprint = {1506.04696},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Ma, Chen, Fox - 2015 - A Complete Recipe for Stochastic Gradient MCMC.pdf:pdf},
issn = {10495258},
journal = {Advances in Neural Information Processing Systems},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
pages = {1--16},
title = {{A Complete Recipe for Stochastic Gradient MCMC}},
url = {http://arxiv.org/abs/1506.04696},
year = {2015}
}
@article{Korattikara2013,
abstract = {Can we make Bayesian posterior MCMC sampling more efficient when faced with very large datasets? We argue that computing the likelihood for N datapoints in the Metropolis-Hastings (MH) test to reach a single binary decision is computationally inefficient. We introduce an approximate MH rule based on a sequential hypothesis test that allows us to accept or reject samples with high confidence using only a fraction of the data required for the exact MH rule. While this method introduces an asymptotic bias, we show that this bias can be controlled and is more than offset by a decrease in variance due to our ability to draw more samples per unit of time.},
archivePrefix = {arXiv},
arxivId = {1304.5299},
author = {Korattikara, Anoop and Chen, Yutian and Welling, Max},
eprint = {1304.5299},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Korattikara, Chen, Welling - 2013 - Austerity in MCMC Land Cutting the Metropolis-Hastings Budget.pdf:pdf},
isbn = {9781634393973},
journal = {International Conference on Machine Learning},
keywords = {parametric-sampling},
mendeley-groups = {VariSampling},
mendeley-tags = {parametric-sampling},
number = {1},
pages = {181--189},
title = {{Austerity in MCMC Land: Cutting the Metropolis-Hastings Budget}},
url = {http://arxiv.org/abs/1304.5299},
volume = {32},
year = {2014}
}
@article{Anaya-Izquierdo2007,
archivePrefix = {arXiv},
arxivId = {arXiv:0709.0447v1},
author = {Anaya-Izquierdo, Karim and Marriott, Paul},
doi = {10.3150/07-BEJ6170},
eprint = {arXiv:0709.0447v1},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Anaya-Izquierdo, Marriott - 2007 - Local mixture models of exponential families.pdf:pdf},
issn = {1350-7265},
journal = {Bernoulli},
keywords = {affine geometry,convex geometry,differential geometry,dispersion model,exponential families,information-geometry,mixture model,parametric-sampling,statistical manifold},
mendeley-tags = {information-geometry,parametric-sampling},
number = {3},
pages = {623--640},
title = {{Local mixture models of exponential families}},
url = {http://projecteuclid.org/euclid.bj/1186503479},
volume = {13},
year = {2007}
}
@article{Anaya-Izquierdo2013,
author = {Anaya-Izquierdo, Karim and Critchley, Frank and Marriott, Paul and Vos, Paul},
doi = {10.1007/978-3-642-40020-9_34},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Anaya-Izquierdo et al. - 2013 - Computational information geometry in statistics Mixture modelling.pdf:pdf},
isbn = {9783642400193},
issn = {03029743},
journal = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
keywords = {information-geometry,parametric-sampling},
mendeley-tags = {information-geometry,parametric-sampling},
pages = {319--326},
title = {{Computational information geometry in statistics: Mixture modelling}},
volume = {8085 LNCS},
year = {2013}
}
@article{Miller2017,
abstract = {We propose a black-box variational inference method to approximate intractable distributions with an increasingly rich approximating class. Our method, termed variational boosting, iteratively refines an existing variational approximation by solving a sequence of optimization problems, allowing the practitioner to trade computation time for accuracy. We show how to expand the variational approximating class by incorporating additional covariance structure and by introducing new components to form a mixture. We apply variational boosting to synthetic and real statistical models, and show that resulting posterior inferences compare favorably to existing posterior approximation algorithms in both accuracy and efficiency.},
archivePrefix = {arXiv},
arxivId = {1611.06585},
author = {Miller, Andrew C. and Foti, Nicholas J. and Adams, Ryan P.},
eprint = {1611.06585},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Miller, Foti, Adams - 2017 - Variational Boosting Iteratively Refining Posterior Approximations.pdf:pdf},
journal = {arXiv},
keywords = {parametric-sampling},
mendeley-groups = {VariSampling},
mendeley-tags = {parametric-sampling},
title = {{Variational Boosting: Iteratively Refining Posterior Approximations}},
url = {http://arxiv.org/abs/1611.06585},
year = {2017}
}
@article{Angelino2016,
annote = {Chapter 1
Main point: with big data, we need to reconsider the guarantees we wanted with our inference algorithms. For example, we're potentially OK with tradeoffs.

Why be bayesian when you have big data? Doesn't big enough data shrink your posterior to a point estimate? No. Here's why:
- more data affords more complex models or more complex hypothesis spaces
- distinction between 'tall data' and 'wide data'

Chapter 2

Exponential families
- the usual story: sufficient statistics and conjugacy
- also neat: size of set of sufficient statistics guaranteed not to grow with the amount of data

Bias + Variance
Mean squared error is a reasonable thing to try to minimize. This is equal to (eq 2.23)
Var[estimator] + Bias[estimator]^2

Some algorithms will have a 'transient bias', e.g. with importance sampling. Only biased for small amounts of time.

Mean field variational inference

MCMC vs Variational
Do you prefer exact inference in an approximate model, or approximate inference in an exact model?

Since the variational lower bound is a lower bound on the marginal likelihood, and since the marginal likelihood is used for model selection in a MDL sense, the variational lower bound may be used for approximate model selection.

2.4 Expectation Propagation
Rather than minimize KL(q||p), we minimize KL(p||q). There are implications.

Intuition: fitting a bimodal distribution with a unimodal one, mean-field will spread out to cover both modes, EP will choose to fit one of them.

Chapter 3: MCMC with data subsets

MH acceptance probability needs to be reworked to have likelihood terms P(x|\theta).

3.2 Adaptive stopping: choose subset of size m. increase m until a criterion is met that guarantees fewer than epsilon-percent errors in your acceptance probability.

-> assume each datum's log likelihood is normally distributed, then the criterion becomes a t-statistic significance test

-> or use concentration inequalities to bound error on each step

3.3 FlyMC: aka "firefly monte carlo". 

Uses auxiliary bernoulli variables to 'mask' data points.

The catch: requires "collapsible" lower bound on likelihood

3.4 Stochastic gradients

Chapter 4: (data or model) parallel MCMC

Plain old multi-chain only reduces variance, not transient bias. So we want something fancier.

4.1.1. distribute product of likelihood terms for MH accept/reject (data parallelism).
Graph coloring of model --> hold one color fixed while Gibbs sampling the other (in parallel). 
(but both of these require frequent synchronization of model params & batched processing, below this constraint is relaxed)

4.1.2 branch prediction of MH accept/reject condition. Speculate into the future on multiple cores, but bad scaling (J cores --> log_2(J) speedup, slightly better if done probabilistically)

4.2 approximations without full synchrony

4.2.1 subposteriors: split data and prior and factors up into J cores. Total posterior is product of subs, but combining samples from subs is hard. 

4.2.2 asynchronous ('hogwild') gibbs},
author = {Angelino, Elaine and Johnson, Matthew James and Adams, Ryan P.},
doi = {10.1561/2200000052},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Angelino, Johnson, Adams - 2016 - Patterns of Scalable Bayesian Inference.pdf:pdf},
journal = {Foundations and Trends in Machine Learning},
keywords = {inference,parametric-sampling},
mendeley-groups = {Robbie Lab},
mendeley-tags = {inference,parametric-sampling},
number = {2-3},
pages = {119--247},
title = {{Patterns of Scalable Bayesian Inference}},
volume = {9},
year = {2016}
}
@article{Salimans2015,
abstract = {Recent advances in stochastic gradient variational inference have made it possible to perform variational Bayesian inference with posterior approximations containing auxiliary random variables. This enables us to explore a new synthesis of variational inference and Monte Carlo methods where we incorporate one or more steps of MCMC into our variational approximation. By doing so we obtain a rich class of inference algorithms bridging the gap between variational methods and MCMC, and offering the best of both worlds: fast posterior approximation through the maximization of an explicit objective, with the option of trading off additional computation for additional accuracy. We describe the theoretical foundations that make this possible and show some promising first results.},
archivePrefix = {arXiv},
arxivId = {1410.6460},
author = {Salimans, Tim and Kingma, Diederik P. and Welling, Max},
eprint = {1410.6460},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Salimans, Kingma, Welling - 2015 - Markov Chain Monte Carlo and Variational Inference Bridging the Gap.pdf:pdf},
isbn = {1410.6460},
journal = {Proceedings of the 32nd International Conference on Machine Learning},
keywords = {parametric-sampling},
mendeley-groups = {VariSampling},
mendeley-tags = {parametric-sampling},
pages = {1218--1226},
title = {{Markov Chain Monte Carlo and Variational Inference: Bridging the Gap}},
url = {http://arxiv.org/abs/1410.6460},
year = {2015}
}
@article{Wei2016,
abstract = {Fisher information is generally believed to represent a lower bound on mutual information (Brunel & Nadal, 1998), a result that is frequently used in the assessment of neural coding efficiency. However, we demon- strate that the relation between these two quantities is more nuanced than previously thought. For example, we find that in the small noise regime, Fisher information actually provides an upper bound on mutual information. Generally our results show that it is more appropriate to consider Fisher information as an approximation rather than a bound on mutual information.We analytically derive the correspondence between the two quantities and the conditions under which the approximation is good. Our results have implications for neural coding theories and the link between neural population coding and psychophysically mea- surable behavior. Specifically, they allow us to formulate the efficient coding problem of maximizing mutual information between a stimu- lus variable and the response of a neural population in terms of Fisher information. We derive a signature of efficient coding expressed as the correspondence between the population Fisher information and the dis- tribution of the stimulus variable. The signature is more general than previously proposed solutions that rely on specific assumptions about the neural tuning characteristics.We demonstrate that it can explain mea- sured tuning characteristics of cortical neural populations that do not agreewith previous models of efficient coding.},
author = {Wei, Xue-Xin and Stocker, Alan A.},
doi = {10.1162/NECO_a_0084},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Wei, Stocker - 2016 - Mutual Information, Fisher Information, and Efficient Coding.pdf:pdf},
journal = {Neural computation},
keywords = {parametric-sampling},
mendeley-groups = {Info. theory MaxEnt etc},
mendeley-tags = {parametric-sampling},
number = {2},
title = {{Mutual Information, Fisher Information, and Efficient Coding}},
volume = {28},
year = {2016}
}
@article{Hoffman2013,
abstract = {Abstract: We develop stochastic variational inference , a scalable algorithm for approximating posterior distributions. We develop this technique for a large class of probabilistic models and we demonstrate it with two probabilistic topic models, latent Dirichlet allocation and ...},
archivePrefix = {arXiv},
arxivId = {1206.7051},
author = {Hoffman, Matthew D. and Blei, David M. and Wang, Chong and Paisley, John},
doi = {citeulike-article-id:10852147},
eprint = {1206.7051},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Hoffman et al. - 2013 - Stochastic variational inference.pdf:pdf},
isbn = {1532-4435},
issn = {1532-4435},
journal = {Journal of Machine Learning Research},
keywords = {parametric-sampling},
mendeley-groups = {Info. theory MaxEnt etc},
mendeley-tags = {parametric-sampling},
pages = {1303--1347},
title = {{Stochastic variational inference}},
url = {http://arxiv.org/abs/1206.7051},
volume = {14},
year = {2013}
}
@article{Neal2010,
abstract = {... I also provided a statistically-oriented tutorial on HMC in a review of MCMC methods ( Neal , 1993, Chapter 5). There have been other applications of HMC to statistical problems (eg Ishwaran, 1999; Schmidt ... 1 Hamilton's Equations Hamiltonian dynamics operates on ...},
author = {Neal, Radford M.},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Neal - 2010 - MCMC using Hamiltonian dynamics.pdf:pdf},
journal = {Handbook of Markov Chain Monte Carlo},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
title = {{MCMC using Hamiltonian dynamics}},
year = {2010}
}
@article{Welling2011,
abstract = {In this paper we propose a new framework\n\nfor learning from large scale datasets based\n\non iterative learning from small mini-batches.\n\nBy adding the right amount of noise to a\n\nstandard stochastic gradient optimization algorithm\n\nwe show that the iterates will converge\n\nto samples from the true posterior distribution\n\nas we anneal the stepsize. This\n\nseamless transition between optimization and\n\nBayesian posterior sampling provides an inbuilt\n\nprotection against overfitting. We also\n\npropose a practical method for Monte Carlo\n\nestimates of posterior statistics which monitors\n\na “sampling threshold” and collects samples\n\nafter it has been surpassed. We apply\n\nthe method to three models: a mixture of\n\nGaussians, logistic regression and ICA with\n\nnatural gradients.},
author = {Welling, M and Teh, Y W},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Welling, Teh - 2011 - Bayesian Learning via Stochastic Gradient Langevin Dynamics.pdf:pdf},
isbn = {978-1-4503-0619-5},
journal = {Proceedings of the 27th International Conference on Machine Learning (ICML)},
keywords = {confirmation-bias,dynamics,parametric-sampling,posterior sampling,stochastic gradient},
mendeley-groups = {Info. theory MaxEnt etc,back burner,Haefner Lab/Confirmation Bias,BCS 582 Grant Writing},
mendeley-tags = {confirmation-bias,parametric-sampling},
title = {{Bayesian Learning via Stochastic Gradient Langevin Dynamics}},
url = {http://machinelearning.wustl.edu/mlpapers/paper_files/ICML2011Welling_398.pdf},
year = {2011}
}
@article{Yao2018,
archivePrefix = {arXiv},
arxivId = {arXiv:1802.02538v2},
author = {Yao, Yulin and Vehtari, Aki and Simpson, Daniel and Gelman, Andrew},
eprint = {arXiv:1802.02538v2},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Yao, Vehtari - 2018 - Yes, but Did It Work Evaluating Variational Inference.pdf:pdf},
journal = {International Conference on Machine Learning},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
pages = {5581--5590},
title = {{Yes, but Did It Work?: Evaluating Variational Inference}},
volume = {80},
year = {2018}
}
@article{Ranganath2016,
abstract = {Black box inference allows researchers to easily prototype and evaluate an array of models. Re-cent advances in variational inference allow such algorithms to scale to high dimensions. However, a central question remains: How to specify an ex-pressive variational distribution which maintains efficient computation? To address this, we de-velop hierarchical variational models. In a HIERAR-CHICAL VM, the variational approximation is aug-mented with a prior on its parameters, such that the latent variables are conditionally independent given this shared structure. This preserves the computational efficiency of the original approx-imation, while admitting hierarchically complex distributions for both discrete and continuous la-tent variables. We study HIERARCHICAL VM on a variety of deep discrete latent variable models. HI-ERARCHICAL VM generalizes other expressive varia-tional distributions and maintains higher fidelity to the posterior.},
archivePrefix = {arXiv},
arxivId = {1511.02386},
author = {Ranganath, Rajesh and Tran, Dustin and Blei, David M.},
eprint = {1511.02386},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Ranganath, Tran, Blei - 2016 - Hierarchical Variational Models.pdf:pdf},
isbn = {1511.02386},
journal = {ICML},
pages = {1--9},
title = {{Hierarchical Variational Models}},
volume = {33},
year = {2016}
}
@article{Blei2017,
archivePrefix = {arXiv},
arxivId = {arXiv:1601.00670v5},
author = {Blei, David M. and Kucukelbir, Alp and Mcauliffe, Jon D},
eprint = {arXiv:1601.00670v5},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Blei, Kucukelbir, Mcauliffe - 2017 - Variational Inference A Review for Statisticians.pdf:pdf},
journal = {arXiv},
keywords = {algorithms,computationally intensive methods,statistical computing},
pages = {1--41},
title = {{Variational Inference: A Review for Statisticians}},
year = {2017}
}
@article{Guo2016,
abstract = {Variational inference (VI) provides fast approximations of a Bayesian posterior in part because it formulates posterior approximation as an optimization problem: to find the closest distribution to the exact posterior over some family of distributions. For practical reasons, the family of distributions in VI is usually constrained so that it does not include the exact posterior, even as a limit point. Thus, no matter how long VI is run, the resulting approximation will not approach the exact posterior. We propose to instead consider a more flexible approximating family consisting of all possible finite mixtures of a parametric base distribution (e.g., Gaussian). For efficient inference, we borrow ideas from gradient boosting to develop an algorithm we call boosting variational inference (BVI). BVI iteratively improves the current approximation by mixing it with a new component from the base distribution family and thereby yields progressively more accurate posterior approximations as more computing time is spent. Unlike a number of common VI variants including mean-field VI, BVI is able to capture multimodality, general posterior covariance, and nonstandard posterior shapes.},
archivePrefix = {arXiv},
arxivId = {1611.05559},
author = {Guo, Fangjian and Wang, Xiangyu and Fan, Kai and Broderick, Tamara and Dunson, David B.},
eprint = {1611.05559},
file = {:Users/richard/Documents/papers/1611.05559.pdf:pdf},
journal = {arXiv},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
title = {{Boosting Variational Inference}},
url = {http://arxiv.org/abs/1611.05559},
year = {2016}
}
@article{Yin2018,
author = {Yin, Mingzhang and Zhou, Mingyuan},
file = {:Users/richard/Documents/papers/yin18b.pdf:pdf},
journal = {International Conference on Machine Learning},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
title = {{Semi-Implicit Variational Inference}},
volume = {35},
year = {2018}
}
@article{Nalisnick2017,
author = {Nalisnick, Eric and Smyth, Padhraic},
doi = {10.1126/science.1070850},
file = {:Users/richard/Library/Application Support/Mendeley Desktop/Downloaded/Nalisnick, Smyth - 2017 - Variational Inference with Stein Mixtures.pdf:pdf},
issn = {00368075},
journal = {NIPS2017 (Workshop)},
keywords = {parametric-sampling},
mendeley-tags = {parametric-sampling},
title = {{Variational Inference with Stein Mixtures}},
url = {https://www.ics.uci.edu/$\sim$enalisni/AABI_paper30-Stein_Mixtures.pdf},
year = {2017}
}
@article{Arenz2018,
abstract = {Inference from complex distributions is a common problem in machine learning needed for many Bayesian methods. We propose an efficient, gradient-free method for learning general GMM approximations of multimodal distributions based on recent insights from stochastic search methods. Our method establishes information-geometric trust regions to ensure efficient exploration of the sampling space and stability of the GMM updates, allowing for efficient estimation of multi-variate Gaussian variational distributions. For GMMs, we apply a variational lower bound to decompose the learning objective into sub-problems given by learning the individual mixture components and the coefficients. The number of mixture components is adapted online in order to allow for arbitrary exact approximations. We demonstrate on several domains that we can learn significantly better approximations than competing variational inference methods and that the quality of samples drawn from our approximations is on par with samples created by state-of-the-art MCMC samplers that require significantly more computational resources.},
author = {Arenz, Oleg and Zhong, Mingjun and Neumann, Gerhard},
file = {:Users/richard/Documents/papers/arenz18a.pdf:pdf},
issn = {2640-3498},
journal = {Proceedings of the 35th International Conference on Machine Learning},
pages = {234--243},
title = {{Efficient Gradient-Free Variational Inference using Policy Search}},
url = {http://proceedings.mlr.press/v80/arenz18a.html},
year = {2018}
}
@article{Zobay2014,
abstract = {Variational Bayesian inference with a Gaussian posterior approximation provides an alternative to the more commonly employed factorization approach and enlarges the range of tractable distributions. In this paper, we propose an extension to the Gaussian approach which uses Gaussian mixtures as approximations. A general problem for variational inference with mixtures is posed by the calculation of the entropy term in the Kullback-Leibler distance, which becomes analytically intractable. We deal with this problem by using a simple lower bound for the entropy and imposing restrictions on the form of the Gaussian covariance matrix. In this way, efficient numerical calculations become possible. To illustrate the method, we discuss its application to an isotropic generalized normal target density, a non-Gaussian state space model, and the Bayesian lasso. For heavy-tailed distributions, the examples show that the mixture approach indeed leads to improved approximations in the sense of a reduced Kullback-Leibler distance. From a more practical point of view, mixtures can improve estimates of posterior marginal variances. Furthermore, they provide an initial estimate of posterior skewness which is not possible with single Gaussians. We also discuss general sufficient conditions under which mixtures are guaranteed to provide improvements over single-component approximations.},
author = {Zobay, O.},
doi = {10.1214/14-EJS887},
file = {:Users/richard/Documents/papers/14-EJS887.pdf:pdf},
issn = {19357524},
journal = {Electronic Journal of Statistics},
keywords = {Approximation methods,Bayesian lasso,Normal mixtures,State-space models,Variational inference,parametric-sampling},
mendeley-tags = {parametric-sampling},
number = {1},
pages = {355--389},
title = {{Variational Bayesian inference with Gaussian-mixture approximations}},
volume = {8},
year = {2014}
}
@article{Durmus2017,
author = {Durmus, Alain and Moulines, {\'{E}}ric},
doi = {10.1214/16-AAP1238},
journal = {The Annals of Applied Probability},
keywords = {Langevin diffusion,Markov chain Monte Carlo,Metropolis adjusted Langevin algorithm,rate of convergence,total variation distance},
number = {3},
pages = {1551--1587},
publisher = {Institute of Mathematical Statistics},
title = {{Nonasymptotic convergence analysis for the unadjusted Langevin algorithm}},
url = {https://doi.org/10.1214/16-AAP1238},
volume = {27},
year = {2017}
}
