@inproceedings{ober2021vardwp,
 author = {Ober, Sebastian and Aitchison, Laurence},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {M. Ranzato and A. Beygelzimer and Y. Dauphin and P.S. Liang and J. Wortman Vaughan},
 pages = {6567--6579},
 publisher = {Curran Associates, Inc.},
 title = {A variational approximate posterior for the deep {W}ishart process},
 url = {https://proceedings.neurips.cc/paper/2021/file/33ceb07bf4eeb3da587e268d663aba1a-Paper.pdf},
 volume = {34},
 year = {2021}
}


@InProceedings{aitchison2021dkp,
  title = 	 {Deep Kernel Processes},
  author =       {Aitchison, Laurence and Yang, Adam and Ober, Sebastian W},
  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
  pages = 	 {130--140},
  year = 	 {2021},
  editor = 	 {Meila, Marina and Zhang, Tong},
  volume = 	 {139},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {18--24 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v139/aitchison21a/aitchison21a.pdf},
  url = 	 {https://proceedings.mlr.press/v139/aitchison21a.html},
  abstract = 	 {We define deep kernel processes in which positive definite Gram matrices are progressively transformed by nonlinear kernel functions and by sampling from (inverse) Wishart distributions. Remarkably, we find that deep Gaussian processes (DGPs), Bayesian neural networks (BNNs), infinite BNNs, and infinite BNNs with bottlenecks can all be written as deep kernel processes. For DGPs the equivalence arises because the Gram matrix formed by the inner product of features is Wishart distributed, and as we show, standard isotropic kernels can be written entirely in terms of this Gram matrix — we do not need knowledge of the underlying features. We define a tractable deep kernel process, the deep inverse Wishart process, and give a doubly-stochastic inducing-point variational inference scheme that operates on the Gram matrices, not on the features, as in DGPs. We show that the deep inverse Wishart process gives superior performance to DGPs and infinite BNNs on fully-connected baselines.}
}

@inproceedings{damianou2013deep,
  title={Deep {G}aussian {P}rocesses},
  author={Damianou, Andreas and Lawrence, Neil},
  booktitle={Artificial Intelligence and Statistics},
  pages={207--215},
  year={2013}
}

@inproceedings{krizhevsky2012cnn,
 author = {Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {F. Pereira and C.J. Burges and L. Bottou and K.Q. Weinberger},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Image{N}et {C}lassification with {D}eep {C}onvolutional {N}eural {N}etworks},
 url = {https://proceedings.neurips.cc/paper/2012/file/c399862d3b9d6b76c8436e924a68c45b-Paper.pdf},
 volume = {25},
 year = {2012}
}
@article{bartlett1933on,
  title={On the {T}heory of {S}tatistical {R}egression},
  author={Bartlett, Maurice Stevenson},
  journal={Proceedings of the Royal Society of Edinburgh},
  volume={53},
  pages={260--283},
  year={1933},
  publisher={Royal Society of Edinburgh Scotland Foundation}
}
@inproceedings{pmlr-v48-gal16,
author = {Gal, Yarin and Ghahramani, Zoubin},
title = {Dropout as a {B}ayesian {A}pproximation: {R}epresenting {M}odel {U}ncertainty in {D}eep {L}earning},
year = {2016},
publisher = {JMLR.org},
abstract = {Deep learning tools have gained tremendous attention in applied machine learning. However such tools for regression and classification do not capture model uncertainty. In comparison, Bayesian models offer a mathematically grounded framework to reason about model uncertainty, but usually come with a prohibitive computational cost. In this paper we develop a new theoretical framework casting dropout training in deep neural networks (NNs) as approximate Bayesian inference in deep Gaussian processes. A direct result of this theory gives us tools to model uncertainty with dropout NNs - extracting information from existing models that has been thrown away so far. This mitigates the problem of representing uncertainty in deep learning without sacrificing either computational complexity or test accuracy. We perform an extensive study of the properties of dropout's uncertainty. Various network architectures and nonlinearities are assessed on tasks of regression and classification, using MNIST as an example. We show a considerable improvement in predictive log-likelihood and RMSE compared to existing state-of-the-art methods, and finish by using dropout's uncertainty in deep reinforcement learning.},
booktitle = {Proceedings of the 33rd International Conference on International Conference on Machine Learning - Volume 48},
pages = {1050–1059},
numpages = {10},
location = {New York, NY, USA},
series = {ICML'16}
}
@inproceedings{kingma14adam,
  author    = {Diederik P. Kingma and
               Jimmy Ba},
  editor    = {Yoshua Bengio and
               Yann LeCun},
  title     = {Adam: {A} Method for Stochastic Optimization},
  booktitle = {3rd International Conference on Learning Representations, {ICLR} 2015,
               San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings},
  year      = {2015},
  url       = {http://arxiv.org/abs/1412.6980},
  timestamp = {Thu, 25 Jul 2019 14:25:37 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/KingmaB14.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}
@InProceedings{ober21globalinducing,
  title = 	 {Global inducing point variational posteriors for {B}ayesian neural networks and deep {G}aussian processes},
  author =       {Ober, Sebastian W and Aitchison, Laurence},
  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
  pages = 	 {8248--8259},
  year = 	 {2021},
  editor = 	 {Meila, Marina and Zhang, Tong},
  volume = 	 {139},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {18--24 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v139/ober21a/ober21a.pdf},
  url = 	 {https://proceedings.mlr.press/v139/ober21a.html},
  abstract = 	 {We consider the optimal approximate posterior over the top-layer weights in a Bayesian neural network for regression, and show that it exhibits strong dependencies on the lower-layer weights. We adapt this result to develop a correlated approximate posterior over the weights at all layers in a Bayesian neural network. We extend this approach to deep Gaussian processes, unifying inference in the two model classes. Our approximate posterior uses learned "global” inducing points, which are defined only at the input layer and propagated through the network to obtain inducing inputs at subsequent layers. By contrast, standard, "local”, inducing point methods from the deep Gaussian process literature optimise a separate set of inducing inputs at every layer, and thus do not model correlations across layers. Our method gives state-of-the-art performance for a variational Bayesian method, without data augmentation or tempering, on CIFAR-10 of 86.7%, which is comparable to SGMCMC without tempering but with data augmentation (88% in Wenzel et al. 2020).}
}


@article{srivastava2003singularwishart,
author = {M.S. Srivastava},
title = {{Singular Wishart and multivariate beta distributions}},
volume = {31},
journal = {The Annals of Statistics},
number = {5},
publisher = {Institute of Mathematical Statistics},
pages = {1537 -- 1560},
keywords = {Jacobian of transformations, normal distribution, pseudo Wishart, singular noncentral Wishart, Stiefel manifold},
year = {2003},
doi = {10.1214/aos/1065705118},
URL = {https://doi.org/10.1214/aos/1065705118}
}
@book{murphy2012machine,
  title={Machine learning: {A} probabilistic perspective},
  author={Murphy, Kevin P},
  year={2012},
  publisher={MIT press}
}
@book{mathai1997jacobians,
  title={Jacobians of matrix transformation and functions of matrix arguments},
  author={Mathai, Arak M},
  year={1997},
  publisher={World Scientific Publishing Company}
}
@book{mathai2008special,
  title={Special functions for applied scientists},
  author={Mathai, Arakaparampil M and Haubold, Hans J},
  year={2008},
  publisher={Springer}
}
@book{williams2006gaussian,
  title={Gaussian {P}rocesses for {M}achine {L}earning},
  author={Williams, Christopher KI and Rasmussen, Carl Edward},
  year={2006},
  publisher={MIT press Cambridge, MA}
}

@inproceedings{wilson2011generalizedwishart,
author = {Wilson, Andrew Gordon and Ghahramani, Zoubin},
title = {Generalised Wishart Processes},
year = {2011},
isbn = {9780974903972},
publisher = {AUAI Press},
address = {Arlington, Virginia, USA},
abstract = {We introduce a new stochastic process called the generalised Wishart process (GWP). It is a collection of positive semi-definite random matrices indexed by any arbitrary input variable. We use this process as a prior over dynamic (e.g. time varying) covariance matrices Σ(t). The GWP captures a diverse class of covariance dynamics, naturally handles missing data, scales nicely with dimension, has easily interpretable parameters, and can use input variables that include covariates other than time. We describe how to construct the GWP, introduce general procedures for inference and prediction, and show that it outperforms its main competitor, multivariate GARCH, even on financial data that especially suits GARCH.},
booktitle = {Proceedings of the Twenty-Seventh Conference on Uncertainty in Artificial Intelligence},
pages = {736–744},
numpages = {9},
location = {Barcelona, Spain},
series = {UAI'11}
}

@article{lee2017deep,
  title={Deep neural networks as gaussian processes},
  author={Lee, Jaehoon and Bahri, Yasaman and Novak, Roman and Schoenholz, Samuel S and Pennington, Jeffrey and Sohl-Dickstein, Jascha},
  journal={arXiv preprint arXiv:1711.00165},
  year={2017}
}

@article{matthews2018gaussian,
  title={Gaussian process behaviour in wide deep neural networks},
  author={Matthews, Alexander G de G and Rowland, Mark and Hron, Jiri and Turner, Richard E and Ghahramani, Zoubin},
  journal={arXiv preprint arXiv:1804.11271},
  year={2018}
}

@article{novak2018bayesian,
  title={Bayesian deep convolutional networks with many channels are gaussian processes},
  author={Novak, Roman and Xiao, Lechao and Lee, Jaehoon and Bahri, Yasaman and Yang, Greg and Hron, Jiri and Abolafia, Daniel A and Pennington, Jeffrey and Sohl-Dickstein, Jascha},
  journal={arXiv preprint arXiv:1810.05148},
  year={2018}
}

@article{garriga2018deep,
  title={Deep convolutional networks as shallow gaussian processes},
  author={Garriga-Alonso, Adri{\`a} and Rasmussen, Carl Edward and Aitchison, Laurence},
  journal={arXiv preprint arXiv:1808.05587},
  year={2018}
}

@inproceedings{aitchison2020bigger,
  title={Why bigger is not always better: on finite and infinite neural networks},
  author={Aitchison, Laurence},
  booktitle={International Conference on Machine Learning},
  pages={156--164},
  year={2020},
  organization={PMLR}
}

@article{yang2020feature,
  title={Feature learning in infinite-width neural networks},
  author={Yang, Greg and Hu, Edward J},
  journal={arXiv preprint arXiv:2011.14522},
  year={2020}
}

@article{dkm23,
  title={A theory of representation learning in deep neural networks gives a deep generalisation of kernel methods},
  author={Yang, Adam X and Robeyns, Maxime and Milsom, Edward and Anson, Ben and Schoots, Nandi and Aitchison, Laurence},
  journal={International Conference on Machine Learning},
  year={2023}
}
@article{cdkm23,
 author = {Milsom, Edward and Anson, Ben and Aitchison, Laurence},
 title = {Convolutional Deep Kernel Machines},
 year = {2023}
}
