@article{Sun2022,
   author = {Li Sun and Junxiang Chen and Yanwu Xu and Mingming Gong and Ke Yu and Kayhan Batmanghelich},
   doi = {10.1109/JBHI.2022.3172976},
   issn = {2168-2194},
   issue = {8},
   journal = {IEEE Journal of Biomedical and Health Informatics},
   month = {8},
   pages = {3966-3975},
   title = {Hierarchical Amortized GAN for 3D High Resolution Medical Image Synthesis},
   volume = {26},
   year = {2022},
}
@article{Chen2019,
   abstract = {The performance on deep learning is significantly affected by volume of training data. Models pre-trained from massive dataset such as ImageNet become a powerful weapon for speeding up training convergence and improving accuracy. Similarly, models based on large dataset are important for the development of deep learning in 3D medical images. However, it is extremely challenging to build a sufficiently large dataset due to difficulty of data acquisition and annotation in 3D medical imaging. We aggregate the dataset from several medical challenges to build 3DSeg-8 dataset with diverse modalities, target organs, and pathologies. To extract general medical three-dimension (3D) features, we design a heterogeneous 3D network called Med3D to co-train multi-domain 3DSeg-8 so as to make a series of pre-trained models. We transfer Med3D pre-trained models to lung segmentation in LIDC dataset, pulmonary nodule classification in LIDC dataset and liver segmentation on LiTS challenge. Experiments show that the Med3D can accelerate the training convergence speed of target 3D medical tasks 2 times compared with model pre-trained on Kinetics dataset, and 10 times compared with training from scratch as well as improve accuracy ranging from 3% to 20%. Transferring our Med3D model on state-the-of-art DenseASPP segmentation network, in case of single model, we achieve 94.6\% Dice coefficient which approaches the result of top-ranged algorithms on the LiTS challenge.},
   author = {Sihong Chen and Kai Ma and Yefeng Zheng},
   month = {4},
   title = {Med3D: Transfer Learning for 3D Medical Image Analysis},
   year = {2019},
}
@article{Heusel2017,
   abstract = {Generative Adversarial Networks (GANs) excel at creating realistic images with complex models for which maximum likelihood is infeasible. However, the convergence of GAN training has still not been proved. We propose a two time-scale update rule (TTUR) for training GANs with stochastic gradient descent on arbitrary GAN loss functions. TTUR has an individual learning rate for both the discriminator and the generator. Using the theory of stochastic approximation, we prove that the TTUR converges under mild assumptions to a stationary local Nash equilibrium. The convergence carries over to the popular Adam optimization, for which we prove that it follows the dynamics of a heavy ball with friction and thus prefers flat minima in the objective landscape. For the evaluation of the performance of GANs at image generation, we introduce the "Fr\'echet Inception Distance" (FID) which captures the similarity of generated images to real ones better than the Inception Score. In experiments, TTUR improves learning for DCGANs and Improved Wasserstein GANs (WGAN-GP) outperforming conventional GAN training on CelebA, CIFAR-10, SVHN, LSUN Bedrooms, and the One Billion Word Benchmark.},
   author = {Martin Heusel and Hubert Ramsauer and Thomas Unterthiner and Bernhard Nessler and Sepp Hochreiter},
   month = {6},
   title = {GANs Trained by a Two Time-Scale Update Rule Converge to a Local Nash Equilibrium},
   year = {2017},
}
@article{Ho2020,
   abstract = {We present high quality image synthesis results using diffusion probabilistic models, a class of latent variable models inspired by considerations from nonequilibrium thermodynamics. Our best results are obtained by training on a weighted variational bound designed according to a novel connection between diffusion probabilistic models and denoising score matching with Langevin dynamics, and our models naturally admit a progressive lossy decompression scheme that can be interpreted as a generalization of autoregressive decoding. On the unconditional CIFAR10 dataset, we obtain an Inception score of 9.46 and a state-of-the-art FID score of 3.17. On 256x256 LSUN, we obtain sample quality similar to ProgressiveGAN. Our implementation is available at https://github.com/hojonathanho/diffusion},
   author = {Jonathan Ho and Ajay Jain and Pieter Abbeel},
   month = {6},
   title = {Denoising Diffusion Probabilistic Models},
   year = {2020},
}
@article{Ramesh2021,
   abstract = {Text-to-image generation has traditionally focused on finding better modeling assumptions for training on a fixed dataset. These assumptions might involve complex architectures, auxiliary losses, or side information such as object part labels or segmentation masks supplied during training. We describe a simple approach for this task based on a transformer that autoregressively models the text and image tokens as a single stream of data. With sufficient data and scale, our approach is competitive with previous domain-specific models when evaluated in a zero-shot fashion.},
   author = {Aditya Ramesh and Mikhail Pavlov and Gabriel Goh and Scott Gray and Chelsea Voss and Alec Radford and Mark Chen and Ilya Sutskever},
   month = {2},
   title = {Zero-Shot Text-to-Image Generation},
   year = {2021},
}
@article{Saharia2022,
   abstract = {We present Imagen, a text-to-image diffusion model with an unprecedented degree of photorealism and a deep level of language understanding. Imagen builds on the power of large transformer language models in understanding text and hinges on the strength of diffusion models in high-fidelity image generation. Our key discovery is that generic large language models (e.g. T5), pretrained on text-only corpora, are surprisingly effective at encoding text for image synthesis: increasing the size of the language model in Imagen boosts both sample fidelity and image-text alignment much more than increasing the size of the image diffusion model. Imagen achieves a new state-of-the-art FID score of 7.27 on the COCO dataset, without ever training on COCO, and human raters find Imagen samples to be on par with the COCO data itself in image-text alignment. To assess text-to-image models in greater depth, we introduce DrawBench, a comprehensive and challenging benchmark for text-to-image models. With DrawBench, we compare Imagen with recent methods including VQ-GAN+CLIP, Latent Diffusion Models, and DALL-E 2, and find that human raters prefer Imagen over other models in side-by-side comparisons, both in terms of sample quality and image-text alignment. See https://imagen.research.google/ for an overview of the results.},
   author = {Chitwan Saharia and William Chan and Saurabh Saxena and Lala Li and Jay Whang and Emily Denton and Seyed Kamyar Seyed Ghasemipour and Burcu Karagol Ayan and S. Sara Mahdavi and Rapha Gontijo Lopes and Tim Salimans and Jonathan Ho and David J Fleet and Mohammad Norouzi},
   month = {5},
   title = {Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding},
   year = {2022},
}
@article{Rombach2021,
   abstract = {By decomposing the image formation process into a sequential application of denoising autoencoders, diffusion models (DMs) achieve state-of-the-art synthesis results on image data and beyond. Additionally, their formulation allows for a guiding mechanism to control the image generation process without retraining. However, since these models typically operate directly in pixel space, optimization of powerful DMs often consumes hundreds of GPU days and inference is expensive due to sequential evaluations. To enable DM training on limited computational resources while retaining their quality and flexibility, we apply them in the latent space of powerful pretrained autoencoders. In contrast to previous work, training diffusion models on such a representation allows for the first time to reach a near-optimal point between complexity reduction and detail preservation, greatly boosting visual fidelity. By introducing cross-attention layers into the model architecture, we turn diffusion models into powerful and flexible generators for general conditioning inputs such as text or bounding boxes and high-resolution synthesis becomes possible in a convolutional manner. Our latent diffusion models (LDMs) achieve a new state of the art for image inpainting and highly competitive performance on various tasks, including unconditional image generation, semantic scene synthesis, and super-resolution, while significantly reducing computational requirements compared to pixel-based DMs. Code is available at https://github.com/CompVis/latent-diffusion .},
   author = {Robin Rombach and Andreas Blattmann and Dominik Lorenz and Patrick Esser and Björn Ommer},
   month = {12},
   title = {High-Resolution Image Synthesis with Latent Diffusion Models},
   year = {2021},
}
@article{Ho2022,
   abstract = {Classifier guidance is a recently introduced method to trade off mode coverage and sample fidelity in conditional diffusion models post training, in the same spirit as low temperature sampling or truncation in other types of generative models. Classifier guidance combines the score estimate of a diffusion model with the gradient of an image classifier and thereby requires training an image classifier separate from the diffusion model. It also raises the question of whether guidance can be performed without a classifier. We show that guidance can be indeed performed by a pure generative model without such a classifier: in what we call classifier-free guidance, we jointly train a conditional and an unconditional diffusion model, and we combine the resulting conditional and unconditional score estimates to attain a trade-off between sample quality and diversity similar to that obtained using classifier guidance.},
   author = {Jonathan Ho and Tim Salimans},
   month = {7},
   title = {Classifier-Free Diffusion Guidance},
   year = {2022},
}
@article{Dhariwal2021,
   abstract = {We show that diffusion models can achieve image sample quality superior to the current state-of-the-art generative models. We achieve this on unconditional image synthesis by finding a better architecture through a series of ablations. For conditional image synthesis, we further improve sample quality with classifier guidance: a simple, compute-efficient method for trading off diversity for fidelity using gradients from a classifier. We achieve an FID of 2.97 on ImageNet 128$\times$128, 4.59 on ImageNet 256$\times$256, and 7.72 on ImageNet 512$\times$512, and we match BigGAN-deep even with as few as 25 forward passes per sample, all while maintaining better coverage of the distribution. Finally, we find that classifier guidance combines well with upsampling diffusion models, further improving FID to 3.94 on ImageNet 256$\times$256 and 3.85 on ImageNet 512$\times$512. We release our code at https://github.com/openai/guided-diffusion},
   author = {Prafulla Dhariwal and Alex Nichol},
   month = {5},
   title = {Diffusion Models Beat GANs on Image Synthesis},
   year = {2021},
}
@article{Creswell2017,
   abstract = {Generative adversarial networks (GANs) provide a way to learn deep representations without extensively annotated training data. They achieve this through deriving backpropagation signals through a competitive process involving a pair of networks. The representations that can be learned by GANs may be used in a variety of applications, including image synthesis, semantic image editing, style transfer, image super-resolution and classification. The aim of this review paper is to provide an overview of GANs for the signal processing community, drawing on familiar analogies and concepts where possible. In addition to identifying different methods for training and constructing GANs, we also point to remaining challenges in their theory and application.},
   author = {Antonia Creswell and Tom White and Vincent Dumoulin and Kai Arulkumaran and Biswa Sengupta and Anil A Bharath},
   doi = {10.1109/MSP.2017.2765202},
   month = {10},
   title = {Generative Adversarial Networks: An Overview},
   year = {2017},
}
@article{Kwon2019,
   abstract = {As deep learning is showing unprecedented success in medical image analysis tasks, the lack of sufficient medical data is emerging as a critical problem. While recent attempts to solve the limited data problem using Generative Adversarial Networks (GAN) have been successful in generating realistic images with diversity, most of them are based on image-to-image translation and thus require extensive datasets from different domains. Here, we propose a novel model that can successfully generate 3D brain MRI data from random vectors by learning the data distribution. Our 3D GAN model solves both image blurriness and mode collapse problems by leveraging alpha-GAN that combines the advantages of Variational Auto-Encoder (VAE) and GAN with an additional code discriminator network. We also use the Wasserstein GAN with Gradient Penalty (WGAN-GP) loss to lower the training instability. To demonstrate the effectiveness of our model, we generate new images of normal brain MRI and show that our model outperforms baseline models in both quantitative and qualitative measurements. We also train the model to synthesize brain disorder MRI data to demonstrate the wide applicability of our model. Our results suggest that the proposed model can successfully generate various types and modalities of 3D whole brain volumes from a small set of training data.},
   author = {Gihyun Kwon and Chihye Han and Dae-shik Kim},
   month = {8},
   title = {Generation of 3D Brain MRI Using Auto-Encoding Generative Adversarial Networks},
   year = {2019},
}
@article{Wang2020,
   author = {Lei Wang and Wei Chen and Wenjia Yang and Fangming Bi and Fei Richard Yu},
   doi = {10.1109/ACCESS.2020.2982224},
   issn = {2169-3536},
   journal = {IEEE Access},
   pages = {63514-63537},
   title = {A State-of-the-Art Review on Image Synthesis With Generative Adversarial Networks},
   volume = {8},
   year = {2020},
}
@article{Wang2021,
   abstract = {<p>This paper reviewed the deep learning‐based studies for medical imaging synthesis and its clinical application. Specifically, we summarized the recent developments of deep learning‐based methods in inter‐ and intra‐modality image synthesis by listing and highlighting the proposed methods, study designs, and reported performances with related clinical applications on representative studies. The challenges among the reviewed studies were then summarized with discussion.</p>},
   author = {Tonghe Wang and Yang Lei and Yabo Fu and Jacob F. Wynne and Walter J. Curran and Tian Liu and Xiaofeng Yang},
   doi = {10.1002/acm2.13121},
   issn = {1526-9914},
   issue = {1},
   journal = {Journal of Applied Clinical Medical Physics},
   month = {1},
   pages = {11-36},
   title = {A review on medical imaging synthesis using deep learning and its clinical applications},
   volume = {22},
   year = {2021},
}
@article{Jordan2020,
   abstract = {Many ground-breaking advancements in machine learning can be attributed to the availability of a large volume of rich data. Unfortunately, many large-scale datasets are highly sensitive, such as healthcare data, and are not widely available to the machine learning community. Generating synthetic data with privacy guarantees provides one such solution, allowing meaningful research to be carried out "at scale" - by allowing the entirety of the machine learning community to potentially accelerate progress within a given field. In this article, we provide a high-level view of synthetic data: what it means, how we might evaluate it and how we might use it.},
   author = {James Jordon and Alan Wilson and Mihaela van der Schaar},
   month = {12},
   title = {Synthetic Data: Opening the data floodgates to enable faster, more directed development of machine learning methods},
   year = {2020},
}
@article{Jordan2022,
   abstract = {This explainer document aims to provide an overview of the current state of the rapidly expanding work on synthetic data technologies, with a particular focus on privacy. The article is intended for a non-technical audience, though some formal definitions have been given to provide clarity to specialists. This article is intended to enable the reader to quickly become familiar with the notion of synthetic data, as well as understand some of the subtle intricacies that come with it. We do believe that synthetic data is a very useful tool, and our hope is that this report highlights that, while drawing attention to nuances that can easily be overlooked in its deployment.},
   author = {James Jordon and Lukasz Szpruch and Florimond Houssiau and Mirko Bottarelli and Giovanni Cherubin and Carsten Maple and Samuel N. Cohen and Adrian Weller},
   month = {5},
   title = {Synthetic Data -- what, why and how?},
   year = {2022},
}
@article{Kiryati2021,
   abstract = {<p>Medical image analysis research requires medical image datasets. Nevertheless, due to various impediments, researchers have been described as “data starved”. We hypothesize that implicit evolving community standards require researchers to use ever-growing datasets. In Phase I of this research, we scanned the MICCAI (Medical Image Computing and Computer-Assisted Intervention) conference proceedings from 2011 to 2018. We identified 907 papers involving human MRI, CT or fMRI datasets and extracted their sizes. The median dataset size had grown by 3–10 times from 2011 to 2018, depending on imaging modality. Statistical analysis revealed exponential growth of the geometric mean dataset size with an annual growth of 21% for MRI, 24% for CT and 31% for fMRI. Thereupon, we had issued a forecast for dataset sizes in MICCAI 2019 well before the conference. In Phase II of this research, we examined the MICCAI 2019 proceedings and analyzed 308 relevant papers. The MICCAI 2019 statistics compare well with the forecast. The revised annual growth rates of the geometric mean dataset size are 27% for MRI, 30% for CT and 32% for fMRI. We predict the respective dataset sizes in the MICCAI 2020 conference (that we have not yet analyzed) and the future MICCAI 2021 conference.</p>},
   author = {Nahum Kiryati and Yuval Landau},
   doi = {10.3390/jimaging7080155},
   issn = {2313-433X},
   issue = {8},
   journal = {Journal of Imaging},
   month = {8},
   pages = {155},
   title = {Dataset Growth in Medical Image Analysis Research},
   volume = {7},
   year = {2021},
}
@inproceedings{Deng2009,
   author = {Jia Deng and Wei Dong and Richard Socher and Li-Jia Li and Kai Li and Li Fei-Fei},
   doi = {10.1109/CVPR.2009.5206848},
   isbn = {978-1-4244-3992-8},
   journal = {2009 IEEE Conference on Computer Vision and Pattern Recognition},
   month = {6},
   pages = {248-255},
   publisher = {IEEE},
   title = {ImageNet: A large-scale hierarchical image database},
   year = {2009},
}
@article{Schuhmann2021,
   abstract = {Multi-modal language-vision models trained on hundreds of millions of image-text pairs (e.g. CLIP, DALL-E) gained a recent surge, showing remarkable capability to perform zero- or few-shot learning and transfer even in absence of per-sample labels on target image data. Despite this trend, to date there has been no publicly available datasets of sufficient scale for training such models from scratch. To address this issue, in a community effort we build and release for public LAION-400M, a dataset with CLIP-filtered 400 million image-text pairs, their CLIP embeddings and kNN indices that allow efficient similarity search.},
   author = {Christoph Schuhmann and Richard Vencu and Romain Beaumont and Robert Kaczmarczyk and Clayton Mullis and Aarush Katta and Theo Coombes and Jenia Jitsev and Aran Komatsuzaki},
   month = {11},
   title = {LAION-400M: Open Dataset of CLIP-Filtered 400 Million Image-Text Pairs},
   year = {2021},
}
@article{Pinaya2022,
   abstract = {Deep neural networks have brought remarkable breakthroughs in medical image analysis. However, due to their data-hungry nature, the modest dataset sizes in medical imaging projects might be hindering their full potential. Generating synthetic data provides a promising alternative, allowing to complement training datasets and conducting medical image research at a larger scale. Diffusion models recently have caught the attention of the computer vision community by producing photorealistic synthetic images. In this study, we explore using Latent Diffusion Models to generate synthetic images from high-resolution 3D brain images. We used T1w MRI images from the UK Biobank dataset (N=31,740) to train our models to learn about the probabilistic distribution of brain images, conditioned on covariables, such as age, sex, and brain structure volumes. We found that our models created realistic data, and we could use the conditioning variables to control the data generation effectively. Besides that, we created a synthetic dataset with 100,000 brain images and made it openly available to the scientific community.},
   author = {Walter H. L. Pinaya and Petru-Daniel Tudosiu and Jessica Dafflon and Pedro F da Costa and Virginia Fernandez and Parashkev Nachev and Sebastien Ourselin and M. Jorge Cardoso},
   month = {9},
   title = {Brain Imaging Generation with Latent Diffusion Models},
   year = {2022},
}
@article{Graham2023,
   abstract = {Methods for out-of-distribution (OOD) detection that scale to 3D data are crucial components of any real-world clinical deep learning system. Classic denoising diffusion probabilistic models (DDPMs) have been recently proposed as a robust way to perform reconstruction-based OOD detection on 2D datasets, but do not trivially scale to 3D data. In this work, we propose to use Latent Diffusion Models (LDMs), which enable the scaling of DDPMs to high-resolution 3D medical data. We validate the proposed approach on near- and far-OOD datasets and compare it to a recently proposed, 3D-enabled approach using Latent Transformer Models (LTMs). Not only does the proposed LDM-based approach achieve statistically significant better performance, it also shows less sensitivity to the underlying latent representation, more favourable memory scaling, and produces better spatial anomaly maps. Code is available at https://github.com/marksgraham/ddpm-ood},
   author = {Mark S. Graham and Walter Hugo Lopez Pinaya and Paul Wright and Petru-Daniel Tudosiu and Yee H. Mah and James T. Teo and H. Rolf Jäger and David Werring and Parashkev Nachev and Sebastien Ourselin and M. Jorge Cardoso},
   month = {7},
   title = {Unsupervised 3D out-of-distribution detection with latent diffusion models},
   year = {2023},
}
@article{pinaya2023generative,
  title={Generative ai for medical imaging: extending the monai framework},
  author={Pinaya, Walter HL and Graham, Mark S and Kerfoot, Eric and Tudosiu, Petru-Daniel and Dafflon, Jessica and Fernandez, Virginia and Sanchez, Pedro and Wolleb, Julia and da Costa, Pedro F and Patel, Ashay and others},
  journal={arXiv preprint arXiv:2307.15208},
  year={2023}
}
@book_section{Patel2023,
   author = {Ashay Patel and Petru-Daniel Tudosiu and Walter Hugo Lopez Pinaya and Olusola Adeleke and Gary Cook and Vicky Goh and Sebastien Ourselin and M. Jorge Cardoso},
   doi = {10.1007/978-3-031-43907-0_29},
   pages = {300-309},
   title = {Geometry-Invariant Abnormality Detection},
   year = {2023},
}
@article{Kim2011,
   abstract = {We propose a method for nonparametric density estimation that exhibits robustness to contamination of the training sample. This method achieves robustness by combining a traditional kernel density estimator (KDE) with ideas from classical $M$-estimation. We interpret the KDE based on a radial, positive semi-definite kernel as a sample mean in the associated reproducing kernel Hilbert space. Since the sample mean is sensitive to outliers, we estimate it robustly via $M$-estimation, yielding a robust kernel density estimator (RKDE). An RKDE can be computed efficiently via a kernelized iteratively re-weighted least squares (IRWLS) algorithm. Necessary and sufficient conditions are given for kernelized IRWLS to converge to the global minimizer of the $M$-estimator objective function. The robustness of the RKDE is demonstrated with a representer theorem, the influence function, and experimental results for density estimation and anomaly detection.},
   author = {JooSeuk Kim and Clayton D. Scott},
   month = {7},
   title = {Robust Kernel Density Estimation},
   year = {2011},
}
@article{Fischler1981,
   abstract = {<p>A new paradigm, Random Sample Consensus (RANSAC), for fitting a model to experimental data is introduced. RANSAC is capable of interpreting/smoothing data containing a significant percentage of gross errors, and is thus ideally suited for applications in automated image analysis where interpretation is based on the data provided by error-prone feature detectors. A major portion of this paper describes the application of RANSAC to the Location Determination Problem (LDP): Given an image depicting a set of landmarks with known locations, determine that point in space from which the image was obtained. In response to a RANSAC requirement, new results are derived on the minimum number of landmarks needed to obtain a solution, and algorithms are presented for computing these minimum-landmark solutions in closed form. These results provide the basis for an automatic system that can solve the LDP under difficult viewing</p>},
   author = {Martin A. Fischler and Robert C. Bolles},
   doi = {10.1145/358669.358692},
   issn = {0001-0782},
   issue = {6},
   journal = {Communications of the ACM},
   month = {6},
   pages = {381-395},
   title = {Random sample consensus},
   volume = {24},
   year = {1981},
}
@article{Rousseeuw1984,
   author = {Peter J. Rousseeuw},
   doi = {10.2307/2288718},
   issn = {01621459},
   issue = {388},
   journal = {Journal of the American Statistical Association},
   month = {12},
   pages = {871},
   title = {Least Median of Squares Regression},
   volume = {79},
   year = {1984},
}
@article{Hatt2017,
   author = {Mathieu Hatt and John A. Lee and Charles R. Schmidtlein and Issam El Naqa and Curtis Caldwell and Elisabetta De Bernardi and Wei Lu and Shiva Das and Xavier Geets and Vincent Gregoire and Robert Jeraj and Michael P. MacManus and Osama R. Mawlawi and Ursula Nestle and Andrei B. Pugachev and Heiko Schöder and Tony Shepherd and Emiliano Spezi and Dimitris Visvikis and Habib Zaidi and Assen S. Kirov},
   doi = {10.1002/mp.12124},
   issn = {00942405},
   issue = {6},
   journal = {Medical Physics},
   month = {6},
   pages = {e1-e42},
   title = {Classification and evaluation strategies of auto-segmentation approaches for PET: Report of AAPM task group No. 211},
   volume = {44},
   year = {2017},
}
@article{Berthon2017,
   author = {Beatrice Berthon and Emiliano Spezi and Paulina Galavis and Tony Shepherd and Aditya Apte and Mathieu Hatt and Hadi Fayad and Elisabetta De Bernardi and Chiara D. Soffientini and C. Ross Schmidtlein and Issam El Naqa and Robert Jeraj and Wei Lu and Shiva Das and Habib Zaidi and Osama R. Mawlawi and Dimitris Visvikis and John A. Lee and Assen S. Kirov},
   doi = {10.1002/mp.12312},
   issn = {0094-2405},
   issue = {8},
   journal = {Medical Physics},
   month = {8},
   pages = {4098-4111},
   title = {Toward a standard for the evaluation of <scp>PET</scp> ‐Auto‐Segmentation methods following the recommendations of AAPM task group No. 211: Requirements and implementation},
   volume = {44},
   year = {2017},
}
@article{Kingma2014,
   abstract = {We introduce Adam, an algorithm for first-order gradient-based optimization of stochastic objective functions, based on adaptive estimates of lower-order moments. The method is straightforward to implement, is computationally efficient, has little memory requirements, is invariant to diagonal rescaling of the gradients, and is well suited for problems that are large in terms of data and/or parameters. The method is also appropriate for non-stationary objectives and problems with very noisy and/or sparse gradients. The hyper-parameters have intuitive interpretations and typically require little tuning. Some connections to related algorithms, on which Adam was inspired, are discussed. We also analyze the theoretical convergence properties of the algorithm and provide a regret bound on the convergence rate that is comparable to the best known results under the online convex optimization framework. Empirical results demonstrate that Adam works well in practice and compares favorably to other stochastic optimization methods. Finally, we discuss AdaMax, a variant of Adam based on the infinity norm.},
   author = {Diederik P. Kingma and Jimmy Ba},
   month = {12},
   title = {Adam: A Method for Stochastic Optimization},
   year = {2014},
}
@article{An2022,
   abstract = {<p>With the prevalence of degenerative diseases due to the increase in the aging population, we have encountered many spine-related disorders. Since the spine is a crucial part of the body, fast and accurate diagnosis is critically important. Generally, clinicians use X-ray images to diagnose the spine, but X-ray images are commonly occluded by the shadows of some bones, making it hard to identify the whole spine. Therefore, recently, various deep-learning-based spinal X-ray image analysis approaches have been proposed to help diagnose the spine. However, these approaches did not consider the characteristics of frequent occlusion in the X-ray image and the properties of the vertebra shape. Therefore, based on the X-ray image properties and vertebra shape, we present a novel landmark detection network specialized in lumbar X-ray images. The proposed network consists of two stages: The first step detects the centers of the lumbar vertebrae and the upper end plate of the first sacral vertebra (S1), and the second step detects the four corner points of each lumbar vertebra and two corner points of S1 from the image obtained in the first step. We used random spine cutout augmentation in the first step to robustify the network against the commonly obscured X-ray images. Furthermore, in the second step, we used CoordConv to make the network recognize the location distribution of landmarks and part affinity fields to understand the morphological features of the vertebrae, resulting in more accurate landmark detection. The proposed network was evaluated using 304 X-ray images, and it achieved 98.02% accuracy in center detection and 8.34% relative distance error in corner detection. This indicates that our network can detect spinal landmarks reliably enough to support radiologists in analyzing the lumbar X-ray images.</p>},
   author = {Chang-Hyeon An and Jeong-Sik Lee and Jun-Su Jang and Hyun-Chul Choi},
   doi = {10.3390/s22228628},
   issn = {1424-8220},
   issue = {22},
   journal = {Sensors},
   month = {11},
   pages = {8628},
   title = {Part Affinity Fields and CoordConv for Detecting Landmarks of Lumbar Vertebrae and Sacrum in X-ray Images},
   volume = {22},
   year = {2022},
}
@article{Jurdi2021,
   author = {R. El Jurdi and C. Petitjean and P. Honeine and F. Abdallah},
   doi = {10.1016/j.irbm.2021.03.002},
   issn = {19590318},
   issue = {6},
   journal = {IRBM},
   month = {12},
   pages = {415-423},
   title = {CoordConv-Unet: Investigating CoordConv for Organ Segmentation},
   volume = {42},
   year = {2021},
}
@article{Wu2021,
   abstract = {Relative position encoding (RPE) is important for transformer to capture sequence ordering of input tokens. General efficacy has been proven in natural language processing. However, in computer vision, its efficacy is not well studied and even remains controversial, e.g., whether relative position encoding can work equally well as absolute position? In order to clarify this, we first review existing relative position encoding methods and analyze their pros and cons when applied in vision transformers. We then propose new relative position encoding methods dedicated to 2D images, called image RPE (iRPE). Our methods consider directional relative distance modeling as well as the interactions between queries and relative position embeddings in self-attention mechanism. The proposed iRPE methods are simple and lightweight. They can be easily plugged into transformer blocks. Experiments demonstrate that solely due to the proposed encoding methods, DeiT and DETR obtain up to 1.5% (top-1 Acc) and 1.3% (mAP) stable improvements over their original versions on ImageNet and COCO respectively, without tuning any extra hyperparameters such as learning rate and weight decay. Our ablation and analysis also yield interesting findings, some of which run counter to previous understanding. Code and models are open-sourced at https://github.com/microsoft/Cream/tree/main/iRPE.},
   author = {Kan Wu and Houwen Peng and Minghao Chen and Jianlong Fu and Hongyang Chao},
   month = {7},
   title = {Rethinking and Improving Relative Position Encoding for Vision Transformer},
   year = {2021},
}
@article{Dosovitskiy2020,
   abstract = {While the Transformer architecture has become the de-facto standard for natural language processing tasks, its applications to computer vision remain limited. In vision, attention is either applied in conjunction with convolutional networks, or used to replace certain components of convolutional networks while keeping their overall structure in place. We show that this reliance on CNNs is not necessary and a pure transformer applied directly to sequences of image patches can perform very well on image classification tasks. When pre-trained on large amounts of data and transferred to multiple mid-sized or small image recognition benchmarks (ImageNet, CIFAR-100, VTAB, etc.), Vision Transformer (ViT) attains excellent results compared to state-of-the-art convolutional networks while requiring substantially fewer computational resources to train.},
   author = {Alexey Dosovitskiy and Lucas Beyer and Alexander Kolesnikov and Dirk Weissenborn and Xiaohua Zhai and Thomas Unterthiner and Mostafa Dehghani and Matthias Minderer and Georg Heigold and Sylvain Gelly and Jakob Uszkoreit and Neil Houlsby},
   month = {10},
   title = {An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale},
   year = {2020},
}
@article{Liu2018,
   abstract = {Few ideas have enjoyed as large an impact on deep learning as convolution. For any problem involving pixels or spatial representations, common intuition holds that convolutional neural networks may be appropriate. In this paper we show a striking counterexample to this intuition via the seemingly trivial coordinate transform problem, which simply requires learning a mapping between coordinates in (x,y) Cartesian space and one-hot pixel space. Although convolutional networks would seem appropriate for this task, we show that they fail spectacularly. We demonstrate and carefully analyze the failure first on a toy problem, at which point a simple fix becomes obvious. We call this solution CoordConv, which works by giving convolution access to its own input coordinates through the use of extra coordinate channels. Without sacrificing the computational and parametric efficiency of ordinary convolution, CoordConv allows networks to learn either complete translation invariance or varying degrees of translation dependence, as required by the end task. CoordConv solves the coordinate transform problem with perfect generalization and 150 times faster with 10--100 times fewer parameters than convolution. This stark contrast raises the question: to what extent has this inability of convolution persisted insidiously inside other tasks, subtly hampering performance from within? A complete answer to this question will require further investigation, but we show preliminary evidence that swapping convolution for CoordConv can improve models on a diverse set of tasks. Using CoordConv in a GAN produced less mode collapse as the transform between high-level spatial latents and pixels becomes easier to learn. A Faster R-CNN detection model trained on MNIST showed 24% better IOU when using CoordConv, and in the RL domain agents playing Atari games benefit significantly from the use of CoordConv layers.},
   author = {Rosanne Liu and Joel Lehman and Piero Molino and Felipe Petroski Such and Eric Frank and Alex Sergeev and Jason Yosinski},
   month = {7},
   title = {An Intriguing Failing of Convolutional Neural Networks and the CoordConv Solution},
   year = {2018},
}
@article{Dinsdale2021,
   author = {Nicola K. Dinsdale and Emma Bluemke and Vaanathi Sundaresan and Mark Jenkinson and Stephen M. Smith and Ana I.L. Namburete},
   doi = {10.1016/j.neuron.2022.09.012},
   issn = {08966273},
   issue = {23},
   journal = {Neuron},
   month = {12},
   pages = {3866-3881},
   title = {Challenges for machine learning in clinical translation of big data imaging studies},
   volume = {110},
   year = {2022},
}
@article{Decuyper2021,
   abstract = {<p>The use of deep learning in medical imaging has increased rapidly over the past few years, finding applications throughout the entire radiology pipeline, from improved scanner performance to automatic disease detection and diagnosis. These advancements have resulted in a wide variety of deep learning approaches being developed, solving unique challenges for various imaging modalities. This paper provides a review on these developments from a technical point of view, categorizing the different methodologies and summarizing their implementation. We provide an introduction to the design of neural networks and their training procedure, after which we take an extended look at their uses in medical imaging. We cover the different sections of the radiology pipeline, highlighting some influential works and discussing the merits and limitations of deep learning approaches compared to other traditional methods. As such, this review is intended to provide a broad yet concise overview for the interested reader, facilitating adoption and interdisciplinary research of deep learning in the field of medical imaging.</p>},
   author = {Milan Decuyper and Jens Maebe and Roel Van Holen and Stefaan Vandenberghe},
   doi = {10.1186/s40658-021-00426-y},
   issn = {2197-7364},
   issue = {1},
   journal = {EJNMMI Physics},
   month = {12},
   pages = {81},
   title = {Artificial intelligence with deep learning in nuclear medicine and radiology},
   volume = {8},
   year = {2021},
}
@article{Varoquaux2022,
   abstract = {<p>Research in computer analysis of medical images bears many promises to improve patients’ health. However, a number of systematic challenges are slowing down the progress of the field, from limitations of the data, such as biases, to research incentives, such as optimizing for publication. In this paper we review roadblocks to developing and assessing methods. Building our analysis on evidence from the literature and data challenges, we show that at every step, potential biases can creep in. On a positive note, we also discuss on-going efforts to counteract these problems. Finally we provide recommendations on how to further address these problems in the future.</p>},
   author = {Gaël Varoquaux and Veronika Cheplygina},
   doi = {10.1038/s41746-022-00592-y},
   issn = {2398-6352},
   issue = {1},
   journal = {npj Digital Medicine},
   month = {4},
   pages = {48},
   title = {Machine learning for medical imaging: methodological failures and recommendations for the future},
   volume = {5},
   year = {2022},
}
@inproceedings{BenDavid2006,
   author = {Shai Ben-David and John Blitzer and Koby Crammer and Fernando Pereira},
   editor = {B Schölkopf and J Platt and T Hoffman},
   journal = {Advances in Neural Information Processing Systems},
   publisher = {MIT Press},
   title = {Analysis of Representations for Domain Adaptation},
   volume = {19},
   url = {https://proceedings.neurips.cc/paper/2006/file/b1b0432ceafb0ce714426e9114852ac7-Paper.pdf},
   year = {2006},
}
@article{Valiant1984,
   author = {Leslie G Valiant},
   journal = {Communications of the ACM},
   title = {A theory of the learnable},
   year = {1984},
}
@article{Feng2021,
   abstract = {Weakly supervised video anomaly detection (WS-VAD) is to distinguish anomalies from normal events based on discriminative representations. Most existing works are limited in insufficient video representations. In this work, we develop a multiple instance self-training framework (MIST)to efficiently refine task-specific discriminative representations with only video-level annotations. In particular, MIST is composed of 1) a multiple instance pseudo label generator, which adapts a sparse continuous sampling strategy to produce more reliable clip-level pseudo labels, and 2) a self-guided attention boosted feature encoder that aims to automatically focus on anomalous regions in frames while extracting task-specific representations. Moreover, we adopt a self-training scheme to optimize both components and finally obtain a task-specific feature encoder. Extensive experiments on two public datasets demonstrate the efficacy of our method, and our method performs comparably to or even better than existing supervised and weakly supervised methods, specifically obtaining a frame-level AUC 94.83% on ShanghaiTech.},
   author = {Jia-Chang Feng and Fa-Ting Hong and Wei-Shi Zheng},
   month = {4},
   title = {MIST: Multiple Instance Self-Training Framework for Video Anomaly Detection},
   year = {2021},
}
@article{AutoPET2022,
   abstract = {<p>We describe a publicly available dataset of annotated Positron Emission Tomography/Computed Tomography (PET/CT) studies. 1014 whole body Fluorodeoxyglucose (FDG)-PET/CT datasets (501 studies of patients with malignant lymphoma, melanoma and non small cell lung cancer (NSCLC) and 513 studies without PET-positive malignant lesions (negative controls)) acquired between 2014 and 2018 were included. All examinations were acquired on a single, state-of-the-art PET/CT scanner. The imaging protocol consisted of a whole-body FDG-PET acquisition and a corresponding diagnostic CT scan. All FDG-avid lesions identified as malignant based on the clinical PET/CT report were manually segmented on PET images in a slice-per-slice (3D) manner. We provide the anonymized original DICOM files of all studies as well as the corresponding DICOM segmentation masks. In addition, we provide scripts for image processing and conversion to different file formats (NIfTI, mha, hdf5). Primary diagnosis, age and sex are provided as non-imaging information. We demonstrate how this dataset can be used for deep learning-based automated analysis of PET/CT data and provide the trained deep learning model.</p>},
   author = {Sergios Gatidis and Tobias Hepp and Marcel Früh and Christian La Fougère and Konstantin Nikolaou and Christina Pfannenberg and Bernhard Schölkopf and Thomas Küstner and Clemens Cyran and Daniel Rubin},
   doi = {10.1038/s41597-022-01718-3},
   issn = {2052-4463},
   issue = {1},
   journal = {Scientific Data},
   month = {10},
   pages = {601},
   title = {A whole-body FDG-PET/CT Dataset with manually annotated Tumor Lesions},
   volume = {9},
   year = {2022},
}
@article{TCIA2013,
   author = {Kenneth Clark and Bruce Vendt and Kirk Smith and John Freymann and Justin Kirby and Paul Koppel and Stephen Moore and Stanley Phillips and David Maffitt and Michael Pringle and Lawrence Tarbox and Fred Prior},
   doi = {10.1007/s10278-013-9622-7},
   issn = {0897-1889},
   issue = {6},
   journal = {Journal of Digital Imaging},
   month = {12},
   pages = {1045-1057},
   title = {The Cancer Imaging Archive (TCIA): Maintaining and Operating a Public Information Repository},
   volume = {26},
   year = {2013},
}
@article{Tudosiu2022,
   abstract = {Human anatomy, morphology, and associated diseases can be studied using medical imaging data. However, access to medical imaging data is restricted by governance and privacy concerns, data ownership, and the cost of acquisition, thus limiting our ability to understand the human body. A possible solution to this issue is the creation of a model able to learn and then generate synthetic images of the human body conditioned on specific characteristics of relevance (e.g., age, sex, and disease status). Deep generative models, in the form of neural networks, have been recently used to create synthetic 2D images of natural scenes. Still, the ability to produce high-resolution 3D volumetric imaging data with correct anatomical morphology has been hampered by data scarcity and algorithmic and computational limitations. This work proposes a generative model that can be scaled to produce anatomically correct, high-resolution, and realistic images of the human brain, with the necessary quality to allow further downstream analyses. The ability to generate a potentially unlimited amount of data not only enables large-scale studies of human anatomy and pathology without jeopardizing patient privacy, but also significantly advances research in the field of anomaly detection, modality synthesis, learning under limited data, and fair and ethical AI. Code and trained models are available at: https://github.com/AmigoLab/SynthAnatomy.},
   author = {Petru-Daniel Tudosiu and Walter Hugo Lopez Pinaya and Mark S. Graham and Pedro Borges and Virginia Fernandez and Dai Yang and Jeremy Appleyard and Guido Novati and Disha Mehra and Mike Vella and Parashkev Nachev and Sebastien Ourselin and Jorge Cardoso},
   month = {9},
   title = {Morphology-preserving Autoregressive 3D Generative Modelling of the Brain},
   year = {2022},
}
@article{Graham2022,
   abstract = {In a clinical setting it is essential that deployed image processing systems are robust to the full range of inputs they might encounter and, in particular, do not make confidently wrong predictions. The most popular approach to safe processing is to train networks that can provide a measure of their uncertainty, but these tend to fail for inputs that are far outside the training data distribution. Recently, generative modelling approaches have been proposed as an alternative; these can quantify the likelihood of a data sample explicitly, filtering out any out-of-distribution (OOD) samples before further processing is performed. In this work, we focus on image segmentation and evaluate several approaches to network uncertainty in the far-OOD and near-OOD cases for the task of segmenting haemorrhages in head CTs. We find all of these approaches are unsuitable for safe segmentation as they provide confidently wrong predictions when operating OOD. We propose performing full 3D OOD detection using a VQ-GAN to provide a compressed latent representation of the image and a transformer to estimate the data likelihood. Our approach successfully identifies images in both the far- and near-OOD cases. We find a strong relationship between image likelihood and the quality of a model's segmentation, making this approach viable for filtering images unsuitable for segmentation. To our knowledge, this is the first time transformers have been applied to perform OOD detection on 3D image data.},
   author = {Mark S Graham and Petru-Daniel Tudosiu and Paul Wright and Walter Hugo Lopez Pinaya and U Jean-Marie and Yee Mah and James Teo and Rolf H Jäger and David Werring and Parashkev Nachev and Sebastien Ourselin and M Jorge Cardoso},
   month = {5},
   title = {Transformer-based out-of-distribution detection for clinically safe segmentation},
   year = {2022},
}
@article{Chang2023,
   abstract = {We present Muse, a text-to-image Transformer model that achieves state-of-the-art image generation performance while being significantly more efficient than diffusion or autoregressive models. Muse is trained on a masked modeling task in discrete token space: given the text embedding extracted from a pre-trained large language model (LLM), Muse is trained to predict randomly masked image tokens. Compared to pixel-space diffusion models, such as Imagen and DALL-E 2, Muse is significantly more efficient due to the use of discrete tokens and requiring fewer sampling iterations; compared to autoregressive models, such as Parti, Muse is more efficient due to the use of parallel decoding. The use of a pre-trained LLM enables fine-grained language understanding, translating to high-fidelity image generation and the understanding of visual concepts such as objects, their spatial relationships, pose, cardinality etc. Our 900M parameter model achieves a new SOTA on CC3M, with an FID score of 6.06. The Muse 3B parameter model achieves an FID of 7.88 on zero-shot COCO evaluation, along with a CLIP score of 0.32. Muse also directly enables a number of image editing applications without the need to fine-tune or invert the model: inpainting, outpainting, and mask-free editing. More results are available at https://muse-model.github.io},
   author = {Huiwen Chang and Han Zhang and Jarred Barber and AJ Maschinot and Jose Lezama and Lu Jiang and Ming-Hsuan Yang and Kevin Murphy and William T. Freeman and Michael Rubinstein and Yuanzhen Li and Dilip Krishnan},
   month = {1},
   title = {Muse: Text-To-Image Generation via Masked Generative Transformers},
   year = {2023},
}
@article{Ding2022,
   abstract = {Despite most existing anomaly detection studies assume the availability of normal training samples only, a few labeled anomaly examples are often available in many real-world applications, such as defect samples identified during random quality inspection, lesion images confirmed by radiologists in daily medical screening, etc. These anomaly examples provide valuable knowledge about the application-specific abnormality, enabling significantly improved detection of similar anomalies in some recent models. However, those anomalies seen during training often do not illustrate every possible class of anomaly, rendering these models ineffective in generalizing to unseen anomaly classes. This paper tackles open-set supervised anomaly detection, in which we learn detection models using the anomaly examples with the objective to detect both seen anomalies (`gray swans') and unseen anomalies (`black swans'). We propose a novel approach that learns disentangled representations of abnormalities illustrated by seen anomalies, pseudo anomalies, and latent residual anomalies (i.e., samples that have unusual residuals compared to the normal data in a latent space), with the last two abnormalities designed to detect unseen anomalies. Extensive experiments on nine real-world anomaly detection datasets show superior performance of our model in detecting seen and unseen anomalies under diverse settings. Code and data are available at: https://github.com/choubo/DRA.},
   author = {Choubo Ding and Guansong Pang and Chunhua Shen},
   month = {3},
   title = {Catching Both Gray and Black Swans: Open-set Supervised Anomaly Detection},
   year = {2022},
}
@article{Mei2022,
   author = {Xueyan Mei and Zelong Liu and Philip M. Robson and Brett Marinelli and Mingqian Huang and Amish Doshi and Adam Jacobi and Chendi Cao and Katherine E. Link and Thomas Yang and Ying Wang and Hayit Greenspan and Timothy Deyer and Zahi A. Fayad and Yang Yang},
   doi = {10.1148/ryai.210315},
   issn = {2638-6100},
   issue = {5},
   journal = {Radiology: Artificial Intelligence},
   month = {9},
   title = {RadImageNet: An Open Radiologic Deep Learning Research Dataset for Effective Transfer Learning},
   volume = {4},
   year = {2022},
}
@book_section{Zhang2022,
   author = {Ke Zhang and Xiahai Zhuang},
   doi = {10.1007/978-3-031-16452-1_16},
   pages = {162-172},
   title = {ShapePU: A New PU Learning Framework Regularized by Global Consistency for Scribble Supervised Cardiac Segmentation},
   year = {2022},
}
@book_section{Zuluaga2011,
   author = {Maria A. Zuluaga and Don Hush and Edgar J. F. Delgado Leyton and Marcela Hernández Hoyos and Maciej Orkisz},
   doi = {10.1007/978-3-642-23626-6_2},
   pages = {9-16},
   title = {Learning from Only Positive and Unlabeled Data to Detect Lesions in Vascular CT Images},
   year = {2011},
}
@inproceedings{Mao2017,
   author = {Xudong Mao and Qing Li and Haoran Xie and Raymond Y.K. Lau and Zhen Wang and Stephen Paul Smolley},
   doi = {10.1109/ICCV.2017.304},
   isbn = {978-1-5386-1032-9},
   journal = {2017 IEEE International Conference on Computer Vision (ICCV)},
   month = {10},
   pages = {2813-2821},
   publisher = {IEEE},
   title = {Least Squares Generative Adversarial Networks},
   year = {2017},
}
@inproceedings{Isola2017,
   author = {Phillip Isola and Jun-Yan Zhu and Tinghui Zhou and Alexei A. Efros},
   doi = {10.1109/CVPR.2017.632},
   isbn = {978-1-5386-0457-1},
   journal = {2017 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
   month = {7},
   pages = {5967-5976},
   publisher = {IEEE},
   title = {Image-to-Image Translation with Conditional Adversarial Networks},
   year = {2017},
}
@article{Devlin2018,
   abstract = {We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models, BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications. BERT is conceptually simple and empirically powerful. It obtains new state-of-the-art results on eleven natural language processing tasks, including pushing the GLUE score to 80.5% (7.7% point absolute improvement), MultiNLI accuracy to 86.7% (4.6% absolute improvement), SQuAD v1.1 question answering Test F1 to 93.2 (1.5 point absolute improvement) and SQuAD v2.0 Test F1 to 83.1 (5.1 point absolute improvement).},
   author = {Jacob Devlin and Ming-Wei Chang and Kenton Lee and Kristina Toutanova},
   month = {10},
   title = {BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding},
   year = {2018},
}
@article{Ruppert1980,
   author = {David Ruppert and Raymond J. Carroll},
   doi = {10.2307/2287169},
   issn = {01621459},
   issue = {372},
   journal = {Journal of the American Statistical Association},
   month = {12},
   pages = {828},
   title = {Trimmed Least Squares Estimation in the Linear Model},
   volume = {75},
   year = {1980},
}
@article{Cho2020,
   abstract = {In contemporary society, surveillance anomaly detection, i.e., spotting anomalous events such as crimes or accidents in surveillance videos, is a critical task. As anomalies occur rarely, most training data consists of unlabeled videos without anomalous events, which makes the task challenging. Most existing methods use an autoencoder (AE) to learn to reconstruct normal videos; they then detect anomalies based on their failure to reconstruct the appearance of abnormal scenes. However, because anomalies are distinguished by appearance as well as motion, many previous approaches have explicitly separated appearance and motion information-for example, using a pre-trained optical flow model. This explicit separation restricts reciprocal representation capabilities between two types of information. In contrast, we propose an implicit two-path AE (ITAE), a structure in which two encoders implicitly model appearance and motion features, along with a single decoder that combines them to learn normal video patterns. For the complex distribution of normal scenes, we suggest normal density estimation of ITAE features through normalizing flow (NF)-based generative models to learn the tractable likelihoods and identify anomalies using out of distribution detection. NF models intensify ITAE performance by learning normality through implicitly learned features. Finally, we demonstrate the effectiveness of ITAE and its feature distribution modeling on six benchmarks, including databases that contain various anomalies in real-world scenarios.},
   author = {MyeongAh Cho and Taeoh Kim and Woo Jin Kim and Suhwan Cho and Sangyoun Lee},
   month = {10},
   title = {Unsupervised Video Anomaly Detection via Normalizing Flows with Implicit Latent Features},
   year = {2020},
}
@book_section{Patel2022,
   author = {Ashay Patel and Petru-Daniel Tudosiu and Walter Hugo Lopez Pinaya and Gary Cook and Vicky Goh and Sebastien Ourselin and M. Jorge Cardoso},
   doi = {10.1007/978-3-031-18576-2_2},
   pages = {14-23},
   title = {Cross Attention Transformers for Multi-modal Unsupervised Whole-Body PET Anomaly Detection},
   year = {2022},
}
@article{Yu2021,
   abstract = {Unsupervised anomaly detection and localization is crucial to the practical application when collecting and labeling sufficient anomaly data is infeasible. Most existing representation-based approaches extract normal image features with a deep convolutional neural network and characterize the corresponding distribution through non-parametric distribution estimation methods. The anomaly score is calculated by measuring the distance between the feature of the test image and the estimated distribution. However, current methods can not effectively map image features to a tractable base distribution and ignore the relationship between local and global features which are important to identify anomalies. To this end, we propose FastFlow implemented with 2D normalizing flows and use it as the probability distribution estimator. Our FastFlow can be used as a plug-in module with arbitrary deep feature extractors such as ResNet and vision transformer for unsupervised anomaly detection and localization. In training phase, FastFlow learns to transform the input visual feature into a tractable distribution and obtains the likelihood to recognize anomalies in inference phase. Extensive experimental results on the MVTec AD dataset show that FastFlow surpasses previous state-of-the-art methods in terms of accuracy and inference efficiency with various backbone networks. Our approach achieves 99.4% AUC in anomaly detection with high inference efficiency.},
   author = {Jiawei Yu and Ye Zheng and Xiang Wang and Wei Li and Yushuang Wu and Rui Zhao and Liwei Wu},
   month = {11},
   title = {FastFlow: Unsupervised Anomaly Detection and Localization via 2D Normalizing Flows},
   year = {2021},
}
@article{Roth2021,
   abstract = {Being able to spot defective parts is a critical component in large-scale industrial manufacturing. A particular challenge that we address in this work is the cold-start problem: fit a model using nominal (non-defective) example images only. While handcrafted solutions per class are possible, the goal is to build systems that work well simultaneously on many different tasks automatically. The best performing approaches combine embeddings from ImageNet models with an outlier detection model. In this paper, we extend on this line of work and propose \textbf\{PatchCore\}, which uses a maximally representative memory bank of nominal patch-features. PatchCore offers competitive inference times while achieving state-of-the-art performance for both detection and localization. On the challenging, widely used MVTec AD benchmark PatchCore achieves an image-level anomaly detection AUROC score of up to $99.6\%$, more than halving the error compared to the next best competitor. We further report competitive results on two additional datasets and also find competitive results in the few samples regime.\freefootnote\{$^*$ Work done during a research internship at Amazon AWS.\} Code: github.com/amazon-research/patchcore-inspection.},
   author = {Karsten Roth and Latha Pemula and Joaquin Zepeda and Bernhard Schölkopf and Thomas Brox and Peter Gehler},
   month = {6},
   title = {Towards Total Recall in Industrial Anomaly Detection},
   year = {2021},
}
@article{Lee2022,
   abstract = {For a long time, anomaly localization has been widely used in industries. Previous studies focused on approximating the distribution of normal features without adaptation to a target dataset. However, since anomaly localization should precisely discriminate normal and abnormal features, the absence of adaptation may make the normality of abnormal features overestimated. Thus, we propose Coupled-hypersphere-based Feature Adaptation (CFA) which accomplishes sophisticated anomaly localization using features adapted to the target dataset. CFA consists of (1) a learnable patch descriptor that learns and embeds target-oriented features and (2) scalable memory bank independent of the size of the target dataset. And, CFA adopts transfer learning to increase the normal feature density so that abnormal features can be clearly distinguished by applying patch descriptor and memory bank to a pre-trained CNN. The proposed method outperforms the previous methods quantitatively and qualitatively. For example, it provides an AUROC score of 99.5% in anomaly detection and 98.5% in anomaly localization of MVTec AD benchmark. In addition, this paper points out the negative effects of biased features of pre-trained CNNs and emphasizes the importance of the adaptation to the target dataset. The code is publicly available at https://github.com/sungwool/CFA_for_anomaly_localization.},
   author = {Sungwook Lee and Seunghyun Lee and Byung Cheol Song},
   month = {6},
   title = {CFA: Coupled-hypersphere-based Feature Adaptation for Target-Oriented Anomaly Localization},
   year = {2022},
}
@inproceedings{Bergmann2019,
   author = {Paul Bergmann and Michael Fauser and David Sattlegger and Carsten Steger},
   doi = {10.1109/CVPR.2019.00982},
   isbn = {978-1-7281-3293-8},
   journal = {2019 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
   month = {6},
   pages = {9584-9592},
   publisher = {IEEE},
   title = {MVTec AD — A Comprehensive Real-World Dataset for Unsupervised Anomaly Detection},
   year = {2019},
}
@article{Schluter2021,
   abstract = {We introduce a simple and intuitive self-supervision task, Natural Synthetic Anomalies (NSA), for training an end-to-end model for anomaly detection and localization using only normal training data. NSA integrates Poisson image editing to seamlessly blend scaled patches of various sizes from separate images. This creates a wide range of synthetic anomalies which are more similar to natural sub-image irregularities than previous data-augmentation strategies for self-supervised anomaly detection. We evaluate the proposed method using natural and medical images. Our experiments with the MVTec AD dataset show that a model trained to localize NSA anomalies generalizes well to detecting real-world a priori unknown types of manufacturing defects. Our method achieves an overall detection AUROC of 97.2 outperforming all previous methods that learn without the use of additional datasets. Code available at https://github.com/hmsch/natural-synthetic-anomalies.},
   author = {Hannah M. Schlüter and Jeremy Tan and Benjamin Hou and Bernhard Kainz},
   month = {9},
   title = {Natural Synthetic Anomalies for Self-Supervised Anomaly Detection and Localization},
   year = {2021},
}
@article{Li2021,
   abstract = {We aim at constructing a high performance model for defect detection that detects unknown anomalous patterns of an image without anomalous data. To this end, we propose a two-stage framework for building anomaly detectors using normal training data only. We first learn self-supervised deep representations and then build a generative one-class classifier on learned representations. We learn representations by classifying normal data from the CutPaste, a simple data augmentation strategy that cuts an image patch and pastes at a random location of a large image. Our empirical study on MVTec anomaly detection dataset demonstrates the proposed algorithm is general to be able to detect various types of real-world defects. We bring the improvement upon previous arts by 3.1 AUCs when learning representations from scratch. By transfer learning on pretrained representations on ImageNet, we achieve a new state-of-theart 96.6 AUC. Lastly, we extend the framework to learn and extract representations from patches to allow localizing defective areas without annotations during training.},
   author = {Chun-Liang Li and Kihyuk Sohn and Jinsung Yoon and Tomas Pfister},
   month = {4},
   title = {CutPaste: Self-Supervised Learning for Anomaly Detection and Localization},
   year = {2021},
}
@article{Sung2021,
   abstract = {This article provides an update on the global cancer burden using the GLOBOCAN 2020 estimates of cancer incidence and mortality produced by the International Agency for Research on Cancer. Worldwide, an estimated 19.3 million new cancer cases (18.1 million excluding nonmelanoma skin cancer) and almost 10.0 million cancer deaths (9.9 million excluding nonmelanoma skin cancer) occurred in 2020. Female breast cancer has surpassed lung cancer as the most commonly diagnosed cancer, with an estimated 2.3 million new cases (11.7%), followed by lung (11.4%), colorectal (10.0 %), prostate (7.3%), and stomach (5.6%) cancers. Lung cancer remained the leading cause of cancer death, with an estimated 1.8 million deaths (18%), followed by colorectal (9.4%), liver (8.3%), stomach (7.7%), and female breast (6.9%) cancers. Overall incidence was from 2-fold to 3-fold higher in transitioned versus transitioning countries for both sexes, whereas mortality varied <2-fold for men and little for women. Death rates for female breast and cervical cancers, however, were considerably higher in transitioning versus transitioned countries (15.0 vs 12.8 per 100,000 and 12.4 vs 5.2 per 100,000, respectively). The global cancer burden is expected to be 28.4 million cases in 2040, a 47% rise from 2020, with a larger increase in transitioning (64% to 95%) versus transitioned (32% to 56%) countries due to demographic changes, although this may be further exacerbated by increasing risk factors associated with globalization and a growing economy. Efforts to build a sustainable infrastructure for the dissemination of cancer prevention measures and provision of cancer care in transitioning countries is critical for global cancer control.},
   author = {Hyuna Sung and Jacques Ferlay and Rebecca L Siegel and Mathieu Laversanne and Isabelle Soerjomataram and Ahmedin Jemal and Freddie Bray},
   doi = {10.3322/caac.21660},
   issn = {1542-4863},
   issue = {3},
   journal = {CA: a cancer journal for clinicians},
   pages = {209-249},
   pmid = {33538338},
   title = {Global Cancer Statistics 2020: GLOBOCAN Estimates of Incidence and Mortality Worldwide for 36 Cancers in 185 Countries.},
   volume = {71},
   year = {2021},
}
@article{Ahmad2015,
   abstract = {BACKGROUND Typically, lifetime risk is calculated by the period method using current risks at different ages. Here, we estimate the probability of being diagnosed with cancer for individuals born in a given year, by estimating future risks as the cohort ages. METHODS We estimated the lifetime risk of cancer in Britain separately for men and women born in each year from 1930 to 1960. We projected rates of all cancers (excluding non-melanoma skin cancer) and of all cancer deaths forwards using a flexible age-period-cohort model and backwards using age-specific extrapolation. The sensitivity of the estimated lifetime risk to the method of projection was explored. RESULTS The lifetime risk of cancer increased from 38.5% for men born in 1930 to 53.5% for men born in 1960. For women it increased from 36.7 to 47.5%. Results are robust to different models for projections of cancer rates. CONCLUSIONS The lifetime risk of cancer for people born since 1960 is >50%. Over half of people who are currently adults under the age of 65 years will be diagnosed with cancer at some point in their lifetime.},
   author = {A S Ahmad and N Ormiston-Smith and P D Sasieni},
   doi = {10.1038/bjc.2014.606},
   issn = {1532-1827},
   issue = {5},
   journal = {British journal of cancer},
   month = {3},
   pages = {943-7},
   pmid = {25647015},
   title = {Trends in the lifetime risk of developing cancer in Great Britain: comparison of risk for those born from 1930 to 1960.},
   volume = {112},
   year = {2015},
}
@article{Endo2006,
   author = {Keigo Endo and Noboru Oriuchi and Tetsuya Higuchi and Yasuhiko Iida and Hirofumi Hanaoka and Mitsuyuki Miyakubo and Tomohiro Ishikita and Keiko Koyama},
   doi = {10.1007/s10147-006-0595-0},
   issn = {1341-9625},
   issue = {4},
   journal = {International Journal of Clinical Oncology},
   month = {9},
   pages = {286-296},
   title = {PET and PET/CT using 18F-FDG in the diagnosis and management of cancer patients},
   volume = {11},
   year = {2006},
}
@article{Wu2021,
   abstract = {Relative position encoding (RPE) is important for transformer to capture sequence ordering of input tokens. General efficacy has been proven in natural language processing. However, in computer vision, its efficacy is not well studied and even remains controversial, e.g., whether relative position encoding can work equally well as absolute position? In order to clarify this, we first review existing relative position encoding methods and analyze their pros and cons when applied in vision transformers. We then propose new relative position encoding methods dedicated to 2D images, called image RPE (iRPE). Our methods consider directional relative distance modeling as well as the interactions between queries and relative position embeddings in self-attention mechanism. The proposed iRPE methods are simple and lightweight. They can be easily plugged into transformer blocks. Experiments demonstrate that solely due to the proposed encoding methods, DeiT and DETR obtain up to 1.5% (top-1 Acc) and 1.3% (mAP) stable improvements over their original versions on ImageNet and COCO respectively, without tuning any extra hyperparameters such as learning rate and weight decay. Our ablation and analysis also yield interesting findings, some of which run counter to previous understanding. Code and models are open-sourced at https://github.com/microsoft/Cream/tree/main/iRPE.},
   author = {Kan Wu and Houwen Peng and Minghao Chen and Jianlong Fu and Hongyang Chao},
   month = {7},
   title = {Rethinking and Improving Relative Position Encoding for Vision Transformer},
   year = {2021},
}
@article{Rousseeuw2018,
   author = {Peter J. Rousseeuw and Mia Hubert},
   doi = {10.1002/widm.1236},
   issn = {1942-4787},
   issue = {2},
   journal = {WIREs Data Mining and Knowledge Discovery},
   month = {3},
   title = {Anomaly detection by robust statistics},
   volume = {8},
   year = {2018},
}
@article{Pinaya2022,
   abstract = {Deep generative models have emerged as promising tools for detecting arbitrary anomalies in data, dispensing with the necessity for manual labelling. Recently, autoregressive transformers have achieved state-of-the-art performance for anomaly detection in medical imaging. Nonetheless, these models still have some intrinsic weaknesses, such as requiring images to be modelled as 1D sequences, the accumulation of errors during the sampling process, and the significant inference times associated with transformers. Denoising diffusion probabilistic models are a class of non-autoregressive generative models recently shown to produce excellent samples in computer vision (surpassing Generative Adversarial Networks), and to achieve log-likelihoods that are competitive with transformers while having fast inference times. Diffusion models can be applied to the latent representations learnt by autoencoders, making them easily scalable and great candidates for application to high dimensional data, such as medical images. Here, we propose a method based on diffusion models to detect and segment anomalies in brain imaging. By training the models on healthy data and then exploring its diffusion and reverse steps across its Markov chain, we can identify anomalous areas in the latent space and hence identify anomalies in the pixel space. Our diffusion models achieve competitive performance compared with autoregressive approaches across a series of experiments with 2D CT and MRI data involving synthetic and real pathological lesions with much reduced inference times, making their usage clinically viable.},
   author = {Walter H. L. Pinaya and Mark S. Graham and Robert Gray and Pedro F Da Costa and Petru-Daniel Tudosiu and Paul Wright and Yee H. Mah and Andrew D. MacKinnon and James T. Teo and Rolf Jager and David Werring and Geraint Rees and Parashkev Nachev and Sebastien Ourselin and M. Jorge Cardoso},
   month = {6},
   title = {Fast Unsupervised Brain Anomaly Detection and Segmentation with Diffusion Models},
   year = {2022},
}
@article{NicholA2021,
   abstract = {Diffusion models have recently been shown to generate high-quality synthetic images, especially when paired with a guidance technique to trade off diversity for fidelity. We explore diffusion models for the problem of text-conditional image synthesis and compare two different guidance strategies: CLIP guidance and classifier-free guidance. We find that the latter is preferred by human evaluators for both photorealism and caption similarity, and often produces photorealistic samples. Samples from a 3.5 billion parameter text-conditional diffusion model using classifier-free guidance are favored by human evaluators to those from DALL-E, even when the latter uses expensive CLIP reranking. Additionally, we find that our models can be fine-tuned to perform image inpainting, enabling powerful text-driven image editing. We train a smaller model on a filtered dataset and release the code and weights at https://github.com/openai/glide-text2im.},
   author = {Alex Nichol and Prafulla Dhariwal and Aditya Ramesh and Pranav Shyam and Pamela Mishkin and Bob McGrew and Ilya Sutskever and Mark Chen},
   month = {12},
   title = {GLIDE: Towards Photorealistic Image Generation and Editing with Text-Guided Diffusion Models},
   year = {2021},
}
@article{Bi2017,
   abstract = {Positron emission tomography (PET) image synthesis plays an important role, which can be used to boost the training data for computer aided diagnosis systems. However, existing image synthesis methods have problems in synthesizing the low resolution PET images. To address these limitations, we propose multi-channel generative adversarial networks (M-GAN) based PET image synthesis method. Different to the existing methods which rely on using low-level features, the proposed M-GAN is capable to represent the features in a high-level of semantic based on the adversarial learning concept. In addition, M-GAN enables to take the input from the annotation (label) to synthesize the high uptake regions e.g., tumors and from the computed tomography (CT) images to constrain the appearance consistency and output the synthetic PET images directly. Our results on 50 lung cancer PET-CT studies indicate that our method was much closer to the real PET images when compared with the existing methods.},
   author = {Lei Bi and Jinman Kim and Ashnil Kumar and Dagan Feng and Michael Fulham},
   month = {7},
   title = {Synthesis of Positron Emission Tomography (PET) Images via Multi-channel Generative Adversarial Networks (GANs)},
   year = {2017},
}
@generic{CPTAC2018,
   author = {CPTAC},
   title = {National Cancer Institute Clinical Proteomic Tumor Analysis Consortium},
   year = {2018},
}
@article{Clark2013,
   author = {Kenneth Clark and Bruce Vendt and Kirk Smith and John Freymann and Justin Kirby and Paul Koppel and Stephen Moore and Stanley Phillips and David Maffitt and Michael Pringle and Lawrence Tarbox and Fred Prior},
   doi = {10.1007/s10278-013-9622-7},
   issn = {0897-1889},
   issue = {6},
   journal = {Journal of Digital Imaging},
   month = {12},
   pages = {1045-1057},
   title = {The Cancer Imaging Archive (TCIA): Maintaining and Operating a Public Information Repository},
   volume = {26},
   year = {2013},
}
@article{Gevaert2012,
   author = {Olivier Gevaert and Jiajing Xu and Chuong D. Hoang and Ann N. Leung and Yue Xu and Andrew Quon and Daniel L. Rubin and Sandy Napel and Sylvia K. Plevritis},
   doi = {10.1148/radiol.12111607},
   issn = {0033-8419},
   issue = {2},
   journal = {Radiology},
   month = {8},
   pages = {387-396},
   title = {Non–Small Cell Lung Cancer: Identifying Prognostic Imaging Biomarkers by Leveraging Public Gene Expression Microarray Data—Methods and Preliminary Results},
   volume = {264},
   year = {2012},
}
@article{Bakr2018,
   author = {Shaimaa Bakr and Olivier Gevaert and Sebastian Echegaray and Kelsey Ayers and Mu Zhou and Majid Shafiq and Hong Zheng and Jalen Anthony Benson and Weiruo Zhang and Ann N. C. Leung and Michael Kadoch and Chuong D. Hoang and Joseph Shrager and Andrew Quon and Daniel L. Rubin and Sylvia K. Plevritis and Sandy Napel},
   doi = {10.1038/sdata.2018.202},
   issn = {2052-4463},
   issue = {1},
   journal = {Scientific Data},
   month = {12},
   pages = {180202},
   title = {A radiogenomic dataset of non-small cell lung cancer},
   volume = {5},
   year = {2018},
}
@generic{Bakr2017,
   author = {Shaimaa Bakr and Olivie Gevaert and Sebastian Echegaray and Kelsey Ayers and Mu Zhou and Majid Shafiq and Hong Zheng and Weiruo Zhang and Ann Leung and Michael Kadoch and Joseph Shrager and Andrew Quon and Daniel Rubin and Sylvia Plevritis and Sandy Napel},
   publisher = {The Cancer Imaging Archive},
   title = {Data for NSCLC Radiogenomics Collection},
   year = {2017},
}
@book{Silverman2018,
   author = {B.W. Silverman},
   doi = {10.1201/9781315140919},
   isbn = {9781315140919},
   month = {2},
   publisher = {Routledge},
   title = {Density Estimation for Statistics and Data Analysis},
   year = {2018},
}
@article{Kodali2017,
   abstract = {We propose studying GAN training dynamics as regret minimization, which is in contrast to the popular view that there is consistent minimization of a divergence between real and generated distributions. We analyze the convergence of GAN training from this new point of view to understand why mode collapse happens. We hypothesize the existence of undesirable local equilibria in this non-convex game to be responsible for mode collapse. We observe that these local equilibria often exhibit sharp gradients of the discriminator function around some real data points. We demonstrate that these degenerate local equilibria can be avoided with a gradient penalty scheme called DRAGAN. We show that DRAGAN enables faster training, achieves improved stability with fewer mode collapses, and leads to generator networks with better modeling performance across a variety of architectures and objective functions.},
   author = {Naveen Kodali and Jacob Abernethy and James Hays and Zsolt Kira},
   month = {5},
   title = {On Convergence and Stability of GANs},
   year = {2017},
}
@article{Sun2018,
   abstract = {The identification of lesion within medical image data is necessary for diagnosis, treatment and prognosis. Segmentation and classification approaches are mainly based on supervised learning with well-paired image-level or voxel-level labels. However, labeling the lesion in medical images is laborious requiring highly specialized knowledge. We propose a medical image synthesis model named abnormal-to-normal translation generative adversarial network (ANT-GAN) to generate a normal-looking medical image based on its abnormal-looking counterpart without the need for paired training data. Unlike typical GANs, whose aim is to generate realistic samples with variations, our more restrictive model aims at producing a normal-looking image corresponding to one containing lesions, and thus requires a special design. Being able to provide a "normal" counterpart to a medical image can provide useful side information for medical imaging tasks like lesion segmentation or classification validated by our experiments. In the other aspect, the ANT-GAN model is also capable of producing highly realistic lesion-containing image corresponding to the healthy one, which shows the potential in data augmentation verified in our experiments.},
   author = {Liyan Sun and Jiexiang Wang and Yue Huang and Xinghao Ding and Hayit Greenspan and John Paisley},
   month = {10},
   title = {An Adversarial Learning Approach to Medical Image Synthesis for Lesion Detection},
   year = {2018},
}
@inproceedings{Alex2017,
   author = {Varghese Alex and Mohammed Safwan K. P. and Sai Saketh Chennamsetty and Ganapathy Krishnamurthi},
   doi = {10.1117/12.2254487},
   editor = {Martin A. Styner and Elsa D. Angelini},
   month = {2},
   pages = {101330G},
   title = {Generative adversarial networks for brain lesion detection},
   year = {2017},
}
@article{Armanious2019,
   abstract = {Image-to-image translation is a new field in computer vision with multiple potential applications in the medical domain. However, for supervised image translation frameworks, co-registered datasets, paired in a pixel-wise sense, are required. This is often difficult to acquire in realistic medical scenarios. On the other hand, unsupervised translation frameworks often result in blurred translated images with unrealistic details. In this work, we propose a new unsupervised translation framework which is titled Cycle-MedGAN. The proposed framework utilizes new non-adversarial cycle losses which direct the framework to minimize the textural and perceptual discrepancies in the translated images. Qualitative and quantitative comparisons against other unsupervised translation approaches demonstrate the performance of the proposed framework for PET-CT translation and MR motion correction.},
   author = {Karim Armanious and Chenming Jiang and Sherif Abdulatif and Thomas Küstner and Sergios Gatidis and Bin Yang},
   doi = {10.23919/EUSIPCO.2019.8902799},
   month = {3},
   title = {Unsupervised Medical Image Translation Using Cycle-MedGAN},
   year = {2019},
}
@article{BenCohen2017,
   abstract = {In this work we present a novel system for PET estimation using CT scans. We explore the use of fully convolutional networks (FCN) and conditional generative adversarial networks (GAN) to export PET data from CT data. Our dataset includes 25 pairs of PET and CT scans where 17 were used for training and 8 for testing. The system was tested for detection of malignant tumors in the liver region. Initial results look promising showing high detection performance with a TPR of 92.3% and FPR of 0.25 per case. Future work entails expansion of the current system to the entire body using a much larger dataset. Such a system can be used for tumor detection and drug treatment evaluation in a CT-only environment instead of the expansive and radioactive PET-CT scan.},
   author = {Avi Ben-Cohen and Eyal Klang and Stephen P. Raskin and Michal Marianne Amitai and Hayit Greenspan},
   doi = {10.1007/978-3-319-68127-6_6},
   month = {7},
   title = {Virtual PET Images from CT Data Using Deep Convolutional Networks: Initial Results},
   year = {2017},
}
@article{Kang2018,
   abstract = {In coronary CT angiography, a series of CT images are taken at different levels of radiation dose during the examination. Although this reduces the total radiation dose, the image quality during the low-dose phases is significantly degraded. To address this problem, here we propose a novel semi-supervised learning technique that can remove the noises of the CT images obtained in the low-dose phases by learning from the CT images in the routine dose phases. Although a supervised learning approach is not possible due to the differences in the underlying heart structure in two phases, the images in the two phases are closely related so that we propose a cycle-consistent adversarial denoising network to learn the non-degenerate mapping between the low and high dose cardiac phases. Experimental results showed that the proposed method effectively reduces the noise in the low-dose CT image while the preserving detailed texture and edge information. Moreover, thanks to the cyclic consistency and identity loss, the proposed network does not create any artificial features that are not present in the input images. Visual grading and quality evaluation also confirm that the proposed method provides significant improvement in diagnostic quality.},
   author = {Eunhee Kang and Hyun Jung Koo and Dong Hyun Yang and Joon Bum Seo and Jong Chul Ye},
   doi = {10.1002/mp.13284},
   month = {6},
   title = {Cycle Consistent Adversarial Denoising Network for Multiphase Coronary CT Angiography},
   year = {2018},
}
@article{Armanious2018,
   abstract = {Image-to-image translation is considered a new frontier in the field of medical image analysis, with numerous potential applications. However, a large portion of recent approaches offers individualized solutions based on specialized task-specific architectures or require refinement through non-end-to-end training. In this paper, we propose a new framework, named MedGAN, for medical image-to-image translation which operates on the image level in an end-to-end manner. MedGAN builds upon recent advances in the field of generative adversarial networks (GANs) by merging the adversarial framework with a new combination of non-adversarial losses. We utilize a discriminator network as a trainable feature extractor which penalizes the discrepancy between the translated medical images and the desired modalities. Moreover, style-transfer losses are utilized to match the textures and fine-structures of the desired target images to the translated images. Additionally, we present a new generator architecture, titled CasNet, which enhances the sharpness of the translated medical outputs through progressive refinement via encoder-decoder pairs. Without any application-specific modifications, we apply MedGAN on three different tasks: PET-CT translation, correction of MR motion artefacts and PET image denoising. Perceptual analysis by radiologists and quantitative evaluations illustrate that the MedGAN outperforms other existing translation approaches.},
   author = {Karim Armanious and Chenming Jiang and Marc Fischer and Thomas Küstner and Konstantin Nikolaou and Sergios Gatidis and Bin Yang},
   doi = {10.1016/j.compmedimag.2019.101684},
   month = {6},
   title = {MedGAN: Medical Image Translation using GANs},
   year = {2018},
}
@article{Quan2018,
   author = {Tran Minh Quan and Thanh Nguyen-Duc and Won-Ki Jeong},
   doi = {10.1109/TMI.2018.2820120},
   issn = {0278-0062},
   issue = {6},
   journal = {IEEE Transactions on Medical Imaging},
   month = {6},
   pages = {1488-1497},
   title = {Compressed Sensing MRI Reconstruction Using a Generative Adversarial Network With a Cyclic Loss},
   volume = {37},
   year = {2018},
}
@article{Yi2019,
   abstract = {Generative adversarial networks have gained a lot of attention in the computer vision community due to their capability of data generation without explicitly modelling the probability density function. The adversarial loss brought by the discriminator provides a clever way of incorporating unlabeled samples into training and imposing higher order consistency. This has proven to be useful in many cases, such as domain adaptation, data augmentation, and image-to-image translation. These properties have attracted researchers in the medical imaging community, and we have seen rapid adoption in many traditional and novel applications, such as image reconstruction, segmentation, detection, classification, and cross-modality synthesis. Based on our observations, this trend will continue and we therefore conducted a review of recent advances in medical imaging using the adversarial training scheme with the hope of benefiting researchers interested in this technique.},
   author = {Xin Yi and Ekta Walia and Paul Babyn},
   doi = {10.1016/j.media.2019.101552},
   issn = {1361-8423},
   journal = {Medical image analysis},
   pages = {101552},
   pmid = {31521965},
   title = {Generative adversarial network in medical imaging: A review.},
   volume = {58},
   year = {2019},
}
@web_page{scikit_kde,
   author = {Scikit-Learn},
   title = {Density Estimation},
   url = {https://scikit-learn.org/stable/modules/density.html},
}
@article{Zhang2018,
   abstract = {While it is nearly effortless for humans to quickly assess the perceptual similarity between two images, the underlying processes are thought to be quite complex. Despite this, the most widely used perceptual metrics today, such as PSNR and SSIM, are simple, shallow functions, and fail to account for many nuances of human perception. Recently, the deep learning community has found that features of the VGG network trained on ImageNet classification has been remarkably useful as a training loss for image synthesis. But how perceptual are these so-called "perceptual losses"? What elements are critical for their success? To answer these questions, we introduce a new dataset of human perceptual similarity judgments. We systematically evaluate deep features across different architectures and tasks and compare them with classic metrics. We find that deep features outperform all previous metrics by large margins on our dataset. More surprisingly, this result is not restricted to ImageNet-trained VGG features, but holds across different deep architectures and levels of supervision (supervised, self-supervised, or even unsupervised). Our results suggest that perceptual similarity is an emergent property shared across deep visual representations.},
   author = {Richard Zhang and Phillip Isola and Alexei A. Efros and Eli Shechtman and Oliver Wang},
   month = {1},
   title = {The Unreasonable Effectiveness of Deep Features as a Perceptual Metric},
   year = {2018},
}
@article{Takaki2018,
   abstract = {This paper proposes a new loss using short-time Fourier transform (STFT) spectra for the aim of training a high-performance neural speech waveform model that predicts raw continuous speech waveform samples directly. Not only amplitude spectra but also phase spectra obtained from generated speech waveforms are used to calculate the proposed loss. We also mathematically show that training of the waveform model on the basis of the proposed loss can be interpreted as maximum likelihood training that assumes the amplitude and phase spectra of generated speech waveforms following Gaussian and von Mises distributions, respectively. Furthermore, this paper presents a simple network architecture as the speech waveform model, which is composed of uni-directional long short-term memories (LSTMs) and an auto-regressive structure. Experimental results showed that the proposed neural model synthesized high-quality speech waveforms.},
   author = {Shinji Takaki and Toru Nakashika and Xin Wang and Junichi Yamagishi},
   month = {10},
   title = {STFT spectral loss for training a neural speech waveform model},
   year = {2018},
}
@article{Dhariwal2020,
   abstract = {We introduce Jukebox, a model that generates music with singing in the raw audio domain. We tackle the long context of raw audio using a multi-scale VQ-VAE to compress it to discrete codes, and modeling those using autoregressive Transformers. We show that the combined model at scale can generate high-fidelity and diverse songs with coherence up to multiple minutes. We can condition on artist and genre to steer the musical and vocal style, and on unaligned lyrics to make the singing more controllable. We are releasing thousands of non cherry-picked samples at https://jukebox.openai.com, along with model weights and code at https://github.com/openai/jukebox},
   author = {Prafulla Dhariwal and Heewoo Jun and Christine Payne and Jong Wook Kim and Alec Radford and Ilya Sutskever},
   month = {4},
   title = {Jukebox: A Generative Model for Music},
   year = {2020},
}
@article{Liu2017,
   author = {Bin Liu and Sujuan Gao and Shuofeng Li},
   doi = {10.1159/000456006},
   issn = {0378-7346},
   issue = {3},
   journal = {Gynecologic and Obstetric Investigation},
   pages = {209-222},
   title = {A Comprehensive Comparison of CT, MRI, Positron Emission Tomography or Positron Emission Tomography/CT, and Diffusion Weighted Imaging-MRI for Detecting the Lymph Nodes Metastases in Patients with Cervical Cancer: A Meta-Analysis Based on 67 Studies},
   volume = {82},
   year = {2017},
}
@article{Yan2021,
   abstract = {We present VideoGPT: a conceptually simple architecture for scaling likelihood based generative modeling to natural videos. VideoGPT uses VQ-VAE that learns downsampled discrete latent representations of a raw video by employing 3D convolutions and axial self-attention. A simple GPT-like architecture is then used to autoregressively model the discrete latents using spatio-temporal position encodings. Despite the simplicity in formulation and ease of training, our architecture is able to generate samples competitive with state-of-the-art GAN models for video generation on the BAIR Robot dataset, and generate high fidelity natural videos from UCF-101 and Tumbler GIF Dataset (TGIF). We hope our proposed architecture serves as a reproducible reference for a minimalistic implementation of transformer based video generation models. Samples and code are available at https://wilson1yan.github.io/videogpt/index.html},
   author = {Wilson Yan and Yunzhi Zhang and Pieter Abbeel and Aravind Srinivas},
   month = {4},
   title = {VideoGPT: Video Generation using VQ-VAE and Transformers},
   year = {2021},
}
@article{Jun2020,
   author = {Heewoo Jun and R Child and Mark Chen and J Schulman},
   title = {Distribution Augmentation for Generative Modeling},
}
@article{Child2019,
   abstract = {Transformers are powerful sequence models, but require time and memory that grows quadratically with the sequence length. In this paper we introduce sparse factorizations of the attention matrix which reduce this to $O(n \sqrt\{n\})$. We also introduce a) a variation on architecture and initialization to train deeper networks, b) the recomputation of attention matrices to save memory, and c) fast attention kernels for training. We call networks with these changes Sparse Transformers, and show they can model sequences tens of thousands of timesteps long using hundreds of layers. We use the same architecture to model images, audio, and text from raw bytes, setting a new state of the art for density modeling of Enwik8, CIFAR-10, and ImageNet-64. We generate unconditional samples that demonstrate global coherence and great diversity, and show it is possible in principle to use self-attention to model sequences of length one million or more.},
   author = {Rewon Child and Scott Gray and Alec Radford and Ilya Sutskever},
   month = {4},
   title = {Generating Long Sequences with Sparse Transformers},
   year = {2019},
}
@article{Chen2020,
   author = {Mark Chen and Alec Radford and Jeff Wu and Jun Heewoo and Prafulla Dhariwal},
   title = {Generative Pretraining From Pixels},
   year = {2020},
}
@article{Vaswani2017,
   abstract = {The dominant sequence transduction models are based on complex recurrent or convolutional neural networks in an encoder-decoder configuration. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing both with large and limited training data.},
   author = {Ashish Vaswani and Noam Shazeer and Niki Parmar and Jakob Uszkoreit and Llion Jones and Aidan N. Gomez and Lukasz Kaiser and Illia Polosukhin},
   month = {6},
   title = {Attention Is All You Need},
   year = {2017},
}
@article{Parzen1962,
   author = {Emanuel Parzen},
   doi = {10.1214/aoms/1177704472},
   issn = {0003-4851},
   issue = {3},
   journal = {The Annals of Mathematical Statistics},
   month = {9},
   pages = {1065-1076},
   title = {On Estimation of a Probability Density Function and Mode},
   volume = {33},
   year = {1962},
}
@article{Burgos2021,
   abstract = {Purpose: In clinical practice, positron emission tomography (PET) images are mostly analyzed visually, but the sensitivity and specificity of this approach greatly depend on the observer's experience. Quantitative analysis of PET images would alleviate this problem by helping define an objective limit between normal and pathological findings. We present an anomaly detection framework for the individual analysis of PET images. Approach: We created subject-specific abnormality maps that summarize the pathology's topographical distribution in the brain by comparing the subject's PET image to a model of healthy PET appearance that is specific to the subject under investigation. This model was generated from demographically and morphologically matched PET scans from a control dataset. Results: We generated abnormality maps for healthy controls, patients at different stages of Alzheimer's disease and with different frontotemporal dementia syndromes. We showed that no anomalies were detected for the healthy controls and that the anomalies detected from the patients with dementia coincided with the regions where abnormal uptake was expected. We also validated the proposed framework using the abnormality maps as inputs of a classifier and obtained higher classification accuracies than when using the PET images themselves as inputs. Conclusions: The proposed method was able to automatically locate and characterize the areas characteristic of dementia from PET images. The abnormality maps are expected to (i) help clinicians in their diagnosis by highlighting, in a data-driven fashion, the pathological areas, and (ii) improve the interpretability of subsequent analyses, such as computer-aided diagnosis or spatiotemporal modeling.},
   author = {Ninon Burgos and M Jorge Cardoso and Jorge Samper-González and Marie-Odile Habert and Stanley Durrleman and Sébastien Ourselin and And and Olivier Colliot and Alzheimer’s Disease Neuroimaging Initiative and Frontotemporal Lobar Degeneration Neuroimaging Initiative},
   doi = {10.1117/1.JMI.8.2.024003},
   issn = {2329-4302},
   issue = {2},
   journal = {Journal of medical imaging (Bellingham, Wash.)},
   month = {3},
   pages = {024003},
   pmid = {33842668},
   title = {Anomaly detection for the individual analysis of brain PET images.},
   volume = {8},
   year = {2021},
}
@inproceedings{Mohla2020,
   author = {Satyam Mohla and Shivam Pande and Biplab Banerjee and Subhasis Chaudhuri},
   doi = {10.1109/CVPRW50498.2020.00054},
   isbn = {978-1-7281-9360-1},
   journal = {2020 IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW)},
   month = {6},
   pages = {416-425},
   publisher = {IEEE},
   title = {FusAtNet: Dual Attention based SpectroSpatial Multimodal Fusion Network for Hyperspectral and LiDAR Classification},
   year = {2020},
}
@article{Chen2021,
   abstract = {The recently developed vision transformer (ViT) has achieved promising results on image classification compared to convolutional neural networks. Inspired by this, in this paper, we study how to learn multi-scale feature representations in transformer models for image classification. To this end, we propose a dual-branch transformer to combine image patches (i.e., tokens in a transformer) of different sizes to produce stronger image features. Our approach processes small-patch and large-patch tokens with two separate branches of different computational complexity and these tokens are then fused purely by attention multiple times to complement each other. Furthermore, to reduce computation, we develop a simple yet effective token fusion module based on cross attention, which uses a single token for each branch as a query to exchange information with other branches. Our proposed cross-attention only requires linear time for both computational and memory complexity instead of quadratic time otherwise. Extensive experiments demonstrate that our approach performs better than or on par with several concurrent works on vision transformer, in addition to efficient CNN models. For example, on the ImageNet1K dataset, with some architectural changes, our approach outperforms the recent DeiT by a large margin of 2\% with a small to moderate increase in FLOPs and model parameters. Our source codes and models are available at \url\{https://github.com/IBM/CrossViT\}.},
   author = {Chun-Fu Chen and Quanfu Fan and Rameswar Panda},
   month = {3},
   title = {CrossViT: Cross-Attention Multi-Scale Vision Transformer for Image Classification},
   year = {2021},
}
@article{Gheini2021,
   abstract = {We study the power of cross-attention in the Transformer architecture within the context of transfer learning for machine translation, and extend the findings of studies into cross-attention when training from scratch. We conduct a series of experiments through fine-tuning a translation model on data where either the source or target language has changed. These experiments reveal that fine-tuning only the cross-attention parameters is nearly as effective as fine-tuning all parameters (i.e., the entire translation model). We provide insights into why this is the case and observe that limiting fine-tuning in this manner yields cross-lingually aligned embeddings. The implications of this finding for researchers and practitioners include a mitigation of catastrophic forgetting, the potential for zero-shot translation, and the ability to extend machine translation models to several new language pairs with reduced parameter storage overhead.},
   author = {Mozhdeh Gheini and Xiang Ren and Jonathan May},
   month = {4},
   title = {Cross-Attention is All You Need: Adapting Pretrained Transformers for Machine Translation},
   year = {2021},
}
@article{ChenX2020,
   abstract = {Unsupervised lesion detection is a challenging problem that requires accurately estimating normative distributions of healthy anatomy and detecting lesions as outliers without training examples. Recently, this problem has received increased attention from the research community following the advances in unsupervised learning with deep learning. Such advances allow the estimation of high-dimensional distributions, such as normative distributions, with higher accuracy than previous methods.The main approach of the recently proposed methods is to learn a latent-variable model parameterized with networks to approximate the normative distribution using example images showing healthy anatomy, perform prior-projection, i.e. reconstruct the image with lesions using the latent-variable model, and determine lesions based on the differences between the reconstructed and original images. While being promising, the prior-projection step often leads to a large number of false positives. In this work, we approach unsupervised lesion detection as an image restoration problem and propose a probabilistic model that uses a network-based prior as the normative distribution and detect lesions pixel-wise using MAP estimation. The probabilistic model punishes large deviations between restored and original images, reducing false positives in pixel-wise detections. Experiments with gliomas and stroke lesions in brain MRI using publicly available datasets show that the proposed approach outperforms the state-of-the-art unsupervised methods by a substantial margin, +0.13 (AUC), for both glioma and stroke detection. Extensive model analysis confirms the effectiveness of MAP-based image restoration.},
   author = {Xiaoran Chen and Suhang You and Kerem Can Tezcan and Ender Konukoglu},
   month = {4},
   title = {Unsupervised Lesion Detection via Image Restoration with a Normative Prior},
   year = {2020},
}
@article{SmithS2009,
   abstract = {Many image enhancement and thresholding techniques make use of spatial neighbourhood information to boost belief in extended areas of signal. The most common such approach in neuroimaging is cluster-based thresholding, which is often more sensitive than voxel-wise thresholding. However, a limitation is the need to define the initial cluster-forming threshold. This threshold is arbitrary, and yet its exact choice can have a large impact on the results, particularly at the lower (e.g., t, z < 4) cluster-forming thresholds frequently used. Furthermore, the amount of spatial pre-smoothing is also arbitrary (given that the expected signal extent is very rarely known in advance of the analysis). In the light of such problems, we propose a new method which attempts to keep the sensitivity benefits of cluster-based thresholding (and indeed the general concept of "clusters" of signal), while avoiding (or at least minimising) these problems. The method takes a raw statistic image and produces an output image in which the voxel-wise values represent the amount of cluster-like local spatial support. The method is thus referred to as "threshold-free cluster enhancement" (TFCE). We present the TFCE approach and discuss in detail ROC-based optimisation and comparisons with cluster-based and voxel-based thresholding. We find that TFCE gives generally better sensitivity than other methods over a wide range of test signal shapes and SNR values. We also show an example on a real imaging dataset, suggesting that TFCE does indeed provide not just improved sensitivity, but richer and more interpretable output than cluster-based thresholding.},
   author = {Stephen M Smith and Thomas E Nichols},
   doi = {10.1016/j.neuroimage.2008.03.061},
   issn = {1095-9572},
   issue = {1},
   journal = {NeuroImage},
   month = {1},
   pages = {83-98},
   pmid = {18501637},
   title = {Threshold-free cluster enhancement: addressing problems of smoothing, threshold dependence and localisation in cluster inference.},
   volume = {44},
   year = {2009},
}
@article{Nalisnick2018,
   abstract = {A neural network deployed in the wild may be asked to make predictions for inputs that were drawn from a different distribution than that of the training data. A plethora of work has demonstrated that it is easy to find or synthesize inputs for which a neural network is highly confident yet wrong. Generative models are widely viewed to be robust to such mistaken confidence as modeling the density of the input features can be used to detect novel, out-of-distribution inputs. In this paper we challenge this assumption. We find that the density learned by flow-based models, VAEs, and PixelCNNs cannot distinguish images of common objects such as dogs, trucks, and horses (i.e. CIFAR-10) from those of house numbers (i.e. SVHN), assigning a higher likelihood to the latter when the model is trained on the former. Moreover, we find evidence of this phenomenon when pairing several popular image data sets: FashionMNIST vs MNIST, CelebA vs SVHN, ImageNet vs CIFAR-10 / CIFAR-100 / SVHN. To investigate this curious behavior, we focus analysis on flow-based generative models in particular since they are trained and evaluated via the exact marginal likelihood. We find such behavior persists even when we restrict the flows to constant-volume transformations. These transformations admit some theoretical analysis, and we show that the difference in likelihoods can be explained by the location and variances of the data and the model curvature. Our results caution against using the density estimates from deep generative models to identify inputs similar to the training distribution until their behavior for out-of-distribution inputs is better understood.},
   author = {Eric Nalisnick and Akihiro Matsukawa and Yee Whye Teh and Dilan Gorur and Balaji Lakshminarayanan},
   month = {10},
   title = {Do Deep Generative Models Know What They Don't Know?},
   year = {2018},
}
@article{ZhangR2018,
   abstract = {While it is nearly effortless for humans to quickly assess the perceptual similarity between two images, the underlying processes are thought to be quite complex. Despite this, the most widely used perceptual metrics today, such as PSNR and SSIM, are simple, shallow functions, and fail to account for many nuances of human perception. Recently, the deep learning community has found that features of the VGG network trained on ImageNet classification has been remarkably useful as a training loss for image synthesis. But how perceptual are these so-called "perceptual losses"? What elements are critical for their success? To answer these questions, we introduce a new dataset of human perceptual similarity judgments. We systematically evaluate deep features across different architectures and tasks and compare them with classic metrics. We find that deep features outperform all previous metrics by large margins on our dataset. More surprisingly, this result is not restricted to ImageNet-trained VGG features, but holds across different deep architectures and levels of supervision (supervised, self-supervised, or even unsupervised). Our results suggest that perceptual similarity is an emergent property shared across deep visual representations.},
   author = {Richard Zhang and Phillip Isola and Alexei A. Efros and Eli Shechtman and Oliver Wang},
   month = {1},
   title = {The Unreasonable Effectiveness of Deep Features as a Perceptual Metric},
   year = {2018},
}
@article{Esser2020,
   abstract = {Designed to learn long-range interactions on sequential data, transformers continue to show state-of-the-art results on a wide variety of tasks. In contrast to CNNs, they contain no inductive bias that prioritizes local interactions. This makes them expressive, but also computationally infeasible for long sequences, such as high-resolution images. We demonstrate how combining the effectiveness of the inductive bias of CNNs with the expressivity of transformers enables them to model and thereby synthesize high-resolution images. We show how to (i) use CNNs to learn a context-rich vocabulary of image constituents, and in turn (ii) utilize transformers to efficiently model their composition within high-resolution images. Our approach is readily applied to conditional synthesis tasks, where both non-spatial information, such as object classes, and spatial information, such as segmentations, can control the generated image. In particular, we present the first results on semantically-guided synthesis of megapixel images with transformers and obtain the state of the art among autoregressive models on class-conditional ImageNet. Code and pretrained models can be found at https://github.com/CompVis/taming-transformers .},
   author = {Patrick Esser and Robin Rombach and Björn Ommer},
   month = {12},
   title = {Taming Transformers for High-Resolution Image Synthesis},
   year = {2020},
}
@article{Larsen2015,
   abstract = {We present an autoencoder that leverages learned representations to better measure similarities in data space. By combining a variational autoencoder with a generative adversarial network we can use learned feature representations in the GAN discriminator as basis for the VAE reconstruction objective. Thereby, we replace element-wise errors with feature-wise errors to better capture the data distribution while offering invariance towards e.g. translation. We apply our method to images of faces and show that it outperforms VAEs with element-wise similarity measures in terms of visual fidelity. Moreover, we show that the method learns an embedding in which high-level abstract visual features (e.g. wearing glasses) can be modified using simple arithmetic.},
   author = {Anders Boesen Lindbo Larsen and Søren Kaae Sønderby and Hugo Larochelle and Ole Winther},
   month = {12},
   title = {Autoencoding beyond pixels using a learned similarity metric},
   year = {2015},
}
@article{Dumoulin2016,
   abstract = {We introduce the adversarially learned inference (ALI) model, which jointly learns a generation network and an inference network using an adversarial process. The generation network maps samples from stochastic latent variables to the data space while the inference network maps training examples in data space to the space of latent variables. An adversarial game is cast between these two networks and a discriminative network is trained to distinguish between joint latent/data-space samples from the generative network and joint samples from the inference network. We illustrate the ability of the model to learn mutually coherent inference and generation networks through the inspections of model samples and reconstructions and confirm the usefulness of the learned representations by obtaining a performance competitive with state-of-the-art on the semi-supervised SVHN and CIFAR10 tasks.},
   author = {Vincent Dumoulin and Ishmael Belghazi and Ben Poole and Olivier Mastropietro and Alex Lamb and Martin Arjovsky and Aaron Courville},
   month = {6},
   title = {Adversarially Learned Inference},
   year = {2016},
}
@article{Lancucki2020,
   abstract = {In this paper we demonstrate methods for reliable and efficient training of discrete representation using Vector-Quantized Variational Auto-Encoder models (VQ-VAEs). Discrete latent variable models have been shown to learn nontrivial representations of speech, applicable to unsupervised voice conversion and reaching state-of-the-art performance on unit discovery tasks. For unsupervised representation learning, they became viable alternatives to continuous latent variable models such as the Variational Auto-Encoder (VAE). However, training deep discrete variable models is challenging, due to the inherent non-differentiability of the discretization operation. In this paper we focus on VQ-VAE, a state-of-the-art discrete bottleneck model shown to perform on par with its continuous counterparts. It quantizes encoder outputs with on-line $k$-means clustering. We show that the codebook learning can suffer from poor initialization and non-stationarity of clustered encoder outputs. We demonstrate that these can be successfully overcome by increasing the learning rate for the codebook and periodic date-dependent codeword re-initialization. As a result, we achieve more robust training across different tasks, and significantly increase the usage of latent codewords even for large codebooks. This has practical benefit, for instance, in unsupervised representation learning, where large codebooks may lead to disentanglement of latent representations.},
   author = {Adrian Łańcucki and Jan Chorowski and Guillaume Sanchez and Ricard Marxer and Nanxin Chen and Hans J. G. A. Dolfing and Sameer Khurana and Tanel Alumäe and Antoine Laurent},
   month = {5},
   title = {Robust Training of Vector Quantized Bottleneck Models},
   year = {2020},
}
@article{Dhariwal2020,
   abstract = {We introduce Jukebox, a model that generates music with singing in the raw audio domain. We tackle the long context of raw audio using a multi-scale VQ-VAE to compress it to discrete codes, and modeling those using autoregressive Transformers. We show that the combined model at scale can generate high-fidelity and diverse songs with coherence up to multiple minutes. We can condition on artist and genre to steer the musical and vocal style, and on unaligned lyrics to make the singing more controllable. We are releasing thousands of non cherry-picked samples at https://jukebox.openai.com, along with model weights and code at https://github.com/openai/jukebox},
   author = {Prafulla Dhariwal and Heewoo Jun and Christine Payne and Jong Wook Kim and Alec Radford and Ilya Sutskever},
   month = {4},
   title = {Jukebox: A Generative Model for Music},
   year = {2020},
}
@inproceedings{XiaY2015,
   author = {Yan Xia and Xudong Cao and Fang Wen and Gang Hua and Jian Sun},
   doi = {10.1109/ICCV.2015.177},
   isbn = {978-1-4673-8391-2},
   journal = {2015 IEEE International Conference on Computer Vision (ICCV)},
   month = {12},
   pages = {1511-1519},
   publisher = {IEEE},
   title = {Learning Discriminative Reconstructions for Unsupervised Outlier Removal},
   year = {2015},
}
@article{Marimont2020,
   abstract = {We propose an out-of-distribution detection method that combines density and restoration-based approaches using Vector-Quantized Variational Auto-Encoders (VQ-VAEs). The VQ-VAE model learns to encode images in a categorical latent space. The prior distribution of latent codes is then modelled using an Auto-Regressive (AR) model. We found that the prior probability estimated by the AR model can be useful for unsupervised anomaly detection and enables the estimation of both sample and pixel-wise anomaly scores. The sample-wise score is defined as the negative log-likelihood of the latent variables above a threshold selecting highly unlikely codes. Additionally, out-of-distribution images are restored into in-distribution images by replacing unlikely latent codes with samples from the prior model and decoding to pixel space. The average L1 distance between generated restorations and original image is used as pixel-wise anomaly score. We tested our approach on the MOOD challenge datasets, and report higher accuracies compared to a standard reconstruction-based approach with VAEs.},
   author = {Sergio Naval Marimont and Giacomo Tarroni},
   month = {12},
   title = {Anomaly detection through latent space restoration using vector-quantized variational autoencoders},
   year = {2020},
}
@article{Wang2020,
   abstract = {<p>Detecting image anomalies automatically in industrial scenarios can improve economic efficiency, but the scarcity of anomalous samples increases the challenge of the task. Recently, autoencoder has been widely used in image anomaly detection without using anomalous images during training. However, it is hard to determine the proper dimensionality of the latent space, and it often leads to unwanted reconstructions of the anomalous parts. To solve this problem, we propose a novel method based on the autoencoder. In this method, the latent space of the autoencoder is estimated using a discrete probability model. With the estimated probability model, the anomalous components in the latent space can be well excluded and undesirable reconstruction of the anomalous parts can be avoided. Specifically, we first adopt VQ-VAE as the reconstruction model to get a discrete latent space of normal samples. Then, PixelSail, a deep autoregressive model, is used to estimate the probability model of the discrete latent space. In the detection stage, the autoregressive model will determine the parts that deviate from the normal distribution in the input latent space. Then, the deviation code will be resampled from the normal distribution and decoded to yield a restored image, which is closest to the anomaly input. The anomaly is then detected by comparing the difference between the restored image and the anomaly image. Our proposed method is evaluated on the high-resolution industrial inspection image datasets MVTec AD which consist of 15 categories. The results show that the AUROC of the model improves by 15% over autoencoder and also yields competitive performance compared with state-of-the-art methods.</p>},
   author = {Lu Wang and Dongkai Zhang and Jiahao Guo and Yuexing Han},
   doi = {10.3390/app10238660},
   issn = {2076-3417},
   issue = {23},
   journal = {Applied Sciences},
   month = {12},
   pages = {8660},
   title = {Image Anomaly Detection Using Normal Data Only by Latent Space Resampling},
   volume = {10},
   year = {2020},
}
@article{Choromanski2020,
   abstract = {We introduce Performers, Transformer architectures which can estimate regular (softmax) full-rank-attention Transformers with provable accuracy, but using only linear (as opposed to quadratic) space and time complexity, without relying on any priors such as sparsity or low-rankness. To approximate softmax attention-kernels, Performers use a novel Fast Attention Via positive Orthogonal Random features approach (FAVOR+), which may be of independent interest for scalable kernel methods. FAVOR+ can be also used to efficiently model kernelizable attention mechanisms beyond softmax. This representational power is crucial to accurately compare softmax with other kernels for the first time on large-scale tasks, beyond the reach of regular Transformers, and investigate optimal attention-kernels. Performers are linear architectures fully compatible with regular Transformers and with strong theoretical guarantees: unbiased or nearly-unbiased estimation of the attention matrix, uniform convergence and low estimation variance. We tested Performers on a rich set of tasks stretching from pixel-prediction through text models to protein sequence modeling. We demonstrate competitive results with other examined efficient sparse and dense attention methods, showcasing effectiveness of the novel attention-learning paradigm leveraged by Performers.},
   author = {Krzysztof Choromanski and Valerii Likhosherstov and David Dohan and Xingyou Song and Andreea Gane and Tamas Sarlos and Peter Hawkins and Jared Davis and Afroz Mohiuddin and Lukasz Kaiser and David Belanger and Lucy Colwell and Adrian Weller},
   month = {9},
   title = {Rethinking Attention with Performers},
   year = {2020},
}
@article{Tay2020,
   abstract = {Transformers do not scale very well to long sequence lengths largely because of quadratic self-attention complexity. In the recent months, a wide spectrum of efficient, fast Transformers have been proposed to tackle this problem, more often than not claiming superior or comparable model quality to vanilla Transformer models. To this date, there is no well-established consensus on how to evaluate this class of models. Moreover, inconsistent benchmarking on a wide spectrum of tasks and datasets makes it difficult to assess relative model quality amongst many models. This paper proposes a systematic and unified benchmark, LRA, specifically focused on evaluating model quality under long-context scenarios. Our benchmark is a suite of tasks consisting of sequences ranging from $1K$ to $16K$ tokens, encompassing a wide range of data types and modalities such as text, natural, synthetic images, and mathematical expressions requiring similarity, structural, and visual-spatial reasoning. We systematically evaluate ten well-established long-range Transformer models (Reformers, Linformers, Linear Transformers, Sinkhorn Transformers, Performers, Synthesizers, Sparse Transformers, and Longformers) on our newly proposed benchmark suite. LRA paves the way towards better understanding this class of efficient Transformer models, facilitates more research in this direction, and presents new challenging tasks to tackle. Our benchmark code will be released at https://github.com/google-research/long-range-arena.},
   author = {Yi Tay and Mostafa Dehghani and Samira Abnar and Yikang Shen and Dara Bahri and Philip Pham and Jinfeng Rao and Liu Yang and Sebastian Ruder and Donald Metzler},
   month = {11},
   title = {Long Range Arena: A Benchmark for Efficient Transformers},
   year = {2020},
}
@article{Radford2018,
   author = {Alec Radford and Karthik Narasimhan},
   title = {Improving Language Understanding by Generative Pre-Training},
   year = {2018},
}
@article{Pinaya2021,
   abstract = {Pathological brain appearances may be so heterogeneous as to be intelligible only as anomalies, defined by their deviation from normality rather than any specific pathological characteristic. Amongst the hardest tasks in medical imaging, detecting such anomalies requires models of the normal brain that combine compactness with the expressivity of the complex, long-range interactions that characterise its structural organisation. These are requirements transformers have arguably greater potential to satisfy than other current candidate architectures, but their application has been inhibited by their demands on data and computational resource. Here we combine the latent representation of vector quantised variational autoencoders with an ensemble of autoregressive transformers to enable unsupervised anomaly detection and segmentation defined by deviation from healthy brain imaging data, achievable at low computational cost, within relative modest data regimes. We compare our method to current state-of-the-art approaches across a series of experiments involving synthetic and real pathological lesions. On real lesions, we train our models on 15,000 radiologically normal participants from UK Biobank, and evaluate performance on four different brain MR datasets with small vessel disease, demyelinating lesions, and tumours. We demonstrate superior anomaly detection performance both image-wise and pixel-wise, achievable without post-processing. These results draw attention to the potential of transformers in this most challenging of imaging tasks.},
   author = {Walter Hugo Lopez Pinaya and Petru-Daniel Tudosiu and Robert Gray and Geraint Rees and Parashkev Nachev and Sebastien Ourselin and M. Jorge Cardoso},
   month = {2},
   title = {Unsupervised Brain Anomaly Detection and Segmentation with Transformers},
   year = {2021},
}
@article{RazaviA2019,
   abstract = {We explore the use of Vector Quantized Variational AutoEncoder (VQ-VAE) models for large scale image generation. To this end, we scale and enhance the autoregressive priors used in VQ-VAE to generate synthetic samples of much higher coherence and fidelity than possible before. We use simple feed-forward encoder and decoder networks, making our model an attractive candidate for applications where the encoding and/or decoding speed is critical. Additionally, VQ-VAE requires sampling an autoregressive model only in the compressed latent space, which is an order of magnitude faster than sampling in the pixel space, especially for large images. We demonstrate that a multi-scale hierarchical organization of VQ-VAE, augmented with powerful priors over the latent codes, is able to generate samples with quality that rivals that of state of the art Generative Adversarial Networks on multifaceted datasets such as ImageNet, while not suffering from GAN's known shortcomings such as mode collapse and lack of diversity.},
   author = {Ali Razavi and Aaron van den Oord and Oriol Vinyals},
   month = {6},
   title = {Generating Diverse High-Fidelity Images with VQ-VAE-2},
   year = {2019},
}
@article{Tudosiu2020,
   abstract = {The increasing efficiency and compactness of deep learning architectures, together with hardware improvements, have enabled the complex and high-dimensional modelling of medical volumetric data at higher resolutions. Recently, Vector-Quantised Variational Autoencoders (VQ-VAE) have been proposed as an efficient generative unsupervised learning approach that can encode images to a small percentage of their initial size, while preserving their decoded fidelity. Here, we show a VQ-VAE inspired network can efficiently encode a full-resolution 3D brain volume, compressing the data to $0.825\%$ of the original size while maintaining image fidelity, and significantly outperforming the previous state-of-the-art. We then demonstrate that VQ-VAE decoded images preserve the morphological characteristics of the original data through voxel-based morphology and segmentation experiments. Lastly, we show that such models can be pre-trained and then fine-tuned on different datasets without the introduction of bias.},
   author = {Petru-Daniel Tudosiu and Thomas Varsavsky and Richard Shaw and Mark Graham and Parashkev Nachev and Sebastien Ourselin and Carole H. Sudre and M. Jorge Cardoso},
   month = {2},
   title = {Neuromorphologicaly-preserving Volumetric data encoding using VQ-VAE},
   year = {2020},
}
@article{Oord2017,
   abstract = {Learning useful representations without supervision remains a key challenge in machine learning. In this paper, we propose a simple yet powerful generative model that learns such discrete representations. Our model, the Vector Quantised-Variational AutoEncoder (VQ-VAE), differs from VAEs in two key ways: the encoder network outputs discrete, rather than continuous, codes; and the prior is learnt rather than static. In order to learn a discrete latent representation, we incorporate ideas from vector quantisation (VQ). Using the VQ method allows the model to circumvent issues of "posterior collapse" -- where the latents are ignored when they are paired with a powerful autoregressive decoder -- typically observed in the VAE framework. Pairing these representations with an autoregressive prior, the model can generate high quality images, videos, and speech as well as doing high quality speaker conversion and unsupervised learning of phonemes, providing further evidence of the utility of the learnt representations.},
   author = {Aaron van den Oord and Oriol Vinyals and Koray Kavukcuoglu},
   month = {11},
   title = {Neural Discrete Representation Learning},
   year = {2017},
}
@article{ChenX2016,
   abstract = {Representation learning seeks to expose certain aspects of observed data in a learned representation that's amenable to downstream tasks like classification. For instance, a good representation for 2D images might be one that describes only global structure and discards information about detailed texture. In this paper, we present a simple but principled method to learn such global representations by combining Variational Autoencoder (VAE) with neural autoregressive models such as RNN, MADE and PixelRNN/CNN. Our proposed VAE model allows us to have control over what the global latent code can learn and , by designing the architecture accordingly, we can force the global latent code to discard irrelevant information such as texture in 2D images, and hence the VAE only "autoencodes" data in a lossy fashion. In addition, by leveraging autoregressive models as both prior distribution $p(z)$ and decoding distribution $p(x|z)$, we can greatly improve generative modeling performance of VAEs, achieving new state-of-the-art results on MNIST, OMNIGLOT and Caltech-101 Silhouettes density estimation tasks.},
   author = {Xi Chen and Diederik P. Kingma and Tim Salimans and Yan Duan and Prafulla Dhariwal and John Schulman and Ilya Sutskever and Pieter Abbeel},
   month = {11},
   title = {Variational Lossy Autoencoder},
   year = {2016},
}
@article{Tomczak2017,
   abstract = {Many different methods to train deep generative models have been introduced in the past. In this paper, we propose to extend the variational auto-encoder (VAE) framework with a new type of prior which we call "Variational Mixture of Posteriors" prior, or VampPrior for short. The VampPrior consists of a mixture distribution (e.g., a mixture of Gaussians) with components given by variational posteriors conditioned on learnable pseudo-inputs. We further extend this prior to a two layer hierarchical model and show that this architecture with a coupled prior and posterior, learns significantly better models. The model also avoids the usual local optima issues related to useless latent dimensions that plague VAEs. We provide empirical studies on six datasets, namely, static and binary MNIST, OMNIGLOT, Caltech 101 Silhouettes, Frey Faces and Histopathology patches, and show that applying the hierarchical VampPrior delivers state-of-the-art results on all datasets in the unsupervised permutation invariant setting and the best results or comparable to SOTA methods for the approach with convolutional networks.},
   author = {Jakub M. Tomczak and Max Welling},
   month = {5},
   title = {VAE with a VampPrior},
   year = {2017},
}
@article{Makhzani2015,
   abstract = {In this paper, we propose the "adversarial autoencoder" (AAE), which is a probabilistic autoencoder that uses the recently proposed generative adversarial networks (GAN) to perform variational inference by matching the aggregated posterior of the hidden code vector of the autoencoder with an arbitrary prior distribution. Matching the aggregated posterior to the prior ensures that generating from any part of prior space results in meaningful samples. As a result, the decoder of the adversarial autoencoder learns a deep generative model that maps the imposed prior to the data distribution. We show how the adversarial autoencoder can be used in applications such as semi-supervised classification, disentangling style and content of images, unsupervised clustering, dimensionality reduction and data visualization. We performed experiments on MNIST, Street View House Numbers and Toronto Face datasets and show that adversarial autoencoders achieve competitive results in generative modeling and semi-supervised classification tasks.},
   author = {Alireza Makhzani and Jonathon Shlens and Navdeep Jaitly and Ian Goodfellow and Brendan Frey},
   month = {11},
   title = {Adversarial Autoencoders},
   year = {2015},
}
@article{Schlegl2017,
   abstract = {Obtaining models that capture imaging markers relevant for disease progression and treatment monitoring is challenging. Models are typically based on large amounts of data with annotated examples of known markers aiming at automating detection. High annotation effort and the limitation to a vocabulary of known markers limit the power of such approaches. Here, we perform unsupervised learning to identify anomalies in imaging data as candidates for markers. We propose AnoGAN, a deep convolutional generative adversarial network to learn a manifold of normal anatomical variability, accompanying a novel anomaly scoring scheme based on the mapping from image space to a latent space. Applied to new data, the model labels anomalies, and scores image patches indicating their fit into the learned distribution. Results on optical coherence tomography images of the retina demonstrate that the approach correctly identifies anomalous images, such as images containing retinal fluid or hyperreflective foci.},
   author = {Thomas Schlegl and Philipp Seeböck and Sebastian M. Waldstein and Ursula Schmidt-Erfurth and Georg Langs},
   month = {3},
   title = {Unsupervised Anomaly Detection with Generative Adversarial Networks to Guide Marker Discovery},
   year = {2017},
}
@article{Zimmerer2018,
   abstract = {Unsupervised learning can leverage large-scale data sources without the need for annotations. In this context, deep learning-based auto encoders have shown great potential in detecting anomalies in medical images. However, state-of-the-art anomaly scores are still based on the reconstruction error, which lacks in two essential parts: it ignores the model-internal representation employed for reconstruction, and it lacks formal assertions and comparability between samples. We address these shortcomings by proposing the Context-encoding Variational Autoencoder (ceVAE) which combines reconstruction- with density-based anomaly scoring. This improves the sample- as well as pixel-wise results. In our experiments on the BraTS-2017 and ISLES-2015 segmentation benchmarks, the ceVAE achieves unsupervised ROC-AUCs of 0.95 and 0.89, respectively, thus outperforming state-of-the-art methods by a considerable margin.},
   author = {David Zimmerer and Simon A. A. Kohl and Jens Petersen and Fabian Isensee and Klaus H. Maier-Hein},
   month = {12},
   title = {Context-encoding Variational Autoencoder for Unsupervised Anomaly Detection},
   year = {2018},
}
@article{ChenX2018,
   abstract = {Lesion detection in brain Magnetic Resonance Images (MRI) remains a challenging task. State-of-the-art approaches are mostly based on supervised learning making use of large annotated datasets. Human beings, on the other hand, even non-experts, can detect most abnormal lesions after seeing a handful of healthy brain images. Replicating this capability of using prior information on the appearance of healthy brain structure to detect lesions can help computers achieve human level abnormality detection, specifically reducing the need for numerous labeled examples and bettering generalization of previously unseen lesions. To this end, we study detection of lesion regions in an unsupervised manner by learning data distribution of brain MRI of healthy subjects using auto-encoder based methods. We hypothesize that one of the main limitations of the current models is the lack of consistency in latent representation. We propose a simple yet effective constraint that helps mapping of an image bearing lesion close to its corresponding healthy image in the latent space. We use the Human Connectome Project dataset to learn distribution of healthy-appearing brain MRI and report improved detection, in terms of AUC, of the lesions in the BRATS challenge dataset.},
   author = {Xiaoran Chen and Ender Konukoglu},
   month = {6},
   title = {Unsupervised Detection of Lesions in Brain MRI using constrained adversarial auto-encoders},
   year = {2018},
}
@inproceedings{SatoD2018,
   author = {Daisuke Sato and Shouhei Hanaoka and Yukihiro Nomura and Tomomi Takenaga and Soichiro Miki and Takeharu Yoshikawa and Naoto Hayashi and Osamu Abe},
   doi = {10.1117/12.2292276},
   editor = {Kensaku Mori and Nicholas Petrick},
   isbn = {9781510616394},
   journal = {Medical Imaging 2018: Computer-Aided Diagnosis},
   month = {2},
   pages = {60},
   publisher = {SPIE},
   title = {A primitive study on unsupervised anomaly detection with an autoencoder in emergency head CT volumes},
   year = {2018},
}
@book_section{Baur2019,
   author = {Christoph Baur and Benedikt Wiestler and Shadi Albarqouni and Nassir Navab},
   doi = {10.1007/978-3-030-11723-8_16},
   pages = {161-169},
   title = {Deep Autoencoding Models for Unsupervised Anomaly Segmentation in Brain MR Images},
   year = {2019},
}
@article{Atlason2018,
   abstract = {Lesions that appear hyperintense in both Fluid Attenuated Inversion Recovery (FLAIR) and T2-weighted magnetic resonance images (MRIs) of the human brain are common in the brains of the elderly population and may be caused by ischemia or demyelination. Lesions are biomarkers for various neurodegenerative diseases, making accurate quantification of them important for both disease diagnosis and progression. Automatic lesion detection using supervised learning requires manually annotated images, which can often be impractical to acquire. Unsupervised lesion detection, on the other hand, does not require any manual delineation; however, these methods can be challenging to construct due to the variability in lesion load, placement of lesions, and voxel intensities. Here we present a novel approach to address this problem using a convolutional autoencoder, which learns to segment brain lesions as well as the white matter, gray matter, and cerebrospinal fluid by reconstructing FLAIR images as conical combinations of softmax layer outputs generated from the corresponding T1, T2, and FLAIR images. Some of the advantages of this model are that it accurately learns to segment lesions regardless of lesion load, and it can be used to quickly and robustly segment new images that were not in the training set. Comparisons with state-of-the-art segmentation methods evaluated on ground truth manual labels indicate that the proposed method works well for generating accurate lesion segmentations without the need for manual annotations.},
   author = {Hans E. Atlason and Askell Love and Sigurdur Sigurdsson and Vilmundur Gudnason and Lotta M. Ellingsen},
   doi = {10.1117/12.2512953},
   month = {11},
   title = {Unsupervised brain lesion segmentation from MRI using a convolutional autoencoder},
   year = {2018},
}
@article{Baur2020,
   abstract = {Deep unsupervised representation learning has recently led to new approaches in the field of Unsupervised Anomaly Detection (UAD) in brain MRI. The main principle behind these works is to learn a model of normal anatomy by learning to compress and recover healthy data. This allows to spot abnormal structures from erroneous recoveries of compressed, potentially anomalous samples. The concept is of great interest to the medical image analysis community as it i) relieves from the need of vast amounts of manually segmented training data---a necessity for and pitfall of current supervised Deep Learning---and ii) theoretically allows to detect arbitrary, even rare pathologies which supervised approaches might fail to find. To date, the experimental design of most works hinders a valid comparison, because i) they are evaluated against different datasets and different pathologies, ii) use different image resolutions and iii) different model architectures with varying complexity. The intent of this work is to establish comparability among recent methods by utilizing a single architecture, a single resolution and the same dataset(s). Besides providing a ranking of the methods, we also try to answer questions like i) how many healthy training subjects are needed to model normality and ii) if the reviewed approaches are also sensitive to domain shift. Further, we identify open challenges and provide suggestions for future community efforts and research directions.},
   author = {Christoph Baur and Stefan Denner and Benedikt Wiestler and Shadi Albarqouni and Nassir Navab},
   month = {4},
   title = {Autoencoders for Unsupervised Anomaly Segmentation in Brain MR Images: A Comparative Study},
   year = {2020},
}
@article{Renard2013,
   abstract = {Only one large series using statistical parametric mapping (SPM) reports on FDG-PET in sporadic (Heidenhain and non-Heidenhain variant) Creutzfeldt-Jakob disease (sCJD), describing hypometabolism in bilateral parietal, frontal, and occipital cortices. Our aim was to study FDG-PET in non-Heidenhain probable sCJD patients in order to assess the most pertinent FDG-PET pattern, and to compare FDG-PET and MRI data. We used both SPM and NeuroGam(®) software analysis, compared with healthy controls, to describe the FDG-PET abnormalities. Individual FDG-PET and MRI-DWI data were compared. SPM group analysis showed lateralized hypometabolism in the medial parietal cortex, the lateral and medial frontal (sparing Brodmann's area 4 and 6 and the anterior cingulate cortex), and lateral parietal cortex, in the absence of basal ganglia or cerebellar hypometabolism. The most severe hypometabolism was seen in Brodmann's area 31, and to a lesser degree area 23 (both areas correspond to the posterior cingulate cortex) and the precuneus. On individual analysis using NeuroGam(®) software, additional variable temporal cortex and frequent basal ganglia (with caudate nucleus as the most frequently involved structure) hypometabolism was seen, in the absence of cerebellar hypometabolism. The cerebral lobe cortex was more frequently and more severely hypometabolic than basal ganglia structures. Concordance between FDG-PET and MRI abnormalities was most often present for both the cerebral lobe cortex and the basal ganglia. In the case of discordance, FDG-PET was more sensitive than MRI for the cortex, whereas MRI was more sensitive than FDG-PET for the basal ganglia. When pathological, both cortical lobe cortex and basal ganglia involvement were slightly more often lateralized on FDG-PET than on MRI. Despite the presence of overlapping features with other diseases presenting with rapidly progressive dementia, the FDG-PET pattern we found in our non-Heidenhain sCJD patients may help in the differential diagnosis of rapidly progressive dementia.},
   author = {Dimitri Renard and Rik Vandenberghe and Laurent Collombier and Pierre-Olivier Kotzki and Jean-Pierre Pouget and Vincent Boudousq},
   doi = {10.1007/s00415-013-7117-6},
   issn = {1432-1459},
   issue = {12},
   journal = {Journal of neurology},
   month = {12},
   pages = {3055-64},
   pmid = {24068371},
   title = {Glucose metabolism in nine patients with probable sporadic Creutzfeldt-Jakob disease: FDG-PET study using SPM and individual patient analysis.},
   volume = {260},
   year = {2013},
}
@article{Drzezga2005,
   abstract = {UNLABELLED Patients with mild cognitive impairment (MCI) represent a risk population for progressing to dementia of the Alzheimer type (DAT). However, clinical criteria do not ensure reliable individual prognosis in these patients. The objective of this longitudinal, prospective study was to examine the value of (18)F-FDG PET of cerebral glucose metabolism and of genetic susceptibility, as defined by an APOEepsilon4-positive genotype, with regard to the early diagnosis of DAT in patients with MCI. METHODS In 30 patients with the diagnosis of MCI (16 female, 14 male; age, 70 +/- 8 y), baseline and follow-up examinations (mean observation period, 16 mo) were performed. In all patients, the APOE genotype was assessed and cerebral glucose metabolism was evaluated at baseline using cranial (18)F-FDG PET. Individual PET data were screened for findings suggestive of Alzheimer's disease (AD), with the help of an automated computer program. After stereotactical normalization of the PET images, this program performs an observer-independent statistical comparison with an age-matched reference database (n = 22). RESULTS In 43% of all MCI subjects, a PET scan suggestive of AD pathology according to our predefined criteria was observed at baseline (PET+); 57% of all MCI patients were carriers of the APOE epsilon4 allele (e4+). In 40% of all patients, progression of symptoms within the observation period justified the clinical diagnosis of probable DAT at the time of follow-up reevaluation. Statistical evaluation revealed the best results for PET with regard to early diagnosis of DAT in MCI patients (sensitivity, 92%; specificity, 89%). Classification according to the APOE genotype was significantly less successful (sensitivity, 75%; specificity, 56%). However, a combination of both diagnostic tests allowed early diagnosis with either very high specificity (PET+ AND e4+: sensitivity, 67%; specificity, 100%) or very high sensitivity (PET+ OR e4+: sensitivity, 100%; specificity, 44%). CONCLUSION (18)F-FDG PET of cerebral glucose metabolism is a valuable diagnostic tool for the prediction of clinical outcome in individual MCI patients. Results are superior to the exclusive assessment of the APOE genotype. A combination of both functional imaging and genotyping may allow an early high-risk or low-risk stratification of patients with either very high sensitivity or very high specificity. This may be valuable, for example, for patient selection in scientific studies.},
   author = {Alexander Drzezga and Timo Grimmer and Matthias Riemenschneider and Nicola Lautenschlager and Hartwig Siebner and Panagiotis Alexopoulus and Satoshi Minoshima and Markus Schwaiger and Alexander Kurz},
   issn = {0161-5505},
   issue = {10},
   journal = {Journal of nuclear medicine : official publication, Society of Nuclear Medicine},
   month = {10},
   pages = {1625-32},
   pmid = {16204712},
   title = {Prediction of individual clinical outcome in MCI by means of genetic assessment and (18)F-FDG PET.},
   volume = {46},
   year = {2005},
}
@article{Signorini1999,
   abstract = {The [18F]fluorodeoxyglucose ([18F]FDG) method for measuring brain metabolism has not the wide clinical application that one might expect, partly because of its high cost and the complexity of the quantification procedure, but also because of reporting techniques based on region of interest (ROI) analysis, which are time-consuming and not fully objective. In this paper we report a clinical validation of statistical parametric mapping (SPM) using rCMRglc (quantitative) and radioactivity distribution (nonquantitative) [18F]FDG PET data. We show that a 10-min noninteractive voxel-based SPM analysis on a standard workstation enables objective assessment, including localization in stereotactic space, of regional glucose consumption abnormalities, whose reliability can be assessed on statistical and clinical grounds. Clinical validity was established using a small series of patients with degenerative or developmental disorders, including probable Alzheimer's disease, progressive aphasia, multiple sclerosis, developmental specific language impairment, and epilepsy. Analysis of quantitative and nonquantitative data showed the same pattern of results, suggesting that, for clinical purposes, quantitation and invasive arterial cannulation can be avoided. This should facilitate a wider application of the technique and the extension of SPM clinical analysis to H215O PET or high resolution SPECT perfusion studies.},
   author = {M Signorini and E Paulesu and K Friston and D Perani and A Colleluori and G Lucignani and F Grassi and V Bettinardi and R S Frackowiak and F Fazio},
   doi = {10.1006/nimg.1998.0381},
   issn = {1053-8119},
   issue = {1},
   journal = {NeuroImage},
   month = {1},
   pages = {63-80},
   pmid = {9918728},
   title = {Rapid assessment of regional cerebral metabolic abnormalities in single subjects with quantitative and nonquantitative [18F]FDG PET: A clinical validation of statistical parametric mapping.},
   volume = {9},
   year = {1999},
}
@article{Verduzco2019,
   author = {Haydeé C Verduzco-Aguirre and Gilberto Lopes and Enrique Soto-Perez-De-Celis},
   doi = {10.3332/ecancer.2019.ed87},
   issn = {17546605},
   journal = {ecancermedicalscience},
   month = {1},
   title = {Implementation of diagnostic resources for cancer in developing countries: a focus on PET/CT},
   volume = {13},
   year = {2019},
}
@article{Bray2018,
   abstract = {This article provides a status report on the global burden of cancer worldwide using the GLOBOCAN 2018 estimates of cancer incidence and mortality produced by the International Agency for Research on Cancer, with a focus on geographic variability across 20 world regions. There will be an estimated 18.1 million new cancer cases (17.0 million excluding nonmelanoma skin cancer) and 9.6 million cancer deaths (9.5 million excluding nonmelanoma skin cancer) in 2018. In both sexes combined, lung cancer is the most commonly diagnosed cancer (11.6% of the total cases) and the leading cause of cancer death (18.4% of the total cancer deaths), closely followed by female breast cancer (11.6%), prostate cancer (7.1%), and colorectal cancer (6.1%) for incidence and colorectal cancer (9.2%), stomach cancer (8.2%), and liver cancer (8.2%) for mortality. Lung cancer is the most frequent cancer and the leading cause of cancer death among males, followed by prostate and colorectal cancer (for incidence) and liver and stomach cancer (for mortality). Among females, breast cancer is the most commonly diagnosed cancer and the leading cause of cancer death, followed by colorectal and lung cancer (for incidence), and vice versa (for mortality); cervical cancer ranks fourth for both incidence and mortality. The most frequently diagnosed cancer and the leading cause of cancer death, however, substantially vary across countries and within each country depending on the degree of economic development and associated social and life style factors. It is noteworthy that high-quality cancer registry data, the basis for planning and implementing evidence-based cancer control programs, are not available in most low- and middle-income countries. The Global Initiative for Cancer Registry Development is an international partnership that supports better estimation, as well as the collection and use of local data, to prioritize and evaluate national cancer control efforts. CA: A Cancer Journal for Clinicians 2018;0:1-31. © 2018 American Cancer Society.},
   author = {Freddie Bray and Jacques Ferlay and Isabelle Soerjomataram and Rebecca L Siegel and Lindsey A Torre and Ahmedin Jemal},
   doi = {10.3322/caac.21492},
   issn = {1542-4863},
   issue = {6},
   journal = {CA: a cancer journal for clinicians},
   pages = {394-424},
   pmid = {30207593},
   title = {Global cancer statistics 2018: GLOBOCAN estimates of incidence and mortality worldwide for 36 cancers in 185 countries.},
   volume = {68},
   year = {2018},
}
@article{Perani2014,
   abstract = {PET based tools can improve the early diagnosis of Alzheimer's disease (AD) and differential diagnosis of dementia. The importance of identifying individuals at risk of developing dementia among people with subjective cognitive complaints or mild cognitive impairment has clinical, social, and therapeutic implications. Within the two major classes of AD biomarkers currently identified, that is, markers of pathology and neurodegeneration, amyloid- and FDG-PET imaging represent decisive tools for their measurement. As a consequence, the PET tools have been recognized to be of crucial value in the recent guidelines for the early diagnosis of AD and other dementia conditions. The references based recommendations, however, include large PET imaging literature based on visual methods that greatly reduces sensitivity and specificity and lacks a clear cut-off between normal and pathological findings. PET imaging can be assessed using parametric or voxel-wise analyses by comparing the subject's scan with a normative data set, significantly increasing the diagnostic accuracy. This paper is a survey of the relevant literature on FDG and amyloid-PET imaging aimed at providing the value of quantification for the early and differential diagnosis of AD. This allowed a meta-analysis and GRADE analysis revealing high values for PET imaging that might be useful in considering recommendations.},
   author = {Daniela Perani and Perani Daniela and Orazio Schillaci and Schillaci Orazio and Alessandro Padovani and Padovani Alessandro and Flavio Mariano Nobili and Nobili Flavio Mariano and Leonardo Iaccarino and Iaccarino Leonardo and Pasquale Anthony Della Rosa and Della Rosa Pasquale Anthony and Giovanni Frisoni and Frisoni Giovanni and Carlo Caltagirone and Caltagirone Carlo},
   doi = {10.1155/2014/785039},
   issn = {2314-6141},
   journal = {BioMed research international},
   pages = {785039},
   pmid = {24772437},
   title = {A survey of FDG- and amyloid-PET imaging in dementia and GRADE analysis.},
   volume = {2014},
   year = {2014},
}
@article{Newman2021,
   abstract = {BACKGROUND Missed vascular events, infections, and cancers account for ~75% of serious harms from diagnostic errors. Just 15 diseases from these "Big Three" categories account for nearly half of all serious misdiagnosis-related harms in malpractice claims. As part of a larger project estimating total US burden of serious misdiagnosis-related harms, we performed a focused literature review to measure diagnostic error and harm rates for these 15 conditions. METHODS We searched PubMed, Google, and cited references. For errors, we selected high-quality, modern, US-based studies, if available, and best available evidence otherwise. For harms, we used literature-based estimates of the generic (disease-agnostic) rate of serious harms (morbidity/mortality) per diagnostic error and applied claims-based severity weights to construct disease-specific rates. Results were validated via expert review and comparison to prior literature that used different methods. We used Monte Carlo analysis to construct probabilistic plausible ranges (PPRs) around estimates. RESULTS Rates for the 15 diseases were drawn from 28 published studies representing 91,755 patients. Diagnostic error (false negative) rates ranged from 2.2% (myocardial infarction) to 62.1% (spinal abscess), with a median of 13.6% [interquartile range (IQR) 9.2-24.7] and an aggregate mean of 9.7% (PPR 8.2-12.3). Serious misdiagnosis-related harm rates per incident disease case ranged from 1.2% (myocardial infarction) to 35.6% (spinal abscess), with a median of 5.5% (IQR 4.6-13.6) and an aggregate mean of 5.2% (PPR 4.5-6.7). Rates were considered face valid by domain experts and consistent with prior literature reports. CONCLUSIONS Diagnostic improvement initiatives should focus on dangerous conditions with higher diagnostic error and misdiagnosis-related harm rates.},
   author = {David E Newman-Toker and Zheyu Wang and Yuxin Zhu and Najlla Nassery and Ali S Saber Tehrani and Adam C Schaffer and Chihwen Winnie Yu-Moe and Gwendolyn D Clemens and Mehdi Fanai and Dana Siegal},
   doi = {10.1515/dx-2019-0104},
   issn = {2194-802X},
   issue = {1},
   journal = {Diagnosis (Berlin, Germany)},
   pages = {67-84},
   pmid = {32412440},
   title = {Rate of diagnostic errors and serious misdiagnosis-related harms for major vascular events, infections, and cancers: toward a national incidence estimate using the "Big Three".},
   volume = {8},
   year = {2021},
}
@web_page{mayoclinic2021,
   title = {Positron emission tomography scan},
   url = {https://www.mayoclinic.org/tests-procedures/pet-scan/about/pac-20385078},
}
@web_page{hopkins2021,
   title = {Positron Emission Tomography (PET)},
   url = {https://www.hopkinsmedicine.org/health/treatment-tests-and-therapies/positron-emission-tomography-pet},
}
@article{Kim2015,
   abstract = {Positron emission tomography / computed tomography (PET/CT), with its metabolic data of (18) F-fluorodeoxyglucose (FDG) cellular uptake in addition to morphologic CT data, is an established technique for staging of lung cancer and has higher sensitivity and accuracy for lung nodule characterization than conventional approaches. Its strength extends outside the chest, with unknown metastases detected or suspected metastases excluded in a significant number of patients. Lastly, PET/CT is used in the assessment of therapy response. Magnetic resonance imaging (MRI) in the chest has been difficult to establish, but with the advent of new sequences is starting to become an increasingly useful alternative to conventional approaches. Diffusion-weighted MRI (DWI) is useful for distinguishing benign and malignant pulmonary nodules, has high sensitivity and specificity for nodal staging, and is helpful for evaluating an early response to systemic chemotherapy. Whole-body MRI/PET promises to contribute additional information with its higher soft-tissue contrast and much less radiation exposure than PET/CT and has become feasible for fast imaging and can be used for cancer staging in patients with a malignant condition.},
   author = {Hyun Su Kim and Kyung Soo Lee and Yoshiharu Ohno and Edwin J R van Beek and Juergen Biederer},
   doi = {10.1002/jmri.24776},
   issn = {1522-2586},
   issue = {2},
   journal = {Journal of magnetic resonance imaging : JMRI},
   month = {8},
   pages = {247-60},
   pmid = {25365936},
   title = {PET/CT versus MRI for diagnosis, staging, and follow-up of lung cancer.},
   volume = {42},
   year = {2015},
}
@article{Almuhaideb2011,
   abstract = {Accurate diagnosis and staging are essential for the optimal management of cancer patients. Positron emission tomography with 2-deoxy-2-[fluorine-18]fluoro-D-glucose integrated with computed tomography (18F-FDG PET/CT) has emerged as a powerful imaging tool for the detection of various cancers. The combined acquisition of PET and CT has synergistic advantages over PET or CT alone and minimizes their individual limitations. It is a valuable tool for staging and restaging of some tumors and has an important role in the detection of recurrence in asymptomatic patients with rising tumor marker levels and patients with negative or equivocal findings on conventional imaging techniques. It also allows for monitoring response to therapy and permitting timely modification of therapeutic regimens. In about 27% of the patients, the course of management is changed. This review provides guidance for oncologists/radiotherapists and clinical and surgical specialists on the use of 18F-FDG PET/CT in oncology.},
   author = {Ahmad Almuhaideb and Nikolaos Papathanasiou and Jamshed Bomanji},
   doi = {10.4103/0256-4947.75771},
   issn = {0975-4466},
   issue = {1},
   journal = {Annals of Saudi medicine},
   pages = {3-13},
   pmid = {21245592},
   title = {18F-FDG PET/CT imaging in oncology.},
   volume = {31},
}
@web_page{CancerStatsUK2020,
   title = {Cancer Statistics for the UK.},
   url = {https://www.cancerresearchuk.org/health-professional/cancer-statistics-for-the-uk},
}
@web_page{oneintwo,
   title = { 1 in 2 people will develop cancer in their lifetime},
   url = {https://www.medicalnewstoday.com/articles/288916#:~:text=In%20the%20US%2C%201%20in,some%20point%20in%20their%20lives.},
}
