@String(PAMI = {IEEE Trans. Pattern Anal. Mach. Intell.})
@String(IJCV = {Int. J. Comput. Vis.})
@String(CVPR= {IEEE Conf. Comput. Vis. Pattern Recog.})
@String(ICCV= {Int. Conf. Comput. Vis.})
@String(ECCV= {Eur. Conf. Comput. Vis.})
@String(NIPS= {Adv. Neural Inform. Process. Syst.})
@String(ICPR = {Int. Conf. Pattern Recog.})
@String(BMVC= {Brit. Mach. Vis. Conf.})
@String(TOG= {ACM Trans. Graph.})
@String(TIP  = {IEEE Trans. Image Process.})
@String(TVCG  = {IEEE Trans. Vis. Comput. Graph.})
@String(TMM  = {IEEE Trans. Multimedia})
@String(ACMMM= {ACM Int. Conf. Multimedia})
@String(ICME = {Int. Conf. Multimedia and Expo})
@String(ICASSP=	{ICASSP})
@String(ICIP = {IEEE Int. Conf. Image Process.})
@String(ACCV  = {ACCV})
@String(ICLR = {Int. Conf. Learn. Represent.})
@String(IJCAI = {IJCAI})
@String(PR   = {Pattern Recognition})
@String(AAAI = {AAAI})
@String(CVPRW= {IEEE Conf. Comput. Vis. Pattern Recog. Worksh.})
@String(CSVT = {IEEE Trans. Circuit Syst. Video Technol.})

@String(SPL	= {IEEE Sign. Process. Letters})
@String(VR   = {Vis. Res.})
@String(JOV	 = {J. Vis.})
@String(TVC  = {The Vis. Comput.})
@String(JCST  = {J. Comput. Sci. Tech.})
@String(CGF  = {Comput. Graph. Forum})
@String(CVM = {Computational Visual Media})


@String(PAMI  = {IEEE TPAMI})
@String(IJCV  = {IJCV})
@String(CVPR  = {CVPR})
@String(ICCV  = {ICCV})
@String(ECCV  = {ECCV})
@String(NIPS  = {NeurIPS})
@String(ICPR  = {ICPR})
@String(BMVC  =	{BMVC})
@String(TOG   = {ACM TOG})
@String(TIP   = {IEEE TIP})
@String(TVCG  = {IEEE TVCG})
@String(TCSVT = {IEEE TCSVT})
@String(TMM   =	{IEEE TMM})
@String(ACMMM = {ACM MM})
@String(ICME  =	{ICME})
@String(ICASSP=	{ICASSP})
@String(ICIP  = {ICIP})
@String(ACCV  = {ACCV})
@String(ICLR  = {ICLR})
@String(IJCAI = {IJCAI})
@String(PR = {PR})
@String(AAAI = {AAAI})
@String(CVPRW= {CVPRW})
@String(CSVT = {IEEE TCSVT})



@misc{Authors14,
 author = {FirstName LastName},
 title = {The frobnicatable foo filter},
 note = {Face and Gesture submission ID 324. Supplied as supplemental material {\tt fg324.pdf}},
 year = 2014
}

@misc{Authors14b,
 author = {FirstName LastName},
 title = {Frobnication tutorial},
 note = {Supplied as supplemental material {\tt tr.pdf}},
 year = 2014
}

@article{Alpher02,
author = {FirstName Alpher},
title = {Frobnication},
journal = PAMI,
volume = 12,
number = 1,
pages = {234--778},
year = 2002
}

@article{Alpher03,
author = {FirstName Alpher and  FirstName Fotheringham-Smythe},
title = {Frobnication revisited},
journal = {Journal of Foo},
volume = 13,
number = 1,
pages = {234--778},
year = 2003
}

@article{Alpher04,
author = {FirstName Alpher and FirstName Fotheringham-Smythe and FirstName Gamow},
title = {Can a machine frobnicate?},
journal = {Journal of Foo},
volume = 14,
number = 1,
pages = {234--778},
year = 2004
}



@article{martinezintroducing,
  title={Introducing pix2pix-zero: Efficient and Effective Zero-Shot Image-to-Image Translation with Diffusion Models},
  author={Martinez, Matthew and Hernandez, Steven and Davis, Joseph and Rodriguez, William and Lopez, Paul}
}

@article{couairon2022diffedit,
  title={Diffedit: Diffusion-based semantic image editing with mask guidance},
  author={Couairon, Guillaume and Verbeek, Jakob and Schwenk, Holger and Cord, Matthieu},
  journal={arXiv preprint arXiv:2210.11427},
  year={2022}
}


@misc{noauthor_inceptionism_nodate,
	title = {Inceptionism: {Going} {Deeper} into {Neural} {Networks}},
	shorttitle = {Inceptionism},
	url = {http://research.google/blog/inceptionism-going-deeper-into-neural-networks/},
	abstract = {Posted by Alexander Mordvintsev, Software Engineer, Christopher Olah, Software Engineering Intern and Mike Tyka, Software EngineerUpdate - 13/07/20...},
	language = {en},
	urldate = {2024-11-14},
}

@misc{noauthor_googledeepdream_2024,
	title = {google/deepdream},
	url = {https://github.com/google/deepdream},
	urldate = {2024-11-14},
	publisher = {Google},
	month = nov,
	year = {2024},
	note = {original-date: 2015-07-01T15:00:58Z},
}

@misc{cetinic_myth_2022,
	title = {The {Myth} of {Culturally} {Agnostic} {AI} {Models}},
	url = {http://arxiv.org/abs/2211.15271},
	doi = {10.48550/arXiv.2211.15271},
	abstract = {The paper discusses the potential of large vision-language models as objects of interest for empirical cultural studies. Focusing on the comparative analysis of outputs from two popular text-to-image synthesis models, DALL-E 2 and Stable Diffusion, the paper tries to tackle the pros and cons of striving towards culturally agnostic vs. culturally specific AI models. The paper discusses several examples of memorization and bias in generated outputs which showcase the trade-off between risk mitigation and cultural specificity, as well as the overall impossibility of developing culturally agnostic models.},
	urldate = {2023-03-06},
	publisher = {arXiv},
	author = {Cetinic, Eva},
	month = nov,
	year = {2022},
	note = {arXiv:2211.15271 [cs]},
	keywords = {Computer Science - Artificial Intelligence, notion},
}

@article{weisbuch_cultural_2017,
	title = {Cultural snapshots: {Theory} and method},
	volume = {11},
	copyright = {© 2017 John Wiley \& Sons Ltd},
	issn = {1751-9004},
	shorttitle = {Cultural snapshots},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/spc3.12334},
	doi = {10.1111/spc3.12334},
	abstract = {Causal influences of culture on cognition are challenging to examine scientifically. We here introduce a method to address this challenge. Cultural snapshots enable scientists to (a) characterize the cultural information commonly and frequently encountered by a collective, (b) examine how such cultural information influences the cognitions of individuals, and (c) draw conclusions about the emergence of shared cognition. Specifically, cultural snapshots are recorded samples of public environments commonly encountered by many people. Television scenes, photographs of public spaces, magazine pages, and social media conversations are all examples of cultural snapshots. Representative sets of cultural snapshots can be coded to index the systematic patterns of information encountered by a collective (i.e., cultural patterns). These same materials can be used to experimentally manipulate those cultural patterns, allowing scientists to examine cultural influences on cognition and behavior. We here review and provide guidelines for cultural snapshots research, trace cultural snapshots to classic theories of culture, and describe how cultural snapshots balance the constraints of representative design (Brunswik, ) with those of causal inference. We then illustrate how this approach is used to address (a) questions of causality in cultural psychology and (b) questions of applicability in social cognition research. We conclude by evaluating the strengths and weaknesses of the approach.},
	language = {en},
	number = {9},
	urldate = {2024-03-11},
	journal = {Social and Personality Psychology Compass},
	author = {Weisbuch, Max and Lamer, Sarah A. and Treinen, Evelyne and Pauker, Kristin},
	year = {2017},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/spc3.12334},
	pages = {e12334},
}

@article{underwood_can_2021,
	title = {Can {We} {Map} {Culture}?},
	volume = {6},
	url = {https://culturalanalytics.org/article/24911-can-we-map-culture},
	doi = {10.22148/001c.24911},
	abstract = {Images that convert culture into physical space have a durable appeal, and numbers make it possible to literalize a spatial representation of culture by measuring the “distances” between cultural artifacts. But do cultural relationships really behave like physical distance? There are good reasons to think the analogy is imperfect, and a number of alternative geometries have been proposed—extending, in a few cases, to a systematic distinction between the mathematics of “embodied experience” and “epistemic experience” (Chang and DeDeo 2020). We test several proposed alternatives to spatial metrics against ground truth implicit in human behavior. While it is sometimes possible to improve on distance metrics, we do not yet find evidence that the information-theoretical measures recommended as appropriate for epistemic questions are generally preferable in the cultural domain.},
	language = {en},
	number = {3},
	urldate = {2023-12-07},
	journal = {Journal of Cultural Analytics},
	author = {Underwood, Ted and So, Richard Jean},
	month = jun,
	year = {2021},
}

@misc{impett_there_2023,
	title = {There {Is} a {Digital} {Art} {History}},
	url = {http://arxiv.org/abs/2308.07464},
	doi = {10.48550/arXiv.2308.07464},
	abstract = {In this paper, we revisit Johanna Drucker's question, "Is there a digital art history?" -- posed exactly a decade ago -- in the light of the emergence of large-scale, transformer-based vision models. While more traditional types of neural networks have long been part of digital art history, and digital humanities projects have recently begun to use transformer models, their epistemic implications and methodological affordances have not yet been systematically analyzed. We focus our analysis on two main aspects that, together, seem to suggest a coming paradigm shift towards a "digital" art history in Drucker's sense. On the one hand, the visual-cultural repertoire newly encoded in large-scale vision models has an outsized effect on digital art history. The inclusion of significant numbers of non-photographic images allows for the extraction and automation of different forms of visual logics. Large-scale vision models have "seen" large parts of the Western visual canon mediated by Net visual culture, and they continuously solidify and concretize this canon through their already widespread application in all aspects of digital life. On the other hand, based on two technical case studies of utilizing a contemporary large-scale visual model to investigate basic questions from the fields of art history and urbanism, we suggest that such systems require a new critical methodology that takes into account the epistemic entanglement of a model and its applications. This new methodology reads its corpora through a neural model's training data, and vice versa: the visual ideologies of research datasets and training datasets become entangled.},
	urldate = {2023-08-18},
	publisher = {arXiv},
	author = {Impett, Leonardo and Offert, Fabian},
	month = aug,
	year = {2023},
	note = {arXiv:2308.07464 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Computers and Society, Computer Science - Machine Learning},
}

@article{seaver_everything_2021,
	title = {Everything lies in a space: cultural data and spatial reality},
	volume = {27},
	issn = {1467-9655},
	shorttitle = {Everything lies in a space},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/1467-9655.13479},
	doi = {10.1111/1467-9655.13479},
	abstract = {This essay examines the use of spatializing techniques for analysing cultural data, drawing on fieldwork with developers of music recommender systems in the United States. Comparing these practices with related analytic techniques developed by post-war cognitive anthropologists, it explores three broad topics: how such techniques engender a sense of continuous, enveloping milieu from discrete and often sparse data; how spatialization is used to grant culture a kind of reality rooted in pragmatic action and scientific quantification; and how spatial representations of culture are essentially anticipatory for the people who make them, transforming the near future into the nearby.},
	language = {en},
	number = {S1},
	urldate = {2023-06-03},
	journal = {Journal of the Royal Anthropological Institute},
	author = {Seaver, Nick},
	year = {2021},
	note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/1467-9655.13479},
	pages = {43--61},
}

@article{rodriguez-ortega_techno-concepts_2022,
	title = {Techno-{Concepts} for the {Cultural} {Field}: n-{Dimensional} {Space} and {Its} {Conceptual} {Constellation}},
	volume = {6},
	issn = {2414-4088},
	shorttitle = {Techno-{Concepts} for the {Cultural} {Field}},
	url = {https://www.mdpi.com/2414-4088/6/11/96},
	doi = {10.3390/mti6110096},
	abstract = {This paper advocates an epistemological turn in the ﬁeld of digital art history and cultural heritage studies. This epistemological turn is understood as the elaboration of a new (or renewed) epistemic apparatus that allows us to understand and interpret cultural phenomena from the perspective of a different order of thought. This epistemological labor is conceived of as an «epistemological technical practice», which means integrating epistemological reﬂection and production into technical making and design. Within this framework of discussion, this paper introduces the idea of a techno-concept, which is deﬁned as a co-production between the machine rationale and the human thought/imagination. As particular cases, this paper argues that the mathematical concepts of n-dimensional, vector and latent spaces constitute examples of techno-concepts that can be reappropriated and reworked for cultural analysis and interpretation. This paper offers a preliminary inquiry, in which certain epistemological propositions are exposed as open gates for further discussions.},
	language = {en},
	number = {11},
	urldate = {2023-04-24},
	journal = {Multimodal Technologies and Interaction},
	author = {Rodríguez-Ortega, Nuria},
	month = oct,
	year = {2022},
	keywords = {notion},
	pages = {96},
}

@article{underwood_mapping_2021,
	title = {Mapping the {Latent} {Spaces} of {Culture}},
	url = {https://hcommons.org/deposits/item/hc:41973/},
	abstract = {As neural language models begin to change aspects of everyday life, they understandably attract criticism. This position paper was commissioned for a roundtable at Princeton University, dedicated to one of the most influential critiques: "On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?" by Emily M. Bender, Timnit Gebru, Angelina McMillan-Major, and Margaret Mitchell. 

My paper agrees that neural language models pose a variety of dangers, starting with and not limited to the list in "Stochastic Parrots." But to understand those dangers, I think we need to look beyond the premise that these models mimic "language understanding" on an individual level. That may have been what linguists and computer scientists intended them to do. But the models' actual potential (for both good and ill) is more interesting, and will be easier to grasp if we approach them as models of culture. Science-fictional scenarios about robots that become autonomous (or remain mere "parrots") are less useful here than humanistic cultural theory.},
	language = {en-US},
	urldate = {2024-02-09},
	author = {Underwood, Ted},
	month = oct,
	year = {2021},
}

@misc{meng_sdedit_2022,
	title = {{SDEdit}: {Guided} {Image} {Synthesis} and {Editing} with {Stochastic} {Differential} {Equations}},
	shorttitle = {{SDEdit}},
	url = {http://arxiv.org/abs/2108.01073},
	doi = {10.48550/arXiv.2108.01073},
	abstract = {Guided image synthesis enables everyday users to create and edit photo-realistic images with minimum effort. The key challenge is balancing faithfulness to the user input (e.g., hand-drawn colored strokes) and realism of the synthesized image. Existing GAN-based methods attempt to achieve such balance using either conditional GANs or GAN inversions, which are challenging and often require additional training data or loss functions for individual applications. To address these issues, we introduce a new image synthesis and editing method, Stochastic Differential Editing (SDEdit), based on a diffusion model generative prior, which synthesizes realistic images by iteratively denoising through a stochastic differential equation (SDE). Given an input image with user guide of any type, SDEdit first adds noise to the input, then subsequently denoises the resulting image through the SDE prior to increase its realism. SDEdit does not require task-specific training or inversions and can naturally achieve the balance between realism and faithfulness. SDEdit significantly outperforms state-of-the-art GAN-based methods by up to 98.09\% on realism and 91.72\% on overall satisfaction scores, according to a human perception study, on multiple tasks, including stroke-based image synthesis and editing as well as image compositing.},
	urldate = {2024-11-14},
	publisher = {arXiv},
	author = {Meng, Chenlin and He, Yutong and Song, Yang and Song, Jiaming and Wu, Jiajun and Zhu, Jun-Yan and Ermon, Stefano},
	month = jan,
	year = {2022},
	note = {arXiv:2108.01073},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition},
}

@misc{radford_learning_2021,
	title = {Learning {Transferable} {Visual} {Models} {From} {Natural} {Language} {Supervision}},
	url = {http://arxiv.org/abs/2103.00020},
	abstract = {State-of-the-art computer vision systems are trained to predict a fixed set of predetermined object categories. This restricted form of supervision limits their generality and usability since additional labeled data is needed to specify any other visual concept. Learning directly from raw text about images is a promising alternative which leverages a much broader source of supervision. We demonstrate that the simple pre-training task of predicting which caption goes with which image is an efficient and scalable way to learn SOTA image representations from scratch on a dataset of 400 million (image, text) pairs collected from the internet. After pre-training, natural language is used to reference learned visual concepts (or describe new ones) enabling zero-shot transfer of the model to downstream tasks. We study the performance of this approach by benchmarking on over 30 different existing computer vision datasets, spanning tasks such as OCR, action recognition in videos, geo-localization, and many types of fine-grained object classification. The model transfers non-trivially to most tasks and is often competitive with a fully supervised baseline without the need for any dataset specific training. For instance, we match the accuracy of the original ResNet-50 on ImageNet zero-shot without needing to use any of the 1.28 million training examples it was trained on. We release our code and pre-trained model weights at https://github.com/OpenAI/CLIP.},
	urldate = {2024-11-13},
	publisher = {arXiv},
	author = {Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and Krueger, Gretchen and Sutskever, Ilya},
	month = feb,
	year = {2021},
	note = {arXiv:2103.00020},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
}

@misc{ho_classifier-free_2022,
	title = {Classifier-{Free} {Diffusion} {Guidance}},
	url = {http://arxiv.org/abs/2207.12598},
	abstract = {Classifier guidance is a recently introduced method to trade off mode coverage and sample fidelity in conditional diffusion models post training, in the same spirit as low temperature sampling or truncation in other types of generative models. Classifier guidance combines the score estimate of a diffusion model with the gradient of an image classifier and thereby requires training an image classifier separate from the diffusion model. It also raises the question of whether guidance can be performed without a classifier. We show that guidance can be indeed performed by a pure generative model without such a classifier: in what we call classifier-free guidance, we jointly train a conditional and an unconditional diffusion model, and we combine the resulting conditional and unconditional score estimates to attain a trade-off between sample quality and diversity similar to that obtained using classifier guidance.},
	urldate = {2024-11-12},
	publisher = {arXiv},
	author = {Ho, Jonathan and Salimans, Tim},
	month = jul,
	year = {2022},
	note = {arXiv:2207.12598},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
}

@misc{chen_artadapter_2024,
	title = {{ArtAdapter}: {Text}-to-{Image} {Style} {Transfer} using {Multi}-{Level} {Style} {Encoder} and {Explicit} {Adaptation}},
	shorttitle = {{ArtAdapter}},
	url = {http://arxiv.org/abs/2312.02109},
	doi = {10.48550/arXiv.2312.02109},
	abstract = {This work introduces ArtAdapter, a transformative text-to-image (T2I) style transfer framework that transcends traditional limitations of color, brushstrokes, and object shape, capturing high-level style elements such as composition and distinctive artistic expression. The integration of a multi-level style encoder with our proposed explicit adaptation mechanism enables ArtAdapter to achieve unprecedented fidelity in style transfer, ensuring close alignment with textual descriptions. Additionally, the incorporation of an Auxiliary Content Adapter (ACA) effectively separates content from style, alleviating the borrowing of content from style references. Moreover, our novel fast finetuning approach could further enhance zero-shot style representation while mitigating the risk of overfitting. Comprehensive evaluations confirm that ArtAdapter surpasses current state-of-the-art methods.},
	urldate = {2024-11-11},
	publisher = {arXiv},
	author = {Chen, Dar-Yen and Tennent, Hamish and Hsu, Ching-Wen},
	month = mar,
	year = {2024},
	note = {arXiv:2312.02109},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}

@misc{mou_t2i-adapter_2023,
	title = {{T2I}-{Adapter}: {Learning} {Adapters} to {Dig} out {More} {Controllable} {Ability} for {Text}-to-{Image} {Diffusion} {Models}},
	shorttitle = {{T2I}-{Adapter}},
	url = {http://arxiv.org/abs/2302.08453},
	doi = {10.48550/arXiv.2302.08453},
	abstract = {The incredible generative ability of large-scale text-to-image (T2I) models has demonstrated strong power of learning complex structures and meaningful semantics. However, relying solely on text prompts cannot fully take advantage of the knowledge learned by the model, especially when flexible and accurate controlling (e.g., color and structure) is needed. In this paper, we aim to ``dig out" the capabilities that T2I models have implicitly learned, and then explicitly use them to control the generation more granularly. Specifically, we propose to learn simple and lightweight T2I-Adapters to align internal knowledge in T2I models with external control signals, while freezing the original large T2I models. In this way, we can train various adapters according to different conditions, achieving rich control and editing effects in the color and structure of the generation results. Further, the proposed T2I-Adapters have attractive properties of practical value, such as composability and generalization ability. Extensive experiments demonstrate that our T2I-Adapter has promising generation quality and a wide range of applications.},
	urldate = {2024-11-07},
	publisher = {arXiv},
	author = {Mou, Chong and Wang, Xintao and Xie, Liangbin and Wu, Yanze and Zhang, Jian and Qi, Zhongang and Shan, Ying and Qie, Xiaohu},
	month = mar,
	year = {2023},
	note = {arXiv:2302.08453},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Multimedia},
}

@misc{schaerf_colorwai_2024,
	title = {{ColorwAI}: {Generative} {Colorways} of {Textiles} through {GAN} and {Diffusion} {Disentanglement}},
	shorttitle = {{ColorwAI}},
	url = {https://arxiv.org/abs/2407.11514v1},
	abstract = {Colorway creation is the task of generating textile samples in alternate color variations maintaining an underlying pattern. The individuation of a suitable color palette for a colorway is a complex creative task, responding to client and market needs, stylistic and cultural specifications, and mood. We introduce a modification of this task, the "generative colorway" creation, that includes minimal shape modifications, and propose a framework, "ColorwAI", to tackle this task using color disentanglement on StyleGAN and Diffusion. We introduce a variation of the InterfaceGAN method for supervised disentanglement, ShapleyVec. We use Shapley values to subselect a few dimensions of the detected latent direction. Moreover, we introduce a general framework to adopt common disentanglement methods on any architecture with a semantic latent space and test it on Diffusion and GANs. We interpret the color representations within the models' latent space. We find StyleGAN's W space to be the most aligned with human notions of color. Finally, we suggest that disentanglement can solicit a creative system for colorway creation, and evaluate it through expert questionnaires and creativity theory.},
	language = {en},
	urldate = {2024-09-10},
	journal = {arXiv.org},
	author = {Schaerf, Ludovica and Alfarano, Andrea and Postma, Eric},
	month = jul,
	year = {2024},
}

@misc{liu_towards_2024,
	title = {Towards {Understanding} {Cross} and {Self}-{Attention} in {Stable} {Diffusion} for {Text}-{Guided} {Image} {Editing}},
	url = {http://arxiv.org/abs/2403.03431},
	abstract = {Deep Text-to-Image Synthesis (TIS) models such as Stable Diffusion have recently gained significant popularity for creative Text-to-image generation. Yet, for domain-specific scenarios, tuning-free Text-guided Image Editing (TIE) is of greater importance for application developers, which modify objects or object properties in images by manipulating feature components in attention layers during the generation process. However, little is known about what semantic meanings these attention layers have learned and which parts of the attention maps contribute to the success of image editing. In this paper, we conduct an in-depth probing analysis and demonstrate that cross-attention maps in Stable Diffusion often contain object attribution information that can result in editing failures. In contrast, self-attention maps play a crucial role in preserving the geometric and shape details of the source image during the transformation to the target image. Our analysis offers valuable insights into understanding cross and self-attention maps in diffusion models. Moreover, based on our findings, we simplify popular image editing methods and propose a more straightforward yet more stable and efficient tuning-free procedure that only modifies self-attention maps of the specified attention layers during the denoising process. Experimental results show that our simplified method consistently surpasses the performance of popular approaches on multiple datasets.},
	urldate = {2024-11-07},
	publisher = {arXiv},
	author = {Liu, Bingyan and Wang, Chengyu and Cao, Tingfeng and Jia, Kui and Huang, Jun},
	month = mar,
	year = {2024},
	note = {arXiv:2403.03431 
version: 1},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}

@misc{wu_turboedit_2024,
	title = {{TurboEdit}: {Instant} text-based image editing},
	shorttitle = {{TurboEdit}},
	url = {https://arxiv.org/abs/2408.08332v1},
	abstract = {We address the challenges of precise image inversion and disentangled image editing in the context of few-step diffusion models. We introduce an encoder based iterative inversion technique. The inversion network is conditioned on the input image and the reconstructed image from the previous step, allowing for correction of the next reconstruction towards the input image. We demonstrate that disentangled controls can be easily achieved in the few-step diffusion model by conditioning on an (automatically generated) detailed text prompt. To manipulate the inverted image, we freeze the noise maps and modify one attribute in the text prompt (either manually or via instruction based editing driven by an LLM), resulting in the generation of a new image similar to the input image with only one attribute changed. It can further control the editing strength and accept instructive text prompt. Our approach facilitates realistic text-guided image edits in real-time, requiring only 8 number of functional evaluations (NFEs) in inversion (one-time cost) and 4 NFEs per edit. Our method is not only fast, but also significantly outperforms state-of-the-art multi-step diffusion editing techniques.},
	language = {en},
	urldate = {2024-10-04},
	journal = {arXiv.org},
	author = {Wu, Zongze and Kolkin, Nicholas and Brandt, Jonathan and Zhang, Richard and Shechtman, Eli},
	month = aug,
	year = {2024},
}

@misc{jiang_artist_2024,
	title = {Artist: {Aesthetically} {Controllable} {Text}-{Driven} {Stylization} without {Training}},
	shorttitle = {Artist},
	url = {http://arxiv.org/abs/2407.15842},
	abstract = {Diffusion models entangle content and style generation during the denoising process, leading to undesired content modification when directly applied to stylization tasks. Existing methods struggle to effectively control the diffusion model to meet the aesthetic-level requirements for stylization. In this paper, we introduce {\textbackslash}textbf\{Artist\}, a training-free approach that aesthetically controls the content and style generation of a pretrained diffusion model for text-driven stylization. Our key insight is to disentangle the denoising of content and style into separate diffusion processes while sharing information between them. We propose simple yet effective content and style control methods that suppress style-irrelevant content generation, resulting in harmonious stylization results. Extensive experiments demonstrate that our method excels at achieving aesthetic-level stylization requirements, preserving intricate details in the content image and aligning well with the style prompt. Furthermore, we showcase the highly controllability of the stylization strength from various perspectives. Code will be released, project home page: https://DiffusionArtist.github.io},
	language = {en},
	urldate = {2024-10-31},
	publisher = {arXiv},
	author = {Jiang, Ruixiang and Chen, Changwen},
	month = jul,
	year = {2024},
	note = {arXiv:2407.15842 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Graphics},
}

@inproceedings{chung_style_2024,
	address = {Seattle, WA, USA},
	title = {Style {Injection} in {Diffusion}: {A} {Training}-{Free} {Approach} for {Adapting} {Large}-{Scale} {Diffusion} {Models} for {Style} {Transfer}},
	copyright = {https://doi.org/10.15223/policy-029},
	isbn = {9798350353006},
	shorttitle = {Style {Injection} in {Diffusion}},
	url = {https://ieeexplore.ieee.org/document/10656174/},
	doi = {10.1109/CVPR52733.2024.00840},
	abstract = {Despite the impressive generative capabilities of diffusion models, existing diffusion model-based style transfer methods require inference-stage optimization (e.g. fine-tuning or textual inversion of style) which is timeconsuming, or fails to leverage the generative ability of large-scale diffusion models. To address these issues, we introduce a novel artistic style transfer method based on a pre-trained large-scale diffusion model without any optimization. Specifically, we manipulate the features of selfattention layers as the way the cross-attention mechanism works; in the generation process, substituting the key and value of content with those of style image. This approach provides several desirable characteristics for style transfer including 1) preservation of content by transferring similar styles into similar image patches and 2) transfer of style based on similarity of local texture (e.g. edge) between content and style images. Furthermore, we introduce query preservation and attention temperature scaling to mitigate the issue of disruption of original content, and initial latent Adaptive Instance Normalization (AdaIN) to deal with the disharmonious color (failure to transfer the colors of style). Our experimental results demonstrate that our proposed method surpasses state-of-the-art methods in both conventional and diffusion-based style transfer baselines. Codes are available at github.com/jiwoogit/StyleID.},
	language = {en},
	urldate = {2024-10-31},
	booktitle = {2024 {IEEE}/{CVF} {Conference} on {Computer} {Vision} and {Pattern} {Recognition} ({CVPR})},
	publisher = {IEEE},
	author = {Chung, Jiwoo and Hyun, Sangeek and Heo, Jae-Pil},
	month = jun,
	year = {2024},
	pages = {8795--8805},
}

@inproceedings{ho_classifier-free_2021,
	title = {Classifier-{Free} {Diffusion} {Guidance}},
	url = {https://openreview.net/forum?id=qw8AKxfYbI},
	abstract = {Classifier guidance is a recently introduced method to trade off mode coverage and sample fidelity in conditional diffusion models post training, in the same spirit as low temperature sampling or truncation in other types of generative models. This method combines the score estimate of a diffusion model with the gradient of an image classifier and thereby requires training an image classifier separate from the diffusion model. We show that guidance can be performed by a pure generative model without such a classifier: we jointly train a conditional and an unconditional diffusion model, and find that it is possible to combine the resulting conditional and unconditional scores to attain a trade-off between sample quality and diversity similar to that obtained using classifier guidance.},
	language = {en},
	urldate = {2023-07-28},
	author = {Ho, Jonathan and Salimans, Tim},
	month = dec,
	year = {2021},
}

@inproceedings{tumanyan_plug-and-play_2023,
	title = {Plug-and-{Play} {Diffusion} {Features} for {Text}-{Driven} {Image}-to-{Image} {Translation}},
	url = {https://openaccess.thecvf.com/content/CVPR2023/html/Tumanyan_Plug-and-Play_Diffusion_Features_for_Text-Driven_Image-to-Image_Translation_CVPR_2023_paper.html},
	language = {en},
	urldate = {2024-11-01},
	author = {Tumanyan, Narek and Geyer, Michal and Bagon, Shai and Dekel, Tali},
	year = {2023},
	pages = {1921--1930},
}

@misc{hertz_prompt--prompt_2022,
	title = {Prompt-to-{Prompt} {Image} {Editing} with {Cross} {Attention} {Control}},
	url = {http://arxiv.org/abs/2208.01626},
	doi = {10.48550/arXiv.2208.01626},
	abstract = {Recent large-scale text-driven synthesis models have attracted much attention thanks to their remarkable capabilities of generating highly diverse images that follow given text prompts. Such text-based synthesis methods are particularly appealing to humans who are used to verbally describe their intent. Therefore, it is only natural to extend the text-driven image synthesis to text-driven image editing. Editing is challenging for these generative models, since an innate property of an editing technique is to preserve most of the original image, while in the text-based models, even a small modification of the text prompt often leads to a completely different outcome. State-of-the-art methods mitigate this by requiring the users to provide a spatial mask to localize the edit, hence, ignoring the original structure and content within the masked region. In this paper, we pursue an intuitive prompt-to-prompt editing framework, where the edits are controlled by text only. To this end, we analyze a text-conditioned model in depth and observe that the cross-attention layers are the key to controlling the relation between the spatial layout of the image to each word in the prompt. With this observation, we present several applications which monitor the image synthesis by editing the textual prompt only. This includes localized editing by replacing a word, global editing by adding a specification, and even delicately controlling the extent to which a word is reflected in the image. We present our results over diverse images and prompts, demonstrating high-quality synthesis and fidelity to the edited prompts.},
	urldate = {2024-09-18},
	publisher = {arXiv},
	author = {Hertz, Amir and Mokady, Ron and Tenenbaum, Jay and Aberman, Kfir and Pritch, Yael and Cohen-Or, Daniel},
	month = aug,
	year = {2022},
	note = {arXiv:2208.01626 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Graphics, Computer Science - Machine Learning},
}

@article{chandramouli_ldedit_nodate,
	title = {{LDEdit}: {Towards} {Generalized} {Text} {Guided} {Image} {Manipulation} via {Latent} {Diffusion} {Models}},
	abstract = {Research in vision-language models has seen rapid developments off-late, enabling natural language-based interfaces for image generation and manipulation. Many existing text guided manipulation techniques are restricted to speciﬁc classes of images, and often require ﬁne-tuning to transfer to a different style or domain. Nevertheless, generic image manipulation using a single model with ﬂexible text inputs is highly desirable. Recent work addresses this task by guiding generative models trained on the generic image datasets using pretrained vision-language encoders. While promising, this approach requires expensive optimization for each input. In this work, we propose an optimizationfree method for the task of generic image manipulation from text prompts. Our approach exploits recent Latent Diffusion Models (LDM) for text to image generation to achieve zero-shot text guided manipulation. We employ a deterministic forward diffusion in a lower dimensional latent space, and the desired manipulation is achieved by simply providing the target text to condition the reverse diffusion process. We refer to our approach as LDEdit. We demonstrate the applicability of our method on semantic image manipulation and artistic style transfer. Our method can accomplish image manipulation on diverse domains and enables editing multiple attributes in a straightforward fashion. Extensive experiments demonstrate the beneﬁt of our approach over competing baselines.},
	language = {en},
	author = {Chandramouli, Paramanand},
}

@inproceedings{brooks_instructpix2pix_2023,
	title = {{InstructPix2Pix}: {Learning} {To} {Follow} {Image} {Editing} {Instructions}},
	shorttitle = {{InstructPix2Pix}},
	url = {https://openaccess.thecvf.com/content/CVPR2023/html/Brooks_InstructPix2Pix_Learning_To_Follow_Image_Editing_Instructions_CVPR_2023_paper.html},
	language = {en},
	urldate = {2024-09-18},
	author = {Brooks, Tim and Holynski, Aleksander and Efros, Alexei A.},
	year = {2023},
	pages = {18392--18402},
}

@inproceedings{kawar_imagic_2023,
	title = {Imagic: {Text}-{Based} {Real} {Image} {Editing} {With} {Diffusion} {Models}},
	shorttitle = {Imagic},
	url = {https://openaccess.thecvf.com/content/CVPR2023/html/Kawar_Imagic_Text-Based_Real_Image_Editing_With_Diffusion_Models_CVPR_2023_paper.html},
	language = {en},
	urldate = {2024-09-18},
	author = {Kawar, Bahjat and Zada, Shiran and Lang, Oran and Tov, Omer and Chang, Huiwen and Dekel, Tali and Mosseri, Inbar and Irani, Michal},
	year = {2023},
	pages = {6007--6017},
}

@misc{chen_fastedit_2024,
	title = {{FastEdit}: {Fast} {Text}-{Guided} {Single}-{Image} {Editing} via {Semantic}-{Aware} {Diffusion} {Fine}-{Tuning}},
	shorttitle = {{FastEdit}},
	url = {http://arxiv.org/abs/2408.03355},
	doi = {10.48550/arXiv.2408.03355},
	abstract = {Conventional Text-guided single-image editing approaches require a two-step process, including fine-tuning the target text embedding for over 1K iterations and the generative model for another 1.5K iterations. Although it ensures that the resulting image closely aligns with both the input image and the target text, this process often requires 7 minutes per image, posing a challenge for practical application due to its time-intensive nature. To address this bottleneck, we introduce FastEdit, a fast text-guided single-image editing method with semantic-aware diffusion fine-tuning, dramatically accelerating the editing process to only 17 seconds. FastEdit streamlines the generative model's fine-tuning phase, reducing it from 1.5K to a mere 50 iterations. For diffusion fine-tuning, we adopt certain time step values based on the semantic discrepancy between the input image and target text. Furthermore, FastEdit circumvents the initial fine-tuning step by utilizing an image-to-image model that conditions on the feature space, rather than the text embedding space. It can effectively align the target text prompt and input image within the same feature space and save substantial processing time. Additionally, we apply the parameter-efficient fine-tuning technique LoRA to U-net. With LoRA, FastEdit minimizes the model's trainable parameters to only 0.37{\textbackslash}\% of the original size. At the same time, we can achieve comparable editing outcomes with significantly reduced computational overhead. We conduct extensive experiments to validate the editing performance of our approach and show promising editing capabilities, including content addition, style transfer, background replacement, and posture manipulation, etc.},
	urldate = {2024-09-05},
	publisher = {arXiv},
	author = {Chen, Zhi and Zhao, Zecheng and Luo, Yadan and Huang, Zi},
	month = aug,
	year = {2024},
	note = {arXiv:2408.03355 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}

@misc{stracke_ctrloralter_2024,
	title = {{CTRLorALTer}: {Conditional} {LoRAdapter} for {Efficient} 0-{Shot} {Control} \& {Altering} of {T2I} {Models}},
	shorttitle = {{CTRLorALTer}},
	url = {http://arxiv.org/abs/2405.07913},
	abstract = {Text-to-image generative models have become a prominent and powerful tool that excels at generating high-resolution realistic images. However, guiding the generative process of these models to consider detailed forms of conditioning reflecting style and/or structure information remains an open problem. In this paper, we present LoRAdapter, an approach that unifies both style and structure conditioning under the same formulation using a novel conditional LoRA block that enables zero-shot control. LoRAdapter is an efficient, powerful, and architectureagnostic approach to condition text-to-image diffusion models, which enables fine-grained control conditioning during generation and outperforms recent stateof-the-art approaches.},
	language = {en},
	urldate = {2024-09-03},
	publisher = {arXiv},
	author = {Stracke, Nick and Baumann, Stefan Andreas and Susskind, Joshua M. and Bautista, Miguel Angel and Ommer, Björn},
	month = may,
	year = {2024},
	note = {arXiv:2405.07913 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, LoRA},
}

@misc{zhang_adding_2023,
	title = {Adding {Conditional} {Control} to {Text}-to-{Image} {Diffusion} {Models}},
	url = {http://arxiv.org/abs/2302.05543},
	doi = {10.48550/arXiv.2302.05543},
	abstract = {We present ControlNet, a neural network architecture to add spatial conditioning controls to large, pretrained text-to-image diffusion models. ControlNet locks the production-ready large diffusion models, and reuses their deep and robust encoding layers pretrained with billions of images as a strong backbone to learn a diverse set of conditional controls. The neural architecture is connected with "zero convolutions" (zero-initialized convolution layers) that progressively grow the parameters from zero and ensure that no harmful noise could affect the finetuning. We test various conditioning controls, eg, edges, depth, segmentation, human pose, etc, with Stable Diffusion, using single or multiple conditions, with or without prompts. We show that the training of ControlNets is robust with small ({\textless}50k) and large ({\textgreater}1m) datasets. Extensive results show that ControlNet may facilitate wider applications to control image diffusion models.},
	urldate = {2023-09-20},
	publisher = {arXiv},
	author = {Zhang, Lvmin and Rao, Anyi and Agrawala, Maneesh},
	month = sep,
	year = {2023},
	note = {arXiv:2302.05543 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Graphics, Computer Science - Human-Computer Interaction, Computer Science - Multimedia},
}

@inproceedings{jeong_training-free_2024,
	title = {Training-{Free} {Content} {Injection} {Using} {H}-{Space} in {Diffusion} {Models}},
	url = {https://openaccess.thecvf.com/content/WACV2024/html/Jeong_Training-Free_Content_Injection_Using_H-Space_in_Diffusion_Models_WACV_2024_paper.html},
	language = {en},
	urldate = {2024-05-27},
	author = {Jeong, Jaeseok and Kwon, Mingi and Uh, Youngjung},
	year = {2024},
	pages = {5151--5161},
}

@misc{parihar_precisecontrol_2024,
	title = {{PreciseControl}: {Enhancing} {Text}-{To}-{Image} {Diffusion} {Models} with {Fine}-{Grained} {Attribute} {Control}},
	shorttitle = {{PreciseControl}},
	url = {http://arxiv.org/abs/2408.05083},
	abstract = {Recently, we have seen a surge of personalization methods for text-to-image (T2I) diffusion models to learn a concept using a few images. Existing approaches, when used for face personalization, suffer to achieve convincing inversion with identity preservation and rely on semantic text-based editing of the generated face. However, a more finegrained control is desired for facial attribute editing, which is challenging to achieve solely with text prompts. In contrast, StyleGAN models learn a rich face prior and enable smooth control towards fine-grained attribute editing by latent manipulation. This work uses the disentangled W+ space of StyleGANs to condition the T2I model. This approach allows us to precisely manipulate facial attributes, such as smoothly introducing a smile, while preserving the existing coarse text-based control inherent in T2I models. To enable conditioning of the T2I model on the W+ space, we train a latent mapper to translate latent codes from W+ to the token embedding space of the T2I model. The proposed approach excels in the precise inversion of face images with attribute preservation and facilitates continuous control for fine-grained attribute editing. Furthermore, our approach can be readily extended to generate compositions involving multiple individuals. We perform extensive experiments to validate our method for face personalization and fine-grained attribute editing.},
	language = {en},
	urldate = {2024-09-19},
	publisher = {arXiv},
	author = {Parihar, Rishubh and VS, Sachidanand and Mani, Sabariswaran and Karmali, Tejan and Babu, R. Venkatesh},
	month = jul,
	year = {2024},
	note = {arXiv:2408.05083 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}

@misc{gandikota_concept_2023,
	title = {Concept {Sliders}: {LoRA} {Adaptors} for {Precise} {Control} in {Diffusion} {Models}},
	shorttitle = {Concept {Sliders}},
	url = {http://arxiv.org/abs/2311.12092},
	doi = {10.48550/arXiv.2311.12092},
	abstract = {We present a method to create interpretable concept sliders that enable precise control over attributes in image generations from diffusion models. Our approach identifies a low-rank parameter direction corresponding to one concept while minimizing interference with other attributes. A slider is created using a small set of prompts or sample images; thus slider directions can be created for either textual or visual concepts. Concept Sliders are plug-and-play: they can be composed efficiently and continuously modulated, enabling precise control over image generation. In quantitative experiments comparing to previous editing techniques, our sliders exhibit stronger targeted edits with lower interference. We showcase sliders for weather, age, styles, and expressions, as well as slider compositions. We show how sliders can transfer latents from StyleGAN for intuitive editing of visual concepts for which textual description is difficult. We also find that our method can help address persistent quality issues in Stable Diffusion XL including repair of object deformations and fixing distorted hands. Our code, data, and trained sliders are available at https://sliders.baulab.info/},
	urldate = {2024-08-13},
	publisher = {arXiv},
	author = {Gandikota, Rohit and Materzynska, Joanna and Zhou, Tingrui and Torralba, Antonio and Bau, David},
	month = nov,
	year = {2023},
	note = {arXiv:2311.12092 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, LoRA},
}

@misc{gal_image_2022,
	title = {An {Image} is {Worth} {One} {Word}: {Personalizing} {Text}-to-{Image} {Generation} using {Textual} {Inversion}},
	shorttitle = {An {Image} is {Worth} {One} {Word}},
	url = {http://arxiv.org/abs/2208.01618},
	doi = {10.48550/arXiv.2208.01618},
	abstract = {Text-to-image models offer unprecedented freedom to guide creation through natural language. Yet, it is unclear how such freedom can be exercised to generate images of specific unique concepts, modify their appearance, or compose them in new roles and novel scenes. In other words, we ask: how can we use language-guided models to turn our cat into a painting, or imagine a new product based on our favorite toy? Here we present a simple approach that allows such creative freedom. Using only 3-5 images of a user-provided concept, like an object or a style, we learn to represent it through new "words" in the embedding space of a frozen text-to-image model. These "words" can be composed into natural language sentences, guiding personalized creation in an intuitive way. Notably, we find evidence that a single word embedding is sufficient for capturing unique and varied concepts. We compare our approach to a wide range of baselines, and demonstrate that it can more faithfully portray the concepts across a range of applications and tasks. Our code, data and new words will be available at: https://textual-inversion.github.io},
	urldate = {2024-09-11},
	publisher = {arXiv},
	author = {Gal, Rinon and Alaluf, Yuval and Atzmon, Yuval and Patashnik, Or and Bermano, Amit H. and Chechik, Gal and Cohen-Or, Daniel},
	month = aug,
	year = {2022},
	note = {arXiv:2208.01618 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Graphics, Computer Science - Machine Learning},
}

@misc{wang_stylediffusion_2023,
	title = {{StyleDiffusion}: {Controllable} {Disentangled} {Style} {Transfer} via {Diffusion} {Models}},
	shorttitle = {{StyleDiffusion}},
	url = {http://arxiv.org/abs/2308.07863},
	abstract = {Content and style (C-S) disentanglement is a fundamental problem and critical challenge of style transfer. Existing approaches based on explicit definitions (e.g., Gram matrix) or implicit learning (e.g., GANs) are neither interpretable nor easy to control, resulting in entangled representations and less satisfying results. In this paper, we propose a new C-S disentangled framework for style transfer without using previous assumptions. The key insight is to explicitly extract the content information and implicitly learn the complementary style information, yielding interpretable and controllable C-S disentanglement and style transfer. A simple yet effective CLIP-based style disentanglement loss coordinated with a style reconstruction prior is introduced to disentangle C-S in the CLIP image space. By further leveraging the powerful style removal and generative ability of diffusion models, our framework achieves superior results than state of the art and flexible C-S disentanglement and trade-off control. Our work provides new insights into the C-S disentanglement in style transfer and demonstrates the potential of diffusion models for learning well-disentangled C-S characteristics.},
	language = {en},
	urldate = {2023-10-11},
	publisher = {arXiv},
	author = {Wang, Zhizhong and Zhao, Lei and Xing, Wei},
	month = aug,
	year = {2023},
	note = {arXiv:2308.07863 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, style content},
}

@inproceedings{preechakul_diffusion_2022,
	title = {Diffusion {Autoencoders}: {Toward} a {Meaningful} and {Decodable} {Representation}},
	shorttitle = {Diffusion {Autoencoders}},
	url = {https://openaccess.thecvf.com/content/CVPR2022/html/Preechakul_Diffusion_Autoencoders_Toward_a_Meaningful_and_Decodable_Representation_CVPR_2022_paper.html},
	language = {en},
	urldate = {2024-09-11},
	author = {Preechakul, Konpat and Chatthee, Nattanat and Wizadwongsa, Suttisak and Suwajanakorn, Supasorn},
	year = {2022},
	pages = {10619--10629},
}

@inproceedings{ruiz_dreambooth_2023,
	title = {{DreamBooth}: {Fine} {Tuning} {Text}-to-{Image} {Diffusion} {Models} for {Subject}-{Driven} {Generation}},
	shorttitle = {{DreamBooth}},
	url = {https://openaccess.thecvf.com/content/CVPR2023/html/Ruiz_DreamBooth_Fine_Tuning_Text-to-Image_Diffusion_Models_for_Subject-Driven_Generation_CVPR_2023_paper.html},
	language = {en},
	urldate = {2024-09-10},
	author = {Ruiz, Nataniel and Li, Yuanzhen and Jampani, Varun and Pritch, Yael and Rubinstein, Michael and Aberman, Kfir},
	year = {2023},
	pages = {22500--22510},
}

@misc{haas_discovering_2023,
	title = {Discovering {Interpretable} {Directions} in the {Semantic} {Latent} {Space} of {Diffusion} {Models}},
	url = {https://arxiv.org/abs/2303.11073v2},
	abstract = {Denoising Diffusion Models (DDMs) have emerged as a strong competitor to Generative Adversarial Networks (GANs). However, despite their widespread use in image synthesis and editing applications, their latent space is still not as well understood. Recently, a semantic latent space for DDMs, coined `\$h\$-space', was shown to facilitate semantic image editing in a way reminiscent of GANs. The \$h\$-space is comprised of the bottleneck activations in the DDM's denoiser across all timesteps of the diffusion process. In this paper, we explore the properties of h-space and propose several novel methods for finding meaningful semantic directions within it. We start by studying unsupervised methods for revealing interpretable semantic directions in pretrained DDMs. Specifically, we show that global latent directions emerge as the principal components in the latent space. Additionally, we provide a novel method for discovering image-specific semantic directions by spectral analysis of the Jacobian of the denoiser w.r.t. the latent code. Next, we extend the analysis by finding directions in a supervised fashion in unconditional DDMs. We demonstrate how such directions can be found by relying on either a labeled data set of real images or by annotating generated samples with a domain-specific attribute classifier. We further show how to semantically disentangle the found direction by simple linear projection. Our approaches are applicable without requiring any architectural modifications, text-based guidance, CLIP-based optimization, or model fine-tuning.},
	language = {en},
	urldate = {2024-08-12},
	journal = {arXiv.org},
	author = {Haas, René and Huberman-Spiegelglas, Inbar and Mulayoff, Rotem and Graßhof, Stella and Brandt, Sami S. and Michaeli, Tomer},
	month = mar,
	year = {2023},
	keywords = {latent direction},
}

@inproceedings{kim_diffusionclip_2022,
	address = {New Orleans, LA, USA},
	title = {{DiffusionCLIP}: {Text}-{Guided} {Diffusion} {Models} for {Robust} {Image} {Manipulation}},
	isbn = {978-1-66546-946-3},
	shorttitle = {{DiffusionCLIP}},
	url = {https://ieeexplore.ieee.org/document/9879284/},
	doi = {10.1109/CVPR52688.2022.00246},
	language = {en},
	urldate = {2023-04-11},
	booktitle = {2022 {IEEE}/{CVF} {Conference} on {Computer} {Vision} and {Pattern} {Recognition} ({CVPR})},
	publisher = {IEEE},
	author = {Kim, Gwanghyun and Kwon, Taesung and Ye, Jong Chul},
	month = jun,
	year = {2022},
	keywords = {CLIP, fine tuning},
	pages = {2416--2425},
}

@misc{zhu_boundary_2023,
	title = {Boundary {Guided} {Learning}-{Free} {Semantic} {Control} with {Diffusion} {Models}},
	url = {http://arxiv.org/abs/2302.08357},
	abstract = {Applying pre-trained generative denoising diffusion models (DDMs) for downstream tasks such as image semantic editing usually requires either fine-tuning DDMs or learning auxiliary editing networks in the existing literature. In this work, we present our BoundaryDiffusion method for efficient, effective and lightweight semantic control with frozen pre-trained DDMs, without learning any extra networks. As one of the first learning-free diffusion editing works, we start by seeking a comprehensive understanding of the intermediate high-dimensional latent spaces by theoretically and empirically analyzing their probabilistic and geometric behaviors in the Markov chain. We then propose to further explore the critical step for editing in the denoising trajectory that characterizes the convergence of a pre-trained DDM and introduce an automatic search method. Last but not least, in contrast to the conventional understanding that DDMs have relatively poor semantic behaviors, we prove that the critical latent space we found already exhibits semantic subspace boundaries at the generic level in unconditional DDMs, which allows us to do controllable manipulation by guiding the denoising trajectory towards the targeted boundary via a single-step operation. We conduct extensive experiments on multiple DPMs architectures (DDPM, iDDPM) and datasets (CelebA, CelebA-HQ, LSUN-church, LSUN-bedroom, AFHQ-dog) with different resolutions (64, 256), achieving superior or state-of-the-art performance in various task scenarios (image semantic editing, text-based editing, unconditional semantic control) to demonstrate the effectiveness. Project page at https://l-yezhu.github.io/BoundaryDiffusion/ .},
	language = {en},
	urldate = {2024-05-27},
	publisher = {arXiv},
	author = {Zhu, Ye and Wu, Yu and Deng, Zhiwei and Russakovsky, Olga and Yan, Yan},
	month = oct,
	year = {2023},
	note = {arXiv:2302.08357 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, latent direction},
}

@misc{kwon_diffusion_2023,
	title = {Diffusion {Models} already have a {Semantic} {Latent} {Space}},
	url = {http://arxiv.org/abs/2210.10960},
	doi = {10.48550/arXiv.2210.10960},
	abstract = {Diffusion models achieve outstanding generative performance in various domains. Despite their great success, they lack semantic latent space which is essential for controlling the generative process. To address the problem, we propose asymmetric reverse process (Asyrp) which discovers the semantic latent space in frozen pretrained diffusion models. Our semantic latent space, named h-space, has nice properties for accommodating semantic image manipulation: homogeneity, linearity, robustness, and consistency across timesteps. In addition, we introduce a principled design of the generative process for versatile editing and quality boost ing by quantifiable measures: editing strength of an interval and quality deficiency at a timestep. Our method is applicable to various architectures (DDPM++, iD- DPM, and ADM) and datasets (CelebA-HQ, AFHQ-dog, LSUN-church, LSUN- bedroom, and METFACES). Project page: https://kwonminki.github.io/Asyrp/},
	urldate = {2024-05-23},
	publisher = {arXiv},
	author = {Kwon, Mingi and Jeong, Jaeseok and Uh, Youngjung},
	month = mar,
	year = {2023},
	note = {arXiv:2210.10960 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, latent direction},
}

@misc{ronneberger_u-net_2015,
	title = {U-{Net}: {Convolutional} {Networks} for {Biomedical} {Image} {Segmentation}},
	shorttitle = {U-{Net}},
	url = {http://arxiv.org/abs/1505.04597},
	doi = {10.48550/arXiv.1505.04597},
	abstract = {There is large consent that successful training of deep networks requires many thousand annotated training samples. In this paper, we present a network and training strategy that relies on the strong use of data augmentation to use the available annotated samples more efficiently. The architecture consists of a contracting path to capture context and a symmetric expanding path that enables precise localization. We show that such a network can be trained end-to-end from very few images and outperforms the prior best method (a sliding-window convolutional network) on the ISBI challenge for segmentation of neuronal structures in electron microscopic stacks. Using the same network trained on transmitted light microscopy images (phase contrast and DIC) we won the ISBI cell tracking challenge 2015 in these categories by a large margin. Moreover, the network is fast. Segmentation of a 512x512 image takes less than a second on a recent GPU. The full implementation (based on Caffe) and the trained networks are available at http://lmb.informatik.uni-freiburg.de/people/ronneber/u-net .},
	urldate = {2024-08-22},
	publisher = {arXiv},
	author = {Ronneberger, Olaf and Fischer, Philipp and Brox, Thomas},
	month = may,
	year = {2015},
	note = {arXiv:1505.04597 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}

@inproceedings{vaswani_attention_2017,
	title = {Attention is {All} you {Need}},
	volume = {30},
	url = {https://proceedings.neurips.cc/paper_files/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html},
	abstract = {The dominant sequence transduction models are based on complex recurrent orconvolutional neural networks in an encoder and decoder configuration. The best performing such models also connect the encoder and decoder through an attentionm echanisms.  We propose a novel, simple network architecture based solely onan attention mechanism, dispensing with recurrence and convolutions entirely.Experiments on two machine translation tasks show these models to be superiorin quality while being more parallelizable and requiring significantly less timeto train. Our single model with 165 million parameters, achieves 27.5 BLEU onEnglish-to-German translation, improving over the existing best ensemble result by over 1 BLEU. On English-to-French translation, we outperform the previoussingle state-of-the-art with model by 0.7 BLEU, achieving a BLEU score of 41.1.},
	urldate = {2023-05-03},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, Łukasz and Polosukhin, Illia},
	year = {2017},
}

@misc{peebles_scalable_2023,
	title = {Scalable {Diffusion} {Models} with {Transformers}},
	url = {http://arxiv.org/abs/2212.09748},
	doi = {10.48550/arXiv.2212.09748},
	abstract = {We explore a new class of diffusion models based on the transformer architecture. We train latent diffusion models of images, replacing the commonly-used U-Net backbone with a transformer that operates on latent patches. We analyze the scalability of our Diffusion Transformers (DiTs) through the lens of forward pass complexity as measured by Gflops. We find that DiTs with higher Gflops -- through increased transformer depth/width or increased number of input tokens -- consistently have lower FID. In addition to possessing good scalability properties, our largest DiT-XL/2 models outperform all prior diffusion models on the class-conditional ImageNet 512x512 and 256x256 benchmarks, achieving a state-of-the-art FID of 2.27 on the latter.},
	urldate = {2024-08-19},
	publisher = {arXiv},
	author = {Peebles, William and Xie, Saining},
	month = mar,
	year = {2023},
	note = {arXiv:2212.09748 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Transformer},
}

@misc{sohl-dickstein_deep_2015,
	title = {Deep {Unsupervised} {Learning} using {Nonequilibrium} {Thermodynamics}},
	url = {http://arxiv.org/abs/1503.03585},
	doi = {10.48550/arXiv.1503.03585},
	abstract = {A central problem in machine learning involves modeling complex data-sets using highly flexible families of probability distributions in which learning, sampling, inference, and evaluation are still analytically or computationally tractable. Here, we develop an approach that simultaneously achieves both flexibility and tractability. The essential idea, inspired by non-equilibrium statistical physics, is to systematically and slowly destroy structure in a data distribution through an iterative forward diffusion process. We then learn a reverse diffusion process that restores structure in data, yielding a highly flexible and tractable generative model of the data. This approach allows us to rapidly learn, sample from, and evaluate probabilities in deep generative models with thousands of layers or time steps, as well as to compute conditional and posterior probabilities under the learned model. We additionally release an open source reference implementation of the algorithm.},
	urldate = {2024-11-07},
	publisher = {arXiv},
	author = {Sohl-Dickstein, Jascha and Weiss, Eric A. and Maheswaranathan, Niru and Ganguli, Surya},
	month = nov,
	year = {2015},
	note = {arXiv:1503.03585},
	keywords = {Computer Science - Machine Learning, Condensed Matter - Disordered Systems and Neural Networks, Quantitative Biology - Neurons and Cognition, Statistics - Machine Learning},
}

@misc{ho_denoising_2020,
	title = {Denoising {Diffusion} {Probabilistic} {Models}},
	url = {http://arxiv.org/abs/2006.11239},
	doi = {10.48550/arXiv.2006.11239},
	abstract = {We present high quality image synthesis results using diffusion probabilistic models, a class of latent variable models inspired by considerations from nonequilibrium thermodynamics. Our best results are obtained by training on a weighted variational bound designed according to a novel connection between diffusion probabilistic models and denoising score matching with Langevin dynamics, and our models naturally admit a progressive lossy decompression scheme that can be interpreted as a generalization of autoregressive decoding. On the unconditional CIFAR10 dataset, we obtain an Inception score of 9.46 and a state-of-the-art FID score of 3.17. On 256x256 LSUN, we obtain sample quality similar to ProgressiveGAN. Our implementation is available at https://github.com/hojonathanho/diffusion},
	urldate = {2024-03-04},
	publisher = {arXiv},
	author = {Ho, Jonathan and Jain, Ajay and Abbeel, Pieter},
	month = dec,
	year = {2020},
	note = {arXiv:2006.11239 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@misc{noauthor_art_nodate,
	title = {Art walks in latent spaces},
	url = {https://www.goethe.de/prj/k40/en/kun/lat.html},
	abstract = {At the Los Angeles-based Refik Anadol Studio, artists, architects and data scientists collaborate with machine intelligence to rethink the layered meanings of consciousness and space. {\textless}a target="\_blank" href="../../en/net/ppp.html"{\textgreater}Pelin Kivrak{\textless}/a{\textgreater}{\textless}a target="\_self" href="../../en/net/ppp.html"{\textgreater},{\textless}/a{\textgreater} the concept developer at RAS, gives an insight into the Studio\&rsquo;s recent work.},
	language = {en},
	urldate = {2024-11-07},
	journal = {@GI\_weltweit},
}

@article{shen_interfacegan_2022,
	title = {{InterFaceGAN}: {Interpreting} the {Disentangled} {Face} {Representation} {Learned} by {GANs}},
	volume = {44},
	issn = {1939-3539},
	shorttitle = {{InterFaceGAN}},
	doi = {10.1109/TPAMI.2020.3034267},
	abstract = {Although generative adversarial networks (GANs) have made significant progress in face synthesis, there lacks enough understanding of what GANs have learned in the latent representation to map a random code to a photo-realistic image. In this work, we propose a framework called InterFaceGAN to interpret the disentangled face representation learned by the state-of-the-art GAN models and study the properties of the facial semantics encoded in the latent space. We first find that GANs learn various semantics in some linear subspaces of the latent space. After identifying these subspaces, we can realistically manipulate the corresponding facial attributes without retraining the model. We then conduct a detailed study on the correlation between different semantics and manage to better disentangle them via subspace projection, resulting in more precise control of the attribute manipulation. Besides manipulating the gender, age, expression, and presence of eyeglasses, we can even alter the face pose and fix the artifacts accidentally made by GANs. Furthermore, we perform an in-depth face identity analysis and a layer-wise analysis to evaluate the editing results quantitatively. Finally, we apply our approach to real face editing by employing GAN inversion approaches and explicitly training feed-forward models based on the synthetic data established by InterFaceGAN. Extensive experimental results suggest that learning to synthesize faces spontaneously brings a disentangled and controllable face representation.},
	number = {4},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	author = {Shen, Yujun and Yang, Ceyuan and Tang, Xiaoou and Zhou, Bolei},
	month = apr,
	year = {2022},
	note = {Conference Name: IEEE Transactions on Pattern Analysis and Machine Intelligence},
	keywords = {Aerospace electronics, Faces, Facial features, Gallium nitride, Generative adversarial network, Generative adversarial networks, Generators, Semantics, disentanglement, explainable artificial intelligence, face editing, interpretability, notion},
	pages = {2004--2018},
}

@inproceedings{higgins_beta-vae_2016,
	title = {beta-{VAE}: {Learning} {Basic} {Visual} {Concepts} with a {Constrained} {Variational} {Framework}},
	shorttitle = {beta-{VAE}},
	url = {https://openreview.net/forum?id=Sy2fzU9gl},
	abstract = {Learning an interpretable factorised representation of the independent data generative factors of the world without supervision is an important precursor for the development of artificial intelligence that is able to learn and reason in the same way that humans do. We introduce beta-VAE, a new state-of-the-art framework for automated discovery of interpretable factorised latent representations from raw image data in a completely unsupervised manner. Our approach is a modification of the variational autoencoder (VAE) framework. We introduce an adjustable hyperparameter beta that balances latent channel capacity and independence constraints with reconstruction accuracy. We demonstrate that beta-VAE with appropriately tuned beta {\textgreater} 1 qualitatively outperforms VAE (beta = 1), as well as state of the art unsupervised (InfoGAN) and semi-supervised (DC-IGN) approaches to disentangled factor learning on a variety of datasets (celebA, faces and chairs). Furthermore, we devise a protocol to quantitatively compare the degree of disentanglement learnt by different models, and show that our approach also significantly outperforms all baselines quantitatively. Unlike InfoGAN, beta-VAE is stable to train, makes few assumptions about the data and relies on tuning a single hyperparameter, which can be directly optimised through a hyper parameter search using weakly labelled data or through heuristic visual inspection for purely unsupervised data.},
	language = {en},
	urldate = {2023-06-23},
	author = {Higgins, Irina and Matthey, Loic and Pal, Arka and Burgess, Christopher and Glorot, Xavier and Botvinick, Matthew and Mohamed, Shakir and Lerchner, Alexander},
	month = nov,
	year = {2016},
}

@misc{kingma_auto-encoding_2022,
	title = {Auto-{Encoding} {Variational} {Bayes}},
	url = {http://arxiv.org/abs/1312.6114},
	doi = {10.48550/arXiv.1312.6114},
	abstract = {How can we perform efficient inference and learning in directed probabilistic models, in the presence of continuous latent variables with intractable posterior distributions, and large datasets? We introduce a stochastic variational inference and learning algorithm that scales to large datasets and, under some mild differentiability conditions, even works in the intractable case. Our contributions are two-fold. First, we show that a reparameterization of the variational lower bound yields a lower bound estimator that can be straightforwardly optimized using standard stochastic gradient methods. Second, we show that for i.i.d. datasets with continuous latent variables per datapoint, posterior inference can be made especially efficient by fitting an approximate inference model (also called a recognition model) to the intractable posterior using the proposed lower bound estimator. Theoretical advantages are reflected in experimental results.},
	urldate = {2024-03-04},
	publisher = {arXiv},
	author = {Kingma, Diederik P. and Welling, Max},
	month = dec,
	year = {2022},
	note = {arXiv:1312.6114 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@inproceedings{karras_style-based_2019,
	title = {A {Style}-{Based} {Generator} {Architecture} for {Generative} {Adversarial} {Networks}},
	url = {https://openaccess.thecvf.com/content_CVPR_2019/html/Karras_A_Style-Based_Generator_Architecture_for_Generative_Adversarial_Networks_CVPR_2019_paper.html},
	urldate = {2023-05-03},
	author = {Karras, Tero and Laine, Samuli and Aila, Timo},
	year = {2019},
	keywords = {notion},
	pages = {4401--4410},
}

@inproceedings{rombach_high-resolution_2022,
	title = {High-{Resolution} {Image} {Synthesis} {With} {Latent} {Diffusion} {Models}},
	url = {https://openaccess.thecvf.com/content/CVPR2022/html/Rombach_High-Resolution_Image_Synthesis_With_Latent_Diffusion_Models_CVPR_2022_paper.html},
	language = {en},
	urldate = {2024-03-12},
	author = {Rombach, Robin and Blattmann, Andreas and Lorenz, Dominik and Esser, Patrick and Ommer, Björn},
	year = {2022},
	pages = {10684--10695},
}

@misc{ramesh_hierarchical_2022,
	title = {Hierarchical {Text}-{Conditional} {Image} {Generation} with {CLIP} {Latents}},
	url = {http://arxiv.org/abs/2204.06125},
	abstract = {Contrastive models like CLIP have been shown to learn robust representations of images that capture both semantics and style. To leverage these representations for image generation, we propose a two-stage model: a prior that generates a CLIP image embedding given a text caption, and a decoder that generates an image conditioned on the image embedding. We show that explicitly generating image representations improves image diversity with minimal loss in photorealism and caption similarity. Our decoders conditioned on image representations can also produce variations of an image that preserve both its semantics and style, while varying the non-essential details absent from the image representation. Moreover, the joint embedding space of CLIP enables language-guided image manipulations in a zero-shot fashion. We use diffusion models for the decoder and experiment with both autoregressive and diffusion models for the prior, finding that the latter are computationally more efficient and produce higher-quality samples.},
	urldate = {2024-11-07},
	publisher = {arXiv},
	author = {Ramesh, Aditya and Dhariwal, Prafulla and Nichol, Alex and Chu, Casey and Chen, Mark},
	month = apr,
	year = {2022},
	note = {arXiv:2204.06125},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}

@misc{song_denoising_2022,
	title = {Denoising {Diffusion} {Implicit} {Models}},
	url = {http://arxiv.org/abs/2010.02502},
	doi = {10.48550/arXiv.2010.02502},
	abstract = {Denoising diffusion probabilistic models (DDPMs) have achieved high quality image generation without adversarial training, yet they require simulating a Markov chain for many steps to produce a sample. To accelerate sampling, we present denoising diffusion implicit models (DDIMs), a more efficient class of iterative implicit probabilistic models with the same training procedure as DDPMs. In DDPMs, the generative process is defined as the reverse of a Markovian diffusion process. We construct a class of non-Markovian diffusion processes that lead to the same training objective, but whose reverse process can be much faster to sample from. We empirically demonstrate that DDIMs can produce high quality samples \$10 {\textbackslash}times\$ to \$50 {\textbackslash}times\$ faster in terms of wall-clock time compared to DDPMs, allow us to trade off computation for sample quality, and can perform semantically meaningful image interpolation directly in the latent space.},
	urldate = {2024-05-27},
	publisher = {arXiv},
	author = {Song, Jiaming and Meng, Chenlin and Ermon, Stefano},
	month = oct,
	year = {2022},
	note = {arXiv:2010.02502 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, DDIM},
}

@article{hertz2022prompt,
  title={Prompt-to-prompt image editing with cross attention control},
  author={Hertz, Amir and Mokady, Ron and Tenenbaum, Jay and Aberman, Kfir and Pritch, Yael and Cohen-Or, Daniel},
  journal={arXiv preprint arXiv:2208.01626},
  year={2022}
}

@inproceedings{tumanyan2023plug,
  title={Plug-and-play diffusion features for text-driven image-to-image translation},
  author={Tumanyan, Narek and Geyer, Michal and Bagon, Shai and Dekel, Tali},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={1921--1930},
  year={2023}
}

@article{meng2021sdedit,
  title={Sdedit: Guided image synthesis and editing with stochastic differential equations},
  author={Meng, Chenlin and He, Yutong and Song, Yang and Song, Jiaming and Wu, Jiajun and Zhu, Jun-Yan and Ermon, Stefano},
  journal={arXiv preprint arXiv:2108.01073},
  year={2021}
}

@inproceedings{cao2023masactrl,
  title={Masactrl: Tuning-free mutual self-attention control for consistent image synthesis and editing},
  author={Cao, Mingdeng and Wang, Xintao and Qi, Zhongang and Shan, Ying and Qie, Xiaohu and Zheng, Yinqiang},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={22560--22570},
  year={2023}
}

@inproceedings{brooks2023instructpix2pix,
  title={Instructpix2pix: Learning to follow image editing instructions},
  author={Brooks, Tim and Holynski, Aleksander and Efros, Alexei A},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={18392--18402},
  year={2023}
}

@inproceedings{park2024shape,
  title={Shape-guided diffusion with inside-outside attention},
  author={Park, Dong Huk and Luo, Grace and Toste, Clayton and Azadi, Samaneh and Liu, Xihui and Karalashvili, Maka and Rohrbach, Anna and Darrell, Trevor},
  booktitle={Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision},
  pages={4198--4207},
  year={2024}
}

@inproceedings{yang_diffusion_2023,
	title = {Diffusion {Model} as {Representation} {Learner}},
	url = {https://openaccess.thecvf.com/content/ICCV2023/html/Yang_Diffusion_Model_as_Representation_Learner_ICCV_2023_paper.html},
	language = {en},
	urldate = {2024-05-27},
	author = {Yang, Xingyi and Wang, Xinchao},
	year = {2023},
	pages = {18938--18949},
}

@misc{schuhmann_laion-5b_2022,
	title = {{LAION}-{5B}: {An} open large-scale dataset for training next generation image-text models},
	shorttitle = {{LAION}-{5B}},
	url = {http://arxiv.org/abs/2210.08402},
	doi = {10.48550/arXiv.2210.08402},
	abstract = {Groundbreaking language-vision architectures like CLIP and DALL-E proved the utility of training on large amounts of noisy image-text data, without relying on expensive accurate labels used in standard vision unimodal supervised learning. The resulting models showed capabilities of strong text-guided image generation and transfer to downstream tasks, while performing remarkably at zero-shot classification with noteworthy out-of-distribution robustness. Since then, large-scale language-vision models like ALIGN, BASIC, GLIDE, Flamingo and Imagen made further improvements. Studying the training and capabilities of such models requires datasets containing billions of image-text pairs. Until now, no datasets of this size have been made openly available for the broader research community. To address this problem and democratize research on large-scale multi-modal models, we present LAION-5B - a dataset consisting of 5.85 billion CLIP-filtered image-text pairs, of which 2.32B contain English language. We show successful replication and fine-tuning of foundational models like CLIP, GLIDE and Stable Diffusion using the dataset, and discuss further experiments enabled with an openly available dataset of this scale. Additionally we provide several nearest neighbor indices, an improved web-interface for dataset exploration and subset generation, and detection scores for watermark, NSFW, and toxic content detection. Announcement page https://laion.ai/laion-5b-a-new-era-of-open-large-scale-multi-modal-datasets/},
	urldate = {2024-11-14},
	publisher = {arXiv},
	author = {Schuhmann, Christoph and Beaumont, Romain and Vencu, Richard and Gordon, Cade and Wightman, Ross and Cherti, Mehdi and Coombes, Theo and Katta, Aarush and Mullis, Clayton and Wortsman, Mitchell and Schramowski, Patrick and Kundurthy, Srivatsa and Crowson, Katherine and Schmidt, Ludwig and Kaczmarczyk, Robert and Jitsev, Jenia},
	month = oct,
	year = {2022},
	note = {arXiv:2210.08402},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
}

@inproceedings{hendrycks2021many,
  title={The many faces of robustness: A critical analysis of out-of-distribution generalization},
  author={Hendrycks, Dan and Basart, Steven and Mu, Norman and Kadavath, Saurav and Wang, Frank and Dorundo, Evan and Desai, Rahul and Zhu, Tyler and Parajuli, Samyak and Guo, Mike and others},
  booktitle={Proceedings of the IEEE/CVF international conference on computer vision},
  pages={8340--8349},
  year={2021}
}

@inproceedings{zhang2018unreasonable,
  title={The unreasonable effectiveness of deep features as a perceptual metric},
  author={Zhang, Richard and Isola, Phillip and Efros, Alexei A and Shechtman, Eli and Wang, Oliver},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={586--595},
  year={2018}
}

@inproceedings{radford2021learning,
  title={Learning transferable visual models from natural language supervision},
  author={Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others},
  booktitle={International conference on machine learning},
  pages={8748--8763},
  year={2021},
  organization={PMLR}
}


@inproceedings{caron2021emerging,
  title={Emerging properties in self-supervised vision transformers},
  author={Caron, Mathilde and Touvron, Hugo and Misra, Ishan and J{\'e}gou, Herv{\'e} and Mairal, Julien and Bojanowski, Piotr and Joulin, Armand},
  booktitle={Proceedings of the IEEE/CVF international conference on computer vision},
  pages={9650--9660},
  year={2021}
}

@misc{podell_sdxl_2023,
	title = {{SDXL}: {Improving} {Latent} {Diffusion} {Models} for {High}-{Resolution} {Image} {Synthesis}},
	shorttitle = {{SDXL}},
	url = {http://arxiv.org/abs/2307.01952},
	doi = {10.48550/arXiv.2307.01952},
	abstract = {We present SDXL, a latent diffusion model for text-to-image synthesis. Compared to previous versions of Stable Diffusion, SDXL leverages a three times larger UNet backbone: The increase of model parameters is mainly due to more attention blocks and a larger cross-attention context as SDXL uses a second text encoder. We design multiple novel conditioning schemes and train SDXL on multiple aspect ratios. We also introduce a refinement model which is used to improve the visual fidelity of samples generated by SDXL using a post-hoc image-to-image technique. We demonstrate that SDXL shows drastically improved performance compared the previous versions of Stable Diffusion and achieves results competitive with those of black-box state-of-the-art image generators. In the spirit of promoting open research and fostering transparency in large model training and evaluation, we provide access to code and model weights at https://github.com/Stability-AI/generative-models},
	urldate = {2024-11-18},
	publisher = {arXiv},
	author = {Podell, Dustin and English, Zion and Lacey, Kyle and Blattmann, Andreas and Dockhorn, Tim and Müller, Jonas and Penna, Joe and Rombach, Robin},
	month = jul,
	year = {2023},
	note = {arXiv:2307.01952},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computer Vision and Pattern Recognition},
}

@misc{noauthor_stable_2022,
	title = {Stable {Diffusion} 1 vs 2 - {What} you need to know},
	url = {https://www.assemblyai.com/blog/stable-diffusion-1-vs-2-what-you-need-to-know/},
	abstract = {Stable Diffusion 2 was released recently, sparking some debate about its performance relative to Stable Diffusion 1. Learn where the differences between the two models stem from and what they mean in practice in this simple guide.},
	language = {en},
	urldate = {2024-11-21},
	journal = {News, Tutorials, AI Research},
	month = dec,
	year = {2022},
}
