@article{lin2023medical,
  title={Medical visual question answering: A survey},
  author={Lin, Zhihong and Zhang, Donghao and Tao, Qingyi and Shi, Danli and Haffari, Gholamreza and Wu, Qi and He, Mingguang and Ge, Zongyuan},
  journal={Artificial Intelligence in Medicine},
  pages={102611},
  year={2023},
  publisher={Elsevier}
}
@article{lee2020changes,
  title={Changes in chest X-ray findings in 1-and 2-month group after treatment initiation for suspected pulmonary tuberculosis},
  author={Lee, Jang Ho and Kim, Ock-Hwa and Kim, Yeon Joo and Shim, Tae Sun and Jo, Kyung-Wook},
  journal={The Korean Journal of Internal Medicine},
  volume={35},
  number={5},
  pages={1145},
  year={2020},
  publisher={Korean Association of Internal Medicine}
}
@inproceedings{eslami2023pubmedclip,
  title={PubMedCLIP: How Much Does CLIP Benefit Visual Question Answering in the Medical Domain?},
  author={Eslami, Sedigheh and Meinel, Christoph and De Melo, Gerard},
  booktitle={Findings of the Association for Computational Linguistics: EACL 2023},
  pages={1151--1163},
  year={2023}
}

@inproceedings{do2021multiple,
  title={Multiple meta-model quantifying for medical visual question answering},
  author={Do, Tuong and Nguyen, Binh X and Tjiputra, Erman and Tran, Minh and Tran, Quang D and Nguyen, Anh},
  booktitle={Medical Image Computing and Computer Assisted Intervention--MICCAI 2021: 24th International Conference, Strasbourg, France, September 27--October 1, 2021, Proceedings, Part V 24},
  pages={64--74},
  year={2021},
  organization={Springer}
}
@article{bao2022vlmo,
  title={Vlmo: Unified vision-language pre-training with mixture-of-modality-experts},
  author={Bao, Hangbo and Wang, Wenhui and Dong, Li and Liu, Qiang and Mohammed, Owais Khan and Aggarwal, Kriti and Som, Subhojit and Piao, Songhao and Wei, Furu},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={32897--32912},
  year={2022}
}
@article{chen2022pali,
  title={Pali: A jointly-scaled multilingual language-image model},
  author={Chen, Xi and Wang, Xiao and Changpinyo, Soravit and Piergiovanni, AJ and Padlewski, Piotr and Salz, Daniel and Goodman, Sebastian and Grycner, Adam and Mustafa, Basil and Beyer, Lucas and others},
  journal={arXiv preprint arXiv:2209.06794},
  year={2022}
}
@article{ordonez2011im2text,
  title={Im2text: Describing images using 1 million captioned photographs},
  author={Ordonez, Vicente and Kulkarni, Girish and Berg, Tamara},
  journal={Advances in neural information processing systems},
  volume={24},
  year={2011}
}
@inproceedings{changpinyo2021conceptual,
  title={Conceptual 12m: Pushing web-scale image-text pre-training to recognize long-tail visual concepts},
  author={Changpinyo, Soravit and Sharma, Piyush and Ding, Nan and Soricut, Radu},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={3558--3568},
  year={2021}
}
@article{sennrich2015neural,
  title={Neural machine translation of rare words with subword units},
  author={Sennrich, Rico and Haddow, Barry and Birch, Alexandra},
  journal={arXiv preprint arXiv:1508.07909},
  year={2015}
}
@inproceedings{lin2014microsoft,
  title={Microsoft coco: Common objects in context},
  author={Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\'a}r, Piotr and Zitnick, C Lawrence},
  booktitle={Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13},
  pages={740--755},
  year={2014},
  organization={Springer}
}
@inproceedings{deng2009imagenet,
  title={Imagenet: A large-scale hierarchical image database},
  author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Li, Kai and Fei-Fei, Li},
  booktitle={2009 IEEE conference on computer vision and pattern recognition},
  pages={248--255},
  year={2009},
  organization={Ieee}
}
@article{krizhevsky2009learning,
  title={Learning multiple layers of features from tiny images},
  author={Krizhevsky, Alex and Hinton, Geoffrey and others},
  year={2009},
  publisher={Toronto, ON, Canada}
}
@article{yuan2022change,
  title={Change detection meets visual question answering},
  author={Yuan, Zhenghang and Mou, Lichao and Xiong, Zhitong and Zhu, Xiao Xiang},
  journal={IEEE Transactions on Geoscience and Remote Sensing},
  volume={60},
  pages={1--13},
  year={2022},
  publisher={IEEE}
}
@inproceedings{moor2023med,
  title={Med-flamingo: a multimodal medical few-shot learner},
  author={Moor, Michael and Huang, Qian and Wu, Shirley and Yasunaga, Michihiro and Dalmia, Yash and Leskovec, Jure and Zakka, Cyril and Reis, Eduardo Pontes and Rajpurkar, Pranav},
  booktitle={Machine Learning for Health (ML4H)},
  pages={353--367},
  year={2023},
  organization={PMLR}
}
@article{li2023llava,
  title={Llava-med: Training a large language-and-vision assistant for biomedicine in one day},
  author={Li, Chunyuan and Wong, Cliff and Zhang, Sheng and Usuyama, Naoto and Liu, Haotian and Yang, Jianwei and Naumann, Tristan and Poon, Hoifung and Gao, Jianfeng},
  journal={arXiv preprint arXiv:2306.00890},
  year={2023}
}
@inproceedings{bigham2010vizwiz,
  title={Vizwiz: nearly real-time answers to visual questions},
  author={Bigham, Jeffrey P and Jayant, Chandrika and Ji, Hanjie and Little, Greg and Miller, Andrew and Miller, Robert C and Miller, Robin and Tatarowicz, Aubrey and White, Brandyn and White, Samual and others},
  booktitle={Proceedings of the 23nd annual ACM symposium on User interface software and technology},
  pages={333--342},
  year={2010}
}
@article{liu2023visual,
  title={Visual instruction tuning},
  author={Liu, Haotian and Li, Chunyuan and Wu, Qingyang and Lee, Yong Jae},
  journal={arXiv preprint arXiv:2304.08485},
  year={2023}
}
@article{openai2023gpt4,
  title={GPT-4 Technical Report},
  author={OpenAI},
  year={2023}
}

@article{heyckendorf2022tuberculosis, title={Tuberculosis treatment monitoring and outcome measures: new interest and new strategies}, author={Heyckendorf, Jan and Georghiou, Sophia B and Frahm, Nicole and Heinrich, Norbert and Kontsevaya, Irina and Reimann, Maja and Holtzman, David and Imperial, Marjorie and Cirillo, Daniela M and Gillespie, Stephen H and others}, journal={Clinical microbiology reviews}, volume={35}, number={3}, pages={e00227--21}, year={2022}, publisher={Am Soc Microbiol} } 
@article{callister2015british,
  title={British Thoracic Society guidelines for the investigation and management of pulmonary nodules: accredited by NICE},
  author={Callister, MEJ and Baldwin, DR and Akram, AR and Barnard, S and Cane, P and Draffan, J and Franks, K and Gleeson, F and Graham, R and Malhotra, P and others},
  journal={Thorax},
  volume={70},
  number={Suppl 2},
  pages={ii1--ii54},
  year={2015},
  publisher={BMJ Publishing Group Ltd}
}
@article{dai2021coatnet,
  title={Coatnet: Marrying convolution and attention for all data sizes},
  author={Dai, Zihang and Liu, Hanxiao and Le, Quoc V and Tan, Mingxing},
  journal={Advances in neural information processing systems},
  volume={34},
  pages={3965--3977},
  year={2021}
}
@article{wang2021simvlm,
  title={Simvlm: Simple visual language model pretraining with weak supervision},
  author={Wang, Zirui and Yu, Jiahui and Yu, Adams Wei and Dai, Zihang and Tsvetkov, Yulia and Cao, Yuan},
  journal={arXiv preprint arXiv:2108.10904},
  year={2021}
}
@article{raffel2020exploring,
  title={Exploring the limits of transfer learning with a unified text-to-text transformer},
  author={Raffel, Colin and Shazeer, Noam and Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and Li, Wei and Liu, Peter J},
  journal={The Journal of Machine Learning Research},
  volume={21},
  number={1},
  pages={5485--5551},
  year={2020},
  publisher={JMLRORG}
}
@article{tiu2022expert,
  title={Expert-level detection of pathologies from unannotated chest X-ray images via self-supervised learning},
  author={Tiu, Ekin and Talius, Ellie and Patel, Pujan and Langlotz, Curtis P and Ng, Andrew Y and Rajpurkar, Pranav},
  journal={Nature Biomedical Engineering},
  volume={6},
  number={12},
  pages={1399--1406},
  year={2022},
  publisher={Nature Publishing Group UK London}
}
@article{demner2016preparing,
  title={Preparing a collection of radiology examinations for distribution and retrieval},
  author={Demner-Fushman, Dina and Kohli, Marc D and Rosenman, Marc B and Shooshan, Sonya E and Rodriguez, Laritza and Antani, Sameer and Thoma, George R and McDonald, Clement J},
  journal={Journal of the American Medical Informatics Association},
  volume={23},
  number={2},
  pages={304--310},
  year={2016},
  publisher={Oxford University Press}
}
@inproceedings{irvin2019chexpert,
  title={Chexpert: A large chest radiograph dataset with uncertainty labels and expert comparison},
  author={Irvin, Jeremy and Rajpurkar, Pranav and Ko, Michael and Yu, Yifan and Ciurea-Ilcus, Silviana and Chute, Chris and Marklund, Henrik and Haghgoo, Behzad and Ball, Robyn and Shpanskaya, Katie and others},
  booktitle={Proceedings of the AAAI conference on artificial intelligence},
  volume={33},
  number={01},
  pages={590--597},
  year={2019}
}
@article{chen2023towards,
  title={Towards Unifying Medical Vision-and-Language Pre-training via Soft Prompts},
  author={Chen, Zhihong and Diao, Shizhe and Wang, Benyou and Li, Guanbin and Wan, Xiang},
  journal={arXiv preprint arXiv:2302.08958},
  year={2023}
}
@inproceedings{sharma2018conceptual,
  title={Conceptual captions: A cleaned, hypernymed, image alt-text dataset for automatic image captioning},
  author={Sharma, Piyush and Ding, Nan and Goodman, Sebastian and Soricut, Radu},
  booktitle={Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={2556--2565},
  year={2018}
}
@article{oluwasanmi2019fully,
  title={Fully convolutional captionnet: Siamese difference captioning attention model},
  author={Oluwasanmi, Ariyo and Frimpong, Enoch and Aftab, Muhammad Umar and Baagyere, Edward Y and Qin, Zhiguang and Ullah, Kifayat},
  journal={IEEE access},
  volume={7},
  pages={175929--175939},
  year={2019},
  publisher={IEEE}
}
@inproceedings{finn2017model,
  title={Model-agnostic meta-learning for fast adaptation of deep networks},
  author={Finn, Chelsea and Abbeel, Pieter and Levine, Sergey},
  booktitle={International conference on machine learning},
  pages={1126--1135},
  year={2017},
  organization={PMLR}
}
@article{lin2023pmc,
  title={Pmc-clip: Contrastive language-image pre-training using biomedical documents},
  author={Lin, Weixiong and Zhao, Ziheng and Zhang, Xiaoman and Wu, Chaoyi and Zhang, Ya and Wang, Yanfeng and Xie, Weidi},
  journal={arXiv preprint arXiv:2303.07240},
  year={2023}
}
@inproceedings{endo2021retrieval,
  title={Retrieval-based chest x-ray report generation using a pre-trained contrastive language-image model},
  author={Endo, Mark and Krishnan, Rayan and Krishna, Viswesh and Ng, Andrew Y and Rajpurkar, Pranav},
  booktitle={Machine Learning for Health},
  pages={209--219},
  year={2021},
  organization={PMLR}
}
@inproceedings{chen2020generating,
  title={Generating Radiology Reports via Memory-driven Transformer},
  author={Chen, Zhihong and Song, Yan and Chang, Tsung-Hui and Wan, Xiang},
  booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
  pages={1439--1449},
  year={2020}
}

@article{qin2022medical,
  title={Medical image understanding with pretrained vision language models: A comprehensive study},
  author={Qin, Ziyuan and Yi, Huahui and Lao, Qicheng and Li, Kang},
  journal={arXiv preprint arXiv:2209.15517},
  year={2022}
}
@article{wu2021chest,
  title={Chest ImaGenome dataset for clinical reasoning},
  author={Wu, Joy T and Agu, Nkechinyere N and Lourentzou, Ismini and Sharma, Arjun and Paguio, Joseph A and Yao, Jasper S and Dee, Edward C and Mitchell, William and Kashyap, Satyananda and Giovannini, Andrea and others},
  journal={arXiv preprint arXiv:2108.00316},
  year={2021}
}
@inproceedings{hou2023recap,
  title={RECAP: Towards Precise Radiology Report Generation via Dynamic Disease Progression Reasoning},
  author={Hou, Wenjun and Cheng, Yi and Xu, Kaishuai and Li, Wenjie and Liu, Jiang},
  booktitle={Findings of the Association for Computational Linguistics: EMNLP 2023},
  pages={2134--2147},
  year={2023}
}
@InProceedings{10.1007/978-3-031-16431-6_55,
author="Karwande, Gaurang
and Mbakwe, Amarachi B.
and Wu, Joy T.
and Celi, Leo A.
and Moradi, Mehdi
and Lourentzou, Ismini",
editor="Wang, Linwei
and Dou, Qi
and Fletcher, P. Thomas
and Speidel, Stefanie
and Li, Shuo",
title="CheXRelNet: An Anatomy-Aware Model for Tracking Longitudinal Relationships Between Chest X-Rays",
booktitle="Medical Image Computing and Computer Assisted Intervention -- MICCAI 2022",
year="2022",
publisher="Springer Nature Switzerland",
address="Cham",
pages="581--591",
abstract="Despite the progress in utilizing deep learning to automate chest radiograph interpretation and disease diagnosis tasks, change between sequential Chest X-rays (CXRs) has received limited attention. Monitoring the progression of pathologies that are visualized through chest imaging poses several challenges in anatomical motion estimation and image registration, i.e., spatially aligning the two images and modeling temporal dynamics in change detection. In this work, we propose CheXRelNet, a neural model that can track longitudinal pathology change relations between two CXRs. CheXRelNet incorporates local and global visual features, utilizes inter-image and intra-image anatomical information, and learns dependencies between anatomical region attributes, to accurately predict disease change for a pair of CXRs. Experimental results on the Chest ImaGenome dataset show increased downstream performance compared to baselines. Code is available at https://github.com/PLAN-Lab/ChexRelNet.",
isbn="978-3-031-16431-6"
}


@InProceedings{10.1007/978-3-031-43904-9_66,
author="Mbakwe, Amarachi B.
and Wang, Lyuyang
and Moradi, Mehdi
and Lourentzou, Ismini",
editor="Greenspan, Hayit
and Madabhushi, Anant
and Mousavi, Parvin
and Salcudean, Septimiu
and Duncan, James
and Syeda-Mahmood, Tanveer
and Taylor, Russell",
title="Hierarchical Vision Transformers for Disease Progression Detection in Chest X-Ray Images",
booktitle="Medical Image Computing and Computer Assisted Intervention -- MICCAI 2023",
year="2023",
publisher="Springer Nature Switzerland",
address="Cham",
pages="685--695",
abstract="Chest radiography is a commonly used diagnostic imaging exam for monitoring disease progression and treatment effectiveness. While machine learning has made significant strides in tasks such as image segmentation, disease diagnosis, and automatic report generation, more intricate tasks such as disease progression monitoring remain fairly underexplored. This task presents a formidable challenge because of the complex and intricate nature of disease appearances on chest X-ray images, which makes distinguishing significant changes from irrelevant variations between images challenging. Motivated by these challenges, this work proposes CheXRelFormer, an end-to-end siamese Transformer disease progression model that takes a pair of images as input and detects whether the patient's condition has improved, worsened, or remained unchanged. The model comprises two hierarchical Transformer encoders, a difference module that compares feature differences across images, and a final classification layer that predicts the change in the patient's condition. Experimental results demonstrate that CheXRelFormer outperforms previous counterparts. Code is available at https://github.com/PLAN-Lab/CheXRelFormer.",
isbn="978-3-031-43904-9"
}


@InProceedings{10.1007/978-3-031-43904-9_19,
author="Zhu, Qingqing
and Mathai, Tejas Sudharshan
and Mukherjee, Pritam
and Peng, Yifan
and Summers, Ronald M.
and Lu, Zhiyong",
editor="Greenspan, Hayit
and Madabhushi, Anant
and Mousavi, Parvin
and Salcudean, Septimiu
and Duncan, James
and Syeda-Mahmood, Tanveer
and Taylor, Russell",
title="Utilizing Longitudinal Chest X-Rays and Reports to Pre-fill Radiology Reports",
booktitle="Medical Image Computing and Computer Assisted Intervention -- MICCAI 2023",
year="2023",
publisher="Springer Nature Switzerland",
address="Cham",
pages="189--198",
abstract="Despite the reduction in turn-around times in radiology reporting with the use of speech recognition software, persistent communication errors can significantly impact the interpretation of radiology reports. Pre-filling a radiology report holds promise in mitigating reporting errors, and despite multiple efforts in literature to generate comprehensive medical reports, there lacks approaches that exploit the longitudinal nature of patient visit records in the MIMIC-CXR dataset. To address this gap, we propose to use longitudinal multi-modal data, i.e., previous patient visit CXR, current visit CXR, and the previous visit report, to pre-fill the ``findings'' section of the patient's current visit. We first gathered the longitudinal visit information for 26,625 patients from the MIMIC-CXR dataset, and created a new dataset called Longitudinal-MIMIC. With this new dataset, a transformer-based model was trained to capture the multi-modal longitudinal information from patient visit records (CXR images + reports) via a cross-attention-based multi-modal fusion module and a hierarchical memory-driven decoder. In contrast to previous works that only uses current visit data as input to train a model, our work exploits the longitudinal information available to pre-fill the ``findings'' section of radiology reports. Experiments show that our approach outperforms several recent approaches by {\$}{\$}{\backslash}ge {\$}{\$}3{\%} on F1 score, and {\$}{\$}{\backslash}ge {\$}{\$}2{\%} for BLEU-4, METEOR and ROUGE-L respectively. Code will be published at https://github.com/CelestialShine/Longitudinal-Chest-X-Ray.",
isbn="978-3-031-43904-9"
}


@article{nicolson2023longitudinal,
  title={Longitudinal Data and a Semantic Similarity Reward for Chest X-Ray Report Generation},
  author={Nicolson, Aaron and Dowling, Jason and Koopman, Bevan},
  journal={arXiv preprint arXiv:2307.09758},
  year={2023}
}
@book{Lamport:Book:1989,
 author = {Lamport, Leslie},
 title = {Latex: A Document Preparation System},
 year = {1986},
 isbn = {0-201-15790-X},
 publisher = {Addison-Wesley Longman Publishing Co., Inc.},
 address = {Boston, MA, USA},
}

@misc{Hinton:arXiv:2015:Distilling,
 title={Distilling the knowledge in a neural network},
 author={Hinton, Geoffrey and Vinyals, Oriol and Dean, Jeff},
 OPTeprinttype={arXiv},
 OPTeprint={1503.02531},
 howpublished={Eprint \href{http://arxiv.org/abs/1503.02531}{arXiv:1503.02531}},
 OPTurl={http://arxiv.org/abs/1503.02531},
 year=2015
}
@inproceedings{radford2021learning,
  title={Learning transferable visual models from natural language supervision},
  author={Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others},
  booktitle={International conference on machine learning},
  pages={8748--8763},
  year={2021},
  organization={PMLR}
}
@inproceedings{wang2022ofa,
  title={Ofa: Unifying architectures, tasks, and modalities through a simple sequence-to-sequence learning framework},
  author={Wang, Peng and Yang, An and Men, Rui and Lin, Junyang and Bai, Shuai and Li, Zhikang and Ma, Jianxin and Zhou, Chang and Zhou, Jingren and Yang, Hongxia},
  booktitle={International Conference on Machine Learning},
  pages={23318--23340},
  year={2022},
  organization={PMLR}
}
@inproceedings{boecking2022making,
  title={Making the most of text semantics to improve biomedical vision--language processing},
  author={Boecking, Benedikt and Usuyama, Naoto and Bannur, Shruthi and Castro, Daniel C and Schwaighofer, Anton and Hyland, Stephanie and Wetscherek, Maria and Naumann, Tristan and Nori, Aditya and Alvarez-Valle, Javier and others},
  booktitle={European conference on computer vision},
  pages={1--21},
  year={2022},
  organization={Springer}
}
@inproceedings{chuang2022robust,
  title={Robust contrastive learning against noisy views},
  author={Chuang, Ching-Yao and Hjelm, R Devon and Wang, Xin and Vineet, Vibhav and Joshi, Neel and Torralba, Antonio and Jegelka, Stefanie and Song, Yale},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={16670--16681},
  year={2022}
}
@article{aideyan1995influence,
  title={Influence of prior radiologic information on the interpretation of radiographic examinations},
  author={Aideyan, Uwa O and Berbaum, Kevin and Smith, Wilbur L},
  journal={Academic Radiology},
  volume={2},
  number={3},
  pages={205--208},
  year={1995},
  publisher={Elsevier}
}
@inproceedings{ramesh2022improving,
  title={Improving radiology report generation systems by removing hallucinated references to non-existent priors},
  author={Ramesh, Vignav and Chi, Nathan A and Rajpurkar, Pranav},
  booktitle={Machine Learning for Health},
  pages={456--473},
  year={2022},
  organization={PMLR}
}
@inproceedings{li2023dynamic,
  title={Dynamic Graph Enhanced Contrastive Learning for Chest X-ray Report Generation},
  author={Li, Mingjie and Lin, Bingqian and Chen, Zicong and Lin, Haokun and Liang, Xiaodan and Chang, Xiaojun},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={3334--3343},
  year={2023}
}
@inproceedings{zhang2020radiology,
  title={When radiology report generation meets knowledge graph},
  author={Zhang, Yixiao and Wang, Xiaosong and Xu, Ziyue and Yu, Qihang and Yuille, Alan and Xu, Daguang},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={07},
  pages={12910--12917},
  year={2020}
}
@inproceedings{liu2021exploring,
  title={Exploring and distilling posterior and prior knowledge for radiology report generation},
  author={Liu, Fenglin and Wu, Xian and Ge, Shen and Fan, Wei and Zou, Yuexian},
  booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
  pages={13753--13762},
  year={2021}
}
@article{demner2016preparing,
  title={Preparing a collection of radiology examinations for distribution and retrieval},
  author={Demner-Fushman, Dina and Kohli, Marc D and Rosenman, Marc B and Shooshan, Sonya E and Rodriguez, Laritza and Antani, Sameer and Thoma, George R and McDonald, Clement J},
  journal={Journal of the American Medical Informatics Association},
  volume={23},
  number={2},
  pages={304--310},
  year={2016},
  publisher={Oxford University Press}
}
@inproceedings{irvin2019chexpert,
  title={Chexpert: A large chest radiograph dataset with uncertainty labels and expert comparison},
  author={Irvin, Jeremy and Rajpurkar, Pranav and Ko, Michael and Yu, Yifan and Ciurea-Ilcus, Silviana and Chute, Chris and Marklund, Henrik and Haghgoo, Behzad and Ball, Robyn and Shpanskaya, Katie and others},
  booktitle={Proceedings of the AAAI conference on artificial intelligence},
  volume={33},
  number={01},
  pages={590--597},
  year={2019}
}
@article{johnson2019mimic,
  title={MIMIC-CXR-JPG, a large publicly available database of labeled chest radiographs},
  author={Johnson, Alistair EW and Pollard, Tom J and Greenbaum, Nathaniel R and Lungren, Matthew P and Deng, Chih-ying and Peng, Yifan and Lu, Zhiyong and Mark, Roger G and Berkowitz, Seth J and Horng, Steven},
  journal={arXiv preprint arXiv:1901.07042},
  year={2019}
}
@article{jain2021radgraph,
  title={Radgraph: Extracting clinical entities and relations from radiology reports},
  author={Jain, Saahil and Agrawal, Ashwin and Saporta, Adriel and Truong, Steven QH and Duong, Du Nguyen and Bui, Tan and Chambon, Pierre and Zhang, Yuhao and Lungren, Matthew P and Ng, Andrew Y and others},
  journal={arXiv preprint arXiv:2106.14463},
  year={2021}
}
@inproceedings{hu2023expert,
  title={Expert Knowledge-Aware Image Difference Graph Representation Learning for Difference-Aware Medical Visual Question Answering},
  author={Hu, Xinyue and Gu, Lin and An, Qiyuan and Zhang, Mengliang and Liu, Liangchen and Kobayashi, Kazuma and Harada, Tatsuya and Summers, Ronald M and Zhu, Yingying},
  booktitle={Proceedings of the 29th ACM SIGKDD Conference on Knowledge Discovery and Data Mining},
  pages={4156--4165},
  year={2023}
}
@article{alayrac2022flamingo,
  title={Flamingo: a visual language model for few-shot learning},
  author={Alayrac, Jean-Baptiste and Donahue, Jeff and Luc, Pauline and Miech, Antoine and Barr, Iain and Hasson, Yana and Lenc, Karel and Mensch, Arthur and Millican, Katherine and Reynolds, Malcolm and others},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={23716--23736},
  year={2022}
}
@article{han2020self,
  title={Self-supervised co-training for video representation learning},
  author={Han, Tengda and Xie, Weidi and Zisserman, Andrew},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={5679--5690},
  year={2020}
}
@inproceedings{recasens2021broaden,
  title={Broaden your views for self-supervised video learning},
  author={Recasens, Adria and Luc, Pauline and Alayrac, Jean-Baptiste and Wang, Luyu and Strub, Florian and Tallec, Corentin and Malinowski, Mateusz and P{\u{a}}tr{\u{a}}ucean, Viorica and Altch{\'e}, Florent and Valko, Michal and others},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={1255--1265},
  year={2021}
}
@article{smit2020chexbert,
  title={CheXbert: combining automatic labelers and expert annotations for accurate radiology report labeling using BERT},
  author={Smit, Akshay and Jain, Saahil and Rajpurkar, Pranav and Pareek, Anuj and Ng, Andrew Y and Lungren, Matthew P},
  journal={arXiv preprint arXiv:2004.09167},
  year={2020}
}
@inproceedings{bannur2023learning,
  title={Learning to exploit temporal structure for biomedical vision-language processing},
  author={Bannur, Shruthi and Hyland, Stephanie and Liu, Qianchu and Perez-Garcia, Fernando and Ilse, Maximilian and Castro, Daniel C and Boecking, Benedikt and Sharma, Harshita and Bouzid, Kenza and Thieme, Anja and others},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={15016--15027},
  year={2023}
}
@inproceedings{qiu2021describing,
  title={Describing and localizing multiple changes with transformers},
  author={Qiu, Yue and Yamamoto, Shintaro and Nakashima, Kodai and Suzuki, Ryota and Iwata, Kenji and Kataoka, Hirokatsu and Satoh, Yutaka},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={1971--1980},
  year={2021}
}
@inproceedings{yao2022image,
  title={Image difference captioning with pre-training and contrastive learning},
  author={Yao, Linli and Wang, Weiying and Jin, Qin},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={36},
  number={3},
  pages={3108--3116},
  year={2022}
}
@inproceedings{dalla2022multimodal,
  title={Multimodal Generation of Radiology Reports using Knowledge-Grounded Extraction of Entities and Relations},
  author={Dalla Serra, Francesco and Clackett, William and MacKinnon, Hamish and Wang, Chaoyang and Deligianni, Fani and Dalton, Jeff and O’Neil, Alison Q},
  booktitle={Proceedings of the 2nd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics and the 12th International Joint Conference on Natural Language Processing},
  pages={615--624},
  year={2022}
}
@inproceedings{huang2023kiut,
  title={KiUT: Knowledge-injected U-Transformer for Radiology Report Generation},
  author={Huang, Zhongzhen and Zhang, Xiaofan and Zhang, Shaoting},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={19809--19818},
  year={2023}
}
@article{nicolson2023improving,
  title={Improving chest X-ray report generation by leveraging warm starting},
  author={Nicolson, Aaron and Dowling, Jason and Koopman, Bevan},
  journal={Artificial Intelligence in Medicine},
  volume={144},
  pages={102633},
  year={2023},
  publisher={Elsevier}
}
@article{zhang2023pmc,
  title={Pmc-vqa: Visual instruction tuning for medical visual question answering},
  author={Zhang, Xiaoman and Wu, Chaoyi and Zhao, Ziheng and Lin, Weixiong and Zhang, Ya and Wang, Yanfeng and Xie, Weidi},
  journal={arXiv preprint arXiv:2305.10415},
  year={2023}
}
@article{zhang2023biomedgpt,
  title={BiomedGPT: A Unified and Generalist Biomedical Generative Pre-trained Transformer for Vision, Language, and Multimodal Tasks},
  author={Zhang, Kai and Yu, Jun and Yan, Zhiling and Liu, Yixin and Adhikarla, Eashan and Fu, Sunyang and Chen, Xun and Chen, Chen and Zhou, Yuyin and Li, Xiang and others},
  journal={arXiv preprint arXiv:2305.17100},
  year={2023}
}



@article{yu2022coca,
  title={Coca: Contrastive captioners are image-text foundation models},
  author={Yu, Jiahui and Wang, Zirui and Vasudevan, Vijay and Yeung, Legg and Seyedhosseini, Mojtaba and Wu, Yonghui},
  journal={arXiv preprint arXiv:2205.01917},
  year={2022}
}
@inproceedings{ben2021overview,
  title={Overview of the vqa-med task at imageclef 2021: Visual question answering and generation in the medical domain},
  author={Ben Abacha, Asma and Sarrouti, Mourad and Demner-Fushman, Dina and Hasan, Sadid A and M{\"u}ller, Henning},
  booktitle={Proceedings of the CLEF 2021 Conference and Labs of the Evaluation Forum-working notes},
  year={2021},
  organization={21-24 September 2021}
}
@inproceedings{liu2021slake,
  title={Slake: A semantically-labeled knowledge-enhanced dataset for medical visual question answering},
  author={Liu, Bo and Zhan, Li-Ming and Xu, Li and Ma, Lin and Yang, Yan and Wu, Xiao-Ming},
  booktitle={2021 IEEE 18th International Symposium on Biomedical Imaging (ISBI)},
  pages={1650--1654},
  year={2021},
  organization={IEEE}
}
@article{lau2018dataset,
  title={A dataset of clinically generated visual questions and answers about radiology images},
  author={Lau, Jason J and Gayen, Soumya and Ben Abacha, Asma and Demner-Fushman, Dina},
  journal={Scientific data},
  volume={5},
  number={1},
  pages={1--10},
  year={2018},
  publisher={Nature Publishing Group}
}
@inproceedings{park2019robust,
  title={Robust change captioning},
  author={Park, Dong Huk and Darrell, Trevor and Rohrbach, Anna},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={4624--4633},
  year={2019}
}
@article{jhamtani2018learning,
  title={Learning to describe differences between pairs of similar images},
  author={Jhamtani, Harsh and Berg-Kirkpatrick, Taylor},
  journal={arXiv preprint arXiv:1808.10584},
  year={2018}
}
@inproceedings{li2020comparison,
  title={A comparison of pre-trained vision-and-language models for multimodal representation learning across medical images and reports},
  author={Li, Yikuan and Wang, Hanyin and Luo, Yuan},
  booktitle={2020 IEEE international conference on bioinformatics and biomedicine (BIBM)},
  pages={1999--2004},
  year={2020},
  organization={IEEE}
}
@inproceedings{khare2021mmbert,
  title={Mmbert: Multimodal bert pretraining for improved medical vqa},
  author={Khare, Yash and Bagal, Viraj and Mathew, Minesh and Devi, Adithi and Priyakumar, U Deva and Jawahar, CV},
  booktitle={2021 IEEE 18th International Symposium on Biomedical Imaging (ISBI)},
  pages={1033--1036},
  year={2021},
  organization={IEEE}
}

@article{moon2022multi,
  title={Multi-modal understanding and generation for medical images and text via vision-language pre-training},
  author={Moon, Jong Hak and Lee, Hyungyung and Shin, Woncheol and Kim, Young-Hak and Choi, Edward},
  journal={IEEE Journal of Biomedical and Health Informatics},
  volume={26},
  number={12},
  pages={6070--6080},
  year={2022},
  publisher={IEEE}
}
@article{devlin2018bert,
  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={arXiv preprint arXiv:1810.04805},
  year={2018}
}
@article{forbes2019neural,
  title={Neural naturalist: generating fine-grained image comparisons},
  author={Forbes, Maxwell and Kaeser-Chen, Christine and Sharma, Piyush and Belongie, Serge},
  journal={arXiv preprint arXiv:1909.04101},
  year={2019}
}
@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}
@article{chipman2010bart,
  title={BART: Bayesian additive regression trees},
  author={Chipman, Hugh A and George, Edward I and McCulloch, Robert E},
  year={2010}
}
@article{radford2018improving,
  title={Improving language understanding by generative pre-training},
  author={Radford, Alec and Narasimhan, Karthik and Salimans, Tim and Sutskever, Ilya and others},
  year={2018},
  publisher={OpenAI}
}
@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}
@inproceedings{papineni2002bleu,
  title={Bleu: a method for automatic evaluation of machine translation},
  author={Papineni, Kishore and Roukos, Salim and Ward, Todd and Zhu, Wei-Jing},
  booktitle={Proceedings of the 40th annual meeting of the Association for Computational Linguistics},
  pages={311--318},
  year={2002}
}
@inproceedings{banerjee2005meteor,
  title={METEOR: An automatic metric for MT evaluation with improved correlation with human judgments},
  author={Banerjee, Satanjeev and Lavie, Alon},
  booktitle={Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and/or summarization},
  pages={65--72},
  year={2005}
}
@inproceedings{lin2004rouge,
  title={Rouge: A package for automatic evaluation of summaries},
  author={Lin, Chin-Yew},
  booktitle={Text summarization branches out},
  pages={74--81},
  year={2004}
}
@inproceedings{vedantam2015cider,
  title={Cider: Consensus-based image description evaluation},
  author={Vedantam, Ramakrishna and Lawrence Zitnick, C and Parikh, Devi},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={4566--4575},
  year={2015}
}
@article{chen2015microsoft,
  title={Microsoft coco captions: Data collection and evaluation server. arXiv 2015},
  author={Chen, Xinlei and Fang, Hao and Lin, Tsung-Yi and Vedantam, Ramakrishna and Gupta, Saurabh and Doll{\'a}r, Piotr and Zitnick, C Lawrence},
  journal={arXiv preprint arXiv:1504.00325},
  year={2015}
}
@article{loshchilov2017decoupled,
  title={Decoupled weight decay regularization},
  author={Loshchilov, Ilya and Hutter, Frank},
  journal={arXiv preprint arXiv:1711.05101},
  year={2017}
}

@inproceedings{ben2019vqa,
  title={Vqa-med: Overview of the medical visual question answering task at imageclef 2019},
  author={Ben Abacha, Asma and Hasan, Sadid A and Datla, Vivek V and Demner-Fushman, Dina and M{\"u}ller, Henning},
  booktitle={Proceedings of CLEF (Conference and Labs of the Evaluation Forum) 2019 Working Notes},
  year={2019},
  organization={9-12 September 2019}
}
@article{xu2023learning,
  title={Learning A Multi-Task Transformer Via Unified And Customized Instruction Tuning For Chest Radiograph Interpretation},
  author={Xu, Lijian and Ni, Ziyu and Liu, Xinglong and Wang, Xiaosong and Li, Hongsheng and Zhang, Shaoting},
  journal={arXiv preprint arXiv:2311.01092},
  year={2023}
}