@incollection{Bengio+chapter2007,
author = {Bengio, Yoshua and LeCun, Yann},
booktitle = {Large Scale Kernel Machines},
publisher = {MIT Press},
title = {Scaling Learning Algorithms Towards {AI}},
year = {2007}
}

@article{Hinton06,
author = {Hinton, Geoffrey E. and Osindero, Simon and Teh, Yee Whye},
journal = {Neural Computation},
pages = {1527--1554},
title = {A Fast Learning Algorithm for Deep Belief Nets},
volume = {18},
year = {2006}
}

@book{goodfellow2016deep,
title={Deep learning},
author={Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron and Bengio, Yoshua},
volume={1},
year={2016},
publisher={MIT Press}
}


%%%%%%%%%%%%%%%%%%%%%%MLLM paper bib%%%%%%

@misc{vaswani2023attentionneed,
      title={Attention Is All You Need}, 
      author={Ashish Vaswani and Noam Shazeer and Niki Parmar and Jakob Uszkoreit and Llion Jones and Aidan N. Gomez and Lukasz Kaiser and Illia Polosukhin},
      year={2023},
      eprint={1706.03762},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/1706.03762}, 
}

@misc{yang2024seedstorymultimodallongstory,
      title={SEED-Story: Multimodal Long Story Generation with Large Language Model}, 
      author={Shuai Yang and Yuying Ge and Yang Li and Yukang Chen and Yixiao Ge and Ying Shan and Yingcong Chen},
      year={2024},
      eprint={2407.08683},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2407.08683}, 
}

@article {Liang2024.09.29.615524,
	author = {Liang, Youwei and Zhang, Ruiyi and Li, Yongce and Huo, Mingjia and Ma, Zinnia and Singh, Digvijay and Gao, Chengzhan and Rahmani, Hamidreza and Bandi, Satvik and Zhang, Li and Weinreb, Robert and Malhotra, Atul and Grotjahn, Danielle A. and Awdishu, Linda and Ideker, Trey and Gilson, Michael and Xie, Pengtao},
	title = {Multi-Modal Large Language Model Enables All-Purpose Prediction of Drug Mechanisms and Properties},
	elocation-id = {2024.09.29.615524},
	year = {2024},
	doi = {10.1101/2024.09.29.615524},
	publisher = {Cold Spring Harbor Laboratory},
	URL = {https://www.biorxiv.org/content/early/2024/10/01/2024.09.29.615524},
	eprint = {https://www.biorxiv.org/content/early/2024/10/01/2024.09.29.615524.full.pdf},
	journal = {bioRxiv}
}


@misc{wang2023finvisgptmultimodallargelanguage,
      title={FinVis-GPT: A Multimodal Large Language Model for Financial Chart Analysis}, 
      author={Ziao Wang and Yuhang Li and Junda Wu and Jaehyeon Soon and Xiaofeng Zhang},
      year={2023},
      eprint={2308.01430},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2308.01430}, 
}

@inproceedings{duan2024vlmevalkit,
  title={Vlmevalkit: An open-source toolkit for evaluating large multi-modality models},
  author={Duan, Haodong and Yang, Junming and Qiao, Yuxuan and Fang, Xinyu and Chen, Lin and Liu, Yuan and Dong, Xiaoyi and Zang, Yuhang and Zhang, Pan and Wang, Jiaqi and others},
  booktitle={Proceedings of the 32nd ACM International Conference on Multimedia},
  pages={11198--11201},
  year={2024}
}

@misc{zhang2024mmllmsrecentadvancesmultimodal,
      title={MM-LLMs: Recent Advances in MultiModal Large Language Models}, 
      author={Duzhen Zhang and Yahan Yu and Jiahua Dong and Chenxing Li and Dan Su and Chenhui Chu and Dong Yu},
      year={2024},
      eprint={2401.13601},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2401.13601}, 
}

@misc{niu2024textmultimodalityexploringevolution,
      title={From Text to Multimodality: Exploring the Evolution and Impact of Large Language Models in Medical Practice}, 
      author={Qian Niu and Keyu Chen and Ming Li and Pohsun Feng and Ziqian Bi and Lawrence KQ Yan and Yichao Zhang and Caitlyn Heqi Yin and Cheng Fei and Junyu Liu and Benji Peng and Tianyang Wang and Yunze Wang and Silin Chen and Ming Liu},
      year={2024},
      eprint={2410.01812},
      archivePrefix={arXiv},
      primaryClass={cs.CY},
      url={https://arxiv.org/abs/2410.01812}, 
}

@article{Bewersdorff_2025,
   title={Taking the next step with generative artificial intelligence: The transformative role of multimodal large language models in science education},
   volume={118},
   ISSN={1041-6080},
   url={http://dx.doi.org/10.1016/j.lindif.2024.102601},
   DOI={10.1016/j.lindif.2024.102601},
   journal={Learning and Individual Differences},
   publisher={Elsevier BV},
   author={Bewersdorff, Arne and Hartmann, Christian and Hornberger, Marie and Seßler, Kathrin and Bannert, Maria and Kasneci, Enkelejda and Kasneci, Gjergji and Zhai, Xiaoming and Nerdel, Claudia},
   year={2025},
   month=feb, pages={102601} 
}


@misc{ghosh2024exploringfrontiervisionlanguagemodels,
      title={Exploring the Frontier of Vision-Language Models: A Survey of Current Methodologies and Future Directions}, 
      author={Akash Ghosh and Arkadeep Acharya and Sriparna Saha and Vinija Jain and Aman Chadha},
      year={2024},
      eprint={2404.07214},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2404.07214}, 
}

@misc{li2025benchmarkevaluationsapplicationschallenges,
      title={Benchmark Evaluations, Applications, and Challenges of Large Vision Language Models: A Survey}, 
      author={Zongxia Li and Xiyang Wu and Hongyang Du and Huy Nghiem and Guangyao Shi},
      year={2025},
      eprint={2501.02189},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2501.02189}, 
}

@misc{xie2024tpevaltapmultimodalllms,
      title={TP-Eval: Tap Multimodal LLMs' Potential in Evaluation by Customizing Prompts}, 
      author={Yuxuan Xie and Tianhua Li and Wenqi Shao and Kaipeng Zhang},
      year={2024},
      eprint={2410.18071},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2410.18071}, 
}


@article{li2025structured,
  title={Structured chain-of-thought prompting for code generation},
  author={Li, Jia and Li, Ge and Li, Yongmin and Jin, Zhi},
  journal={ACM Transactions on Software Engineering and Methodology},
  volume={34},
  number={2},
  pages={1--23},
  year={2025},
  publisher={ACM New York, NY}
}

@article{Yin_2024,
   title={A survey on multimodal large language models},
   volume={11},
   ISSN={2053-714X},
   url={http://dx.doi.org/10.1093/nsr/nwae403},
   DOI={10.1093/nsr/nwae403},
   number={12},
   journal={National Science Review},
   publisher={Oxford University Press (OUP)},
   author={Yin, Shukang and Fu, Chaoyou and Zhao, Sirui and Li, Ke and Sun, Xing and Xu, Tong and Chen, Enhong},
   year={2024},
   month=nov }


@article{Jiang_2024,
   title={Effectiveness assessment of recent large vision-language models},
   volume={2},
   ISSN={2731-9008},
   url={http://dx.doi.org/10.1007/s44267-024-00050-1},
   DOI={10.1007/s44267-024-00050-1},
   number={1},
   journal={Visual Intelligence},
   publisher={Springer Science and Business Media LLC},
   author={Jiang, Yao and Yan, Xinyu and Ji, Ge-Peng and Fu, Keren and Sun, Meijun and Xiong, Huan and Fan, Deng-Ping and Khan, Fahad Shahbaz},
   year={2024},
   month=jun }


@misc{hao2025mllmsreasonmultimodalityemma,
      title={Can MLLMs Reason in Multimodality? EMMA: An Enhanced MultiModal ReAsoning Benchmark}, 
      author={Yunzhuo Hao and Jiawei Gu and Huichen Will Wang and Linjie Li and Zhengyuan Yang and Lijuan Wang and Yu Cheng},
      year={2025},
      eprint={2501.05444},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2501.05444}, 
}

@inproceedings{kour2014real,
  title={Real-time segmentation of on-line handwritten arabic script},
  author={Kour, George and Saabne, Raid},
  booktitle={Frontiers in Handwriting Recognition (ICFHR), 2014 14th International Conference on},
  pages={417--422},
  year={2014},
  organization={IEEE}
}

@inproceedings{kour2014fast,
  title={Fast classification of handwritten on-line Arabic characters},
  author={Kour, George and Saabne, Raid},
  booktitle={Soft Computing and Pattern Recognition (SoCPaR), 2014 6th International Conference of},
  pages={312--318},
  year={2014},
  organization={IEEE}
}

@article{hadash2018estimate,
  title={Estimate and Replace: A Novel Approach to Integrating Deep Neural Networks with Existing Applications},
  author={Hadash, Guy and Kermany, Einat and Carmeli, Boaz and Lavi, Ofer and Kour, George and Jacovi, Alon},
  journal={arXiv preprint arXiv:1804.09028},
  year={2018}
}



% OUR REFERENCE 
@article{jiang2022promptmaker,
  title={Promptmaker: Prompt-based prototyping with large language models},
  author={Jiang, E and Olson, K and Toh, E and Molina, A and Donsbach, A and Terry, M and Cai, C.J},
  journal={CHI Conference on Human Factors in Computing Systems Extended Abstracts},
  year={2022},
  pages={1--8}
}

@article{bhattacharjya2024foundation,
  title={Foundation Model Sherpas: Guiding Foundation Models through Knowledge and Reasoning},
  author={Bhattacharjya, D and Lee, J and Agravante, D.J and Ganesan, B and Marinescu, R},
  journal={arXiv preprint arXiv:2402.01602},
  year={2024}
}

@article{zamfirescu2023johnny,
  title={Why Johnny Can’t Prompt: How Non-AI Experts Try (and Fail) to Design LLM Prompts},
  author={Zamfirescu-Pereira, J.D and Wong, R.Y and Hartmann, B and Yang, Q},
  journal={Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems},
  year={2023},
  pages={1--21}
}

@article{wu2023scattershot,
  title={Scattershot: Interactive In-Context Example Curation for Text Transformation},
  author={Wu, S and Shen, H and Weld, D.S and Heer, J and Ribeiro, M.T},
  journal={Proceedings of the 28th International Conference on Intelligent User Interfaces},
  year={2023},
  pages={353--367}
}

@misc{chen2024internvlscalingvisionfoundation,
  title={InternVL: Scaling up Vision Foundation Models and Aligning for Generic Visual-Linguistic Tasks},
  author={Zhe Chen and Jiannan Wu and Wenhai Wang and Weijie Su and Guo Chen and Sen Xing and Muyan Zhong and Qinglong Zhang and Xizhou Zhu and Lewei Lu and Bin Li and Ping Luo and Tong Lu and Yu Qiao and Jifeng Dai},
  year={2024},
  eprint={2312.14238},
  archivePrefix={arXiv},
  primaryClass={cs.CV},
  url={https://arxiv.org/abs/2312.14238}
}

@article{yang2024qwen2,
  title={Qwen2.5 Technical Report},
  author={Yang, A and Yang, B and Zhang, B and Hui, B and Zheng, B and Yu, B and Li, C and Liu, D and Huang, F and Wei, H and others},
  journal={arXiv preprint arXiv:2412.15115},
  year={2024}
}

@article{chen2024internvl,
  title={Internvl: Scaling Up Vision Foundation Models and Aligning for Generic Visual-Linguistic Tasks},
  author={Chen, Z and Wu, J and Wang, W and Su, W and Chen, G and Xing, S and Zhong, M and Zhang, Q and Zhu, X and Lu, L and others},
  journal={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  year={2024},
  pages={24185--24198}
}

@article{wang2024qwen2,
  title={Qwen2-vl: Enhancing Vision-Language Model's Perception of the World at Any Resolution},
  author={Wang, P and Bai, S and Tan, S and Wang, S and Fan, Z and Bai, J and Chen, K and Liu, X and Wang, J and Ge, W and others},
  journal={arXiv preprint arXiv:2409.12191},
  year={2024}
}

@article{huang2024mini,
  title={Mini-Monkey: Multi-Scale Adaptive Cropping for Multimodal Large Language Models},
  author={Huang, M and Liu, Y and Liang, D and Jin, L and Bai, X},
  journal={CoRR},
  year={2024}
}

@article{dong2024internlm,
  title={InternLM-XComposer2: Mastering Free-Form Text-Image Composition and Comprehension in Vision-Language Large Model},
  author={Dong, X and Zhang, P and Zang, Y and Cao, Y and Wang, B and Ouyang, L and Wei, X and Zhang, S and Duan, H and Cao, M and others},
  journal={arXiv preprint arXiv:2401.16420},
  year={2024}
}

@article{beyer2024paligemma,
  title={Paligemma: A Versatile 3B VLM for Transfer},
  author={Beyer, L and Steiner, A and Pinto, A.S and Kolesnikov, A and Wang, X and Salz, D and Neumann, M and Alabdulmohsin, I and Tschannen, M and Bugliarello, E and others},
  journal={arXiv preprint arXiv:2407.07726},
  year={2024}
}

@article{zhai2023sigmoid,
  title={Sigmoid Loss for Language Image Pre-Training},
  author={Zhai, X and Mustafa, B and Kolesnikov, A and Beyer, L},
  journal={Proceedings of the 2023 IEEE/CVF International Conference on Computer Vision (ICCV)},
  year={2023},
  pages={11941--11952}
}

@article{team2024gemma,
  title={Gemma: Open Models Based on Gemini Research and Technology},
  author={Team, G and Mesnard, T and Hardin, C and Dadashi, R and Bhupatiraju, S and Pathak, S and Sifre, L and Rivi{\`e}re, M and Kale, M.S and Love, J and others},
  journal={arXiv preprint arXiv:2403.08295},
  year={2024}
}

@article{abdin2024phi,
  title={Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone},
  author={Abdin, M and Aneja, J and Awadalla, H and Awadallah, A and Awan, A.A and Bach, N and Bahree, A and Bakhtiari, A and Bao, J and Behl, H and others},
  journal={arXiv preprint arXiv:2404.14219},
  year={2024}
}

@article{radford2021learning,
  title={Learning Transferable Visual Models from Natural Language Supervision},
  author={Radford, A and Kim, J.W and Hallacy, C and Ramesh, A and Goh, G and Agarwal, S and Sastry, G and Askell, A and Mishkin, P and Clark, J and others},
  journal={Proceedings of the International Conference on Machine Learning},
  year={2021},
  pages={8748--8763}
}

@article{li2024llava,
  title={Llava-OneVision: Easy Visual Task Transfer},
  author={Li, B and Zhang, Y and Guo, D and Zhang, R and Li, F and Zhang, H and Zhang, K and Zhang, P and Li, Y and Liu, Z and others},
  journal={arXiv preprint arXiv:2408.03326},
  year={2024}
}

@article{lu2024ovis,
  title={Ovis: Structural Embedding Alignment for Multimodal Large Language Model},
  author={Lu, S and Li, Y and Chen, Q.G and Xu, Z and Luo, W and Zhang, K and Ye, H.J},
  journal={arXiv preprint arXiv:2405.20797},
  year={2024}
}

@article{meta2024llama3,
  title={Build the Future of AI with Meta LLaMA 3},
  author={Meta},
  journal={Technical Report},
  year={2024},
  url={https://llama.meta.com/llama3/},
  note={accessed on 13 February 2025}
}

@article{glm2024chatglm,
  title={ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools},
  author={GLM, T and Zeng, A and Xu, B and Wang, B and Zhang, C and Yin, D and Zhang, D and Rojas, D and Feng, G and Zhao, H and others},
  journal={arXiv preprint arXiv:2406.12793},
  year={2024}
}

@article{fang2024eva,
  title={Eva-02: A Visual Representation for Neon Genesis},
  author={Fang, Y and Sun, Q and Wang, X and Huang, T and Wang, X and Cao, Y},
  journal={Image and Vision Computing},
  year={2024},
  volume={149},
  pages={105171}
}

@article{meta2024llama32,
  title={LLaMA 3.2-11B Vision},
  author={Meta},
  journal={Available online},
  year={2024},
  url={https://huggingface.co/meta-llama/Llama-3.2-11B-Vision},
  note={accessed on 13 February 2025}
}

@article{agrawal2024pixtral,
  title={Pixtral 12B},
  author={Agrawal, P and Antoniak, S and Hanna, E.B and Bout, B and Chaplot, D and Chudnovsky, J and Costa, D and De Monicault, B and Garg, S and Gervet, T and others},
  journal={arXiv preprint arXiv:2410.07073},
  year={2024}
}

@article{mistral2024pixtral,
  title={Pixtral-12B-2409},
  author={MistralAI},
  journal={Available online},
  year={2024},
  url={https://huggingface.co/mistralai/Pixtral-12B-2409},
  note={accessed on 13 February 2025}
}

@article{mistral2024nemo,
  title={Mistral Nemo},
  author={MistralAI},
  journal={Available online},
  year={2024},
  url={https://mistral.ai/en/news/mistral-nemo},
  note={accessed on 13 February 2025}
}

@article{zhao2024omchat,
  title={OmChat: A Recipe to Train Multimodal Language Models with Strong Long Context and Video Understanding},
  author={Zhao, T and Zhang, Q and Lee, K and Liu, P and Zhang, L and Fang, C and Liao, J and Jiang, K and Ma, Y and Xu, R},
  journal={arXiv preprint arXiv:2407.04923},
  year={2024}
}

@article{omchat2024v2,
  title={OmChat v2.0-13B Single Beta},
  author={OmLab},
  journal={Available online},
  year={2024},
  url={https://huggingface.co/omlab/omchat-v2.0-13B-single-beta_hf},
  note={accessed on 13 February 2025}
}

@article{chen2024far,
  title={How Far Are We to GPT-4V? Closing the Gap to Commercial Multimodal Models with Open-Source Suites},
  author={Chen, Z and Wang, W and Tian, H and Ye, S and Gao, Z and Cui, E and Tong, W and Hu, K and Luo, J and Ma, Z and others},
  journal={Science China Information Sciences},
  year={2024},
  volume={67},
  pages={220101}
}

@article{internvl2024,
  title={InternVL 2.0 Blog Post},
  author={InternVL},
  journal={Available online},
  year={2024},
  url={https://internvl.github.io/blog/2024-07-02-InternVL-2.0/},
  note={accessed on 12 February 2025}
}


@article{radford2019language,
  title={Language models are unsupervised multitask learners},
  author={Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya and others},
  journal={OpenAI blog},
  volume={1},
  number={8},
  pages={9},
  year={2019}
}

@article{mann2020language,
  title={Language models are few-shot learners},
  author={Mann, Ben and Ryder, N and Subbiah, M and Kaplan, J and Dhariwal, P and Neelakantan, A and Shyam, P and Sastry, G and Askell, A and Agarwal, S and others},
  journal={arXiv preprint arXiv:2005.14165},
  volume={1},
  year={2020}
}


@article{wei2022chain,
  title={Chain-of-thought prompting elicits reasoning in large language models},
  author={Wei, Jason and Wang, Xuezhi and Schuurmans, Dale and Bosma, Maarten and Xia, Fei and Chi, Ed and Le, Quoc V and Zhou, Denny and others},
  journal={Advances in neural information processing systems},
  volume={35},
  pages={24824--24837},
  year={2022}
}

@article{liu2021generated,
  title={Generated knowledge prompting for commonsense reasoning},
  author={Liu, Jiacheng and Liu, Alisa and Lu, Ximing and Welleck, Sean and West, Peter and Bras, Ronan Le and Choi, Yejin and Hajishirzi, Hannaneh},
  journal={arXiv preprint arXiv:2110.08387},
  year={2021}
}


@article{yao2024tree,
  title={Tree of thoughts: Deliberate problem solving with large language models},
  author={Yao, Shunyu and Yu, Dian and Zhao, Jeffrey and Shafran, Izhak and Griffiths, Tom and Cao, Yuan and Narasimhan, Karthik},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2024}
}


@article{yasunaga2023large,
  title={Large language models as analogical reasoners},
  author={Yasunaga, Michihiro and Chen, Xinyun and Li, Yujia and Pasupat, Panupong and Leskovec, Jure and Liang, Percy and Chi, Ed H and Zhou, Denny},
  journal={arXiv preprint arXiv:2310.01714},
  year={2023}
}

@misc{russell2020artificial,
  title={Artificial intelligence: a modern approach. Hoboken},
  author={Russell, Stuart and Norvig, Peter},
  year={2020},
  publisher={NJ: Pearson}
}


%EA part
@article{fu2024mme,
  title={Mme-survey: A comprehensive survey on evaluation of multimodal llms},
  author={Fu, Chaoyou and Zhang, Yi-Fan and Yin, Shukang and Li, Bo and Fang, Xinyu and Zhao, Sirui and Duan, Haodong and Sun, Xing and Liu, Ziwei and Wang, Liang and others},
  journal={arXiv preprint arXiv:2411.15296},
  year={2024}
}

@article{nwae403,
    author = {Yin, Shukang and Fu, Chaoyou and Zhao, Sirui and Li, Ke and Sun, Xing and Xu, Tong and Chen, Enhong},
    title = {A survey on multimodal large language models},
    journal = {National Science Review},
    volume = {11},
    number = {12},
    pages = {nwae403},
    year = {2024},
    month = {11},
    abstract = {Recently, the multimodal large language model (MLLM) represented by GPT-4V has been a new rising research hotspot, which uses powerful large language models (LLMs) as a brain to perform multimodal tasks. The surprising emergent capabilities of the MLLM, such as writing stories based on images and optical character recognition–free math reasoning, are rare in traditional multimodal methods, suggesting a potential path to artificial general intelligence. To this end, both academia and industry have endeavored to develop MLLMs that can compete with or even outperform GPT-4V, pushing the limit of research at a surprising speed.}
}

@article{huang2024survey,
  title={A survey on evaluation of multimodal large language models},
  author={Huang, Jiaxing and Zhang, Jingyi},
  journal={arXiv preprint arXiv:2408.15769},
  year={2024}
}

@inproceedings{tong2024eyes,
  title={Eyes wide shut? exploring the visual shortcomings of multimodal llms},
  author={Tong, Shengbang and Liu, Zhuang and Zhai, Yuexiang and Ma, Yi and LeCun, Yann and Xie, Saining},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={9568--9578},
  year={2024}
}

@article{zhou2025aretheythesame,
  title={Are They the Same? Exploring Visual Correspondence Shortcomings of Multimodal LLMs},
  author={Zhou, Yikang and Zhang, Tao and Xu, Shilin and Chen, Shihao and Zhou, Qianyu and Tong, Yunhai and Ji, Shunping and Zhang, Jiangning and Li, Xiangtai and Qi, Lu},
  journal={arXiv preprint arXiv:2501.04670},
  year={2025}
}

@inproceedings{cai2024vip,
  title={Vip-llava: Making large multimodal models understand arbitrary visual prompts},
  author={Cai, Mu and Liu, Haotian and Mustikovela, Siva Karthik and Meyer, Gregory P and Chai, Yuning and Park, Dennis and Lee, Yong Jae},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={12914--12923},
  year={2024}
}


@article{wu2025controlmllm,
  title={Controlmllm: Training-free visual prompt learning for multimodal large language models},
  author={Wu, Mingrui and Cai, Xinyue and Ji, Jiayi and Li, Jiale and Huang, Oucheng and Luo, Gen and Fei, Hao and Jiang, Guannan and Sun, Xiaoshuai and Ji, Rongrong},
  journal={Advances in Neural Information Processing Systems},
  volume={37},
  pages={45206--45234},
  year={2025}
}

@article{yu2023mmvet,
  title={Mm-vet: Evaluating large multimodal models for integrated capabilities},
  author={Yu, Weihao and Yang, Zhengyuan and Li, Linjie and Wang, Jianfeng and Lin, Kevin and Liu, Zicheng and Wang, Xinchao and Wang, Lijuan},
  journal={arXiv preprint arXiv:2308.02490},
  year={2023}
}

@article{wang2024exploring,
  title={Exploring the reasoning abilities of multimodal large language models (mllms): A comprehensive survey on emerging trends in multimodal reasoning},
  author={Wang, Yiqi and Chen, Wentao and Han, Xiaotian and Lin, Xudong and Zhao, Haiteng and Liu, Yongfei and Zhai, Bohan and Yuan, Jianbo and You, Quanzeng and Yang, Hongxia},
  journal={arXiv preprint arXiv:2401.06805},
  year={2024}
}

@article{li2024surveying,
  title={Surveying the mllm landscape: A meta-review of current surveys},
  author={Li, Ming and Chen, Keyu and Bi, Ziqian and Liu, Ming and Peng, Benji and Niu, Qian and Liu, Junyu and Wang, Jinlang and Zhang, Sen and Pan, Xuanhe and others},
  journal={arXiv preprint arXiv:2409.18991},
  year={2024}
}


@article{wang2024comprehensive,
  title={A comprehensive review of multimodal large language models: Performance and challenges across different tasks},
  author={Wang, Jiaqi and Jiang, Hanqi and Liu, Yiheng and Ma, Chong and Zhang, Xu and Pan, Yi and Liu, Mengyuan and Gu, Peiran and Xia, Sichen and Li, Wenjun and others},
  journal={arXiv preprint arXiv:2408.01319},
  year={2024}
}


@article{royer2024multimedeval,
  title={Multimedeval: A benchmark and a toolkit for evaluating medical vision-language models},
  author={Royer, Corentin and Menze, Bjoern and Sekuboyina, Anjany},
  journal={arXiv preprint arXiv:2402.09262},
  year={2024}
}

@article{li2023llava,
  title={Llava-med: Training a large language-and-vision assistant for biomedicine in one day},
  author={Li, Chunyuan and Wong, Cliff and Zhang, Sheng and Usuyama, Naoto and Liu, Haotian and Yang, Jianwei and Naumann, Tristan and Poon, Hoifung and Gao, Jianfeng},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={28541--28564},
  year={2023}
}

@article{zhang2024generalist,
  title={A generalist vision--language foundation model for diverse biomedical tasks},
  author={Zhang, Kai and Zhou, Rong and Adhikarla, Eashan and Yan, Zhiling and Liu, Yixin and Yu, Jun and Liu, Zhengliang and Chen, Xun and Davison, Brian D and Ren, Hui and others},
  journal={Nature Medicine},
  pages={1--13},
  year={2024},
  publisher={Nature Publishing Group US New York}
}


@inproceedings{sima2024drivelm,
  title={Drivelm: Driving with graph visual question answering},
  author={Sima, Chonghao and Renz, Katrin and Chitta, Kashyap and Chen, Li and Zhang, Hanxue and Xie, Chengen and Bei{\ss}wenger, Jens and Luo, Ping and Geiger, Andreas and Li, Hongyang},
  booktitle={European Conference on Computer Vision},
  pages={256--274},
  year={2024},
  organization={Springer}
}

@article{shi2023exploring,
  title={Exploring ocr capabilities of gpt-4v (ision): A quantitative and in-depth evaluation},
  author={Shi, Yongxin and Peng, Dezhi and Liao, Wenhui and Lin, Zening and Chen, Xinhong and Liu, Chongyu and Zhang, Yuyi and Jin, Lianwen},
  journal={arXiv preprint arXiv:2310.16809},
  year={2023}
}

@inproceedings{kim2018textual,
  title={Textual explanations for self-driving vehicles},
  author={Kim, Jinkyu and Rohrbach, Anna and Darrell, Trevor and Canny, John and Akata, Zeynep},
  booktitle={Proceedings of the European conference on computer vision (ECCV)},
  pages={563--578},
  year={2018}
}

@inproceedings{qian2024nuscenes,
  title={Nuscenes-qa: A multi-modal visual question answering benchmark for autonomous driving scenario},
  author={Qian, Tianwen and Chen, Jingjing and Zhuo, Linhai and Jiao, Yang and Jiang, Yu-Gang},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={38},
  pages={4542--4550},
  year={2024}
}

@article{zhang2024benchmarking,
  title={Benchmarking trustworthiness of multimodal large language models: A comprehensive study},
  author={Zhang, Yichi and Huang, Yao and Sun, Yitong and Liu, Chang and Zhao, Zhe and Fang, Zhengwei and Wang, Yifan and Chen, Huanran and Yang, Xiao and Wei, Xingxing and others},
  journal={arXiv preprint arXiv:2406.07057},
  year={2024}
}

@article{li2024red,
  title={Red teaming visual language models},
  author={Li, Mukai and Li, Lei and Yin, Yuwei and Ahmed, Masood and Liu, Zhenguang and Liu, Qi},
  journal={arXiv preprint arXiv:2401.12915},
  year={2024}
}

@article{adler2024gpt,
  title={Gpt-4 technical report, 2024},
  author={Adler, Steven and Agarwal, Sandhini and Ahmad, Lama and Akkaya, Ilge and Aleman, Florencia Leoni and Almeida, Diogo and Altenschmidt, Janko and Altman, Sam and Anadkat, Shyamal and Avila, Red and others},
  journal={URL https://arxiv. org/abs/2303.08774},
  volume={2},
  pages={6},
  year={2024}
}

@article{anthropic2024claude,
  title={The claude 3 model family: Opus, sonnet, haiku},
  author={Anthropic, AI},
  journal={Claude-3 Model Card},
  volume={1},
  pages={1},
  year={2024}
}

@article{wang2024vlbiasbench,
  title={Vlbiasbench: A comprehensive benchmark for evaluating bias in large vision-language model},
  author={Wang, Sibo and Cao, Xiangkui and Zhang, Jie and Yuan, Zheng and Shan, Shiguang and Chen, Xilin and Gao, Wen},
  journal={arXiv preprint arXiv:2406.14194},
  year={2024}
}


@article{dang2024exploring,
  title={Exploring response uncertainty in mllms: An empirical evaluation under misleading scenarios},
  author={Dang, Yunkai and Gao, Mengxi and Yan, Yibo and Zou, Xin and Gu, Yanggan and Liu, Aiwei and Hu, Xuming},
  journal={arXiv preprint arXiv:2411.02708},
  year={2024}
}

@article{bai2024hallucination,
  title={Hallucination of multimodal large language models: A survey},
  author={Bai, Zechen and Wang, Pichao and Xiao, Tianjun and He, Tong and Han, Zongbo and Zhang, Zheng and Shou, Mike Zheng},
  journal={arXiv preprint arXiv:2404.18930},
  year={2024}
}

@article{yin2024woodpecker,
  title={Woodpecker: Hallucination correction for multimodal large language models},
  author={Yin, Shukang and Fu, Chaoyou and Zhao, Sirui and Xu, Tong and Wang, Hao and Sui, Dianbo and Shen, Yunhang and Li, Ke and Sun, Xing and Chen, Enhong},
  journal={Science China Information Sciences},
  volume={67},
  number={12},
  pages={220105},
  year={2024},
  publisher={Springer}
}

@article{yang2023mm,
  title={Mm-bigbench: Evaluating multimodal models on multimodal content comprehension tasks},
  author={Yang, Xiaocui and Wu, Wenfang and Feng, Shi and Wang, Ming and Wang, Daling and Li, Yang and Sun, Qi and Zhang, Yifei and Fu, Xiaoming and Poria, Soujanya},
  journal={arXiv preprint arXiv:2310.09036},
  year={2023}
}

@article{liu2023pre,
  title={Pre-train, prompt, and predict: A systematic survey of prompting methods in natural language processing},
  author={Liu, Pengfei and Yuan, Weizhe and Fu, Jinlan and Jiang, Zhengbao and Hayashi, Hiroaki and Neubig, Graham},
  journal={ACM computing surveys},
  volume={55},
  number={9},
  pages={1--35},
  year={2023},
  publisher={ACM New York, NY}
}

@article{xie2024tp,
  title={TP-Eval: Tap Multimodal LLMs' Potential in Evaluation by Customizing Prompts},
  author={Xie, Yuxuan and Li, Tianhua and Shao, Wenqi and Zhang, Kaipeng},
  journal={arXiv preprint arXiv:2410.18071},
  year={2024}
}


@article{kojima2022large,
  title={Large language models are zero-shot reasoners},
  author={Kojima, Takeshi and Gu, Shixiang Shane and Reid, Machel and Matsuo, Yutaka and Iwasawa, Yusuke},
  journal={Advances in neural information processing systems},
  volume={35},
  pages={22199--22213},
  year={2022}
}

@article{zhang2022automatic,
  title={Automatic chain of thought prompting in large language models},
  author={Zhang, Zhuosheng and Zhang, Aston and Li, Mu and Smola, Alex},
  journal={arXiv preprint arXiv:2210.03493},
  year={2022}
}

@article{rose2023visual,
  title={Visual chain of thought: bridging logical gaps with multimodal infillings},
  author={Rose, Daniel and Himakunthala, Vaishnavi and Ouyang, Andy and He, Ryan and Mei, Alex and Lu, Yujie and Saxon, Michael and Sonar, Chinmay and Mirza, Diba and Wang, William Yang},
  journal={arXiv preprint arXiv:2305.02317},
  year={2023}
}

@article{zhang2023multimodal,
  title={Multimodal chain-of-thought reasoning in language models},
  author={Zhang, Zhuosheng and Zhang, Aston and Li, Mu and Zhao, Hai and Karypis, George and Smola, Alex},
  journal={arXiv preprint arXiv:2302.00923},
  year={2023}
}

@article{ge2023chain,
  title={Chain of thought prompt tuning in vision language models},
  author={Ge, Jiaxin and Luo, Hongyin and Qian, Siyuan and Gan, Yulu and Fu, Jie and Zhang, Shanghang},
  journal={arXiv preprint arXiv:2304.07919},
  year={2023}
}

@article{razavi2025benchmarking,
  title={Benchmarking Prompt Sensitivity in Large Language Models},
  author={Razavi, Amirhossein and Soltangheis, Mina and Arabzadeh, Negar and Salamat, Sara and Zihayat, Morteza and Bagheri, Ebrahim},
  journal={arXiv preprint arXiv:2502.06065},
  year={2025}
}

@article{petroni2019language,
  title={Language models as knowledge bases?},
  author={Petroni, Fabio and Rockt{\"a}schel, Tim and Lewis, Patrick and Bakhtin, Anton and Wu, Yuxiang and Miller, Alexander H and Riedel, Sebastian},
  journal={arXiv preprint arXiv:1909.01066},
  year={2019}
}


@article{brown2020language,
  title={Language models are few-shot learners},
  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={1877--1901},
  year={2020}
}


@article{ouyang2020ernie,
  title={ERNIE-M: Enhanced multilingual representation by aligning cross-lingual semantics with monolingual corpora},
  author={Ouyang, Xuan and Wang, Shuohuan and Pang, Chao and Sun, Yu and Tian, Hao and Wu, Hua and Wang, Haifeng},
  journal={arXiv preprint arXiv:2012.15674},
  year={2020}
}

@article{schick2020exploiting,
  title={Exploiting cloze questions for few shot text classification and natural language inference},
  author={Schick, Timo and Sch{\"u}tze, Hinrich},
  journal={arXiv preprint arXiv:2001.07676},
  year={2020}
}

@article{schick2020s,
  title={It's not just size that matters: Small language models are also few-shot learners},
  author={Schick, Timo and Sch{\"u}tze, Hinrich},
  journal={arXiv preprint arXiv:2009.07118},
  year={2020}
}


@article{dang2024explainable,
  title={Explainable and interpretable multimodal large language models: A comprehensive survey},
  author={Dang, Yunkai and Huang, Kaichen and Huo, Jiahao and Yan, Yibo and Huang, Sirui and Liu, Dongrui and Gao, Mengxi and Zhang, Jie and Qian, Chen and Wang, Kun and others},
  journal={arXiv preprint arXiv:2412.02104},
  year={2024}
}


@article{nie2024survey,
  title={A survey of large language models for financial applications: Progress, prospects and challenges},
  author={Nie, Yuqi and Kong, Yaxuan and Dong, Xiaowen and Mulvey, John M and Poor, H Vincent and Wen, Qingsong and Zohren, Stefan},
  journal={arXiv preprint arXiv:2406.11903},
  year={2024}
}


@article{chakraborty2025hallucination,
  title={Hallucination detection in foundation models for decision-making: A flexible definition and review of the state of the art},
  author={Chakraborty, Neeloy and Ornik, Melkior and Driggs-Campbell, Katherine},
  journal={ACM Computing Surveys},
  year={2025},
  publisher={ACM New York, NY}
}

@article{honovich2022true,
  title={TRUE: Re-evaluating factual consistency evaluation},
  author={Honovich, Or and Aharoni, Roee and Herzig, Jonathan and Taitelbaum, Hagai and Kukliansy, Doron and Cohen, Vered and Scialom, Thomas and Szpektor, Idan and Hassidim, Avinatan and Matias, Yossi},
  journal={arXiv preprint arXiv:2204.04991},
  year={2022}
}

@article{laban2022summac,
  title={SummaC: Re-visiting NLI-based models for inconsistency detection in summarization},
  author={Laban, Philippe and Schnabel, Tobias and Bennett, Paul N and Hearst, Marti A},
  journal={Transactions of the Association for Computational Linguistics},
  volume={10},
  pages={163--177},
  year={2022},
  publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…}
}


@misc{vectara_hallucination_leaderboard,
  author    = {Vectara},
  title     = {Hallucination Leaderboard},
  year      = {2024},
  howpublished = {GitHub repository},
  url       = {https://github.com/vectara/hallucination-leaderboard}
}

@misc{galileo_hallucination_index,
  author    = {Galileo AI},
  title     = {The Hallucination Index: A New Paradigm for Evaluating AI Generated Content},
  year      = {2024},
  howpublished = {Galileo AI Blog},
  url       = {https://www.galileo.ai/blog/hallucination-index}
}





@article{tong2025cambrian,
  title={Cambrian-1: A fully open, vision-centric exploration of multimodal llms},
  author={Tong, Peter and Brown, Ellis and Wu, Penghao and Woo, Sanghyun and IYER, Adithya Jairam Vedagiri and Akula, Sai Charitha and Yang, Shusheng and Yang, Jihan and Middepogu, Manoj and Wang, Ziteng and others},
  journal={Advances in Neural Information Processing Systems},
  volume={37},
  pages={87310--87356},
  year={2025}
}

@article{malartic2024falcon2,
  title={Falcon2-11b technical report},
  author={Malartic, Quentin and Chowdhury, Nilabhra Roy and Cojocaru, Ruxandra and Farooq, Mugariya and Campesan, Giulia and Djilali, Yasser Abdelaziz Dahou and Narayan, Sanath and Singh, Ankit and Velikanov, Maksim and Boussaha, Basma El Amel and others},
  journal={arXiv preprint arXiv:2407.14885},
  year={2024}
}

@article{yao2024minicpm,
  title={Minicpm-v: A gpt-4v level mllm on your phone},
  author={Yao, Yuan and Yu, Tianyu and Zhang, Ao and Wang, Chongyi and Cui, Junbo and Zhu, Hongji and Cai, Tianchi and Li, Haoyu and Zhao, Weilin and He, Zhihui and others},
  journal={arXiv preprint arXiv:2408.01800},
  year={2024}
}

@inproceedings{ye2024mplug,
  title={mplug-owl3: Towards long image-sequence understanding in multi-modal large language models},
  author={Ye, Jiabo and Xu, Haiyang and Liu, Haowei and Hu, Anwen and Yan, Ming and Qian, Qi and Zhang, Ji and Huang, Fei and Zhou, Jingren},
  booktitle={The Thirteenth International Conference on Learning Representations},
  year={2024}
}

@article{deitke2024molmo,
  title={Molmo and pixmo: Open weights and open data for state-of-the-art multimodal models},
  author={Deitke, Matt and Clark, Christopher and Lee, Sangho and Tripathi, Rohun and Yang, Yue and Park, Jae Sung and Salehi, Mohammadreza and Muennighoff, Niklas and Lo, Kyle and Soldaini, Luca and others},
  journal={arXiv preprint arXiv:2409.17146},
  year={2024}
}


@misc{liu2024llavanext,
  title={Llavanext: Improved reasoning, ocr, and world knowledge},
  author={Liu, Haotian and Li, Chunyuan and Li, Yuheng and Li, Bo and Zhang, Yuanhan and Shen, Sheng and Lee, Yong Jae},
  year={2024}
}


@article{zeng2025scaling,
  title={Scaling Sparse and Dense Retrieval in Decoder-Only LLMs},
  author={Zeng, Hansi and Killingback, Julian and Zamani, Hamed},
  journal={arXiv preprint arXiv:2502.15526},
  year={2025}
}



@article{oquab2023dinov2,
  title={Dinov2: Learning robust visual features without supervision},
  author={Oquab, Maxime and Darcet, Timoth{\'e}e and Moutakanni, Th{\'e}o and Vo, Huy and Szafraniec, Marc and Khalidov, Vasil and Fernandez, Pierre and Haziza, Daniel and Massa, Francisco and El-Nouby, Alaaeldin and others},
  journal={arXiv preprint arXiv:2304.07193},
  year={2023}
}

@article{lu2023reference,
  title={A Reference Architecture for Designing Foundation Model based Systems},
  author={Lu, Qinghua and Zhu, Liming and Xu, Xiwei and Xing, Zhenchang and Whittle, Jon},
  journal={arXiv preprint arXiv:2304.11090},
  year={2023}
}

@article{arif2025fixing,
  title={Fixing Imbalanced Attention to Mitigate In-Context Hallucination of Large Vision-Language Model},
  author={Arif, Kazi Hasan Ibn and Dip, Sajib Acharjee and Hussain, Khizar and Zhang, Lang and Thomas, Chris},
  journal={arXiv preprint arXiv:2501.12206},
  year={2025}
}

@misc{OpenAI2025Models,
    author = {OpenAI},
    title = {OpenAI Models Documentation},
    year = {2025},
    url = {https://platform.openai.com/docs/models},
    note = {Accessed: 25 February 2025}
}

@misc{DeepMind2025Gemini,
    author = {DeepMind},
    title = {Gemini: Multimodal AI by DeepMind},
    year = {2025},
    url = {https://deepmind.google/technologies/gemini/},
    note = {Accessed: 25 February 2025}
}


@article{deepseek2025deepseek,
  title={Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning},
  author={DeepSeek-AI, Daya Guo and Yang, Dejian and Zhang, Haowei and Song, Junxiao and Zhang, Ruoyu and Xu, Runxin and Zhu, Qihao and Ma, Shirong and Wang, Peiyi and Bi, Xiao and others},
  journal={arXiv preprint arXiv:2501.12948},
  year={2025}
}

@article{jiang2024chatrex,
  title={ChatRex: Taming Multimodal LLM for Joint Perception and Understanding},
  author={Jiang, Qing and Yang, Yuqin and Xiong, Yuda and Chen, Yihao and Zeng, Zhaoyang and Ren, Tianhe and Zhang, Lei and others},
  journal={arXiv preprint arXiv:2411.18363},
  year={2024}
}

@article{fu2024vita,
  title={Vita: Towards open-source interactive omni multimodal llm},
  author={Fu, Chaoyou and Lin, Haojia and Long, Zuwei and Shen, Yunhang and Zhao, Meng and Zhang, Yifan and Dong, Shaoqi and Wang, Xiong and Yin, Di and Ma, Long and others},
  journal={arXiv preprint arXiv:2408.05211},
  year={2024}
}

@article{shen2025long,
  title={Long-VITA: Scaling Large Multi-modal Models to 1 Million Tokens with Leading Short-Context Accuray},
  author={Shen, Yunhang and Fu, Chaoyou and Dong, Shaoqi and Wang, Xiong and Chen, Peixian and Zhang, Mengdan and Cao, Haoyu and Li, Ke and Zheng, Xiawu and Zhang, Yan and others},
  journal={arXiv preprint arXiv:2502.05177},
  year={2025}
}

@article{lee2025meteor,
  title={Meteor: Mamba-based traversal of rationale for large language and vision models},
  author={Lee, Byung-Kwan and Kim, Chae Won and Park, Beomchan and Ro, Yong Man},
  journal={Advances in Neural Information Processing Systems},
  volume={37},
  pages={40278--40315},
  year={2025}
}

@inproceedings{lee2024moai,
  title={Moai: Mixture of all intelligence for large language and vision models},
  author={Lee, Byung-Kwan and Park, Beomchan and Won Kim, Chae and Man Ro, Yong},
  booktitle={European Conference on Computer Vision},
  pages={273--302},
  year={2024},
  organization={Springer}
}

@article{lu2021fantastically,
  title={Fantastically ordered prompts and where to find them: Overcoming few-shot prompt order sensitivity},
  author={Lu, Yao and Bartolo, Max and Moore, Alastair and Riedel, Sebastian and Stenetorp, Pontus},
  journal={arXiv preprint arXiv:2104.08786},
  year={2021}
}

@article{wu2022self,
  title={Self-adaptive in-context learning: An information compression perspective for in-context example selection and ordering},
  author={Wu, Zhiyong and Wang, Yaoxiang and Ye, Jiacheng and Kong, Lingpeng},
  journal={arXiv preprint arXiv:2212.10375},
  year={2022}
}

@article{guo2024can,
  title={Can Multimodal Large Language Model Think Analogically?},
  author={Guo, Diandian and Cao, Cong and Yuan, Fangfang and Wang, Dakui and Ma, Wei and Liu, Yanbing and Fu, Jianhui},
  journal={arXiv preprint arXiv:2411.01307},
  year={2024}
}


@article{dosovitskiy2020image,
  title={An image is worth 16x16 words: Transformers for image recognition at scale},
  author={Dosovitskiy, Alexey and Beyer, Lucas and Kolesnikov, Alexander and Weissenborn, Dirk and Zhai, Xiaohua and Unterthiner, Thomas and Dehghani, Mostafa and Minderer, Matthias and Heigold, Georg and Gelly, Sylvain and others},
  journal={arXiv preprint arXiv:2010.11929},
  year={2020}
}

@article{wu2020visual,
  title={Visual transformers: Token-based image representation and processing for computer vision},
  author={Wu, Bichen and Xu, Chenfeng and Dai, Xiaoliang and Wan, Alvin and Zhang, Peizhao and Yan, Zhicheng and Tomizuka, Masayoshi and Gonzalez, Joseph and Keutzer, Kurt and Vajda, Peter},
  journal={arXiv preprint arXiv:2006.03677},
  year={2020}
}

@article{xiao2021early,
  title={Early convolutions help transformers see better},
  author={Xiao, Tete and Singh, Mannat and Mintun, Eric and Darrell, Trevor and Doll{\'a}r, Piotr and Girshick, Ross},
  journal={Advances in neural information processing systems},
  volume={34},
  pages={30392--30400},
  year={2021}
}

@inproceedings{guan2024hallusionbench,
  title={Hallusionbench: an advanced diagnostic suite for entangled language hallucination and visual illusion in large vision-language models},
  author={Guan, Tianrui and Liu, Fuxiao and Wu, Xiyang and Xian, Ruiqi and Li, Zongxia and Liu, Xiaoyu and Wang, Xijun and Chen, Lichang and Huang, Furong and Yacoob, Yaser and others},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={14375--14385},
  year={2024}
}

@article{sivarajkumar2024empirical,
  title={An empirical evaluation of prompting strategies for large language models in zero-shot clinical natural language processing: algorithm development and validation study},
  author={Sivarajkumar, Sonish and Kelley, Mark and Samolyk-Mazzanti, Alyssa and Visweswaran, Shyam and Wang, Yanshan},
  journal={JMIR Medical Informatics},
  volume={12},
  pages={e55318},
  year={2024},
  publisher={JMIR Publications Toronto, Canada}
}

@article{jaech2024openai,
  title={Openai o1 system card},
  author={Jaech, Aaron and Kalai, Adam and Lerer, Adam and Richardson, Adam and El-Kishky, Ahmed and Low, Aiden and Helyar, Alec and Madry, Aleksander and Beutel, Alex and Carney, Alex and others},
  journal={arXiv preprint arXiv:2412.16720},
  year={2024}
}


@article{chen2023minigpt,
  title={Minigpt-v2: large language model as a unified interface for vision-language multi-task learning},
  author={Chen, Jun and Zhu, Deyao and Shen, Xiaoqian and Li, Xiang and Liu, Zechun and Zhang, Pengchuan and Krishnamoorthi, Raghuraman and Chandra, Vikas and Xiong, Yunyang and Elhoseiny, Mohamed},
  journal={arXiv preprint arXiv:2310.09478},
  year={2023}
}


@inproceedings{liu2024improved,
  title={Improved baselines with visual instruction tuning},
  author={Liu, Haotian and Li, Chunyuan and Li, Yuheng and Lee, Yong Jae},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={26296--26306},
  year={2024}
}

@article{chen2023shikra,
  title={Shikra: Unleashing multimodal llm's referential dialogue magic},
  author={Chen, Keqin and Zhang, Zhao and Zeng, Weili and Zhang, Richong and Zhu, Feng and Zhao, Rui},
  journal={arXiv preprint arXiv:2306.15195},
  year={2023}
}

@inproceedings{jiangmmad,
  title={MMAD: A Comprehensive Benchmark for Multimodal Large Language Models in Industrial Anomaly Detection},
  author={Jiang, Xi and Li, Jian and Deng, Hanqiu and Liu, Yong and Gao, Bin-Bin and Zhou, Yifeng and Li, Jialin and Wang, Chengjie and Zheng, Feng},
  booktitle={The Thirteenth International Conference on Learning Representations},
year={2025}
}

@article{huang2023language,
  title={Language is not all you need: Aligning perception with language models},
  author={Huang, Shaohan and Dong, Li and Wang, Wenhui and Hao, Yaru and Singhal, Saksham and Ma, Shuming and Lv, Tengchao and Cui, Lei and Mohammed, Owais Khan and Patra, Barun and others},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={72096--72109},
  year={2023}
}

@article{zhang2024mme,
  title={MME-RealWorld: Could Your Multimodal LLM Challenge High-Resolution Real-World Scenarios that are Difficult for Humans?},
  author={Zhang, Yi-Fan and Zhang, Huanyu and Tian, Haochen and Fu, Chaoyou and Zhang, Shuangqing and Wu, Junfei and Li, Feng and Wang, Kun and Wen, Qingsong and Zhang, Zhang and others},
  journal={arXiv preprint arXiv:2408.13257},
  year={2024}
}


@article{bao2021beit,
  title={Beit: Bert pre-training of image transformers},
  author={Bao, Hangbo and Dong, Li and Piao, Songhao and Wei, Furu},
  journal={arXiv preprint arXiv:2106.08254},
  year={2021}
}