\relax 
\providecommand\hyper@newdestlabel[2]{}
\providecommand\HyField@AuxAddToFields[1]{}
\providecommand\HyField@AuxAddToCoFields[2]{}
\bibstyle{plainnat}
\citation{vorontsovVirchowMillionSlideDigital2024,ikezogwoQuilt1MOneMillion2023}
\citation{wangPathologyFoundationModel2024a,chenGeneralpurposeFoundationModel2024b}
\citation{luVisualLanguageFoundationModel2023}
\citation{luVisualLanguageFoundationModel2023,shaikovskiPRISMMultiModalGenerative2024,ahmedPathAlignVisionlanguageModel2024,zhangContrastiveLearningMedical2022a}
\citation{luVisualLanguageFoundationModel2023}
\citation{luVisualLanguageFoundationModel2023,shaikovskiPRISMMultiModalGenerative2024}
\citation{tran2024generating}
\citation{laurenconWhatMattersWhen2024,karamchetiPrismaticVLMsInvestigating2024a,zhangWhyAreVisuallyGrounded2024a,tongEyesWideShut2024,zhaiInvestigatingCatastrophicForgetting2023}
\citation{zhangWhyAreVisuallyGrounded2024a}
\citation{zhaiInvestigatingCatastrophicForgetting2023}
\citation{kirkpatrickOvercomingCatastrophicForgetting2017,perkoniggDynamicMemoryAlleviate2021,bandiContinualLearningStrategies2023}
\citation{kirkpatrickOvercomingCatastrophicForgetting2017}
\citation{rusuProgressiveNeuralNetworks2022}
\citation{rebuffiICaRLIncrementalClassifier2017}
\providecommand \oddpage@label [2]{}
\jmlr@workshop{Full Paper -- MIDL 2025}
\jmlr@title{Balancing Classification and Retrieval in Cross-modal Vision Models}{A Balancing Act: Optimizing Classification and Retrieval in Cross-Modal Vision Models}
\jmlr@author{\begin {NoHyper}\Name {Judith Lefkes\nametag {$^{1,2}$}} \Email {judith.lefkes@radboudumc.nl}\\ \Name {Cl\'ement Grisi\nametag {$^{1,2}$}} \Email {clement.grisi@radboudumc.nl} \\ \Name {Geert Litjens\nametag {$^{1,2}$}} \Email {geert.litjens@radboudumc.nl}\\ \addr $^{1}$ Computational Pathology Group, Radboudumc, Nijmegen, Netherlands \\ \addr $^{2}$ Oncode Institute, Utrecht, the Netherlands \end {NoHyper}}{\begin {NoHyper}\Name {Judith Lefkes\nametag {$^{1,2}$}} \Email {judith.lefkes@radboudumc.nl}\\ \Name {Cl\'ement Grisi\nametag {$^{1,2}$}} \Email {clement.grisi@radboudumc.nl} \\ \Name {Geert Litjens\nametag {$^{1,2}$}} \Email {geert.litjens@radboudumc.nl}\\ \addr $^{1}$ Computational Pathology Group, Radboudumc, Nijmegen, Netherlands \\ \addr $^{2}$ Oncode Institute, Utrecht, the Netherlands \end {NoHyper}}
\newlabel{jmlrstart}{{}{1}{}{Doc-Start}{}}
\@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{1}{section.0.1}\protected@file@percent }
\citation{linMicrosoftCOCOCommon2015}
\@writefile{toc}{\contentsline {section}{\numberline {2}Methods}{2}{section.0.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {2.1}Experimental setup}{2}{subsection.0.2.1}\protected@file@percent }
\citation{wuVisualTransformersTokenbased2020}
\citation{reimersSentenceBERTSentenceEmbeddings2019a}
\citation{radfordLearningTransferableVisual2021b}
\citation{schroffFaceNetUnifiedEmbedding2015}
\citation{bultenArtificialIntelligenceDiagnosis2022}
\citation{grisiHierarchicalVisionTransformers2023}
\citation{joeranbosmaDragonrobertabasedomainspecific2024}
\citation{leeBioBERTPretrainedBiomedical2020}
\@writefile{toc}{\contentsline {subsection}{\numberline {2.2}COCO Experiments}{3}{subsection.0.2.2}\protected@file@percent }
\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces \textbf  {(A)} An example case from the COCO dataset comprising a natural image, one or multiple label(s), and five reference captions. \textbf  {(B)} An example case from the prostate biopsy data comprising a thumbnail of a WSI, its corresponding ISUP grade, and the pathology report. \textbf  {(C)} Label distribution of the prostate biopsy data.}}{4}{figure.1}\protected@file@percent }
\newlabel{training_sample}{{1}{4}{\textbf {(A)} An example case from the COCO dataset comprising a natural image, one or multiple label(s), and five reference captions. \textbf {(B)} An example case from the prostate biopsy data comprising a thumbnail of a WSI, its corresponding ISUP grade, and the pathology report. \textbf {(C)} Label distribution of the prostate biopsy data}{figure.1}{}}
\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces Implementation Details for COCO and Prostate Biopsy Grading Experiments. Full loss formulations per task are provided in Appendix \ref {losses}.}}{4}{table.1}\protected@file@percent }
\newlabel{tab:implementation_details}{{1}{4}{Implementation Details for COCO and Prostate Biopsy Grading Experiments. Full loss formulations per task are provided in Appendix \ref {losses}}{table.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {2.3}Prostate biopsy grading experiments}{4}{subsection.0.2.3}\protected@file@percent }
\@writefile{lof}{\contentsline {figure}{\numberline {2}{\ignorespaces Validation performance metrics during contrastive tuning using $\lambda = 1.0$ for the COCO dataset in \textbf  {(A)} and the prostate biopsy grading experiments in \textbf  {(B)}. Lines represent the medians across five folds, with shaded areas indicating the interquartile range.}}{5}{figure.2}\protected@file@percent }
\newlabel{fig:lambda_1.0 results}{{2}{5}{Validation performance metrics during contrastive tuning using $\lambda = 1.0$ for the COCO dataset in \textbf {(A)} and the prostate biopsy grading experiments in \textbf {(B)}. Lines represent the medians across five folds, with shaded areas indicating the interquartile range}{figure.2}{}}
\@writefile{toc}{\contentsline {section}{\numberline {3}Results}{5}{section.0.3}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {3.1}COCO}{5}{subsection.0.3.1}\protected@file@percent }
\newlabel{roberta_coco_results}{{3.1}{5}{COCO}{subsection.0.3.1}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {3}{\ignorespaces Impact of $\lambda $ on the classification-alignment trade-off for COCO with \textbf  {(A)} $\lambda \in [0.1, 0.9]$ and \textbf  {(B)} $\lambda \in [0.9, 1.0)$.}}{6}{figure.3}\protected@file@percent }
\newlabel{fig:combined_results_coco}{{3}{6}{Impact of $\lambda $ on the classification-alignment trade-off for COCO with \textbf {(A)} $\lambda \in [0.1, 0.9]$ and \textbf {(B)} $\lambda \in [0.9, 1.0)$}{figure.3}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.2}Prostate biopsy grading}{7}{subsection.0.3.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {3.3}Test performances COCO and Prostate grading experiments}{7}{subsection.0.3.3}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {4}Discussion}{7}{section.0.4}\protected@file@percent }
\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces Impact of $\lambda $ on the classification-alignment trade-off for the prostate biopsy grading experiments with \textbf  {(A)} $\lambda \in [0.1, 0.9]$ and \textbf  {(B)} $\lambda \in [0.9, 1.0]$.}}{8}{figure.4}\protected@file@percent }
\newlabel{fig:combined_results_medical}{{4}{8}{Impact of $\lambda $ on the classification-alignment trade-off for the prostate biopsy grading experiments with \textbf {(A)} $\lambda \in [0.1, 0.9]$ and \textbf {(B)} $\lambda \in [0.9, 1.0]$}{figure.4}{}}
\@writefile{lot}{\contentsline {table}{\numberline {2}{\ignorespaces Test performance on COCO ($N=4,690$) and Prostate Biopsies ($N=35$) with early stopping at peak retrieval performance. $\Delta \%$ denotes the relative change in testset classification performance w.r.t the baseline for COCO ($0.77$) and Prostate Biopsies ($0.80$). We report median values (Q1–Q3) across five folds.}}{8}{table.2}\protected@file@percent }
\newlabel{tab:performance}{{2}{8}{Test performance on COCO ($N=4,690$) and Prostate Biopsies ($N=35$) with early stopping at peak retrieval performance. $\Delta \%$ denotes the relative change in testset classification performance w.r.t the baseline for COCO ($0.77$) and Prostate Biopsies ($0.80$). We report median values (Q1–Q3) across five folds}{table.2}{}}
\bibdata{midl25_202}
\bibcite{ahmedPathAlignVisionlanguageModel2024}{{1}{2024}{{Ahmed et~al.}}{{Ahmed, Sellergren, Yang, Xu, Babenko, Ward, Olson, Mohtashamian, Matias, Corrado, et~al.}}}
\bibcite{bandiContinualLearningStrategies2023}{{2}{2023}{{B{\'a}ndi et~al.}}{{B{\'a}ndi, Balkenhol, Van~Dijk, Kok, van Ginneken, van~der Laak, and Litjens}}}
\bibcite{joeranbosmaDragonrobertabasedomainspecific2024}{{3}{2025}{{Bosma et~al.}}{{Bosma, Dercksen, Builtjes, Andr{\'e}, Roest, Fransen, Noordman, Navarro-Padilla, Lefkes, Alves, et~al.}}}
\bibcite{bultenArtificialIntelligenceDiagnosis2022}{{4}{2022}{{Bulten et~al.}}{{Bulten, Kartasalo, Chen, Str{\"o}m, Pinckaers, Nagpal, Cai, Steiner, Van~Boven, Vink, et~al.}}}
\bibcite{chenGeneralpurposeFoundationModel2024b}{{5}{2024}{{Chen et~al.}}{{Chen, Ding, Lu, Williamson, Jaume, Song, Chen, Zhang, Shao, Shaban, et~al.}}}
\bibcite{grisiHierarchicalVisionTransformers2023}{{6}{2023}{{Grisi et~al.}}{{Grisi, Litjens, and van~der Laak}}}
\bibcite{ikezogwoQuilt1MOneMillion2023}{{7}{2023}{{Ikezogwo et~al.}}{{Ikezogwo, Seyfioglu, Ghezloo, Geva, Sheikh~Mohammed, Anand, Krishna, and Shapiro}}}
\bibcite{karamchetiPrismaticVLMsInvestigating2024a}{{8}{2024}{{Karamcheti et~al.}}{{Karamcheti, Nair, Balakrishna, Liang, Kollar, and Sadigh}}}
\bibcite{kirkpatrickOvercomingCatastrophicForgetting2017}{{9}{2017}{{Kirkpatrick et~al.}}{{Kirkpatrick, Pascanu, Rabinowitz, Veness, Desjardins, Rusu, Milan, Quan, Ramalho, Grabska-Barwinska, et~al.}}}
\bibcite{laurenconWhatMattersWhen2024}{{10}{2024}{{Lauren{\c {c}}on et~al.}}{{Lauren{\c {c}}on, Tronchon, Cord, and Sanh}}}
\bibcite{leeBioBERTPretrainedBiomedical2020}{{11}{2020}{{Lee et~al.}}{{Lee, Yoon, Kim, Kim, Kim, So, and Kang}}}
\bibcite{linMicrosoftCOCOCommon2015}{{12}{2014}{{Lin et~al.}}{{Lin, Maire, Belongie, Hays, Perona, Ramanan, Doll{\'a}r, and Zitnick}}}
\bibcite{luVisualLanguageFoundationModel2023}{{13}{2024}{{Lu et~al.}}{{Lu, Chen, Williamson, Chen, Liang, Ding, Jaume, Odintsov, Le, Gerber, et~al.}}}
\bibcite{perkoniggDynamicMemoryAlleviate2021}{{14}{2021}{{Perkonigg et~al.}}{{Perkonigg, Hofmanninger, Herold, Brink, Pianykh, Prosch, and Langs}}}
\bibcite{radfordLearningTransferableVisual2021b}{{15}{2021}{{Radford et~al.}}{{Radford, Kim, Hallacy, Ramesh, Goh, Agarwal, Sastry, Askell, Mishkin, Clark, et~al.}}}
\bibcite{rebuffiICaRLIncrementalClassifier2017}{{16}{2017}{{Rebuffi et~al.}}{{Rebuffi, Kolesnikov, Sperl, and Lampert}}}
\bibcite{reimersSentenceBERTSentenceEmbeddings2019a}{{17}{2019}{{Reimers and Gurevych}}{{}}}
\bibcite{rusuProgressiveNeuralNetworks2022}{{18}{2016}{{Rusu et~al.}}{{Rusu, Rabinowitz, Desjardins, Soyer, Kirkpatrick, Kavukcuoglu, Pascanu, and Hadsell}}}
\bibcite{schroffFaceNetUnifiedEmbedding2015}{{19}{2015}{{Schroff et~al.}}{{Schroff, Kalenichenko, and Philbin}}}
\bibcite{shaikovskiPRISMMultiModalGenerative2024}{{20}{2024}{{Shaikovski et~al.}}{{Shaikovski, Casson, Severson, Zimmermann, Wang, Kunz, Retamero, Oakley, Klimstra, Kanan, et~al.}}}
\bibcite{tongEyesWideShut2024}{{21}{2024}{{Tong et~al.}}{{Tong, Liu, Zhai, Ma, LeCun, and Xie}}}
\bibcite{tran2024generating}{{22}{2024}{{Tran et~al.}}{{Tran, Schmidle, Wagner, Koch, Lupperger, Feuchtinger, B{\"o}hner, Kaczmarczyk, Biedermann, Eyerich, et~al.}}}
\bibcite{vorontsovVirchowMillionSlideDigital2024}{{23}{2023}{{Vorontsov et~al.}}{{Vorontsov, Bozkurt, Casson, Shaikovski, Zelechowski, Liu, Severson, Zimmermann, Hall, Tenenholtz, et~al.}}}
\bibcite{wangPathologyFoundationModel2024a}{{24}{2024}{{Wang et~al.}}{{Wang, Zhao, Marostica, Yuan, Jin, Zhang, Li, Tang, Wang, Li, et~al.}}}
\bibcite{wuVisualTransformersTokenbased2020}{{25}{2020}{{Wu et~al.}}{{Wu, Xu, Dai, Wan, Zhang, Yan, Tomizuka, Gonzalez, Keutzer, and Vajda}}}
\bibcite{zhaiInvestigatingCatastrophicForgetting2023}{{26}{2023}{{Zhai et~al.}}{{Zhai, Tong, Li, Cai, Qu, Lee, and Ma}}}
\bibcite{zhangContrastiveLearningMedical2022a}{{27}{2022}{{Zhang et~al.}}{{Zhang, Jiang, Miura, Manning, and Langlotz}}}
\bibcite{zhangWhyAreVisuallyGrounded2024a}{{28}{2024}{{Zhang et~al.}}{{Zhang, Unell, Wang, Ghosh, Su, Schmidt, and Yeung-Levy}}}
\citation{joeranbosmaDragonrobertabasedomainspecific2024}
\@writefile{toc}{\contentsline {section}{\numberline {A}Fine-tuning Details for the Task-specific Vision Encoder in COCO}{13}{section.0.A}\protected@file@percent }
\newlabel{finetuning_coco_vision_encoder}{{A}{13}{Fine-tuning Details for the Task-specific Vision Encoder in COCO}{section.0.A}{}}
\@writefile{toc}{\contentsline {section}{\numberline {B}Fine-Tuning Details for the Language Encoder Pretrained on Dutch Medical Reports}{13}{section.0.B}\protected@file@percent }
\newlabel{finetuning details roberta language model}{{B}{13}{Fine-Tuning Details for the Language Encoder Pretrained on Dutch Medical Reports}{section.0.B}{}}
\@writefile{toc}{\contentsline {section}{\numberline {C}Impact of Language Encoder Choice: Results with BioBERT and MPNet}{14}{section.0.C}\protected@file@percent }
\newlabel{appendix_dual_plot_coco_medical}{{C}{14}{Impact of Language Encoder Choice: Results with BioBERT and MPNet}{section.0.C}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {5}{\ignorespaces Validation performance metrics during contrastive tuning with $\lambda = 1.0$. Panel \textbf  {(A)} presents classification loss, mAP, and Recall@5 for the COCO dataset, where text embeddings are computed using the \texttt  {MPNet} model. Panel \textbf  {(B)} shows validation classification loss, $\kappa ^2$ and \textit  {retrieval} $\kappa ^2$ for prostate cancer grading experiments, where report embeddings are derived from the \texttt  {BioBERT} model. Both panels illustrate a clear trade-off, where classification performance is sacrificed in exchange for improved retrieval. In all figures, metrics are reported starting from the first epoch of fine-tuning. Lines represent the median across five folds, with shaded areas indicating the interquartile range (IQR).}}{14}{figure.5}\protected@file@percent }
\newlabel{fig:lambda_1.0 results_different_language_encoder}{{5}{14}{Validation performance metrics during contrastive tuning with $\lambda = 1.0$. Panel \textbf {(A)} presents classification loss, mAP, and Recall@5 for the COCO dataset, where text embeddings are computed using the \texttt {MPNet} model. Panel \textbf {(B)} shows validation classification loss, $\kappa ^2$ and \textit {retrieval} $\kappa ^2$ for prostate cancer grading experiments, where report embeddings are derived from the \texttt {BioBERT} model. Both panels illustrate a clear trade-off, where classification performance is sacrificed in exchange for improved retrieval. In all figures, metrics are reported starting from the first epoch of fine-tuning. Lines represent the median across five folds, with shaded areas indicating the interquartile range (IQR)}{figure.5}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {C.1}Contrastive Tuning Experiments on COCO using \textbf  {MPNet} as a Language Encoder}{15}{subsection.0.C.1}\protected@file@percent }
\newlabel{appendix_mpnet_results}{{C.1}{15}{Contrastive Tuning Experiments on COCO using \textbf {MPNet} as a Language Encoder}{subsection.0.C.1}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {6}{\ignorespaces Impact of $\lambda $ on the classification-alignment trade-off for COCO with \textbf  {(A)} $\lambda \in [0.1, 0.9]$ and \textbf  {(B)} $\lambda \in [0.9, 1.0)$. We used the same Vit-Base model for the vision encoder while generating text embeddings using the frozen \texttt  {multi-qa-mpnet-base-dot-v1} model as the language encoder. }}{15}{figure.6}\protected@file@percent }
\newlabel{fig:mpnet_appendix}{{6}{15}{Impact of $\lambda $ on the classification-alignment trade-off for COCO with \textbf {(A)} $\lambda \in [0.1, 0.9]$ and \textbf {(B)} $\lambda \in [0.9, 1.0)$. We used the same Vit-Base model for the vision encoder while generating text embeddings using the frozen \texttt {multi-qa-mpnet-base-dot-v1} model as the language encoder}{figure.6}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {C.2}Contrastive Tuning Experiments on the Prostate Biopsy Data using \textbf  {BioBERT} as a Language Encoder}{16}{subsection.0.C.2}\protected@file@percent }
\newlabel{appendix_bioebert_results}{{C.2}{16}{Contrastive Tuning Experiments on the Prostate Biopsy Data using \textbf {BioBERT} as a Language Encoder}{subsection.0.C.2}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {7}{\ignorespaces Impact of $\lambda $ on the classification-alignment trade-off for the prostate cancer grading task with \textbf  {(A)} $\lambda \in [0.1, 0.9]$ and \textbf  {(B)} $\lambda \in [0.9, 1.0)$. We use the same H-ViT model as the vision encoder while generating text embeddings with the frozen \texttt  {BioBERT} model as the language encoder.}}{16}{figure.7}\protected@file@percent }
\newlabel{fig:mi}{{7}{16}{Impact of $\lambda $ on the classification-alignment trade-off for the prostate cancer grading task with \textbf {(A)} $\lambda \in [0.1, 0.9]$ and \textbf {(B)} $\lambda \in [0.9, 1.0)$. We use the same H-ViT model as the vision encoder while generating text embeddings with the frozen \texttt {BioBERT} model as the language encoder}{figure.7}{}}
\@writefile{toc}{\contentsline {section}{\numberline {D}Loss functions}{16}{section.0.D}\protected@file@percent }
\newlabel{losses}{{D}{16}{Loss functions}{section.0.D}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {D.1}COCO}{16}{subsection.0.D.1}\protected@file@percent }
\newlabel{loss_coco}{{D.1}{16}{COCO}{subsection.0.D.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {D.2}Prostate biopsy grading experiments}{17}{subsection.0.D.2}\protected@file@percent }
\newlabel{loss_prostate}{{D.2}{17}{Prostate biopsy grading experiments}{subsection.0.D.2}{}}
\newlabel{jmlrend}{{D.2}{17}{end of Balancing Classification and Retrieval in Cross-modal Vision Models}{section*.2}{}}
\gdef \@abspage@last{17}
