\PassOptionsToPackage{table}{xcolor}
\documentclass{midl}
% The following packages will be automatically loaded:
% jmlr, amsmath, amssymb, natbib, graphicx, url, algorithm2e
% ifoddpage, relsize and probably more
% make sure they are installed with your latex distribution
\usepackage{booktabs}
\usepackage[table]{xcolor}
\usepackage{multirow}
\usepackage{amssymb}% http://ctan.org/pkg/amssymb
\usepackage{pifont}% http://ctan.org/pkg/pifont
\newcommand{\cmark}{\ding{51}}%
\newcommand{\xmark}{\ding{55}}%
\usepackage{mwe} % to get dummy images

\jmlryear{2024}
\jmlrworkshop{Full Paper -- MIDL 2024}
\jmlrvolume{-- 169}
\editors{Accepted for publication at MIDL 2024}

\title[Segmentation-guided multimodal registration]{Multimodal Image Registration Guided by Few Segmentations from One Modality}

\midlauthor{\Name{Ba\c{s}ar Demir} \Email{bdemir@cs.unc.edu}\\
\Name{Marc Niethammer} \Email{mn@cs.unc.edu}\\
\addr University of North Carolina at Chapel Hill, North Carolina, USA}

\begin{document}
\maketitle
      
\begin{abstract}
Registration of multimodal images is challenging, especially when dealing with different anatomical structures and samples without segmentations. The main difficulty arises from the use of registration loss functions that are inadequate in the absence of corresponding regions. In this work, we present the first registration and segmentation approach tailored to this challenge. In particular, we assume the practically highly relevant scenario that only a limited number of segmentations are available for one modality and none for the other. First, we augment our few segmented samples using unsupervised deep registration within one modality, thereby providing many anatomically plausible samples to train a segmentation network. The resulting segmentation network then allows us to train a segmentation network on the target modality without available segmentations by using an unsupervised domain adaptation architecture. Finally, we train a deep registration network to register multimodal image pairs purely based on predictions of their segmentation networks. Our work demonstrates that using a small number of segmentations from one modality enables training a segmentation network on a target modality without the need for additional manual segmentations on that modality. Additionally, we show that registration based on these segmentations provides smooth and accurate deformation fields on anatomically different image pairs, unlike previous methods. We evaluate our approach on 2D medical image segmentation and registration between knee DXA and X-ray images. Our experiments show that our approach outperforms existing methods. \emph{Code is available at} \url{https://github.com/uncbiag/SegGuidedMMReg}.

\end{abstract}

\begin{keywords}
multimodal image registration, few-shot learning %, UK Biobank
\end{keywords}

\section{Introduction}
\label{sec:intro}
Image registration is a crucial medical imaging task in establishing correspondences between image pairs. It is commonly used for disease diagnosis \cite{cardiac-diagnosis}, disease progression monitoring \cite{progression-tracking}, and organ motion tracking \cite{lungtracking}. At its core is the solution of an inverse problem to determine the unknown spatial transformation between a moving and a fixed image, such that the deformed moving image matches the fixed image well. Solving this inverse problem involves solving a possibly high-dimensional optimization problem. One can \emph{directly} optimize over the parameters of a chosen transformation model (e.g., a displacement vector field) or over the parameters of a deep neural network which then \emph{indirectly} produces the desired transformation by regression. Deep-learning approaches~\cite{voxelmorph, quicksilver} are typically significantly faster than direct optimization approaches~\cite{ants, convex-opt} and can achieve state-of-the-art (SOTA) accuracies~\cite{gradicon, lapirn} especially when combined with instance optimization.

All of these methods aim to optimize a loss function that is a compromise between image pair dissimilarity and transformation irregularity. Multimodal image registration is still significantly more challenging than monomodal image registration because it requires image dissimilarity costs that can assess whether two images from different modalities differ or should be considered spatially well-matched. Further, Fig.~\ref{fig:loss-visuals} illustrates that the visibility of different anatomical structures across different modalities may cause a registration model to predict incorrect correspondences between different anatomical structures. \emph{We hypothesize that a registration model trained using supervision based on segmentations available for both modalities can assist the registration model in focusing on common regions of image pairs.} However, this presupposes the availability of these segmentations. Our focus in this work is, therefore, to develop an approach that solves both image segmentation and registration for multimodal image pairs, assuming that 1) anatomies are not consistently visualized between the modalities (e.g., one might focus on bone visualization while another might visualize both bones and soft tissue) and 2) only a few manual segmentations are available in one modality and none in the other.

\noindent{\textbf{In summary, the contributions of our work are:}}
\begin{itemize}
\item We propose the first deep image registration approach for multimodal images where image pairs visualize different anatomical parts. Our approach is based on registering segmentations while only requiring a few segmentations from one modality and none for the other.
\item We test our approach for 2D knee datasets between DXA images which focus on imaging bone, and radiographs (termed X-rays in what follows) which visualize bones and soft tissue.
\item Compared to existing loss functions and approaches, our approach can reliably register DXA and X-ray image pairs and predicts smooth deformation fields on anatomically different image pairs.
\end{itemize}

\begin{figure}[htbp]
\floatconts
  {fig:loss-visuals}
  {\caption{Multimodal registration results for registration using different similarity measures. Existing multimodal registration approaches based on image intensities fail to capture anatomical correspondences, whereas our segmentation-based approach (see {\color{red}red box}) predicts smooth and accurate deformation fields. \emph{DXA images reproduced by kind permission of the UK Biobank\textsuperscript{\textregistered}.}}}
  {\includegraphics[width=1\linewidth]{figures/different_losses.png}}
\end{figure}

\section{Related Work}
\label{sec:related_work}
\noindent \textbf{Image Registration.} Image registration is a key tool of medical image analysis to estimate spatial correspondences between images. Recently, deep neural networks have enabled population-based registration approaches that require one-time training but then provide very fast inference~\cite{voxelmorph, cnnregression, icon, gradicon, iconbyconstr}. However, registration performance still heavily depends on selecting a correct task-specific image similarity measure.

For monomodal applications, mean squared error (MSE) \cite{birnet, hypermorph, cyclemorph} and normalized cross-correlation (NCC)~\cite{diffusemorph, admir} are popular image similarity measures. In contrast, multimodal registration cannot rely on matching image intensities directly. Hence, measures of statistical dependence such as NCC and Mutual Information~\cite{MI} are popular choices. Another option, Modality Independent Neighbourhood Descriptors (MIND) \cite{mind}, operates by comparing local image neighborhoods in both the moving and fixed images. Many other image similarity measures exist (e.g., structural similarity index (SSIM) \cite{ssim} and normalized gradient fields (NGF) \cite{gradient_field}). However, none of these similarity measures are usable for image pairs which show different anatomical parts (see Fig.~\ref{fig:loss-visuals}). Specifically, registrations based on these similarity measures fail to find appropriate correspondences even between common anatomical structures when one of the images contains anatomical structures that are not visualized in the other image.

Recent works \cite{deep_atlas, segmentation_registration, segmentation_registration2} show that segmentation can supervise registration and improve registration performance with better border alignment. Further, such supervision encourages segmentations to match after registration, ensuring the preservation of anatomical consistency. Inspired by these works, we propose a \emph{registration strategy that can find correspondences fully based on given segmentations tailored for anatomically different and multimodal image pairs.}

\noindent \textbf{Domain Adaptation for Segmentation.} 
Machine learning models often struggle with data that differs significantly from their training data. This is a major challenge in medical imaging, where datasets can vary greatly in modality (e.g., MRI vs. CT) even when imaging the same body region. Moreover, manual segmentations are typically available for only a limited number of images and modalities. Unsupervised domain adaptation (UDA) offers a solution to this challenge. UDA techniques adapt models trained on one domain (source) to perform well on a related but different domain (target) without requiring segmented target data. This is crucial for generalizing models in medical imaging, where acquiring large segmented datasets is expensive and time-consuming. Building on the success of adversarial image translation methods \cite{cogan, pix2pix, cyclegan}, several techniques have been proposed for pixel-level UDA \cite{translation1, translation2, translation3}. These methods initially establish a mapping function between the source and target domains, enabling inter-domain image translation.

Recent works \cite{cycada, xie2018learning} enhance UDA by combining feature alignment and pixel-level adversarial training. In our approach, we use an \emph{CycleGAN \cite{cyclegan} unsupervised domain adaptation strategy \emph{with semantic losses} at each step of translation~\cite{irospaper}}. These semantic losses ensure that modality-translated images not only appear to belong to the target image domain but also maintain consistent anatomical boundaries during image translation. This is an essential property for our registration approach. \emph{Further, the approach allows us to train a segmentation network on the target modality using only source domain segmentations.}

\section{Methodology}
\subsection{Overview}
Our source dataset $D_{S}=\{\left(I_{i}\right)\}_{i \in [n]}$  includes at least one segmentation $\{\left(S_{i}\right)\}_{i \in [l]}$ ($1 \leq l \leq n$). No segmentations are available for our target dataset $D_{T}=\{(I_{i})\}_{i \in [m]}$. First, we augment the source dataset $D_{S}$ with a registration network (Sec.~\ref{sub:augmentation}). Subsequently, we train the segmentation network $S_{S}$ for the source domain using the augmented dataset and freeze weights of the network. Next, we obtain a segmentation network $S_{T}$ for the target dataset with the guidance of $S_{S}$ via an unsupervised domain adaptation network (Sec.~\ref{sub:domain-adaptation}). We predict segmentations for the non-segmented part of the source dataset with $S_{S}$ and apply $S_{T}$ to the target dataset. Finally, we train a registration network solely based on the segmentations we obtained (Sec.~\ref{sub:segmentation_registration}). This registration network then enables us to register two images according to their predicted or provided segmentations.

\subsection{Augmentation with Registration}
\label{sub:augmentation}
A non-parametric registration network predicts a displacement field $u = R(I_{m}, I_{f}; \theta)$ between the moving image $I_{m}$ and the fixed image $I_{f}$ for given network parameters $\theta$. We can warp the moving image to the fixed image as $I_{m}^{w} = I_{m} \circ \Phi^{-1}$, where $\Phi^{-1} = u + id$ is a transformation map, and $id$ is the identity transform. Since the registration network can find correspondences between images without supervision, we use predicted transformation maps to create artificial samples by warping the images with segmentations to images without available segmentations. First, we sample moving image segmentation pair $\left(I_{m}, S_{m} \right) \in D_{S}$ and fixed image $I_{f} \in D_{S}$ and we obtain the transformation map $\Phi^{-1}$ between $I_{m}$ and $I_{f}$. Next, we create a new sample by warping both the moving image $I_{m}$ \emph{and} its segmentation $S_{m}$ as 
\begin{equation}
\begin{aligned}
    \left(I_{m}^{w}, S_{m}^{w} \right) &= \left(I_{m} \circ \Phi^{-1}, S_{m} \circ \Phi^{-1} \right)\,.
\end{aligned}
\end{equation}

Warping both the image and its segmentation helps preserve semantic consistency between them. This artificially created sample, $\left(I_{m}^{w}, S_{m}^{w}\right)$, is anatomically similar to $I_{f}$. When a limited number of segmented samples is available, we can use this approach to create additional samples. Sec.~\ref{appendix:augmentation} provides details of the implementation. 

\subsection{Domain Adaptation}
\label{sub:domain-adaptation}

\begin{figure}[htbp]
\floatconts
  {fig:domain-adaptation}
  {\caption{\textbf{Domain Adaptation Overview.} Our domain adaptation architecture consists of two image translation cycles. Each cycle translates images from one domain (source or target) to the other and back again. Our pre-trained segmentation network $S_{S}$ enables training of $S_{T}$ based on semantic consistency. \emph{DXA images reproduced by kind permission of the UK Biobank\textsuperscript{\textregistered}.}}}
  {\includegraphics[width=1\linewidth]{figures/domain-adaptation-single.png}}
\end{figure}

Inspired by \cite{irospaper}, we use a CycleGAN \cite{cyclegan}-based domain adaptation network for medical image \emph{segmentation}. We assume that we already have a pre-trained segmentation network $S_{S}$ for the source domain. Our goal is to train a segmentation network $S_{T}$ for the target domain by transferring information from $S_{S}$.

We use an adversarial training procedure. Two generators, $G_{ST}$ and $G_{TS}$, are designed to learn mapping functions from the source domain to the target domain and vice versa. Concurrently, discriminators $D_{S}$ and $D_{T}$ are responsible for distinguishing between real and translated samples. We integrate these generators and discriminators into the image translation process with adversarial loss ($\ell_{GAN}$) and cycle consistency loss ($\ell_{\text{cycle}}$) terms, following the principles of CycleGAN \cite{cyclegan}. However, it is important to note that this process does not ensure the semantic consistency between real and translated images, i.e., while translated images look realistic for a CycleGAN approach the synthesized image will not necessarily be spatially consistent, and anatomical boundaries might shift. This would, of course, be problematic for any subsequent registration. Hence, we incorporate semantic segmentation networks $S_{S}$ and $S_{T}$ to encourage the semantic segmentation of the images at each step of the cycle to remain consistent~\cite{irospaper}. The work by \cite{irospaper} assumes that all segmentations are available for the source domain, and hence manual segmentations can be used for network training. However, in our case, only a few segmentations are available for the source domain. Therefore, we augment our source dataset with registration (see Sec.~\ref{sub:augmentation}), and use all images in $D_{S}$ for training.

To encourage consistent segmentation predictions at each cycle phase we use the losses
\begin{equation}
\begin{aligned}
        \ell_{\text {sem-ST}} := & \mathbb{E}_{\left(I_{S},M_{S}\right) \sim \mathcal{D}^{aug}_{S}} \mathcal{L}_{sem}\left(S_T\left(G_{S T}(I_{S})\right), M_{S}\right)+  \mathcal{L}_{sem}\left(S_S\left(G_{T S}\left(G_{S T}(I_{S})\right)\right), M_{S} \right), \\
         \ell_{\text {sem-TS}} := & \mathbb{E}_{I_{T} \sim \mathcal{D}_T}  \mathcal{L}_{sem}\left(S_T(I_{T}), S_S\left(G_{T S}(I_{T})\right)\right) +  \mathcal{L}_{sem}\left(S_T(I_{T}), S_T\left(G_{S T}\left(G_{T S}(I_{T})\right)\right)\right)\,
    \end{aligned}
\end{equation}
where $\mathcal{D}^{aug}_{S}$ is augmented source dataset containing image segmentation pairs $\left(I_{S},M_{S}\right)$. These losses enable us to train $S_{T}$ based on $S_{S}$ which was trained with only a few segmented samples. The total loss is

\begin{equation}
    \begin{aligned}
        \ell_{\text {total}} = & \lambda_{G A N} \left( \ell_{G A N}^{S T} + \ell_{G A N}^{T S} \right) + \lambda_{cycle} \ell_{\text {cycle}} + \lambda_{sem} \left( \ell_{\text {sem-ST}} + \ell_{\text {sem-TS}} \right)\,.
    \end{aligned}
\end{equation}

In summary, we use an unsupervised domain adaptation method for medical image segmentation \cite{irospaper}. In doing so, we jointly learn image segmentation and translation using a pre-trained segmentation network trained with only a few segmentations. Sec.~\ref{appendix:domain-adaptation} provides implementation details. 

\subsection{Registration by Segmentation}
\label{sub:segmentation_registration}

\begin{figure}[htbp]
\floatconts
  {fig:registration-approach}
  {\caption{\textbf{Registration Approach.} We first predict segmentations of the images by our $S_{S}$ and $S_{T}$ segmentation models. Then, we pass the segmentations to the registration network and predict transformation maps. Finally, we warp images based on the predicted transformation maps. \emph{DXA images reproduced by kind permission of the UK Biobank\textsuperscript{\textregistered}.}}}
  {\includegraphics[width=1\linewidth]{figures/registration-single.png}}
\end{figure}

Instead of proposing a new registration approach, we use an inverse-consistent registration network \cite{gradicon}. Since our goal is to register anatomically different pairs, we register images based on their segmentations. Note that while our domain adaptation approach of Sec.~\ref{sub:domain-adaptation} generates translated images and segmentations, we only utilize its capability to train a segmentation network for the target domain.

Our image-segmentation pairs for moving and fixed images are $\left(I_{m}, S_{m}\right)$ and $\left(I_{f}, S_{f}\right)$. In our registration setting, the registration network $R$ finds correspondences between images based on their segmentations. The registration network predicts the transformation map $\Phi = R \left(S_{m}, S_{f}; \theta \right) + id $, where $\theta$ denotes the network parameters. We denote by $\Phi_\theta[S_{m}, S_{f}]$ the transformation mapping the moving segmentation, $S_m$ to the space of the fixed segmentation, $S_f$. During training, we register segmentations of image pairs, using gradient inverse consistency of the transformation maps as the regularizer and 1-Dice score on the segmentations as the similarity loss $\mathcal{L}_{\operatorname{sim}}$. The training loss function is

\begin{equation}
    \begin{aligned}
    \ell =  \mathcal{L}_{\operatorname{sim}}\left(S_{m} \circ \Phi_{\theta}\left[S_{m}, S_{f}\right], S_{f}\right)+  &\mathcal{L}_{\operatorname{sim}}\left(S_{f} \circ \Phi_{\theta}\left[S_{f}, S_{m}\right], S_{m}\right)+ \\
    & +\lambda\left\|\nabla\left(\Phi_{\theta}\left[S_{m}, S_{f}\right] \circ \Phi_{\theta}\left[S_{f}, S_{m}\right]\right)-\mathbf{I}\right\|_F^{2}.
    \end{aligned}
\end{equation}

During inference, we first register the segmentation pair $\left(S_{m}, S_{f} \right)$ and obtain the corresponding transformation map $\Phi_\theta[S_{m},S_{f}]$. Then, we use this transformation map to warp the real multimodal image pair $\left(I_{m}, I_{f}\right)$ as $I_{m}^{w} = I_{m} \circ \Phi_\theta[S_{m},S_{f}]$.

In summary, we use our pre-trained segmentation networks (see Sec.~\ref{sub:augmentation}, Sec.~\ref{sub:domain-adaptation}) for inference. First, we predict the segmentations for a given image pair. Then, we predict the transformation map based on the predicted segmentation pair. Lastly, we warp the real images using this predicted transformation map. Thus, we preserve anatomical consistency while finding correspondences between images. Sec.~\ref{appendix:registration} provides implementation details. 

\section{Experiments}
\label{sec:experiments}
We conduct experiments to evaluate the effectiveness of each step of our approach. We use 123 DXA samples provided by the UK Biobank \cite{ukbiobank} and 123 X-rays from The Osteoarthritis Initiative (OAI) \cite{oai}\footnote{This registration task is motivated by our desire to extract spatially localized biomarkers across multiple large-scale, but modality-diverse datasets, e.g., to enable imaging genetics studies for large sample sizes.}. We resized each sample to $256 \times 256$ and manually segmented the femur, tibia, and fibula for 100 samples from each dataset. We performed affine pre-alignment on all samples to one random sample using the affine layers of~\cite{iconbyconstr}. See Table \ref{tab:dataset} for the dataset splits.

\noindent \textbf{Augmentation with Registration:} We compare the effectiveness of our augmentation strategy with raw data (without any augmentation) and elastic augmentation \cite{unet}. Our method outperforms elastic augmentation in most cases. Additionally, we investigate the impact of the number of segmented training samples on test performance. Table \ref{tab:segmentation-aug} (highlights in \textcolor{orange}{orange} cells) shows that we achieve comparable performance to the segmentation network trained on 75 segmented samples using only 10 segmented samples for DXA (86.6\% fewer) or 15 samples for X-rays (80\% fewer). Additionally, our method does not require any parameters unlike elastic augmentation (e.g., number of control points and maximum displacement).

\noindent \textbf{Domain Adaptation for Segmentation:} We evaluate our domain adaptation network in both the DXA to X-ray and the X-ray to DXA adaptation directions. We assume that we already have a pre-trained segmentation model on the source domain trained with varying numbers of segmented samples. We investigate the impact of different ways of creating the source dataset. We conduct experiments using the following approaches: 1) "Baseline," which only uses images with available segmentations; 2) "Baseline + Predictions," which additionally uses pre-trained network predictions for all available images without segmentation; and 3) "Baseline + Augmentation," which augments the source dataset with our augmentation strategy. We observe that augmenting the source dataset is the most reliable approach when the number of segmentations is less than 5. However, for 15 segmentations or more, all approaches converge to similar Dice scores.

The domain adaptation network significantly improves performance on the target modality. The "Baseline + Augmentation" approach improves over directly using a segmentation network trained \emph{only} on the source domain and then applied to the target domain ("w/o Adaptation"). For example, when only 5 segmented samples are available for the source domain, we observe a 7.1\% increase in mean Dice score when comparing the "Baseline + Augmentation" and the "w/o Adaptation" approaches when adapting from the X-ray to the DXA domain, and an increase of 50.5\% in Dice score when adapting from the DXA to the X-ray domain. Table~\ref{tab:domain-adaptation} shows quantitative results, and we highlight the results discussed herein \textcolor{orange}{orange} cells. Figures \ref{fig:xray-seg} and \ref{fig:dxa-seg} provide visual comparisons.


\noindent \textbf{Registration:} We compare our approach to deep registration approaches that differ with respect to their input (either images or segmentations) and the losses that were used during training (either using 1-Dice for segmentation-based losses or various image-based similarity measures). Specifically, we compare our registration approach with 1) An oracle approach that has access to manual segmentations for all images and uses manual segmentations as input and for the loss (to provide an upper bound on performance); 2) Multimodal registration where inputs are real images and our loss uses images; 3) Multimodal registration where inputs are real images and our loss uses images but only within manually segmented regions (Sec. \ref{appendix:constraint-sim}); 4) Multimodal registration where inputs are real images and our loss uses manual segmentations (Sec. \ref{appendix:semantic}); 5) Monomodal registration where inputs are real-fake image pairs generated by our image translation network, where loss uses images (Sec. \ref{appendix:fake-registration}). 

Note that while our approach only assumes that few segmentations are available on the source domain, approaches 1, 3, 4, and 5 above may use manual segmentations for all images in the source and target domain to provide strong competing approaches.

Table \ref{tab:registration} shows that our approach achieves a mean Dice score of 95.7\% with only 5 segmentations from the DXA domain or 96\% with 10 segmentations from the X-ray domain, which is better than the non-oracle best-performing approach (5) that uses 150 manual segmentations in the training process and results in a Dice score of 94.6\%. For more detailed results, please refer to Table \ref{tab:registration-all} where we compare different approaches with varying training losses. Additionally, Figures \ref{fig:dxa-reg} and \ref{fig:xray-reg} demonstrate that our method aligns bone borders with smooth deformation fields.

\begin{table}
\centering
\resizebox{\linewidth}{!}{%
\begin{tabular}{c|c|c|c|c|ccccc|ccccc|l} 
\hline
\multicolumn{5}{c|}{}                                                                                                                                                                                                           & \multicolumn{5}{c|}{DXA $\xrightarrow{}$ X-ray}                                                                                                                                                                       & \multicolumn{5}{c|}{X-ray $\xrightarrow{}$ DXA}                                                                                                                                                                       &                                           \\ 
\hline
Approach               & Input Type                                         & \#DXA Seg.                                & \#X-ray Seg.                              & Loss                                                      & Femur                                    & Tibia                                    & Fibula                                   & Mean                                     & $\operatorname{\%}\left|J_\varphi\right|$ & Femur                                    & Tibia                                    & Fibula                                   & Mean                                     & $\operatorname{\%}\left|J_\varphi\right|$ & Mean                                      \\ 
\hline
1                      & Manual Segmentations                               & 75                                        & 75                                        & Dice                                                      & 0.993                                    & 0.993                                    & 0.990                                    & 0.992                                    & 0.000                                     & 0.994                                    & 0.995                                    & 0.990                                    & 0.993                                    & 0.000                                     & 0.993                                     \\ 
\hline
2                      & Real DXA - Real X-ray                              & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}LNCC                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.834 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.813 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.610 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.752 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.952 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.841 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.914 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.845  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.883  \\ 
\hline
3                      & Real DXA - Real X-ray                              & 75                                        & 75                                        & NGF                                                       & 0.966                                    & 0.963                                    & 0.869                                    & 0.933                                    & 0.000                                     & 0.964                                    & 0.960                                    & 0.865                                    & 0.930                                    & 0.000                                     & 0.931                                     \\ 
\hline
4                      & Real DXA - Real X-ray                              & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.965 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.863 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.931 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.844 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.926 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.929  \\ 
\hline
\multirow{4}{*}{5}     & \multirow{2}{*}{\textbf{Real DXA - Fake DXA}}      & \xmark                                    & \xmark                                    & \multirow{2}{*}{\textbf{LNCC}}                            & 0.970                                    & 0.964                                    & 0.802                                    & 0.912                                    & 0.000                                     & 0.964                                    & 0.960                                    & 0.817                                    & 0.914                                    & 0.000                                     & 0.913                                     \\
                       &                                                    & \textbf{75}                               & \textbf{75}                               &                                                           & \textbf{0.969}                           & \textbf{0.967}                           & \textbf{0.920}                           & \textbf{0.952}                           & \textbf{0.000}                            & \textbf{0.954}                           & \textbf{0.958}                           & \textbf{0.908}                           & \textbf{0.940}                           & \textbf{0.000}                            & \textbf{0.946}                            \\ 
\cline{2-16}
                       & \multirow{2}{*}{Real X-ray - Fake X-ray}           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}                       & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.962 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.806 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.911 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.818 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.915 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.913  \\
                       &                                                    & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & \multirow{-2}{*}{{\cellcolor[rgb]{0.855,0.91,0.988}}LNCC} & {\cellcolor[rgb]{0.855,0.91,0.988}}0.961 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.866 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.929 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.962 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.880 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.936 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.933  \\ 
\hline
\multirow{10}{*}{Ours} & \multirow{10}{*}{\textbf{Predicted Segmentations}} & 1                                         & \xmark                                    & Dice                                                      & 0.956                                    & 0.945                                    & 0.899                                    & 0.933                                    & 0.000                                     & 0.957                                    & 0.944                                    & 0.902                                    & 0.934                                    & 0.000                                     & 0.891                                     \\
                       &                                                    & {\cellcolor[rgb]{0.855,0.91,0.988}}3      & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.964 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.917 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.949 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.965 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.919 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950  \\
                       &                                                    & \textbf{5}                                & \xmark                                    & \textbf{Dice}                                             & \textbf{0.970}                           & \textbf{0.966}                           & \textbf{0.932}                           & \textbf{0.956}                           & \textbf{0.000}                            & \textbf{0.971}                           & \textbf{0.967}                           & \textbf{0.934}                           & \textbf{0.957}                           & \textbf{0.000}                            & \textbf{0.957}                            \\
                       &                                                    & {\cellcolor[rgb]{0.855,0.91,0.988}}10     & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.925 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.956 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.973 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.928 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.958 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.957  \\
                       &                                                    & 75                                        & \xmark                                    & Dice                                                      & 0.970                                    & 0.970                                    & 0.938                                    & 0.959                                    & 0.000                                     & 0.971                                    & 0.971                                    & 0.939                                    & 0.960                                    & 0.000                                     & 0.960                                     \\ 
\cline{3-16}
                       &                                                    & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}1      & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.917 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.942 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.793 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.884 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.002  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.906 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.946 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.825 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.892 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.008  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.888  \\
                       &                                                    & \xmark                                    & 3                                         & Dice                                                      & 0.971                                    & 0.965                                    & 0.901                                    & 0.946                                    & 0.000                                     & 0.969                                    & 0.967                                    & 0.916                                    & 0.951                                    & 0.001                                     & 0.948                                     \\
                       &                                                    & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}5      & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.965 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.889 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.940 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.965 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.905 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.945 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.943  \\
                       &                                                    & \xmark                                    & \textbf{10}                               & \textbf{Dice}                                             & \textbf{0.976}                           & \textbf{0.967}                           & \textbf{0.937}                           & \textbf{0.960}                           & \textbf{0.000}                            & \textbf{0.977}                           & \textbf{0.969}                           & \textbf{0.935}                           & \textbf{0.960}                           & \textbf{0.000}                            & \textbf{0.960}                            \\
                       &                                                    & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.977 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.978 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.975 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.953 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.969 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.968  \\
\hline
\end{tabular}}
\caption{Dice and \% of voxels with neg. Jacobian ($\operatorname{\%}\left|J_\varphi\right|$) for multimodal registration.}
\label{tab:registration}
\end{table}

\section{Conclusion}

We demonstrated that our proposed approach enables the registration of 2D multimodal image pairs visualizing different anatomical structures using a few segmentations from one modality. This work is essential when we have limited numbers of segmented samples for one modality and potentially none for other modalities. To the best of our knowledge, this is the first work addressing this particular problem. Our results showed that our approach using registration-based data augmentation, unsupervised domain adaptation, and segmentation-based registration outperforms existing multimodal registration approaches. Future work will explore 3D extensions to our approach (see Sec.~\ref{sec:limitations} for a related discussion on the limitations of our approach).

\clearpage
\midlacknowledgments{This work was supported by NIH grants 1R01AR072013 and 1R01AR082684. The work expresses the views of the authors, not of the NIH. This research has been conducted using the UK Biobank Resource under Application Number 22783.}

\bibliography{midl24_169}

\appendix

\section{Limitations}
\label{sec:limitations}
Our current approach has several limitations.\\
\noindent \textbf{2D Images.} Our 2D knee datasets are projections of 3D structures. This projection results in complex image intensity distributions because of the multi-layer structure of the knee, which cannot easily be disentangled based on a 2D registration (See Sec. \ref{abs:similar-pairs}). This is the main reason why our approach currently uses segmentations for registration.

\noindent \textbf{Registration with Segmentation.} Although segmentations allow us to align object borders well, they disregard image texture. Hence, while our approach establishes correspondences inside the bones, those are purely based on a smooth interpolation of the deformation field.

\noindent \textbf{Training Cost.} Our approach involves several sequential training phases to train deep networks. This increases the training time.

\noindent \textbf{Future work.} As a next step, we will extend our work to 3D images. Unlike our 2D images, these are not projection images and will therefore allow us to incorporate a suitable loss function that can directly operate on images without solely relying on segmentations.


\section{Implementation and Network Parameters}
\subsection{Dataset Splits}
We crop left and right knee 2D X-ray images from the raw OAI dataset and select 123 random images from the dataset. Then, we resize both the DXA images from the UK Biobank and the OAI X-ray images to $256 \times 256$. Since segmentations are not provided for these datasets, we manually segment them into femur, tibia, and fibula. Additionally, we perform affine alignment on all images with respect to one randomly selected image, using the affine registration layers from \cite{iconbyconstr}. In unpaired image translation networks, the discriminator learns how anatomical regions should appear across a variety of images without seeing real paired examples. Affine alignment helps this process by standardizing the shapes and positions of these regions. Table \ref{tab:dataset} shows train/validation/test splits.

\begin{table}[htbp]
  \centering
  \begin{tabular}{cccc}
    \toprule
    Dataset & Train & Validation & Test  \\
    \midrule
    DXA & $75 + 23^{\dagger}$ & $10$ & $15$ \\
    X-ray & $75 + 23^{\dagger}$ & $10$ & $15$ \\
    \bottomrule
  \end{tabular}
  \caption{Dataset splits. $\dagger$ denotes samples without segmentations.}
  \label{tab:dataset}
\end{table}


\subsection{Augmentation with Registration}
\label{appendix:augmentation}

\begin{figure}[htbp]
\floatconts
  {fig:augmentation-single}
  {\caption{For augmentation, we first register images with segmentations $\left(I_{m}, S_{m}\right)$ to images without segmentations $I_{f}$. Then, we warp both $I_{m}$ and $S_{m}$ to create an artificial sample based on the predicted transformation map. \emph{DXA images reproduced by kind permission of the UK Biobank\textsuperscript{\textregistered}.}}}
  {\includegraphics[width=0.7\linewidth]{figures/augmentation-single.png}}
\end{figure}

We train a two-stage registration network \cite{gradicon} for each modality. We set the learning rate to $5 \times 10^{-5}$, use 20000  iterations per stage, use a batch size of $32$, and penalize with localized normalized cross-correlation (LNCC) with $\sigma=3$. For random elastic augmentation, we set the augmentation parameters to 3 control points and use a Gaussian with a standard deviation $\sigma$ of 3. We use a 2D Residual U-Net \cite{res-unet} from the MONAI Framework \cite{monai} setting channels to (4, 8, 16, 32, 64, 128), strides to (1, 1, 1, 1, 1) and using 8 residual units.

\subsection{Domain Adaptation}
\label{appendix:domain-adaptation}
We implement generators using ResNet \cite{resnet}, discriminators following the PatchGAN \cite{pix2pix} approach, and segmentation models using a Residual U-Net \cite{res-unet} that shares the same architecture introduced in Section \ref{appendix:augmentation}. We embed our source segmentation network, which is pre-trained with our augmentation strategy, into the domain adaptation architecture and freeze it. Additionally, we initialize the segmentation network for the target modality with the same weights. We set $\lambda_{GAN}$ to 1, $\lambda_{cycle}$ to 10, and $\lambda_{sem}$ to 5 and train the network for 400 epochs with a batch size of 1. We use 1-Dice as a semantic loss $\mathcal{L}_{sem}$.

\subsection{Registration}
\label{appendix:registration}
For our registration task, we prefer using a single-stage registration network \cite{gradicon} over a two-stage network. Since registering segmentations is an easier task than registering images, even a single-stage model can converge easily and achieve high performance. We set the learning rate to $10^{-4}$, batch size to $32$, and use 1-Dice as a similarity loss for $5000$ steps. During inference, we do not employ any instance optimization.

\subsection{Computational Cost}
\label{appendix:cost}
We train our registration network for augmentation for $\sim$4.5 hours, the segmentation network on the source dataset for $\sim$1 hour, the UDA network for $\sim$3.5 hours, and the registration network for registering segmentations for $\sim$0.5 hours. We use one NVIDIA GeForce RTX 3090 for all of our experiments. 

\section{Experimental Details}
This section provides additional details and experimental results for the registration approaches 3-5 of Sec.~\ref{sec:experiments}. Specifically, Sec.~\ref{appendix:constraint-sim} provides experimental details for the multi-modal registration approach 3) where inputs are real images and our loss uses images, but only within manually segmented regions. Sec.~\ref{appendix:semantic} provides experimental details for the multi-modal registration approach 4) where inputs are real images and our loss uses manual segmentations. Sec.~\ref{appendix:fake-registration} provides experimental details for the multi-modal registration approach 5) where inputs are real-fake image pairs generated by our image translation network and the loss uses images. Sec.~\ref{abs:similar-pairs} discusses the challenges of 2D knee registration and provides registration visualizations for the inner regions of the bones when using different similarity measures. Sec.~\ref{appendix:both-modality} provides experimental results for the applicability of our method to the case where we assume segmentations from both modalities are available. Sec.~\ref{appendix:robustness-cyclegan} discusses the robustness of our domain adaptation strategy.

\subsection{Registration Training with Segmentation Constrained Similarity Loss}
\label{appendix:constraint-sim}
We design an experimental setting where the registration network takes real multimodal images as input, and the similarity loss is calculated using images but only within the segmented regions. First, we create binary masks from our manual segmentations by assigning 1 to segmented regions and 0 to the background. Then, we assume that image-mask pairs are given as moving $\left(I_{m}, M_{m}\right)$ and fixed images $\left(I_{f}, M_{f}\right)$. The registration network estimates the transformation map $\Phi = R \left(I_{m}, I_{f}; \theta \right) + id$ based on image intensities. If we denote the mapping from the moving image to the fixed image as $\Phi_\theta[I_{m},I_{f}]$, the training loss function is

\begin{equation}
    \begin{aligned}
    \ell =  \sum_{i \in M_{f}} \mathcal{L}_{\operatorname{sim}}\left(I_{m} \circ \Phi_{\theta}\left[I_{m}, I_{f}\right], I_{f}\right)_{i}
    & +\sum_{i \in M_{m}} \mathcal{L}_{\operatorname{sim}}\left(I_{f} \circ \Phi_{\theta}\left[I_{f}, I_{m}\right], I_{m}\right)_{i}+ \\
        & +\lambda\left\|\nabla\left(\Phi_{\theta}\left[I_{m}, I_{f}\right] \circ \Phi_{\theta}\left[I_{f}, I_{m}\right]\right)-\mathbf{I}\right\|_F^{2}\,,
        \end{aligned}
\end{equation}

where $\mathcal{L}_{\operatorname{sim}}()_i$ denotes the value of the similarity measure at location $i$. Here, we sum over all the foreground pixels of the fixed ($M_f$) and moving ($M_m$) image, respectively. 

During training, we use gradient inverse consistency as the regularizer. Additionally, we conduct experiments with several similarity losses. During inference, this approach does not require any segmentations since it operates on image space.

Table~\ref{tab:registration-all} shows that our approach, with 3 segmentations from either the DXA or X-ray modality, outperforms this approach (registration approach 3 of Sec.~\ref{sec:experiments}) trained with several similarity losses while using 75 segmentations from each of the modalities. Therefore, optimizing over images even if constrained by segmentations, is not the best way for this registration task.

\subsection{Registration with Semantic Loss}
\label{appendix:semantic}
Inspired by \cite{segmentation-loss}, we create an experiment where the registration network takes real multimodal images as input but where the similarity loss is based on manual segmentations, assuming manual segmentations are available for \emph{all} images. Based on a similar setting as in Sec. \ref{sub:segmentation_registration}, we assume that image-segmentation pairs are given as moving $\left(I_{m}, S_{m}\right)$ and fixed images $\left(I_{f}, S_{f}\right)$. The registration network estimates the transformation map $\Phi = R \left(I_{m}, I_{f}; \theta \right) + id $ based on image intensities. If we denote the mapping from the moving image to the fixed image $\Phi_\theta[I_{m}, I_{f}]$, the training loss function is 

\begin{equation}
    \begin{aligned}
    \ell =  \mathcal{L}_{\operatorname{sim}}\left(S_{m} \circ \Phi_{\theta}\left[I_{m}, I_{f}\right], S_{f}\right)+  &\mathcal{L}_{\operatorname{sim}}\left(S_{f} \circ \Phi_{\theta}\left[I_{f}, I_{m}\right], S_{m}\right)+ \\
    & +\lambda\left\|\nabla\left(\Phi_{\theta}\left[I_{m}, I_{f}\right] \circ \Phi_{\theta}\left[I_{f}, I_{m}\right]\right)-\mathbf{I}\right\|_F^{2}\,.
    \end{aligned}
\end{equation}

We use gradient inverse consistency of the transformation maps as the regularizer and 1-Dice score on the segmentations as the similarity loss $\mathcal{L}_{\operatorname{sim}}$. During inference, this approach does not require any segmentations since it operates on image space.

Table \ref{tab:registration-all} shows that this approach (registration approach 4 of Sec.~\ref{sec:experiments}) performs worse when registering the fibula, even if we incorporate all manual segmentations for both modalities. We hypothesize that in contrast to a registration network that uses segmentations as inputs, a registration network that uses the acquired images directly will likely have to internally form some notion of a region of interest, essentially performing a form of latent segmentation. Hence, using the segmentations already as input is likely an easier task.

\subsection{Registration of Real-Fake Image Pairs}
\label{appendix:fake-registration}
In our approach, we use a CycleGAN-based image translation network to train a segmentation network on the target domain, and we use this segmentation network for registration. However, we can also use our image translation network to convert our multimodal registration problem into a monomodal one. To obtain a strong competitor to our approach, we assume that we have manual segmentations for all images of both domains in contrast to our approach. 

We train both a standard CycleGAN and a CycleGAN using manual segmentations for both modalities as semantic losses. Then, we translate all images in our training dataset to the same domain and create DXA \& fake DXA and X-ray \& fake X-ray image pairs. Based on these datasets, we train monomodal registration networks. During inference, we initially translate images to the same domain and then use these real-fake image pairs as the input to a monomodal registration network. Finally, we predict deformation fields. 

Table \ref{tab:registration-all} shows that registration via a CycleGAN using manual segmentations for both modalities as semantic losses is better than using a standard CycleGAN. However, because registering based on images is still difficult, the registration accuracy of our proposed approach remains better. Therefore, in our approach, it is beneficial to use the segmentation-guided CycleGAN to train the segmentation networks and work with the predicted segmentations.

Figure \ref{fig:fake-reg} illustrates the moving, fixed, fake fixed, and registered images produced using the CycleGAN monomodal registration approach with semantic loss for several cases. Table \ref{tab:registration-all} presents the Dice scores of the CycleGAN monomodal registration approach, comparing results with and without the use of semantic loss.

\begin{figure*}[ht]
  \centering
   \includegraphics[width=1\linewidth]{figures/fake.png}
   \caption{Qualitative results for multimodal registration with image translation. While registering between DXA and fake DXA images (bottom two rows) works reasonably well, registering between X-rays and fake X-rays (top two rows) is much harder and results in nonsensical registration results for some of the similarity measures. Best viewed zoomed. \emph{DXA images reproduced by kind permission of the UK Biobank\textsuperscript{\textregistered}.}}
   \label{fig:fake-reg}
\end{figure*}
\newpage
\subsection{Registration of Anatomically Similar Pairs}
\label{abs:similar-pairs}
In our registration setting, we register pairs based on their provided or predicted segmentations. Although registering based on segmentations aligns the borders of the bones well, it does not establish correspondences between the inner regions of the bones. To decouple registration problems due to visualization of different anatomical regions from registration problems due to image appearance differences for the different modalities within the same anatomical regions, we conducted an experiment by using images that only visualize common anatomical structures. We achieve this by masking the images of both modalities based on our manual segmentations. We thereby remove uncommon structures, in this case, soft tissue. This masking results in an artificial experimental dataset for multimodal registration that only shows anatomically similar regions. Then, we trained a registration network with several image similarity loss functions.

Figure \ref{fig:masked-reg} illustrates predictions of grid-based displacement fields based on these different image similarity measures. We observe each loss causes the network to behave significantly differently for inner bone regions due to appearance differences between the DXA and X-ray images. Combined with the multi-layer structure of the knee (including the presence of the patella), this leads to some similarity measures to highly spatially irregular deformation fields.

\begin{figure*}[ht]
  \centering
   \includegraphics[width=1\linewidth]{figures/masked.png}
   \caption{Qualitative results for multimodal registration using different similarity measures. The DXA and X-ray images have been masked to obtain pairs showing the same anatomical structure. Due to appearance differences within the bones, registration results strongly differ between the different similarity measures. \emph{DXA images reproduced by kind permission of the UK Biobank\textsuperscript{\textregistered}.}}
   \label{fig:masked-reg}
\end{figure*}

\newpage
\subsection{Registration where Segmentations for Both Modalities Exist}
\label{appendix:both-modality}
Our approach assumes that a small number of segmentations are provided for only one of the modalities. However, we also explore the applicability of our approach to a case where we have segmentations for both modalities. In this setting, the unsupervised domain adaptation phase is no longer necessary. Instead, we directly train segmentation networks for each modality with augmentation based on registration (see \ref{sub:augmentation}) for which we illustrated its effectiveness in Table \ref{tab:segmentation-aug}. Then, we register image pairs again with predicted segmentations as we do in our proposed approach. 

Table \ref{tab:both-segmented} compares two approaches: one that assumes segmentations are available from only one modality (ours) and another that utilizes segmentations from both modalities (ours w/o UDA). When the same number of segmentations are provided, our approach can outperform the approach without unsupervised domain adaptation (UDA) in some cases. For instance, ours reaches a 95.7\% Dice score with 10 DXA segmentations and 96\% with 10 X-ray segmentations, while ours w/o UDA with 5 segmentations for each modality only achieves a 94.2\% Dice score. We observe similar results for the case of 20 segmentations: our approach reaches a 96.4\% Dice score with 20 DXA segmentations and 96.5\% with 20 X-ray segmentations while ours w/o UDA with 20 segmentations achieves a 96\% Dice score. However, ours w/o UDA slightly outperforms with a 97\% Dice score when 40 segmentations are provided, while ours reaches 96\% with DXA and 96.8\% with X-ray segmentations. These results demonstrate that our method, even when utilizing segmentations from only one modality, performs competitively compared to the approach that benefits from segmentations from both modalities. This highlights the effectiveness of unsupervised domain adaptation in our approach.

\begin{table}[ht]
\centering
\resizebox{\linewidth}{!}{%
\begin{tabular}{c|c|c|ccccc|ccccc|c} 
\hline
\multicolumn{3}{c|}{}                                                                                                 & \multicolumn{5}{c|}{DXA $\xrightarrow{}$ X-ray}                                                                                                                                                                       & \multicolumn{5}{c|}{X-ray $\xrightarrow{}$ DXA}                                                                                                                                                                       &                                           \\ 
\hline
Approach                      & \#DXA Seg.                                & \#X-ray Seg.                              & Femur                                    & Tibia                                    & Fibula                                   & Mean                                     & $\operatorname{\%}\left|J_\varphi\right|$ & Femur                                    & Tibia                                    & Fibula                                   & Mean                                     & $\operatorname{\%}\left|J_\varphi\right|$ & Mean                                      \\ 
\hline
\multirow{3}{*}{Ours w/o UDA} & 5                                         & 5                                         & 0.967                                    & 0.968                                    & 0.883                                    & 0.939                                    & 0.000                                     & 0.968                                    & 0.971                                    & 0.893                                    & 0.944                                    & 0.000                                     & 0.942                                     \\
                              & {\cellcolor[rgb]{0.855,0.91,0.988}}10     & {\cellcolor[rgb]{0.855,0.91,0.988}}10     & {\cellcolor[rgb]{0.855,0.91,0.988}}0.971 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.969 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.941 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.973 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.935 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.959 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960  \\
                              & 20                                        & 20                                        & 0.980                                    & 0.974                                    & 0.950                                    & 0.968                                    & 0.000                                     & 0.982                                    & 0.977                                    & 0.955                                    & 0.971                                    & 0.000                                     & 0.970                                     \\ 
\hline
\multirow{6}{*}{Ours}         & {\cellcolor[rgb]{0.855,0.91,0.988}}10     & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.925 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.956 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.973 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.928 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.958 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.957  \\
                              & 20                                        & \xmark                                    & 0.976                                    & 0.970                                    & 0.944                                    & 0.963                                    & 0.000                                     & 0.976                                    & 0.970                                    & 0.945                                    & 0.964                                    & 0.000                                     & 0.964                                     \\
                              & {\cellcolor[rgb]{0.855,0.91,0.988}}40     & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}0.971 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.941 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.968 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.941 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960  \\ 
\cline{2-14}
                              & \xmark                                    & 10                                        & 0.976                                    & 0.967                                    & 0.937                                    & 0.960                                    & 0.000                                     & 0.977                                    & 0.969                                    & 0.935                                    & 0.960                                    & 0.000                                     & 0.960                                     \\
                              & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}20     & {\cellcolor[rgb]{0.855,0.91,0.988}}0.976 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.945 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.964 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.977 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.973 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.965  \\
                              & \xmark                                    & 40                                        & 0.979                                    & 0.971                                    & 0.952                                    & 0.967                                    & 0.000                                     & 0.980                                    & 0.973                                    & 0.954                                    & 0.969                                    & 0.000                                     & 0.968                                     \\
\hline
\end{tabular}}
\caption{
Class-based Dice scores for multimodal registration where we compare our approach (ours), which assumes manual segmentations from one of the modalities are provided for training, to ours w/o UDA, which assumes manual segmentations from both modalities are provided for training.}
\label{tab:both-segmented}
\end{table}

\subsection{Robustness of CycleGAN Based Unsupervised Domain Adaptation}
\label{appendix:robustness-cyclegan}
Training a robust CycleGAN typically requires a large and diverse dataset. However, collecting such a dataset may be a challenging task. Under our experimental constraints, where we have a small number of images, this task becomes even more difficult. To investigate the robustness of our unsupervised domain adaptation approach, we run our experiments with 5 random seeds and report the mean and standard deviation of Dice scores for each class. Table \ref{tab:robustness} shows that our approach is reliable for domain adaptation, with the highest standard deviation being 0.028 for the worst-case scenario.

\begin{table}[ht]
\centering
\resizebox{\linewidth}{!}{%
\begin{tabular}{c|ccc|ccc} 
\hline
                                    & \multicolumn{3}{c|}{DXA $\xrightarrow{}$ X-ray}           & \multicolumn{3}{c}{X-ray $\xrightarrow{}$ DXA}                                         \\ 
\hline
\#Segmentations                       & Femur             & Tibia             & Fibula            & Femur             & Tibia             & Fibula                                         \\ 
\hline
1                                   & 0.955 $\pm$ 0.004 & 0.935 $\pm$ 0.007 & 0.922 $\pm$ 0.008 & 0.984 $\pm$ 0.002 & 0.981 $\pm$ 0.001 & {\cellcolor[rgb]{1,0.647,0}}0.922 $\pm$ 0.028  \\
\rowcolor[rgb]{0.855,0.91,0.988} 3  & 0.968 $\pm$ 0.002 & 0.962 $\pm$ 0.002 & 0.932 $\pm$ 0.012 & 0.986 $\pm$ 0.001 & 0.984 $\pm$ 0.002 & 0.967 $\pm$ 0.001                              \\
5                                   & 0.972 $\pm$ 0.002 & 0.969 $\pm$ 0.002 & 0.914 $\pm$ 0.011 & 0.986 $\pm$ 0.003 & 0.983 $\pm$ 0.002 & 0.969 $\pm$ 0.001                              \\
\rowcolor[rgb]{0.855,0.91,0.988} 10 & 0.978 $\pm$ 0.003 & 0.976 $\pm$ 0.004 & 0.922 $\pm$ 0.011 & 0.988 $\pm$ 0.001 & 0.984 $\pm$ 0.001 & 0.964 $\pm$ 0.002                              \\
15                                  & 0.975 $\pm$ 0.001 & 0.970 $\pm$ 0.004 & 0.931 $\pm$ 0.004 & 0.987 $\pm$ 0.001 & 0.983 $\pm$ 0.002 & 0.966 $\pm$ 0.004                              \\
\rowcolor[rgb]{0.855,0.91,0.988} 20 & 0.974 $\pm$ 0.002 & 0.970 $\pm$ 0.002 & 0.923 $\pm$ 0.009 & 0.986 $\pm$ 0.002 & 0.984 $\pm$ 0.002 & 0.966 $\pm$ 0.004                              \\
40                                  & 0.977 $\pm$ 0.001 & 0.967 $\pm$ 0.004 & 0.924 $\pm$ 0.011 & 0.988 $\pm$ 0.001 & 0.985 $\pm$ 0.001 & 0.965 $\pm$ 0.001                              \\
\rowcolor[rgb]{0.855,0.91,0.988} 75 & 0.977 $\pm$ 0.003 & 0.971 $\pm$ 0.004 & 0.918 $\pm$ 0.009 & 0.985 $\pm$ 0.001 & 0.981 $\pm$ 0.001 & 0.963 $\pm$ 0.006                              \\
\hline
\end{tabular}
}
\caption{Class-based Dice score means and standard deviations for unsupervised domain adaptation with 5 random training seeds. Dice scores are based on comparing the segmentation network results for the domain without available segmentations. The \textcolor{orange}{orange} cell indicates a result discussed in the main text.}
\label{tab:robustness}
\end{table}


\newpage
\section{Qualitative and Quantitative Results}
This section provides the experimental results of our experiments. Specifically, Sec. \ref{appendix:augmentation-results} provides a table comparing segmentation performances of networks without augmentation (raw), with elastic augmentation, and our augmentation with registration strategies. Sec. \ref{appendix:adaptation-results} provides qualitative results that compare segmentation results without adaptation (w/o Adaptation) with several domain adaptation approaches: 1) "Baseline," which only uses images with available segmentations; 2) "Baseline + Predictions," which additionally uses pre-trained network predictions for all available images without segmentation; and 3) "Baseline + Augmentation," which augments the source dataset with our augmentation strategy. Additionally, it provides resulting visuals of our domain adaptation strategy ("Baseline+Augmentation") for a varying number of source dataset segmentations. Lastly, Sec. \ref{appendix:registration-results} provides Dice scores and the percentage of voxels with a negative Jacobian ($\operatorname{\%}\left|J_\varphi\right|$) for our registration approach compared to 1) an oracle approach; 2) multimodal registration where inputs are real images and our loss uses images; 3) multimodal registration where inputs are real images and our loss uses images but only within manually segmented regions; 4) multimodal registration where inputs are real images and our loss uses manual segmentations; 5) monomodal registration where inputs are real-fake image pairs generated by our image translation network, where loss uses images. Moreover, it provides registration visuals of our approach for a varying number of provided source dataset segmentations.


\subsection{Results of Augmentation Experiments}
\label{appendix:augmentation-results}
\begin{table}[ht]
\centering
\resizebox{\linewidth}{!}{
\begin{tabular}{c|cccl|cccl|cccl} 
\hline
                                    & \multicolumn{12}{c}{DXA}                                                                                                                                           \\ 
\hline
                                    & \multicolumn{4}{c|}{Raw}                                   & \multicolumn{4}{c|}{Elastic}   & \multicolumn{4}{c}{Registration}                                     \\ 
\hline
\#Segmentations                       & Femur & Tibia & Fibula & Mean                              & Femur & Tibia & Fibula & Mean  & Femur & Tibia & Fibula & Mean                                        \\ 
\hline
\rowcolor[rgb]{0.855,0.91,0.988} 1  & 0.700 & 0.681 & 0.710  & 0.697                             & 0.850 & 0.865 & 0.714  & 0.810 & 0.962 & 0.949 & 0.907  & \textbf{0.928}                              \\
3                                   & 0.917 & 0.917 & 0.828  & 0.887                             & 0.963 & 0.958 & 0.938  & 0.953 & 0.984 & 0.981 & 0.951  & \textbf{0.972}                              \\
\rowcolor[rgb]{0.855,0.91,0.988} 5  & 0.928 & 0.923 & 0.885  & 0.912                             & 0.971 & 0.964 & 0.951  & 0.962 & 0.982 & 0.980 & 0.950  & \textbf{0.971}                              \\
10                                  & 0.967 & 0.966 & 0.916  & 0.950                             & 0.977 & 0.974 & 0.956  & 0.969 & 0.987 & 0.984 & 0.970  & {\cellcolor[rgb]{1,0.647,0}}\textbf{0.980}  \\
\rowcolor[rgb]{0.855,0.91,0.988} 15 & 0.968 & 0.963 & 0.916  & 0.949                             & 0.981 & 0.978 & 0.957  & 0.972 & 0.987 & 0.982 & 0.968  & \textbf{0.979}                              \\
20                                  & 0.974 & 0.972 & 0.956  & 0.967                             & 0.984 & 0.981 & 0.966  & 0.977 & 0.987 & 0.983 & 0.973  & \textbf{0.981}                              \\
\rowcolor[rgb]{0.855,0.91,0.988} 40 & 0.988 & 0.985 & 0.964  & 0.979                             & 0.990 & 0.989 & 0.966  & 0.982 & 0.990 & 0.988 & 0.975  & \textbf{0.984}                              \\
75                                  & 0.982 & 0.981 & 0.974  & {\cellcolor[rgb]{1,0.647,0}}0.979 & 0.983 & 0.982 & 0.977  & 0.981 & 0.987 & 0.985 & 0.973  & \textbf{0.982}                              \\
\hline
\end{tabular}
}

\resizebox{\linewidth}{!}{
\bigskip
\bigskip
\bigskip

\begin{tabular}{c|cccl|cccl|cccl} 
\hline
                                    & \multicolumn{12}{c}{X-ray}                                                                                                                                                  \\ 
\hline
                                    & \multicolumn{4}{c|}{Raw}                                   & \multicolumn{4}{c|}{Elastic}            & \multicolumn{4}{c}{Registration}                                     \\ 
\hline
\#Segmentations                       & Femur & Tibia & Fibula & Mean                              & Femur & Tibia & Fibula & Mean           & Femur & Tibia & Fibula & Mean                                        \\ 
\hline
\rowcolor[rgb]{0.855,0.91,0.988} 1  & 0.736 & 0.607 & 0.243  & 0.529                             & 0.729 & 0.706 & 0.345  & \textbf{0.593} & 0.772 & 0.666 & 0.291  & 0.576                                       \\
3                                   & 0.763 & 0.745 & 0.421  & 0.643                             & 0.882 & 0.878 & 0.643  & 0.801          & 0.941 & 0.936 & 0.811  & \textbf{0.896}                              \\
\rowcolor[rgb]{0.855,0.91,0.988} 5  & 0.778 & 0.751 & 0.642  & 0.724                             & 0.900 & 0.882 & 0.699  & 0.827          & 0.946 & 0.920 & 0.759  & \textbf{0.875}                              \\
10                                  & 0.911 & 0.900 & 0.736  & 0.849                             & 0.945 & 0.941 & 0.885  & 0.924          & 0.976 & 0.959 & 0.918  & \textbf{0.951}                              \\
\rowcolor[rgb]{0.855,0.91,0.988} 15 & 0.953 & 0.941 & 0.837  & 0.910                             & 0.980 & 0.970 & 0.887  & 0.946          & 0.976 & 0.967 & 0.948  & {\cellcolor[rgb]{1,0.647,0}}\textbf{0.964}  \\
20                                  & 0.956 & 0.956 & 0.879  & 0.930                             & 0.982 & 0.978 & 0.932  & \textbf{0.964} & 0.978 & 0.975 & 0.935  & 0.963                                       \\
\rowcolor[rgb]{0.855,0.91,0.988} 40 & 0.976 & 0.964 & 0.902  & 0.947                             & 0.977 & 0.979 & 0.939  & 0.965          & 0.982 & 0.980 & 0.955  & \textbf{0.972}                              \\
75                                  & 0.977 & 0.981 & 0.934  & {\cellcolor[rgb]{1,0.647,0}}0.964 & 0.982 & 0.986 & 0.956  & 0.975          & 0.984 & 0.985 & 0.959  & \textbf{0.976}                              \\
\hline
\end{tabular}
}
\caption{Class-based Dice scores for augmentation strategies evaluated on the DXA and X-ray segmentation tasks. Our method outperforms other strategies in most cases for a given number of available manual segmentations. The \textcolor{orange}{orange} cells indicate the results discussed in the main text.}
\label{tab:segmentation-aug}%
\end{table}


\subsection{Results of Domain Adaptation Experiments}
\label{appendix:adaptation-results}
\begin{table}[ht]
\resizebox{\linewidth}{!}{
\begin{tabular}{c|cccl|cccl|cccl|cccl} 
\hline
                                        & \multicolumn{16}{c}{DXA $\xrightarrow{}$ X-ray}                                                                                                                                                                                                                                            \\ 
\hline
                                        & \multicolumn{4}{c|}{w/o Adaptation}                        & \multicolumn{4}{c|}{Baseline}                            & \multicolumn{4}{c|}{Baseline + Pred.}                             & \multicolumn{4}{c}{Baseline + Aug.}                                                            \\ 
\hline
\#Segmentations                           & Femur & Tibia & Fibula & Mean                              & Femur          & Tibia          & Fibula         & Mean  & Femur          & Tibia          & Fibula         & Mean           & Femur          & Tibia          & Fibula         & Mean                                        \\ 
\hline
\rowcolor[rgb]{0.855,0.91,0.988} Oracle & 0.984 & 0.985 & 0.959  & 0.976                             & 0.984          & 0.985          & 0.959          & 0.976 & 0.984          & 0.985          & 0.959          & 0.976          & 0.984          & 0.985          & 0.959          & 0.976                                       \\ 
\hline
1                                       & 0.689 & 0.635 & 0.232  & 0.519                             & 0.944          & \textbf{0.954} & 0.742          & 0.880 & \textbf{0.953} & 0.938          & 0.899          & 0.930          & 0.952          & 0.932          & \textbf{0.910} & \textbf{0.931}                              \\
\rowcolor[rgb]{0.855,0.91,0.988} 3      & 0.657 & 0.672 & 0.272  & 0.534                             & 0.969          & 0.963          & 0.856          & 0.929 & \textbf{0.976} & \textbf{0.969} & \textbf{0.937} & \textbf{0.961} & 0.968          & 0.965          & 0.903          & 0.945                                       \\
5                                       & 0.611 & 0.388 & 0.331  & {\cellcolor[rgb]{1,0.647,0}}0.443 & 0.971          & 0.969          & 0.897          & 0.946 & \textbf{0.972} & \textbf{0.972} & 0.889          & 0.944          & 0.967          & 0.966          & \textbf{0.911} & {\cellcolor[rgb]{1,0.647,0}}\textbf{0.948}  \\
\rowcolor[rgb]{0.855,0.91,0.988} 10     & 0.788 & 0.759 & 0.358  & 0.635                             & 0.978          & \textbf{0.978} & \textbf{0.928} & 0.961 & \textbf{0.979} & 0.974          & 0.897          & \textbf{0.950} & 0.977          & \textbf{0.978} & 0.891          & 0.949                                       \\
15                                      & 0.782 & 0.729 & 0.476  & 0.662                             & 0.973          & 0.965          & 0.926          & 0.955 & 0.975          & 0.970          & \textbf{0.936} & 0.960          & \textbf{0.977} & \textbf{0.971} & 0.934          & \textbf{0.961}                              \\
\rowcolor[rgb]{0.855,0.91,0.988} 20     & 0.729 & 0.780 & 0.533  & 0.681                             & 0.977          & 0.972          & 0.926          & 0.958 & \textbf{0.981} & \textbf{0.975} & \textbf{0.932} & \textbf{0.963} & 0.980          & 0.973          & 0.920          & 0.958                                       \\
40                                      & 0.807 & 0.837 & 0.562  & 0.735                             & \textbf{0.978} & 0.965          & 0.932          & 0.958 & 0.975          & \textbf{0.969} & \textbf{0.940} & \textbf{0.961} & 0.977          & 0.968          & 0.920          & 0.955                                       \\
\rowcolor[rgb]{0.855,0.91,0.988} 75     & 0.698 & 0.615 & 0.462  & 0.592                             & \textbf{0.979} & 0.974          & 0.909          & 0.954 & 0.978          & \textbf{0.978} & 0.929          & \textbf{0.962} & 0.972          & 0.976          & \textbf{0.930} & 0.959                                       \\
\hline
\end{tabular}
}

\resizebox{\linewidth}{!}{
\bigskip
\bigskip
\bigskip

\begin{tabular}{c|cccl|cccl|cccl|cccl} 
\hline
                                        & \multicolumn{16}{c}{X-ray~$\xrightarrow{}$ DXA}                                                                                                                                                                                                                                                                \\ 
\hline
                                        & \multicolumn{4}{c|}{w/o Adaptation}                        & \multicolumn{4}{c|}{Baseline}                                                         & \multicolumn{4}{c|}{Baseline + Pred.}                             & \multicolumn{4}{c}{Baseline + Aug.}                                                   \\ 
\hline
\#Segmentations                           & Femur & Tibia & Fibula & Mean                              & Femur          & Tibia          & Fibula         & Mean                               & Femur          & Tibia          & Fibula         & Mean           & Femur          & Tibia          & Fibula         & Mean                               \\ 
\hline
\rowcolor[rgb]{0.855,0.91,0.988} Oracle & 0.987 & 0.985 & 0.973  & 0.982                             & 0.987          & 0.985          & 0.973          & 0.982                              & 0.987          & 0.985          & 0.973          & 0.982          & 0.987          & 0.985          & 0.973          & 0.982                              \\ 
\hline
1                                       & 0.696 & 0.720 & 0.438  & 0.618                             & 0.971          & 0.964          & 0.855          & 0.930                              & 0.976          & 0.878          & 0.064          & 0.639          & \textbf{0.982} & \textbf{0.978} & \textbf{0.930} & \textbf{0.954}                     \\
\rowcolor[rgb]{0.855,0.91,0.988} 3      & 0.820 & 0.871 & 0.843  & 0.845                             & 0.982          & 0.981          & 0.954          & 0.972                              & 0.981          & 0.978          & \textbf{0.970} & 0.976          & \textbf{0.987} & \textbf{0.983} & 0.964          & \textbf{0.978}                     \\
5                                       & 0.929 & 0.947 & 0.845  & {\cellcolor[rgb]{1,0.647,0}}0.907 & 0.985          & 0.980          & 0.958          & 0.974                              & \textbf{0.988} & \textbf{0.982} & \textbf{0.969} & \textbf{0.980} & 0.986          & 0.981          & 0.967          & {\cellcolor[rgb]{1,0.647,0}}0.978  \\
\rowcolor[rgb]{0.855,0.91,0.988} 10     & 0.936 & 0.967 & 0.886  & 0.930                             & 0.985          & \textbf{0.985} & 0.953          & 0.974                              & 0.985          & 0.980          & \textbf{0.973} & 0.979          & \textbf{0.990} & 0.984          & 0.970          & \textbf{0.981}                     \\
15                                      & 0.886 & 0.923 & 0.818  & 0.876                             & 0.985          & 0.984          & 0.968          & 0.979                              & \textbf{0.990} & \textbf{0.985} & \textbf{0.976} & \textbf{0.984} & 0.989          & 0.984          & 0.963          & 0.979                              \\
\rowcolor[rgb]{0.855,0.91,0.988} 20     & 0.927 & 0.925 & 0.928  & 0.927                             & \textbf{0.988} & 0.984          & 0.970          & \textbf{0.981}                     & 0.985          & \textbf{0.987} & \textbf{0.972} & \textbf{0.981} & 0.987          & 0.985          & 0.969          & 0.980                              \\
40                                      & 0.957 & 0.958 & 0.944  & 0.953                             & 0.988          & 0.983          & 0.966          & 0.979                              & 0.988          & 0.984          & \textbf{0.972} & \textbf{0.981} & \textbf{0.989} & \textbf{0.985} & 0.970          & \textbf{0.981}                     \\
\rowcolor[rgb]{0.855,0.91,0.988} 75     & 0.940 & 0.958 & 0.781  & 0.893                             & \textbf{0.988} & 0.980          & \textbf{0.972} & \multicolumn{1}{l}{\textbf{0.980}} & 0.986          & 0.981          & 0.971          & 0.979          & \textbf{0.988} & \textbf{0.982} & 0.967          & 0.979                              \\
\hline
\end{tabular}
}
\caption{Class-based Dice scores for unsupervised domain adaptation for segmentation. We test our networks on the target modality with a limited number of source segmentations. Additionally, as an oracle method, we trained and tested a segmentation network on the target modality with full supervision. We observe that Dice scores converge when around 15 segmentations are provided. The results of domain adaptation are competitive with the oracle method. The \textcolor{orange}{orange} cells indicate the results discussed in the main text.}
\label{tab:domain-adaptation}
\end{table}

\begin{figure*}[htbp]
  \centering
   \includegraphics[width=0.95\linewidth]{figures/xray-seg.png}
   \caption{Qualitative results of our domain adaptation strategy ("Baseline+Augmentation") on the X-ray modality with varying numbers of provided DXA segmented samples.}
   \label{fig:xray-seg}
\end{figure*}

\begin{figure*}[htbp]
  \centering
   \includegraphics[width=0.95\linewidth]{figures/dxa-seg.png}
   \caption{Qualitative results of our domain adaptation strategy ("Baseline+Augmentation") on the DXA modality for varying numbers of provided X-ray segmented samples. \emph{DXA images reproduced by kind permission of the UK Biobank\textsuperscript{\textregistered}.}}
   \label{fig:dxa-seg}
\end{figure*}

\subsection{Results of Registration Experiments}
\label{appendix:registration-results}
\begin{table}
\centering
\resizebox{0.96\linewidth}{!}{%
\begin{tabular}{c|c|c|c|c|ccccc|ccccc|c} 
\hline
\multicolumn{5}{c|}{}                                                                                                                                                                                                         & \multicolumn{5}{c|}{DXA $\xrightarrow{}$ X-ray}                                                                                                                                                                       & \multicolumn{5}{c|}{X-ray $\xrightarrow{}$ DXA}                                                                                                                                                                       &                                           \\ 
\hline
Approach                      & Input Type                                & \#DXA Seg.                                & \#X-ray Seg.                              & Loss                                                      & Femur                                    & Tibia                                    & Fibula                                   & Mean                                     & $\operatorname{\%}\left|J_\varphi\right|$ & Femur                                    & Tibia                                    & Fibula                                   & Mean                                     & $\operatorname{\%}\left|J_\varphi\right|$ & Mean                                      \\ 
\hline
1                             & Manual Segmentations                      & 75                                        & 75                                        & Dice                                                      & 0.993                                    & 0.993                                    & 0.990                                    & 0.992                                    & 0.000                                     & 0.994                                    & 0.995                                    & 0.990                                    & 0.993                                    & 0.000                                     & 0.993                                     \\ 
\hline
\multirow{7}{*}{2}            & \multirow{7}{*}{Real DXA - Real X-ray}    & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}LNCC                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.834 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.813 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.610 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.752 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.952 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.841 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.914 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.845  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.833  \\
                              &                                           & \xmark                                    & \xmark                                    & MI                                                        & 0.658                                    & 0.770                                    & 0.460                                    & 0.629                                    & 0.030                                     & 0.787                                    & 0.948                                    & 0.634                                    & 0.790                                    & 7.422                                     & 0.710                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}MIND                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.866 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.893 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.528 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.762 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.847 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.888 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.493 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.867 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.753  \\
                              &                                           & \xmark                                    & \xmark                                    & NCC                                                       & 0.643                                    & 0.721                                    & 0.399                                    & 0.588                                    & 0.000                                     & 0.727                                    & 0.907                                    & 0.357                                    & 0.664                                    & 7.308                                     & 0.626                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}NGF                    & {\cellcolor[rgb]{0.855,0.91,0.988}}0.678 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.712 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.368 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.586 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.758 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.794 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.356 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.636 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.611  \\
                              &                                           & \xmark                                    & \xmark                                    & SSD                                                       & 0.671                                    & 0.656                                    & 0.177                                    & 0.501                                    & 0.000                                     & 0.698                                    & 0.664                                    & 0.164                                    & 0.509                                    & 5.309                                     & 0.505                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}SSIM                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.642 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.702 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.443 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.596 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.852 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.911 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.390 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.718 & {\cellcolor[rgb]{0.855,0.91,0.988}}4.694  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.657  \\ 
\hline
\multirow{7}{*}{3}            & \multirow{7}{*}{Real DXA - Real X-ray}    & 75                                        & 75                                        & LNCC                                                      & 0.965                                    & 0.962                                    & 0.882                                    & 0.936                                    & 0.000                                     & 0.962                                    & 0.959                                    & 0.845                                    & 0.922                                    & 0.004                                     & 0.929                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}MI                     & {\cellcolor[rgb]{0.855,0.91,0.988}}0.964 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.962 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.874 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.933 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.963 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.856 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.926 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.001  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.930  \\
                              &                                           & 75                                        & 75                                        & MIND                                                      & 0.698                                    & 0.445                                    & 0.310                                    & 0.484                                    & 0.000                                     & 0.735                                    & 0.645                                    & 0.285                                    & 0.555                                    & 1.597                                     & 0.520                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}NCC                    & {\cellcolor[rgb]{0.855,0.91,0.988}}0.964 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.963 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.875 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.934 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.001  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.964 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.961 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.848 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.924 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.002  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.929  \\
                              &                                           & 75                                        & 75                                        & NGF                                                       & 0.966                                    & 0.963                                    & 0.869                                    & 0.933                                    & 0.000                                     & 0.964                                    & 0.960                                    & 0.865                                    & 0.930                                    & 0.000                                     & 0.931                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}SSD                    & {\cellcolor[rgb]{0.855,0.91,0.988}}0.597 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.563 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.387 & {\cellcolor[rgb]{0.855,0.91,0.988}}99.737 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.630 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.769 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.466 & {\cellcolor[rgb]{0.855,0.91,0.988}}75.812 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.427  \\
                              &                                           & 75                                        & 75                                        & SSIM                                                      & 0.965                                    & 0.962                                    & 0.867                                    & 0.931                                    & 0.001                                     & 0.963                                    & 0.960                                    & 0.852                                    & 0.925                                    & 0.000                                     & 0.928                                     \\ 
\hline
4                             & Real DXA - Real X-ray                     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.965 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.863 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.931 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.844 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.926 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.929  \\ 
\hline
\multirow{28}{*}{5}           & \multirow{14}{*}{Real DXA - Fake DXA}     & \xmark                                    & \xmark                                    & \multirow{2}{*}{LNCC}                                     & 0.970                                    & 0.964                                    & 0.802                                    & 0.912                                    & 0.000                                     & 0.964                                    & 0.960                                    & 0.817                                    & 0.914                                    & 0.000                                     & 0.913                                     \\
                              &                                           & 75                                        & 75                                        &                                                           & 0.969                                    & 0.967                                    & 0.920                                    & 0.952                                    & 0.000                                     & 0.954                                    & 0.958                                    & 0.908                                    & 0.940                                    & 0.000                                     & 0.946                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}                       & {\cellcolor[rgb]{0.855,0.91,0.988}}0.969 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.963 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.770 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.901 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.001  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.799 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.908 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.002  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.905  \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & \multirow{-2}{*}{{\cellcolor[rgb]{0.855,0.91,0.988}}MI}   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.968 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.909 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.948 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.959 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.957 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.868 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.928 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.002  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.938  \\
                              &                                           & \xmark                                    & \xmark                                    & \multirow{2}{*}{MIND}                                     & 0.950                                    & 0.947                                    & 0.696                                    & 0.864                                    & 0.000                                     & 0.953                                    & 0.949                                    & 0.698                                    & 0.867                                    & 0.000                                     & 0.866                                     \\
                              &                                           & 75                                        & 75                                        &                                                           & 0.953                                    & 0.949                                    & 0.699                                    & 0.867                                    & 0.000                                     & 0.953                                    & 0.949                                    & 0.697                                    & 0.866                                    & 0.000                                     & 0.867                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}                       & {\cellcolor[rgb]{0.855,0.91,0.988}}0.968 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.961 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.777 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.902 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.969 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.963 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.773 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.902 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.902  \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & \multirow{-2}{*}{{\cellcolor[rgb]{0.855,0.91,0.988}}NCC}  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.965 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.963 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.874 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.934 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.963 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.871 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.933 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.934  \\
                              &                                           & \xmark                                    & \xmark                                    & \multirow{2}{*}{NGF}                                      & 0.957                                    & 0.955                                    & 0.779                                    & 0.897                                    & 5.483                                     & 0.962                                    & 0.956                                    & 0.768                                    & 0.895                                    & 5.477                                     & 0.896                                     \\
                              &                                           & 75                                        & 75                                        &                                                           & 0.951                                    & 0.955                                    & 0.820                                    & 0.909                                    & 5.234                                     & 0.952                                    & 0.952                                    & 0.816                                    & 0.907                                    & 4.090                                     & 0.908                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}                       & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.764 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.897 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.961 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.760 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.896 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.896  \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & \multirow{-2}{*}{{\cellcolor[rgb]{0.855,0.91,0.988}}SSD}  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.837 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.919 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.961 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.842 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.921 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.920  \\
                              &                                           & \xmark                                    & \xmark                                    & \multirow{2}{*}{SSIM}                                     & 0.968                                    & 0.964                                    & 0.775                                    & 0.902                                    & 0.000                                     & 0.961                                    & 0.958                                    & 0.791                                    & 0.903                                    & 0.000                                     & 0.903                                     \\
                              &                                           & 75                                        & 75                                        &                                                           & 0.966                                    & 0.966                                    & 0.908                                    & 0.947                                    & 0.000                                     & 0.948                                    & 0.951                                    & 0.864                                    & 0.921                                    & 0.000                                     & 0.934                                     \\ 
\cline{2-16}
                              & \multirow{14}{*}{Real X-ray - Fake X-ray} & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}                       & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.962 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.806 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.911 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.818 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.915 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.913  \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & \multirow{-2}{*}{{\cellcolor[rgb]{0.855,0.91,0.988}}LNCC} & {\cellcolor[rgb]{0.855,0.91,0.988}}0.961 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.866 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.929 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.962 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.880 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.936 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.933  \\
                              &                                           & \xmark                                    & \xmark                                    & \multirow{2}{*}{MI}                                       & 0.947                                    & 0.950                                    & 0.761                                    & 0.886                                    & 0.000                                     & 0.952                                    & 0.949                                    & 0.796                                    & 0.899                                    & 0.000                                     & 0.893                                     \\
                              &                                           & 75                                        & 75                                        &                                                           & 0.928                                    & 0.921                                    & 0.820                                    & 0.890                                    & 0.000                                     & 0.951                                    & 0.947                                    & 0.816                                    & 0.905                                    & 0.000                                     & 0.897                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}                       & {\cellcolor[rgb]{0.855,0.91,0.988}}0.927 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.875 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.693 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.832 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.923 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.871 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.695 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.830 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.831  \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & \multirow{-2}{*}{{\cellcolor[rgb]{0.855,0.91,0.988}}MIND} & {\cellcolor[rgb]{0.855,0.91,0.988}}0.894 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.800 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.682 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.792 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.900 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.819 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.696 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.805 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.799  \\
                              &                                           & \xmark                                    & \xmark                                    & \multirow{2}{*}{NCC}                                      & 0.929                                    & 0.916                                    & 0.658                                    & 0.834                                    & 0.000                                     & 0.921                                    & 0.910                                    & 0.680                                    & 0.837                                    & 0.000                                     & 0.836                                     \\
                              &                                           & 75                                        & 75                                        &                                                           & 0.872                                    & 0.834                                    & 0.652                                    & 0.786                                    & 0.000                                     & 0.911                                    & 0.889                                    & 0.597                                    & 0.799                                    & 0.000                                     & 0.793                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}                       & {\cellcolor[rgb]{0.855,0.91,0.988}}0.951 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.944 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.703 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.866 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.952 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.945 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.684 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.860 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.863  \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & \multirow{-2}{*}{{\cellcolor[rgb]{0.855,0.91,0.988}}NGF}  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.943 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.938 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.686 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.856 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.949 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.944 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.677 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.857 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.856  \\
                              &                                           & \xmark                                    & \xmark                                    & \multirow{2}{*}{SSD}                                      & 0.738                                    & 0.694                                    & 0.349                                    & 0.594                                    & 0.000                                     & 0.790                                    & 0.743                                    & 0.351                                    & 0.628                                    & 0.000                                     & 0.611                                     \\
                              &                                           & 75                                        & 75                                        &                                                           & 0.705                                    & 0.644                                    & 0.150                                    & 0.500                                    & 0.000                                     & 0.530                                    & 0.487                                    & 0.148                                    & 0.388                                    & 0.000                                     & 0.444                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}                       & {\cellcolor[rgb]{0.855,0.91,0.988}}0.878 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.836 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.562 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.759 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.901 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.864 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.571 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.779 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.769  \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & \multirow{-2}{*}{{\cellcolor[rgb]{0.855,0.91,0.988}}SSIM} & {\cellcolor[rgb]{0.855,0.91,0.988}}0.799 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.758 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.389 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.649 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.804 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.704 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.371 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.626 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.638  \\ 
\hline
\multirow{8}{*}{Ours w/o UDA} & \multirow{8}{*}{Predicted Segmentations}  & 1                                         & 1                                         & Dice                                                      & 0.918                                    & 0.947                                    & 0.802                                    & 0.888                                    & 0.002                                     & 0.909                                    & 0.949                                    & 0.821                                    & 0.893                                    & 0.000                                     & 0.891                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}3      & {\cellcolor[rgb]{0.855,0.91,0.988}}3      & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.969 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.904 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.946 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.968 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.912 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.001  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.948  \\
                              &                                           & 5                                         & 5                                         & Dice                                                      & 0.967                                    & 0.968                                    & 0.883                                    & 0.939                                    & 0.000                                     & 0.968                                    & 0.971                                    & 0.893                                    & 0.944                                    & 0.000                                     & 0.942                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}10     & {\cellcolor[rgb]{0.855,0.91,0.988}}10     & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.971 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.969 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.941 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.973 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.935 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.959 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960  \\
                              &                                           & 15                                        & 15                                        & Dice                                                      & 0.978                                    & 0.975                                    & 0.949                                    & 0.967                                    & 0.000                                     & 0.979                                    & 0.978                                    & 0.951                                    & 0.969                                    & 0.000                                     & 0.968                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}20     & {\cellcolor[rgb]{0.855,0.91,0.988}}20     & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.980 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.974 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.968 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.982 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.977 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.955 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.971 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970  \\
                              &                                           & 40                                        & 40                                        & Dice                                                      & 0.978                                    & 0.974                                    & 0.959                                    & 0.970                                    & 0.000                                     & 0.979                                    & 0.977                                    & 0.960                                    & 0.972                                    & 0.000                                     & 0.971                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.975 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.976 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.959 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.976 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.979 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.971 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.971  \\ 
\hline
\multirow{16}{*}{Ours}        & \multirow{16}{*}{Predicted Segmentations} & 1                                         & \xmark                                    & Dice                                                      & 0.956                                    & 0.945                                    & 0.899                                    & 0.933                                    & 0.000                                     & 0.957                                    & 0.944                                    & 0.902                                    & 0.934                                    & 0.000                                     & 0.934                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}3      & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.964 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.917 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.949 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.965 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.919 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950  \\
                              &                                           & 5                                         & \xmark                                    & Dice                                                      & 0.970                                    & 0.966                                    & 0.932                                    & 0.956                                    & 0.000                                     & 0.971                                    & 0.967                                    & 0.934                                    & 0.957                                    & 0.000                                     & 0.957                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}10     & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.925 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.956 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.973 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.928 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.958 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.957  \\
                              &                                           & 15                                        & \xmark                                    & Dice                                                      & 0.975                                    & 0.971                                    & 0.938                                    & 0.961                                    & 0.000                                     & 0.976                                    & 0.972                                    & 0.939                                    & 0.962                                    & 0.000                                     & 0.962                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}20     & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.976 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.944 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.963 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.976 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.945 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.964 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.964  \\
                              &                                           & 40                                        & \xmark                                    & Dice                                                      & 0.971                                    & 0.967                                    & 0.941                                    & 0.960                                    & 0.000                                     & 0.972                                    & 0.968                                    & 0.941                                    & 0.960                                    & 0.000                                     & 0.960                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.938 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.959 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.971 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.971 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.939 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960  \\ 
\cline{3-16}
                              &                                           & \xmark                                    & 1                                         & Dice                                                      & 0.917                                    & 0.942                                    & 0.793                                    & 0.884                                    & 0.002                                     & 0.906                                    & 0.946                                    & 0.825                                    & 0.892                                    & 0.008                                     & 0.888                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}3      & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.971 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.965 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.901 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.946 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.969 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.916 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.951 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.001  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.948  \\
                              &                                           & \xmark                                    & 5                                         & Dice                                                      & 0.966                                    & 0.965                                    & 0.889                                    & 0.940                                    & 0.000                                     & 0.966                                    & 0.965                                    & 0.905                                    & 0.945                                    & 0.000                                     & 0.943                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}10     & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.976 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.937 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.977 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.969 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.935 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.960  \\
                              &                                           & \xmark                                    & 15                                        & Dice                                                      & 0.976                                    & 0.971                                    & 0.941                                    & 0.963                                    & 0.000                                     & 0.977                                    & 0.974                                    & 0.945                                    & 0.965                                    & 0.000                                     & 0.964                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}20     & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.976 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.970 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.945 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.964 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.977 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.973 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.967 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.965  \\
                              &                                           & \xmark                                    & 40                                        & Dice                                                      & 0.979                                    & 0.971                                    & 0.952                                    & 0.967                                    & 0.000                                     & 0.980                                    & 0.973                                    & 0.954                                    & 0.969                                    & 0.000                                     & 0.968                                     \\
                              &                                           & {\cellcolor[rgb]{0.855,0.91,0.988}}\xmark & {\cellcolor[rgb]{0.855,0.91,0.988}}75     & {\cellcolor[rgb]{0.855,0.91,0.988}}Dice                   & {\cellcolor[rgb]{0.855,0.91,0.988}}0.977 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.972 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.950 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.966 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.978 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.975 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.953 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.969 & {\cellcolor[rgb]{0.855,0.91,0.988}}0.000  & {\cellcolor[rgb]{0.855,0.91,0.988}}0.968  \\
\hline
\end{tabular}
}
\caption{Class-based Dice scores for multimodal registration. This table shows quantitative results for our approach comparing the following approaches: 1) An oracle approach (to provide an upper performance bound); 2) Multimodal registration where inputs are real images and our loss uses images; 3) Multimodal registration where inputs are real images and our loss uses images but only within manually segmented regions; 4) Multimodal registration where inputs are real images and our loss uses manual segmentations; 5) Monomodal registration where inputs are real-fake image pairs generated by our image translation network, where loss uses images.}
\label{tab:registration-all}
\end{table}





\begin{figure*}[htbp]
  \centering
   \includegraphics[width=1\linewidth]{figures/dxa_registration.png}
   \caption{Qualitative results for our multimodal registration approach for different numbers of segmented DXA samples. \emph{DXA images reproduced by kind permission of the UK Biobank\textsuperscript{\textregistered}.}}
   \label{fig:dxa-reg}
\end{figure*}

\begin{figure*}[htbp]
  \centering
   \includegraphics[width=1\linewidth]{figures/xray_registration.png}
   \caption{Qualitative results for our multimodal registration approach for different numbers of segmented X-ray samples. \emph{DXA images reproduced by kind permission of the UK Biobank\textsuperscript{\textregistered}.}}
   \label{fig:xray-reg}
\end{figure*}

\end{document} 