% DO NOT EDIT - automatically generated from metadata.yaml

\def \codeURL{https://github.com/zseljee/re-faircal}
\def \codeDOI{}
\def \codeSWH{swh:TODO}
\def \dataURL{http://arxiv.org/abs/1411.7923}
\def \dataDOI{}
\def \editorNAME{}
\def \editorORCID{}
\def \reviewerINAME{}
\def \reviewerIORCID{}
\def \reviewerIINAME{}
\def \reviewerIIORCID{}
\def \dateRECEIVED{}
\def \dateACCEPTED{}
\def \datePUBLISHED{}
\def \articleTITLE{[Re] Reproducing FairCal: Fairness Calibration for Face Verification}
\def \articleTYPE{Replication}
\def \articleDOMAIN{ML Reproducibility Challenge 2022}
\def \articleBIBLIOGRAPHY{bibliography.bib}
\def \articleYEAR{2023}
\def \reviewURL{https://openreview.net/forum?id=jDBYRwDpeW}
\def \articleABSTRACT{\subsubsection*{Scope of Reproducibility} This reproducibility paper verifies the claim by \citeauthor{salvador2022faircal} in ``\textit{FairCal: Fairness Calibration for Face Verification}'' \cite{salvador2022faircal} that the FairCal and Oracle methods are fair with respect to sensitive attributes and obtain SOTA accuracy results in face verification when compared to FSN and FTC. The aim is to reproduce the relative values in Tables 2, 3 and 4 of the original paper for these methods. We also provide and support an intuitive explanation of why FairCal outperforms Oracle. \subsubsection*{Methodology} The authors provided partial code to create the results; Code to create and preprocess embeddings was missing, but code to run the experiments on these embeddings was provided. Nevertheless, we re-implement the code from scratch, keeping the data structure identical. Hardware used are personal laptops without GPU and a desktop with an MSI GeForce GTX 1060-3GB GPU.  \subsubsection*{Results}Compared to the data reported in the original paper, the reproduced results vary across embedding models and evaluation metrics, where some combinations perform very similarly to the original paper while other combinations deviate significantly. Despite this, the claims of the original paper have been confirmed, which include no loss of accuracy, fairly calibrated subgroups and predictive equality.  \subsubsection*{What was easy} Some parts of the reproduction went smoothly such as the accessibility of the data and models and the quick execution of the experiments. Also, the paper was clear about the evaluation metrics. Finally, the code for the figures worked straight out of the box. \subsubsection*{What was difficult} Obtaining and running the correct ArcFace model from its ONNX file was difficult because we never worked with ONNX and initially also downloaded the wrong instance. Additionally, the exact steps of the original implementation were unclear to us because the provided code had few comments and its structure was not immediately obvious. \subsubsection*{Communication with original authors} We had indirect contact with the first author who provided an example of the required metadata structure and clarified that all unmentioned hyperparameters were kept at their default values.}
\def \replicationCITE{T. Salvador, S. Cairns, V. Voleti, N. Marshall, and A. M. Oberman. “FairCal: Fairness Calibration for Face Verification” International Conference on Learning Representations. 2022.}
\def \replicationBIB{salvador2022faircal}
\def \replicationURL{https://openreview.net/pdf?id=nRj0NcmSuxb}
\def \replicationDOI{10.48550/arXiv.2106.03761}
\def \contactNAME{Zirk Seljee}
\def \contactEMAIL{zirk.seljee@student.uva.nl}
\def \articleKEYWORDS{rescience c, rescience x, machine learning, deep learning, python, pytorch}
\def \journalNAME{ReScience C}
\def \journalVOLUME{9}
\def \journalISSUE{2}
\def \articleNUMBER{1}
\def \articleDOI{}
\def \authorsFULL{Jip Greven, Simon Stallinga and Zirk Seljee}
\def \authorsABBRV{J. Greven, S. Stallinga and Z. Seljee}
\def \authorsSHORT{Greven, Stallinga and Seljee}
\title{\articleTITLE}
\date{}
\author[1,\textdagger,\orcid{0009-0003-5763-5699}]{Jip Greven}
\author[1,\textdagger,\orcid{0009-0009-2773-4200}]{Simon Stallinga}
\author[1,\textdagger,\orcid{0009-0007-2829-1494}]{Zirk Seljee}
\affil[1]{Informatics Institute, University of Amsterdam, The Netherlands}
\affil[{}\textdagger]{Equal contributions}
