@software{transferlab_pydvl_2022,
  title = {{{pyDVL}}: {{The Python Data Valuation Library}}},
  shorttitle = {{{pyDVL}}},
  author = {{TransferLab Team}},
  year = {2022},
  version={0.4.0},
  url = {https://pypi.org/project/pyDVL/},
  abstract = {pyDVL collects algorithms for Data Valuation and Influence Function computation.},
  copyright = {GNU Lesser General Public License v3},
  howpublished = {appliedAI Institute gGmbH}
}

@inproceedings{ghorbani_data_2019,
  title = {Data {{Shapley}}: {{Equitable Valuation}} of {{Data}} for {{Machine Learning}}},
  shorttitle = {Data {{Shapley}}},
  booktitle = {Proceedings of the 36th {{International Conference}} on {{Machine Learning}}, {{PMLR}}},
  author = {Ghorbani, Amirata and Zou, James},
  year = {2019},
  month = may,
  eprint = {1904.02868},
  eprinttype = {arxiv},
  pages = {2242--2251},
  publisher = {{PMLR}},
  issn = {2640-3498},
  url = {http://proceedings.mlr.press/v97/ghorbani19c.html},
  urldate = {2020-11-01},
  abstract = {As data becomes the fuel driving technological and economic growth, a fundamental challenge is how to quantify the value of data in algorithmic predictions and decisions. For example, in healthcare and consumer markets, it has been suggested that individuals should be compensated for the data that they generate, but it is not clear what is an equitable valuation for individual data. In this work, we develop a principled framework to address data valuation in the context of supervised machine learning. Given a learning algorithm trained on n data points to produce a predictor, we propose data Shapley as a metric to quantify the value of each training datum to the predictor performance. Data Shapley uniquely satisfies several natural properties of equitable data valuation. We develop Monte Carlo and gradient-based methods to efficiently estimate data Shapley values in practical settings where complex learning algorithms, including neural networks, are trained on large datasets. In addition to being equitable, extensive experiments across biomedical, image and synthetic data demonstrate that data Shapley has several other benefits: 1) it is more powerful than the popular leave-one-out or leverage score in providing insight on what data is more valuable for a given learning task; 2) low Shapley value data effectively capture outliers and corruptions; 3) high Shapley value data inform what type of new data to acquire to improve the predictor.},
  archiveprefix = {arXiv},
  langid = {english}
}

@inproceedings{jia_efficient_2019,
  title = {Towards {{Efficient Data Valuation Based}} on the {{Shapley Value}}},
  booktitle = {Proceedings of the 22nd {{International Conference}} on {{Artificial Intelligence}} and {{Statistics}}},
  author = {Jia, Ruoxi and Dao, David and Wang, Boxin and Hubis, Frances Ann and Hynes, Nick and G{\"u}rel, Nezihe Merve and Li, Bo and Zhang, Ce and Song, Dawn and Spanos, Costas J.},
  year = {2019},
  month = apr,
  pages = {1167--1176},
  publisher = {{PMLR}},
  issn = {2640-3498},
  url = {http://proceedings.mlr.press/v89/jia19a.html},
  urldate = {2021-02-12},
  abstract = {``How much is my data worth?'' is an increasingly common question posed by organizations and individuals alike. An answer to this question could allow, for instance, fairly distributing profits...},
  langid = {english}
}

@misc{kuprieiev_dvc_2023,
  title = {{{DVC}}: {{Data Version Control}} - {{Git}} for {{Data}} \& {{Models}}},
  shorttitle = {{{DVC}}},
  author = {Kuprieiev, Ruslan and {skshetry} and Petrov, Dmitry and Rowlands (변기호), Peter and Redzy{\'n}ski, Pawe{\l} and da {Costa-Luis}, Casper and Schepanovski, Alexander and Shcheklein, Ivan and Gao and Castro, David de la Iglesia and Taskaya, Batuhan and Orpinel, Jorge and Santos, F{\'a}bio and Berenbaum, Dave and {daniele} and Lamy, Ronan and Sharma, Aman and Zhanibek and Hodovic, Dani and Kodenko, Nikita and Grigorev, Andrew and Earl and Dash, Nabanita and Vyshnya, George and {maykulkarni} and Hora, Max and Vera and Mangal, Sanidhya},
  year = {2023},
  month = jan,
  doi = {10.5281/zenodo.7559368},
  url = {https://zenodo.org/record/7559368},
  urldate = {2023-01-29},
  abstract = {What's Changed Other Changes deps: bump dvc-azure to 2.21.0 by @efiop in https://github.com/iterative/dvc/pull/8864 deps: bump dvc-ssh to 2.21.0 by @efiop in https://github.com/iterative/dvc/pull/8865 Full Changelog: https://github.com/iterative/dvc/compare/2.43.0...2.43.1},
  howpublished = {Zenodo}
}

@inproceedings{metsis_spam_2006,
  title = {Spam {{Filtering}} with {{Naive Bayes-Which Naive Bayes}}?},
  booktitle = {3rd {{Conference}} on {{Email}} and {{Anti-Spam}}},
  author = {Metsis, Vangelis and Androutsopoulos, Ion and Paliouras, Georgios},
  year = {2006},
  month = jul,
  address = {{Mountain View, California USA}},
  url = {https://cir.nii.ac.jp/crid/1571135650462485632},
  urldate = {2023-01-29},
  abstract = {Naive Bayes is very popular in commercial and open-source anti-spam e-mail filters. There are, however, several forms of Naive Bayes, something the anti-spam literature does not always acknowledge. We discuss five different versions of Naive Bayes, and compare them on six new, non-encoded datasets, that contain ham messages of particular Enron users and fresh spam messages. The new datasets, which we make publicly available, are more realistic than previous comparable benchmarks, because they maintain the temporal order of the messages in the two categories, and they emulate the varying proportion of spam and ham messages that users receive over time. We adopt an experimental procedure that emulates the incremental training of personalized spam filters, and we plot roc curves that allow us to compare the different versions of nb over the entire tradeoff between true positives and true negatives.},
  langid = {english}
}

@inproceedings{sim_data_2022,
  title = {Data {{Valuation}} in {{Machine Learning}}: "{{Ingredients}}", {{Strategies}}, and {{Open Challenges}}},
  shorttitle = {Data {{Valuation}} in {{Machine Learning}}},
  booktitle = {Thirty-{{First International Joint Conference}} on {{Artificial Intelligence}}},
  author = {Sim, Rachael Hwee Ling and Xu, Xinyi and Low, Bryan Kian Hsiang},
  year = {2022},
  month = jul,
  volume = {6},
  pages = {5607--5614},
  issn = {1045-0823},
  doi = {10.24963/ijcai.2022/782},
  url = {https://www.ijcai.org/proceedings/2022/782},
  urldate = {2023-01-29},
  abstract = {Electronic proceedings of IJCAI 2022},
  langid = {english}
}

@misc{wang_data_2022,
  title = {Data {{Banzhaf}}: {{A Robust Data Valuation Framework}} for {{Machine Learning}}},
  shorttitle = {Data {{Banzhaf}}},
  author = {Wang, Jiachen T. and Jia, Ruoxi},
  year = {2022},
  month = oct,
  number = {arXiv:2205.15466},
  eprint = {2205.15466},
  eprinttype = {arxiv},
  publisher = {{arXiv}},
  doi = {10.48550/arXiv.2205.15466},
  url = {http://arxiv.org/abs/2205.15466},
  urldate = {2022-10-28},
  abstract = {This paper studies the robustness of data valuation to noisy model performance scores. Particularly, we find that the inherent randomness of the widely used stochastic gradient descent can cause existing data value notions (e.g., the Shapley value and the Leave-one-out error) to produce inconsistent data value rankings across different runs. To address this challenge, we first pose a formal framework within which one can measure the robustness of a data value notion. We show that the Banzhaf value, a value notion originated from cooperative game theory literature, achieves the maximal robustness among all semivalues -- a class of value notions that satisfy crucial properties entailed by ML applications. We propose an algorithm to efficiently estimate the Banzhaf value based on the Maximum Sample Reuse (MSR) principle. We derive the lower bound sample complexity for Banzhaf value estimation, and we show that our MSR algorithm's sample complexity is close to the lower bound. Our evaluation demonstrates that the Banzhaf value outperforms the existing semivalue-based data value notions on several downstream ML tasks such as learning with weighted samples and noisy label detection. Overall, our study suggests that when the underlying ML algorithm is stochastic, the Banzhaf value is a promising alternative to the semivalue-based data value schemes given its computational advantage and ability to robustly differentiate data quality.},
  archiveprefix = {arXiv}
}

@inproceedings{yan_if_2021,
  title = {If {{You Like Shapley Then You}}'ll {{Love}} the {{Core}}},
  booktitle = {Proceedings of the 35th {{AAAI Conference}} on {{Artificial Intelligence}}, 2021},
  author = {Yan, Tom and Procaccia, Ariel D.},
  year = {2021},
  month = may,
  volume = {6},
  pages = {5751--5759},
  publisher = {{Association for the Advancement of Artificial Intelligence}},
  address = {{Virtual conference}},
  doi = {10.1609/aaai.v35i6.16721},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/16721},
  urldate = {2021-04-23},
  abstract = {The prevalent approach to problems of credit assignment in machine learning \textemdash{} such as feature and data valuation\textemdash{} is to model the problem at hand as a cooperative game and apply the Shapley value. But cooperative game theory offers a rich menu of alternative solution concepts, which famously includes the core and its variants. Our goal is to challenge the machine learning community's current consensus around the Shapley value, and make a case for the core as a viable alternative. To that end, we prove that arbitrarily good approximations to the least core \textemdash{} a core relaxation that is always feasible \textemdash{} can be computed efficiently (but prove an impossibility for a more refined solution concept, the nucleolus). We also perform experiments that corroborate these theoretical results and shed light on settings where the least core may be preferable to the Shapley value.},
  copyright = {Copyright (c) 2021, Association for the Advancement of Artificial Intelligence (www.aaai.org). All rights reserved.},
  langid = {english}
}

@software{benmerzoug_mlrc22_2022,
  title = {Mlrc22 - {{If}} You like {{Shapley}} Then You'll Love the {{Core}}},
  author = {Benmerzoug, Anes},
  year = {2022},
  url = {https://github.com/aai-institute/mlrc22-like-shapley-love-the-core},
  abstract = {Code for the submission to the ML Reproducibility Challenge 2022, reproducing "If you like Shapley then you'll love the core"},
  copyright = {GNU Lesser General Public License v3},
  howpublished = {appliedAI Institute gGmbH}
}