@article{saleiro2018aequitas,
  title={Aequitas: A bias and fairness audit toolkit},
  author={Saleiro, Pedro and Kuester, Benedict and Hinkson, Loren and London, Jesse and Stevens, Abby and Anisfeld, Ari and Rodolfa, Kit T and Ghani, Rayid},
  journal={arXiv preprint arXiv:1811.05577},
  year={2018}
}

@article{kulis2012metric,
  title={Metric learning: A survey},
  author={Kulis, Brian and others},
  journal={Foundations and Trends in Machine Learning},
  volume={5},
  number={4},
  pages={287--364},
  year={2012}
}

@incollection{furnkranz2010preference,
  title={Preference learning and ranking by pairwise comparison},
  author={F{\"u}rnkranz, Johannes and H{\"u}llermeier, Eyke},
  booktitle={Preference Learning},
  pages={65--82},
  year={2010},
  publisher={Springer}
}

@article{fu2017learning,
  title={Learning robust rewards with adversarial inverse reinforcement learning},
  author={Fu, Justin and Luo, Katie and Levine, Sergey},
  journal={ArXiv preprint, arXiv:1710.11248},
  year={2017}
}

@inproceedings{ng2000algorithms,
  title={Algorithms for inverse reinforcement learning.},
  author={Ng, Andrew Y and Russell, Stuart J and others},
  booktitle={ICML},
  year={2000}
}

@article{shen2020designing,
  title={Designing Alternative Representations of Confusion Matrices to Support Non-Expert Public Understanding of Algorithm Performance},
  author={Shen, Hong and Jin, H. and Cabrera, A. A. and Perer, A. and Zhu, H. and Hong, Jason I},
  journal={CSCW2},
  year={2020},
  publisher={ACM New York, NY, USA}
}

@inproceedings{lopes2009active,
  title={Active learning for reward estimation in inverse reinforcement learning},
  author={Lopes, Manuel and Melo, Francisco and Montesano, Luis},
  booktitle={ECML-PKDD},
  pages={31--46},
  year={2009},
  organization={Springer}
}

@inproceedings{sadigh2017active,
  title={Active preference-based learning of reward functions},
  author={Sadigh, Dorsa and Dragan, Anca D and Sastry, Shankar and Seshia, Sanjit A},
  booktitle={RSS},
  year={2017}
}

@article{levine2011nonlinear,
  title={Nonlinear inverse reinforcement learning with gaussian processes},
  author={Levine, Sergey and Popovic, Zoran and Koltun, Vladlen},
  journal={NeurIPS},
  volume={24},
  pages={19--27},
  year={2011}
}

@inproceedings{abbeel2004apprenticeship,
  title={Apprenticeship learning via inverse reinforcement learning},
  author={Abbeel, Pieter and Ng, Andrew Y},
  booktitle={ICML},
  year={2004}
}

@article{wu2020efficient,
  title={Efficient sampling-based maximum entropy inverse reinforcement learning with application to autonomous driving},
  author={Wu, Zheng and Sun, Liting and Zhan, Wei and Yang, Chenyu and Tomizuka, Masayoshi},
  journal={IEEE Robotics and Automation Letters},
  volume={5},
  number={4},
  pages={5355--5362},
  year={2020},
  publisher={IEEE}
}

@inproceedings{niranjan2017inductive,
  title={Inductive pairwise ranking: going beyond the n log (n) barrier},
  author={Niranjan, UN and Rajkumar, Arun},
  booktitle={AAAI},
  volume={31},
  number={1},
  year={2017}
}

@inproceedings{shah2015estimation,
  title={Estimation from pairwise comparisons: Sharp minimax bounds with topology dependence},
  author={Shah, Nihar and Balakrishnan, S. and Bradley, J. and Parekh, A. and Ramchandran, K. and Wainwright, Martin},
  booktitle={AISTATS},
  year={2015},
}

@inproceedings{mohajer2017active,
  title={Active Learning for Top-$ K $ Rank Aggregation from Noisy Comparisons},
  author={Mohajer, Soheil and Suh, Changho and Elmahdy, Adel},
  booktitle={ICML},
  year={2017}}

@article{stone1948generalized,
  title={The generalized Weierstrass approximation theorem},
  author={Stone, Marshall H},
  journal={Mathematics Magazine},
  volume={21},
  number={5},
  pages={237--254},
  year={1948},
  publisher={JSTOR}
}

@article{benabbou2017incremental,
  title={Incremental elicitation of Choquet capacities for multicriteria choice, ranking and sorting problems},
  author={Benabbou, Nawal and Perny, Patrice and Viappiani, Paolo},
  journal={Artificial Intelligence},
  volume={246},
  pages={152--180},
  year={2017},
  publisher={Elsevier}
}

@inproceedings{perny2016incremental,
  title={Incremental preference elicitation for decision making under risk with the rank-dependent utility model},
  author={Perny, Patrice and Viappiani, P. and Boukhatem, A.},
  booktitle={UAI},
  year={2016}
}

@inproceedings{weng2013interactive,
  title={Interactive value iteration for markov decision processes with unknown rewards},
  author={Weng, Paul and Zanuttini, Bruno},
  booktitle={IJCAI'13-Twenty-Third international joint conference on Artificial Intelligence},
  pages={2415--2421},
  year={2013},
  organization={AAAI Press}
}

@phdthesis{braziunas2012decision,
  title={Decision-theoretic elicitation of generalized additive utilities},
  author={Braziunas, Darius},
  year={2012}
}

@article{boutilier2006constraint,
  title={Constraint-based optimization and utility elicitation using the minimax decision criterion},
  author={Boutilier, Craig and Patrascu, Relu and Poupart, Pascal and Schuurmans, Dale},
  journal={Artificial Intelligence},
  volume={170},
  number={8-9},
  pages={686--713},
  year={2006},
  publisher={Elsevier}
}

@inproceedings{chajewska2000making,
  title={Making rational decisions using adaptive utility elicitation},
  author={Chajewska, Urszula and Koller, Daphne and Parr, Ronald},
  booktitle={Aaai/Iaai},
  pages={363--369},
  year={2000}
}

@article{white1984model,
  title={A model of multiattribute decisionmaking and trade-off weight determination under uncertainty},
  author={White, Chelsea C and Sage, Andrew P and Dozono, Shigeru},
  journal={IEEE Transactions on Systems, Man, and Cybernetics},
  number={2},
  pages={223--229},
  year={1984},
  publisher={IEEE}
}

@article{mcgill1978variations,
  title={Variations of box plots},
  author={McGill, Robert and Tukey, John W and Larsen, Wayne A},
  journal={The American Statistician},
  volume={32},
  number={1},
  pages={12--16},
  year={1978},
  publisher={Taylor \& Francis Group}
}

@inproceedings{kane2017active,
  title={Active classification with comparison queries},
  author={Kane, Daniel M and Lovett, Shachar and Moran, Shay and Zhang, Jiapeng},
  booktitle={FOCS},
  year={2017},
}

@techreport{settles2009active,
  title={Active learning literature survey},
  author={Settles, Burr},
  year={2009},
  institution={University of Wisconsin-Madison Department of Computer Sciences}
}

@inproceedings{beauxis2014visualization,
  title={Visualization of confusion matrix for non-expert users},
  author={Beauxis-Aussalet, Emma and Hardman, Lynda},
  booktitle={IEEE Conference on Visual Analytics Science and Technology, Poster Proceedings},
  year={2014}
}

@inproceedings{hiranandani2020fair,
  title={Fair Performance Metric Elicitation},
  author={Hiranandani, Gaurush and Narasimhan, Harikrishna and Koyejo, Oluwasanmi},
  booktitle={NeurIPS},
  year={2020}
}

@inproceedings{mukherjee2020two,
  title={Two Simple Ways to Learn Individual Fairness Metric from Data},
  author={Mukherjee, Debarghya and Yurochkin, Mikhail and Banerjee, Moulinath and Sun, Yuekai},
  booktitle={ICML},
  year={2020}
}

@article{barocas2017fairness,
  title={Fairness in machine learning},
  author={Barocas, Solon and Hardt, Moritz and Narayanan, Arvind},
  journal={NeurIPS Tutorial},
  year={2017}
}

@inproceedings{zafar2017fairness,
  title={Fairness beyond disparate treatment \& disparate impact: Learning classification without disparate mistreatment},
  author={Zafar, Muhammad Bilal and Valera, Isabel and Gomez Rodriguez, Manuel and Gummadi, Krishna P},
  booktitle={Proceedings of the 26th international conference on world wide web},
  pages={1171--1180},
  year={2017}
}

@inproceedings{woodworth2017learning,
  title={Learning Non-Discriminatory Predictors},
  author={Woodworth, Blake and Gunasekar, Suriya and Ohannessian, Mesrob I and Srebro, Nathan},
  booktitle={Conference on Learning Theory},
  pages={1920--1953},
  year={2017}
}

@article{calders2010three,
  title={Three naive Bayes approaches for discrimination-free classification},
  author={Calders, Toon and Verwer, Sicco},
  journal={Data Mining and Knowledge Discovery},
  volume={21},
  number={2},
  pages={277--292},
  year={2010},
  publisher={Springer}
}

@article{opotow1996affirmative,
  title={Affirmative action, fairness, and the scope of justice},
  author={Opotow, Susan},
  journal={Journal of Social Issues},
  volume={52},
  number={4},
  pages={19--24},
  year={1996},
  publisher={Wiley Online Library}
}

@article{barocas2016big,
  title={Big data's disparate impact},
  author={Barocas, Solon and Selbst, Andrew D},
  journal={Calif. L. Rev.},
  volume={104},
  pages={671},
  year={2016},
  publisher={HeinOnline}
}

@article{berk2018fairness,
  title={Fairness in criminal justice risk assessments: The state of the art},
  author={Berk, Richard and Heidari, Hoda and Jabbari, Shahin and Kearns, Michael and Roth, Aaron},
  journal={Sociological Methods \& Research},
  pages={0049124118782533},
  year={2018},
  publisher={Sage Publications Sage CA: Los Angeles, CA}
}

@article{chouldechova2017fair,
  title={Fair prediction with disparate impact: A study of bias in recidivism prediction instruments},
  author={Chouldechova, Alexandra},
  journal={Big Data},
  volume={5},
  number={2},
  pages={153--163},
  year={2017},
  publisher={Mary Ann Liebert, Inc. 140 Huguenot Street, 3rd Floor New Rochelle, NY 10801 USA}
}

@inproceedings{pleiss2017fairness,
  title={On fairness and calibration},
  author={Pleiss, Geoff and Raghavan, Manish and Wu, Felix and Kleinberg, Jon and Weinberger, Kilian Q},
  booktitle={NeurIPS},
  pages={5680--5689},
  year={2017}
}

@article{zhao2019conditional,
  title={Conditional learning of fair representations},
  author={Zhao, Han and Coston, Amanda and Adel, Tameem and Gordon, Geoffrey J},
  journal={arXiv preprint arXiv:1910.07162},
  year={2019}
}

@article{shieh1998weighted,
  title={A weighted Kendall's tau statistic},
  author={Shieh, Grace S},
  journal={Statistics \& probability letters},
  volume={39},
  number={1},
  pages={17--24},
  year={1998},
  publisher={Elsevier}
}

@inproceedings{valizadegan2009learning,
  title={Learning to rank by optimizing {NDCG} measure},
  author={Valizadegan, H. and Jin, R. and Zhang, R. and Mao, J.},
  booktitle={NeurIPS},
  year={2009}
}

@inproceedings{narasimhan2019optimizing,
  title={Optimizing generalized rate metrics with three players},
  author={Narasimhan, Harikrishna and Cotter, Andrew and Gupta, Maya},
  booktitle={NeurIPS},
  pages={10746--10757},
  year={2019}
}

@inproceedings{ke2017lightgbm,
  title={LightGBM: A highly efficient gradient boosting decision tree},
  author={Ke, Guolin and Meng, Qi and Finley, Thomas and Wang, Taifeng and Chen, Wei and Ma, Weidong and Ye, Qiwei and Liu, Tie-Yan},
  booktitle={NeurIPS},
  year={2017}
}

@article{joachims1999svmlight,
  title={SVM-Light: Support vector machine},
  author={Joachims, Thorsten},
  journal={URL: \url{http://svmlight. joachims. org/}, University of Dortmund},
  year={1999}
}

@article{pal1992multilayer,
  title={Multilayer perceptron, fuzzy sets, and classification},
  author={Pal, SK and Mitra, S},
  journal={IEEE Transactions on Neural Networks},
  volume={3},
  number={5},
  pages={683--697},
  year={1992},
}

@book{kleinbaum2002logistic,
  title={Logistic regression},
  author={Kleinbaum, David G and Dietz, K and Gail, M and Klein, Mitchel and Klein, Mitchell},
  year={2002},
  publisher={Springer}
}


@inproceedings{goh2016satisfying,
  title={Satisfying real-world goals with dataset constraints},
  author={Goh, Gabriel and Cotter, Andrew and Gupta, Maya and Friedlander, Michael P},
  booktitle={NeurIPS},
  year={2016}
}

@inproceedings{noriega2019active,
  title={Active fairness in algorithmic decision making},
  author={Noriega-Campero, Alejandro and Bakker, Michiel A and Garcia-Bulle, Bernardo and Pentland, Alex'Sandy'},
  booktitle={Proceedings of the 2019 AAAI/ACM Conference on AI, Ethics, and Society},
  year={2019}
}

@inproceedings{agarwal2018reductions,
  title={A Reductions Approach to Fair Classification},
  author={Agarwal, Alekh and Beygelzimer, Alina and Dudik, Miroslav and Langford, John and Wallach, Hanna},
  booktitle={ICML},
  year={2018}
}

@article{kulis2013metric,
  title={Metric learning: A survey},
  author={Kulis, Brian and others},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={5},
  number={4},
  pages={287--364},
  year={2013},
  publisher={Now Publishers, Inc.}
}

@article{ilvento2019metric,
  title={Metric Learning for Individual Fairness},
  author={Ilvento, Christina},
  booktitle={1st Symposium on Foundations of Responsible Computing},
  year={2020},
}

@inproceedings{bechavod2017learning,
  title={Learning fair classifiers: A regularization-inspired approach},
  author={Bechavod, Yahav and Ligett, Katrina},
  booktitle={4th Workshop on FAccT in Machine Learning},
  year={2017}
}

@inproceedings{kamishima2012fairness,
  title={Fairness-aware classifier with prejudice remover regularizer},
  author={Kamishima, Toshihiro and Akaho, Shotaro and Asoh, Hideki and Sakuma, Jun},
  booktitle={ECML-PKDD},
  pages={35--50},
  year={2012},
  organization={Springer}
}

@inproceedings{zafar2017constraints,
  title={Fairness Constraints: Mechanisms for Fair Classification},
  author={Zafar, Muhammad Bilal and Valera, Isabel and Rogriguez, Manuel Gomez and Gummadi, Krishna P},
  booktitle={AISTATS},
  year={2017}
}

@inproceedings{dwork2012fairness,
  title={Fairness through awareness},
  author={Dwork, Cynthia and Hardt, Moritz and Pitassi, Toniann and Reingold, Omer and Zemel, Richard},
  booktitle={ITCS},
  pages={214--226},
  year={2012}
}

@inproceedings{menon2018cost,
  title={The cost of fairness in binary classification},
  author={Menon, Aditya Krishna and Williamson, Robert C},
  booktitle={FAccT},
  year={2018}
}

@inproceedings{hardt2016equality,
  title={Equality of opportunity in supervised learning},
  author={Hardt, Moritz and Price, Eric and Srebro, Nati},
  booktitle={NeurIPS},
  pages={3315--3323},
  year={2016}
}

@inproceedings{zhang2020joint,
  title={Joint optimization of AI fairness and utility: A human-centered approach},
  author={Zhang, Yunfeng and Bellamy, Rachel and Varshney, Kush},
  booktitle={AIES},
  pages={400--406},
  year={2020}
}

@inproceedings{kleinberg2017inherent,
  title={Inherent Trade-Offs in the Fair Determination of Risk Scores},
  author={Kleinberg, Jon and Mullainathan, Sendhil and Raghavan, Manish},
  booktitle={ITCS},
  year={2017}
  }

@inproceedings{lahoti2019ifair,
  title={ifair: Learning individually fair data representations for algorithmic decision making},
  author={Lahoti, Preethi and Gummadi, Krishna P and Weikum, Gerhard},
  booktitle={2019 IEEE 35th International Conference on Data Engineering (ICDE)},
  pages={1334--1345},
  year={2019},
  organization={IEEE}
}

@article{angwin2016machine,
  title={Machine bias risk assessments in criminal sentencing},
  author={Angwin, Julia and Larson, Jeff and Mattu, Surya and Kirchner, Lauren},
  journal={ProPublica, May},
  volume={23},
  year={2016}
}

@inproceedings{corbett2017algorithmic,
  title={Algorithmic decision making and the cost of fairness},
  author={Corbett-Davies, Sam and Pierson, Emma and Feller, Avi and Goel, Sharad and Huq, Aziz},
  booktitle={ACM SIGKDD},
  year={2017}
}

@inproceedings{friedler2019comparative,
  title={A comparative study of fairness-enhancing interventions in machine learning},
  author={Friedler, Sorelle A and Scheidegger, Carlos and Venkatasubramanian, Suresh and Choudhary, Sonam and Hamilton, Evan P and Roth, Derek},
  booktitle={Proceedings of the Conference on Fairness, Accountability, and Transparency},
  pages={329--338},
  year={2019}
}

@inproceedings{singla2015learning,
  title={Learning to hire teams},
  author={Singla, Adish and Horvitz, Eric and Kohli, Pushmeet and Krause, Andreas},
  booktitle={Third AAAI Conference on Human Computation and Crowdsourcing},
  year={2015}
}

@inproceedings{hiranandani2019multiclass,
  title={Multiclass Performance Metric Elicitation},
  author={Hiranandani, Gaurush and Boodaghians, Shant and Mehta, Ruta and Koyejo, Oluwasanmi O},
  booktitle={NeurIPS},
  year={2019}
}

@inproceedings{kaariainen2006active,
  title={Active learning in the non-realizable case},
  author={K{\"a}{\"a}ri{\"a}inen, Matti},
  booktitle={International Conference on Algorithmic Learning Theory},
  pages={63--77},
  year={2006},
  organization={Springer}
}

@book{bishop2006pattern,
  title={Pattern recognition and machine learning},
  author={Bishop, Christopher M},
  year={2006},
  publisher={springer}
}

@article{siebert1987vehicle,
  title={Vehicle recognition using rule based methods},
  author={Siebert, J Paul},
  year={1987},
  publisher={Turing Institute}
}

@article{duarte2004vehicle,
  title={Vehicle classification in distributed sensor networks},
  author={Duarte, Marco F and Hu, Yu Hen},
  journal={Journal of Parallel and Distributed Computing},
  volume={64},
  number={7},
  pages={826--838},
  year={2004},
  publisher={Elsevier}
}

@article{qian2015learning,
  title={Learning user preferences by adaptive pairwise comparison},
  author={Qian, Li and Gao, Jinyang and Jagadish, HV},
  journal={Proceedings of the VLDB Endowment},
  volume={8},
  number={11},
  pages={1322--1333},
  year={2015},
  publisher={VLDB Endowment}
}

@inproceedings{joachims2002optimizing,
  title={Optimizing search engines using clickthrough data},
  author={Joachims, Thorsten},
  booktitle={ACM SIGKDD},
  year={2002},
}

@inproceedings{narasimhan2018learning,
  title={Learning with complex loss functions and constraints},
  author={Narasimhan, Harikrishna},
  booktitle={AISTATS},
  year={2018}
}


@inproceedings{jamieson2012query,
  title={Query complexity of derivative-free optimization},
  author={Jamieson, Kevin G and Nowak, Robert and Recht, Ben},
  booktitle={NeurIPS},
  pages={2672--2680},
  year={2012}
}

@inproceedings{abe2004iterative,
  title={An iterative method for multi-class cost-sensitive learning},
  author={Abe, Naoki and Zadrozny, Bianca and Langford, John},
  booktitle={ACM SIGKDD},
  year={2004},
}

@article{zhou2010multi,
  title={On multi-class cost-sensitive learning},
  author={Zhou, Zhi-Hua and Liu, Xu-Ying},
  journal={Computational Intelligence},
  volume={26},
  number={3},
  pages={232--257},
  year={2010},
  publisher={Wiley Online Library}
}

@article{yang2014multiclass,
  title={Multiclass cancer classification based on gene expression comparison},
  author={Yang, Sitan and Naiman, Daniel Q},
  journal={Statistical Applications in Genetics and Molecular Biology},
  volume={13},
  number={4},
  pages={477--496},
  year={2014},
  publisher={De Gruyter}
}

@inproceedings{hiranandani2018eliciting,
  title={Performance Metric Elicitation from Pairwise Classifier Comparisons},
  author={Hiranandani, Gaurush and Boodaghians, Shant and Mehta, Ruta and Koyejo, Oluwasanmi},
  booktitle={AISTATS},
  year={2019}
}

@book{boucheron2013concentration,
  title={Concentration inequalities: A nonasymptotic theory of independence},
  author={Boucheron, St{\'e}phane and Lugosi, G{\'a}bor and Massart, Pascal},
  year={2013},
  publisher={Oxford university press}
}

@inproceedings{qian2013active,
  title={Active Learning from Relative Queries.},
  author={Qian, Buyue and Wang, Xiang and Wang, Fei and Li, Hongfei and Ye, Jieping and Davidson, Ian},
  booktitle={IJCAI},
  year={2013}
}

@inproceedings{Dmitriev2016MeasuringM,
  title={Measuring Metrics},
  author={Pavel Dmitriev and Xian Wu},
  booktitle={CIKM},
  year={2016}
}

@inproceedings{peyrard2017learning,
  title={Learning to Score System Summaries for Better Content Selection Evaluation.},
  author={Peyrard, Maxime and Botschen, Teresa and Gurevych, Iryna},
  booktitle={Proceedings of the Workshop on New Frontiers in Summarization},
  year={2017}
}

@inproceedings{janssen2007meta,
  title={On meta-learning rule learning heuristics},
  author={Janssen, Frederik and Furnkranz, Johannes},
  booktitle={ICDM},
  pages={529--534},
  year={2007},
  organization={IEEE}
}

@inproceedings{wauthier2013efficient,
  title={Efficient ranking from pairwise comparisons},
  author={Wauthier, Fabian and Jordan, Michael and Jojic, Nebojsa},
  booktitle={ICML},
  pages={109--117},
  year={2013}
}

@book{boyd2004convex,
  title={Convex optimization},
  author={Boyd, Stephen and Vandenberghe, Lieven},
  year={2004},
  publisher={Cambridge university press}
}

@article{czubenko2015autonomous,
  title={Autonomous driver based on an intelligent system of decision-making},
  author={Czubenko, Micha{\l} and Kowalczuk, Zdzis{\l}aw and Ordys, Andrew},
  journal={Cognitive computation},
  volume={7},
  number={5},
  pages={569--581},
  year={2015},
  publisher={Springer}
}

@inproceedings{kumar2012web,
  title={Web analytics and metrics: a survey},
  author={Kumar, Lakhwinder and Singh, Hardeep and Kaur, Ramandeep},
  booktitle={Proceedings of the International Conference on Advances in Computing, Communications and Informatics},
  pages={966--971},
  year={2012},
  organization={ACM}
}

@article{ferri2009experimental,
  title={An experimental comparison of performance measures for classification},
  author={Ferri, C{\'e}sar and Hern{\'a}ndez-Orallo, Jos{\'e} and Modroiu, R},
  journal={Pattern Recognition Letters},
  volume={30},
  number={1},
  pages={27--38},
  year={2009},
  publisher={Elsevier}
}

@inproceedings{caruana2004data,
  title={Data mining in metric space: an empirical analysis of supervised learning performance criteria},
  author={Caruana, Rich and Niculescu-Mizil, Alexandru},
  booktitle={ACM SIGKDD},
  year={2004}
}

@inproceedings{jamieson2011active,
  title={Active ranking using pairwise comparisons},
  author={Jamieson, Kevin G and Nowak, Robert},
  booktitle={NeurIPS},
  pages={2240--2248},
  year={2011}
}

@inproceedings{herbrich2000large,
  title={Large margin rank boundaries for ordinal regression},
  author={Herbrich, Ralf},
  booktitle={Advances in large margin classifiers},
  pages={115--132},
  year={2000},
  publisher={The MIT Press}
}

@ARTICLE{doshi2017towards,
   author = {{Doshi-Velez}, F. and {Kim}, B.},
    title = "{Towards A Rigorous Science of Interpretable Machine Learning}",
  journal = {ArXiv e-prints:1702.08608},
archivePrefix = "arXiv",
   eprint = {1702.08608},
 primaryClass = "stat.ML",
 keywords = {Statistics - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Learning},
     year = 2017,
   adsurl = {http://adsabs.harvard.edu/abs/2017arXiv170208608D},
  adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}

@article{doshi2017towards1,
  title={Towards a rigorous science of interpretable machine learning},
  archivePrefix = {"arXiv"},
       eprint = {1702.08608},
 primaryClass = {"quant-ph"},
  author={Doshi-Velez, Finale and Kim, Been},
  year={2017}
}

@inproceedings{ribeiro2016should,
  title={Why should i trust you?: Explaining the predictions of any classifier},
  author={Ribeiro, Marco Tulio and Singh, Sameer and Guestrin, Carlos},
  booktitle={ACM SIGKDD},
  year={2016},
}

@book{sox1988medical,
  title={Medical decision making},
  author={Sox, Harold C},
  year={1988},
  publisher={ACP Press}
}

@book{cormen2009introduction,
  title={Introduction to algorithms},
  author={Cormen, Thomas H},
  year={2009},
  publisher={MIT press}
}

@book{varian2005,
author={Hal R. Varian},
 editor = {M. Szenberg and L. Ramrattand and A. A. Gottesman}, 
 title={Revealed preference. In Samuelsonian Economics and the 21st Century by M. Szenberg and L. Ramrattand and A. A. Gottesman (editors)},
booktitle = {In Samuelsonian Economics and the 21st Century},
  year={2005},
  pages={99--115},
  publisher={Oxford University Press}
}

@article{mas1977recoverability,
  title={The recoverability of consumers' preferences from market demand behavior},
  author={Mas-Colell, Andreu},
  journal={Econometrica: Journal of the Econometric Society},
  pages={1409--1430},
  year={1977},
  publisher={JSTOR}
}

@article{afriat1967construction,
  title={The construction of utility functions from expenditure data},
  author={Afriat, Sydney N},
  journal={International economic review},
  volume={8},
  number={1},
  pages={67--77},
  year={1967},
  publisher={JSTOR}
}

@article{samuelson1938note,
  title={A note on the pure theory of consumer's behaviour},
  author={Samuelson, Paul A},
  journal={Economica},
  volume={5},
  number={17},
  pages={61--71},
  year={1938},
  publisher={JSTOR}
}

@inproceedings{Chen2004SurveyOP,
  title={Survey of Preference Elicitation Methods},
  author={Li Chen},
  year={2004}
}

@inproceedings{Hines2010PreferenceEF,
  title={Preference elicitation for risky prospects},
  author={Greg Hines and Kate Larson},
  booktitle={AAMAS},
  year={2010}
}

@article{nasrabadi2007pattern,
  title={Pattern recognition and machine learning},
  author={Nasrabadi, Nasser M},
  journal={Journal of electronic imaging},
  volume={16},
  number={4},
  pages={049901},
  year={2007},
  publisher={International Society for Optics and Photonics}
}

@inproceedings{cortes2004auc,
  title={\emph{AUC} optimization vs. error rate minimization},
  author={Cortes, Corinna and Mohri, Mehryar},
  booktitle={NeurIPS},
  pages={313--320},
  year={2004}
}

@inproceedings{street1993nuclear,
  title={Nuclear feature extraction for breast tumor diagnosis},
  author={Street, W Nick and Wolberg, William H and Mangasarian, Olvi L},
  booktitle={Biomedical Image Processing and Biomedical Visualization},
  volume={1905},
  pages={861--871},
  year={1993},
  organization={International Society for Optics and Photonics}
}

@inproceedings{dvovrak2007softening,
  title={Softening splits in decision trees using simulated annealing},
  author={Dvo{\v{r}}{\'a}k, Jakub and Savick{\`y}, Petr},
  booktitle={International Conference on Adaptive and Natural Computing Algorithms},
  year={2007},
}

@inproceedings{webb2006introducing,
  title={Introducing the Webb Spam Corpus: Using Email Spam to Identify Web Spam Automatically.},
  author={Webb, Steve and Caverlee, James and Pu, Calton},
  booktitle={CEAS},
  year={2006}
}

@techreport{heck1998corsika,
  title={CORSIKA: A Monte Carlo code to simulate extensive air showers},
  author={Heck, Dieter and Schatz, G and Knapp, J and Thouw, T and Capdevielle, JN},
  year={1998}
}

@article{hernandez2013roc,
  title={ROC curves in cost space},
  author={Hern{\'a}ndez-Orallo, Jos{\'e} and Flach, Peter and Ferri, C{\'e}sar},
  journal={Machine learning},
  volume={93},
  number={1},
  pages={71--91},
  year={2013},
  publisher={Springer}
}

@inproceedings{davis2006relationship,
  title={The relationship between Precision-Recall and ROC curves},
  author={Davis, Jesse and Goadrich, Mark},
  booktitle={ICML},
  year={2006},
}

@article{liu2009learning,
  title={Learning to rank for information retrieval},
  author={Liu, Tie-Yan and others},
  journal={Foundations and Trends in Information Retrieval},
  volume={3},
  number={3},
  pages={225--331},
  year={2009},
  publisher={Now Publishers, Inc.}
}

@inproceedings{negahban2012iterative,
  title={Iterative ranking from pair-wise comparisons},
  author={Negahban, Sahand and Oh, Sewoong and Shah, Devavrat},
  booktitle={NeurIPS},
  pages={2474--2482},
  year={2012}
}

@inproceedings{braziunas2007minimax,
  title={Minimax regret based elicitation of generalized additive utilities.},
  author={Braziunas, Darius and Boutilier, Craig},
  booktitle={UAI},
  pages={25--32},
  year={2007}
}

@inproceedings{tamburrelli2014towards,
  title={Towards automated \emph{A/B} testing},
  author={Tamburrelli, Giordano and Margara, Alessandro},
  booktitle={International Symposium on Search Based Software Engineering},
  year={2014},
}

@inproceedings{hacker2009matchin,
  title={Matchin: eliciting user preferences with an online game},
  author={Hacker, Severin and Von Ahn, Luis},
  booktitle={Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
  pages={1207--1216},
  year={2009},
  organization={ACM}
}

@book{dhami2016foundations,
  title={The foundations of behavioral economic analysis},
  author={Dhami, Sanjit},
  year={2016},
  publisher={Oxford University Press}
}

@article{sokolova2009systematic,
  title={A systematic analysis of performance measures for classification tasks},
  author={Sokolova, Marina and Lapalme, Guy},
  journal={Information Processing \& Management},
  volume={45},
  number={4},
  pages={427--437},
  year={2009},
  publisher={Elsevier}
}

@inproceedings{elkan2001foundations,
  title={The foundations of cost-sensitive learning},
  author={Elkan, Charles},
  booktitle={International joint conference on artificial intelligence},
  volume={17},
  pages={973--978},
  year={2001},
  organization={Lawrence Erlbaum Associates Ltd}
}

@article{zhang2004statistical,
  title={Statistical analysis of some multi-category large margin classification methods},
  author={Zhang, Tong},
  journal={Journal of Machine Learning Research},
  volume={5},
  number={Oct},
  pages={1225--1251},
  year={2004}
}

@article{zhang2004statistical,
  title={Statistical behavior and consistency of classification methods based on convex risk minimization},
  author={Zhang, Tong},
  journal={Annals of Statistics},
  pages={56--85},
  year={2004},
  publisher={JSTOR}
}

@article{tewari2007consistency,
  title={On the consistency of multiclass classification methods},
  author={Tewari, Ambuj and Bartlett, Peter L},
  journal={Journal of Machine Learning Research},
  volume={8},
  number={May},
  pages={1007--1025},
  year={2007}
}

@article{lee2004multicategory,
  title={Multicategory support vector machines: Theory and application to the classification of microarray data and satellite radiance data},
  author={Lee, Yoonkyung and Lin, Yi and Wahba, Grace},
  journal={Journal of the American Statistical Association},
  volume={99},
  number={465},
  pages={67--81},
  year={2004},
  publisher={Taylor \& Francis}
}

@article{wang2012multiclass,
  title={Multiclass imbalance problems: Analysis and potential solutions},
  author={Wang, Shuo and Yao, Xin},
  journal={IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics)},
  volume={42},
  number={4},
  pages={1119--1130},
  year={2012},
  publisher={IEEE}
}

@inproceedings{narasimhan2014statistical,
  title={On the statistical consistency of plug-in classifiers for non-decomposable performance measures},
  author={Narasimhan, Harikrishna and Vaish, Rohit and Agarwal, Shivani},
  booktitle={NeurIPS},
  pages={1493--1501},
  year={2014}
}

@inproceedings{menon2013statistical,
  title={On the statistical consistency of algorithms for binary classification under class imbalance},
  author={Menon, Aditya and Narasimhan, Harikrishna and Agarwal, Shivani and Chawla, Sanjay},
  booktitle={ICML},
  year={2013}
}

@inproceedings{nan2012optimizing,
  title={Optimizing F-measure: A Tale of Two Approaches},
  author={Nan Ye and Kian Ming Adam Chai and Wee Sun Lee and Hai Leong Chieu},
  booktitle={ICML},
  year={2012}
}

@inproceedings{parambath2014optimizing,
  title={Optimizing F-measures by cost-sensitive classification},
  author={Parambath, Shameem Puthiya and Usunier, Nicolas and Grandvalet, Yves},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2123--2131},
  year={2014}
}

@inproceedings{musicant2003optimizing,
  title={Optimizing F-Measure with Support Vector Machines.},
  author={Musicant, David R and Kumar, Vipin and Ozgur, Aysel and others},
  booktitle={FLAIRS conference},
  pages={356--360},
  year={2003}
}

@inproceedings{joachims2005support,
  title={A support vector method for multivariate performance measures},
  author={Joachims, Thorsten},
  booktitle={ICML},
  year={2005},
  organization={ACM}
}

@book{poor2013introduction,
  title={An introduction to signal detection and estimation},
  author={Poor, H Vincent},
  year={2013},
  publisher={Springer Science \& Business Media}
}

@inproceedings{lewis1991evaluating,
  title={Evaluating text categorization i},
  author={Lewis, David D},
  booktitle={Speech and Natural Language: Proceedings of a Workshop Held at Pacific Grove, California, February 19-22, 1991},
  year={1991}
}

@incollection{lawrence1998neural,
  title={Neural network classification and prior class probabilities},
  author={Lawrence, Steve and Burns, Ian and Back, Andrew and Tsoi, Ah Chung and Giles, C Lee},
  booktitle={Neural networks: Tricks of the trade},
  pages={299--313},
  year={1998},
  publisher={Springer}
}

@inproceedings{kim2013genia,
  title={The genia event extraction shared task, 2013 edition-overview},
  author={Kim, Jin-Dong and Wang, Yue and Yasunori, Yamamoto},
  booktitle={Proceedings of the BioNLP Shared Task 2013 Workshop},
  pages={8--15},
  year={2013}
}

@inproceedings{kennedy2009learning,
  title={Learning without default: A study of one-class classification and the low-default portfolio problem},
  author={Kennedy, Kenneth and Mac Namee, Brian and Delany, Sarah Jane},
  booktitle={Irish Conference on Artificial Intelligence and Cognitive Science},
  pages={174--187},
  year={2009},
  organization={Springer}
}

@article{wang2012multiclass,
  title={Multiclass imbalance problems: Analysis and potential solutions},
  author={Wang, Shuo and Yao, Xin},
  journal={IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics)},
  volume={42},
  number={4},
  pages={1119--1130},
  year={2012},
  publisher={IEEE}
}

@inproceedings{sun2006boosting,
  title={Boosting for learning multiple classes with imbalanced class distribution},
  author={Sun, Yanmin and Kamel, Mohamed S and Wang, Yang},
  booktitle={Data Mining, 2006. ICDM'06. Sixth International Conference on},
  pages={592--602},
  year={2006},
  organization={IEEE}
}



@article{steinwart2007compare,
  title={How to compare different loss functions and their risks},
  author={Steinwart, Ingo},
  journal={Constructive Approximation},
  volume={26},
  number={2},
  pages={225--287},
  year={2007},
  publisher={Springer}
}

@article{bartlett2006convexity,
  title={Convexity, classification, and risk bounds},
  author={Bartlett, Peter L and Jordan, Michael I and McAuliffe, Jon D},
  journal={Journal of the American Statistical Association},
  volume={101},
  number={473},
  pages={138--156},
  year={2006},
  publisher={Taylor \& Francis}
}

@article{he2009learning,
  title={Learning from imbalanced data},
  author={He, Haibo and Garcia, Edwardo A},
  journal={IEEE Transactions on knowledge and data engineering},
  volume={21},
  number={9},
  pages={1263--1284},
  year={2009},
  publisher={Ieee}
}

@inproceedings{gu2009evaluation,
  title={Evaluation measures of the classification performance of imbalanced data sets},
  author={Gu, Qiong and Zhu, Li and Cai, Zhihua},
  booktitle={International Symposium on Intelligence Computation and Applications},
  pages={461--471},
  year={2009},
  organization={Springer}
}

@inproceedings{drummond2005severe,
  title={Severe class imbalance: Why better algorithms aren’t the answer},
  author={Drummond, Chris and Holte, Robert C},
  booktitle={ECML},
  pages={539--546},
  year={2005},
  organization={Springer}
}

@inproceedings{lewis1994sequential,
  title={A sequential algorithm for training text classifiers},
  author={Lewis, David D and Gale, William A},
  booktitle={Proceedings of the 17th annual international ACM SIGIR conference on Research and development in information retrieval},
  pages={3--12},
  year={1994},
  organization={Springer-Verlag New York, Inc.}
}

@inproceedings{koyejo2014consistent,
  title={Consistent binary classification with generalized performance metrics},
  author={Koyejo, Oluwasanmi O and Natarajan, Nagarajan and Ravikumar, Pradeep K and Dhillon, Inderjit S},
  booktitle={NeurIPS},
  pages={2744--2752},
  year={2014}
}

@inproceedings{narasimhan2015consistent,
  title={Consistent multiclass algorithms for complex performance measures},
  author={Narasimhan, Harikrishna and Ramaswamy, Harish and Saha, Aadirupa and Agarwal, Shivani},
  booktitle={ICML},
  year={2015}
}

@inproceedings{koyejo2015consistent,
  title={Consistent multilabel classification},
  author={Koyejo, Oluwasanmi O and Natarajan, Nagarajan and Ravikumar, Pradeep K and Dhillon, Inderjit S},
  booktitle={NeurIPS},
  year={2015}
}

@inproceedings{langley00,
 author    = {P. Langley},
 title     = {Crafting Papers on Machine Learning},
 year      = {2000},
 pages     = {1207--1216},
 editor    = {Pat Langley},
 booktitle     = {Proceedings of the 17th International Conference
              on Machine Learning (ICML 2000)},
 address   = {Stanford, CA},
 publisher = {Morgan Kaufmann}
}

@TechReport{mitchell80,
  author = 	 "T. M. Mitchell",
  title = 	 "The Need for Biases in Learning Generalizations",
  institution =  "Computer Science Department, Rutgers University",
  year = 	 "1980",
  address =	 "New Brunswick, MA",
}

@phdthesis{kearns89,
  author = {M. J. Kearns},
  title =  {Computational Complexity of Machine Learning},
  school = {Department of Computer Science, Harvard University},
  year =   {1989}
}

@Book{MachineLearningI,
  editor = 	 "R. S. Michalski and J. G. Carbonell and T.
		  M. Mitchell",
  title = 	 "Machine Learning: An Artificial Intelligence
		  Approach, Vol. I",
  publisher = 	 "Tioga",
  year = 	 "1983",
  address =	 "Palo Alto, CA"
}

@Book{DudaHart2nd,
  author =       "R. O. Duda and P. E. Hart and D. G. Stork",
  title =        "Pattern Classification",
  publisher =    "John Wiley and Sons",
  edition =      "2nd",
  year =         "2000"
}

@misc{anonymous,
  title= {Suppressed for Anonymity},
  author= {Author, N. N.},
  year= {2018}
}

@InCollection{Newell81,
  author =       "A. Newell and P. S. Rosenbloom",
  title =        "Mechanisms of Skill Acquisition and the Law of
                  Practice", 
  booktitle =    "Cognitive Skills and Their Acquisition",
  pages =        "1--51",
  publisher =    "Lawrence Erlbaum Associates, Inc.",
  year =         "1981",
  editor =       "J. R. Anderson",
  chapter =      "1",
  address =      "Hillsdale, NJ"
}


@Article{Samuel59,
  author = 	 "A. L. Samuel",
  title = 	 "Some Studies in Machine Learning Using the Game of
		  Checkers",
  journal =	 "IBM Journal of Research and Development",
  year =	 "1959",
  volume =	 "3",
  number =	 "3",
  pages =	 "211--229"
}

@inproceedings{langley00,
 author    = {P. Langley},
 title     = {Crafting Papers on Machine Learning},
 year      = {2000},
 pages     = {1207--1216},
 editor    = {Pat Langley},
 booktitle     = {Proceedings of the 17th International Conference
              on Machine Learning (ICML 2000)},
 address   = {Stanford, CA},
 publisher = {Morgan Kaufmann}
}

@TechReport{mitchell80,
  author = 	 "T. M. Mitchell",
  title = 	 "The Need for Biases in Learning Generalizations",
  institution =  "Computer Science Department, Rutgers University",
  year = 	 "1980",
  address =	 "New Brunswick, MA",
}

@phdthesis{kearns89,
  author = {M. J. Kearns},
  title =  {Computational Complexity of Machine Learning},
  school = {Department of Computer Science, Harvard University},
  year =   {1989}
}

@Book{MachineLearningI,
  editor = 	 "R. S. Michalski and J. G. Carbonell and T.
		  M. Mitchell",
  title = 	 "Machine Learning: An Artificial Intelligence
		  Approach, Vol. I",
  publisher = 	 "Tioga",
  year = 	 "1983",
  address =	 "Palo Alto, CA"
}

@Book{DudaHart2nd,
  author =       "R. O. Duda and P. E. Hart and D. G. Stork",
  title =        "Pattern Classification",
  publisher =    "John Wiley and Sons",
  edition =      "2nd",
  year =         "2000"
}

@misc{anonymous,
  title= {Suppressed for Anonymity},
  author= {Author, N. N.},
  year= {2019}
}

@InCollection{Newell81,
  author =       "A. Newell and P. S. Rosenbloom",
  title =        "Mechanisms of Skill Acquisition and the Law of
                  Practice", 
  booktitle =    "Cognitive Skills and Their Acquisition",
  pages =        "1--51",
  publisher =    "Lawrence Erlbaum Associates, Inc.",
  year =         "1981",
  editor =       "J. R. Anderson",
  chapter =      "1",
  address =      "Hillsdale, NJ"
}


@Article{Samuel59,
  author = 	 "A. L. Samuel",
  title = 	 "Some Studies in Machine Learning Using the Game of
		  Checkers",
  journal =	 "IBM Journal of Research and Development",
  year =	 "1959",
  volume =	 "3",
  number =	 "3",
  pages =	 "211--229"
}



@InProceedings{Fab2,
  Title                    = {Tweet Sentiment: From Classification to Quantification},
  Author                   = {Gao, W. and Sebastiani, F.},
  Booktitle                = {ASONAM},
  Year                     = {2015}
 }


@Article{Fab1,
  Title                    = {Optimizing Text Quantifiers for Multivariate Loss Functions},
  Author                   = {Esuli, A. and Sebastiani, F.},
  Journal                  = {ACM Transactions on Knowledge Discovery and Data},
  Year                     = {2015},
  Number                   = {4},
  Volume                   = {9},
}

@incollection{Lawrence+98,
	author={Lawrence, S. and Burns, I. and Back, A. and Tsoi, A-C. and Giles, C.L.},
	title={Neural Network Classification and Prior Class Probabilities},
	booktitle={Neural Networks: Tricks of the Trade},
	pages={1524:299--313},
	year={1998},
    publisher={Springer},
	series={LNCS}
}



@INPROCEEDINGS{LiuCh11,
  author = {W. Liu and S. Chawla},
  title = {A Quadratic Mean based Supervised Learning Model for Managing Data
	Skewness},
  booktitle = {SDM},
  year = {2011}
}

@inproceedings{Kar16,
  title={Online optimization methods for the quantification problem},
  author={Kar, Purushottam and Li, Shuai and Narasimhan, Harikrishna and Chawla, Sanjay and Sebastiani, Fabrizio},
  booktitle={ACM SIGKDD},
  year={2016}
}

@article{Lindsay08,
  title={Quadratic distances on probabilities: A unified foundation},
  author={Lindsay, Bruce G and Markatou, Marianthi and Ray, Surajit and Yang, Ke and Chen, Shu-Chuan and others},
  journal={The Annals of Statistics},
  volume={36},
  number={2},
  pages={983--1006},
  year={2008},
  publisher={Institute of Mathematical Statistics}
}


@inproceedings{Cotter:2019,
  title = {On Making Stochastic Classifiers Deterministic},
  author = {Cotter, A. and  Narasimhan, H. and Gupta, M.},
  booktitle = {{NeurIPS}},
  year = {{2019}}
}


@article{Daniely:2015,
  author = {Daniely, Amit and Sabato, Sivan and Ben-David, Shai and Shalev-Shwartz, Shai},
  title = {Multiclass Learnability and the {ERM} Principle},
  year = {2015},
  issue_date = {January 2015},
  volume = {16},
  number = {1},
  journal = {{JMLR}},
  month = jan,
  pages = {2377–2404},
  numpages = {28},
}

@article{Natarajan:1989,
  author = {Natarajan, B. K.},
  journal = {Machine Learning},
  number = 1,
  pages = {67-97},
  title = {On learning sets and functions},
  volume = 4,
  year = 1989,
}