@article{VanHorn:2017,
  title={The Devil is in the Tails: Fine-grained Classification in the Wild},
  author={Van Horn, Grant and Perona, Pietro},
  journal={arXiv preprint arXiv:1709.01450},
  year={2017}
}

@article{d2021tale,
  title={A Tale Of Two Long Tails},
  author={D'souza, Daniel and Nussbaum, Zach and Agarwal, Chirag and Hooker, Sara},
  journal={arXiv preprint arXiv:2107.13098},
  year={2021}
}

@misc{feldman2021does,
      title={Does Learning Require Memorization? A Short Tale about a Long Tail}, 
      author={Vitaly Feldman},
      year={2021},
      eprint={1906.05271},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{
lukasik2021teachers,
title={Teacher{\textquoteright}s pet: understanding and mitigating biases in distillation},
author={Michal Lukasik and Srinadh Bhojanapalli and Aditya Krishna Menon and Sanjiv Kumar},
journal={Transactions on Machine Learning Research},
issn={2835-8856},
year={2022},
url={https://openreview.net/forum?id=ph3AYXpwEb},
note={}
}

@article{46642,
  title={Large scale distributed neural network training through online distillation},
  author={Anil, Rohan and Pereyra, Gabriel and Passos, Alexandre and Ormandi, Robert and Dahl, George E and Hinton, Geoffrey E},
  journal={arXiv preprint arXiv:1804.03235},
  year={2018}
}



@inproceedings{Furlanello2018BornAN,
  title={Born Again Neural Networks},
  author={Tommaso Furlanello and Zachary Chase Lipton and Michael Tschannen and Laurent Itti and Anima Anandkumar},
  booktitle={ICML},
  year={2018}
}

@article{roberts2020,
  author    = {Adam Roberts and
               Colin Raffel and
               Noam Shazeer},
  title     = {How Much Knowledge Can You Pack Into the Parameters of a Language
               Model?},
  journal   = {CoRR},
  volume    = {abs/2002.08910},
  year      = {2020},
  url       = {https://arxiv.org/abs/2002.08910},
  eprinttype = {arXiv},
  eprint    = {2002.08910},
  timestamp = {Mon, 02 Mar 2020 16:46:06 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-2002-08910.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{10.1145/1150402.1150464,
author = {Buciluundefined, Cristian and Caruana, Rich and Niculescu-Mizil, Alexandru},
title = {Model Compression},
year = {2006},
isbn = {1595933395},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/1150402.1150464},
doi = {10.1145/1150402.1150464},
abstract = {Often the best performing supervised learning models are ensembles of hundreds or thousands of base-level classifiers. Unfortunately, the space required to store this many classifiers, and the time required to execute them at run-time, prohibits their use in applications where test sets are large (e.g. Google), where storage space is at a premium (e.g. PDAs), and where computational power is limited (e.g. hearing aids). We present a method for "compressing" large, complex ensembles into smaller, faster models, usually without significant loss in performance.},
booktitle = {Proceedings of the 12th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining},
pages = {535–541},
numpages = {7},
keywords = {model compression, supervised learning},
location = {Philadelphia, PA, USA},
series = {KDD '06}
}

@misc{rae2021scaling,
      title={Scaling Language Models: Methods, Analysis & Insights from Training Gopher}, 
      author={Jack W. Rae and Sebastian Borgeaud and Trevor Cai and Katie Millican and Jordan Hoffmann and Francis Song and John Aslanides and Sarah Henderson and Roman Ring and Susannah Young and Eliza Rutherford and Tom Hennigan and Jacob Menick and Albin Cassirer and Richard Powell and George van den Driessche and Lisa Anne Hendricks and Maribeth Rauh and Po-Sen Huang and Amelia Glaese and Johannes Welbl and Sumanth Dathathri and Saffron Huang and Jonathan Uesato and John Mellor and Irina Higgins and Antonia Creswell and Nat McAleese and Amy Wu and Erich Elsen and Siddhant Jayakumar and Elena Buchatskaya and David Budden and Esme Sutherland and Karen Simonyan and Michela Paganini and Laurent Sifre and Lena Martens and Xiang Lorraine Li and Adhiguna Kuncoro and Aida Nematzadeh and Elena Gribovskaya and Domenic Donato and Angeliki Lazaridou and Arthur Mensch and Jean-Baptiste Lespiau and Maria Tsimpoukelli and Nikolai Grigorev and Doug Fritz and Thibault Sottiaux and Mantas Pajarskas and Toby Pohlen and Zhitao Gong and Daniel Toyama and Cyprien de Masson d'Autume and Yujia Li and Tayfun Terzi and Vladimir Mikulik and Igor Babuschkin and Aidan Clark and Diego de Las Casas and Aurelia Guy and Chris Jones and James Bradbury and Matthew Johnson and Blake Hechtman and Laura Weidinger and Iason Gabriel and William Isaac and Ed Lockhart and Simon Osindero and Laura Rimell and Chris Dyer and Oriol Vinyals and Kareem Ayoub and Jeff Stanway and Lorrayne Bennett and Demis Hassabis and Koray Kavukcuoglu and Geoffrey Irving},
      year={2021},
      eprint={2112.11446},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{bommasani2021opportunities,
      title={On the Opportunities and Risks of Foundation Models}, 
      author={Rishi Bommasani and Drew A. Hudson and Ehsan Adeli and Russ Altman and Simran Arora and Sydney von Arx and Michael S. Bernstein and Jeannette Bohg and Antoine Bosselut and Emma Brunskill and Erik Brynjolfsson and Shyamal Buch and Dallas Card and Rodrigo Castellon and Niladri Chatterji and Annie Chen and Kathleen Creel and Jared Quincy Davis and Dora Demszky and Chris Donahue and Moussa Doumbouya and Esin Durmus and Stefano Ermon and John Etchemendy and Kawin Ethayarajh and Li Fei-Fei and Chelsea Finn and Trevor Gale and Lauren Gillespie and Karan Goel and Noah Goodman and Shelby Grossman and Neel Guha and Tatsunori Hashimoto and Peter Henderson and John Hewitt and Daniel E. Ho and Jenny Hong and Kyle Hsu and Jing Huang and Thomas Icard and Saahil Jain and Dan Jurafsky and Pratyusha Kalluri and Siddharth Karamcheti and Geoff Keeling and Fereshte Khani and Omar Khattab and Pang Wei Koh and Mark Krass and Ranjay Krishna and Rohith Kuditipudi and Ananya Kumar and Faisal Ladhak and Mina Lee and Tony Lee and Jure Leskovec and Isabelle Levent and Xiang Lisa Li and Xuechen Li and Tengyu Ma and Ali Malik and Christopher D. Manning and Suvir Mirchandani and Eric Mitchell and Zanele Munyikwa and Suraj Nair and Avanika Narayan and Deepak Narayanan and Ben Newman and Allen Nie and Juan Carlos Niebles and Hamed Nilforoshan and Julian Nyarko and Giray Ogut and Laurel Orr and Isabel Papadimitriou and Joon Sung Park and Chris Piech and Eva Portelance and Christopher Potts and Aditi Raghunathan and Rob Reich and Hongyu Ren and Frieda Rong and Yusuf Roohani and Camilo Ruiz and Jack Ryan and Christopher Ré and Dorsa Sadigh and Shiori Sagawa and Keshav Santhanam and Andy Shih and Krishnan Srinivasan and Alex Tamkin and Rohan Taori and Armin W. Thomas and Florian Tramèr and Rose E. Wang and William Wang and Bohan Wu and Jiajun Wu and Yuhuai Wu and Sang Michael Xie and Michihiro Yasunaga and Jiaxuan You and Matei Zaharia and Michael Zhang and Tianyi Zhang and Xikun Zhang and Yuhui Zhang and Lucia Zheng and Kaitlyn Zhou and Percy Liang},
      year={2021},
      eprint={2108.07258},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}


@misc{raffel2020exploring,
      title={Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer}, 
      author={Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu},
      year={2020},
      eprint={1910.10683},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}



@inproceedings{menon2020long,
  title={Long-tail learning via logit adjustment},
  author={Menon, Aditya Krishna and Jayasumana, Sadeep and Rawat, Ankit Singh and Jain, Himanshu and Veit, Andreas and Kumar, Sanjiv},
  booktitle={International Conference on Learning Representations (ICLR)},
  year={2021}
}

@article{narasimhan2021training,
  title={Training Over-parameterized Models with Non-decomposable Objectives},
  author={Narasimhan, Harikrishna and Menon, Aditya K},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@inproceedings{Sagawa2020Distributionally,
title={Distributionally Robust Neural Networks},
author={Shiori Sagawa and Pang Wei Koh and Tatsunori B. Hashimoto and Percy Liang},
booktitle={International Conference on Learning Representations},
year={2020},
}

@inproceedings{kini2021labelimbalanced,
title={Label-Imbalanced and Group-Sensitive Classification under Overparameterization},
author={Ganesh Ramachandra Kini and Orestis Paraskevas and Samet Oymak and Christos Thrampoulidis},
booktitle={Advances in Neural Information Processing Systems},
year={2021},
}

@article{piratla2021focus,
  title={Focus on the Common Good: Group Distributional Robustness Follows},
  author={Piratla, Vihari and Netrapalli, Praneeth and Sarawagi, Sunita},
  journal={arXiv preprint arXiv:2110.02619},
  year={2021}
}

@inproceedings{menon2021statistical,
  title={A statistical perspective on distillation},
  author={Menon, Aditya K and Rawat, Ankit Singh and Reddi, Sashank and Kim, Seungyeon and Kumar, Sanjiv},
  booktitle={International Conference on Machine Learning},
  pages={7632--7642},
  year={2021},
  organization={PMLR}
}

@article{cotter2019optimization,
  title={Optimization with non-differentiable constraints with applications to fairness, recall, churn, and other goals},
  author={Cotter, Andrew and Jiang, Heinrich and Wang, Serena and Narayan, Taman and You, Seungil and Sridharan, Karthik and Gupta, Maya R.},
  journal={Journal of Machine Learning Research (JMLR)},
  volume={20},
  number={172},
  pages={1--59},
  year={2019}
}

@inproceedings{cotter2019training,
  title={Training well-generalizing classifiers for fairness metrics and other data-dependent constraints},
  author={Cotter, Andrew and Gupta, Maya and Jiang, Heinrich and Srebro, Nathan and Sridharan, Karthik and Wang, Serena and Woodworth, Blake and You, Seungil},
  booktitle={International Conference on Machine Learning},
  pages={1397--1405},
  year={2019},
  organization={PMLR}
}

@article{AbbasiAsl2021StructuralCO,
  title={Structural Compression of Convolutional Neural Networks with Applications in Interpretability},
  author={Reza Abbasi-Asl and Bin Yu},
  journal={Frontiers in Big Data},
  year={2021},
  volume={4}
}

@misc{park2020attribution,
      title={Attribution Preservation in Network Compression for Reliable Network Interpretation}, 
      author={Geondo Park and June Yong Yang and Sung Ju Hwang and Eunho Yang},
      year={2020},
      eprint={2010.15054},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}


@article{liebenwein2021lost,
  title={Lost in Pruning: The Effects of Pruning Neural Networks beyond Test Accuracy},
  author={Liebenwein, Lucas and Baykal, Cenk and Carter, Brandon and Gifford, David and Rus, Daniela},
  journal={Proceedings of Machine Learning and Systems},
  volume={3},
  year={2021}
}


@article{Sehwag2019,
  author        = {Vikash Sehwag and
               Shiqi Wang and
               Prateek Mittal and
               Suman Jana},
  title         = {Towards Compact and Robust Deep Neural Networks},
  journal       = {CoRR},
  volume        = {abs/1906.06110},
  year          = {2019},
  url           = {http://arxiv.org/abs/1906.06110},
  archiveprefix = {arXiv},
  eprint        = {1906.06110},
  timestamp     = {Mon, 24 Jun 2019 17:28:45 +0200},
  biburl        = {https://dblp.org/rec/journals/corr/abs-1906-06110.bib},
  bibsource     = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{chen2017robust,
  title={Robust Optimization for Non-Convex Objectives},
  author={Chen, Robert S and Lucier, Brendan and Singer, Yaron and Syrgkanis, Vasilis},
  booktitle={NeurIPS},
  year={2017}
}

@incollection{NIPS2018_7308,
title = {Sparse DNNs with Improved Adversarial Robustness},
author = {Guo, Yiwen and Zhang, Chao and Zhang, Changshui and Chen, Yurong},
booktitle = {Advances in Neural Information Processing Systems 31},
pages = {242--251},
year = {2018},
}

@inproceedings{NEURIPS2020_3f5ee243,
 author = {Wang, Wenhui and Wei, Furu and Dong, Li and Bao, Hangbo and Yang, Nan and Zhou, Ming},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 title = {MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers},
 year = {2020}
}


@misc{menghani2021efficient,
      title={Efficient Deep Learning: A Survey on Making Deep Learning Models Smaller, Faster, and Better}, 
      author={Gaurav Menghani},
      year={2021},
      eprint={2106.08962},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{williamson2016composite,
  title={Composite Multiclass Losses},
  author={Williamson, Robert C and Vernet, Elodie and Reid, Mark D},
  journal={Journal of Machine Learning Research},
  volume={17},
  pages={1--52},
  year={2016}
}

@misc{du2021compressed,
      title={What do Compressed Large Language Models Forget? Robustness Challenges in Model Compression}, 
      author={Mengnan Du and Subhabrata Mukherjee and Yu Cheng and Milad Shokouhi and Xia Hu and Ahmed Hassan Awadallah},
      year={2021},
      eprint={2110.08419},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@inproceedings{zhao2020maintaining,
  title={Maintaining discrimination and fairness in class incremental learning},
  author={Zhao, Bowen and Xiao, Xi and Gan, Guojun and Zhang, Bin and Xia, Shu-Tao},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={13208--13217},
  year={2020}
}

@article{Hinton2015DistillingTK,
  title={Distilling the Knowledge in a Neural Network},
  author={Geoffrey E. Hinton and Oriol Vinyals and Jeffrey Dean},
  journal={ArXiv},
  year={2015},
  volume={abs/1503.02531}
}

@inproceedings{xu2020class,
  title={Class-Weighted Classification: Trade-offs and Robust Approaches},
  author={Xu, Ziyu and Dan, Chen and Khim, Justin and Ravikumar, Pradeep},
  booktitle={International Conference on Machine Learning},
  pages={10544--10554},
  year={2020},
  organization={PMLR}
}

@inproceedings{goldblum2020adversarially,
  title={Adversarially robust distillation},
  author={Goldblum, Micah and Fowl, Liam and Feizi, Soheil and Goldstein, Tom},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={3996--4003},
  year={2020}
}

@inproceedings{zi2021revisiting,
  title={Revisiting adversarial robustness distillation: Robust soft labels make student better},
  author={Zi, Bojia and Zhao, Shihao and Ma, Xingjun and Jiang, Yu-Gang},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={16443--16452},
  year={2021}
}

@inproceedings{carmon2019unlabeled,
  title={Unlabeled data improves adversarial robustness},
  author={Carmon, Yair and Raghunathan, Aditi and Schmidt, Ludwig and Liang, Percy and Duchi, John C},
  booktitle={Proceedings of the 33rd International Conference on Neural Information Processing Systems},
  pages={11192--11203},
  year={2019}
}

@article{raghunathan2020understanding,
  title={Understanding and mitigating the tradeoff between robustness and accuracy},
  author={Raghunathan, Aditi and Xie, Sang Michael and Yang, Fanny and Duchi, John and Liang, Percy},
  journal={arXiv preprint arXiv:2002.10716},
  year={2020}
}

@inproceedings{sagawa2020investigation,
  title={An investigation of why overparameterization exacerbates spurious correlations},
  author={Sagawa, Shiori and Raghunathan, Aditi and Koh, Pang Wei and Liang, Percy},
  booktitle={International Conference on Machine Learning},
  pages={8346--8356},
  year={2020},
  organization={PMLR}
}

@inproceedings{Bucilua:2006,
	author = {Bucilua, Cristian and Caruana, Rich and Niculescu-Mizil, Alexandru},
	title = {Model Compression},
	year = {2006},
	booktitle = {{KDD}},
}

@article{Hinton:2015,
	title = {Distilling the knowledge in a neural network},
	author = {G. Hinton and O. Vinyals and J. Dean},
	journal = {arXiv:1503.02531},
	year = {2015},
}

@inproceedings{Radosavovic:2018,
	author={I. Radosavovic and P. Doll\'{a}r and R. Girshick and G. Gkioxari and K. He},
	title={Data Distillation: Towards Omni-Supervised Learning}, 
	booktitle={{CVPR}},
	year={2018},
}

@inproceedings{Hardt:2016,
  author = {Moritz Hardt and Eric Price and Nathan Srebro},
  title = {Equality of Opportunity in Supervised Learning},
  booktitle = {{NIPS}},
  year = {2016},
}

@article{sohoni2020no,
  title={No Subclass Left Behind: Fine-Grained Robustness in Coarse-Grained Classification Problems},
  author={Sohoni, Nimit and Dunnmon, Jared and Angus, Geoffrey and Gu, Albert and R{\'e}, Christopher},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}

@inproceedings{zafar2017fairness,
  title={Fairness constraints: Mechanisms for fair classification},
  author={Zafar, Muhammad Bilal and Valera, Isabel and Rogriguez, Manuel Gomez and Gummadi, Krishna P},
  booktitle={Artificial Intelligence and Statistics},
  pages={962--970},
  year={2017},
  organization={PMLR}
}

@article{hooker2020characterising,
  title={Characterising bias in compressed models},
  author={Hooker, Sara and Moorosi, Nyalleng and Clark, Gregory and Bengio, Samy and Denton, Emily},
  journal={arXiv preprint arXiv:2010.03058},
  year={2020}
}

@inproceedings{agarwal2018reductions,
  title={A reductions approach to fair classification},
  author={Agarwal, Alekh and Beygelzimer, Alina and Dud{\'\i}k, Miroslav and Langford, John and Wallach, Hanna},
  booktitle={International Conference on Machine Learning},
  pages={60--69},
  year={2018},
  organization={PMLR}
}

@inproceedings{xu-etal-2021-beyond,
    title = "Beyond Preserved Accuracy: Evaluating Loyalty and Robustness of {BERT} Compression",
    author = "Xu, Canwen  and
      Zhou, Wangchunshu  and
      Ge, Tao  and
      Xu, Ke  and
      McAuley, Julian  and
      Wei, Furu",
    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
    year = "2021",
}

@inproceedings{menon2020overparameterisation,
  title={Overparameterisation and worst-case generalisation: friend or foe?},
  author={Menon, Aditya Krishna and Rawat, Ankit Singh and Kumar, Sanjiv},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@inproceedings{pham2021meta,
  title={Meta pseudo labels},
  author={Pham, Hieu and Dai, Zihang and Xie, Qizhe and Le, Quoc V},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={11557--11568},
  year={2021}
}

@article{shalev2011online,
  title={Online learning and online convex optimization},
  author={Shalev-Shwartz, Shai and others},
  journal={Foundations and trends in Machine Learning},
  volume={4},
  number={2},
  pages={107--194},
  year={2011}
}

@inproceedings{rusu2016policy,
  title={Policy Distillation},
  author={Rusu, Andrei A and Colmenarejo, Sergio Gomez and G{\"u}l{\c{c}}ehre, {\c{C}}aglar and Desjardins, Guillaume and Kirkpatrick, James and Pascanu, Razvan and Mnih, Volodymyr and Kavukcuoglu, Koray and Hadsell, Raia},
  booktitle={ICLR},
  year={2016}
}

@inproceedings{yang2019training,
  title={Training deep neural networks in generations: A more tolerant teacher educates better students},
  author={Yang, Chenglin and Xie, Lingxi and Qiao, Siyuan and Yuille, Alan L},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={33},
  number={01},
  pages={5628--5635},
  year={2019}
}

@inproceedings{xie2020self,
  title={Self-training with noisy student improves imagenet classification},
  author={Xie, Qizhe and Luong, Minh-Thang and Hovy, Eduard and Le, Quoc V},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={10687--10698},
  year={2020}
}

@TECHREPORT{Krizhevsky09learningmultiple,
    author = {Alex Krizhevsky},
    title = {Learning multiple layers of features from tiny images},
    institution = {University of Toronto},
    year = {2009}
}

@misc{le2015tiny,
  title={Tiny imagenet visual recognition challenge},
  author={Le, Ya and Yang, Xuan},
  howpublished={CS 231N},
  year={2015}
}

@article{ILSVRC15,
  Author = {Olga Russakovsky and Jia Deng and Hao Su and Jonathan Krause and Sanjeev Satheesh and Sean Ma and Zhiheng Huang and Andrej Karpathy and Aditya Khosla and Michael Bernstein and Alexander C. Berg and Li Fei-Fei},
  Title = {{ImageNet Large Scale Visual Recognition Challenge}},
  Year = {2015},
  journal = {International Journal of Computer Vision (IJCV)},
  volume={115},
  number={3},
  pages={211-252}
}


@article{cao2019learning,
  title={Learning Imbalanced Datasets with Label-Distribution-Aware Margin Loss},
  author={Cao, Kaidi and Wei, Colin and Gaidon, Adrien and Arechiga, Nikos and Ma, Tengyu},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  pages={1567--1578},
  year={2019}
}

@inproceedings{cui2019class,
  title={Class-balanced loss based on effective number of samples},
  author={Cui, Yin and Jia, Menglin and Lin, Tsung-Yi and Song, Yang and Belongie, Serge},
  booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
  pages={9268--9277},
  year={2019}
}

@article{loshchilov2016sgdr,
  title={Sgdr: Stochastic gradient descent with warm restarts},
  author={Loshchilov, Ilya and Hutter, Frank},
  journal={arXiv preprint arXiv:1608.03983},
  year={2016}
}

@article{sion1958general,
  title={On general minimax theorems.},
  author={Sion, Maurice},
  journal={Pacific Journal of mathematics},
  volume={8},
  number={1},
  pages={171--176},
  year={1958},
  publisher={Pacific Journal of Mathematics, A Non-profit Corporation}
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@inproceedings{liu2017sphereface,
  title={Sphereface: Deep hypersphere embedding for face recognition},
  author={Liu, Weiyang and Wen, Yandong and Yu, Zhiding and Li, Ming and Raj, Bhiksha and Song, Le},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={212--220},
  year={2017}
}

@inproceedings{Cao:2019,
	author = {Cao, Kaidi and Wei, Colin and Gaidon, Adrien and Arechiga, Nikos and Ma, Tengyu},
	booktitle = {Advances in Neural Information Processing Systems},
	title = {Learning Imbalanced Datasets with Label-Distribution-Aware Margin Loss},
	year = {2019}}

@inproceedings{Cui:2019,
	author = {Cui, Yin and Jia, Menglin and Lin, Tsung-Yi and Song, Yang and Belongie, Serge},
	booktitle = {CVPR},
	title = {Class-Balanced Loss Based on Effective Number of Samples},
	year = {2019}}

@inproceedings{Yin:2018,
	author = {Yin, Xi and Yu, Xiang and Sohn, Kihyuk and Liu, Xiaoming and Chandraker, Manmohan},
	booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
	month = {June},
	title = {Feature Transfer Learning for Face Recognition With Under-Represented Data},
	year = {2019}}
	
	
@misc{Zhang:2019,
	archiveprefix = {arXiv},
	author = {Junjie Zhang and Lingqiao Liu and Peng Wang and Chunhua Shen},
	eprint = {1912.04486},
	primaryclass = {cs.CV},
	title = {To Balance or Not to Balance: A Simple-yet-Effective Approach for Learning with Long-Tailed Distributions},
	year = {2019}}
	
@inproceedings{cui2021parametric,
  title={Parametric contrastive learning},
  author={Cui, Jiequan and Zhong, Zhisheng and Liu, Shu and Yu, Bei and Jia, Jiaya},
  booktitle={Proceedings of the IEEE/CVF international conference on computer vision},
  pages={715--724},
  year={2021}
}

@inproceedings{Ren:2020,
	author = {Ren, Jiawei and Yu, Cunjun and sheng, shunan and Ma, Xiao and Zhao, Haiyu and Yi, Shuai and Li, hongsheng},
	booktitle = {Advances in Neural Information Processing Systems},
	editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
	pages = {4175--4186},
	publisher = {Curran Associates, Inc.},
	title = {Balanced Meta-Softmax for Long-Tailed Visual Recognition},
	volume = {33},
	year = {2020}}
	
	
@article{cui2022reslt,
  title={Reslt: Residual learning for long-tailed recognition},
  author={Cui, Jiequan and Liu, Shu and Tian, Zhuotao and Zhong, Zhisheng and Jia, Jiaya},
  journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
  year={2022},
  publisher={IEEE}
}

@inproceedings{wang2020long,
  title={Long-tailed Recognition by Routing Diverse Distribution-Aware Experts},
  author={Wang, Xudong and Lian, Long and Miao, Zhongqi and Liu, Ziwei and Yu, Stella},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@article{iscen2021class,
  title={Class-balanced distillation for long-tailed visual recognition},
  author={Iscen, Ahmet and Araujo, Andr{\'e} and Gong, Boqing and Schmid, Cordelia},
  journal={arXiv preprint arXiv:2104.05279},
  year={2021}
}

@article{zhang2021balanced,
  title={Balanced knowledge distillation for long-tailed learning},
  author={Zhang, Shaoyu and Chen, Chen and Hu, Xiyuan and Peng, Silong},
  journal={arXiv preprint arXiv:2104.10510},
  year={2021}
}

@inproceedings{xiang2020learning,
  title={Learning from multiple experts: Self-paced knowledge distillation for long-tailed classification},
  author={Xiang, Liuyu and Ding, Guiguang and Han, Jungong},
  booktitle={European Conference on Computer Vision},
  pages={247--263},
  year={2020},
  organization={Springer}
}

@inproceedings{he2021distilling,
  title={Distilling virtual examples for long-tailed recognition},
  author={He, Yin-Yin and Wu, Jianxin and Wei, Xiu-Shen},
  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
  pages={235--244},
  year={2021}
}

@article{hooker2019compressed,
  title={What do compressed deep neural networks forget?},
  author={Hooker, Sara and Courville, Aaron and Clark, Gregory and Dauphin, Yann and Frome, Andrea},
  journal={arXiv preprint arXiv:1911.05248},
  year={2019}
}

@inproceedings{Lukasik2020,
author = {Lukasik, Michal and Bhojanapalli, Srinadh and Menon, Aditya Krishna and Kumar, Sanjiv},
title = {Does Label Smoothing Mitigate Label Noise?},
year = {2020},
publisher = {JMLR.org},
abstract = {Label smoothing is commonly used in training deep learning models, wherein one-hot training labels are mixed with uniform label vectors. Empirically, smoothing has been shown to improve both predictive performance and model calibration. In this paper, we study whether label smoothing is also effective as a means of coping with label noise. While label smoothing apparently amplifies this problem--being equivalent to injecting symmetric noise to the labels--we show how it relates to a general family of loss-correction techniques from the label noise literature. Building on this connection, we show that label smoothing is competitive with loss-correction under label noise. Further, we show that when distilling models from noisy data, label smoothing of the teacher is beneficial; this is in contrast to recent findings for noise-free problems, and sheds further light on settings where label smoothing is beneficial.},
booktitle = {Proceedings of the 37th International Conference on Machine Learning},
articleno = {598},
numpages = {11},
series = {ICML'20}
}

@inproceedings{
shen2021is,
title={Is Label Smoothing Truly Incompatible with Knowledge Distillation: An Empirical Study},
author={Zhiqiang Shen and Zechun Liu and Dejia Xu and Zitian Chen and Kwang-Ting Cheng and Marios Savvides},
booktitle={International Conference on Learning Representations},
year={2021},
url={https://openreview.net/forum?id=PObuuGVrGaZ}
}


@misc{RevisitingLabelSmoothing22,
  doi = {10.48550/ARXIV.2206.14532},
  
  url = {https://arxiv.org/abs/2206.14532},
  
  author = {Chandrasegaran, Keshigeyan and Tran, Ngoc-Trung and Zhao, Yunqing and Cheung, Ngai-Man},
  
  keywords = {Machine Learning (cs.LG), FOS: Computer and information sciences, FOS: Computer and information sciences},
  
  title = {Revisiting Label Smoothing and Knowledge Distillation Compatibility: What was Missing?},
  
  publisher = {arXiv},
  
  year = {2022},
  
  copyright = {arXiv.org perpetual, non-exclusive license}
}

@article{muller2019labelsmoothing,
  title={When does label smoothing help?},
  author={M{\"u}ller, Rafael and Kornblith, Simon and Hinton, Geoffrey E},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}
