@inproceedings{brown2020gpt3,
  title={Language models are few-shot learners},
  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  booktitle={NeurIPS},
  year={2020}
}
@article{rae2021gopher,
  title={Scaling language models: Methods, analysis \& insights from training gopher},
  author={Rae, Jack W and Borgeaud, Sebastian and Cai, Trevor and Millican, Katie and Hoffmann, Jordan and Song, Francis and Aslanides, John and Henderson, Sarah and Ring, Roman and Young, Susannah and others},
  journal={arXiv},
  year={2021}
}

@article{J1WhitePaper,
  author = {Lieber, Opher and Sharir, Or and Lenz, Barak and Shoham, Yoav},
  title = {Jurassic-1: Technical Details And Evaluation},
  journal = {AI21 Labs},
  year = {2021}
}

@inproceedings{frantar-spdy,
  title={{SPDY}: Accurate Pruning with Speedup Guarantees}, 
  author={Elias Frantar and Dan Alistarh},
  booktitle={ICML},
  year={2022}
}

@inproceedings{tanaka2020synflow,
 author = {Tanaka, Hidenori and Kunin, Daniel and Yamins, Daniel L and Ganguli, Surya},
 booktitle = {NeurIPS},
 title = {Pruning neural networks without any data by iteratively conserving synaptic flow},
 year = {2020}
}

@article{elsen2019sparse,
  author = {Elsen, Erich and Dukhan, Marat and Gale, Trevor and Simonyan, Karen},
  
  title = {Fast Sparse ConvNets},
  
  journal = {arXiv},
  
  year = {2019}
}


@misc{graphcore_2022, 
 title={Overview - PopSparse Matrix Multiplication (Dynamic Pattern) on the IPU}, 
 url={https://docs.graphcore.ai/projects/dynamic-sparsity/en/latest/dynamic-sparsity.html}, 
 journal={Graphcore Poplar AND PopLibs API Reference}, 
 author={Graphcore}, 
 year={2022}
 } 

@article{Wang2021SparseDNNFS,
  title={SparseDNN: Fast Sparse Deep Learning Inference on CPUs},
  author={Ziheng Wang},
  journal={arXiv},
  year={2021}
}

@inproceedings{Ashby2019ExploitingUS,
  title={Exploiting Unstructured Sparsity on Next-Generation Datacenter Hardware},
  author={Mike Ashby and Christiaan Baaij and Peter Baldwin and Martijn Bastiaan and Oliver Bunting and Aiken Cairncross and Christopher Chalmers and Liz Corrigan and Sam Davis and Nathan van Doorn and Jon Fowler and Graham Hazel and Basile Henry and David Page and Jonny Shipton and Shaun. Steenkamp},
  year={2019}
}

@inproceedings{zhu2018toprune,
  author    = {Michael Zhu and
               Suyog Gupta},
  title     = {To Prune, or Not to Prune: Exploring the Efficacy of Pruning for Model
               Compression},
  booktitle = {ICLR},
  year      = {2018}
}

 @misc{lie_2021, 
 title={Thinking Outside the Die: Architecting the ML Accelerator of the Future}, 
 url={https://www.microarch.org/micro54/media/lie-keynote.pdf}, 
 publisher={MICRO 2021 Keynote}, 
 author={Lie, Sean}, 
 year={2021}, 
 month={Nov}} 

@inproceedings{
ma2022effective,
title={Effective Model Sparsification by Scheduled Grow-and-Prune Methods},
author={Xiaolong Ma and Minghai Qin and Fei Sun and Zejiang Hou and Kun Yuan and Yi Xu and Yanzhi Wang and Yen-Kuang Chen and Rong Jin and Yuan Xie},
booktitle={ICLR},
year={2022}
}

@article{sanh2021multitask,
  title={Multitask prompted training enables zero-shot task generalization},
  author={Sanh, Victor and Webson, Albert and Raffel, Colin and Bach, Stephen H and Sutawika, Lintang and Alyafeai, Zaid and Chaffin, Antoine and Stiegler, Arnaud and Scao, Teven Le and Raja, Arun and others},
  journal={arXiv},
  year={2021}
}

@article{chowdhery2022palm,
  title={Palm: Scaling language modeling with pathways},
  author={Chowdhery, Aakanksha and Narang, Sharan and Devlin, Jacob and Bosma, Maarten and Mishra, Gaurav and Roberts, Adam and Barham, Paul and Chung, Hyung Won and Sutton, Charles and Gehrmann, Sebastian and others},
  journal={arXiv},
  year={2022}
}

@article{smith2022using,
  title={Using deepspeed and megatron to train megatron-turing nlg 530b, a large-scale generative language model},
  author={Smith, Shaden and Patwary, Mostofa and Norick, Brandon and LeGresley, Patrick and Rajbhandari, Samyam and Casper, Jared and Liu, Zhun and Prabhumoye, Shrimai and Zerveas, George and Korthikanti, Vijay and others},
  journal={arXiv},
  year={2022}
}

@inproceedings{Li2021ImprovedRA,
  title={Improved Regularization and Robustness for Fine-tuning in Neural Networks},
  author={Dongyue Li and Hongyang Zhang},
  booktitle={NeurIPS},
  year={2021}
}

@article{Evci2019TheDO,
  title={The Difficulty of Training Sparse Neural Networks},
  author={Utku Evci and Fabian Pedregosa and Aidan N. Gomez and Erich Elsen},
  journal={arXiv},
  year={2019}
}

@inproceedings{jayakumar2020top,
  title={Top-kast: Top-k always sparse training},
  author={Jayakumar, Siddhant and Pascanu, Razvan and Rae, Jack and Osindero, Simon and Elsen, Erich},
  booktitle={NeurIPS},
  year={2020}
}

@inproceedings{
han2017dsd,
title={{DSD}: Dense-Sparse-Dense Training for Deep Neural Networks},
author={Song Han and Jeff Pool and Sharan Narang and Huizi Mao and Enhao Gong and Shijian Tang and Erich Elsen and Peter Vajda and Manohar Paluri and John Tran and Bryan Catanzaro and William J. Dally},
booktitle={ICLR},
year={2017}
}

@inproceedings{
jiang2022exposing,
title={Exposing and Exploiting Fine-Grained Block Structures for Fast and Accurate Sparse Training},
author={Peng Jiang and Lihan Hu and Shihui Song},
booktitle={NeurIPS},
year={2022}
}

@article{Devlin2019BERTPO,
  title={BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding},
  author={Jacob Devlin and Ming-Wei Chang and Kenton Lee and Kristina Toutanova},
  journal={arXiv},
  year={2019}
}

@article{Radford2018ImprovingLU,
  title={Improving Language Understanding by Generative Pre-Training},
  author={Alec Radford and Karthik Narasimhan},
  year={2018},
  journal={OpenAI Blog}
}

@article{radfordGPT2,
  title={Language Models are Unsupervised Multitask Learners},
  author={Alec Radford and Jeffrey Wu and Rewon Child and David Luan and Dario Amodei and Ilya Sutskever},
  year={2019},
  journal={OpenAI Blog}
}

@inproceedings{chen2020lth,
 author = {Chen, Tianlong and Frankle, Jonathan and Chang, Shiyu and Liu, Sijia and Zhang, Yang and Wang, Zhangyang and Carbin, Michael},
 booktitle = {NeurIPS},
 title = {The Lottery Ticket Hypothesis for Pre-trained BERT Networks},
 year = {2020}
}

@article{li2021prefix,
  title={Prefix-tuning: Optimizing continuous prompts for generation},
  author={Li, Xiang Lisa and Liang, Percy},
  journal={arXiv},
  year={2021}
}

@inproceedings{RadiyaDixit2020HowFC,
  title={How fine can fine-tuning be? Learning efficient language models},
  author={Evani Radiya-Dixit and Xin Wang},
  booktitle={AISTATS},
  year={2020}
}

@article{zaken2021bitfit,
  title={Bitfit: Simple parameter-efficient fine-tuning for transformer-based masked language-models},
  author={Zaken, Elad Ben and Ravfogel, Shauli and Goldberg, Yoav},
  journal={arXiv},
  year={2021}
}

@article{gale2019state,
  author = {Gale, Trevor and Elsen, Erich and Hooker, Sara},
  title = {The State of Sparsity in Deep Neural Networks},
  journal = {arXiv},
  year = {2019}
}

@article{Li2021PrefixTuningOC,
  title={Prefix-Tuning: Optimizing Continuous Prompts for Generation},
  author={Xiang Lisa Li and Percy Liang},
  journal={ACL},
  year={2021}
}

@inproceedings{ben-zaken-etal-2022-bitfit,
    title = "{B}it{F}it: Simple Parameter-efficient Fine-tuning for Transformer-based Masked Language-models",
    author = "Ben Zaken, Elad  and
      Goldberg, Yoav  and
      Ravfogel, Shauli",
    booktitle = "ACL",
    year = "2022",
}


@InProceedings{pmlr-v97-houlsby19a,
  title = 	 {Parameter-Efficient Transfer Learning for {NLP}},
  author =       {Houlsby, Neil and Giurgiu, Andrei and Jastrzebski, Stanislaw and Morrone, Bruna and De Laroussilhe, Quentin and Gesmundo, Andrea and Attariyan, Mona and Gelly, Sylvain},
  booktitle = 	 {ICML},
  year = 	 {2019}
}


@article{bengio2003pretrain,
author = {Bengio, Yoshua and Ducharme, R\'{e}jean and Vincent, Pascal and Janvin, Christian},
title = {A Neural Probabilistic Language Model},
year = {2003},
abstract = {A goal of statistical language modeling is to learn the joint probability function of sequences of words in a language. This is intrinsically difficult because of the curse of dimensionality: a word sequence on which the model will be tested is likely to be different from all the word sequences seen during training. Traditional but very successful approaches based on n-grams obtain generalization by concatenating very short overlapping sequences seen in the training set. We propose to fight the curse of dimensionality by learning a distributed representation for words which allows each training sentence to inform the model about an exponential number of semantically neighboring sentences. The model learns simultaneously (1) a distributed representation for each word along with (2) the probability function for word sequences, expressed in terms of these representations. Generalization is obtained because a sequence of words that has never been seen before gets high probability if it is made of words that are similar (in the sense of having a nearby representation) to words forming an already seen sentence. Training such large models (with millions of parameters) within a reasonable time is itself a significant challenge. We report on experiments using neural networks for the probability function, showing on two text corpora that the proposed approach significantly improves on state-of-the-art n-gram models, and that the proposed approach allows to take advantage of longer contexts.},
journal = {JMLR},
}

@inproceedings{vaswani2017transformers,
 author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, \L ukasz and Polosukhin, Illia},
 booktitle = {NeurIPS},
 editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
 title = {Attention is All you Need},
 year = {2017}
}



@article{teven2022bigsci,
Author = {Teven Le Scao and Thomas Wang and Daniel Hesslow and Lucile Saulnier and Stas Bekman and M Saiful Bari and Stella Biderman and Hady Elsahar and Niklas Muennighoff and Jason Phang and Ofir Press and Colin Raffel and Victor Sanh and Sheng Shen and Lintang Sutawika and Jaesung Tae and Zheng Xin Yong and Julien Launay and Iz Beltagy},
Title = {What Language Model to Train if You Have One Million GPU Hours?},
Year = {2022},
journal = {arXiv},
}

@inproceedings{lester-etal-2021-power,
    title = "The Power of Scale for Parameter-Efficient Prompt Tuning",
    author = "Lester, Brian  and
      Al-Rfou, Rami  and
      Constant, Noah",
    booktitle = "EMNLP",
    year = "2021",
    abstract = "In this work, we explore {``}prompt tuning,{''} a simple yet effective mechanism for learning {``}soft prompts{''} to condition frozen language models to perform specific downstream tasks. Unlike the discrete text prompts used by GPT-3, soft prompts are learned through backpropagation and can be tuned to incorporate signals from any number of labeled examples. Our end-to-end learned approach outperforms GPT-3{'}s few-shot learning by a large margin. More remarkably, through ablations on model size using T5, we show that prompt tuning becomes more competitive with scale: as models exceed billions of parameters, our method {``}closes the gap{''} and matches the strong performance of model tuning (where all model weights are tuned). This finding is especially relevant because large models are costly to share and serve and the ability to reuse one frozen model for multiple downstream tasks can ease this burden. Our method can be seen as a simplification of the recently proposed {``}prefix tuning{''} of Li and Liang (2021) and we provide a comparison to this and other similar approaches. Finally, we show that conditioning a frozen model with soft prompts confers benefits in robustness to domain transfer and enables efficient {``}prompt ensembling.{''} We release code and model checkpoints to reproduce our experiments.",
}
@inproceedings{
ouyang2022training,
title={Training language models to follow instructions with human feedback},
author={Long Ouyang and Jeffrey Wu and Xu Jiang and Diogo Almeida and Carroll Wainwright and Pamela Mishkin and Chong Zhang and Sandhini Agarwal and Katarina Slama and Alex Gray and John Schulman and Jacob Hilton and Fraser Kelton and Luke Miller and Maddie Simens and Amanda Askell and Peter Welinder and Paul Christiano and Jan Leike and Ryan Lowe},
booktitle={NeurIPS},
year={2022}
}

@inproceedings{novikona2017e2e,
    title = "The {E}2{E} Dataset: New Challenges For End-to-End Generation",
    author = "Novikova, Jekaterina  and
      Du{\v{s}}ek, Ond{\v{r}}ej  and
      Rieser, Verena",
    booktitle = "{SIG}dial Meeting on Discourse and Dialogue",
    year = "2017",
    abstract = "This paper describes the E2E data, a new dataset for training end-to-end, data-driven natural language generation systems in the restaurant domain, which is ten times bigger than existing, frequently used datasets in this area. The E2E dataset poses new challenges: (1) its human reference texts show more lexical richness and syntactic variation, including discourse phenomena; (2) generating from this set requires content selection. As such, learning from this dataset promises more natural, varied and less template-like system utterances. We also establish a baseline on this dataset, which illustrates some of the difficulties associated with this data.",
}

@inproceedings{kishore2002bleu,
author = {Papineni, Kishore and Roukos, Salim and Ward, Todd and Zhu, Wei-Jing},
title = {BLEU: A Method for Automatic Evaluation of Machine Translation},
year = {2002},
abstract = {Human evaluations of machine translation are extensive but expensive. Human evaluations can take months to finish and involve human labor that can not be reused. We propose a method of automatic machine translation evaluation that is quick, inexpensive, and language-independent, that correlates highly with human evaluation, and that has little marginal cost per run. We present this method as an automated understudy to skilled human judges which substitutes for them when there is need for quick or frequent evaluations.},
booktitle = {ACL}
}

@inproceedings{belz2006comparing,
    title = "Comparing Automatic and Human Evaluation of {NLG} Systems",
    author = "Belz, Anja  and
      Reiter, Ehud",
    booktitle = "ACL",
    year = "2006",
}

@inproceedings{alon2007meteor,
author = {Lavie, Alon and Agarwal, Abhaya},
title = {Meteor: An Automatic Metric for MT Evaluation with High Levels of Correlation with Human Judgments},
year = {2007},
publisher = {ACL},
abstract = {Meteor is an automatic metric for Machine Translation evaluation which has been demonstrated to have high levels of correlation with human judgments of translation quality, significantly outperforming the more commonly used Bleu metric. It is one of several automatic metrics used in this year's shared task within the ACL WMT-07 workshop. This paper recaps the technical details underlying the metric and describes recent improvements in the metric. The latest release includes improved metric parameters and extends the metric to support evaluation of MT output in Spanish, French and German, in addition to English.},
booktitle = {Second Workshop on Statistical Machine Translation},
}

@inproceedings{lin2004rouge,
    title = "{ROUGE}: A Package for Automatic Evaluation of Summaries",
    author = "Lin, Chin-Yew",
    booktitle = "Text Summarization Branches Out",
    year = "2004",
    publisher = "ACL",
}

@inproceedings{ramakrishna2015cider,  
author= "Vedantam, Ramakrishna and Zitnick, C. Lawrence and Parikh, Devi",  booktitle= "CVPR",   
title= "CIDEr: Consensus-based image description evaluation",   
year= "2015"
}

@inproceedings{gardent2017webnlg,
    title = "The {W}eb{NLG} Challenge: Generating Text from {RDF} Data",
    author = "Gardent, Claire  and
      Shimorina, Anastasia  and
      Narayan, Shashi  and
      Perez-Beltrachini, Laura",
    booktitle = "INLG",
    year = "2017",
    abstract = "The WebNLG challenge consists in mapping sets of RDF triples to text. It provides a common benchmark on which to train, evaluate and compare {``}microplanners{''}, i.e. generation systems that verbalise a given content by making a range of complex interacting choices including referring expression generation, aggregation, lexicalisation, surface realisation and sentence segmentation. In this paper, we introduce the microplanning task, describe data preparation, introduce our evaluation methodology, analyse participant results and provide a brief description of the participating systems.",
}

@inproceedings{nan2021dart,
    title = "{DART}: Open-Domain Structured Data Record to Text Generation",
    author = "Nan, Linyong  and
      Radev, Dragomir  and
      Zhang, Rui  and
      Rau, Amrit  and
      Sivaprasad, Abhinand  and
      Hsieh, Chiachun  and
      Tang, Xiangru  and
      Vyas, Aadit  and
      Verma, Neha  and
      Krishna, Pranav  and
      Liu, Yangxiaokang  and
      Irwanto, Nadia  and
      Pan, Jessica  and
      Rahman, Faiaz  and
      Zaidi, Ahmad  and
      Mutuma, Mutethia  and
      Tarabar, Yasin  and
      Gupta, Ankit  and
      Yu, Tao  and
      Tan, Yi Chern  and
      Lin, Xi Victoria  and
      Xiong, Caiming  and
      Socher, Richard  and
      Rajani, Nazneen Fatema",
    booktitle = "ACL",
    year = "2021",
    abstract = "We present DART, an open domain structured DAta Record to Text generation dataset with over 82k instances (DARTs). Data-to-text annotations can be a costly process, especially when dealing with tables which are the major source of structured data and contain nontrivial structures. To this end, we propose a procedure of extracting semantic triples from tables that encodes their structures by exploiting the semantic dependencies among table headers and the table title. Our dataset construction framework effectively merged heterogeneous sources from open domain semantic parsing and spoken dialogue systems by utilizing techniques including tree ontology annotation, question-answer pair to declarative sentence conversion, and predicate unification, all with minimum post-editing. We present systematic evaluation on DART as well as new state-of-the-art results on WebNLG 2017 to show that DART (1) poses new challenges to existing data-to-text datasets and (2) facilitates out-of-domain generalization. Our data and code can be found at https://github.com/Yale-LILY/dart.",
}

@inproceedings{
liu2023sparsity,
title={Sparsity May Cry: Let Us Fail (Current) Sparse Neural Networks Together!},
author={Shiwei Liu and Tianlong Chen and Zhenyu Zhang and Xuxi Chen and Tianjin Huang and Ajay Kumar Jaiswal and Zhangyang Wang},
booktitle={ICLR},
year={2023}
}

@inproceedings{snover2006study,
    title = "A Study of Translation Edit Rate with Targeted Human Annotation",
    author = "Snover, Matthew  and
      Dorr, Bonnie  and
      Schwartz, Rich  and
      Micciulla, Linnea  and
      Makhoul, John",
    booktitle = "Association for Machine Translation",
    year = "2006",
    abstract = "We examine a new, intuitive measure for evaluating machine-translation output that avoids the knowledge intensiveness of more meaning-based approaches, and the labor-intensiveness of human judgments. Translation Edit Rate (TER) measures the amount of editing that a human would have to perform to change a system output so it exactly matches a reference translation. We show that the single-reference variant of TER correlates as well with human judgments of MT quality as the four-reference variant of BLEU. We also define a human-targeted TER (or HTER) and show that it yields higher correlations with human judgments than BLEU{---}even when BLEU is given human-targeted references. Our results indicate that HTER correlates with human judgments better than HMETEOR and that the four-reference variants of TER and HTER correlate with human judgments as well as{---}or better than{---}a second human judgment does.",
}

@inproceedings{
chen2022sparsity,
title={Sparsity Winning Twice: Better Robust Generalization from More Efficient Training},
author={Tianlong Chen and Zhenyu Zhang and pengjun wang and Santosh Balachandra and Haoyu Ma and Zehao Wang and Zhangyang Wang},
booktitle={ICLR},
year={2022}
}

@inproceedings{loshchilov2017decoupled,
  title={Decoupled Weight Decay Regularization},
  author={Ilya Loshchilov and Frank Hutter},
  booktitle={ICLR},
  year={2017}
}

@inproceedings{
lee2018snip,
title={{SNIP}: {Single}-{Shot} {Network} {Pruning} {based} {on} {Connection} {Sensitivity}},
author={Namhoon Lee and Thalaiyasingam Ajanthan and Philip Torr},
booktitle={ICLR},
year={2019}
}

@article{shaoyi2022betterrigl,
  author = {Huang, Shaoyi and Lei, Bowen and Xu, Dongkuan and Peng, Hongwu and Sun, Yue and Xie, Mimi and Ding, Caiwen},
  title = {Dynamic Sparse Training via More Exploration},
  journal = {arXiv},
  year = {2022}
}


@inproceedings{
Wang2020Picking,
title={Picking Winning Tickets Before Training by Preserving Gradient Flow},
author={Chaoqi Wang and Guodong Zhang and Roger Grosse},
booktitle={ICLR},
year={2020}
}

@inproceedings{gale2020sparse,
author = {Gale, Trevor and Zaharia, Matei and Young, Cliff and Elsen, Erich},
title = {Sparse GPU Kernels for Deep Learning},
year = {2020},
abstract = {Scientific workloads have traditionally exploited high levels of sparsity to accelerate computation and reduce memory requirements. While deep neural networks can be made sparse, achieving practical speedups on GPUs is difficult because these applications have relatively moderate levels of sparsity that are not sufficient for existing sparse kernels to outperform their dense counterparts. In this work, we study sparse matrices from deep learning applications and identify favorable properties that can be exploited to accelerate computation. Based on these insights, we develop high-performance GPU kernels for two sparse matrix operations widely applicable in neural networks: sparse matrix-dense matrix multiplication and sampled dense-dense matrix multiplication. Our kernels reach 27% of single-precision peak on Nvidia V100 GPUs. Using our kernels, we demonstrate sparse Transformer and MobileNet models that achieve 1.2--2.1X speedups and up to 12.8X memory savings without sacrificing accuracy.},
booktitle = {SC}
}

@incollection{lamda2022,
title	= {LaMDA: Language Models for Dialog Applications},
author	= {Aaron Daniel Cohen and Adam Roberts and Alejandra Molina and Alena Butryna and Alicia Jin and Apoorv Kulshreshtha and Ben Hutchinson and Ben Zevenbergen and Blaise Hilary Aguera-Arcas and Chung-ching Chang and Claire Cui and Cosmo Du and Daniel De Freitas Adiwardana and Dehao Chen and Dmitry (Dima) Lepikhin and Ed H. Chi and Erin Hoffman-John and others},
year	= {2022},
booktitle	= {arXiv}
}

@inproceedings{
yu2022differentially,
title={Differentially Private Fine-tuning of Language Models},
author={Da Yu and Saurabh Naik and Arturs Backurs and Sivakanth Gopi and Huseyin A Inan and Gautam Kamath and Janardhan Kulkarni and Yin Tat Lee and Andre Manoel and Lukas Wutschitz and Sergey Yekhanin and Huishuai Zhang},
booktitle={ICLR},
year={2022}
}

@inproceedings{irene2021,
 author = {Solaiman, Irene and Dennison, Christy},
 booktitle = {NeurIPS},
 title = {Process for Adapting Language Models to Society (PALMS) with Values-Targeted Datasets},
 year = {2021}
}


 @misc{krashinsky_2020, title={Nvidia ampere architecture in-depth}, url={https://developer.nvidia.com/blog/nvidia-ampere-architecture-in-depth/}, journal={NVIDIA Ampere Architecture In-Depth}, author={Krashinsky, Ronny and Giroux, Olivier and Jones, Stephen and Stam, Nick and Ramaswamy, Sridhar}, year={2020}, month={May}} 


@inproceedings{xin2020sparsetpu,
author = {He, Xin and Pal, Subhankar and Amarnath, Aporva and Feng, Siying and Park, Dong-Hyeon and Rovinski, Austin and Ye, Haojie and Chen, Yuhan and Dreslinski, Ronald and Mudge, Trevor},
title = {Sparse-TPU: Adapting Systolic Arrays for Sparse Matrices},
year = {2020},
abstract = {While systolic arrays are widely used for dense-matrix operations, they are seldom used for sparse-matrix operations. In this paper, we show how a systolic array of Multiply-and-Accumulate (MAC) units, similar to Google's Tensor Processing Unit (TPU), can be adapted to efficiently handle sparse matrices. TPU-like accelerators are built upon a 2D array of MAC units and have demonstrated high throughput and efficiency for dense matrix multiplication, which is a key kernel in machine learning algorithms and is the target of the TPU. In this work, we employ a co-designed approach of first developing a packing technique to condense a sparse matrix and then propose a systolic array based system, Sparse-TPU, abbreviated to STPU, to accommodate the matrix computations for the packed denser matrix counterparts. To demonstrate the efficacy of our co-designed approach, we evaluate sparse matrix-vector multiplication on a broad set of synthetic and real-world sparse matrices. Experimental results show that STPU delivers 16.08X higher performance while consuming 4.39X and 19.79X lower energy for integer (int8) and floating point (float32) implementations, respectively, over a TPU baseline. Meanwhile, STPU has 12.93\% area overhead and an average of 4.14\% increase in dynamic energy over the TPU baseline for the float32 implementation.},
booktitle = {ACM International Conference on Supercomputing}
}

@inproceedings{
liu2021sparsegranet,
title={Sparse Training via Boosting Pruning Plasticity  with Neuroregeneration},
author={Shiwei Liu and Tianlong Chen and Xiaohan Chen and Zahra Atashgahi and Lu Yin and Huanyu Kou and Li Shen and Mykola Pechenizkiy and Zhangyang Wang and Decebal Constantin Mocanu},
booktitle={NeurIPS},
year={2021}
}

@article{Ding2022DeltaTA,
  title={Delta Tuning: A Comprehensive Study of Parameter Efficient Methods for Pre-trained Language Models},
  author={Ning Ding and Yujia Qin and Guang Yang and Fu Wei and Zonghan Yang and Yusheng Su and Shengding Hu and Yulin Chen and Chi-Min Chan and Weize Chen and Jing Yi and Weilin Zhao and Xiaozhi Wang and Zhiyuan Liu and Haitao Zheng and Jianfei Chen and Yang Liu and Jie Tang and Juan Li and Maosong Sun},
  journal={arXiv},
  year={2022}
}

@misc{openai_chatgpt_2022, 
 title={ChatGPT: Optimizing language models for dialogue}, 
 url={https://openai.com/blog/chatgpt/}, 
 journal={OpenAI}, 
 publisher={OpenAI}, 
 author={OpenAI}, 
 year={2022}, 
 month={Nov}} 

@misc{pichai_2023, 
 title={An important next step on our ai journey}, 
 url={https://blog.google/technology/ai/bard-google-ai-search-updates/}, journal={Google}, 
 publisher={Google}, 
 author={Pichai, Sundar}, 
 year={2023}, 
 month={Feb}} 

@inproceedings{
li2018measuring,
title={Measuring the Intrinsic Dimension of Objective Landscapes},
author={Chunyuan Li and Heerad Farkhoor and Rosanne Liu and Jason Yosinski},
booktitle={ICLR},
year={2018}
}

@article{child2019generating,
   title={Generating long sequences with sparse transformers},
   author={Child, Rewon and Gray, Scott and Radford, Alec and Sutskever, Ilya},
   journal={arXiv},
   year={2019}
 }

@misc{neural_magic_2021, 
 title={DeepSparse }, 
 url={https://github.com/neuralmagic/deepsparse}, 
 journal={GitHub}, 
 author={NeuralMagic}, 
 year={2021}} 

@inproceedings{hoefler2022sparsity,
author = {Hoefler, Torsten and Alistarh, Dan and Ben-Nun, Tal and Dryden, Nikoli and Peste, Alexandra},
title = {Sparsity in Deep Learning: Pruning and Growth for Efficient Inference and Training in Neural Networks},
year = {2022},
abstract = {The growing energy and performance costs of deep learning have driven the community to reduce the size of neural networks by selectively pruning components. Similarly to their biological counterparts, sparse networks generalize just as well, sometimes even better than, the original dense networks. Sparsity promises to reduce the memory footprint of regular networks to fit mobile devices, as well as shorten training time for ever growing networks. In this paper, we survey prior work on sparsity in deep learning and provide an extensive tutorial of sparsification for both inference and training. We describe approaches to remove and add elements of neural networks, different training strategies to achieve model sparsity, and mechanisms to exploit sparsity in practice. Our work distills ideas from more than 300 research papers and provides guidance to practitioners who wish to utilize sparsity today, as well as to researchers whose goal is to push the frontier forward. We include the necessary background on mathematical methods in sparsification, describe phenomena such as early structure adaptation, the intricate relations between sparsity and the training process, and show techniques for achieving acceleration on real hardware. We also define a metric of pruned parameter efficiency that could serve as a baseline for comparison of different sparse networks. We close by speculating on how sparsity can improve future workloads and outline major open problems in the field.},
booktitle = {JMLR},
}

@inproceedings{evci2020rigl,
author = {Evci, Utku and Gale, Trevor and Menick, Jacob and Castro, Pablo Samuel and Elsen, Erich},
title = {Rigging the Lottery: Making All Tickets Winners},
year = {2020},
abstract = {Many applications require sparse neural networks due to space or inference time restrictions. There is a large body of work on training dense networks to yield sparse networks for inference, but this limits the size of the largest trainable sparse model to that of the largest trainable dense model. In this paper we introduce a method to train sparse neural networks with a fixed parameter count and a fixed computational cost throughout training, without sacrificing accuracy relative to existing dense-to-sparse training methods. Our method updates the topology of the sparse network during training by using parameter magnitudes and infrequent gradient calculations. We show that this approach requires fewer floating-point operations (FLOPs) to achieve a given level of accuracy compared to prior techniques. We demonstrate state-of-the-art sparse training results on a variety of networks and datasets, including ResNet-50, MobileNets on Imagenet-2012, and RNNs on WikiText-103. Finally, we provide some insights into why allowing the topology to change during the optimization can overcome local minima encountered when the topology remains static*.},
booktitle = {ICML},
}

@article{mocanu2018,
title = "Scalable training of artificial neural networks with adaptive sparse connectivity inspired by network science",
author = "Mocanu, {Decebal Constantin} and Elena Mocanu and Peter Stone and Nguyen, {Phuong H.} and Madeleine Gibescu and Antonio Liotta",
year = "2018",
journal = "Nature Communications",
}

@article{liu2019sparse,
title = "Sparse evolutionary deep learning with over one million artificial neurons on commodity hardware",
abstract = "Artificial neural networks (ANNs) have emerged as hot topics in the research community. Despite the success of ANNs, it is challenging to train and deploy modern ANNs on commodity hardware due to the ever-increasing model size and the unprecedented growth in the data volumes. Particularly for microarray data, the very high dimensionality and the small number of samples make it difficult for machine learning techniques to handle. Furthermore, specialized hardware such as graphics processing unit (GPU) is expensive. Sparse neural networks are the leading approaches to address these challenges. However, off-the-shelf sparsity-inducing techniques either operate from a pretrained model or enforce the sparse structure via binary masks. The training efficiency of sparse neural networks cannot be obtained practically. In this paper, we introduce a technique allowing us to train truly sparse neural networks with fixed parameter count throughout training. Our experimental results demonstrate that our method can be applied directly to handle high-dimensional data, while achieving higher accuracy than the traditional two-phase approaches. Moreover, we have been able to create truly sparse multilayer perceptron models with over one million neurons and to train them on a typical laptop without GPU (https://github.com/dcmocanu/sparse-evolutionary-artificial-neural-networks/tree/master/SET-MLP-Sparse-Python-Data-Structures), this being way beyond what is possible with any state-of-the-art technique.",
author = "Shiwei Liu and Mocanu, {Decebal Constantin} and Matavalam, {Amarsagar Reddy Ramapuram} and Yulong Pei and Mykola Pechenizkiy",
journal = "Neural Computing and Applications",
yeah = "2019"
}

@inproceedings{
hoffmann2022an,
title={An empirical analysis of compute-optimal large language model training},
author={Jordan Hoffmann and Sebastian Borgeaud and Arthur Mensch and Elena Buchatskaya and Trevor Cai and Eliza Rutherford and Diego de las Casas and Lisa Anne Hendricks and Johannes Welbl and Aidan Clark and Tom Hennigan and Eric Noland and Katherine Millican and George van den Driessche and Bogdan Damoc and Aurelia Guy and Simon Osindero and Karen Simonyan and Erich Elsen and Oriol Vinyals and Jack William Rae and Laurent Sifre},
booktitle={NeurIPS},
year={2022}
}

@inproceedings{merity2017wiki,
  author    = {Stephen Merity and
               Caiming Xiong and
               James Bradbury and
               Richard Socher},
  title     = {Pointer Sentinel Mixture Models},
  booktitle = {ICLR},
  year      = {2017}
}

@misc{curationcorpusbase:2020,
  title={Curation Corpus Base},
  author={Curation},
  year={2020}
}

@inproceedings{alt-etal-2019-fine,
    title = "Fine-tuning Pre-Trained Transformer Language Models to Distantly Supervised Relation Extraction",
    author = {Alt, Christoph  and
      H{\"u}bner, Marc  and
      Hennig, Leonhard},
    booktitle = "ACL",
    year = "2019",
    abstract = "Distantly supervised relation extraction is widely used to extract relational facts from text, but suffers from noisy labels. Current relation extraction methods try to alleviate the noise by multi-instance learning and by providing supporting linguistic and contextual information to more efficiently guide the relation classification. While achieving state-of-the-art results, we observed these models to be biased towards recognizing a limited set of relations with high precision, while ignoring those in the long tail. To address this gap, we utilize a pre-trained language model, the OpenAI Generative Pre-trained Transformer (GPT) (Radford et al., 2018). The GPT and similar models have been shown to capture semantic and syntactic features, and also a notable amount of {``}common-sense{''} knowledge, which we hypothesize are important features for recognizing a more diverse set of relations. By extending the GPT to the distantly supervised setting, and fine-tuning it on the NYT10 dataset, we show that it predicts a larger set of distinct relation types with high confidence. Manual and automated evaluation of our model shows that it achieves a state-of-the-art AUC score of 0.422 on the NYT10 dataset, and performs especially well at higher recall levels.",
}


@article{
wei2022emergent,
title={Emergent Abilities of Large Language Models},
author={Jason Wei and Yi Tay and Rishi Bommasani and Colin Raffel and Barret Zoph and Sebastian Borgeaud and Dani Yogatama and Maarten Bosma and Denny Zhou and Donald Metzler and Ed H. Chi and Tatsunori Hashimoto and Oriol Vinyals and Percy Liang and Jeff Dean and William Fedus},
journal={TMLR},
year={2022},
note={Survey Certification}
}

@inproceedings{marfurt2021sentence,
    title = "Sentence-level Planning for Especially Abstractive Summarization",
    author = "Marfurt, Andreas  and
      Henderson, James",
    booktitle = "Third Workshop on New Frontiers in Summarization",
    year = "2021",
    abstract = "Abstractive summarization models heavily rely on copy mechanisms, such as the pointer network or attention, to achieve good performance, measured by textual overlap with reference summaries. As a result, the generated summaries stay close to the formulations in the source document. We propose the *sentence planner* model to generate more abstractive summaries. It includes a hierarchical decoder that first generates a representation for the next summary sentence, and then conditions the word generator on this representation. Our generated summaries are more abstractive and at the same time achieve high ROUGE scores when compared to human reference summaries. We verify the effectiveness of our design decisions with extensive evaluations.",
}

@inproceedings{houlsby2019parameter,
  title={Parameter-efficient transfer learning for NLP},
  author={Houlsby, Neil and Giurgiu, Andrei and Jastrzebski, Stanislaw and Morrone, Bruna and De Laroussilhe, Quentin and Gesmundo, Andrea and Attariyan, Mona and Gelly, Sylvain},
  booktitle={ICML},
  year={2019}
}

@inproceedings{
hu2022lora,
title={Lo{RA}: Low-Rank Adaptation of Large Language Models},
author={Edward J Hu and yelong shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen},
booktitle={ICLR},
year={2022}
}

@article{zhong2018seqsql,

  author = {Zhong, Victor and Xiong, Caiming and Socher, Richard},
  title = {Seq2SQL: Generating Structured Queries from Natural Language using Reinforcement Learning},
  journal={arXiv},
  year = {2017},
  
  copyright = {arXiv.org perpetual, non-exclusive license}
}


@inproceedings{pasupat2015compositional,
    title = "Compositional Semantic Parsing on Semi-Structured Tables",
    author = "Pasupat, Panupong  and
      Liang, Percy",
    booktitle = "ACL and IJCNN",
    year = "2015"
}

@article{Kornblith2019DoBI,
  title={Do Better ImageNet Models Transfer Better?},
  author={Simon Kornblith and Jonathon Shlens and Quoc V. Le},
  journal={2019 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year={2019}
}

@inproceedings{raffel2022transfer,
author = {Raffel, Colin and Shazeer, Noam and Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and Li, Wei and Liu, Peter J.},
title = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer},
year = {2022},
abstract = {Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts all text-based language problems into a text-to-text format. Our systematic study compares pretraining objectives, architectures, unlabeled data sets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new "Colossal Clean Crawled Corpus", we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our data set, pre-trained models, and code.},
booktitle = {JMLR},
}

@inproceedings{schick2021smallllm,
    title = "It{'}s Not Just Size That Matters: Small Language Models Are Also Few-Shot Learners",
    author = {Schick, Timo  and
      Sch{\"u}tze, Hinrich},
    booktitle = "ACL",
    year = "2021",
    abstract = "When scaled to hundreds of billions of parameters, pretrained language models such as GPT-3 (Brown et al., 2020) achieve remarkable few-shot performance. However, enormous amounts of compute are required for training and applying such big models, resulting in a large carbon footprint and making it difficult for researchers and practitioners to use them. We show that performance similar to GPT-3 can be obtained with language models that are much {``}greener{''} in that their parameter count is several orders of magnitude smaller. This is achieved by converting textual inputs into cloze questions that contain a task description, combined with gradient-based optimization; exploiting unlabeled data gives further improvements. We identify key factors required for successful natural language understanding with small language models.",
}

@inproceedings{dao2022monarch,
  title={Monarch: Expressive structured matrices for efficient and accurate training},
  author={Dao, Tri and Chen, Beidi and Sohoni, Nimit S and Desai, Arjun and Poli, Michael and Grogan, Jessica and Liu, Alexander and Rao, Aniruddh and Rudra, Atri and R{\'e}, Christopher},
  booktitle={ICML},
  year={2022}
}


@article{gao2020pile,
  title={The pile: An 800gb dataset of diverse text for language modeling},
  author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and others},
  journal={arXiv},
  year={2020}
}

@article{li_2022, 
 title={OpenAI's GPT-3 Language model: A technical overview}, 
 url={https://lambdalabs.com/blog/demystifying-gpt-3}, 
 journal={GPU Cloud, Workstations, Servers, Laptops for Deep Learning}, publisher={Lambda, Inc.}, 
 author={Li, Chuan}, 
 year={2022}, 
 month={Aug}
 } 


@inproceedings{
chen2022pixelated,
title={Pixelated Butterfly: Simple and Efficient Sparse training for Neural Network Models},
author={Beidi Chen and Tri Dao and Kaizhao Liang and Jiaming Yang and Zhao Song and Atri Rudra and Christopher Re},
booktitle={ICLR},
year={2022}
}

@inproceedings{
zhou2021learning,
title={Learning N:M  Fine-grained Structured Sparse Neural Networks From Scratch},
author={Aojun Zhou and Yukun Ma and Junnan Zhu and Jianbo Liu and Zhijie Zhang and Kun Yuan and Wenxiu Sun and Hongsheng Li},
booktitle={ICLR},
year={2021}
}

@article{Gholami2021ASO,
  title={A Survey of Quantization Methods for Efficient Neural Network Inference},
  author={Amir Gholami and Sehoon Kim and Zhen Dong and Zhewei Yao and Michael W. Mahoney and Kurt Keutzer},
  journal={ArXiv},
  year={2021}
}

@inproceedings{
zhou2017incremental,
title={Incremental Network Quantization: Towards Lossless {CNN}s with Low-precision Weights},
author={Aojun Zhou and Anbang Yao and Yiwen Guo and Lin Xu and Yurong Chen},
booktitle={ICLR},
year={2017}
}

@inproceedings{wang2019cvpr,
  author    = {Kuan Wang and
               Zhijian Liu and
               Yujun Lin and
               Ji Lin and
               Song Han},
  title     = {{HAQ:} Hardware-Aware Automated Quantization With Mixed Precision},
  booktitle = {CVPR},
  year      = {2019}
}

@inproceedings{Shen2019QBERTHB,
  title={Q-BERT: Hessian Based Ultra Low Precision Quantization of BERT},
  author={Sheng Shen and Zhen Dong and Jiayu Ye and Linjian Ma and Zhewei Yao and Amir Gholami and Michael W. Mahoney and Kurt Keutzer},
  booktitle={AAAI},
  year={2019}
}

@inproceedings{dao2022flashattention,
title={FlashAttention: Fast and Memory-Efficient Exact Attention with {IO}-Awareness},
author={Tri Dao and Daniel Y Fu and Stefano Ermon and Atri Rudra and Christopher Re},
booktitle={NeurIPS},
year={2022}
}

@inproceedings{jaszczur2021sparse,
title={Sparse is Enough in Scaling Transformers},
author={Sebastian Jaszczur and Aakanksha Chowdhery and Afroz Mohiuddin and Lukasz Kaiser and Wojciech Gajewski and Henryk Michalewski and Jonni Kanerva},
booktitle={NeurIPS},
year={2021}
}

@inproceedings{
li2022the,
title={The Stability-Efficiency Dilemma: Investigating Sequence Length Warmup for Training {GPT} Models},
author={Conglong Li and Minjia Zhang and Yuxiong He},
booktitle={NeurIPS},
year={2022}
}

@inproceedings{frankle2018lottery,
  title={The lottery ticket hypothesis: Finding sparse, trainable neural networks},
  author={Frankle, Jonathan and Carbin, Michael},
  booktitle={ICLR},
  year={2018}
}

@article{tang20221bitadam,
  
  author = {Tang, Hanlin and Gan, Shaoduo and Awan, Ammar Ahmad and Rajbhandari, Samyam and Li, Conglong and Lian, Xiangru and Liu, Ji and Zhang, Ce and He, Yuxiong},
  
  title = {1-bit Adam: Communication Efficient Large-Scale Training with Adam's Convergence Speed},
  
  journal = {arXiv},
  
  year = {2021}
}

@inproceedings{aghajanyan-etal-2021-intrinsic,
    title = "Intrinsic Dimensionality Explains the Effectiveness of Language Model Fine-Tuning",
    author = "Aghajanyan, Armen  and
      Gupta, Sonal  and
      Zettlemoyer, Luke",
    booktitle = "ACL",
    year = "2021",
    abstract = "Although pretrained language models can be fine-tuned to produce state-of-the-art results for a very wide range of language understanding tasks, the dynamics of this process are not well understood, especially in the low data regime. Why can we use relatively vanilla gradient descent algorithms (e.g., without strong regularization) to tune a model with hundreds of millions of parameters on datasets with only hundreds or thousands of labeled examples? In this paper, we argue that analyzing fine-tuning through the lens of intrinsic dimension provides us with empirical and theoretical intuitions to explain this remarkable phenomenon. We empirically show that common pre-trained models have a very low intrinsic dimension; in other words, there exists a low dimension reparameterization that is as effective for fine-tuning as the full parameter space. For example, by optimizing only 200 trainable parameters randomly projected back into the full space, we can tune a RoBERTa model to achieve 90{\%} of the full parameter performance levels on MRPC. Furthermore, we empirically show that pre-training implicitly minimizes intrinsic dimension and, perhaps surprisingly, larger models tend to have lower intrinsic dimension after a fixed number of pre-training updates, at least in part explaining their extreme effectiveness. Lastly, we connect intrinsic dimensionality with low dimensional task representations and compression based generalization bounds to provide intrinsic-dimension-based generalization bounds that are independent of the full parameter count.",
}

@inproceedings{
neyshabur2018the,
title={The role of over-parametrization in generalization of neural networks},
author={Behnam Neyshabur and Zhiyuan Li and Srinadh Bhojanapalli and Yann LeCun and Nathan Srebro},
booktitle={ICLR},
year={2019}
}


@InProceedings{pmlr-v97-allen-zhu19a,
  title = 	 {A Convergence Theory for Deep Learning via Over-Parameterization},
  author =       {Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  booktitle = 	 {ICML},
  year = 	 {2019},
  abstract = 	 {Deep neural networks (DNNs) have demonstrated dominating performance in many fields; since AlexNet, networks used in practice are going wider and deeper. On the theoretical side, a long line of works have been focusing on why we can train neural networks when there is only one hidden layer. The theory of multi-layer networks remains unsettled. In this work, we prove simple algorithms such as stochastic gradient descent (SGD) can find Global Minima on the training objective of DNNs in Polynomial Time. We only make two assumptions: the inputs do not degenerate and the network is over-parameterized. The latter means the number of hidden neurons is sufficiently large: polynomial in L, the number of DNN layers and in n, the number of training samples. As concrete examples, starting from randomly initialized weights, we show that SGD attains 100% training accuracy in classification tasks, or minimizes regression loss in linear convergence speed eps &nbsp; e^{-T}, with running time polynomial in n and L. Our theory applies to the widely-used but non-smooth ReLU activation, and to any smooth and possibly non-convex loss functions. In terms of network architectures, our theory at least applies to fully-connected neural networks, convolutional neural networks (CNN), and residual neural networks (ResNet).}
}

@inproceedings{
liu2022the,
title={The Unreasonable Effectiveness of Random Pruning: Return of the Most Naive Baseline for Sparse Training},
author={Shiwei Liu and Tianlong Chen and Xiaohan Chen and Li Shen and Decebal Constantin Mocanu and Zhangyang Wang and Mykola Pechenizkiy},
booktitle={ICLR},
year={2022}
}

@article{Soltanolkotabi2019,
  author={Soltanolkotabi, Mahdi and Javanmard, Adel and Lee, Jason D.},
  journal={IEEE Transactions on Information Theory}, 
  title={Theoretical Insights Into the Optimization Landscape of Over-Parameterized Shallow Neural Networks}, 
  year={2019}
  }

@article{sara2020lottery,

  author = {Hooker, Sara},
  
  title = {The Hardware Lottery},
  
  journal = {arXiv},
  
  year = {2020}
}

@article{ba2016ln,

  author = {Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E.},
  
  title = {Layer Normalization},
  
  journal = {arXiv},
  
  year = {2016}
}


@article{ren2022ns,
  author    = {Pengzhen Ren and
               Yun Xiao and
               Xiaojun Chang and
               Poyao Huang and
               Zhihui Li and
               Xiaojiang Chen and
               Xin Wang},
  title     = {A Comprehensive Survey of Neural Architecture Search: Challenges and
               Solutions},
  journal   = {{ACM} Comput. Surv.},
  year      = {2022},
}

@article{lie_2022, 
title={Hot Chips 34. Cerebras Architecture Deep Dive: First Look Inside the Hardware/Software Co-Design for Deep Learning}, 
journal={IEEE Micro}, 
author={Lie, Sean}, 
year={2023}
}

@article{tan2018mnasnet,
  author    = {Mingxing Tan and
               Bo Chen and
               Ruoming Pang and
               Vijay Vasudevan and
               Quoc V. Le},
  title     = {MnasNet: Platform-Aware Neural Architecture Search for Mobile},
  journal   = {arXiv},
  year      = {2018}
}

@inproceedings{tao-etal-2022-compression,
    title = "Compression of Generative Pre-trained Language Models via Quantization",
    author = "Tao, Chaofan  and
      Hou, Lu  and
      Zhang, Wei  and
      Shang, Lifeng  and
      Jiang, Xin  and
      Liu, Qun  and
      Luo, Ping  and
      Wong, Ngai",
    booktitle = "ACL",
    year = "2022",
    abstract = "The increasing size of generative Pre-trained Language Models (PLMs) have greatly increased the demand for model compression. Despite various methods to compress BERT or its variants, there are few attempts to compress generative PLMs, and the underlying difficulty remains unclear. In this paper, we compress generative PLMs by quantization. We find that previous quantization methods fail on generative tasks due to the homogeneous word embeddings caused by reduced capacity and the varied distribution of weights. Correspondingly, we propose a token-level contrastive distillation to learn distinguishable word embeddings, and a module-wise dynamic scaling to make quantizers adaptive to different modules. Empirical results on various tasks show that our proposed method outperforms the state-of-the-art compression methods on generative PLMs by a clear margin. With comparable performance with the full-precision models, we achieve 14.4x and 13.4x compression rate on GPT-2 and BART, respectively.",
}