@article{radford2018improving,
  title={Improving language understanding by generative pre-training},
  author={Radford, Alec and Narasimhan, Karthik and Salimans, Tim and Sutskever, Ilya and others},
  year={2018},
  publisher={OpenAI}
}

@article{radford2019language,
  title={Language models are unsupervised multitask learners},
  author={Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya and others},
  journal={OpenAI blog},
  volume={1},
  number={8},
  pages={9},
  year={2019}
}

@article{devlin2018bert,
  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={arXiv preprint arXiv:1810.04805},
  year={2018}
}

@article{ren2015faster,
  title={Faster r-cnn: Towards real-time object detection with region proposal networks},
  author={Ren, Shaoqing and He, Kaiming and Girshick, Ross and Sun, Jian},
  journal={Advances in neural information processing systems},
  volume={28},
  year={2015}
}

@article{simonyan2014very,
  title={Very deep convolutional networks for large-scale image recognition},
  author={Simonyan, Karen and Zisserman, Andrew},
  journal={arXiv preprint arXiv:1409.1556},
  year={2014}
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@article{denton2014exploiting,
  title={Exploiting linear structure within convolutional networks for efficient evaluation},
  author={Denton, Emily L and Zaremba, Wojciech and Bruna, Joan and LeCun, Yann and Fergus, Rob},
  journal={Advances in neural information processing systems},
  volume={27},
  year={2014}
}

@article{lebedev2014speeding,
  title={Speeding-up convolutional neural networks using fine-tuned cp-decomposition},
  author={Lebedev, Vadim and Ganin, Yaroslav and Rakhuba, Maksim and Oseledets, Ivan and Lempitsky, Victor},
  journal={arXiv preprint arXiv:1412.6553},
  year={2014}
}

@article{courbariaux2016binarized,
  title={Binarized neural networks: Training deep neural networks with weights and activations constrained to+ 1 or-1},
  author={Courbariaux, Matthieu and Hubara, Itay and Soudry, Daniel and El-Yaniv, Ran and Bengio, Yoshua},
  journal={arXiv preprint arXiv:1602.02830},
  year={2016}
}

@inproceedings{rastegari2016xnor,
  title={Xnor-net: Imagenet classification using binary convolutional neural networks},
  author={Rastegari, Mohammad and Ordonez, Vicente and Redmon, Joseph and Farhadi, Ali},
  booktitle={European conference on computer vision},
  pages={525--542},
  year={2016},
  organization={Springer}
}

@article{hinton2015distilling,
  title={Distilling the knowledge in a neural network},
  author={Hinton, Geoffrey and Vinyals, Oriol and Dean, Jeff},
  journal={arXiv preprint arXiv:1503.02531},
  year={2015}
}

@article{romero2014fitnets,
  title={Fitnets: Hints for thin deep nets},
  author={Romero, Adriana and Ballas, Nicolas and Kahou, Samira Ebrahimi and Chassang, Antoine and Gatta, Carlo and Bengio, Yoshua},
  journal={arXiv preprint arXiv:1412.6550},
  year={2014}
}

@article{lecun1989optimal,
  title={Optimal brain damage},
  author={LeCun, Yann and Denker, John and Solla, Sara},
  journal={Advances in neural information processing systems},
  volume={2},
  year={1989}
}

@article{hassibi1992second,
  title={Second order derivatives for network pruning: Optimal brain surgeon},
  author={Hassibi, Babak and Stork, David},
  journal={Advances in neural information processing systems},
  volume={5},
  year={1992}
}

@article{han2015learning,
  title={Learning both weights and connections for efficient neural network},
  author={Han, Song and Pool, Jeff and Tran, John and Dally, William},
  journal={Advances in neural information processing systems},
  volume={28},
  year={2015}
}

@article{molchanov2016pruning,
  title={Pruning convolutional neural networks for resource efficient inference},
  author={Molchanov, Pavlo and Tyree, Stephen and Karras, Tero and Aila, Timo and Kautz, Jan},
  journal={arXiv preprint arXiv:1611.06440},
  year={2016}
}

@article{li2016pruning,
  title={Pruning filters for efficient convnets},
  author={Li, Hao and Kadav, Asim and Durdanovic, Igor and Samet, Hanan and Graf, Hans Peter},
  journal={arXiv preprint arXiv:1608.08710},
  year={2016}
}

@article{frankle2018lottery,
  title={The lottery ticket hypothesis: Finding sparse, trainable neural networks},
  author={Frankle, Jonathan and Carbin, Michael},
  journal={arXiv preprint arXiv:1803.03635},
  year={2018}
}

@article{han2016eie,
  title={EIE: Efficient inference engine on compressed deep neural network},
  author={Han, Song and Liu, Xingyu and Mao, Huizi and Pu, Jing and Pedram, Ardavan and Horowitz, Mark A and Dally, William J},
  journal={ACM SIGARCH Computer Architecture News},
  volume={44},
  number={3},
  pages={243--254},
  year={2016},
  publisher={ACM New York, NY, USA}
}

@InProceedings{He_2019_CVPR,
author = {He, Yang and Liu, Ping and Wang, Ziwei and Hu, Zhilan and Yang, Yi},
title = {Filter Pruning via Geometric Median for Deep Convolutional Neural Networks Acceleration},
booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
month = {June},
year = {2019}
}

@article{zhu2023survey,
  title={A Survey on Model Compression for Large Language Models},
  author={Zhu, Xunyu and Li, Jian and Liu, Yong and Ma, Can and Wang, Weiping},
  journal={arXiv preprint arXiv:2308.07633},
  year={2023}
}

@misc{he2023structured,
      title={Structured Pruning for Deep Convolutional Neural Networks: A survey}, 
      author={Yang He and Lingao Xiao},
      year={2023},
      eprint={2303.00566},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

@inproceedings{lin2020hrank,
  title={Hrank: Filter pruning using high-rank feature map},
  author={Lin, Mingbao and Ji, Rongrong and Wang, Yan and Zhang, Yichen and Zhang, Baochang and Tian, Yonghong and Shao, Ling},
  booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
  pages={1529--1538},
  year={2020}
}

@article{sui2021chip,
  title={Chip: Channel independence-based pruning for compact neural networks},
  author={Sui, Yang and Yin, Miao and Xie, Yi and Phan, Huy and Aliari Zonouz, Saman and Yuan, Bo},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={24604--24616},
  year={2021}
}

@inproceedings{luo2017thinet,
  title={Thinet: A filter level pruning method for deep neural network compression},
  author={Luo, Jian-Hao and Wu, Jianxin and Lin, Weiyao},
  booktitle={Proceedings of the IEEE international conference on computer vision},
  pages={5058--5066},
  year={2017}
}

@inproceedings{yu2018nisp,
  title={Nisp: Pruning networks using neuron importance score propagation},
  author={Yu, Ruichi and Li, Ang and Chen, Chun-Fu and Lai, Jui-Hsin and Morariu, Vlad I and Han, Xintong and Gao, Mingfei and Lin, Ching-Yung and Davis, Larry S},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={9194--9203},
  year={2018}
}

@inproceedings{luo2020neural,
  title={Neural network pruning with residual-connections and limited-data},
  author={Luo, Jian-Hao and Wu, Jianxin},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={1458--1467},
  year={2020}
}

@inproceedings{liu2017learning,
  title={Learning efficient convolutional networks through network slimming},
  author={Liu, Zhuang and Li, Jianguo and Shen, Zhiqiang and Huang, Gao and Yan, Shoumeng and Zhang, Changshui},
  booktitle={Proceedings of the IEEE international conference on computer vision},
  pages={2736--2744},
  year={2017}
}

@inproceedings{molchanov2019importance,
  title={Importance estimation for neural network pruning},
  author={Molchanov, Pavlo and Mallya, Arun and Tyree, Stephen and Frosio, Iuri and Kautz, Jan},
  booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
  pages={11264--11272},
  year={2019}
}

@inproceedings{li2022revisiting,
  title={Revisiting random channel pruning for neural network compression},
  author={Li, Yawei and Adamczewski, Kamil and Li, Wen and Gu, Shuhang and Timofte, Radu and Van Gool, Luc},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={191--201},
  year={2022}
}

@article{liu2018rethinking,
  title={Rethinking the value of network pruning},
  author={Liu, Zhuang and Sun, Mingjie and Zhou, Tinghui and Huang, Gao and Darrell, Trevor},
  journal={arXiv preprint arXiv:1810.05270},
  year={2018}
}

@inproceedings{wang2020pruning,
  title={Pruning from scratch},
  author={Wang, Yulong and Zhang, Xiaolu and Xie, Lingxi and Zhou, Jun and Su, Hang and Zhang, Bo and Hu, Xiaolin},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={07},
  pages={12273--12280},
  year={2020}
}

@article{dherin2022neural,
  title={Why neural networks find simple solutions: The many regularizers of geometric complexity},
  author={Dherin, Benoit and Munn, Michael and Rosca, Mihaela and Barrett, David},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={2333--2349},
  year={2022}
}

@inproceedings{NEURIPS2020_6f5216f8,
 author = {Hou, Lu and Huang, Zhiqi and Shang, Lifeng and Jiang, Xin and Chen, Xiao and Liu, Qun},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M.F. Balcan and H. Lin},
 pages = {9782--9793},
 publisher = {Curran Associates, Inc.},
 title = {DynaBERT: Dynamic BERT with Adaptive Width and Depth},
 url = {https://proceedings.neurips.cc/paper_files/paper/2020/file/6f5216f8d89b086c18298e043bfe48ed-Paper.pdf},
 volume = {33},
 year = {2020}
}

@article{fan2019reducing,
  title={Reducing transformer depth on demand with structured dropout},
  author={Fan, Angela and Grave, Edouard and Joulin, Armand},
  journal={arXiv preprint arXiv:1909.11556},
  year={2019}
}

@article{voita2019analyzing,
  title={Analyzing multi-head self-attention: Specialized heads do the heavy lifting, the rest can be pruned},
  author={Voita, Elena and Talbot, David and Moiseev, Fedor and Sennrich, Rico and Titov, Ivan},
  journal={arXiv preprint arXiv:1905.09418},
  year={2019}
}

@article{mccarley2019structured,
  title={Structured pruning of a bert-based question answering model},
  author={McCarley, JS and Chakravarti, Rishav and Sil, Avirup},
  journal={arXiv preprint arXiv:1910.06360},
  year={2019}
}

@article{xia2022structured,
  title={Structured pruning learns compact and accurate models},
  author={Xia, Mengzhou and Zhong, Zexuan and Chen, Danqi},
  journal={arXiv preprint arXiv:2204.00408},
  year={2022}
}

@article{ma2023llm,
  title={LLM-Pruner: On the Structural Pruning of Large Language Models},
  author={Ma, Xinyin and Fang, Gongfan and Wang, Xinchao},
  journal={arXiv preprint arXiv:2305.11627},
  year={2023}
}

@article{krizhevsky2009learning,
  title={Learning multiple layers of features from tiny images},
  author={Krizhevsky, Alex and Hinton, Geoffrey and others},
  year={2009},
  publisher={Toronto, ON, Canada}
}

@article{le2015tiny,
  title={Tiny imagenet visual recognition challenge},
  author={Le, Ya and Yang, Xuan},
  journal={CS 231N},
  volume={7},
  number={7},
  pages={3},
  year={2015}
}

@InProceedings{maas-EtAl:2011:ACL-HLT2011,
  author    = {Maas, Andrew L.  and  Daly, Raymond E.  and  Pham, Peter T.  and  Huang, Dan  and  Ng, Andrew Y.  and  Potts, Christopher},
  title     = {Learning Word Vectors for Sentiment Analysis},
  booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies},
  month     = {June},
  year      = {2011},
  address   = {Portland, Oregon, USA},
  publisher = {Association for Computational Linguistics},
  pages     = {142--150},
  url       = {http://www.aclweb.org/anthology/P11-1015}
}

@article{yoshida2017spectral,
  title={Spectral norm regularization for improving the generalizability of deep learning},
  author={Yoshida, Yuichi and Miyato, Takeru},
  journal={arXiv preprint arXiv:1705.10941},
  year={2017}
}

@article{paszke2019pytorch,
  title={Pytorch: An imperative style, high-performance deep learning library},
  author={Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and others},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@article{loshchilov2016sgdr,
  title={Sgdr: Stochastic gradient descent with warm restarts},
  author={Loshchilov, Ilya and Hutter, Frank},
  journal={arXiv preprint arXiv:1608.03983},
  year={2016}
}

@inproceedings{fang2023depgraph,
  title={Depgraph: Towards any structural pruning},
  author={Fang, Gongfan and Ma, Xinyin and Song, Mingli and Mi, Michael Bi and Wang, Xinchao},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={16091--16101},
  year={2023}
}

@article{chen2023otov2,
  title={Otov2: Automatic, generic, user-friendly},
  author={Chen, Tianyi and Liang, Luming and Ding, Tianyu and Zhu, Zhihui and Zharkov, Ilya},
  journal={arXiv preprint arXiv:2303.06862},
  year={2023}
}

@inproceedings{glorot2010understanding,
  title={Understanding the difficulty of training deep feedforward neural networks},
  author={Glorot, Xavier and Bengio, Yoshua},
  booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
  pages={249--256},
  year={2010},
  organization={JMLR Workshop and Conference Proceedings}
}

@inproceedings{he2015delving,
  title={Delving deep into rectifiers: Surpassing human-level performance on imagenet classification},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE international conference on computer vision},
  pages={1026--1034},
  year={2015}
}

@inproceedings{NEURIPS2020_46a4378f,
 author = {Tanaka, Hidenori and Kunin, Daniel and Yamins, Daniel L and Ganguli, Surya},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M.F. Balcan and H. Lin},
 pages = {6377--6389},
 publisher = {Curran Associates, Inc.},
 title = {Pruning neural networks without any data by iteratively conserving synaptic flow},
 url = {https://proceedings.neurips.cc/paper_files/paper/2020/file/46a4378f835dc8040c8057beb6a2da52-Paper.pdf},
 volume = {33},
 year = {2020}
}

@article{jiao2019tinybert,
  title={Tinybert: Distilling bert for natural language understanding},
  author={Jiao, Xiaoqi and Yin, Yichun and Shang, Lifeng and Jiang, Xin and Chen, Xiao and Li, Linlin and Wang, Fang and Liu, Qun},
  journal={arXiv preprint arXiv:1909.10351},
  year={2019}
}

@article{loshchilov2017decoupled,
  title={Decoupled weight decay regularization},
  author={Loshchilov, Ilya and Hutter, Frank},
  journal={arXiv preprint arXiv:1711.05101},
  year={2017}
}

@article{goyal2017accurate,
  title={Accurate, large minibatch sgd: Training imagenet in 1 hour},
  author={Goyal, Priya and Doll{\'a}r, Piotr and Girshick, Ross and Noordhuis, Pieter and Wesolowski, Lukasz and Kyrola, Aapo and Tulloch, Andrew and Jia, Yangqing and He, Kaiming},
  journal={arXiv preprint arXiv:1706.02677},
  year={2017}
}

@article{liu2021latency,
  title={Latency-aware automatic CNN channel pruning with GPU runtime analysis},
  author={Liu, Jiaqiang and Sun, Jingwei and Xu, Zhongtian and Sun, Guangzhong},
  journal={BenchCouncil Transactions on Benchmarks, Standards and Evaluations},
  volume={1},
  number={1},
  pages={100009},
  year={2021},
  publisher={Elsevier}
}

@inproceedings{liang2023less,
  title={Less is more: Task-aware layer-wise distillation for language model compression},
  author={Liang, Chen and Zuo, Simiao and Zhang, Qingru and He, Pengcheng and Chen, Weizhu and Zhao, Tuo},
  booktitle={International Conference on Machine Learning},
  pages={20852--20867},
  year={2023},
  organization={PMLR}
}

@inproceedings{wang2023yolov7,
  title={YOLOv7: Trainable bag-of-freebies sets new state-of-the-art for real-time object detectors},
  author={Wang, Chien-Yao and Bochkovskiy, Alexey and Liao, Hong-Yuan Mark},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={7464--7475},
  year={2023}
}

@article{wu2023brief,
  title={A brief overview of ChatGPT: The history, status quo and potential future development},
  author={Wu, Tianyu and He, Shizhu and Liu, Jingping and Sun, Siqi and Liu, Kang and Han, Qing-Long and Tang, Yang},
  journal={IEEE/CAA Journal of Automatica Sinica},
  volume={10},
  number={5},
  pages={1122--1136},
  year={2023},
  publisher={IEEE}
}

@article{yuan2021florence,
  title={Florence: A new foundation model for computer vision},
  author={Yuan, Lu and Chen, Dongdong and Chen, Yi-Ling and Codella, Noel and Dai, Xiyang and Gao, Jianfeng and Hu, Houdong and Huang, Xuedong and Li, Boxin and Li, Chunyuan and others},
  journal={arXiv preprint arXiv:2111.11432},
  year={2021}
}

@article{blalock2020state,
  title={What is the state of neural network pruning?},
  author={Blalock, Davis and Gonzalez Ortiz, Jose Javier and Frankle, Jonathan and Guttag, John},
  journal={Proceedings of machine learning and systems},
  volume={2},
  pages={129--146},
  year={2020}
}

@article{li2023losparse,
  title={LoSparse: Structured Compression of Large Language Models based on Low-Rank and Sparse Approximation},
  author={Li, Yixiao and Yu, Yifan and Zhang, Qingru and Liang, Chen and He, Pengcheng and Chen, Weizhu and Zhao, Tuo},
  journal={arXiv preprint arXiv:2306.11222},
  year={2023}
}

@article{swaminathan2020sparse,
  title={Sparse low rank factorization for deep neural network compression},
  author={Swaminathan, Sridhar and Garg, Deepak and Kannan, Rajkumar and Andres, Frederic},
  journal={Neurocomputing},
  volume={398},
  pages={185--196},
  year={2020},
  publisher={Elsevier}
}

@article{gong2020vecq,
  title={VecQ: Minimal loss DNN model compression with vectorized weight quantization},
  author={Gong, Cheng and Chen, Yao and Lu, Ye and Li, Tao and Hao, Cong and Chen, Deming},
  journal={IEEE Transactions on Computers},
  volume={70},
  number={5},
  pages={696--710},
  year={2020},
  publisher={IEEE}
}

@article{li2023model,
  title={Model Compression for Deep Neural Networks: A Survey},
  author={Li, Zhuo and Li, Hengyi and Meng, Lin},
  journal={Computers},
  volume={12},
  number={3},
  pages={60},
  year={2023},
  publisher={MDPI}
}

@article{pan2020meta,
  title={Meta-KD: A meta knowledge distillation framework for language model compression across domains},
  author={Pan, Haojie and Wang, Chengyu and Qiu, Minghui and Zhang, Yichang and Li, Yaliang and Huang, Jun},
  journal={arXiv preprint arXiv:2012.01266},
  year={2020}
}

@inproceedings{dupont2021weight,
  title={Weight Reparametrization for Budget-Aware Network Pruning},
  author={Dupont, Robin and Sahbi, Hichem and Michel, Guillaume},
  booktitle={2021 IEEE International Conference on Image Processing (ICIP)},
  pages={789--793},
  year={2021},
  organization={IEEE}
}

@article{tiwari2021chipnet,
  title={Chipnet: Budget-aware pruning with heaviside continuous approximations},
  author={Tiwari, Rishabh and Bamba, Udbhav and Chavan, Arnav and Gupta, Deepak K},
  journal={arXiv preprint arXiv:2102.07156},
  year={2021}
}

@inproceedings{Guyon2003DesignOE,
  title={Design of experiments for the NIPS 2003 variable selection benchmark},
  author={Isabelle M Guyon},
  year={2003},
  url={https://api.semanticscholar.org/CorpusID:115452637}
}

@article{scikit-learn,
  title={Scikit-learn: Machine Learning in {P}ython},
  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
          and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
          and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
          Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
  journal={Journal of Machine Learning Research},
  volume={12},
  pages={2825--2830},
  year={2011}
}

@article{kendall1938new,
  title={A new measure of rank correlation},
  author={Kendall, Maurice G},
  journal={Biometrika},
  volume={30},
  number={1/2},
  pages={81--93},
  year={1938},
  publisher={JSTOR}
}

@article{kingma2014adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik P and Ba, Jimmy},
  journal={arXiv preprint arXiv:1412.6980},
  year={2014}
}

@article{li2021survey,
  title={A survey of convolutional neural networks: analysis, applications, and prospects},
  author={Li, Zewen and Liu, Fan and Yang, Wenjie and Peng, Shouheng and Zhou, Jun},
  journal={IEEE transactions on neural networks and learning systems},
  year={2021},
  publisher={IEEE}
}

@article{koroteev2021bert,
  title={BERT: a review of applications in natural language processing and understanding},
  author={Koroteev, MV},
  journal={arXiv preprint arXiv:2103.11943},
  year={2021}
}

@article{hoang2023revisiting,
  title={Revisiting pruning at initialization through the lens of ramanujan graph},
  author={Hoang, Duc NM and Liu, Shiwei},
  journal={ICLR 2023},
  year={2023}
}

@article{krishnan2019structural,
  title={Structural pruning in deep neural networks: A small-world approach},
  author={Krishnan, Gokul and Du, Xiaocong and Cao, Yu},
  journal={arXiv preprint arXiv:1911.04453},
  year={2019}
}

@article{anwar2017structured,
  title={Structured pruning of deep convolutional neural networks},
  author={Anwar, Sajid and Hwang, Kyuyeon and Sung, Wonyong},
  journal={ACM Journal on Emerging Technologies in Computing Systems (JETC)},
  volume={13},
  number={3},
  pages={1--18},
  year={2017},
  publisher={ACM New York, NY, USA}
}

@InProceedings{Lemaire_2019_CVPR,
author = {Lemaire, Carl and Achkar, Andrew and Jodoin, Pierre-Marc},
title = {Structured Pruning of Neural Networks With Budget-Aware Regularization},
booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
month = {June},
year = {2019}
}


@InProceedings{pmlr-v119-goyal20a,
  title = 	 {{P}o{WER}-{BERT}: Accelerating {BERT} Inference via Progressive Word-vector Elimination},
  author =       {Goyal, Saurabh and Choudhury, Anamitra Roy and Raje, Saurabh and Chakaravarthy, Venkatesan and Sabharwal, Yogish and Verma, Ashish},
  booktitle = 	 {Proceedings of the 37th International Conference on Machine Learning},
  pages = 	 {3690--3699},
  year = 	 {2020},
  editor = 	 {III, Hal Daumé and Singh, Aarti},
  volume = 	 {119},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {13--18 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v119/goyal20a/goyal20a.pdf},
  url = 	 {https://proceedings.mlr.press/v119/goyal20a.html},
  abstract = 	 {We develop a novel method, called PoWER-BERT, for improving the inference time of the popular BERT model, while maintaining the accuracy. It works by: a) exploiting redundancy pertaining to word-vectors (intermediate transformer block outputs) and eliminating the redundant vectors. b) determining which word-vectors to eliminate by developing a strategy for measuring their significance, based on the self-attention mechanism. c) learning how many word-vectors to eliminate by augmenting the BERT model and the loss function. Experiments on the standard GLUE benchmark shows that PoWER-BERT achieves up to 4.5x reduction in inference time over BERT with &lt; 1% loss in accuracy. We show that PoWER-BERT offers significantly better trade-off between accuracy and inference time compared to prior methods. We demonstrate that our method attains up to 6.8x reduction in inference time with &lt; 1% loss in accuracy when applied over ALBERT, a highly compressed version of BERT. The code for PoWER-BERT is publicly available at https://github.com/IBM/PoWER-BERT.}
}

@inproceedings{deng2009imagenet,
  title={Imagenet: A large-scale hierarchical image database},
  author={Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Li, Kai and Fei-Fei, Li},
  booktitle={2009 IEEE conference on computer vision and pattern recognition},
  pages={248--255},
  year={2009},
  organization={Ieee}
}

@article{wang2022trainability,
  title={Trainability preserving neural pruning},
  author={Wang, Huan and Fu, Yun},
  journal={arXiv preprint arXiv:2207.12534},
  year={2022}
}

@inproceedings{howard2019searching,
  title={Searching for mobilenetv3},
  author={Howard, Andrew and Sandler, Mark and Chu, Grace and Chen, Liang-Chieh and Chen, Bo and Tan, Mingxing and Wang, Weijun and Zhu, Yukun and Pang, Ruoming and Vasudevan, Vijay and others},
  booktitle={Proceedings of the IEEE/CVF international conference on computer vision},
  pages={1314--1324},
  year={2019}
}

@article{howard2017mobilenets,
  title={Mobilenets: Efficient convolutional neural networks for mobile vision applications},
  author={Howard, Andrew G and Zhu, Menglong and Chen, Bo and Kalenichenko, Dmitry and Wang, Weijun and Weyand, Tobias and Andreetto, Marco and Adam, Hartwig},
  journal={arXiv preprint arXiv:1704.04861},
  year={2017}
}

@inproceedings{DBLP:conf/icml/RadfordKXBMS23,
  author       = {Alec Radford and
                  Jong Wook Kim and
                  Tao Xu and
                  Greg Brockman and
                  Christine McLeavey and
                  Ilya Sutskever},
  title        = {Robust Speech Recognition via Large-Scale Weak Supervision},
  booktitle    = {International Conference on Machine Learning, {ICML} 2023, 23-29 July
                  2023, Honolulu, Hawaii, {USA}},
  series       = {Proceedings of Machine Learning Research},
  volume       = {202},
  pages        = {28492--28518},
  publisher    = {{PMLR}},
  year         = {2023},
  timestamp    = {Mon, 28 Aug 2023 17:23:08 +0200},
  biburl       = {https://dblp.org/rec/conf/icml/RadfordKXBMS23.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{DBLP:conf/nips/BaevskiZMA20,
  author       = {Alexei Baevski and
                  Yuhao Zhou and
                  Abdelrahman Mohamed and
                  Michael Auli},
  title        = {wav2vec 2.0: {A} Framework for Self-Supervised Learning of Speech
                  Representations},
  booktitle    = {Advances in Neural Information Processing Systems 33: Annual Conference
                  on Neural Information Processing Systems 2020, NeurIPS 2020, December
                  6-12, 2020, virtual},
  year         = {2020},
  timestamp    = {Tue, 19 Jan 2021 15:57:22 +0100},
  biburl       = {https://dblp.org/rec/conf/nips/BaevskiZMA20.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{cordts2016cityscapes,
  title={The cityscapes dataset for semantic urban scene understanding},
  author={Cordts, Marius and Omran, Mohamed and Ramos, Sebastian and Rehfeld, Timo and Enzweiler, Markus and Benenson, Rodrigo and Franke, Uwe and Roth, Stefan and Schiele, Bernt},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={3213--3223},
  year={2016}
}

@article{badrinarayanan2017segnet,
  title={Segnet: A deep convolutional encoder-decoder architecture for image segmentation},
  author={Badrinarayanan, Vijay and Kendall, Alex and Cipolla, Roberto},
  journal={IEEE transactions on pattern analysis and machine intelligence},
  volume={39},
  number={12},
  pages={2481--2495},
  year={2017},
  publisher={IEEE}
}