@inproceedings{fu2021learn,
  title={Learn-to-share: A hardware-friendly transfer learning framework exploiting computation and parameter sharing},
  author={Fu, Cheng and Huang, Hanxian and Chen, Xinyun and Tian, Yuandong and Zhao, Jishen},
  booktitle={International Conference on Machine Learning},
  pages={3469--3479},
  year={2021},
  organization={PMLR}
}

@inproceedings{cao2022attention,
  title={Attention Fusion: a light yet efficient late fusion mechanism for task adaptation in NLU},
  author={Cao, Jin and Prakash, Chandana Satya and Hamza, Wael},
  booktitle={Findings of the Association for Computational Linguistics: NAACL 2022},
  pages={857--866},
  year={2022}
}

@inproceedings{malladi2023kernel,
  title={A kernel-based view of language model fine-tuning},
  author={Malladi, Sadhika and Wettig, Alexander and Yu, Dingli and Chen, Danqi and Arora, Sanjeev},
  booktitle={International Conference on Machine Learning},
  pages={23610--23641},
  year={2023},
  organization={PMLR}
}

@article{maurer2016benefit,
  title={The benefit of multitask representation learning},
  author={Maurer, Andreas and Pontil, Massimiliano and Romera-Paredes, Bernardino},
  journal={Journal of Machine Learning Research},
  volume={17},
  number={81},
  pages={1--32},
  year={2016}
}

@article{du2020few,
  title={Few-shot learning via learning the representation, provably},
  author={Du, Simon S and Hu, Wei and Kakade, Sham M and Lee, Jason D and Lei, Qi},
  journal={arXiv preprint arXiv:2002.09434},
  year={2020}
}

@article{oymak2023role,
  title={On the Role of Attention in Prompt-tuning},
  author={Oymak, Samet and Rawat, Ankit Singh and Soltanolkotabi, Mahdi and Thrampoulidis, Christos},
  journal={arXiv preprint arXiv:2306.03435},
  year={2023}
}

@article{englert2022adversarial,
  title={Adversarial reprogramming revisited},
  author={Englert, Matthias and Lazic, Ranko},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={28588--28600},
  year={2022}
}

@article{giannou2023expressive,
  title={The Expressive Power of Tuning Only the Norm Layers},
  author={Giannou, Angeliki and Rajput, Shashank and Papailiopoulos, Dimitris},
  journal={arXiv preprint arXiv:2302.07937},
  year={2023}
}

@article{zeng2023expressive,
  title={The expressive power of low-rank adaptation},
  author={Zeng, Yuchen and Lee, Kangwook},
  journal={arXiv preprint arXiv:2310.17513},
  year={2023}
}

@article{sheng2023s,
  title={S-lora: Serving thousands of concurrent lora adapters},
  author={Sheng, Ying and Cao, Shiyi and Li, Dacheng and Hooper, Coleman and Lee, Nicholas and Yang, Shuo and Chou, Christopher and Zhu, Banghua and Zheng, Lianmin and Keutzer, Kurt and others},
  journal={arXiv preprint arXiv:2311.03285},
  year={2023}
}

@inproceedings{zhang2020side,
  title={Side-tuning: a baseline for network adaptation via additive side networks},
  author={Zhang, Jeffrey O and Sax, Alexander and Zamir, Amir and Guibas, Leonidas and Malik, Jitendra},
  booktitle={Computer Vision--ECCV 2020: 16th European Conference, Glasgow, UK, August 23--28, 2020, Proceedings, Part III 16},
  pages={698--714},
  year={2020},
  organization={Springer}
}


@article{vardi2021optimal,
  title={On the optimal memorization power of relu neural networks},
  author={Vardi, Gal and Yehudai, Gilad and Shamir, Ohad},
  journal={arXiv preprint arXiv:2110.03187},
  year={2021}
}

@article{ippolito2022preventing,
  title={Preventing verbatim memorization in language models gives a false sense of privacy},
  author={Ippolito, Daphne and Tram{\`e}r, Florian and Nasr, Milad and Zhang, Chiyuan and Jagielski, Matthew and Lee, Katherine and Choquette-Choo, Christopher A and Carlini, Nicholas},
  journal={arXiv preprint arXiv:2210.17546},
  year={2022}
}

@article{carlini2022quantifying,
  title={Quantifying memorization across neural language models},
  author={Carlini, Nicholas and Ippolito, Daphne and Jagielski, Matthew and Lee, Katherine and Tramer, Florian and Zhang, Chiyuan},
  journal={arXiv preprint arXiv:2202.07646},
  year={2022}
}

@article{tripuraneni2020theory,
  title={On the theory of transfer learning: The importance of task diversity},
  author={Tripuraneni, Nilesh and Jordan, Michael and Jin, Chi},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={7852--7862},
  year={2020}
}

@article{wu2022power,
  title={The power and limitation of pretraining-finetuning for linear regression under covariate shift},
  author={Wu, Jingfeng and Zou, Difan and Braverman, Vladimir and Gu, Quanquan and Kakade, Sham},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={33041--33053},
  year={2022}
}

@article{he2021towards,
  title={Towards a unified view of parameter-efficient transfer learning},
  author={He, Junxian and Zhou, Chunting and Ma, Xuezhe and Berg-Kirkpatrick, Taylor and Neubig, Graham},
  journal={arXiv preprint arXiv:2110.04366},
  year={2021}
}

@article{ramesh2022hierarchical,
  title={Hierarchical text-conditional image generation with clip latents},
  author={Ramesh, Aditya and Dhariwal, Prafulla and Nichol, Alex and Chu, Casey and Chen, Mark},
  journal={arXiv preprint arXiv:2204.06125},
  volume={1},
  number={2},
  pages={3},
  year={2022}
}

@inproceedings{radford2021learning,
  title={Learning transferable visual models from natural language supervision},
  author={Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others},
  booktitle={International conference on machine learning},
  pages={8748--8763},
  year={2021},
  organization={PMLR}
}

@article{brown2020language,
  title={Language models are few-shot learners},
  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={1877--1901},
  year={2020}
}
openai2023gpt,
@article{openai2023gpt,
  title={GPT-4 technical report},
  author={OpenAI, R},
  journal={arXiv},
  pages={2303--08774},
  year={2023}
}
ouyang2022training,
@article{ouyang2022training,
  title={Training language models to follow instructions with human feedback},
  author={Ouyang, Long and Wu, Jeffrey and Jiang, Xu and Almeida, Diogo and Wainwright, Carroll and Mishkin, Pamela and Zhang, Chong and Agarwal, Sandhini and Slama, Katarina and Ray, Alex and others},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={27730--27744},
  year={2022}
}

chowdhery2022palm,
@article{chowdhery2022palm,
  title={Palm: Scaling language modeling with pathways},
  author={Chowdhery, Aakanksha and Narang, Sharan and Devlin, Jacob and Bosma, Maarten and Mishra, Gaurav and Roberts, Adam and Barham, Paul and Chung, Hyung Won and Sutton, Charles and Gehrmann, Sebastian and others},
  journal={arXiv preprint arXiv:2204.02311},
  year={2022}
}

zhang2022opt,
@article{zhang2022opt,
  title={Opt: Open pre-trained transformer language models},
  author={Zhang, Susan and Roller, Stephen and Goyal, Naman and Artetxe, Mikel and Chen, Moya and Chen, Shuohui and Dewan, Christopher and Diab, Mona and Li, Xian and Lin, Xi Victoria and others},
  journal={arXiv preprint arXiv:2205.01068},
  year={2022}
}

touvron2023llama
@article{touvron2023llama,
  title={Llama 2: Open foundation and fine-tuned chat models},
  author={Touvron, Hugo and Martin, Louis and Stone, Kevin and Albert, Peter and Almahairi, Amjad and Babaei, Yasmine and Bashlykov, Nikolay and Batra, Soumya and Bhargava, Prajjwal and Bhosale, Shruti and others},
  journal={arXiv preprint arXiv:2307.09288},
  year={2023}
}

@article{bommasani2021opportunities,
  title={On the opportunities and risks of foundation models},
  author={Bommasani, Rishi and Hudson, Drew A and Adeli, Ehsan and Altman, Russ and Arora, Simran and von Arx, Sydney and Bernstein, Michael S and Bohg, Jeannette and Bosselut, Antoine and Brunskill, Emma and others},
  journal={arXiv preprint arXiv:2108.07258},
  year={2021}
}

@article{huang2003learning,
  title={Learning capability and storage capacity of two-hidden-layer feedforward networks},
  author={Huang, Guang-Bin},
  journal={IEEE transactions on neural networks},
  volume={14},
  number={2},
  pages={274--281},
  year={2003},
  publisher={IEEE}
}


@article{huang1998upper,
  title={Upper bounds on the number of hidden neurons in feedforward networks with arbitrary bounded nonlinear activation functions},
  author={Huang, Guang-Bin and Babri, Haroon A},
  journal={IEEE transactions on neural networks},
  volume={9},
  number={1},
  pages={224--229},
  year={1998},
  publisher={IEEE}
}


@article{baum1988capabilities,
  title={On the capabilities of multilayer perceptrons},
  author={Baum, Eric B},
  journal={Journal of complexity},
  volume={4},
  number={3},
  pages={193--215},
  year={1988},
  publisher={Elsevier}
}

@inproceedings{kidger2020universal,
  title={Universal approximation with deep narrow networks},
  author={Kidger, Patrick and Lyons, Terry},
  booktitle={Conference on learning theory},
  pages={2306--2327},
  year={2020},
  organization={PMLR}
}

@article{hanin2017approximating,
  title={Approximating continuous functions by relu nets of minimal width},
  author={Hanin, Boris and Sellke, Mark},
  journal={arXiv preprint arXiv:1710.11278},
  year={2017}
}

@article{lu2017expressive,
  title={The expressive power of neural networks: A view from the width},
  author={Lu, Zhou and Pu, Hongming and Wang, Feicheng and Hu, Zhiqiang and Wang, Liwei},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@article{park2020minimum,
  title={Minimum width for universal approximation},
  author={Park, Sejun and Yun, Chulhee and Lee, Jaeho and Shin, Jinwoo},
  journal={arXiv preprint arXiv:2006.08859},
  year={2020}
}

@article{hornik1989multilayer,
  title={Multilayer feedforward networks are universal approximators},
  author={Hornik, Kurt and Stinchcombe, Maxwell and White, Halbert},
  journal={Neural networks},
  volume={2},
  number={5},
  pages={359--366},
  year={1989},
  publisher={Elsevier}
}

@article{funahashi1989approximate,
  title={On the approximate realization of continuous mappings by neural networks},
  author={Funahashi, Ken-Ichi},
  journal={Neural networks},
  volume={2},
  number={3},
  pages={183--192},
  year={1989},
  publisher={Elsevier}
}

@article{barron1993universal,
  title={Universal approximation bounds for superpositions of a sigmoidal function},
  author={Barron, Andrew R},
  journal={IEEE Transactions on Information theory},
  volume={39},
  number={3},
  pages={930--945},
  year={1993},
  publisher={IEEE}
}


@article{cybenko1989approximation,
  title={Approximation by superpositions of a sigmoidal function},
  author={Cybenko, George},
  journal={Mathematics of control, signals and systems},
  volume={2},
  number={4},
  pages={303--314},
  year={1989},
  publisher={Springer}
}

@article{zeng2023exploring,
  title={Exploring Memorization in Fine-tuned Language Models},
  author={Zeng, Shenglai and Li, Yaxin and Ren, Jie and Liu, Yiding and Xu, Han and He, Pengfei and Xing, Yue and Wang, Shuaiqiang and Tang, Jiliang and Yin, Dawei},
  journal={arXiv preprint arXiv:2310.06714},
  year={2023}
}

@article{hu2021lora,
  title={Lora: Low-rank adaptation of large language models},
  author={Hu, Edward J and Shen, Yelong and Wallis, Phillip and Allen-Zhu, Zeyuan and Li, Yuanzhi and Wang, Shean and Wang, Lu and Chen, Weizhu},
  journal={arXiv preprint arXiv:2106.09685},
  year={2021}
}

@article{zaken2021bitfit,
  title={Bitfit: Simple parameter-efficient fine-tuning for transformer-based masked language-models},
  author={Zaken, Elad Ben and Ravfogel, Shauli and Goldberg, Yoav},
  journal={arXiv preprint arXiv:2106.10199},
  year={2021}
}

@inproceedings{houlsby2019parameter,
  title={Parameter-efficient transfer learning for NLP},
  author={Houlsby, Neil and Giurgiu, Andrei and Jastrzebski, Stanislaw and Morrone, Bruna and De Laroussilhe, Quentin and Gesmundo, Andrea and Attariyan, Mona and Gelly, Sylvain},
  booktitle={International Conference on Machine Learning},
  pages={2790--2799},
  year={2019},
  organization={PMLR}
}

@article{yun2019small,
  title={Small ReLU networks are powerful memorizers: a tight analysis of memorization capacity},
  author={Yun, Chulhee and Sra, Suvrit and Jadbabaie, Ali},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@article{zhang2016understanding,
  title={Understanding deep learning requires rethinking generalization},
  author={Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
  journal={arXiv preprint arXiv:1611.03530},
  year={2016}
}


@article{hardt2016identity,
  title={Identity matters in deep learning},
  author={Hardt, Moritz and Ma, Tengyu},
  journal={arXiv preprint arXiv:1611.04231},
  year={2016}
}


@inproceedings{nguyen2018optimization,
  title={Optimization landscape and expressivity of deep CNNs},
  author={Nguyen, Quynh and Hein, Matthias},
  booktitle={International conference on machine learning},
  pages={3730--3739},
  year={2018},
  organization={PMLR}
}


@inproceedings{kim2023provable,
  title={Provable Memorization Capacity of Transformers},
  author={Kim, Junghwan and Kim, Michelle and Mozafari, Barzan},
  booktitle={International Conference on Learning Representation (ICLR)},
  year={2023},
}

@misc{vershynin2020memory,
      title={Memory capacity of neural networks with threshold and ReLU activations}, 
      author={Roman Vershynin},
      year={2020},
      eprint={2001.06938},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{rajput2021exponential,
      title={An Exponential Improvement on the Memorization Capacity of Deep Threshold Networks}, 
      author={Shashank Rajput and Kartik Sreenivasan and Dimitris Papailiopoulos and Amin Karbasi},
      year={2021},
      eprint={2106.07724},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{bubeck2020network,
      title={Network size and weights size for memorization with two-layers neural networks}, 
      author={Sébastien Bubeck and Ronen Eldan and Yin Tat Lee and Dan Mikulincer},
      year={2020},
      eprint={2006.02855},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{bartlett2017nearlytight,
      title={Nearly-tight VC-dimension and pseudodimension bounds for piecewise linear neural networks}, 
      author={Peter L. Bartlett and Nick Harvey and Chris Liaw and Abbas Mehrabian},
      year={2017},
      eprint={1703.02930},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}