@inproceedings{Eldad2019,
   abstract = {Quantization of neural networks has become common practice, driven by the need for efficient implementations of deep neural networks on embedded devices. In this paper, we exploit an oft-overlooked degree of freedom in most networks - for a given layer, individual output channels can be scaled by any factor provided that the corresponding weights of the next layer are inversely scaled. Therefore, a given network has many factorizations which change the weights of the network without changing its function. We present a conceptually simple and easy to implement method that uses this property and show that proper factorizations significantly decrease the degradation caused by quantization. We show improvement on a wide variety of networks and achieve state-of-the-art degradation results for MobileNets. While our focus is on quantization, this type of factorization is applicable to other domains such as network-pruning, neural nets regularization and network interpretability.},
   author = {Meller Eldad and Finkelstein Alexander and Almog Uri and Grobman Mark},
   journal = {36th International Conference on Machine Learning, ICML 2019},
   title = {Same, same but different: Recovering neural network quantization error through weight factorization},
   volume = {2019-June},
   year = {2019},
}

@inproceedings{nagel2020up,
  title={Up or down? adaptive rounding for post-training quantization},
  author={Nagel, Markus and Amjad, Rana Ali and Van Baalen, Mart and Louizos, Christos and Blankevoort, Tijmen},
  booktitle={International Conference on Machine Learning},
  pages={7197--7206},
  year={2020},
  organization={PMLR}
}

@article{Hubara2020,
   abstract = {Lately, post-training quantization methods have gained considerable
attention, as they are simple to use, and require only a small unlabeled
calibration set. This small dataset cannot be used to fine-tune the model
without significant over-fitting. Instead, these methods only use the
calibration set to set the activations' dynamic ranges. However, such methods
always resulted in significant accuracy degradation, when used below 8-bits
(except on small datasets). Here we aim to break the 8-bit barrier. To this
end, we minimize the quantization errors of each layer separately by optimizing
its parameters over the calibration set. We empirically demonstrate that this
approach is: (1) much less susceptible to over-fitting than the standard
fine-tuning approaches, and can be used even on a very small calibration set;
and (2) more powerful than previous methods, which only set the activations'
dynamic ranges. Furthermore, we demonstrate how to optimally allocate the
bit-widths for each layer, while constraining accuracy degradation or model
compression by proposing a novel integer programming formulation. Finally, we
suggest model global statistics tuning, to correct biases introduced during
quantization. Together, these methods yield state-of-the-art results for both
vision and text models. For instance, on ResNet50, we obtain less than 1\%
accuracy degradation --- with 4-bit weights and activations in all layers, but
the smallest two. We open-sourced our code.},
   author = {Itay Hubara and Yury Nahshan and Yair Hanani and Ron Banner and Daniel Soudry},
   doi = {10.48550/arxiv.2006.10518},
   month = {6},
   title = {Improving Post Training Neural Quantization: Layer-wise Calibration and Integer Programming},
   url = {https://arxiv.org/abs/2006.10518},
   year = {2020},
}

@inproceedings{BayesianBits,
   abstract = {We introduce Bayesian Bits, a practical method for joint mixed precision quantization and pruning through gradient based optimization. Bayesian Bits employs a novel decomposition of the quantization operation, which sequentially considers doubling the bit width. At each new bit width, the residual error between the full precision value and the previously rounded value is quantized. We then decide whether or not to add this quantized residual error for a higher effective bit width and lower quantization noise. By starting with a power-of-two bit width, this decomposition will always produce hardware-friendly configurations, and through an additional 0-bit option, serves as a unified view of pruning and quantization. Bayesian Bits then introduces learnable stochastic gates, which collectively control the bit width of the given tensor. As a result, we can obtain low bit solutions by performing approximate inference over the gates, with prior distributions that encourage most of them to be switched off. We experimentally validate our proposed method on several benchmark datasets and show that we can learn pruned, mixed precision networks that provide a better trade-off between accuracy and efficiency than their static bit width equivalents.},
   author = {Mart van Baalen and Christos Louizos and Markus Nagel and Rana Ali Amjad and Ying Wang and Tijmen Blankevoort and Max Welling},
   issn = {10495258},
   journal = {Advances in Neural Information Processing Systems},
   title = {Bayesian bits: Unifying quantization and pruning},
   volume = {2020-December},
   year = {2020},
}

@article{kashin1977diameters,
  title={Diameters of some finite-dimensional sets and classes of smooth functions},
  author={Kashin, Boris Sergeevich},
  journal={Izvestiya Rossiiskoi Akademii Nauk. Seriya Matematicheskaya},
  volume={41},
  number={2},
  pages={334--351},
  year={1977},
  publisher={Russian Academy of Sciences, Steklov Mathematical Institute of Russian~…}
}

@article{temlyakov2008greedy,
  title={Greedy approximation},
  author={Temlyakov, Vladimir N},
  journal={Acta Numerica},
  volume={17},
  pages={235--409},
  year={2008},
  publisher={Cambridge University Press}
}


%---------------------------Examples---------------------------
@inproceedings{langley00,
 author    = {P. Langley},
 title     = {Crafting Papers on Machine Learning},
 year      = {2000},
 pages     = {1207--1216},
 editor    = {Pat Langley},
 booktitle     = {Proceedings of the 17th International Conference
              on Machine Learning (ICML 2000)},
 address   = {Stanford, CA},
 publisher = {Morgan Kaufmann}
}

@TechReport{mitchell80,
  author = 	 "T. M. Mitchell",
  title = 	 "The Need for Biases in Learning Generalizations",
  institution =  "Computer Science Department, Rutgers University",
  year = 	 "1980",
  address =	 "New Brunswick, MA",
}

@phdthesis{kearns89,
  author = {M. J. Kearns},
  title =  {Computational Complexity of Machine Learning},
  school = {Department of Computer Science, Harvard University},
  year =   {1989}
}

@Book{MachineLearningI,
  editor = 	 "R. S. Michalski and J. G. Carbonell and T.
		  M. Mitchell",
  title = 	 "Machine Learning: An Artificial Intelligence
		  Approach, Vol. I",
  publisher = 	 "Tioga",
  year = 	 "1983",
  address =	 "Palo Alto, CA"
}

@Book{DudaHart2nd,
  author =       "R. O. Duda and P. E. Hart and D. G. Stork",
  title =        "Pattern Classification",
  publisher =    "John Wiley and Sons",
  edition =      "2nd",
  year =         "2000"
}

@misc{anonymous,
  title= {Suppressed for Anonymity},
  author= {Author, N. N.},
  year= {2021}
}

@InCollection{Newell81,
  author =       "A. Newell and P. S. Rosenbloom",
  title =        "Mechanisms of Skill Acquisition and the Law of
                  Practice", 
  booktitle =    "Cognitive Skills and Their Acquisition",
  pages =        "1--51",
  publisher =    "Lawrence Erlbaum Associates, Inc.",
  year =         "1981",
  editor =       "J. R. Anderson",
  chapter =      "1",
  address =      "Hillsdale, NJ"
}


@Article{Samuel59,
  author = 	 "A. L. Samuel",
  title = 	 "Some Studies in Machine Learning Using the Game of
		  Checkers",
  journal =	 "IBM Journal of Research and Development",
  year =	 "1959",
  volume =	 "3",
  number =	 "3",
  pages =	 "211--229"
}

@article{samuel1959some,
  title={Some studies in machine learning using the game of checkers},
  author={Samuel, Arthur L},
  journal={IBM Journal of research and development},
  volume={3},
  number={3},
  pages={210--229},
  year={1959},
  publisher={IBM}
}
%--------------------------------------------------------------------

@inproceedings{Paperno_2016,
   title={The LAMBADA dataset: Word prediction requiring a broad discourse context},
   url={http://dx.doi.org/10.18653/v1/P16-1144},
   DOI={10.18653/v1/p16-1144},
   booktitle={Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
   publisher={Association for Computational Linguistics},
   author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Ngoc Quan and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernandez, Raquel},
   year={2016} }

@misc{zhang2022opt,
      title={OPT: Open Pre-trained Transformer Language Models}, 
      author={Susan Zhang and Stephen Roller and Naman Goyal and Mikel Artetxe and Moya Chen and Shuohui Chen and Christopher Dewan and Mona Diab and Xian Li and Xi Victoria Lin and Todor Mihaylov and Myle Ott and Sam Shleifer and Kurt Shuster and Daniel Simig and Punit Singh Koura and Anjali Sridhar and Tianlu Wang and Luke Zettlemoyer},
      year={2022},
      eprint={2205.01068},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{dettmers2023qlora,
      title={QLoRA: Efficient Finetuning of Quantized LLMs}, 
      author={Tim Dettmers and Artidoro Pagnoni and Ari Holtzman and Luke Zettlemoyer},
      year={2023},
      eprint={2305.14314},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{dettmers2022llmint8,
      title={LLM.int8(): 8-bit Matrix Multiplication for Transformers at Scale}, 
      author={Tim Dettmers and Mike Lewis and Younes Belkada and Luke Zettlemoyer},
      year={2022},
      eprint={2208.07339},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{nagel2021white,
      title={A White Paper on Neural Network Quantization}, 
      author={Markus Nagel and Marios Fournarakis and Rana Ali Amjad and Yelysei Bondarenko and Mart van Baalen and Tijmen Blankevoort},
      year={2021},
      eprint={2106.08295},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{Lyubarskii_2010,
   title={Uncertainty Principles and Vector Quantization},
   volume={56},
   ISSN={1557-9654},
   url={http://dx.doi.org/10.1109/TIT.2010.2048458},
   DOI={10.1109/tit.2010.2048458},
   number={7},
   journal={IEEE Transactions on Information Theory},
   publisher={Institute of Electrical and Electronics Engineers (IEEE)},
   author={Lyubarskii, Yurii and Vershynin, Roman},
   year={2010},
   month=jul, pages={3491–3501} }

\Bibitem{GarGlu84}
\by A.~Yu.~Garnaev, E.~D.~Gluskin
\paper The widths of a Euclidean ball
\jour Dokl. Akad. Nauk SSSR
\yr 1984
\vol 277
\issue 5
\pages 1048--1052
\mathnet{http://mi.mathnet.ru/dan9506}
\mathscinet{http://mathscinet.ams.org/mathscinet-getitem?mr=759962}
\zmath{https://zbmath.org/?q=an:0588.41022}

@Article{GarGlu1984,
 Author = {Garnaev, A. Yu. and Gluskin, E. D.},
 Title = {On widths of the {Euclidean} ball},
 FJournal = {Soviet Mathematics. Doklady},
 Journal = {Sov. Math., Dokl.},
 ISSN = {0197-6788},
 Volume = {30},
 Pages = {200--204},
 Year = {1984},
 Language = {English},
 Keywords = {41A46,41A65,46B20},
 zbMATH = {3944477},
 Zbl = {0588.41022}
}

@misc{guedon2008majorizing,
      title={Majorizing measures and proportional subsets of bounded orthonormal systems}, 
      author={Olivier Guedon and Shahar Mendelson and Alain Pajor and Nicole Tomczak-Jaegermann},
      year={2008},
      eprint={0801.3556},
      archivePrefix={arXiv},
      primaryClass={math.FA}
}

@misc{bondarenko2023quantizable,
      title={Quantizable Transformers: Removing Outliers by Helping Attention Heads Do Nothing}, 
      author={Yelysei Bondarenko and Markus Nagel and Tijmen Blankevoort},
      year={2023},
      eprint={2306.12929},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{bondarenko2021understanding,
      title={Understanding and Overcoming the Challenges of Efficient Transformer Quantization}, 
      author={Yelysei Bondarenko and Markus Nagel and Tijmen Blankevoort},
      year={2021},
      eprint={2109.12948},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{safaryan2022uncertainty,
  title={Uncertainty principle for communication compression in distributed and federated learning and the search for an optimal compressor},
  author={Safaryan, Mher and Shulgin, Egor and Richt{\'a}rik, Peter},
  journal={Information and Inference: A Journal of the IMA},
  volume={11},
  number={2},
  pages={557--580},
  year={2022},
  publisher={Oxford University Press}
}

@article{touvron2023llama,
  title={Llama: Open and efficient foundation language models},
  author={Touvron, Hugo and Lavril, Thibaut and Izacard, Gautier and Martinet, Xavier and Lachaux, Marie-Anne and Lacroix, Timoth{\'e}e and Rozi{\`e}re, Baptiste and Goyal, Naman and Hambro, Eric and Azhar, Faisal and others},
  journal={arXiv preprint arXiv:2302.13971},
  year={2023}
}

@article{touvron2023llama2,
  title={Llama 2: Open foundation and fine-tuned chat models},
  author={Touvron, Hugo and Martin, Louis and Stone, Kevin and Albert, Peter and Almahairi, Amjad and Babaei, Yasmine and Bashlykov, Nikolay and Batra, Soumya and Bhargava, Prajjwal and Bhosale, Shruti and others},
  journal={arXiv preprint arXiv:2307.09288},
  year={2023}
}

@article{almazrouei2023falcon,
  title={The falcon series of open language models},
  author={Almazrouei, Ebtesam and Alobeidli, Hamza and Alshamsi, Abdulaziz and Cappelli, Alessandro and Cojocaru, Ruxandra and Debbah, M{\'e}rouane and Goffinet, {\'E}tienne and Hesslow, Daniel and Launay, Julien and Malartic, Quentin and others},
  journal={arXiv preprint arXiv:2311.16867},
  year={2023}
}

@article{kashin2023efficient,
  title={An Efficient Algorithm for Decomposing a Vector into Two Vectors with a Small Uniform Norm},
  author={Kashin, Boris Sergeevich and Romskii, DG},
  journal={Matematicheskie Zametki},
  volume={114},
  number={6},
  pages={945--948},
  year={2023},
  publisher={Russian Academy of Sciences, Steklov Mathematical Institute of Russian~…}
}

@article{bershatsky2022memory,
  title={Memory-efficient backpropagation through large linear layers},
  author={Bershatsky, Daniel and Mikhalev, Aleksandr and Katrutsa, Alexandr and Gusak, Julia and Merkulov, Daniil and Oseledets, Ivan},
  journal={arXiv preprint arXiv:2201.13195},
  year={2022}
}

@article{micikevicius2017mixed,
  title={Mixed precision training},
  author={Micikevicius, Paulius and Narang, Sharan and Alben, Jonah and Diamos, Gregory and Elsen, Erich and Garcia, David and Ginsburg, Boris and Houston, Michael and Kuchaiev, Oleksii and Venkatesh, Ganesh and others},
  journal={arXiv preprint arXiv:1710.03740},
  year={2017}
}

@inproceedings{novikov2023few,
  title={Few-bit backward: Quantized gradients of activation functions for memory footprint reduction},
  author={Novikov, Georgii Sergeevich and Bershatsky, Daniel and Gusak, Julia and Shonenkov, Alex and Dimitrov, Denis Valerievich and Oseledets, Ivan},
  booktitle={International Conference on Machine Learning},
  pages={26363--26381},
  year={2023},
  organization={PMLR}
}

@article{gusak2022survey,
  title={Survey on large scale neural network training},
  author={Gusak, Julia and Cherniuk, Daria and Shilova, Alena and Katrutsa, Alexander and Bershatsky, Daniel and Zhao, Xunyi and Eyraud-Dubois, Lionel and Shlyazhko, Oleg and Dimitrov, Denis and Oseledets, Ivan and others},
  journal={arXiv preprint arXiv:2202.10435},
  year={2022}
}

@article{chen2016training,
  title={Training deep nets with sublinear memory cost},
  author={Chen, Tianqi and Xu, Bing and Zhang, Chiyuan and Guestrin, Carlos},
  journal={arXiv preprint arXiv:1604.06174},
  year={2016}
}

@article{temlyakov1998nonlinear,
  title={Nonlinear Kolmogorov widths},
  author={Temlyakov, Vladimir Nikolaevich},
  journal={Mathematical Notes},
  volume={63},
  number={6},
  pages={785--795},
  year={1998},
  publisher={Springer}
}