@inproceedings{DBLP:conf/nips/Johnson013,
  author       = {Rie Johnson and
                  Tong Zhang},
  title        = {Accelerating Stochastic Gradient Descent using Predictive Variance
                  Reduction},
  booktitle    = {NIPS},
  pages        = {315--323},
  year         = {2013},
}

@inproceedings{DBLP:conf/nips/DefazioBL14,
  author       = {Aaron Defazio and
                  Francis R. Bach and
                  Simon Lacoste{-}Julien},
  title        = {{SAGA:} {A} Fast Incremental Gradient Method With Support for Non-Strongly
                  Convex Composite Objectives},
  booktitle    = {NIPS},
  pages        = {1646--1654},
  year         = {2014},
}

@article{bottou2018optimization,
  title={Optimization methods for large-scale machine learning},
  author={Bottou, L{\'e}on and Curtis, Frank E and Nocedal, Jorge},
  journal={SIAM review},
  volume={60},
  number={2},
  pages={223--311},
  year={2018},
  publisher={SIAM}
}

@article{polyak1977comparison,
  title={Comparison of the convergence rates for single-step and multi-step optimization algorithms in the presence of noise},
  author={Polyak, Boris T},
  journal={Engineering Cybernetics},
  volume={15},
  number={1},
  pages={6--10},
  year={1977}
}

@article{polyak1991,
author = {Polyak, Boris},
year = {1991},
month = {01},
pages = {937-946},
title = {New method of stochastic approximation type},
volume = {7},
journal = {Autom. Remote Control}
}

@book{nesterov2013introductory,
  title={Introductory lectures on convex optimization: A basic course},
  author={Nesterov, Yurii},
  volume={87},
  year={2013},
  publisher={Springer Science \& Business Media}
}

@inproceedings{DBLP:journals/corr/KingmaB14,
  author       = {Diederik P. Kingma and
                  Jimmy Ba},
  title        = {Adam: {A} Method for Stochastic Optimization},
  booktitle    = {ICLR},
  year         = {2015}
}

@inproceedings{jin2017escape,
  title={How to escape saddle points efficiently},
  author={Jin, Chi and Ge, Rong and Netrapalli, Praneeth and Kakade, Sham M and Jordan, Michael I},
  booktitle={ICML},
  pages={1724--1732},
  year={2017},
}

@article{duchi2011adaptive,
  title={Adaptive subgradient methods for online learning and stochastic optimization.},
  author={Duchi, John and Hazan, Elad and Singer, Yoram},
  journal={Journal of machine learning research},
  volume={12},
  number={7},
  year={2011}
}

@article{hinton2012neural,
  title={Neural networks for machine learning lecture 6a overview of mini-batch gradient descent},
  author={Hinton, Geoffrey and Srivastava, Nitish and Swersky, Kevin},
  journal={Cited on},
  volume={14},
  number={8},
  pages={2},
  year={2012}
}

@article{kingma2014adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik P and Ba, Jimmy},
  journal={arXiv preprint arXiv:1412.6980},
  year={2014}
}

@article{loshchilov2017decoupled,
  title={Decoupled weight decay regularization},
  author={Loshchilov, Ilya and Hutter, Frank},
  journal={arXiv preprint arXiv:1711.05101},
  year={2017}
}

@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={NIPS},
  volume={30},
  year={2017}
}

@article{dozat2016incorporating,
  title={Incorporating nesterov momentum into adam},
  author={Dozat, Timothy},
  journal={ICLR Workshop},
  year={2016}
}

@inproceedings{shazeer2018adafactor,
  title={Adafactor: Adaptive learning rates with sublinear memory cost},
  author={Shazeer, Noam and Stern, Mitchell},
  booktitle={ICML},
  pages={4596--4604},
  year={2018},
}

@article{reddi2019convergence,
  title={On the convergence of adam and beyond},
  author={Reddi, Sashank J and Kale, Satyen and Kumar, Sanjiv},
  journal={arXiv preprint arXiv:1904.09237},
  year={2019}
}

@article{zhuang2020adabelief,
  title={Adabelief optimizer: Adapting stepsizes by the belief in observed gradients},
  author={Zhuang, Juntang and Tang, Tommy and Ding, Yifan and Tatikonda, Sekhar C and Dvornek, Nicha and Papademetris, Xenophon and Duncan, James},
  journal={NeurIPS},
  volume={33},
  pages={18795--18806},
  year={2020}
}

@article{you2019large,
  title={Large batch optimization for deep learning: Training bert in 76 minutes},
  author={You, Yang and Li, Jing and Reddi, Sashank and Hseu, Jonathan and Kumar, Sanjiv and Bhojanapalli, Srinadh and Song, Xiaodan and Demmel, James and Keutzer, Kurt and Hsieh, Cho-Jui},
  journal={arXiv preprint arXiv:1904.00962},
  year={2019}
}

@article{chen2023symbolic,
  title={Symbolic discovery of optimization algorithms},
  author={Chen, Xiangning and Liang, Chen and Huang, Da and Real, Esteban and Wang, Kaiyuan and Liu, Yao and Pham, Hieu and Dong, Xuanyi and Luong, Thang and Hsieh, Cho-Jui and others},
  journal={arXiv preprint arXiv:2302.06675},
  year={2023}
}

@article{xie2022adan,
  title={Adan: Adaptive nesterov momentum algorithm for faster optimizing deep models},
  author={Xie, Xingyu and Zhou, Pan and Li, Huan and Lin, Zhouchen and Yan, Shuicheng},
  journal={arXiv preprint arXiv:2208.06677},
  year={2022}
}

@article{broyden1970convergence,
  title={The convergence of a class of double-rank minimization algorithms 1. general considerations},
  author={Broyden, Charles George},
  journal={IMA Journal of Applied Mathematics},
  volume={6},
  number={1},
  pages={76--90},
  year={1970},
  publisher={Oxford University Press}
}

@article{nesterov2006cubic,
  title={Cubic regularization of Newton method and its global performance},
  author={Nesterov, Yurii and Polyak, Boris T},
  journal={Mathematical Programming},
  volume={108},
  number={1},
  pages={177--205},
  year={2006},
  publisher={Springer}
}

@book{conn2000trust,
  title={Trust region methods},
  author={Conn, Andrew R and Gould, Nicholas IM and Toint, Philippe L},
  year={2000},
  publisher={SIAM}
}

@inproceedings{becker1988improving,
  title={Improving the convergence of back-propagation learning with second order methods},
  author={Becker, Sue and Le Cun, Yann and others},
  booktitle={Proceedings of the 1988 connectionist models summer school},
  pages={29--37},
  year={1988}
}

@inproceedings{martens2010deep,
  title={Deep learning via hessian-free optimization.},
  author={Martens, James and others},
  booktitle={ICML},
  volume={27},
  pages={735--742},
  year={2010}
}

@inproceedings{schaul2013no,
  title={No more pesky learning rates},
  author={Schaul, Tom and Zhang, Sixin and LeCun, Yann},
  booktitle={ICML},
  pages={343--351},
  year={2013},
}

@article{pascanu2013revisiting,
  title={Revisiting natural gradient for deep networks},
  author={Pascanu, Razvan and Bengio, Yoshua},
  journal={arXiv preprint arXiv:1301.3584},
  year={2013}
}

@inproceedings{martens2015optimizing,
  title={Optimizing neural networks with kronecker-factored approximate curvature},
  author={Martens, James and Grosse, Roger},
  booktitle={ICML},
  pages={2408--2417},
  year={2015},
}

@inproceedings{ba2016distributed,
  title={Distributed second-order optimization using Kronecker-factored approximations},
  author={Ba, Jimmy and Grosse, Roger and Martens, James},
  booktitle={ICLR},
  year={2016}
}

@article{george2018fast,
  title={Fast approximate natural gradient descent in a kronecker factored eigenbasis},
  author={George, Thomas and Laurent, C{\'e}sar and Bouthillier, Xavier and Ballas, Nicolas and Vincent, Pascal},
  journal={NIPS},
  volume={31},
  year={2018}
}

@inproceedings{martens2018kronecker,
  title={Kronecker-factored curvature approximations for recurrent neural networks},
  author={Martens, James and Ba, Jimmy and Johnson, Matt},
  booktitle={ICLR},
  year={2018}
}

@inproceedings{yao2021adahessian,
  title={Adahessian: An adaptive second order optimizer for machine learning},
  author={Yao, Zhewei and Gholami, Amir and Shen, Sheng and Mustafa, Mustafa and Keutzer, Kurt and Mahoney, Michael},
  booktitle={AAAI},
  volume={35},
  number={12},
  pages={10665--10673},
  year={2021}
}

@article{jahani2021doubly,
  title={Doubly adaptive scaled algorithm for machine learning using second-order information},
  author={Jahani, Majid and Rusakov, Sergey and Shi, Zheng and Richt{\'a}rik, Peter and Mahoney, Michael W and Tak{\'a}{\v{c}}, Martin},
  journal={arXiv preprint arXiv:2109.05198},
  year={2021}
}

@article{jamieson2012query,
  title={Query complexity of derivative-free optimization},
  author={Jamieson, Kevin G and Nowak, Robert and Recht, Ben},
  journal={NIPS},
  volume={25},
  year={2012}
}

@article{agarwal2009information,
  title={Information-theoretic lower bounds on the oracle complexity of convex optimization},
  author={Agarwal, Alekh and Wainwright, Martin J and Bartlett, Peter and Ravikumar, Pradeep},
  journal={NIPS},
  volume={22},
  year={2009}
}

@article{raginsky2011information,
  title={Information-based complexity, feedback and dynamics in convex programming},
  author={Raginsky, Maxim and Rakhlin, Alexander},
  journal={IEEE Transactions on Information Theory},
  volume={57},
  number={10},
  pages={7036--7056},
  year={2011},
  publisher={IEEE}
}

@article{duchi2015optimal,
  title={Optimal rates for zero-order convex optimization: The power of two function evaluations},
  author={Duchi, John C and Jordan, Michael I and Wainwright, Martin J and Wibisono, Andre},
  journal={IEEE Transactions on Information Theory},
  volume={61},
  number={5},
  pages={2788--2806},
  year={2015},
  publisher={IEEE}
}

@article{shamir2017optimal,
  title={An optimal algorithm for bandit and zero-order convex optimization with two-point feedback},
  author={Shamir, Ohad},
  journal={The Journal of Machine Learning Research},
  volume={18},
  number={1},
  pages={1703--1713},
  year={2017},
  publisher={JMLR. org}
}

@article{wang2020zeroth,
  title={Zeroth-order algorithms for nonconvex minimax problems with improved complexities},
  author={Wang, Zhongruo and Balasubramanian, Krishnakumar and Ma, Shiqian and Razaviyayn, Meisam},
  journal={arXiv preprint arXiv:2001.07819},
  year={2020}
}

@inproceedings{wang2018stochastic,
  title={Stochastic zeroth-order optimization in high dimensions},
  author={Wang, Yining and Du, Simon and Balakrishnan, Sivaraman and Singh, Aarti},
  booktitle={AISTATS},
  pages={1356--1365},
  year={2018},
}

@article{balasubramanian2018zeroth,
  title={Zeroth-order (non)-convex stochastic optimization via conditional gradient and gradient updates},
  author={Balasubramanian, Krishnakumar and Ghadimi, Saeed},
  journal={NIPS},
  volume={31},
  year={2018}
}

@article{cai2022zeroth,
  title={Zeroth-order regularized optimization (zoro): Approximately sparse gradients and adaptive sampling},
  author={Cai, HanQin and Mckenzie, Daniel and Yin, Wotao and Zhang, Zhenliang},
  journal={SIAM Journal on Optimization},
  volume={32},
  number={2},
  pages={687--714},
  year={2022},
  publisher={SIAM}
}

@article{bollapragada2018adaptive,
  title={Adaptive sampling strategies for stochastic optimization},
  author={Bollapragada, Raghu and Byrd, Richard and Nocedal, Jorge},
  journal={SIAM Journal on Optimization},
  volume={28},
  number={4},
  pages={3312--3343},
  year={2018},
  publisher={SIAM}
}

@inproceedings{ji2019improved,
  title={Improved zeroth-order variance reduced algorithms and analysis for nonconvex optimization},
  author={Ji, Kaiyi and Wang, Zhe and Zhou, Yi and Liang, Yingbin},
  booktitle={ICML},
  pages={3100--3109},
  year={2019},
}

@article{liu2018zeroth,
  title={Zeroth-order stochastic variance reduction for nonconvex optimization},
  author={Liu, Sijia and Kailkhura, Bhavya and Chen, Pin-Yu and Ting, Paishun and Chang, Shiyu and Amini, Lisa},
  journal={NIPS},
  volume={31},
  year={2018}
}

@article{lv2023full,
  title={Full Parameter Fine-tuning for Large Language Models with Limited Resources},
  author={Lv, Kai and Yang, Yuqing and Liu, Tengxiao and Gao, Qinghui and Guo, Qipeng and Qiu, Xipeng},
  journal={arXiv preprint arXiv:2306.09782},
  year={2023}
}

@article{luo2023came,
  title={CAME: Confidence-guided Adaptive Memory Efficient Optimization},
  author={Luo, Yang and Ren, Xiaozhe and Zheng, Zangwei and Jiang, Zhuo and Jiang, Xin and You, Yang},
  journal={arXiv preprint arXiv:2307.02047},
  year={2023}
}

@article{malladi2023fine,
  title={Fine-Tuning Language Models with Just Forward Passes},
  author={Malladi, Sadhika and Gao, Tianyu and Nichani, Eshaan and Damian, Alex and Lee, Jason D and Chen, Danqi and Arora, Sanjeev},
  journal={arXiv preprint arXiv:2305.17333},
  year={2023}
}

@article{spall1992multivariate,
  title={Multivariate stochastic approximation using a simultaneous perturbation gradient approximation},
  author={Spall, James C},
  journal={IEEE Transactions on Automatic Control},
  volume={37},
  number={3},
  pages={332--341},
  year={1992},
  publisher={IEEE}
}

@article{touvron2023llama,
  title={Llama 2: Open foundation and fine-tuned chat models},
  author={Touvron, Hugo and Martin, Louis and Stone, Kevin and Albert, Peter and Almahairi, Amjad and Babaei, Yasmine and Bashlykov, Nikolay and Batra, Soumya and Bhargava, Prajjwal and Bhosale, Shruti and others},
  journal={arXiv preprint arXiv:2307.09288},
  year={2023}
}

@article{touvron2023llama1,
  title={Llama: Open and efficient foundation language models},
  author={Touvron, Hugo and Lavril, Thibaut and Izacard, Gautier and Martinet, Xavier and Lachaux, Marie-Anne and Lacroix, Timoth{\'e}e and Rozi{\`e}re, Baptiste and Goyal, Naman and Hambro, Eric and Azhar, Faisal and others},
  journal={arXiv preprint arXiv:2302.13971},
  year={2023}
}

@article{rafailov2023direct,
  title={Direct preference optimization: Your language model is secretly a reward model},
  author={Rafailov, Rafael and Sharma, Archit and Mitchell, Eric and Ermon, Stefano and Manning, Christopher D and Finn, Chelsea},
  journal={arXiv preprint arXiv:2305.18290},
  year={2023}
}

@article{radford2019language,
  title={Language models are unsupervised multitask learners},
  author={Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya and others},
  journal={OpenAI blog},
  volume={1},
  number={8},
  pages={9},
  year={2019}
}

@article{shoeybi2019megatron,
  title={Megatron-lm: Training multi-billion parameter language models using model parallelism},
  author={Shoeybi, Mohammad and Patwary, Mostofa and Puri, Raul and LeGresley, Patrick and Casper, Jared and Catanzaro, Bryan},
  journal={arXiv preprint arXiv:1909.08053},
  year={2019}
}

@inproceedings{rasley2020deepspeed,
  title={Deepspeed: System optimizations enable training deep learning models with over 100 billion parameters},
  author={Rasley, Jeff and Rajbhandari, Samyam and Ruwase, Olatunji and He, Yuxiong},
  booktitle={SIGKDD},
  pages={3505--3506},
  year={2020}
}

@online{gibiansky2017, 
title={Bringing HPC Techniques to Deep Learning},
author={Andrew Giniansky},
url={https://andrew.gibiansky.com/blog/machine-learning/baidu-allreduce/}, 
year={2017},
} 

@inproceedings{mcmahan2017communication,
  title={Communication-efficient learning of deep networks from decentralized data},
  author={McMahan, Brendan and Moore, Eider and Ramage, Daniel and Hampson, Seth and y Arcas, Blaise Aguera},
  booktitle={AISTATS},
  pages={1273--1282},
  year={2017},
}

@article{reddi2020adaptive,
  title={Adaptive federated optimization},
  author={Reddi, Sashank and Charles, Zachary and Zaheer, Manzil and Garrett, Zachary and Rush, Keith and Kone{\v{c}}n{\`y}, Jakub and Kumar, Sanjiv and McMahan, H Brendan},
  journal={arXiv preprint arXiv:2003.00295},
  year={2020}
}

@misc{alpaca,
  author = {Rohan Taori and Ishaan Gulrajani and Tianyi Zhang and Yann Dubois and Xuechen Li and Carlos Guestrin and Percy Liang and Tatsunori B. Hashimoto },
  title = {Stanford Alpaca: An Instruction-following LLaMA model},
  year = {2023},
  publisher = {GitHub},
  journal = {GitHub repository},
  howpublished = {\url{https://github.com/tatsu-lab/stanford_alpaca}},
}

@article{javaheripi2phi,
  title={Phi-2: The surprising power of small language models},
  author={Javaheripi, Mojan and Bubeck, S{\'e}bastien and others},
  journal={URL https://www. microsoft. com/en-us/research/blog/phi-2-the-surprising-power-of-small-language-models},
  year={2023}
}

@ONLINE{wikidump,
    author = "Wikimedia Foundation",
    title  = "Wikimedia Downloads",
    url    = "https://dumps.wikimedia.org"
}

@article{gautam2024variance,
  title={Variance-reduced zeroth-order methods for fine-tuning language models},
  author={Gautam, Tanmay and Park, Youngsuk and Zhou, Hao and Raman, Parameswaran and Ha, Wooseok},
  year={2024}
}

@article{kaplan2020scaling,
  title={Scaling laws for neural language models},
  author={Kaplan, Jared and McCandlish, Sam and Henighan, Tom and Brown, Tom B and Chess, Benjamin and Child, Rewon and Gray, Scott and Radford, Alec and Wu, Jeffrey and Amodei, Dario},
  journal={arXiv preprint arXiv:2001.08361},
  year={2020}
}

@article{brown2020language,
  title={Language models are few-shot learners},
  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  journal={NeurIPS},
  volume={33},
  pages={1877--1901},
  year={2020}
}

@article{zhang2022opt,
  title={Opt: Open pre-trained transformer language models},
  author={Zhang, Susan and Roller, Stephen and Goyal, Naman and Artetxe, Mikel and Chen, Moya and Chen, Shuohui and Dewan, Christopher and Diab, Mona and Li, Xian and Lin, Xi Victoria and others},
  journal={arXiv preprint arXiv:2205.01068},
  year={2022}
}

@article{achiam2023gpt,
  title={Gpt-4 technical report},
  author={Achiam, Josh and Adler, Steven and Agarwal, Sandhini and Ahmad, Lama and Akkaya, Ilge and Aleman, Florencia Leoni and Almeida, Diogo and Altenschmidt, Janko and Altman, Sam and Anadkat, Shyamal and others},
  journal={arXiv preprint arXiv:2303.08774},
  year={2023}
}

@article{hoffmann2022training,
  title={Training compute-optimal large language models},
  author={Hoffmann, Jordan and Borgeaud, Sebastian and Mensch, Arthur and Buchatskaya, Elena and Cai, Trevor and Rutherford, Eliza and Casas, Diego de Las and Hendricks, Lisa Anne and Welbl, Johannes and Clark, Aidan and others},
  journal={arXiv preprint arXiv:2203.15556},
  year={2022}
}

@article{bi2024deepseek,
  title={Deepseek llm: Scaling open-source language models with longtermism},
  author={Bi, Xiao and Chen, Deli and Chen, Guanting and Chen, Shanhuang and Dai, Damai and Deng, Chengqi and Ding, Honghui and Dong, Kai and Du, Qiushi and Fu, Zhe and others},
  journal={arXiv preprint arXiv:2401.02954},
  year={2024}
}

@article{ouyang2022training,
  title={Training language models to follow instructions with human feedback},
  author={Ouyang, Long and Wu, Jeffrey and Jiang, Xu and Almeida, Diogo and Wainwright, Carroll and Mishkin, Pamela and Zhang, Chong and Agarwal, Sandhini and Slama, Katarina and Ray, Alex and others},
  journal={NeurIPS},
  volume={35},
  pages={27730--27744},
  year={2022}
}

@article{mccandlish2018empirical,
  title={An empirical model of large-batch training},
  author={McCandlish, Sam and Kaplan, Jared and Amodei, Dario and Team, OpenAI Dota},
  journal={arXiv preprint arXiv:1812.06162},
  year={2018}
}

@misc{Lee_2022, 
    title={Introducing the AI Research SuperCluster — Meta’s cutting-edge AI supercomputer for AI research}, 
    author={Kevin Lee and Shubho Sengupta}, 
    url={https://ai.meta.com/blog/ai-rsc/}, 
    year={2022}
} 

@Misc{swift,
  title = {SWIFT:Scalable lightWeight Infrastructure for Fine-Tuning},
  author = {The ModelScope Team},
  howpublished = {\url{https://github.com/modelscope/swift}},
  year = {2024}
}

@article{bai2022training,
  title={Training a helpful and harmless assistant with reinforcement learning from human feedback},
  author={Bai, Yuntao and Jones, Andy and Ndousse, Kamal and Askell, Amanda and Chen, Anna and DasSarma, Nova and Drain, Dawn and Fort, Stanislav and Ganguli, Deep and Henighan, Tom and others},
  journal={arXiv preprint arXiv:2204.05862},
  year={2022}
}

@article{paszke2019pytorch,
  title={Pytorch: An imperative style, high-performance deep learning library},
  author={Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and others},
  journal={NeurIPS},
  volume={32},
  year={2019}
}

@article{hendrycks2020measuring,
  title={Measuring massive multitask language understanding},
  author={Hendrycks, Dan and Burns, Collin and Basart, Steven and Zou, Andy and Mazeika, Mantas and Song, Dawn and Steinhardt, Jacob},
  journal={arXiv preprint arXiv:2009.03300},
  year={2020}
}

@article{bai2023qwen,
  title={Qwen technical report},
  author={Bai, Jinze and Bai, Shuai and Chu, Yunfei and Cui, Zeyu and Dang, Kai and Deng, Xiaodong and Fan, Yang and Ge, Wenbin and Han, Yu and Huang, Fei and others},
  journal={arXiv preprint arXiv:2309.16609},
  year={2023}
}

@inproceedings{killamsetty2021grad,
  title={Grad-match: Gradient matching based data subset selection for efficient deep model training},
  author={Killamsetty, Krishnateja and Durga, Sivasubramanian and Ramakrishnan, Ganesh and De, Abir and Iyer, Rishabh},
  booktitle={ICML},
  pages={5464--5474},
  year={2021},
}

@article{ash2019deep,
  title={Deep batch active learning by diverse, uncertain gradient lower bounds},
  author={Ash, Jordan T and Zhang, Chicheng and Krishnamurthy, Akshay and Langford, John and Agarwal, Alekh},
  journal={arXiv preprint arXiv:1906.03671},
  year={2019}
}

@inproceedings{killamsetty2021glister,
  title={Glister: Generalization based data subset selection for efficient and robust learning},
  author={Killamsetty, Krishnateja and Sivasubramanian, Durga and Ramakrishnan, Ganesh and Iyer, Rishabh},
  booktitle={AAAI},
  volume={35},
  number={9},
  pages={8110--8118},
  year={2021}
}

@inproceedings{mirzasoleiman2020coresets,
  title={Coresets for data-efficient training of machine learning models},
  author={Mirzasoleiman, Baharan and Bilmes, Jeff and Leskovec, Jure},
  booktitle={ICML},
  pages={6950--6960},
  year={2020},
}

@article{luo2024badam,
  title={BAdam: A Memory Efficient Full Parameter Training Method for Large Language Models},
  author={Luo, Qijun and Yu, Hengxu and Li, Xiao},
  journal={arXiv preprint arXiv:2404.02827},
  year={2024}
}

@article{liu2023sophia,
  title={Sophia: A scalable stochastic second-order optimizer for language model pre-training},
  author={Liu, Hong and Li, Zhiyuan and Hall, David and Liang, Percy and Ma, Tengyu},
  journal={arXiv preprint arXiv:2305.14342},
  year={2023}
}

@inproceedings{bisla2022low,
  title={Low-pass filtering sgd for recovering flat optima in the deep learning optimization landscape},
  author={Bisla, Devansh and Wang, Jing and Choromanska, Anna},
  booktitle={AISTATS},
  pages={8299--8339},
  year={2022},
}

@article{cutkosky2019momentum,
  title={Momentum-based variance reduction in non-convex sgd},
  author={Cutkosky, Ashok and Orabona, Francesco},
  journal={NeurIPS},
  volume={32},
  year={2019}
}

@inproceedings{nguyen2017sarah,
  title={SARAH: A novel method for machine learning problems using stochastic recursive gradient},
  author={Nguyen, Lam M and Liu, Jie and Scheinberg, Katya and Tak{\'a}{\v{c}}, Martin},
  booktitle={ICML},
  pages={2613--2621},
  year={2017},
}

@article{kunstner2023noise,
  title={Noise is not the main factor behind the gap between sgd and adam on transformers, but sign descent might be},
  author={Kunstner, Frederik and Chen, Jacques and Lavington, Jonathan Wilder and Schmidt, Mark},
  journal={arXiv preprint arXiv:2304.13960},
  year={2023}
}

@article{fu2023and,
  title={When and why momentum accelerates SGD: An empirical study},
  author={Fu, Jingwen and Wang, Bohan and Zhang, Huishuai and Zhang, Zhizheng and Chen, Wei and Zheng, Nanning},
  journal={arXiv preprint arXiv:2306.09000},
  year={2023}
}

@article{yuan2016influence,
  title={On the influence of momentum acceleration on online learning},
  author={Yuan, Kun and Ying, Bicheng and Sayed, Ali H},
  journal={Journal of Machine Learning Research},
  volume={17},
  number={192},
  pages={1--66},
  year={2016}
}

@article{yang2024qwen2,
  title={Qwen2 technical report},
  author={Yang, An and Yang, Baosong and Hui, Binyuan and Zheng, Bo and Yu, Bowen and Zhou, Chang and Li, Chengpeng and Li, Chengyuan and Liu, Dayiheng and Huang, Fei and others},
  journal={arXiv preprint arXiv:2407.10671},
  year={2024}
}

@article{dubey2024llama,
  title={The llama 3 herd of models},
  author={Dubey, Abhimanyu and Jauhri, Abhinav and Pandey, Abhinav and Kadian, Abhishek and Al-Dahle, Ahmad and Letman, Aiesha and Mathur, Akhil and Schelten, Alan and Yang, Amy and Fan, Angela and others},
  journal={arXiv preprint arXiv:2407.21783},
  year={2024}
}

@article{openaio1,
  title={Learning to Reason with LLMs},
  author={OpenAI},
  journal={https://openai.com/index/learning-to-reason-with-llms/},
  year={2024}
}

@article{pham2023combined,
  title={Combined scaling for zero-shot transfer learning},
  author={Pham, Hieu and Dai, Zihang and Ghiasi, Golnaz and Kawaguchi, Kenji and Liu, Hanxiao and Yu, Adams Wei and Yu, Jiahui and Chen, Yi-Ting and Luong, Minh-Thang and Wu, Yonghui and others},
  journal={Neurocomputing},
  volume={555},
  pages={126658},
  year={2023},
  publisher={Elsevier}
}

@article{zhang2024adam,
  title={Adam-mini: Use Fewer Learning Rates To Gain More},
  author={Zhang, Yushun and Chen, Congliang and Li, Ziniu and Ding, Tian and Wu, Chenwei and Ye, Yinyu and Luo, Zhi-Quan and Sun, Ruoyu},
  journal={arXiv preprint arXiv:2406.16793},
  year={2024}
}

@article{tang2020dureader_robust,
  title={DuReader\_robust: A Chinese dataset towards evaluating robustness and generalization of machine reading comprehension in real-world applications},
  author={Tang, Hongxuan and Li, Hongyu and Liu, Jing and Hong, Yu and Wu, Hua and Wang, Haifeng},
  journal={arXiv preprint arXiv:2004.11142},
  year={2020}
}

@misc{qwen1.5,
    title = {Introducing Qwen1.5},
    url = {https://qwenlm.github.io/blog/qwen1.5/},
    author = {Qwen Team},
    month = {February},
    year = {2024}
}

@article{zhao2024galore,
  title={Galore: Memory-efficient llm training by gradient low-rank projection},
  author={Zhao, Jiawei and Zhang, Zhenyu and Chen, Beidi and Wang, Zhangyang and Anandkumar, Anima and Tian, Yuandong},
  journal={arXiv preprint arXiv:2403.03507},
  year={2024}
}

@article{cao2024grams,
  title={Grams: Gradient descent with adaptive momentum scaling},
  author={Cao, Yang and Li, Xiaoyu and Song, Zhao},
  journal={arXiv preprint arXiv:2412.17107},
  year={2024}
}