@inproceedings{av22,
  title={Building normalizing flows with stochastic interpolants},
  author={Albergo, Michael S and Vanden-Eijnden, Eric},
  booktitle={ICLR},
  year={2023}
}

@inproceedings{rectified_flow,
  title={Flow straight and fast: Learning to generate and transfer data with rectified flow},
  author={Liu, Xingchao and Gong, Chengyue and Liu, Qiang},
  booktitle={ICLR},
  year={2023}
}

@article{k11,
  title={Equivalence of stochastic equations and martingale problems},
  author={Kurtz, Thomas G},
  journal={Stochastic analysis 2010},
  pages={113--130},
  year={2011},
  publisher={Springer}
}



@book{acd+08,
  title={Transport equations and multi-D hyperbolic conservation laws},
  author={Ambrosio, Luigi and Crippa, Gianluca and De Lellis, Camillo and Otto, Felix and Westdickenberg, Michael},
  volume={5},
  year={2008},
  publisher={Springer}
}

@article{dpm-solver,
  title={Dpm-solver: A fast ode solver for diffusion probabilistic model sampling in around 10 steps},
  author={Lu, Cheng and Zhou, Yuhao and Bao, Fan and Chen, Jianfei and Li, Chongxuan and Zhu, Jun},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={5775--5787},
  year={2022}
}

@article{lhh+24,
  title={Flow Matching Guide and Code},
  author={Lipman, Yaron and Havasi, Marton and Holderrieth, Peter and Shaul, Neta and Le, Matt and Karrer, Brian and Chen, Ricky TQ and Lopez-Paz, David and Ben-Hamu, Heli and Gat, Itai},
  journal={arXiv preprint arXiv:2412.06264},
  year={2024}
}

@inproceedings{llly24,
  title={Common diffusion noise schedules and sample steps are flawed},
  author={Lin, Shanchuan and Liu, Bingchen and Li, Jiashi and Yang, Xiao},
  booktitle={WACV},
  year={2024}
}

@article{hg24,
  title={Improved noise schedule for diffusion training},
  author={Hang, Tiankai and Gu, Shuyang},
  journal={arXiv preprint arXiv:2407.03297},
  year={2024}
}

@article{c23,
  title={On the importance of noise scheduling for diffusion models},
  author={Chen, Ting},
  journal={arXiv preprint arXiv:2301.10972},
  year={2023}
}

@inproceedings{hwa+24,
  title={Flow Matching for Conditional Text Generation in a Few Sampling Steps},
  author={Hu, Vincent and Wu, Di and Asano, Yuki and Mettes, Pascal and Fernando, Basura and Ommer, Bj{\"o}rn and Snoek, Cees},
  booktitle={EACL},
  year={2024}
}

@inproceedings{wet+24,
  title={An Investigation of Noise Robustness for Flow-Matching-Based Zero-Shot {TTS}},
  author={Wang, Xiaofei and Eskimez, Sefik Emre and Thakker, Manthan and Yang, Hemin and Zhu, Zirun and Tang, Min and Xia, Yufei and Li, Jinzhu and Zhao, Sheng and Li, Jinyu and others},
  booktitle={Interspeech},
  year={2024}
}

@inproceedings{bah+23,
  title={{SE(3)}-stochastic flow matching for protein backbone generation},
  author={Bose, Avishek Joey and Akhound-Sadegh, Tara and Huguet, Guillaume and Fatras, Kilian and Rector-Brooks, Jarrid and Liu, Cheng-Hao and Nica, Andrei Cristian and Korablyov, Maksym and Bronstein, Michael and Tong, Alexander},
  booktitle={ICLR},
  year={2024}
}

@inproceedings{ekb+24,
  title={Scaling rectified flow transformers for high-resolution image synthesis},
  author={Esser, Patrick and Kulal, Sumith and Blattmann, Andreas and Entezari, Rahim and M{\"u}ller, Jonas and Saini, Harry and Levi, Yam and Lorenz, Dominik and Sauer, Axel and Boesel, Frederic and others},
  booktitle={ICML},
  year={2024}
}

@book{e10_pde,
  title={Partial Differential Equations},
  author={Evans, Lawrence C.},
  series={Graduate Studies in Mathematics},
  volume={19},
  year={2010},
  publisher={American Mathematical Society}
}

@book{sb_14_book,
  title={Understanding machine learning: From theory to algorithms},
  author={Shalev-Shwartz, Shai and Ben-David, Shai},
  year={2014},
  publisher={Cambridge university press}
}

@inproceedings{flow_matching,
  title={Flow matching for generative modeling},
  author={Lipman, Yaron and Chen, Ricky TQ and Ben-Hamu, Heli and Nickel, Maximilian and Le, Matt},
  booktitle={ICLR},
  year={2023}
}

@article{bm02,
  title={Rademacher and Gaussian complexities: Risk bounds and structural results},
  author={Bartlett, Peter L and Mendelson, Shahar},
  journal={Journal of Machine Learning Research},
  volume={3},
  number={Nov},
  pages={463--482},
  year={2002}
}

@article{lsw+24,
  title={How to Inverting the Leverage Score Distribution?},
  author={Li, Zhihang and Song, Zhao and Wang, Weixin and Yin, Junze and Yu, Zheng},
  journal={arXiv preprint arXiv:2404.13785},
  year={2024}
}

@article{l11,
  title={On the spectral radius and the spectral norm of Hadamard products of nonnegative matrices},
  author={Huang, Zejun},
  journal={Linear algebra and its applications},
  volume={434},
  number={2},
  pages={457--462},
  year={2011},
  publisher={Elsevier}
}

@article{dls23,
  title={Attention scheme inspired softmax regression},
  author={Deng, Yichuan and Li, Zhihang and Song, Zhao},
  journal={arXiv preprint arXiv:2304.10411},
  year={2023}
}

@article{lsz23,
  title={Solving regularized exp, cosh and sinh regression problems},
  author={Li, Zhihang and Song, Zhao and Zhou, Tianyi},
  journal={arXiv preprint arXiv:2303.15725},
  year={2023}
}

@article{syyz22,
  title={Faster algorithm for structured john ellipsoid computation},
  author={Song, Zhao and Yang, Xin and Yang, Yuanyuan and Zhou, Tianyi},
  journal={arXiv preprint arXiv:2211.14407},
  year={2022}
}

@article{dsw22,
  title={Discrepancy minimization in input-sparsity time},
  author={Deng, Yichuan and Song, Zhao and Weinstein, Omri},
  journal={arXiv preprint arXiv:2210.12468},
  year={2022}
}


@article{lswy23,
  title={Local Convergence of Approximate Newton Method for Two Layer Nonlinear Regression},
  author={Li, Zhihang and Song, Zhao and Wang, Zifan and Yin, Junze},
  journal={arXiv preprint arXiv:2311.15390},
  year={2023}
}

@article{swy23,
  title={A unified scheme of resnet and softmax},
  author={Song, Zhao and Wang, Weixin and Yin, Junze},
  journal={arXiv preprint arXiv:2309.13482},
  year={2023}
}

@inproceedings{gswy23,
  title={A fast optimization view: Reformulating single layer attention in llm based on tensor and svm trick, and solving it in matrix multiplication time},
  author={Gao, Yeqi and Song, Zhao and Wang, Weixin and Yin, Junze},
  booktitle={UAI},
  year={2025}
}

@article{zlh19,
  title={Deep leakage from gradients},
  author={Zhu, Ligeng and Liu, Zhijian and Han, Song},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@inproceedings{cle+19,
  title={The secret sharer: Evaluating and testing unintended memorization in neural networks},
  author={Carlini, Nicholas and Liu, Chang and Erlingsson, {\'U}lfar and Kos, Jernej and Song, Dawn},
  booktitle={28th USENIX Security Symposium (USENIX Security 19)},
  pages={267--284},
  year={2019}
}





@article{b65,
  title={A class of methods for solving nonlinear simultaneous equations},
  author={Broyden, Charles G},
  journal={Mathematics of computation},
  volume={19},
  number={92},
  pages={577--593},
  year={1965}
}

@book{nw99,
  title={Numerical optimization},
  author={Nocedal, Jorge and Wright, Stephen J},
  year={1999},
  publisher={Springer}
}

@inproceedings{zz23,
  title={A Hyperbolic Extension of Kadison-Singer Type Results},
  author={Zhang, Ruizhe and Zhang, Xinzhi},
  booktitle={ICALP},
  year={2023}
}

@article{bsy23,
  title={Federated Empirical Risk Minimization via Second-Order Method},
  author={Bian, Song and Song, Zhao and Yin, Junze},
  journal={arXiv preprint arXiv:2305.17482},
  year={2023}
}

@article{bs23,
    author = {Brand, Jan den van and Song, Zhao},
    title = {A $\sqrt{n}$ passes streaming algorithm for solving bipartite matching exactly},
    journal = {Manuscript},
    year = {2023}
}

@inproceedings{dwz23,
  title = {Faster Matrix Multiplication via Asymmetric Hashing},
  author = {Duan, Ran and Wu, Hongxun and Zhou, Renfei},
  booktitle = {FOCS},
  year = {2023}
}

@article{lg23,
  title={Faster Rectangular Matrix Multiplication by Combination Loss Analysis},
  author={Le Gall, Fran{\c{c}}ois },
  journal={arXiv preprint arXiv:2307.06535},
  year={2023}
}

@inproceedings{wxxz23,
  title={New bounds for matrix multiplication: from alpha to omega},
  author={Williams, Virginia Vassilevska and Xu, Yinzhan and Xu, Zixuan and Zhou, Renfei},
  booktitle={Proceedings of the 2024 Annual ACM-SIAM Symposium on Discrete Algorithms (SODA)},
  pages={3792--3835},
  year={2024},
  organization={SIAM}
}

@article{dszz23,
  title={Efficient Algorithm for Solving Hyperbolic Programs},
  author={Deng, Yichuan and Song, Zhao and Zhang, Lichen and Zhang, Ruizhe},
  journal={arXiv preprint arXiv:2306.07587},
  year={2023}
}

@inproceedings{lsz+23,
  title = {Space-efficient interior point method, with applications to linear programming and maximum weight bipartite matching},
  author = {Liu, S. Cliff and Song, Zhao and Zhang, Hengjie and Zhang, Lichen and Zhou, Tianyi},
  booktitle = {International Colloquium on Automata, Languages and Programming (ICALP)},
  pages = {88:1--88:14},
  year = {2023}
}


@article{syz23_sdp,
    author = {Zhao Song and Mingquan Ye and Lichen Zhang},
    title = {Streaming Semidefinite Programs: $O(\sqrt{n})$ Passes, Small Space and Fast Runtime},
    journal = {Manuscript},
    year = {2023}
}

@inproceedings{s90,
  title={A weighted-least-squares matrix decomposition method with application to the design of two-dimensional digital filters},
  author={Shpak, Dale J},
  booktitle={Proceedings of the 33rd Midwest Symposium on Circuits and Systems},
  pages={1070--1073},
  year={1990},
  organization={IEEE}
}

@inproceedings{la03,
  title={New method for weighted low-rank approximation of complex-valued matrices and its application for the design of 2-d digital filters},
  author={Lu, Wu-Sheng and Antoniou, Andreas},
  booktitle={Proceedings of the 2003 International Symposium on Circuits and Systems, 2003. ISCAS'03.},
  volume={3},
  pages={III--III},
  year={2003},
  organization={IEEE}
}
@article{lpw97,
  title={Weighted low-rank approximation of general complex matrices and its application in the design of 2-D digital filters},
  author={Lu, W-S and Pei, S-C and Wang, P-H},
  journal={IEEE Transactions on Circuits and Systems I: Fundamental Theory and Applications},
  volume={44},
  number={7},
  pages={650--655},
  year={1997},
  publisher={IEEE}
}

@inproceedings{jlsw20,
  title={An improved cutting plane method for convex optimization, convex-concave games, and its applications},
  author={Jiang, Haotian and Lee, Yin Tat and Song, Zhao and Wong, Sam Chiu-wai},
  booktitle={Proceedings of the 52nd Annual ACM SIGACT Symposium on Theory of Computing},
  pages={944--953},
  year={2020}
}


@inproceedings{lsw15,
  title={A faster cutting plane method and its implications for combinatorial and convex optimization},
  author={Lee, Yin Tat and Sidford, Aaron and Wong, Sam Chiu-wai},
  booktitle={2015 IEEE 56th Annual Symposium on Foundations of Computer Science},
  pages={1049--1065},
  year={2015},
  organization={IEEE}
}


@inproceedings{clz+15,
  title={WEMAREC: Accurate and scalable recommendation through weighted and ensemble matrix approximation},
  author={Chen, Chao and Li, Dongsheng and Zhao, Yingying and Lv, Qin and Shang, Li},
  booktitle={Proceedings of the 38th international ACM SIGIR conference on research and development in information retrieval},
  pages={303--312},
  year={2015}
}



@article{y41,
  title={Maximum likelihood estimation and factor analysis},
  author={Young, Gale},
  journal={Psychometrika},
  volume={6},
  number={1},
  pages={49--53},
  year={1941},
  publisher={Springer}
}

@inproceedings{s90,
  title={A weighted-least-squares matrix decomposition method with application to the design of two-dimensional digital filters},
  author={Shpak, Dale J},
  booktitle={Proceedings of the 33rd Midwest Symposium on Circuits and Systems},
  pages={1070--1073},
  year={1990},
  organization={IEEE}
}

@inproceedings{la03,
  title={New method for weighted low-rank approximation of complex-valued matrices and its application for the design of 2-d digital filters},
  author={Lu, Wu-Sheng and Antoniou, Andreas},
  booktitle={Proceedings of the 2003 International Symposium on Circuits and Systems, 2003. ISCAS'03.},
  volume={3},
  pages={III--III},
  year={2003},
  organization={IEEE}
}
@article{lpw97,
  title={Weighted low-rank approximation of general complex matrices and its application in the design of 2-D digital filters},
  author={Lu, W-S and Pei, S-C and Wang, P-H},
  journal={IEEE Transactions on Circuits and Systems I: Fundamental Theory and Applications},
  volume={44},
  number={7},
  pages={650--655},
  year={1997},
  publisher={IEEE}
}


@inproceedings{emz21,
  title={Almost linear time density level set estimation via DBSCAN},
  author={Esfandiari, Hossein and Mirrokni, Vahab and Zhong, Peilin},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={35},
  pages={7349--7357},
  year={2021}
}

@inproceedings{lsz19,
  title={Solving empirical risk minimization in the current matrix multiplication time},
  author={Lee, Yin Tat and Song, Zhao and Zhang, Qiuyi},
  booktitle={Conference on Learning Theory},
  pages={2140--2157},
  year={2019},
  organization={PMLR}
}

@inproceedings{jswz21,
  title={A faster algorithm for solving general LPs},
  author={Jiang, Shunhua and Song, Zhao and Weinstein, Omri and Zhang, Hengjie},
  booktitle={Proceedings of the 53rd Annual ACM SIGACT Symposium on Theory of Computing},
  pages={823--832},
  year={2021}
}

@inproceedings{sy21,
  title={Oblivious sketching-based central path method for solving linear programming problems},
  author={Song, Zhao and Yu, Zheng},
  year={2021},
  booktitle={38th International Conference on Machine Learning (ICML)}
}


 
@article{gsz23,
  title={A nearly-linear time algorithm for structured support vector machines},
  author={Gu, Yuzhou and Song, Zhao and Zhang, Lichen},
  journal={arXiv preprint arXiv:2307.07735},
  year={2023}
}


@inproceedings{b20,
  title={A deterministic linear program solver in current matrix multiplication time},
  author={ Brand, Jan van den},
  booktitle={Proceedings of the Fourteenth Annual ACM-SIAM Symposium on Discrete Algorithms (SODA)},
  pages={259--278},
  year={2020},
  organization={SIAM}
}

@article{gs22,
  title={A faster small treewidth sdp solver},
  author={Gu, Yuzhou and Song, Zhao},
  journal={arXiv preprint arXiv:2211.06033},
  year={2022}
}

@article{jlsz23,
title={Convex Minimization with Integer Minima in $\widetilde{O}(n^4)$ Time},
author={Haotian Jiang and Yin Tat Lee and Zhao Song and Lichen Zhang},
journal={arXiv preprint arXiv:2304.03426},
year={2023}
}



@inproceedings{hlz23,
  title={Upper and Lower Bounds on the Smoothed Complexity of the Simplex Method},
  author={Huiberts, Sophie and Lee, Yin Tat and Zhang, Xinzhi},
  booktitle={Proceedings of the 55th Annual ACM Symposium on Theory of Computing},
  pages={1904--1917},
  year={2023}
}


@inproceedings{cls19,
  title={Solving Linear Programs in the Current Matrix Multiplication Time},
  author={Cohen, Michael B and Lee, Yin Tat and Song, Zhao},
  booktitle= {STOC},
  year={2019}
}

@inproceedings{qszz23,
  title={An online and unified algorithm for projection matrix vector multiplication with application to empirical risk minimization},
  author={Qin, Lianke and Song, Zhao and Zhang, Lichen and Zhuo, Danyang},
  booktitle={International Conference on Artificial Intelligence and Statistics (AISTATS)},
  pages={101--156},
  year={2023},
  organization={PMLR}
}

@article{bsz23,
  title={Algorithm and Hardness for Dynamic Attention Maintenance in Large Language Models},
  author={Brand, Jan van den and Song, Zhao and Zhou, Tianyi},
  journal={arXiv preprint arXiv:2304.02207},
  year={2023}
}

 

 





@article{b15,
  title={Convex optimization: Algorithms and complexity},
  author={Bubeck, S{\'e}bastien and others},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={8},
  number={3-4},
  pages={231--357},
  year={2015},
  publisher={Now Publishers, Inc.}
}

@book{bv04,
  title={Convex optimization},
  author={Boyd, Stephen P and Vandenberghe, Lieven},
  year={2004},
  publisher={Cambridge university press}
}

@inproceedings{akk+20,
  title={Leverage score sampling for faster accelerated regression and ERM},
  author={Agarwal, Naman and Kakade, Sham and Kidambi, Rahul and Lee, Yin-Tat and Netrapalli, Praneeth and Sidford, Aaron},
  booktitle={Algorithmic Learning Theory},
  pages={22--47},
  year={2020},
  organization={PMLR}
}

@inproceedings{ss08,
  title={Graph sparsification by effective resistances},
  author={Spielman, Daniel A and Srivastava, Nikhil},
  booktitle={Proceedings of the fortieth annual ACM symposium on Theory of computing},
  pages={563--568},
  year={2008}
}


@inproceedings{lg14,
  title={Powers of tensors and fast matrix multiplication},
  author={Le Gall, Fran{\c{c}}ois},
  booktitle={Proceedings of the 39th international symposium on symbolic and algebraic computation},
  pages={296--303},
  year={2014}
}

@inproceedings{w12,
  title={Multiplying matrices faster than Coppersmith-Winograd},
  author={Williams, Virginia Vassilevska},
  booktitle={Proceedings of the forty-fourth annual ACM symposium on Theory of computing},
  pages={887--898},
  year={2012}
}

@inproceedings{aw21,
  title={A refined laser method and faster matrix multiplication},
  author={Alman, Josh and Williams, Virginia Vassilevska},
  booktitle={Proceedings of the 2021 ACM-SIAM Symposium on Discrete Algorithms (SODA)},
  pages={522--539},
  year={2021},
  organization={SIAM}
}


@article{a00,
  title={The volumetric barrier for semidefinite programming},
  author={Anstreicher, Kurt M},
  journal={Mathematics of Operations Research},
  year={2000},
  publisher={INFORMS}
}

@inproceedings{jkl+20,
  title={A faster interior point method for semidefinite programming},
  author={Jiang, Haotian and Kathuria, Tarun and Lee, Yin Tat and Padmanabhan, Swati and Song, Zhao},
  booktitle={2020 IEEE 61st annual symposium on foundations of computer science (FOCS)},
  pages={910--918},
  year={2020},
  organization={IEEE}
}

@inproceedings{bpsw21,
  title={Training (overparametrized) neural networks in near-linear time},
  author={Brand, Jan van den and Peng, Binghui and Song, Zhao and Weinstein, Omri},
  booktitle={ITCS},
  year={2021}
}

@article{szz21,
  title={Training multi-layer over-parametrized neural network in subquadratic time},
  author={Song, Zhao and Zhang, Lichen and Zhang, Ruizhe},
  journal={arXiv preprint arXiv:2112.07628},
  year={2021}
}


@inproceedings{hjs+22,
  title={Solving sdp faster: A robust ipm framework and efficient implementation},
  author={Huang, Baihe and Jiang, Shunhua and Song, Zhao and Tao, Runzhou and Zhang, Ruizhe},
  booktitle={2022 IEEE 63rd Annual Symposium on Foundations of Computer Science (FOCS)},
  pages={233--244},
  year={2022},
  organization={IEEE}
}



@inproceedings{cy21,
  title={Fast statistical leverage score approximation in kernel ridge regression},
  author={Chen, Yifan and Yang, Yun},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2935--2943},
  year={2021},
  organization={PMLR}
}

@article{m18,
  title={Ridge regression and provable deterministic ridge leverage score sampling},
  author={McCurdy, Shannon},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}

@article{m11,
  title={Randomized algorithms for matrices and data},
  author={Mahoney, Michael W and others},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={3},
  number={2},
  pages={123--224},
  year={2011},
  publisher={Now Publishers, Inc.}
}


@article{dmmw12,
  title={Fast approximation of matrix coherence and statistical leverage},
  author={Drineas, Petros and Magdon-Ismail, Malik and Mahoney, Michael W and Woodruff, David P},
  journal={The Journal of Machine Learning Research},
  volume={13},
  number={1},
  pages={3475--3506},
  year={2012},
  publisher={JMLR. org}
}



leverage score

@inproceedings{swz19,
  title={Relative error tensor low rank approximation},
  author={Song, Zhao and Woodruff, David P and Zhong, Peilin},
  booktitle={Proceedings of the Thirtieth Annual ACM-SIAM Symposium on Discrete Algorithms},
  pages={2772--2789},
  year={2019},
  organization={SIAM}
}

@inproceedings{swz17,
  title={Low rank approximation with entrywise l1-norm error},
  author={Song, Zhao and Woodruff, David P and Zhong, Peilin},
  booktitle={Proceedings of the 49th Annual ACM SIGACT Symposium on Theory of Computing},
  pages={688--701},
  year={2017}
}

@article{ch86,
  title={Influential observations, high leverage points, and outliers in linear regression},
  author={Chatterjee, Samprit and Hadi, Ali S},
  journal={Statistical science},
  pages={379--393},
  year={1986},
  publisher={JSTOR}
}

@article{dkm06,
  title={Fast Monte Carlo algorithms for matrices I: Approximating matrix multiplication},
  author={Drineas, Petros and Kannan, Ravi and Mahoney, Michael W},
  journal={SIAM Journal on Computing},
  volume={36},
  number={1},
  pages={132--157},
  year={2006},
  publisher={SIAM}
}

@inproceedings{dmm06,
  title={Sampling algorithms for $l_2$ regression and applications},
  author={Drineas, Petros and Mahoney, Michael W and Muthukrishnan, Shan},
  booktitle={Proceedings of the seventeenth annual ACM-SIAM symposium on Discrete algorithm},
  pages={1127--1136},
  year={2006}
}

@article{md09,
  title={CUR matrix decompositions for improved data analysis},
  author={Mahoney, Michael W and Drineas, Petros},
  journal={Proceedings of the National Academy of Sciences},
  volume={106},
  number={3},
  pages={697--702},
  year={2009},
  publisher={National Acad Sciences}
}

@article{dmm08,
  title={Relative-error CUR matrix decompositions},
  author={Drineas, Petros and Mahoney, Michael W and Muthukrishnan, Shan},
  journal={SIAM Journal on Matrix Analysis and Applications},
  volume={30},
  number={2},
  pages={844--881},
  year={2008},
  publisher={SIAM}
}

@inproceedings{bmd09,
  title={An improved approximation algorithm for the column subset selection problem},
  author={Boutsidis, Christos and Mahoney, Michael W and Drineas, Petros},
  booktitle={Proceedings of the twentieth annual ACM-SIAM symposium on Discrete algorithms},
  pages={968--977},
  year={2009},
  organization={SIAM}
}

@inproceedings{pkb14,
  title={Provable deterministic leverage score sampling},
  author={Papailiopoulos, Dimitris and Kyrillidis, Anastasios and Boutsidis, Christos},
  booktitle={Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining},
  pages={997--1006},
  year={2014}
}

@inproceedings{gs12,
  title={Optimal column-based low-rank matrix reconstruction},
  author={Guruswami, Venkatesan and Sinop, Ali Kemal},
  booktitle={Proceedings of the twenty-third annual ACM-SIAM symposium on Discrete Algorithms},
  pages={1207--1214},
  year={2012},
  organization={SIAM}
}

@article{bdm14,
  title={Near-optimal column-based matrix reconstruction},
  author={Boutsidis, Christos and Drineas, Petros and Magdon-Ismail, Malik},
  journal={SIAM Journal on Computing},
  volume={43},
  number={2},
  pages={687--717},
  year={2014},
  publisher={SIAM}
}

@article{cm12,
  title={Column subset selection via sparse approximation of SVD},
  author={Civril, Ali and Magdon-Ismail, Malik},
  journal={Theoretical Computer Science},
  volume={421},
  pages={1--14},
  year={2012},
  publisher={Elsevier}
}

@inproceedings{abf+16,
  title={Greedy column subset selection: New bounds and distributed algorithms},
  author={Altschuler, Jason and Bhaskara, Aditya and Fu, Gang and Mirrokni, Vahab and Rostamizadeh, Afshin and Zadimoghaddam, Morteza},
  booktitle={International conference on machine learning},
  pages={2539--2548},
  year={2016},
  organization={PMLR}
}

@inproceedings{fgk11,
  title={An efficient greedy method for unsupervised feature selection},
  author={Farahat, Ahmed K and Ghodsi, Ali and Kamel, Mohamed S},
  booktitle={2011 IEEE 11th International Conference on Data Mining},
  pages={161--170},
  year={2011},
  organization={IEEE}
}

@book{ss02,
  title={Learning with kernels: support vector machines, regularization, optimization, and beyond},
  author={Sch{\"o}lkopf, Bernhard and Smola, Alexander J},
  year={2002},
  publisher={MIT press}
}

@article{cv95,
  title={Support-vector networks},
  author={Cortes, Corinna and Vapnik, Vladimir},
  journal={Machine learning},
  volume={20},
  pages={273--297},
  year={1995},
  publisher={Springer}
}

@article{am15,
  title={Fast randomized kernel ridge regression with statistical guarantees},
  author={Alaoui, Ahmed and Mahoney, Michael W},
  journal={Advances in neural information processing systems},
  volume={28},
  year={2015}
}

@article{mm17,
  title={Recursive sampling for the nystrom method},
  author={Musco, Cameron and Musco, Christopher},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@inproceedings{clv17,
  title={Distributed adaptive sampling for kernel matrix approximation},
  author={Calandriello, Daniele and Lazaric, Alessandro and Valko, Michal},
  booktitle={Artificial Intelligence and Statistics},
  pages={1421--1429},
  year={2017},
  organization={PMLR}
}

@inproceedings{ltos19,
  title={Towards a unified analysis of random Fourier features},
  author={Li, Zhu and Ton, Jean-Francois and Oglic, Dino and Sejdinovic, Dino},
  booktitle={International conference on machine learning},
  pages={3905--3914},
  year={2019},
  organization={PMLR}
}

@article{emm20,
  title={Fourier sparse leverage scores and approximate kernel learning},
  author={Erd{\'e}lyi, Tam{\'a}s and Musco, Cameron and Musco, Christopher},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={109--122},
  year={2020}
}

@inproceedings{lhc+20,
  title={Random Fourier features via fast surrogate leverage weighted sampling},
  author={Liu, Fanghui and Huang, Xiaolin and Chen, Yudong and Yang, Jie and Suykens, Johan},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={34},
  number={04},
  pages={4844--4851},
  year={2020}
}

@article{lhcs21,
  title={Random features for kernel approximation: A survey on algorithms, theory, and beyond},
  author={Liu, Fanghui and Huang, Xiaolin and Chen, Yudong and Suykens, Johan AK},
  journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
  volume={44},
  number={10},
  pages={7128--7148},
  year={2021},
  publisher={IEEE}
}


@article{mt20,
  title={Randomized numerical linear algebra: Foundations and algorithms},
  author={Martinsson, Per-Gunnar and Tropp, Joel A},
  journal={Acta Numerica},
  volume={29},
  pages={403--572},
  year={2020},
  publisher={Cambridge University Press}
}

@article{gsyz23,
  title={Low rank matrix completion via robust alternating minimization in nearly linear time},
  author={Gu, Yuzhou and Song, Zhao and Yin, Junze and Zhang, Lichen},
  journal={arXiv preprint arXiv:2302.11068},
  year={2023}
}

@article{syyz23,
  title={Efficient Alternating Minimization with Applications to Weighted Low Rank Approximation},
  author={Song, Zhao and Ye, Mingquan and Yin, Junze and Zhang, Lichen},
  journal={arXiv preprint arXiv:2306.04169},
  year={2023}
}


@article{css+23,
  title={Query Complexity of Active Learning for Function Family With Nearly Orthogonal Basis},
  author={Chen, Xiang and Song, Zhao and Sun, Baocheng and Yin, Junze and Zhuo, Danyang},
  journal={arXiv preprint arXiv:2306.03356},
  year={2023}
}


attack

@article{szs13,
  title={Intriguing properties of neural networks},
  author={Szegedy, Christian and Zaremba, Wojciech and Sutskever, Ilya and Bruna, Joan and Erhan, Dumitru and Goodfellow, Ian and Fergus, Rob},
  journal={arXiv preprint arXiv:1312.6199},
  year={2013}
}


@article{mms+17,
  title={Towards deep learning models resistant to adversarial attacks},
  author={Madry, Aleksander and Makelov, Aleksandar and Schmidt, Ludwig and Tsipras, Dimitris and Vladu, Adrian},
  journal={arXiv preprint arXiv:1706.06083},
  year={2017}
}

@article{gss14,
  title={Explaining and harnessing adversarial examples},
  author={Goodfellow, Ian J and Shlens, Jonathon and Szegedy, Christian},
  journal={arXiv preprint arXiv:1412.6572},
  year={2014}
}

@inproceedings{zyj+19,
  title={Theoretically principled trade-off between robustness and accuracy},
  author={Zhang, Hongyang and Yu, Yaodong and Jiao, Jiantao and Xing, Eric and El Ghaoui, Laurent and Jordan, Michael},
  booktitle={International conference on machine learning},
  pages={7472--7482},
  year={2019},
  organization={PMLR}
}

@article{gmp+17,
  title={On the (statistical) detection of adversarial examples},
  author={Grosse, Kathrin and Manoharan, Praveen and Papernot, Nicolas and Backes, Michael and McDaniel, Patrick},
  journal={arXiv preprint arXiv:1702.06280},
  year={2017}
}

@article{fcsg17,
  title={Detecting adversarial samples from artifacts},
  author={Feinman, Reuben and Curtin, Ryan R and Shintre, Saurabh and Gardner, Andrew B},
  journal={arXiv preprint arXiv:1703.00410},
  year={2017}
}

@inproceedings{fjr15,
  title={Model inversion attacks that exploit confidence information and basic countermeasures},
  author={Fredrikson, Matt and Jha, Somesh and Ristenpart, Thomas},
  booktitle={Proceedings of the 22nd ACM SIGSAC conference on computer and communications security},
  pages={1322--1333},
  year={2015}
}

@inproceedings{ssss17,
  author={Shokri, Reza and Stronati, Marco and Song, Congzheng and Shmatikov, Vitaly},
  booktitle={2017 IEEE Symposium on Security and Privacy (SP)}, 
  title={Membership Inference Attacks Against Machine Learning Models}, 
  year={2017},
}

@article{dr14,
  title={The algorithmic foundations of differential privacy},
  author={Dwork, Cynthia and Roth, Aaron and others},
  journal={Foundations and Trends{\textregistered} in Theoretical Computer Science},
  volume={9},
  number={3--4},
  pages={211--407},
  year={2014},
  publisher={Now Publishers, Inc.}
}

@inproceedings{je19,
  title={Evaluating differentially private machine learning in practice},
  author={Jayaraman, Bargav and Evans, David},
  booktitle={28th USENIX Security Symposium (USENIX Security 19)},
  pages={1895--1912},
  year={2019}
}

@article{lllw21,
  title={Differentially private ensemble learning for classification},
  author={Li, Xianxian and Liu, Jing and Liu, Songfeng and Wang, Jinyan},
  journal={Neurocomputing},
  volume={430},
  pages={34--46},
  year={2021},
  publisher={Elsevier}
}


inverting

@article{cwd+18,
  title={Generative adversarial networks: An overview},
  author={Creswell, Antonia and White, Tom and Dumoulin, Vincent and Arulkumaran, Kai and Sengupta, Biswa and Bharath, Anil A},
  journal={IEEE signal processing magazine},
  volume={35},
  number={1},
  pages={53--65},
  year={2018},
  publisher={IEEE}
}

@inproceedings{db16,
  title={Inverting visual representations with convolutional networks},
  author={Dosovitskiy, Alexey and Brox, Thomas},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={4829--4837},
  year={2016}
}

@article{lluz16,
  title={Understanding the effective receptive field in deep convolutional neural networks},
  author={Luo, Wenjie and Li, Yujia and Urtasun, Raquel and Zemel, Richard},
  journal={Advances in neural information processing systems},
  volume={29},
  year={2016}
}

@article{bwz+21,
  title={Targeted attack against deep neural networks via flipping limited weight bits},
  author={Bai, Jiawang and Wu, Baoyuan and Zhang, Yong and Li, Yiming and Li, Zhifeng and Xia, Shu-Tao},
  journal={arXiv preprint arXiv:2102.10496},
  year={2021}
}

@inproceedings{mv15,
  title={Understanding deep image representations by inverting them},
  author={Mahendran, Aravindh and Vedaldi, Andrea},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={5188--5196},
  year={2015}
}

@article{jrm+99,
  title={Inversion of feedforward neural networks: algorithms and applications},
  author={Jensen, Craig A and Reed, Russell D and Marks, Robert Jackson and El-Sharkawi, Mohamed A and Jung, Jae-Byung and Miyamoto, Robert T and Anderson, Gregory M and Eggen, Christian J},
  journal={Proceedings of the IEEE},
  volume={87},
  number={9},
  pages={1536--1549},
  year={1999},
  publisher={IEEE}
}

@article{lkn99,
  title={Inverting feedforward neural networks using linear and nonlinear programming},
  author={Lu, Bao-Liang and Kita, Hajime and Nishikawa, Yoshikazu},
  journal={IEEE Transactions on Neural networks},
  volume={10},
  number={6},
  pages={1271--1290},
  year={1999},
  publisher={IEEE}
}

@inproceedings{zjp+20,
  title={The secret revealer: Generative model-inversion attacks against deep neural networks},
  author={Zhang, Yuheng and Jia, Ruoxi and Pei, Hengzhi and Wang, Wenxiao and Li, Bo and Song, Dawn},
  booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
  pages={253--261},
  year={2020}
}





@article{bdm73,
  title={On the local and superlinear convergence of quasi-Newton methods},
  author={Broyden, Charles George and Dennis Jr, John E and Mor{\'e}, Jorge J},
  journal={IMA Journal of Applied Mathematics},
  volume={12},
  number={3},
  pages={223--245},
  year={1973},
  publisher={Oxford University Press}
}

@article{ln89,
  title={On the limited memory BFGS method for large scale optimization},
  author={Liu, Dong C and Nocedal, Jorge},
  journal={Mathematical programming},
  volume={45},
  number={1},
  pages={503--528},
  year={1989},
  publisher={Springer}
}

@inproceedings{m10,
  title={Deep learning via hessian-free optimization.},
  author={Martens, James and others},
  booktitle={ICML},
  year={2010}
}

@inproceedings{vp12,
  title={{Krylov} subspace descent for deep learning},
  author={Vinyals, Oriol and Povey, Daniel},
  booktitle={AISTATS},
  year={2012},
}

@article{a98,
  title={Natural gradient works efficiently in learning},
  author={Amari, Shun-Ichi},
  journal={Neural computation},
  year={1998},
  publisher={MIT Press}
}

@inproceedings{mg15,
  title={Optimizing neural networks with kronecker-factored approximate curvature},
  author={Martens, James and Grosse, Roger},
  booktitle={International conference on machine learning},
  pages={2408--2417},
  year={2015},
  organization={PMLR}
}

@inproceedings{gm16,
  title={A kronecker-factored approximate fisher matrix for convolution layers},
  author={Grosse, Roger and Martens, James},
  booktitle={ICML},
  year={2016},
}

@inproceedings{bgm22,
  title={Distributed second-order optimization using {Kronecker}-factored approximations},
  author={Ba, Jimmy and Grosse, Roger and Martens, James},
  booktitle={ICLR},
  year={2022}
}


@article{lssz24_dp_tree,
  title={Differential Privacy of Cross-Attention with Provable Guarantee},
  author={Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Zhou, Yufa},
  journal={arXiv preprint arXiv:2407.14717},
  year={2024}
}

@article{lssz24_diffusion,
  title={Unraveling the Smoothness Properties of Diffusion Models: A Gaussian Mixture Perspective},
  author={Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Zhou, Yufa},
  journal={arXiv preprint arXiv:2405.16418},
  year={2024}
}

@article{lssz24_tensor,
  title={Tensor attention training: Provably efficient learning of higher-order transformers},
  author={Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Zhou, Yufa},
  journal={arXiv preprint arXiv:2405.16411},
  year={2024}
}


@inproceedings{wsd+23,
  title={Tokencompose: Text-to-image diffusion with token-level supervision},
  author={Wang, Zirui and Sha, Zhizhou and Ding, Zheng and Wang, Yilin and Tu, Zhuowen},
  booktitle={CVPR},
  year={2024}
}

@inproceedings{wcz+23,
  title={Dolfin: Diffusion Layout Transformers without Autoencoder},
  author={Wang, Yilin and Chen, Zeyuan and Zhong, Liangjun and Ding, Zheng and Sha, Zhizhou and Tu, Zhuowen},
  booktitle={ECCV},
  year={2024}
}

@inproceedings{lsss24_dp_ntk,
  title={Differential Privacy Mechanisms in Neural Tangent Kernel Regression},
  author={Liang, Yingyu and Sha, Zhizhou and Shi, Zhenmei and Song, Zhao},
  booktitle={WACV},
  year={2025}
}

@inproceedings{wxz+24,
  title={{OmniControlNet}: Dual-stage Integration for Conditional Image Generation},
  author={Wang, Yilin and Xu, Haiyang and Zhang, Xiang and Chen, Zeyuan and Sha, Zhizhou and Wang, Zirui and Tu, Zhuowen},
  booktitle={CVPR},
  year={2024}
}

@article{lls+24c,
  title={Fast John Ellipsoid Computation with Differential Privacy Optimization},
  author={Li, Xiaoyu and Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Yu, Junwei},
  journal={arXiv preprint arXiv:2408.06395},
  year={2024}
}

@inproceedings{abadi2016deep,
  title={Deep learning with differential privacy},
  author={Abadi, Martin and Chu, Andy and Goodfellow, Ian and McMahan, H Brendan and Mironov, Ilya and Talwar, Kunal and Zhang, Li},
  booktitle={Proceedings of the 2016 ACM SIGSAC conference on computer and communications security},
  pages={308--318},
  year={2016}
}

@inproceedings{papernot2018scalable,
  title={Scalable private learning with pate},
  author={Papernot, Nicolas and Song, Shuang and Mironov, Ilya and Raghunathan, Ananth and Talwar, Kunal and Erlingsson, {\'U}lfar},
  booktitle={International conference on learning representations},
  year={2018}
}



@inproceedings{gsy23_hyper,
  title={An iterative algorithm for rescaled hyperbolic functions regression},
  author={Gao, Yeqi and Song, Zhao and Yin, Junze},
  booktitle={AISTATS},
  year={2025}
}

@article{gsy23_coin,
  title={GradientCoin: A Peer-to-Peer Decentralized Large Language Models},
  author={Gao, Yeqi and Song, Zhao and Yin, Junze},
  journal={arXiv preprint arXiv:2308.10502},
  year={2023}
}

@inproceedings{syyz23_dp,
  title = {Sketching meets differential privacy: fast algorithm for dynamic Kronecker projection maintenance},
  author = {Song, Zhao and Yang, Xin and Yang, Yuanyuan and Zhang, Lichen},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {32418--32462},
  year = {2023},
  organization = {PMLR}
}

@inproceedings{ssz+24_dit,
  title={Lazy{DiT}: Lazy Learning for the Acceleration of Diffusion Transformers},
  author={Shen, Xuan and Song, Zhao and Zhou, Yufa and Chen, Bo and Li, Yanyu and Gong, Yifan and Zhang, Kai and Tan, Hao and Kuen, Jason and Ding, Henghui and others},
  booktitle={AAAI},
  year={2025}
}

@inproceedings{ssz+24_pruning,
  title={Numerical Pruning for Efficient Autoregressive Models},
  author={Shen, Xuan and Song, Zhao and Zhou, Yufa and Chen, Bo and Liu, Jing and Zhang, Ruiyi and Rossi, Ryan A and Tan, Hao and Yu, Tong and Chen, Xiang and others},
  booktitle={AAAI},
  year={2025}
}



@book{v09,
  title={Optimal transport: old and new},
  author={Villani, Cedric},
  volume={338},
  year={2009},
  publisher={Springer}
}

@article{pc19,
  title={Computational optimal transport: With applications to data science},
  author={Peyre, Gabriel and Cuturi, Marco},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={11},
  number={5-6},
  pages={355--607},
  year={2019},
  publisher={Now Publishers, Inc.}
}

@article{tfm+23,
  title={Improving and generalizing flow-based generative models with minibatch optimal transport},
  author={Tong, Alexander and Fatras, Kilian and Malkin, Nikolay and Huguet, Guillaume and Zhang, Yanlei and Rector-Brooks, Jarrid and Wolf, Guy and Bengio, Yoshua},
  journal={Transactions on Machine Learning Research},
  year={2024}
}

@article{hppa24,
  title={{Wasserstein} Flow Matching: Generative modeling over families of distributions},
  author={Haviv, Doron and Pooladian, Aram-Alexandre and Pe'er, Dana and Amos, Brandon},
  journal={arXiv preprint arXiv:2411.00698},
  year={2024}
}

@inproceedings{kkn24,
  title={Equivariant flow matching},
  author={Klein, Leon and Kramer, Andreas and Noe, Frank},
  booktitle={NeurIPS},
  year={2024}
}

@inproceedings{cl24,
  title={Flow matching on general geometries},
  author={Chen, Ricky TQ and Lipman, Yaron},
  booktitle={ICLR},
  year={2024}
}

@inproceedings{pbd+23,
  title={Multisample flow matching: Straightening flows with minibatch couplings},
  author={Pooladian, Aram-Alexandre and Ben-Hamu, Heli and Domingo-Enrich, Carles and Amos, Brandon and Lipman, Yaron and Chen, Ricky TQ},
  booktitle={ICML},
  year={2023}
}

@inproceedings{xzc+22,
  title={Gmflow: Learning optical flow via global matching},
  author={Xu, Haofei and Zhang, Jing and Cai, Jianfei and Rezatofighi, Hamid and Tao, Dacheng},
  booktitle={CVPR},
  year={2022}
}

@inproceedings{dwb+23,
  title={Flow matching for scalable simulation-based inference},
  author={Dax, Maximilian and Wildberger, Jonas and Buchholz, Simon and Green, Stephen R and Macke, Jakob H and Scholkopf, Bernhard},
  booktitle={NeurIPS},
  year={2023}
}

@article{cll+24_rope,
  title={Circuit Complexity Bounds for {RoPE}-based Transformer Architecture},
  author={Chen, Bo and Li, Xiaoyu and Liang, Yingyu and Long, Jiangxuan and Shi, Zhenmei and Song, Zhao},
  journal={arXiv preprint arXiv:2411.07602},
  year={2024}
}

@article{qss23_gnn,
  title={Is Solving Graph Neural Tangent Kernel Equivalent to Training Graph Neural Network?},
  author={Qin, Lianke and Song, Zhao and Sun, Baocheng},
  journal={arXiv preprint arXiv:2309.07452},
  year={2023}
}


@article{ssx23_nns,
  title={A Theoretical Analysis Of Nearest Neighbor Search On Approximate Near Neighbor Graph},
  author={Shrivastava, Anshumali and Song, Zhao and Xu, Zhaozhuo},
  journal={arXiv preprint arXiv:2303.06210},
  year={2023}
}

@article{gms23,
  title={An over-parameterized exponential regression},
  author={Gao, Yeqi and Mahadevan, Sridhar and Song, Zhao},
  journal={arXiv preprint arXiv:2303.16504},
  year={2023}
}

@article{dms23_spar,
  title={Randomized and deterministic attention sparsification algorithms for over-parameterized feature dimension},
  author={Deng, Yichuan and Mahadevan, Sridhar and Song, Zhao},
  journal={arXiv preprint arXiv:2304.04397},
  year={2023}
}

@article{dswy22_coreset,
  title={A nearly optimal size coreset algorithm with nearly linear time},
  author={Deng, Yichuan and Song, Zhao and Wang, Yitan and Yang, Yuanyuan},
  journal={arXiv preprint arXiv:2210.08361},
  year={2022}
}

@inproceedings{gsr+24,
  title={Can Looped Transformers Learn to Implement Multi-step Gradient Descent for In-context Learning?},
  author={Gatmiry, Khashayar and Saunshi, Nikunj and Reddi, Sashank J and Jegelka, Stefanie and Kumar, Sanjiv},
  booktitle={Forty-first International Conference on Machine Learning},
    year={2024}
}

@article{acds24,
  title={Transformers learn to implement preconditioned gradient descent for in-context learning},
  author={Ahn, Kwangjun and Cheng, Xiang and Daneshmand, Hadi and Sra, Suvrit},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2024}
}

@article{gg23,
  title={Handbook of convergence theorems for (stochastic) gradient methods},
  author={Garrigos, Guillaume and Gower, Robert M},
  journal={arXiv preprint arXiv:2301.11235},
  year={2023}
}

@inproceedings{grs+23,
  title={Looped transformers as programmable computers},
  author={Giannou, Angeliki and Rajput, Shashank and Sohn, Jy-yong and Lee, Kangwook and Lee, Jason D and Papailiopoulos, Dimitris},
  booktitle={International Conference on Machine Learning},
  pages={11398--11442},
  year={2023},
  organization={PMLR}
}

@article{ylnp23,
  title={Looped transformers are better at learning learning algorithms},
  author={Yang, Liu and Lee, Kangwook and Nowak, Robert and Papailiopoulos, Dimitris},
  journal={arXiv preprint arXiv:2311.12424},
  year={2023}
}

@article{gzx+24,
  title={On the expressive power of a variant of the looped transformer},
  author={Gao, Yihang and Zheng, Chuanyang and Xie, Enze and Shi, Han and Hu, Tianyang and Li, Yu and Ng, Michael K and Li, Zhenguo and Liu, Zhaoqiang},
  journal={arXiv preprint arXiv:2402.13572},
  year={2024}
}

@article{fdrl24,
  title={Looped Transformers for Length Generalization},
  author={Fan, Ying and Du, Yilun and Ramchandran, Kannan and Lee, Kangwook},
  journal={arXiv preprint arXiv:2409.15647},
  year={2024}
}

@article{zfb23,
  title={Trained Transformers Learn Linear Models In-Context},
  author={Zhang, Ruiqi and Frei, Spencer and Bartlett, Peter L},
  journal={arXiv preprint arXiv:2306.09927},
  year={2023}
}

@article{mhm23,
  title={One step of gradient descent is provably the optimal in-context learner with one layer of linear self-attention},
  author={Mahankali, Arvind and Hashimoto, Tatsunori B and Ma, Tengyu},
  journal={arXiv preprint arXiv:2307.03576},
  year={2023}
}

@article{gtl+22,
  title={What can transformers learn in-context? a case study of simple function classes},
  author={Garg, Shivam and Tsipras, Dimitris and Liang, Percy S and Valiant, Gregory},
  journal={Advances in Neural Information Processing Systems},
  year={2022}
}

@article{wick1950evaluation,
  title={The evaluation of the collision matrix},
  author={Wick, Gian-Carlo},
  journal={Physical review},
  year={1950},
  publisher={APS}
}

@article{michalowicz2009isserlis,
  title={An Isserlis’ theorem for mixed Gaussian variables: Application to the auto-bispectral density},
  author={Michalowicz, JV and Nichols, JM and Bucholtz, F and Olson, CC},
  journal={Journal of Statistical Physics},
  year={2009},
  publisher={Springer}
}

@inproceedings{hsw+22,
  title={Lo{RA}: Low-Rank Adaptation of Large Language Models},
  author={Edward J Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen},
  booktitle={ICLR},
  year={2022}
}

@article{chen2021scatterbrain,
  title={Scatterbrain: Unifying sparse and low-rank attention},
  author={Chen, Beidi and Dao, Tri and Winsor, Eric and Song, Zhao and Rudra, Atri and R{\'e}, Christopher},
  journal={Advances in Neural Information Processing Systems},
  year={2021}
}

@inproceedings{bhojanapalli2020low,
  title={Low-rank bottleneck in multi-head attention models},
  author={Bhojanapalli, Srinadh and Yun, Chulhee and Rawat, Ankit Singh and Reddi, Sashank and Kumar, Sanjiv},
  booktitle={International conference on machine learning},
  year={2020},
  organization={PMLR}
}

@inproceedings{fan2021lighter,
  title={Lighter and better: low-rank decomposed self-attention networks for next-item recommendation},
  author={Fan, Xinyan and Liu, Zheng and Lian, Jianxun and Zhao, Wayne Xin and Xie, Xing and Wen, Ji-Rong},
  booktitle={Proceedings of the 44th international ACM SIGIR conference on research and development in information retrieval},
  year={2021}
}

@inproceedings{dass2023vitality,
  title={Vitality: Unifying low-rank and sparse approximation for vision transformer acceleration with a linear taylor attention},
  author={Dass, Jyotikrishna and Wu, Shang and Shi, Huihong and Li, Chaojian and Ye, Zhifan and Wang, Zhongfeng and Lin, Yingyan},
  booktitle={2023 IEEE International Symposium on High-Performance Computer Architecture (HPCA)},
  year={2023},
  organization={IEEE}
}


@inproceedings{vnr+23,
  title={Transformers learn in-context by gradient descent},
  author={Von Oswald, Johannes and Niklasson, Eyvind and Randazzo, Ettore and Sacramento, Jo{\~a}o and Mordvintsev, Alexander and Zhmoginov, Andrey and Vladymyrov, Max},
  booktitle={ICML},
  year={2023}
}


@InProceedings{pmlr-v202-li23l,
  title = 	 {Transformers as Algorithms: Generalization and Stability in In-context Learning},
  author =       {Li, Yingcong and Ildiz, Muhammed Emrullah and Papailiopoulos, Dimitris and Oymak, Samet},
  booktitle = 	 {Proceedings of the 40th International Conference on Machine Learning},
  year = 	 {2023},
  series = 	 {Proceedings of Machine Learning Research},
  publisher =    {PMLR}
}

@inproceedings{
edj+23,
title={What learning algorithm is in-context learning? Investigations with linear models},
author={Ekin Akyurek and Dale Schuurmans and Jacob Andreas and Tengyu Ma and Denny Zhou},
booktitle={The Eleventh International Conference on Learning Representations },
year={2023}
}

@inproceedings{schlag2021linear,
  title={Linear transformers are secretly fast weight programmers},
  author={Schlag, Imanol and Irie, Kazuki and Schmidhuber, J{\"u}rgen},
  booktitle={International Conference on Machine Learning},
  year={2021},
  organization={PMLR}
}
@article{bmr+20,
  title={Language models are few-shot learners},
  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  journal={Advances in neural information processing systems},
  year={2020}
}

@article{mlzh21,
  title={Metaicl: Learning to learn in context},
  author={Min, Sewon and Lewis, Mike and Zettlemoyer, Luke and Hajishirzi, Hannaneh},
  journal={arXiv preprint arXiv:2110.15943},
  year={2021}
}

@article{wei2023larger,
  title={Larger language models do in-context learning differently},
  author={Wei, Jerry and Wei, Jason and Tay, Yi and Tran, Dustin and Webson, Albert and Lu, Yifeng and Chen, Xinyun and Liu, Hanxiao and Huang, Da and Zhou, Denny and others},
  journal={arXiv preprint arXiv:2303.03846},
  year={2023}
}

@misc{chatgpt,
  author={OpenAI},
  title = {Introducing {ChatGPT}},
  year={2024},
}

@article{lcb+22,
  title={Flow matching for generative modeling},
  author={Lipman, Yaron and Chen, Ricky TQ and Ben-Hamu, Heli and Nickel, Maximilian and Le, Matt},
  journal={arXiv preprint arXiv:2210.02747},
  year={2022}
}


@article{o23,
  title={{GPT-4} technical report},
  author={OpenAI},
  journal={arXiv preprint arXiv:2303.08774},
  year={2023}
}

@misc{c24,
      author={Anthropic.},
      title={Claude 3.5 Sonnet}, 
      year={2024},
}


@article{
jyr+22,
title={Emergent Abilities of Large Language Models},
author={Jason Wei and Yi Tay and Rishi Bommasani and Colin Raffel and Barret Zoph and Sebastian Borgeaud and Dani Yogatama and Maarten Bosma and Denny Zhou and Donald Metzler and Ed H. Chi and Tatsunori Hashimoto and Oriol Vinyals and Percy Liang and Jeff Dean and William Fedus},
journal={Transactions on Machine Learning Research},
year={2022}
}



@article{malladi2023fine,
  title={Fine-Tuning Language Models with Just Forward Passes},
  author={Malladi, Sadhika and Gao, Tianyu and Nichani, Eshaan and Damian, Alex and Lee, Jason D and Chen, Danqi and Arora, Sanjeev},
  journal={Advances in Neural Information Processing Systems},
  year={2023}
}


@inproceedings{shi2023large,
  title={Large language models can be easily distracted by irrelevant context},
  author={Shi, Freda and Chen, Xinyun and Misra, Kanishka and Scales, Nathan and Dohan, David and Chi, Ed H and Sch{\"a}rli, Nathanael and Zhou, Denny},
  booktitle={International Conference on Machine Learning},
  year={2023},
  organization={PMLR}
}

@inproceedings{min2022rethinking,
  title={Rethinking the Role of Demonstrations: What Makes In-Context Learning Work?},
  author={Min, Sewon and Lyu, Xinxi and Holtzman, Ari and Artetxe, Mikel and Lewis, Mike and Hajishirzi, Hannaneh and Zettlemoyer, Luke},
  booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing},
  year={2022}
}

@inproceedings{pgcc23,
   title={What In-Context Learning 'Learns' In-Context: Disentangling Task Recognition and Task Learning},
   author={Pan, Jane and Gao, Tianyu and Chen, Howard and Chen, Danqi},
   booktitle={Findings of Association for Computational Linguistics (ACL)},
   year={2023}
}


### exp
@inproceedings{conneau-kiela-2018-senteval,
    title = "{S}ent{E}val: An Evaluation Toolkit for Universal Sentence Representations",
    author = "Conneau, Alexis  and
      Kiela, Douwe",
    booktitle = "Proceedings of the Eleventh International Conference on Language Resources and Evaluation ({LREC} 2018)",
    year = "2018",
    publisher = "European Language Resources Association (ELRA)",
}

@inproceedings{wang-etal-2018-glue,
    title = "{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding",
    author = "Wang, Alex  and
      Singh, Amanpreet  and
      Michael, Julian  and
      Hill, Felix  and
      Levy, Omer  and
      Bowman, Samuel",
    booktitle = "Proceedings of the 2018 {EMNLP} Workshop {B}lackbox{NLP}: Analyzing and Interpreting Neural Networks for {NLP}",
    year = "2018",
    publisher = "Association for Computational Linguistics"
}

@article{tli+23,
  title={Llama: Open and efficient foundation language models},
  author={Touvron, Hugo and Lavril, Thibaut and Izacard, Gautier and Martinet, Xavier and Lachaux, Marie-Anne and Lacroix, Timoth{\'e}e and Rozi{\`e}re, Baptiste and Goyal, Naman and Hambro, Eric and Azhar, Faisal and others},
  journal={arXiv preprint arXiv:2302.13971},
  year={2023}
}

@article{touvron2023llama2,
  title={Llama 2: Open foundation and fine-tuned chat models},
  author={Touvron, Hugo and Martin, Louis and Stone, Kevin and Albert, Peter and Almahairi, Amjad and Babaei, Yasmine and Bashlykov, Nikolay and Batra, Soumya and Bhargava, Prajjwal and Bhosale, Shruti and others},
  journal={arXiv preprint arXiv:2307.09288},
  year={2023}
}

@article{vsp+17,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={NeurIPS},
  year={2017}
}

@inproceedings{dclt19,
    title = "{BERT}: Pre-training of Deep Bidirectional Transformers for Language Understanding",
    author = "Devlin, Jacob  and
      Chang, Ming-Wei  and
      Lee, Kenton  and
      Toutanova, Kristina",
    booktitle = "NAACL",
    year = "2019",
}

@article{cnd+22,
  title={{PaLM}: Scaling language modeling with pathways},
  author={Chowdhery, Aakanksha and Narang, Sharan and Devlin, Jacob and Bosma, Maarten and Mishra, Gaurav and Roberts, Adam and Barham, Paul and Chung, Hyung Won and Sutton, Charles and Gehrmann, Sebastian and others},
  journal={Journal of Machine Learning Research},
  year={2023}
}

@article{bha+21,
  title={On the opportunities and risks of foundation models},
  author={Bommasani, Rishi and Hudson, Drew A and Adeli, Ehsan and Altman, Russ and Arora, Simran and von Arx, Sydney and Bernstein, Michael S and Bohg, Jeannette and Bosselut, Antoine and Brunskill, Emma and others},
  journal={arXiv preprint arXiv:2108.07258},
  year={2021}
}

@article{bce+23,
  title={Sparks of artificial general intelligence: Early experiments with {GPT-4}},
  author={Bubeck, S{\'e}bastien and Chandrasekaran, Varun and Eldan, Ronen and Gehrke, Johannes and Horvitz, Eric and Kamar, Ece and Lee, Peter and Lee, Yin Tat and Li, Yuanzhi and Lundberg, Scott and others},
  journal={arXiv preprint arXiv:2303.12712},
  year={2023}
}

@inproceedings{gfc+21,
    title = "Making Pre-trained Language Models Better Few-shot Learners",
    author = "Gao, Tianyu  and
      Fisch, Adam  and
      Chen, Danqi",
    booktitle = "ACL",
    year = "2021",
}

@article{chl+22,
  title={Scaling instruction-finetuned language models},
  author={Chung, Hyung Won and Hou, Le and Longpre, Shayne and Zoph, Barret and Tay, Yi and Fedus, William and Li, Eric and Wang, Xuezhi and Dehghani, Mostafa and Brahma, Siddhartha and others},
  journal={Journal of Machine Learning Research},
  year={2024}
}

@inproceedings{zhz+23,
  title={Llama-adapter: Efficient fine-tuning of language models with zero-init attention},
  author={Zhang, Renrui and Han, Jiaming and Zhou, Aojun and Hu, Xiangfei and Yan, Shilin and Lu, Pan and Li, Hongsheng and Gao, Peng and Qiao, Yu},
  booktitle={ICLR},
  year={2024}
}

@inproceedings{lac+21,
    title = "The Power of Scale for Parameter-Efficient Prompt Tuning",
    author = "Lester, Brian  and
      Al-Rfou, Rami  and
      Constant, Noah",
    booktitle = "EMNLP",
    year = "2021",
}

@inproceedings{ll21,
    title = "{Prefix-Tuning}: Optimizing Continuous Prompts for Generation",
    author = "Li, Xiang Lisa  and
      Liang, Percy",
    booktitle = "ACL",
    year = "2021",
}

@inproceedings{mkd+22,
  title={Cross-Task Generalization via Natural Language Crowdsourcing Instructions},
  author={Mishra, Swaroop and Khashabi, Daniel and Baral, Chitta and Hajishirzi, Hannaneh},
  booktitle={ACL},
  year={2022}
}


@article{naa+21,
  title={Show your work: Scratchpads for intermediate computation with language models},
  author={Nye, Maxwell and Andreassen, Anders Johan and Gur AriGuy and Michalewski, Henryk and Austin, Jacob and Bieber, David and Dohan, David and Lewkowycz, Aitor and Bosma, Maarten and Luan, David and others},
  journal={arXiv preprint arXiv:2112.00114},
  year={2021}
}

@article{wws+22,
  title={Chain-of-thought prompting elicits reasoning in large language models},
  author={Wei, Jason and Wang, Xuezhi and Schuurmans, Dale and Bosma, Maarten and Xia, Fei and Chi, Ed and Le, Quoc V and Zhou, Denny and others},
  journal={Advances in Neural Information Processing Systems},
  year={2022}
}

@article{ksk+22,
  title={Demonstrate-Search-Predict: Composing retrieval and language models for knowledge-intensive NLP},
  author={Khattab, Omar and Santhanam, Keshav and Li, Xiang Lisa and Hall, David and Liang, Percy and Potts, Christopher and Zaharia, Matei},
  journal={arXiv preprint arXiv:2212.14024},
  year={2022}
}

@article{ghz+23,
  title={Llama-adapter v2: Parameter-efficient visual instruction model},
  author={Gao, Peng and Han, Jiaming and Zhang, Renrui and Lin, Ziyi and Geng, Shijie and Zhou, Aojun and Zhang, Wei and Lu, Pan and He, Conghui and Yue, Xiangyu and others},
  journal={arXiv preprint arXiv:2304.15010},
  year={2023}
}

@inproceedings{ssy+22,
  title={Black-box tuning for language-model-as-a-service},
  author={Sun, Tianxiang and Shao, Yunfan and Qian, Hong and Huang, Xuanjing and Qiu, Xipeng},
  booktitle={ICML},
  year={2022},

}

@inproceedings{zwf+21,
  title={Calibrate before use: Improving few-shot performance of language models},
  author={Zhao, Zihao and Wallace, Eric and Feng, Shi and Klein, Dan and Singh, Sameer},
  booktitle={ICML},
  year={2021},
}

@article{znl+22,
  title={Teaching algorithmic reasoning via in-context learning},
  author={Zhou, Hattie and Nova, Azade and Larochelle, Hugo and Courville, Aaron and Neyshabur, Behnam and Sedghi, Hanie},
  journal={arXiv preprint arXiv:2211.09066},
  year={2022}
}

@article{ag23,
  title={A theory for emergence of complex skills in language models},
  author={Arora, Sanjeev and Goyal, Anirudh},
  journal={arXiv preprint arXiv:2307.15936},
  year={2023}
}

@inproceedings{
hmsp23,
title={A Theoretical Understanding of Shallow Vision Transformers: Learning, Generalization, and Sample Complexity},
author={Hongkang Li and Meng Wang and Sijia Liu and Pin-Yu Chen},
booktitle={The Eleventh International Conference on Learning Representations },
year={2023}
}


@InProceedings{llr23,
  title = 	 {How Do Transformers Learn Topic Structure: Towards a Mechanistic Understanding},
  author =       {Li, Yuchen and Li, Yuanzhi and Risteski, Andrej},
  booktitle = 	 {Proceedings of the 40th International Conference on Machine Learning},
  year = 	 {2023},
  series = 	 {Proceedings of Machine Learning Research},
  publisher =    {PMLR}
}

@article{fpk+23,
  title={Improving language model negotiation with self-play and in-context learning from ai feedback},
  author={Fu, Yao and Peng, Hao and Khot, Tushar and Lapata, Mirella},
  journal={arXiv preprint arXiv:2305.10142},
  year={2023}
}
@article{al23,
  title={Physics of Language Models: Part 1, Context-Free Grammar},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi},
  journal={arXiv preprint arXiv:2305.13673},
  year={2023}
}


@article{twcd23,
  title={Scan and Snap: Understanding Training Dynamics and Token Composition in 1-layer Transformer},
  author={Tian, Yuandong and Wang, Yiping and Chen, Beidi and Du, Simon},
  journal={Advances in Neural Information Processing Systems},
  year={2023}
}

@inproceedings{owj+22,
  title={Training language models to follow instructions with human feedback},
  author={Ouyang, Long and Wu, Jeffrey and Jiang, Xu and Almeida, Diogo and Wainwright, Carroll and Mishkin, Pamela and Zhang, Chong and Agarwal, Sandhini and Slama, Katarina and Ray, Alex and others},
  journal={NeurIPS},
  year={2022}
}

@article{jsl+22,
  title={Vision transformers provably learn spatial structure},
  author={Jelassi, Samy and Sander, Michael and Li, Yuanzhi},
  journal={Advances in Neural Information Processing Systems},
  year={2022}
}

@inproceedings{
zsc+23,
title={Understanding The Robustness of Self-supervised Learning Through Topic Modeling},
author={Zeping Luo and Shiyou Wu and Cindy Weng and Mo Zhou and Rong Ge},
booktitle={The Eleventh International Conference on Learning Representations },
year={2023}
}

@inproceedings{
sapt22,
title={An Explanation of In-context Learning as Implicit Bayesian Inference},
author={Sang Michael Xie and Aditi Raghunathan and Percy Liang and Tengyu Ma},
booktitle={International Conference on Learning Representations},
year={2022},
}

@article{liu2022few,
  title={Few-shot parameter-efficient fine-tuning is better and cheaper than in-context learning},
  author={Liu, Haokun and Tam, Derek and Muqeeth, Mohammed and Mohta, Jay and Huang, Tenghao and Bansal, Mohit and Raffel, Colin A},
  journal={Advances in Neural Information Processing Systems},
  year={2022}
}

@article{pr23,
  title={Din-sql: Decomposed in-context learning of text-to-sql with self-correction},
  author={Pourreza, Mohammadreza and Rafiei, Davood},
  journal={arXiv preprint arXiv:2304.11015},
  year={2023}
}

@article{azl+22,
  title={In-context examples selection for machine translation},
  author={Agrawal, Sweta and Zhou, Chunting and Lewis, Mike and Zettlemoyer, Luke and Ghazvininejad, Marjan},
  journal={arXiv preprint arXiv:2212.02437},
  year={2022}
}

@article{twz+23,
  title={Joma: Demystifying multilayer transformers via joint dynamics of mlp and attention},
  author={Tian, Yuandong and Wang, Yiping and Zhang, Zhenyu and Chen, Beidi and Du, Simon},
  journal={arXiv preprint arXiv:2310.00535},
  year={2023}
}

@inproceedings{wma+22,
  title={Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks},
  author={Wang, Yizhong and Mishra, Swaroop and Alipoormolabashi, Pegah and Kordi, Yeganeh and Mirzaei, Amirreza and Naik, Atharva and Ashok, Arjun and Dhanasekaran, Arut Selvan and Arunkumar, Anjana and Stap, David and others},
  booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing},
  pages={5085--5109},
  year={2022}
}

@article{dong2022survey,
  title={A survey for in-context learning},
  author={Dong, Qingxiu and Li, Lei and Dai, Damai and Zheng, Ce and Wu, Zhiyong and Chang, Baobao and Sun, Xu and Xu, Jingjing and Sui, Zhifang},
  journal={arXiv preprint arXiv:2301.00234},
  year={2022}
}

@inproceedings{
jmv+22,
title={Finetuned Language Models are Zero-Shot Learners},
author={Jason Wei and Maarten Bosma and Vincent Zhao and Kelvin Guu and Adams Wei Yu and Brian Lester and Nan Du and Andrew M. Dai and Quoc V Le},
booktitle={International Conference on Learning Representations},
year={2022}
}

@article{ilp+22,
  title={Opt-iml: Scaling language model instruction meta learning through the lens of generalization},
  author={Iyer, Srinivasan and Lin, Xi Victoria and Pasunuru, Ramakanth and Mihaylov, Todor and Simig, Daniel and Yu, Ping and Shuster, Kurt and Wang, Tianlu and Liu, Qing and Koura, Punit Singh and others},
  journal={arXiv preprint arXiv:2212.12017},
  year={2022}
}

@inproceedings{
shi2023domain,
title={Domain Generalization via Nuclear Norm Regularization},
author={Zhenmei Shi and Yifei Ming and Ying Fan and Frederic Sala and Yingyu Liang},
booktitle={Conference on Parsimony and Learning (Proceedings Track)},
year={2023}
}

@inproceedings{
zjk+23,
title={The Trade-off between Universality and Label Efficiency of Representations from Contrastive Learning},
author={Zhenmei Shi and Jiefeng Chen and Kunyang Li and Jayaram Raghuram and Xi Wu and Yingyu Liang and Somesh Jha},
booktitle={ICLR},
year={2023}
}



@inproceedings{shalev2017failures,
  title={Failures of gradient-based deep learning},
  author={Shalev-Shwartz, Shai and Shamir, Ohad and Shammah, Shaked},
  booktitle={International Conference on Machine Learning},
  pages={3067--3075},
  year={2017},
  organization={PMLR}
}

@article{dm20,
  title={Learning parities with neural networks},
  author={Daniely, Amit and Malach, Eran},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={20356--20365},
  year={2020}
}
@article{beg+22,
  title={Hidden progress in deep learning: Sgd learns parities near the computational limit},
  author={Barak, Boaz and Edelman, Benjamin and Goel, Surbhi and Kakade, Sham and Malach, Eran and Zhang, Cyril},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={21750--21764},
  year={2022}
}


@inproceedings{
swl22,
title={A Theoretical Analysis on Feature Learning in Neural Networks: Emergence from Inputs and Advantage over Fixed Features},
author={Zhenmei Shi and Junyi Wei and Yingyu Liang},
booktitle={International Conference on Learning Representations},
year={2022}
}

@inproceedings{
shi2023provable,
title={Provable Guarantees for Neural Networks via Gradient Feature Learning},
author={Zhenmei Shi and Junyi Wei and Yingyu Liang},
booktitle={Thirty-seventh Conference on Neural Information Processing Systems},
year={2023}
}


@article{Olshausen1997SparseCW,
  title={Sparse coding with an overcomplete basis set: A strategy employed by V1?},
  author={B. Olshausen and D. Field},
  journal={Vision Research},
  year={1997},
  volume={37},
  pages={3311-3325}
}


@article{vinje2000sparse,
  title={Sparse coding and decorrelation in primary visual cortex during natural vision},
  author={Vinje, William E and Gallant, Jack L},
  journal={Science},
  volume={287},
  number={5456},
  pages={1273--1276},
  year={2000},
  publisher={American Association for the Advancement of Science}
}

@article{blei2003latent,
  title={Latent dirichlet allocation},
  author={Blei, David M and Ng, Andrew Y and Jordan, Michael I},
  journal={the Journal of machine Learning research},
  volume={3},
  pages={993--1022},
  year={2003},
  publisher={JMLR. org}
}

@inproceedings{zzj+23,
  title={	Towards Few-Shot Adaptation of Foundation Models via Multitask Finetuning},
  author={Zhuoyan Xu and Zhenmei Shi and Junyi Wei and Fangzhou Mu and Yin Li and Yingyu Liang},
  booktitle={ICLR},
  year={2024}
}


@inproceedings{r24,
  title={The mechanistic basis of data dependence and abrupt learning in an in-context classification task},
  author={Reddy, Gautam},
  booktitle={The Twelfth International Conference on Learning Representations },
  year={2024}
}

@inproceedings{ghm+24,
  title={How do transformers learn in-context beyond simple functions? a case study on learning with representations},
  author={Guo, Tianyu and Hu, Wei and Mei, Song and Wang, Huan and Xiong, Caiming and Savarese, Silvio and Bai, Yu},
  booktitle={The Twelfth International Conference on Learning Representations },
  year={2024}
}

@inproceedings{wzc+24,
  title={How Many Pretraining Tasks Are Needed for In-Context Learning of Linear Regression?},
  author={Wu, Jingfeng and Zou, Difan and Chen, Zixiang and Braverman, Vladimir and Gu, Quanquan and Bartlett, Peter L},
  booktitle={The Twelfth International Conference on Learning Representations },
  year={2024}
}

@article{hcl23,
  title={In-context convergence of transformers},
  author={Huang, Yu and Cheng, Yuan and Liang, Yingbin},
  journal={arXiv preprint arXiv:2310.05249},
  year={2023}
}

@inproceedings{lwl+23,
  title={Transformers as Multi-Task Feature Selectors: Generalization Analysis of In-Context Learning},
  author={Li, Hongkang and Wang, Meng and Lu, Songtao and Wan, Hui and Cui, Xiaodong and Chen, Pin-Yu},
  booktitle={NeurIPS 2023 Workshop on Mathematics of Modern Machine Learning},
  year={2023}
}

@article{ccs23,
  title={Transformers Implement Functional Gradient Descent to Learn Non-Linear Functions In Context},
  author={Cheng, Xiang and Chen, Yuxin and Sra, Suvrit},
  journal={arXiv preprint arXiv:2312.06528},
  year={2023}
}

@inproceedings{zbl+23,
  title={What Algorithms can Transformers Learn? A Study in Length Generalization},
  author={Zhou, Hattie and Bradley, Arwen and Littwin, Etai and Razin, Noam and Saremi, Omid and Susskind, Joshua and Bengio, Samy and Nakkiran, Preetum},
  booktitle={The 3rd Workshop on Mathematical Reasoning and AI at NeurIPS'23},
  year={2023}
}
@inproceedings{
avd+23,
title={Birth of a Transformer: A Memory Viewpoint},
author={Alberto Bietti and Vivien Cabannes and Diane Bouchacourt and Herve Jegou and Leon Bottou},
booktitle={Thirty-seventh Conference on Neural Information Processing Systems},
year={2023}
}

@inproceedings{ww23,
  title={On the Role of Unstructured Training Data in Transformers' In-Context Learning Capabilities},
  author={Wibisono, Kevin Christian and Wang, Yixin},
  booktitle={NeurIPS 2023 Workshop on Mathematics of Modern Machine Learning},
  year={2023}
}

@inproceedings{
yfh+23,
title={Transformers as Statisticians: Provable In-Context Learning with In-Context Algorithm Selection},
author={Yu Bai and Fan Chen and Huan Wang and Caiming Xiong and Song Mei},
booktitle={Thirty-seventh Conference on Neural Information Processing Systems},
year={2023}
}

@inproceedings{gsb+21,
  title={Transformer Feed-Forward Layers Are Key-Value Memories},
  author={Geva, Mor and Schuster, Roei and Berant, Jonathan and Levy, Omer},
  booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},
  pages={5484--5495},
  year={2021}
}

@inproceedings{lsg+23,
  title={Dissecting Chain-of-Thought: Compositionality through In-Context Filtering and Learning},
  author={Li, Yingcong and Sreenivasan, Kartik and Giannou, Angeliki and Papailiopoulos, Dimitris and Oymak, Samet},
  booktitle={Thirty-seventh Conference on Neural Information Processing Systems},
  year={2023}
}

@inproceedings{
zmc+24,
title={Step-Back Prompting Enables Reasoning Via Abstraction in Large Language Models},
author={Zheng, Huaixiu Steven and Mishra, Swaroop and Chen, Xinyun and Cheng, Heng-Tze and Chi, Ed H and Le, Quoc V and Zhou, Denny},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024}
}

@article{zzy+23,
  title={A Study on the Calibration of In-context Learning},
  author={Zhang, Hanlin and Zhang, Yi-Fan and Yu, Yaodong and Madeka, Dhruv and Foster, Dean and Xing, Eric and Lakkaraju, Hima and Kakade, Sham},
  journal={arXiv preprint arXiv:2312.04021},
  year={2023}
}

@inproceedings{
sdj+23,
title={Tree of Thoughts: Deliberate Problem Solving with Large Language Models},
author={Shunyu Yao and Dian Yu and Jeffrey Zhao and Izhak Shafran and Thomas L. Griffiths and Yuan Cao and Karthik R Narasimhan},
booktitle={Thirty-seventh Conference on Neural Information Processing Systems},
year={2023}
}

@inproceedings{
ahn2023linear,
title={Linear attention is (maybe) all you need (to understand Transformer optimization)},
author={Ahn, Kwangjun and Cheng, Xiang and Song, Minhak and Yun, Chulhee and Jadbabaie, Ali and Sra, Suvrit},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024}
}

@inproceedings{
kxhs23,
title={Transformers learn to implement preconditioned gradient descent for in-context learning},
author={Kwangjun Ahn and Xiang Cheng and Hadi Daneshmand and Suvrit Sra},
booktitle={Thirty-seventh Conference on Neural Information Processing Systems},
year={2023}
}

@inproceedings{
amfs23,
title={Pretraining task diversity and the emergence of non-Bayesian in-context learning for regression},
author={Allan Raventos and Mansheej Paul and Feng Chen and Surya Ganguli},
booktitle={Thirty-seventh Conference on Neural Information Processing Systems},
year={2023}
}

@inproceedings{
cpp+23,
title={{LIMA}: Less Is More for Alignment},
author={Chunting Zhou and Pengfei Liu and Puxin Xu and Srini Iyer and Jiao Sun and Yuning Mao and Xuezhe Ma and Avia Efrat and Ping Yu and LILI YU and Susan Zhang and Gargi Ghosh and Mike Lewis and Luke Zettlemoyer and Omer Levy},
booktitle={NeurIPS},
year={2023}
}

@article{panigrahi2023trainable,
  title={Trainable transformer in transformer},
  author={Panigrahi, Abhishek and Malladi, Sadhika and Xia, Mengzhou and Arora, Sanjeev},
  journal={arXiv preprint arXiv:2307.01189},
  year={2023}
}



@article{an2023context,
  title={How Do In-Context Examples Affect Compositional Generalization?},
  author={An, Shengnan and Lin, Zeqi and Fu, Qiang and Chen, Bei and Zheng, Nanning and Lou, Jian-Guang and Zhang, Dongmei},
  journal={arXiv preprint arXiv:2305.04835},
  year={2023}
}

@InProceedings{ye2023compositional,
  title = 	 {Compositional Exemplars for In-context Learning},
  author =       {Ye, Jiacheng and Wu, Zhiyong and Feng, Jiangtao and Yu, Tao and Kong, Lingpeng},
  booktitle = 	 {Proceedings of the 40th International Conference on Machine Learning},
  pages = 	 {39818--39833},
  year = 	 {2023},
  editor = 	 {Krause, Andreas and Brunskill, Emma and Cho, Kyunghyun and Engelhardt, Barbara and Sabato, Sivan and Scarlett, Jonathan},
  volume = 	 {202},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {23--29 Jul},
  publisher =    {PMLR}
}

@article{lls+24_conv,
  title={Conv-Basis: A New Paradigm for Efficient Attention Inference and Gradient Computation in Transformers},
  author={Liang, Yingyu and Liu, Heshan and Shi, Zhenmei and Song, Zhao and Yin, Junze},
  journal={arXiv preprint arXiv:2405.05219},
  year={2024}
}

@article{gll+24b,
  title={Exploring the Frontiers of Softmax: Provable Optimization, Applications in Diffusion Model, and Beyond},
  author={Li, Chenyang and Liang, Yingyu and Shi, Zhenmei and Song, Zhao},
  journal={arXiv preprint arXiv:2405.03251},
  year={2024}
}

@inproceedings{xsl24,
  title={Do large language models have compositional ability? an investigation into limitations and scalability},
  author={Xu, Zhuoyan and Shi, Zhenmei and Liang, Yingyu},
  booktitle={ICLR 2024 Workshop on Mathematical and Empirical Understanding of Foundation Models},
  year={2024}
}

@article{gls+24b,
  title={Unraveling the Smoothness Properties of Diffusion Models: A Gaussian Mixture Perspective},
  author={Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Zhou, Yufa},
  journal={arXiv preprint arXiv:2405.16418},
  year={2024}
}

@article{dgv+18,
  title={Universal transformers},
  author={Dehghani, Mostafa and Gouws, Stephan and Vinyals, Oriol and Uszkoreit, Jakob and Kaiser, {\L}ukasz},
  journal={arXiv preprint arXiv:1807.03819},
  year={2018}
}

@inproceedings{ccw+21,
  title={Audio albert: A lite bert for self-supervised learning of audio representation},
  author={Chi, Po-Han and Chung, Pei-Hung and Wu, Tsung-Han and Hsieh, Chun-Cheng and Chen, Yen-Hao and Li, Shang-Wen and Lee, Hung-yi},
  booktitle={2021 IEEE Spoken Language Technology Workshop (SLT)},
  pages={344--350},
  year={2021},
  organization={IEEE}
}

@article{af23,
  title={Simulation of graph algorithms with looped transformers},
  author={Artur Back, de Luca and Fountoulakis, Kimon},
  journal={arXiv preprint arXiv:2402.01107},
  year={2024}
}

@article{gyw+24,
  title={How Well Can Transformers Emulate In-context Newton's Method?},
  author={Giannou, Angeliki and Yang, Liu and Wang, Tianhao and Papailiopoulos, Dimitris and Lee, Jason D},
  journal={arXiv preprint arXiv:2403.03183},
  year={2024}
}

@article{dld+22,
  title={A survey on in-context learning},
  author={Dong, Qingxiu and Li, Lei and Dai, Damai and Zheng, Ce and Wu, Zhiyong and Chang, Baobao and Sun, Xu and Xu, Jingjing and Sui, Zhifang},
  journal={arXiv preprint arXiv:2301.00234},
  year={2022}
}



@inproceedings{whl+23,
  title={Symbol tuning improves in-context learning in language models},
  author={Wei, Jerry and Hou, Le and Lampinen, Andrew Kyle and Chen, Xiangning and Huang, Da and Tay, Yi and Chen, Xinyun and Lu, Yifeng and Zhou, Denny and Ma, Tengyu and others},
  booktitle={EMNLP},
  year={2023}
}

@article{bcw+24,
  title={Transformers as statisticians: Provable in-context learning with in-context algorithm selection},
  author={Bai, Yu and Chen, Fan and Wang, Huan and Xiong, Caiming and Mei, Song},
  journal={Advances in neural information processing systems},
  volume={36},
  year={2024}
}


@inproceedings{hzrs16,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@inproceedings{cls+24,
  title={{HSR}-Enhanced Sparse Attention Acceleration},
  author={Chen, Bo and Liang, Yingyu and Sha, Zhizhou and Shi, Zhenmei and Song, Zhao},
  booktitle={CPAL},
  year={2025}
}

@article{lss+24_multi_layer,
  title={Multi-Layer Transformers Gradient Can be Approximated in Almost Linear Time},
  author={Liang, Yingyu and Sha, Zhizhou and Shi, Zhenmei and Song, Zhao and Zhou, Yufa},
  journal={arXiv preprint arXiv:2408.13233},
  year={2024}
}

@article{lssz24_tat,
  title={Tensor Attention Training: Provably Efficient Learning of Higher-order Transformers},
  author={Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Zhou, Yufa},
  journal={arXiv preprint arXiv:2405.16411},
  year={2024}
}

@inproceedings{lls+24_prune,
  title={Beyond Linear Approximations: A Novel Pruning Approach for Attention Matrix},
  author={Liang, Yingyu and Long, Jiangxuan and Shi, Zhenmei and Song, Zhao and Zhou, Yufa},
  booktitle={ICLR},
  year={2025}
}

@inproceedings{hyw+23,
  title={On Sparse Modern {Hopfield} Model},
  author={Hu, Jerry Yao-Chieh and Yang, Donglin and Wu, Dennis and Xu, Chenwei and Chen, Bo-Yu and Liu, Han},
  booktitle={NeurIPS},
  year={2023}
}

@inproceedings{whl+24,
  title={{ST}anHop: Sparse Tandem Hopfield Model for Memory-Enhanced Time Series Prediction},
  author={Wu, Dennis and Hu, Jerry Yao-Chieh and Li, Weijian and Chen, Bo-Yu and Liu, Han},
  booktitle={ICLR},
  year={2024}
}

@inproceedings{hlsl24,
  title={On Computational Limits of Modern {Hopfield} Models: A Fine-Grained Complexity Analysis},
  author={Hu, Jerry Yao-Chieh and Lin, Thomas and Song, Zhao and Liu, Han},
  booktitle={ICML},
  year={2024}
}

@inproceedings{xhh+24,
  title={BiSHop: Bi-Directional Cellular Learning for Tabular Data with Generalized Sparse Modern Hopfield Model},
  author={Xu, Chenwei and Huang, Yu-Chao and Hu, Jerry Yao-Chieh and Li, Weijian and Gilani, Ammar and Goan, Hsi-Sheng and Liu, Han},
  booktitle={ICML},
  year={2024}
}

@inproceedings{whhl24,
  title={Uniform Memory Retrieval with Larger Capacity for Modern {Hopfield} Models},
  author={Wu, Dennis and Hu, Jerry Yao-Chieh and Hsiao, Teng-Yun and Liu, Han},
  booktitle={ICML},
  year={2024}
}

@inproceedings{hcl+24,
  title={Outlier-Efficient Hopfield Layers for Large Transformer-Based Models},
  author={Hu, Jerry Yao-Chieh and Chang, Pei-Hsuan and Luo, Haozheng and Chen, Hong-Yu and Li, Weijian and Wang, Wei-Po and Liu, Han},
  booktitle={ICML},
  year={2024}
}
@article{hcw+24,
  title={Nonparametric Modern {Hopfield} Models},
  author={Hu, Jerry Yao-Chieh and Chen, Bo-Yu and Wu, Dennis and Ruan, Feng and Liu, Han},
  journal={arXiv preprint arXiv:2404.03900},
  year={2024}
}

@article{llss24_sparsegpt,
  title={A Tighter Complexity Analysis of {SparseGPT}},
  author={Li, Xiaoyu and Liang, Yingyu and Shi, Zhenmei and Song, Zhao},
  journal={arXiv preprint arXiv:2408.12151},
  year={2024}
}

@article{syz23,
  title={Streaming Semidefinite Programs: $ {O} (\sqrt n)$ Passes, Small Space and Fast Runtime},
  author={Song, Zhao and Ye, Mingquan and Zhang, Lichen},
  journal={arXiv preprint arXiv:2309.05135},
  year={2023}
}

@article{kll+25,
  title={On computational limits and provably efficient criteria of visual autoregressive models: A fine-grained complexity analysis},
  author={Ke, Yekun and Li, Xiaoyu and Liang, Yingyu and Sha, Zhizhou and Shi, Zhenmei and Song, Zhao},
  journal={arXiv preprint arXiv:2501.04377},
  year={2025}
}

@article{kll+24,
  title={Advancing the understanding of fixed point iterations in deep neural networks: A detailed analytical study},
  author={Ke, Yekun and Li, Xiaoyu and Liang, Yingyu and Shi, Zhenmei and Song, Zhao},
  journal={arXiv preprint arXiv:2410.11279},
  year={2024}
}

@inproceedings{lss+24_relu,
  title={Looped ReLU MLPs May Be All You Need as Practical Programmable Computers},
  author={Liang, Yingyu and Sha, Zhizhou and Shi, Zhenmei and Song, Zhao and Zhou, Yufa},
  booktitle={AISTATS},
  year={2025}
}

@article{smn+24,
  title={Discovering the Gems in Early Layers: Accelerating Long-Context {LLM}s with 1000x Input Token Reduction},
  author={Shi, Zhenmei and Ming, Yifei and Nguyen, Xuan-Phi and Liang, Yingyu and Joty, Shafiq},
  journal={arXiv preprint arXiv:2409.17422},
  year={2024}
}

@article{lssy24,
  title={Toward Infinite-Long Prefix in Transformer},
  author={Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Yang, Chiwun},
  journal={arXiv preprint arXiv:2406.14036},
  year={2024}
}

@inproceedings{swxl24,
  title={Why Larger Language Models Do In-context Learning Differently?},
  author={Shi, Zhenmei and Wei, Junyi and Xu, Zhuoyan and Liang, Yingyu},
  booktitle={ICML},
  year={2024}
}

@inproceedings{hwl24,
  title={Provably Optimal Memory Capacity for Modern Hopfield Models: Tight Analysis for Transformer-Compatible Dense Associative Memories},
  author={Hu, Jerry Yao-Chieh and Wu, Dennis and Liu, Han},
  booktitle={NeurIPS},
  year={2024}
}

@inproceedings{px23,
  title={Scalable diffusion models with transformers},
  author={Peebles, William and Xie, Saining},
  booktitle={ICCV},
  year={2023}
}

@inproceedings{bnx+23,
  title={All are worth words: A {ViT} backbone for diffusion models},
  author={Bao, Fan and Nie, Shen and Xue, Kaiwen and Cao, Yue and Li, Chongxuan and Su, Hang and Zhu, Jun},
  booktitle={CVPR},
  year={2023}
}

@inproceedings{se19,
  title={Generative modeling by estimating gradients of the data distribution},
  author={Song, Yang and Ermon, Stefano},
  booktitle={NeurIPS},
  year={2019}
}

@inproceedings{sme20,
  title={Denoising diffusion implicit models},
  author={Song, Jiaming and Meng, Chenlin and Ermon, Stefano},
  booktitle={ICLR},
  year={2021}
}

@inproceedings{lzb+22,
  title={Dpm-solver: A fast ode solver for diffusion probabilistic model sampling in around 10 steps},
  author={Lu, Cheng and Zhou, Yuhao and Bao, Fan and Chen, Jianfei and Li, Chongxuan and Zhu, Jun},
  booktitle={NeurIPS},
  year={2022}
}

@inproceedings{rbl+22,
  title={High-resolution image synthesis with latent diffusion models},
  author={Rombach, Robin and Blattmann, Andreas and Lorenz, Dominik and Esser, Patrick and Ommer, Bj{\"o}rn},
  booktitle={CVPR},
  year={2022}
}

@inproceedings{hwsl24,
  title={On statistical rates and provably efficient criteria of latent diffusion transformers ({DiT}s)},
  author={Hu, Jerry Yao-Chieh and Wu, Weimin and Song, Zhao and Liu, Han},
  booktitle={NeurIPS},
  year={2024}
}


@inproceedings{ylp+24,
  title={Perflow: Piecewise rectified flow as universal plug-and-play accelerator},
  author={Yan, Hanshu and Liu, Xingchao and Pan, Jiachun and Liew, Jun Hao and Liu, Qiang and Feng, Jiashi},
  booktitle={NeurIPS},
  year={2024}
}

@article{brsr24,
  title={On the {Wasserstein} Convergence and Straightness of Rectified Flow},
  author={Bansal, Vansh and Roy, Saptarshi and Sarkar, Purnamrita and Rinaldo, Alessandro},
  journal={arXiv preprint arXiv:2410.14949},
  year={2025}
}

@inproceedings{clpl24,
  title={Categorical Flow Matching on Statistical Manifolds},
  author={Cheng, Chaoran and Li, Jiahan and Peng, Jian and Liu, Ge},
  booktitle={NeurIPS},
  year={2024}
}

@inproceedings{ddt19,
  title={Augmented neural {ODE}s},
  author={Dupont, Emilien and Doucet, Arnaud and Teh, Yee Whye},
  booktitle={NeurIPS},
  year={2019}
}

@inproceedings{nbd+20,
  title={On second order behaviour in augmented neural {ODE}s},
  author={Norcliffe, Alexander and Bodnar, Cristian and Day, Ben and Simidjievski, Nikola and Li{\`o}, Pietro},
  booktitle={NeurIPS},
  year={2020}
}

@book{monge1781,
  title={Memoire sur la th{\'e}orie des d{\'e}blais et des remblais},
  author={Monge, G.},
  year={1781},
  publisher={Imprimerie royale}
}


@article{k58,
  title={On the translocation of masses},
  author={Kantorovitch, Leonid},
  journal={Management science},
  year={1958},
}

@article{m97,
  title={A convexity principle for interacting gases},
  author={McCann, Robert J},
  journal={Advances in mathematics},
  year={1997},
}

@inproceedings{scn23,
  title={On kinetic optimal probability paths for generative models},
  author={Shaul, Neta and Chen, Ricky TQ and Nickel, Maximilian and Le, Matthew and Lipman, Yaron},
  booktitle={ICML},
  year={2023}
}

@inproceedings{llsz24_nn_tw,
  title={Fast Second-order Method for Neural Network under Small Treewidth Setting},
  author={Li, Xiaoyu and Long, Jiangxuan and Song, Zhao and Zhou, Tianyi},
  booktitle={IEEE BigData},
  year={2024}
}

@inproceedings{cll+24_ssm,
  title={The Computational Limits of State-Space Models and Mamba via the Lens of Circuit Complexity},
  author={Chen, Yifang and Li, Xiaoyu and Liang, Yingyu and Shi, Zhenmei and Song, Zhao},
  booktitle={CPAL},
  year={2025}
}

@article{chl+24_rope_grad,
  title={Fast Gradient Computation for {RoPE} Attention in Almost Linear Time},
  author={Chen, Yifang and Huo, Jiayan and Li, Xiaoyu and Liang, Yingyu and Shi, Zhenmei and Song, Zhao},
  journal={arXiv preprint arXiv:2412.17316},
  year={2024}
}

@inproceedings{lkw+24,
  title={Latent diffusion for language generation},
  author={Lovelace, Justin and Kishore, Varsha and Wan, Chao and Shekhtman, Eliot and Weinberger, Kilian Q},
  booktitle={NeurIPS},
  year={2024}
}

@inproceedings{jzx+25,
  title={Autodir: Automatic all-in-one image restoration with latent diffusion},
  author={Jiang, Yitong and Zhang, Zhaoyang and Xue, Tianfan and Gu, Jinwei},
  booktitle={ECCV},
  year={2025},
}

@inproceedings{fmzz24,
  title={Latent diffusion transformer for probabilistic time series forecasting},
  author={Feng, Shibo and Miao, Chunyan and Zhang, Zhong and Zhao, Peilin},
  booktitle={AAAI},
  year={2024}
}

@inproceedings{rckc24,
  title={Beyond first-order tweedie: Solving inverse problems using latent diffusion},
  author={Rout, Litu and Chen, Yujia and Kumar, Abhishek and Caramanis, Constantine and Shakkottai, Sanjay and Chu, Wen-Sheng},
  booktitle={CVPR},
  year={2024}
}

@inproceedings{lyhz24,
  title={Diff-foley: Synchronized video-to-audio synthesis with latent diffusion models},
  author={Luo, Simian and Yan, Chuanhao and Hu, Chenxu and Zhao, Hang},
  booktitle={NeurIPS},
  year={2024}
}

@phdthesis{s19,
  title={Matrix theory: optimization, concentration, and algorithms},
  author={Song, Zhao},
  year={2019},
  school={The University of Texas at Austin}
}

@article{ccl+25,
  title={Force Matching with Relativistic Constraints: A Physics-Inspired Approach to Stable and Efficient Generative Modeling},
  author={Cao, Yang and Chen, Bo and Li, Xiaoyu and Liang, Yingyu and Sha, Zhizhou and Shi, Zhenmei and Song, Zhao and Wan, Mingda},
  journal={arXiv preprint arXiv:2502.08150},
  year={2025}
}

@article{cs25,
  title={Sorsa: Singular values and orthonormal regularized singular vectors adaptation of large language models},
  author={Cao, Yang and Song, Zhao},
  journal={arXiv preprint arXiv:2409.00055},
  year={2025}
}

@article{csy25,
  title={Video Latent Flow Matching: Optimal Polynomial Projections for Video Interpolation and Extrapolation},
  author={Cao, Yang and Song, Zhao and Yang, Chiwun},
  journal={arXiv preprint arXiv:2502.00500},
  year={2025}
}

@article{cgh+25,
  title={Text-to-Image Diffusion Models Cannot Count, and Prompt Refinement Cannot Help},
  author={Cao, Yuefan and Guo, Xuyang and Huo, Jiayan and Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Zhang, Jiahao and Zhuang, Zhen},
  journal={arXiv preprint arXiv:2503.06884},
  year={2025}
}

@article{ghh+25,
  title={Can You Count to Nine? A Human Evaluation Benchmark for Counting Limits in Modern Text-to-Video Models},
  author={Guo, Xuyang and Huang, Zekai and Huo, Jiayan and Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Zhang, Jiahao},
  journal={arXiv preprint arXiv:2504.04051},
  year={2025}
}

@article{ghs+25_physical,
  title={T2VPhysBench: A First-Principles Benchmark for Physical Consistency in Text-to-Video Generation},
  author={Guo, Xuyang and Huo, Jiayan and Shi, Zhenmei and Song, Zhao and Zhang, Jiahao and Zhao, Jiale},
  journal={arXiv preprint arXiv:2505.00337},
  year={2025}
}

@article{ghs+25_text,
  title={T2VTextBench: A Human Evaluation Benchmark for Textual Control in Video Generation Models}, 
  author={Xuyang Guo and Jiayan Huo and Zhenmei Shi and Zhao Song and Jiahao Zhang and Jiale Zhao},
  journal={arXiv preprint arXiv:2505.04946},
  year={2025}
}

@article{lzw+24,
  title={Score-based Generative Diffusion Models for Social Recommendations},
  author={Liu, Chengyi and Zhang, Jiahao and Wang, Shijie and Fan, Wenqi and Li, Qing},
  journal={arXiv preprint arXiv:2412.15579},
  year={2024}
}

@inproceedings{hwl+24,
  title={On Statistical Rates of Conditional Diffusion Transformers: Approximation, Estimation and Minimax Optimality},
  author={Hu, Jerry Yao-Chieh and Wu, Weimin and Lee, Yi-Chen and Huang, Yu-Chao and Chen, Minshuo and Liu, Han},
  booktitle={ICLR},
  year={2025}
}

@article{wxhl24,
  title={AlignAb: Pareto-Optimal Energy Alignment for Designing Nature-Like Antibodies},
  author={Wen, Yibo and Xu, Chenwei and Hu, Jerry Yao-Chieh and Liu, Han},
  journal={arXiv preprint arXiv:2412.20984},
  year={2024}
}

@article{rpcs23,
  title={A theoretical justification for image inpainting using denoising diffusion probabilistic models},
  author={Rout, Litu and Parulekar, Advait and Caramanis, Constantine and Shakkottai, Sanjay},
  journal={arXiv preprint arXiv:2302.01217},
  year={2023}
}

@inproceedings{hsk+24,
  title={Computational Limits of Low-Rank Adaptation (LoRA) Fine-Tuning for Transformer Models},
  author={Hu, Jerry Yao-Chieh and Su, Maojiang and Kuo, En-Jui and Song, Zhao and Liu, Han},
  booktitle={ICLR},
  year={2025}
}

@inproceedings{hwg+24,
  title={Fundamental Limits of Prompt Tuning Transformers: Universality, Capacity and Efficiency},
  author={Hu, Jerry Yao-Chieh and Wang, Wei-Po and Gilani, Ammar and Li, Chenyang and Song, Zhao and Liu, Han},
  booktitle={ICLR},
  year={2025}
}

@inproceedings{wsh+24,
  title={In-Context Deep Learning via Transformer Models},
  author={Wu, Weimin and Su, Maojiang and Hu, Jerry Yao-Chieh and Song, Zhao and Liu, Han},
  booktitle={ICML},
  year={2025},
}

@article{lwz+22,
  title={EvoGAN: An evolutionary computation assisted GAN},
  author={Liu, Feng and Wang, Hanyang and Zhang, Jiahao and Fu, Ziwang and Zhou, Aimin and Qi, Jiayin and Li, Zhibin},
  journal={Neurocomputing},
  year={2022},
}

@inproceedings{zlz21,
  title={Off-tanet: A lightweight neural micro-expression recognizer with optical flow features and integrated attention mechanism},
  author={Zhang, Jiahao and Liu, Feng and Zhou, Aimin},
  booktitle={PRICAI},
  year={2021},
}

@article{bdk+23,
  title={Stable video diffusion: Scaling latent video diffusion models to large datasets},
  author={Blattmann, Andreas and Dockhorn, Tim and Kulal, Sumith and Mendelevitch, Daniel and Kilian, Maciej and Lorenz, Dominik and Levi, Yam and English, Zion and Voleti, Vikram and Letts, Adam and others},
  journal={arXiv preprint arXiv:2311.15127},
  year={2023}
}

@inproceedings{dbk+20,
  title={An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale},
  author={Dosovitskiy, Alexey and Beyer, Lucas and Kolesnikov, Alexander and Weissenborn, Dirk and Zhai, Xiaohua and Unterthiner, Thomas and Dehghani, Mostafa and Minderer, Matthias and Heigold, Georg and Gelly, Sylvain and others},
  booktitle={ICLR},
  year={2021}
}

@article{ytz+24,
  title={Cogvideox: Text-to-video diffusion models with an expert transformer},
  author={Yang, Zhuoyi and Teng, Jiayan and Zheng, Wendi and Ding, Ming and Huang, Shiyu and Xu, Jiazheng and Yang, Yuanming and Hong, Wenyi and Zhang, Xiaohan and Feng, Guanyu and others},
  journal={arXiv preprint arXiv:2408.06072},
  year={2024}
}

@inproceedings{kzl+23,
  title={Imagic: Text-based real image editing with diffusion models},
  author={Kawar, Bahjat and Zada, Shiran and Lang, Oran and Tov, Omer and Chang, Huiwen and Dekel, Tali and Mosseri, Inbar and Irani, Michal},
  booktitle={CVPR},
  year={2023}
}

@inproceedings{gpv+24,
  title={Renoise: Real image inversion through iterative noising},
  author={Garibi, Daniel and Patashnik, Or and Voynov, Andrey and Averbuch-Elor, Hadar and Cohen-Or, Daniel},
  booktitle={ECCV},
  year={2024},
}

@inproceedings{wxf+23,
  title={Diffusion recommender model},
  author={Wang, Wenjie and Xu, Yiyan and Feng, Fuli and Lin, Xinyu and He, Xiangnan and Chua, Tat-Seng},
  booktitle={SIGIR},
  year={2023}
}

@inproceedings{zwx+24,
  title={Denoising diffusion recommender model},
  author={Zhao, Jujia and Wang, Wenjie and Xu, Yiyan and Sun, Teng and Feng, Fuli and Chua, Tat-Seng},
  booktitle={SIGIR},
  year={2024}
}

@inproceedings{ccp+23,
  title={Intriguing properties of synthetic images: from generative adversarial networks to diffusion models},
  author={Corvi, Riccardo and Cozzolino, Davide and Poggi, Giovanni and Nagano, Koki and Verdoliva, Luisa},
  booktitle={CVPR},
  year={2023}
}

@article{gpm+20,
  title={Generative adversarial networks},
  author={Goodfellow, Ian and Pouget-Abadie, Jean and Mirza, Mehdi and Xu, Bing and Warde-Farley, David and Ozair, Sherjil and Courville, Aaron and Bengio, Yoshua},
  journal={Communications of the ACM},
  year={2020},
}

@inproceedings{kw14,
  title={Auto-encoding variational bayes},
  author={Kingma, Diederik P and Welling, Max},
  year={2014},
  booktitle={ICLR}
}

@article{cssz25,
  title={Provable Failure of Language Models in Learning Majority Boolean Logic via Gradient Descent},
  author={Chen, Bo and Shi, Zhenmei and Song, Zhao and Zhang, Jiahao},
  journal={arXiv preprint arXiv:2504.04702},
  year={2025}
}

@article{hzs+25,
  title={Minimalist Softmax Attention Provably Learns Constrained Boolean Functions},
  author={Hu, Jerry Yao-Chieh and Zhang, Xiwen and Su, Maojiang and Song, Zhao and Liu, Han},
  journal={arXiv preprint arXiv:2505.19531},
  year={2025}
}

@inproceedings{
ks25,
title={Transformers Provably Solve Parity Efficiently with Chain of Thought},
author={Juno Kim and Taiji Suzuki},
booktitle={ICLR},
year={2025},
}

@inproceedings{as23,
  title={Fast attention requires bounded entries},
  author={Alman, Josh and Song, Zhao},
  booktitle={NeurIPS},
  year={2023}
}

@inproceedings{as24,
  title={The fine-grained complexity of gradient computation for training large language models},
  author={Alman, Josh and Song, Zhao},
  booktitle={NeurIPS},
  year={2024}
}


@inproceedings{as24_tensor,
  title={How to Capture Higher-order Correlations? Generalizing Matrix Softmax Attention to Kronecker Computation},
  author={Alman, Josh and Song, Zhao},
  booktitle={ICLR},
  year={2024}
}

@article{as25,
  title={Fast RoPE Attention: Combining the Polynomial Method and Fast Fourier Transform},
  author={Alman, Josh and Song, Zhao},
  journal={arXiv preprint arXiv:2505.11892},
  year={2025}
}

@article{lls+25,
  title={On the Computational Capability of Graph Neural Networks: A Circuit Complexity Bound Perspective},
  author={Li, Xiaoyu and Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Wang, Wei and Zhang, Jiahao},
  journal={arXiv preprint arXiv:2501.06444},
  year={2025}
}

@inproceedings{llzm24,
  title={Chain of thought empowers transformers to solve inherently serial problems},
  author={Li, Zhiyuan and Liu, Hong and Zhou, Denny and Ma, Tengyu},
  booktitle={ICLR},
  year={2024}
}

@inproceedings{whhz+25,
  author    = {Wu, Weimin and Hsiao, Teng{-}Yun and Hu, Jerry Yao{-}Chieh and Zhang, Wenxin and Liu, Han},
  title     = {In-Context Learning as Conditioned Associative Memory Retrieval},
  booktitle={ICML},
  year={2025}
}

@inproceedings{dvk22,
  title={Score-based generative modeling with critically-damped langevin diffusion},
  author={Dockhorn, Tim and Vahdat, Arash and Kreis, Karsten},
  booktitle={ICLR},
  year={2022}
}

@article{wzkz23,
  title={Fast diffusion model},
  author={Wu, Zike and Zhou, Pan and Kawaguchi, Kenji and Zhang, Hanwang},
  journal={arXiv preprint arXiv:2306.06991},
  year={2023}
}

@article{as25_rank,
  title={Only large weights (and not skip connections) can prevent the perils of rank collapse},
  author={Alman, Josh and Song, Zhao},
  journal={arXiv preprint arXiv:2505.16284},
  year={2025}
}

%%% Yang: Need to add the following citations

@inproceedings{lls+24_grok,
  title={Fourier Circuits in Neural Networks and Transformers: A Case Study of Modular Arithmetic with Multiple Inputs},
  author={Li, Chenyang and Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Zhou, Tianyi},
  booktitle={AISTATS},
  year={2025}
}

@inproceedings{cll+24_icl,
  title={Bypassing the Exponential Dependency: Looped Transformers Efficiently Learn In-context by Multi-step Gradient Descent},
  author={Chen, Bo and Li, Xiaoyu and Liang, Yingyu and Shi, Zhenmei and Song, Zhao},
  booktitle={AISTATS},
  year={2025}
}

@inproceedings{cll+25_var,
  title={Fundamental Limits of Visual Autoregressive Transformers: Universal Approximation Abilities},
  author={Chen, Yifang and Li, Xiaoyu and Liang, Yingyu and Shi, Zhenmei and Song, Zhao},
  booktitle={ICML},
  year={2025}
}

@article{lhsl25,
  title={Attention Mechanism, Max-Affine Partition, and Universal Approximation},
  author={Liu, Hude and Hu, Jerry Yao-Chieh and Song, Zhao and Liu, Han},
  journal={arXiv preprint arXiv:2504.19901},
  year={2025}
}

@article{hlcw+25,
  title={Universal Approximation with Softmax Attention},
  author={Hu, Jerry Yao-Chieh and Liu, Hude and Chen, Hong-Yu and Wu, Weimin and Liu, Han},
  journal={arXiv preprint arXiv:2504.15956},
  year={2025}
}

@inproceedings{kzld22,
  title={Universal approximation under constraints is possible with transformers},
  author={Kratsios, Anastasis and Zamanlooy, Behnoosh and Liu, Tianlin and Dokmani{\'c}, Ivan},
  booktitle={ICLR},
  year={2022}
}