@article{kajitsuka2023transformers,
  title={Are Transformers with One Layer Self-Attention Using Low-Rank Weight Matrices Universal Approximators?},
  author={Kajitsuka, Tokio and Sato, Issei},
  journal={arXiv preprint arXiv:2307.14023},
  year={2023}
}
@article{gu2023mamba,
  title={Mamba: Linear-time sequence modeling with selective state spaces},
  author={Gu, Albert and Dao, Tri},
  journal={arXiv preprint arXiv:2312.00752},
  year={2023}
}
@article{gu2020hippo,
  title={Hippo: Recurrent memory with optimal polynomial projections},
  author={Gu, Albert and Dao, Tri and Ermon, Stefano and Rudra, Atri and R{\'e}, Christopher},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={1474--1487},
  year={2020}
}
@article{gu2021efficiently,
  title={Efficiently modeling long sequences with structured state spaces},
  author={Gu, Albert and Goel, Karan and R{\'e}, Christopher},
  journal={arXiv preprint arXiv:2111.00396},
  year={2021}
}
@article{gu2021combining,
  title={Combining recurrent, convolutional, and continuous-time models with linear state space layers},
  author={Gu, Albert and Johnson, Isys and Goel, Karan and Saab, Khaled and Dao, Tri and Rudra, Atri and R{\'e}, Christopher},
  journal={Advances in neural information processing systems},
  volume={34},
  pages={572--585},
  year={2021}
}
@article{gupta2022diagonal,
  title={Diagonal state spaces are as effective as structured state spaces},
  author={Gupta, Ankit and Gu, Albert and Berant, Jonathan},
  journal={Advances in Neural Information Processing Systems},
  volume={35},
  pages={22982--22994},
  year={2022}
}
@inproceedings{li2024videomamba,
  title={Videomamba: State space model for efficient video understanding},
  author={Li, Kunchang and Li, Xinhao and Wang, Yi and He, Yinan and Wang, Yali and Wang, Limin and Qiao, Yu},
  booktitle={European Conference on Computer Vision},
  pages={237--255},
  year={2024},
  organization={Springer}
}
@article{kosma2023time,
  title={Time-parameterized convolutional neural networks for irregularly sampled time series},
  author={Kosma, Chrysoula and Nikolentzos, Giannis and Vazirgiannis, Michalis},
  journal={arXiv preprint arXiv:2308.03210},
  year={2023}
}
@article{muca2024theoretical,
  title={Theoretical foundations of deep selective state-space models},
  author={Muca Cirone, Nicola and Orvieto, Antonio and Walker, Benjamin and Salvi, Cristopher and Lyons, Terry},
  journal={Advances in Neural Information Processing Systems},
  volume={37},
  pages={127226--127272},
  year={2024}
}
@article{smith2022simplified,
  title={Simplified state space layers for sequence modeling},
  author={Smith, Jimmy TH and Warrington, Andrew and Linderman, Scott W},
  journal={arXiv preprint arXiv:2208.04933},
  year={2022}
}
@inproceedings{goel2022s,
  title={It’s raw! audio generation with state-space models},
  author={Goel, Karan and Gu, Albert and Donahue, Chris and R{\'e}, Christopher},
  booktitle={International conference on machine learning},
  pages={7616--7633},
  year={2022},
  organization={PMLR}
}
@article{nguyen2022s4nd,
  title={S4nd: Modeling images and videos as multidimensional signals with state spaces},
  author={Nguyen, Eric and Goel, Karan and Gu, Albert and Downs, Gordon and Shah, Preey and Dao, Tri and Baccus, Stephen and R{\'e}, Christopher},
  journal={Advances in neural information processing systems},
  volume={35},
  pages={2846--2861},
  year={2022}
}
@inproceedings{saon2023diagonal,
  title={Diagonal state space augmented transformers for speech recognition},
  author={Saon, George and Gupta, Ankit and Cui, Xiaodong},
  booktitle={ICASSP 2023-2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
  pages={1--5},
  year={2023},
  organization={IEEE}
}
@article{tay2020long,
  title={Long range arena: A benchmark for efficient transformers},
  author={Tay, Yi and Dehghani, Mostafa and Abnar, Samira and Shen, Yikang and Bahri, Dara and Pham, Philip and Rao, Jinfeng and Yang, Liu and Ruder, Sebastian and Metzler, Donald},
  journal={arXiv preprint arXiv:2011.04006},
  year={2020}
}
@inproceedings{siivola2003state,
  title={A state-space method for language modeling},
  author={Siivola, Vesa and Honkela, Antti},
  booktitle={2003 IEEE Workshop on Automatic Speech Recognition and Understanding (IEEE Cat. No. 03EX721)},
  pages={548--553},
  year={2003},
  organization={IEEE}
}
@article{fu2022hungry,
  title={Hungry hungry hippos: Towards language modeling with state space models},
  author={Fu, Daniel Y and Dao, Tri and Saab, Khaled K and Thomas, Armin W and Rudra, Atri and R{\'e}, Christopher},
  journal={arXiv preprint arXiv:2212.14052},
  year={2022}
}
@article{wang2022pretraining,
  title={Pretraining without attention},
  author={Wang, Junxiong and Yan, Jing Nathan and Gu, Albert and Rush, Alexander M},
  journal={arXiv preprint arXiv:2212.10544},
  year={2022}
}
@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}
@article{vo2024demystifying,
  title={Demystifying the Token Dynamics of Deep Selective State Space Models},
  author={Vo, Thieu N and Pham, Tung D and Tong, Xin T and Nguyen, Tan Minh},
  journal={arXiv preprint arXiv:2410.03292},
  year={2024}
}
@article{geshkovski2023emergence,
  title={The emergence of clusters in self-attention dynamics},
  author={Geshkovski, Borjan and Letrouit, Cyril and Polyanskiy, Yury and Rigollet, Philippe},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={57026--57037},
  year={2023}
}
@article{kratsios2021universal,
  title={Universal approximation under constraints is possible with transformers},
  author={Kratsios, Anastasis and Zamanlooy, Behnoosh and Liu, Tianlin and Dokmani{\'c}, Ivan},
  journal={arXiv preprint arXiv:2110.03303},
  year={2021}
}
@inproceedings{edelman2022inductive,
  title={Inductive biases and variable creation in self-attention mechanisms},
  author={Edelman, Benjamin L and Goel, Surbhi and Kakade, Sham and Zhang, Cyril},
  booktitle={International Conference on Machine Learning},
  pages={5793--5831},
  year={2022},
  organization={PMLR}
}
@article{jiang2024approximation,
  title={Approximation rate of the transformer architecture for sequence modeling},
  author={Jiang, Haotian and Li, Qianxiao},
  journal={Advances in Neural Information Processing Systems},
  volume={37},
  pages={68926--68955},
  year={2024}
}
@inproceedings{orvieto2023resurrecting,
  title={Resurrecting recurrent neural networks for long sequences},
  author={Orvieto, Antonio and Smith, Samuel L and Gu, Albert and Fernando, Anushan and Gulcehre, Caglar and Pascanu, Razvan and De, Soham},
  booktitle={International Conference on Machine Learning},
  pages={26670--26698},
  year={2023},
  organization={PMLR}
}
@article{zou2006adaptive,
  title={The adaptive lasso and its oracle properties},
  author={Zou, Hui},
  journal={Journal of the American statistical association},
  volume={101},
  number={476},
  pages={1418--1429},
  year={2006},
  publisher={Taylor \& Francis}
}
@article{hoerl1970ridge,
  title={Ridge regression: Biased estimation for nonorthogonal problems},
  author={Hoerl, Arthur E and Kennard, Robert W},
  journal={Technometrics},
  volume={12},
  number={1},
  pages={55--67},
  year={1970},
  publisher={Taylor \& Francis}
}
@inproceedings{pascanu2013difficulty,
  title={On the difficulty of training recurrent neural networks},
  author={Pascanu, Razvan and Mikolov, Tomas and Bengio, Yoshua},
  booktitle={International conference on machine learning},
  pages={1310--1318},
  year={2013},
  organization={Pmlr}
}
@article{merity2017regularizing,
  title={Regularizing and optimizing LSTM language models},
  author={Merity, Stephen and Keskar, Nitish Shirish and Socher, Richard},
  journal={arXiv preprint arXiv:1708.02182},
  year={2017}
}
@article{miyato2018spectral,
  title={Spectral normalization for generative adversarial networks},
  author={Miyato, Takeru and Kataoka, Toshiki and Koyama, Masanori and Yoshida, Yuichi},
  journal={arXiv preprint arXiv:1802.05957},
  year={2018}
}
@article{arora2019implicit,
  title={Implicit regularization in deep matrix factorization},
  author={Arora, Sanjeev and Cohen, Nadav and Hu, Wei and Luo, Yuping},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}
@article{menon2024geometry,
  title={The geometry of the deep linear network},
  author={Menon, Govind},
  journal={arXiv preprint arXiv:2411.09004},
  year={2024}
}
@article{wang2023state,
  title={State-space models with layer-wise nonlinearity are universal approximators with exponential decaying memory},
  author={Wang, Shida and Xue, Beichen},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={74021--74038},
  year={2023}
}
@article{arora2018convergence,
  title={A convergence analysis of gradient descent for deep linear neural networks},
  author={Arora, Sanjeev and Cohen, Nadav and Golowich, Noah and Hu, Wei},
  journal={arXiv preprint arXiv:1810.02281},
  year={2018}
}
@article{bah2022learning,
  title={Learning deep linear neural networks: Riemannian gradient flows and convergence to global minimizers},
  author={Bah, Bubacarr and Rauhut, Holger and Terstiege, Ulrich and Westdickenberg, Michael},
  journal={Information and Inference: A Journal of the IMA},
  volume={11},
  number={1},
  pages={307--353},
  year={2022},
  publisher={Oxford University Press}
}
@article{gruber2024role,
  title={On the Role of Initialization on the Implicit Bias in Deep Linear Networks},
  author={Gruber, Oria and Avron, Haim},
  journal={arXiv preprint arXiv:2402.02454},
  year={2024}
}
@article{smekal2024interplay,
  title={On the interplay between learning and memory in deep state space models},
  author={Smekal, Jakub and Zucchet, Nicolas and Biderman, Dan and Buchanan, E Kelly and Smith, Jimmy TH and Linderman, Scott},
  year={2024}
}
@article{saxe2013exact,
  title={Exact solutions to the nonlinear dynamics of learning in deep linear neural networks},
  author={Saxe, Andrew M and McClelland, James L and Ganguli, Surya},
  journal={arXiv preprint arXiv:1312.6120},
  year={2013}
}
@article{saxe2019mathematical,
  title={A mathematical theory of semantic development in deep neural networks},
  author={Saxe, Andrew M and McClelland, James L and Ganguli, Surya},
  journal={Proceedings of the National Academy of Sciences},
  volume={116},
  number={23},
  pages={11537--11546},
  year={2019},
  publisher={National Academy of Sciences}
}
@article{jorda2005estimation,
  title={Estimation and inference of impulse responses by local projections},
  author={Jord{\`a}, {\`O}scar},
  journal={American economic review},
  volume={95},
  number={1},
  pages={161--182},
  year={2005},
  publisher={American Economic Association}
}
@article{xiao2017fashion,
  title={Fashion-mnist: a novel image dataset for benchmarking machine learning algorithms},
  author={Xiao, Han and Rasul, Kashif and Vollgraf, Roland},
  journal={arXiv preprint arXiv:1708.07747},
  year={2017}
}