@article{Rougier:2018,
  doi = {10.1038/d41586-018-04628-w},
  year = {2018},
  month = apr,
  publisher = {Springer Nature},
  volume = {556},
  number = {7701},
  pages = {309--309},
  author = {Rougier, Nicolas P. and Hinsen, Konrad},
  title = {Code reviewing puts extra demands on referees },
  journal = {Nature},
}

@article{Science:2018,
  doi = {10.1126/science.359.6377.725},
  author = {Hutson, Matthew},
  url = {http://science.sciencemag.org/content/359/6377/725},
  year = {2018},
  month = feb,
  volume = {359},
  number = {6377},
  title = {Artificial intelligence faces a replication crisis},
  journal = {Science},
}

@Misc{Roboto:2011,
  author =       {Christian Robertson},
  title =        {The Roboto family of fonts (Google)},
  url =          {https://github.com/google/roboto},
  year =         2011,
  note =         {Apache License, verison 2.0},
}

@Misc{SourceSerifPro:2014,
  author =       {Frank Grießhammer},
  title =        {Source Serif Pro (Adobe Systems)},
  url =          {https://github.com/adobe-fonts/source-serif-pro},
  year =         2014,
  note =         {SIL Open Font License, version 1.1},
}

@Misc{SourceCodePro:2012,
  author =       {Paul D. Hunt},
  title =        {Source Code Pro (Adobe Systems)},
  url =          {https://github.com/adobe-fonts/source-code-pro},
  year =         2012,
  note =         {SIL Open Font License, version 1.1},
}

@article{Topalidou:2015,
  author =       {Topalidou, Meropi and Rougier, Nicolas P.},
  title =        {{[Re] Interaction between cognitive and motor cortico-basal
                  ganglia loops during decision making: a computational study}},
  journal =      {ReScience},
  year =         2015,
  volume =       1,
  number =       1,
  doi =          {10.5281/zenodo.27944},
}

@article{Rougier:2017,
  doi =          {10.7717/peerj-cs.142},
  author =       {Nicolas P. Rougier and Konrad Hinsen and Frédéric Alexandre
                  and Thomas Arildsen and Lorena Barba and Fabien
                  C. Y. Benureau and C. Titus Brown and Pierre de Buyl and Ozan
                  Caglayan and Andrew P. Davison and Marc André Delsuc and
                  Georgios Detorakis and Alexandra K. Diem and Damien Drix and
                  Pierre Enel and Benoît Girard and Olivia Guest and Matt
                  G. Hall and Rafael Neto Henriques and Xavier Hinaut and Kamil
                  S Jaron and Mehdi Khamassi and Almar Klein and Tiina Manninen
                  and Pietro Marchesi and Dan McGlinn and Christoph Metzner and
                  Owen L. Petchey and Hans Ekkehard Plesser and Timothée Poisot
                  and Karthik Ram and Yoav Ram and Etienne Roesch and Cyrille
                  Rossant and Vahid Rostami and Aaron Shifman and Joseph
                  Stachelek and Marcel Stimberg and Frank Stollmeier and
                  Federico Vaggi and Guillaume Viejo and Julien Vitay and Anya
                  Vostinar and Roman Yurchak and Tiziano Zito},
  title =        {{Sustainable computational science: the ReScience
                  initiative}},
  journal =      {{PeerJ} Computer Science},
  month =        12,
  volume =       3,
  pages =        {e142},
  year =         2017,
  github =       {https://github.com/ReScience/ReScience-article-2},
  keywords =     {journal},
  tags =         {OS},
}

@article{Sinha:2022,
  author = {Sinha, Koustuv and Dodge, Jesse and Luccioni, Sasha and Forde, Jessica Zosa and Raparthy, Sharath Chandra and Pineau, Joelle and Stojnic, Robert},
  title = {{ML Reproducibility Challenge 2021}},
  journal = {ReScience C},
  year = {2022},
  month = may,
  volume = {8},
  number = {2},
  pages = {{#48}},
  doi = {10.5281/zenodo.6574723},
  url = {https://zenodo.org/record/6574723/files/article.pdf},
  code_url = {},
  code_doi = {},
  code_swh = {},
  data_url = {},
  data_doi = {},
  review_url = {},
  type = {Editorial},
  language = {Python},
  domain = {ML Reproducibility Challenge 2021},
  keywords = {rescience c, machine learning, deep learning, python, pytorch}
}

@article{Sinha:2021,
  author = {Sinha, Koustuv and Dodge, Jesse and Luccioni, Sasha and Forde, Jessica Zosa and Stojnic, Robert and Pineau, Joelle},
  title = {{ML Reproducibility Challenge 2020}},
  journal = {ReScience C},
  year = {2021},
  month = may,
  volume = {7},
  number = {2},
  pages = {{#1}},
  doi = {10.5281/zenodo.4833117},
  url = {https://zenodo.org/record/4833117/files/article.pdf},
  code_url = {},
  code_doi = {},
  code_swh = {},
  data_url = {},
  data_doi = {},
  review_url = {},
  type = {Editorial},
  language = {},
  domain = {},
  keywords = {machine learning, neurips, reproducibility challenge}
}

@article{Sinha:2020,
  author = {Sinha, Koustuv and Pineau, Joelle and Forde, Jessica and Ke, Rosemary Nan and Larochelle, Hugo},
  title = {{NeurIPS 2019 Reproducibility Challenge}},
  journal = {ReScience C},
  year = {2020},
  month = may,
  volume = {6},
  number = {2},
  pages = {{#11}},
  doi = {10.5281/zenodo.3818627},
  url = {https://zenodo.org/record/3818627/files/article.pdf},
  code_url = {},
  code_doi = {},
  code_swh = {},
  data_url = {},
  data_doi = {},
  review_url = {},
  type = {Editorial},
  language = {},
  domain = {},
  keywords = {machine learning, neurips, reproducibility challenge}
}

@article{Pineau:2019,
  author = {Pineau, Joelle and Sinha, Koustuv and Fried, Genevieve and Ke, Rosemary Nan and Larochelle, Hugo},
  title = {{ICLR Reproducibility Challenge 2019}},
  journal = {ReScience C},
  year = {2019},
  month = may,
  volume = {5},
  number = {2},
  pages = {{#5}},
  doi = {10.5281/zenodo.3158244},
  url = {https://zenodo.org/record/3158244/files/article.pdf},
  code_url = {},
  code_doi = {},
  data_url = {},
  data_doi = {},
  review_url = {https://github.com/ReScience/submissions/issues/5},
  type = {Editorial},
  language = {},
  domain = {},
  keywords = {machine learning, ICLR, reproducibility challenge}
}

@inproceedings{shaw-etal-2021-compositional,
    title = "Compositional Generalization and Natural Language Variation: Can a Semantic Parsing Approach Handle Both?",
    author = "Shaw, Peter  and
      Chang, Ming-Wei  and
      Pasupat, Panupong  and
      Toutanova, Kristina",
    booktitle = "Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)",
    month = aug,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.acl-long.75",
    doi = "10.18653/v1/2021.acl-long.75",
    pages = "922--938",
}

@inproceedings{kim-linzen-2020-cogs,
    title = "{COGS}: A Compositional Generalization Challenge Based on Semantic Interpretation",
    author = "Kim, Najoung  and
      Linzen, Tal",
    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    month = nov,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.emnlp-main.731",
    doi = "10.18653/v1/2020.emnlp-main.731",
    pages = "9087--9105",
    abstract = "Natural language is characterized by compositionality: the meaning of a complex expression is constructed from the meanings of its constituent parts. To facilitate the evaluation of the compositional abilities of language processing architectures, we introduce COGS, a semantic parsing dataset based on a fragment of English. The evaluation portion of COGS contains multiple systematic gaps that can only be addressed by compositional generalization; these include new combinations of familiar syntactic structures, or new combinations of familiar words and familiar structures. In experiments with Transformers and LSTMs, we found that in-distribution accuracy on the COGS test set was near-perfect (96{--}99{\%}), but generalization accuracy was substantially lower (16{--}35{\%}) and showed high sensitivity to random seed (+-6{--}8{\%}). These findings indicate that contemporary standard NLP models are limited in their compositional generalization capacity, and position COGS as a good way to measure progress.",
}

@inproceedings{
kim2021sequencetosequence,
title={Sequence-to-Sequence Learning with Latent Neural Grammars},
author={Yoon Kim},
booktitle={Advances in Neural Information Processing Systems},
editor={A. Beygelzimer and Y. Dauphin and P. Liang and J. Wortman Vaughan},
year={2021},
url={https://openreview.net/forum?id=pbfAgoc_l2w}
}

@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@inproceedings{tang2001using,
  title={Using multiple clause constructors in inductive logic programming for semantic parsing},
  author={Tang, Lappoon R and Mooney, Raymond J},
  booktitle={European Conference on Machine Learning},
  pages={466--477},
  year={2001},
  organization={Springer}
}

@inproceedings{zelle1996learning,
  title={Learning to parse database queries using inductive logic programming},
  author={Zelle, John M and Mooney, Raymond J},
  booktitle={Proceedings of the national conference on artificial intelligence},
  pages={1050--1055},
  year={1996}
}

@article{raffel2020exploring,
  title={Exploring the limits of transfer learning with a unified text-to-text transformer.},
  author={Raffel, Colin and Shazeer, Noam and Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and Li, Wei and Liu, Peter J and others},
  journal={J. Mach. Learn. Res.},
  volume={21},
  number={140},
  pages={1--67},
  year={2020}
}

@inproceedings{
keysers2020measuring,
title={Measuring Compositional Generalization: A Comprehensive Method on Realistic Data},
author={Daniel Keysers and Nathanael Sch{\"a}rli and Nathan Scales and Hylke Buisman and Daniel Furrer and Sergii Kashubin and Nikola Momchev and Danila Sinopalnikov and Lukasz Stafiniak and Tibor Tihon and Dmitry Tsarkov and Xiao Wang and Marc van Zee and Olivier Bousquet},
booktitle={International Conference on Learning Representations},
year={2020},
url={https://openreview.net/forum?id=SygcCnNKwr}
}
@inproceedings{lake2018generalization,
  title={Generalization without systematicity: On the compositional skills of sequence-to-sequence recurrent networks},
  author={Lake, Brenden and Baroni, Marco},
  booktitle={International conference on machine learning},
  pages={2873--2882},
  year={2018},
  organization={PMLR}
}

inproceedings{lake2018generalization,
  title={Generalization without systematicity: On the compositional skills of sequence-to-sequence recurrent networks},
  author={Lake, Brenden and Baroni, Marco},
  booktitle={International conference on machine learning},
  pages={2873--2882},
  year={2018},
  organization={PMLR}
}



@article{dessi2019cnns,
  title={CNNs found to jump around more skillfully than RNNs: Compositional generalization in seq2seq convolutional networks},
  author={Dess{\`\i}, Roberto and Baroni, Marco},
  journal={arXiv preprint arXiv:1905.08527},
  year={2019}
}

@article{russin2019compositional,
  title={Compositional generalization in a deep seq2seq model by separating syntax and semantics},
  author={Russin, Jake and Jo, Jason and O'Reilly, Randall C and Bengio, Yoshua},
  journal={arXiv preprint arXiv:1904.09708},
  year={2019}
}

@article{li2019compositional,
  title={Compositional generalization for primitive substitutions},
  author={Li, Yuanpeng and Zhao, Liang and Wang, Jianyu and Hestness, Joel},
  journal={arXiv preprint arXiv:1910.02612},
  year={2019}
}

@inproceedings{gordon2020permutation,
  title={Permutation equivariant models for compositional generalization in language},
  author={Gordon, Jonathan and Lopez-Paz, David and Baroni, Marco and Bouchacourt, Diane},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@article{liu2020compositional,
  title={Compositional generalization by learning analytical expressions},
  author={Liu, Qian and An, Shengnan and Lou, Jian-Guang and Chen, Bei and Lin, Zeqi and Gao, Yan and Zhou, Bin and Zheng, Nanning and Zhang, Dongmei},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={11416--11427},
  year={2020}
}

@article{nye2020learning,
  title={Learning compositional rules via neural program synthesis},
  author={Nye, Maxwell and Solar-Lezama, Armando and Tenenbaum, Josh and Lake, Brenden M},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={10832--10842},
  year={2020}
}

@article{chen2020compositional,
  title={Compositional generalization via neural-symbolic stack machines},
  author={Chen, Xinyun and Liang, Chen and Yu, Adams Wei and Song, Dawn and Zhou, Denny},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={1690--1701},
  year={2020}
}

@article{yu2018spider,
  title={Spider: A large-scale human-labeled dataset for complex and cross-domain semantic parsing and text-to-sql task},
  author={Yu, Tao and Zhang, Rui and Yang, Kai and Yasunaga, Michihiro and Wang, Dongxu and Li, Zifan and Ma, James and Li, Irene and Yao, Qingning and Roman, Shanelle and others},
  journal={arXiv preprint arXiv:1809.08887},
  year={2018}
}

@article{hochreiter1997long,
  title={Long short-term memory},
  author={Hochreiter, Sepp and Schmidhuber, J{\"u}rgen},
  journal={Neural computation},
  volume={9},
  number={8},
  pages={1735--1780},
  year={1997},
  publisher={MIT press}
}

@inproceedings{Lake2018GeneralizationWS,
  title={Generalization without Systematicity: On the Compositional Skills of
         Sequence-to-Sequence Recurrent Networks},
  author={Brenden M. Lake and Marco Baroni},
  booktitle={ICML},
  year={2018},
  url={https://arxiv.org/pdf/1711.00350.pdf},
}

@article{yu2018spider,
  title={Spider: A large-scale human-labeled dataset for complex and cross-domain semantic parsing and text-to-sql task},
  author={Yu, Tao and Zhang, Rui and Yang, Kai and Yasunaga, Michihiro and Wang, Dongxu and Li, Zifan and Ma, James and Li, Irene and Yao, Qingning and Roman, Shanelle and others},
  journal={arXiv preprint arXiv:1809.08887},
  year={2018}
}

@article{chen2020compositional,
  title={Compositional generalization via neural-symbolic stack machines},
  author={Chen, Xinyun and Liang, Chen and Yu, Adams Wei and Song, Dawn and Zhou, Denny},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={1690--1701},
  year={2020}
}

@article{nye2020learning,
  title={Learning compositional rules via neural program synthesis},
  author={Nye, Maxwell and Solar-Lezama, Armando and Tenenbaum, Josh and Lake, Brenden M},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={10832--10842},
  year={2020}
}

@article{liu2020compositional,
  title={Compositional generalization by learning analytical expressions},
  author={Liu, Qian and An, Shengnan and Lou, Jian-Guang and Chen, Bei and Lin, Zeqi and Gao, Yan and Zhou, Bin and Zheng, Nanning and Zhang, Dongmei},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={11416--11427},
  year={2020}
}

@article{lake2019compositional,
  title={Compositional generalization through meta sequence-to-sequence learning},
  author={Lake, Brenden M},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}
@inproceedings{
Gordon2020Permutation,
title={Permutation Equivariant Models for Compositional Generalization in Language},
author={Jonathan Gordon and David Lopez-Paz and Marco Baroni and Diane Bouchacourt},
booktitle={International Conference on Learning Representations},
year={2020},
url={https://openreview.net/forum?id=SylVNerFvr}
}
@article{russin1904compositional,
  title={Compositional generalization in a deep seq2seq model by separating syntax and semantics. apr 2019},
  author={Russin, J and Jo, J and O’Reilly, RC and Bengio, Y},
  journal={URL http://arxiv. org/abs/1904},
  volume={9708}
}

@inproceedings{li-etal-2019-compositional,
    title = "Compositional Generalization for Primitive Substitutions",
    author = "Li, Yuanpeng  and
      Zhao, Liang  and
      Wang, Jianyu  and
      Hestness, Joel",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)",
    month = nov,
    year = "2019",
    address = "Hong Kong, China",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/D19-1438",
    doi = "10.18653/v1/D19-1438",
    pages = "4293--4302",
    abstract = "Compositional generalization is a basic mechanism in human language learning, but current neural networks lack such ability. In this paper, we conduct fundamental research for encoding compositionality in neural networks. Conventional methods use a single representation for the input sentence, making it hard to apply prior knowledge of compositionality. In contrast, our approach leverages such knowledge with two representations, one generating attention maps, and the other mapping attended input words to output symbols. We reduce the entropy in each representation to improve generalization. Our experiments demonstrate significant improvements over the conventional methods in five NLP tasks including instruction learning and machine translation. In the SCAN domain, it boosts accuracies from 14.0{\%} to 98.8{\%} in Jump task, and from 92.0{\%} to 99.7{\%} in TurnLeft task. It also beats human performance on a few-shot learning task. We hope the proposed approach can help ease future research towards human-level compositional language learning.",
}

@inproceedings{reddy-etal-2017-universal,
    title = "Universal Semantic Parsing",
    author = {Reddy, Siva  and
      T{\"a}ckstr{\"o}m, Oscar  and
      Petrov, Slav  and
      Steedman, Mark  and
      Lapata, Mirella},
    booktitle = "Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing",
    month = sep,
    year = "2017",
    address = "Copenhagen, Denmark",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/D17-1009",
    doi = "10.18653/v1/D17-1009",
    pages = "89--101",
    abstract = "Universal Dependencies (UD) offer a uniform cross-lingual syntactic representation, with the aim of advancing multilingual applications. Recent work shows that semantic parsing can be accomplished by transforming syntactic dependencies to logical forms. However, this work is limited to English, and cannot process dependency graphs, which allow handling complex phenomena such as control. In this work, we introduce UDepLambda, a semantic interface for UD, which maps natural language to logical forms in an almost language-independent fashion and can process dependency graphs. We perform experiments on question answering against Freebase and provide German and Spanish translations of the WebQuestions and GraphQuestions datasets to facilitate multilingual evaluation. Results show that UDepLambda outperforms strong baselines across languages and datasets. For English, it achieves a 4.9 F1 point improvement over the state-of-the-art on GraphQuestions.",
}

@inproceedings{kingma-ba-2015-adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik P. and Ba, Jimmy},
  booktitle    = {3rd International Conference for Learning Representations},
  year={2015}
}

@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}
@article{kim2022uncontrolled,
  title={Uncontrolled Lexical Exposure Leads to Overestimation of Compositional Generalization in Pretrained Models},
  author={Kim, Najoung and Linzen, Tal and Smolensky, Paul},
  journal={arXiv preprint arXiv:2212.10769},
  year={2022}
}

@article{csordas2021devil,
  title={The devil is in the detail: Simple tricks improve systematic generalization of transformers},
  author={Csord{\'a}s, R{\'o}bert and Irie, Kazuki and Schmidhuber, J{\"u}rgen},
  journal={arXiv preprint arXiv:2108.12284},
  year={2021}
}

@article{hupkes2022sotageneralisation, 
  author       = {Dieuwke Hupkes and
                  Mario Giulianelli and
                  Verna Dankers and
                  Mikel Artetxe and
                  Yanai Elazar and
                  Tiago Pimentel and
                  Christos Christodoulopoulos and
                  Karim Lasri and
                  Naomi Saphra and
                  Arabella Sinclair and
                  Dennis Ulmer and
                  Florian Schottmann and
                  Khuyagbaatar Batsuren and
                  Kaiser Sun and
                  Koustuv Sinha and
                  Leila Khalatbari and
                  Maria Ryskina and
                  Rita Frieske and
                  Ryan Cotterell and
                  Zhijing Jin},
  title        = {State-of-the-art generalisation research in {NLP:} a taxonomy and
                  review},
  journal      = {CoRR},
  volume       = {abs/2210.03050},
  year         = {2022},
  url          = {https://doi.org/10.48550/arXiv.2210.03050},
}



% ACL
