@article{gehrmann2022repairing,
  title={Repairing the cracked foundation: A survey of obstacles in evaluation practices for generated text},
  author={Gehrmann, Sebastian and Clark, Elizabeth and Sellam, Thibault},
  journal={arXiv preprint arXiv:2202.06935},
  year={2022}
}

@inproceedings{deutsch-roth-2021-understanding,
    title = "Understanding the Extent to which Content Quality Metrics Measure the Information Quality of Summaries",
    author = "Deutsch, Daniel  and
      Roth, Dan",
    booktitle = "Proceedings of the 25th Conference on Computational Natural Language Learning",
    month = nov,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.conll-1.24",
    doi = "10.18653/v1/2021.conll-1.24",
    pages = "300--309",
    abstract = "Reference-based metrics such as ROUGE or BERTScore evaluate the content quality of a summary by comparing the summary to a reference. Ideally, this comparison should measure the summary{'}s information quality by calculating how much information the summaries have in common. In this work, we analyze the token alignments used by ROUGE and BERTScore to compare summaries and argue that their scores largely cannot be interpreted as measuring information overlap. Rather, they are better estimates of the extent to which the summaries discuss the same topics. Further, we provide evidence that this result holds true for many other summarization evaluation metrics. The consequence of this result is that the most frequently used summarization evaluation metrics do not align with the community{'}s research goal, to generate summaries with high-quality information. However, we conclude by demonstrating that a recently proposed metric, QAEval, which scores summaries using question-answering, appears to better capture information quality than current evaluations, highlighting a direction for future research.",
}

@inproceedings{gliwa2019samsum,
  title={SAMSum: Sentence-Aligned Multilingual Summarization Evaluation},
  author={Gliwa, Dominik and Schutze, Hinrich},
  booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
  pages={3778--3788},
  year={2019}
}

@inproceedings{banerjee-lavie:2005:ACL,
  author    = {Banerjee, Satanjeev  and  Lavie, Alon},
  title     = {METEOR: An Automatic Metric for MT Evaluation with Improved Correlation with Human Judgments},
  booktitle = {Proceedings of the 43rd Annual Meeting of the Association for Computational Linguistics (ACL)},
  month     = {06},
  year      = {2005},
  address   = {Ann Arbor, Michigan},
  publisher = {Association for Computational Linguistics},
  pages     = {24--31},
  url       = {https://www.aclweb.org/anthology/P05-1073}
}

@inproceedings{zhang-etal-2020-bertscore,
    title = "{{BERT}Score}: Evaluating Text Generation with BERT",
    author = "Zhang, Jianmo  and
      Liu, Yichong  and
      Gao, Jiaming  and
      Toutanova, Kristina  and
      Chen, Zhifang",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/2020.acl-main.332",
    pages = "4920--4926",
}
@inproceedings{zhao-etal-2019-moverscore,
    title = "{M}over{S}core: Text Generation Evaluation as Empirical Learning-to-Rank",
    author = "Zhao, Kai  and
      Wu, Yingce  and
      Eskenazi, Maxine  and
      Tian, Yafang",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)",
    month = nov,
    year = "2019",
    address = "Hong Kong, China",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/D19-1292",
    pages = "3071--3081",
}
@article{yuan-etal-2021-bartscore,
    title = "{BART}Score: Better Evaluation for Text Generation Models",
    author = "Yuan, Fandong  and
      Liu, Yichong  and
      Toutanova, Kristina  and
      Chen, Zhifang",
    journal = "Transactions of the Association for Computational Linguistics",
    volume = "9",
    pages = "243--257",
    year = "2021",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/2021.tacl-1.21",
    doi = "10.1162/tacl_a_00285",
}
@inproceedings{clark-etal-2019-sentence,
    title = "Sentence-{M}over: A Distance Metric for Sentence Embeddings",
    author = "Clark, Kevin  and
      Khandelwal, Urvashi  and
      Levy, Omer  and
      Manning, Christopher D.",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)",
    month = nov,
    year = "2019",
    address = "Hong Kong, China",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/D19-1203",
    pages = "1765--1775",
}
@article{landauer1997solution,
  title={A solution to Plato's problem: The latent semantic analysis theory of acquisition, induction, and representation of knowledge},
  author={Landauer, Thomas K and Dumais, Susan T},
  journal={Psychological review},
  volume={104},
  number={2},
  pages={211},
  year={1997},
  publisher={American Psychological Association}
}
@inproceedings{forgues2014automatic,
  title={Automatic evaluation of summaries using vector extrema},
  author={Forgues, Gabriel and Lavergne, Tarek and Allan, James},
  booktitle={Proceedings of the 52nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={36--46},
  year={2014}
}
@inproceedings{Durmus_2020,
  title={FEQA: Fine-Grained Factual Consistency Evaluation for Text Generation},
  author={Durmus, Emre and Van Asch, Vincent and Gatt, Albert and Georgila, Katerina and Moens, Marie-Francine},
  booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
  year={2020},
  pages={6644--6653},
}
@inproceedings{Scialom_2019,
  title={SummaQA: Answering Questions on Summaries using Pre-Trained Models},
  author={Scialom, Thomas and Delabrouille, Ludovic and Lavergne, Tarek},
  booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
  year={2019},
  pages={1687--1697},
}
@inproceedings{Scialom_2021,
  title={QuestEval: A Simple QA-based Metric for Referenceless Evaluation of Text Generation},
  author={Scialom, Thomas and Lavergne, Tarek and Delabrouille, Ludovic},
  booktitle={Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 2 (Short and Student Papers)},
  year={2021},
  pages={620--628},
}
@inproceedings{kryscinski-etal-2020-factcc,
  title={FactCC: Evaluating Factual Consistency of Generated Text},
  author={Kryscinski, Diana  and
          Burchardt, Aljoscha  and
          Specia, Lucia},
  booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
  pages={6654--6665},
  year={2020}
}
@inproceedings{goyal-durrett-2020-dae,
  title={DAE: Dependency-Based Factual Consistency Evaluation for Text Generation},
  author={Goyal, Ananya  and
          Durrett, Greg},
  booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing},
  pages={6801--6812},
  year={2020}
}


@inproceedings{lin2004rouge,
title={ROUGE: A Package for Automatic Evaluation of Summaries},
author={Lin, Chin-Yew and Hovy, Eduard},
booktitle={Proceedings of the Workshop on Text Summarization Branches Out (WAS 2004)},
pages={74-81},
year={2004}
}

@inproceedings{papineni2002bleu,
title={BLEU: a Method for Automatic Evaluation of Machine Translation},
author={Papineni, Kishore and Roukos, Salim and Ward, Todd and Zhu, Wei-Jing},
booktitle={Proceedings of the 40th annual meeting on association for computational linguistics},
pages={311-318},
year={2002}
}

@inproceedings{gao2022dialsummeval,
  title={DialSummEval: Revisiting Summarization Evaluation for Dialogues},
  author={Gao, Mingqi and Wan, Xiaojun},
  booktitle={Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
  pages={5693--5709},
  year={2022}
}
@misc{genie,
  doi = {10.48550/ARXIV.2101.06561},
  
  url = {https://arxiv.org/abs/2101.06561},
  
  author = {Khashabi, Daniel and Stanovsky, Gabriel and Bragg, Jonathan and Lourie, Nicholas and Kasai, Jungo and Choi, Yejin and Smith, Noah A. and Weld, Daniel S.},
  
  keywords = {Computation and Language (cs.CL), Artificial Intelligence (cs.AI), FOS: Computer and information sciences, FOS: Computer and information sciences},
  
  title = {GENIE: Toward Reproducible and Standardized Human Evaluation for Text Generation},
  
  publisher = {arXiv},
  
  year = {2021},
  
  copyright = {arXiv.org perpetual, non-exclusive license}
}
@inproceedings{bhandari-etal-2020-evaluating,
    title = "Re-evaluating Evaluation in Text Summarization",
    author = "Bhandari, Manik  and
      Gour, Pranav Narayan  and
      Ashfaq, Atabak  and
      Liu, Pengfei  and
      Neubig, Graham",
    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    month = nov,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.emnlp-main.751",
    doi = "10.18653/v1/2020.emnlp-main.751",
    pages = "9347--9359",
    abstract = "Automated evaluation metrics as a stand-in for manual evaluation are an essential part of the development of text-generation tasks such as text summarization. However, while the field has progressed, our standard metrics have not {--} for nearly 20 years ROUGE has been the standard evaluation in most summarization papers. In this paper, we make an attempt to re-evaluate the evaluation method for text summarization: assessing the reliability of automatic metrics using top-scoring system outputs, both abstractive and extractive, on recently popular datasets for both system-level and summary-level evaluation settings. We find that conclusions about evaluation metrics on older datasets do not necessarily hold on modern datasets and systems. We release a dataset of human judgments that are collected from 25 top-scoring neural summarization systems (14 abstractive and 11 extractive).",
}
@inproceedings{iskender-etal-2021-reliability,
    title = "Reliability of Human Evaluation for Text Summarization: Lessons Learned and Challenges Ahead",
    author = {Iskender, Neslihan  and
      Polzehl, Tim  and
      M{\"o}ller, Sebastian},
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.humeval-1.10",
    pages = "86--96",
    abstract = "Only a small portion of research papers with human evaluation for text summarization provide information about the participant demographics, task design, and experiment protocol. Additionally, many researchers use human evaluation as gold standard without questioning the reliability or investigating the factors that might affect the reliability of the human evaluation. As a result, there is a lack of best practices for reliable human summarization evaluation grounded by empirical evidence. To investigate human evaluation reliability, we conduct a series of human evaluation experiments, provide an overview of participant demographics, task design, experimental set-up and compare the results from different experiments. Based on our empirical analysis, we provide guidelines to ensure the reliability of expert and non-expert evaluations, and we determine the factors that might affect the reliability of the human evaluation.",
}
@inproceedings{van-der-lee-etal-2019-best,
    title = "Best practices for the human evaluation of automatically generated text",
    author = "van der Lee, Chris  and
      Gatt, Albert  and
      van Miltenburg, Emiel  and
      Wubben, Sander  and
      Krahmer, Emiel",
    booktitle = "Proceedings of the 12th International Conference on Natural Language Generation",
    month = oct,
    year = "2019",
    address = "Tokyo, Japan",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/W19-8643",
    doi = "10.18653/v1/W19-8643",
    pages = "355--368",
    abstract = "Currently, there is little agreement as to how Natural Language Generation (NLG) systems should be evaluated. While there is some agreement regarding automatic metrics, there is a high degree of variation in the way that human evaluation is carried out. This paper provides an overview of how human evaluation is currently conducted, and presents a set of best practices, grounded in the literature. With this paper, we hope to contribute to the quality and consistency of human evaluations in NLG.",
}
@inproceedings{zhou-etal-2022-deconstructing,
    title = "Deconstructing {NLG} Evaluation: Evaluation Practices, Assumptions, and Their Implications",
    author = "Zhou, Kaitlyn  and
      Blodgett, Su Lin  and
      Trischler, Adam  and
      Daum{\'e} III, Hal  and
      Suleman, Kaheer  and
      Olteanu, Alexandra",
    booktitle = "Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",
    month = jul,
    year = "2022",
    address = "Seattle, United States",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.naacl-main.24",
    doi = "10.18653/v1/2022.naacl-main.24",
    pages = "314--324",
    abstract = "There are many ways to express similar things in text, which makes evaluating natural language generation (NLG) systems difficult. Compounding this difficulty is the need to assess varying quality criteria depending on the deployment setting. While the landscape of NLG evaluation has been well-mapped, practitioners{'} goals, assumptions, and constraints{---}which inform decisions about what, when, and how to evaluate{---}are often partially or implicitly stated, or not stated at all. Combining a formative semi-structured interview study of NLG practitioners (N=18) with a survey study of a broader sample of practitioners (N=61), we surface goals, community practices, assumptions, and constraints that shape NLG evaluations, examining their implications and how they embody ethical considerations.",
}

@inproceedings{tuggener-etal-2021-summarizing,
    title = "Are We Summarizing the Right Way? A Survey of Dialogue Summarization Data Sets",
    author = "Tuggener, Don  and
      Mieskes, Margot  and
      Deriu, Jan  and
      Cieliebak, Mark",
    booktitle = "Proceedings of the Third Workshop on New Frontiers in Summarization",
    month = nov,
    year = "2021",
    address = "Online and in Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.newsum-1.12",
    doi = "10.18653/v1/2021.newsum-1.12",
    pages = "107--118",
    abstract = "Dialogue summarization is a long-standing task in the field of NLP, and several data sets with dialogues and associated human-written summaries of different styles exist. However, it is unclear for which type of dialogue which type of summary is most appropriate. For this reason, we apply a linguistic model of dialogue types to derive matching summary items and NLP tasks. This allows us to map existing dialogue summarization data sets into this model and identify gaps and potential directions for future work. As part of this process, we also provide an extensive overview of existing dialogue summarization data sets.",
}
@misc{reproducibility_in_nlp,
  doi = {10.48550/ARXIV.2103.07929},
  
  url = {https://arxiv.org/abs/2103.07929},
  
  author = {Belz, Anya and Agarwal, Shubham and Shimorina, Anastasia and Reiter, Ehud},
  
  keywords = {Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences},
  
  title = {A Systematic Review of Reproducibility Research in Natural Language Processing},
  
  publisher = {arXiv},
  
  year = {2021},
  
  copyright = {Creative Commons Attribution 4.0 International}
}
@inproceedings{belz-etal-2021-reprogen,
    title = "The {R}epro{G}en Shared Task on Reproducibility of Human Evaluations in {NLG}: Overview and Results",
    author = "Belz, Anya  and
      Shimorina, Anastasia  and
      Agarwal, Shubham  and
      Reiter, Ehud",
    booktitle = "Proceedings of the 14th International Conference on Natural Language Generation",
    month = aug,
    year = "2021",
    address = "Aberdeen, Scotland, UK",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.inlg-1.24",
    pages = "249--258",
    abstract = "The NLP field has recently seen a substantial increase in work related to reproducibility of results, and more generally in recognition of the importance of having shared definitions and practices relating to evaluation. Much of the work on reproducibility has so far focused on metric scores, with reproducibility of human evaluation results receiving far less attention. As part of a research programme designed to develop theory and practice of reproducibility assessment in NLP, we organised the first shared task on reproducibility of human evaluations, ReproGen 2021. This paper describes the shared task in detail, summarises results from each of the reproduction studies submitted, and provides further comparative analysis of the results. Out of nine initial team registrations, we received submissions from four teams. Meta-analysis of the four reproduction studies revealed varying degrees of reproducibility, and allowed very tentative first conclusions about what types of evaluation tend to have better reproducibility.",
}



@article{Rougier:2018,
  doi = {10.1038/d41586-018-04628-w},
  year = {2018},
  month = apr,
  publisher = {Springer Nature},
  volume = {556},
  number = {7701},
  pages = {309--309},
  author = {Rougier, Nicolas P. and Hinsen, Konrad},
  title = {Code reviewing puts extra demands on referees },
  journal = {Nature},
}

@article{Science:2018,
  doi = {10.1126/science.359.6377.725},
  author = {Hutson, Matthew},
  url = {http://science.sciencemag.org/content/359/6377/725},
  year = {2018},
  month = feb,
  volume = {359},
  number = {6377},
  title = {Artificial intelligence faces a replication crisis},
  journal = {Science},
}

@Misc{Roboto:2011,
  author =       {Christian Robertson},
  title =        {The Roboto family of fonts (Google)},
  url =          {https://github.com/google/roboto},
  year =         2011,
  note =         {Apache License, verison 2.0},
}

@Misc{SourceSerifPro:2014,
  author =       {Frank Grießhammer},
  title =        {Source Serif Pro (Adobe Systems)},
  url =          {https://github.com/adobe-fonts/source-serif-pro},
  year =         2014,
  note =         {SIL Open Font License, version 1.1},
}

@Misc{SourceCodePro:2012,
  author =       {Paul D. Hunt},
  title =        {Source Code Pro (Adobe Systems)},
  url =          {https://github.com/adobe-fonts/source-code-pro},
  year =         2012,
  note =         {SIL Open Font License, version 1.1},
}

@article{Topalidou:2015,
  author =       {Topalidou, Meropi and Rougier, Nicolas P.},
  title =        {{[Re] Interaction between cognitive and motor cortico-basal
                  ganglia loops during decision making: a computational study}},
  journal =      {ReScience},
  year =         2015,
  volume =       1,
  number =       1,
  doi =          {10.5281/zenodo.27944},
}

@article{Rougier:2017,
  doi =          {10.7717/peerj-cs.142},
  author =       {Nicolas P. Rougier and Konrad Hinsen and Frédéric Alexandre
                  and Thomas Arildsen and Lorena Barba and Fabien
                  C. Y. Benureau and C. Titus Brown and Pierre de Buyl and Ozan
                  Caglayan and Andrew P. Davison and Marc André Delsuc and
                  Georgios Detorakis and Alexandra K. Diem and Damien Drix and
                  Pierre Enel and Benoît Girard and Olivia Guest and Matt
                  G. Hall and Rafael Neto Henriques and Xavier Hinaut and Kamil
                  S Jaron and Mehdi Khamassi and Almar Klein and Tiina Manninen
                  and Pietro Marchesi and Dan McGlinn and Christoph Metzner and
                  Owen L. Petchey and Hans Ekkehard Plesser and Timothée Poisot
                  and Karthik Ram and Yoav Ram and Etienne Roesch and Cyrille
                  Rossant and Vahid Rostami and Aaron Shifman and Joseph
                  Stachelek and Marcel Stimberg and Frank Stollmeier and
                  Federico Vaggi and Guillaume Viejo and Julien Vitay and Anya
                  Vostinar and Roman Yurchak and Tiziano Zito},
  title =        {{Sustainable computational science: the ReScience
                  initiative}},
  journal =      {{PeerJ} Computer Science},
  month =        12,
  volume =       3,
  pages =        {e142},
  year =         2017,
  github =       {https://github.com/ReScience/ReScience-article-2},
  keywords =     {journal},
  tags =         {OS},
}

@article{Sinha:2022,
  author = {Sinha, Koustuv and Dodge, Jesse and Luccioni, Sasha and Forde, Jessica Zosa and Raparthy, Sharath Chandra and Pineau, Joelle and Stojnic, Robert},
  title = {{ML Reproducibility Challenge 2021}},
  journal = {ReScience C},
  year = {2022},
  month = may,
  volume = {8},
  number = {2},
  pages = {{48}},
  doi = {10.5281/zenodo.6574723},
  url = {https://zenodo.org/record/6574723/files/article.pdf},
  code_url = {},
  code_doi = {},
  code_swh = {},
  data_url = {},
  data_doi = {},
  review_url = {},
  type = {Editorial},
  language = {Python},
  domain = {ML Reproducibility Challenge 2021},
  keywords = {rescience c, machine learning, deep learning, python, pytorch}
}

@article{Sinha:2021,
  author = {Sinha, Koustuv and Dodge, Jesse and Luccioni, Sasha and Forde, Jessica Zosa and Stojnic, Robert and Pineau, Joelle},
  title = {{ML Reproducibility Challenge 2020}},
  journal = {ReScience C},
  year = {2021},
  month = may,
  volume = {7},
  number = {2},
  pages = {{1}},
  doi = {10.5281/zenodo.4833117},
  url = {https://zenodo.org/record/4833117/files/article.pdf},
  code_url = {},
  code_doi = {},
  code_swh = {},
  data_url = {},
  data_doi = {},
  review_url = {},
  type = {Editorial},
  language = {},
  domain = {},
  keywords = {machine learning, neurips, reproducibility challenge}
}

@article{Sinha:2020,
  author = {Sinha, Koustuv and Pineau, Joelle and Forde, Jessica and Ke, Rosemary Nan and Larochelle, Hugo},
  title = {{NeurIPS 2019 Reproducibility Challenge}},
  journal = {ReScience C},
  year = {2020},
  month = may,
  volume = {6},
  number = {2},
  pages = {{11}},
  doi = {10.5281/zenodo.3818627},
  url = {https://zenodo.org/record/3818627/files/article.pdf},
  code_url = {},
  code_doi = {},
  code_swh = {},
  data_url = {},
  data_doi = {},
  review_url = {},
  type = {Editorial},
  language = {},
  domain = {},
  keywords = {machine learning, neurips, reproducibility challenge}
}

@article{Pineau:2019,
  author = {Pineau, Joelle and Sinha, Koustuv and Fried, Genevieve and Ke, Rosemary Nan and Larochelle, Hugo},
  title = {{ICLR Reproducibility Challenge 2019}},
  journal = {ReScience C},
  year = {2019},
  month = may,
  volume = {5},
  number = {2},
  pages = {{5}},
  doi = {10.5281/zenodo.3158244},
  url = {https://zenodo.org/record/3158244/files/article.pdf},
  code_url = {},
  code_doi = {},
  data_url = {},
  data_doi = {},
  review_url = {https://github.com/ReScience/submissions/issues/5},
  type = {Editorial},
  language = {},
  domain = {},
  keywords = {machine learning, ICLR, reproducibility challenge}
}
@article{see2017get,
  title={Get to the point: Summarization with pointer-generator networks},
  author={See, Abigail and Liu, Peter J and Manning, Christopher D},
  journal={arXiv preprint arXiv:1704.04368},
  year={2017}
}
@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}
@article{lewis2019bart,
  title={Bart: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension},
  author={Lewis, Mike and Liu, Yinhan and Goyal, Naman and Ghazvininejad, Marjan and Mohamed, Abdelrahman and Levy, Omer and Stoyanov, Ves and Zettlemoyer, Luke},
  journal={arXiv preprint arXiv:1910.13461},
  year={2019}
}
@inproceedings{zhang2020pegasus,
  title={Pegasus: Pre-training with extracted gap-sentences for abstractive summarization},
  author={Zhang, Jingqing and Zhao, Yao and Saleh, Mohammad and Liu, Peter},
  booktitle={International Conference on Machine Learning},
  pages={11328--11339},
  year={2020},
  organization={PMLR}
}
@article{dong2019unified,
  title={Unified language model pre-training for natural language understanding and generation},
  author={Dong, Li and Yang, Nan and Wang, Wenhui and Wei, Furu and Liu, Xiaodong and Wang, Yu and Gao, Jianfeng and Zhou, Ming and Hon, Hsiao-Wuen},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}
@article{wu2021controllable,
  title={Controllable abstractive dialogue summarization with sketch supervision},
  author={Wu, Chien-Sheng and Liu, Linqing and Liu, Wenhao and Stenetorp, Pontus and Xiong, Caiming},
  journal={arXiv preprint arXiv:2105.14064},
  year={2021}
}
@article{fabbri2021convosumm,
  title={ConvoSumm: Conversation summarization benchmark and improved abstractive summarization with argument mining},
  author={Fabbri, Alexander R and Rahman, Faiaz and Rizvi, Imad and Wang, Borui and Li, Haoran and Mehdad, Yashar and Radev, Dragomir},
  journal={arXiv preprint arXiv:2106.00829},
  year={2021}
}
@article{chen2020multi, 
title={Multi-view sequence-to-sequence models with conversational structure for abstractive dialogue summarization}, 
author={Chen, Jiaao and Yang, Diyi}, 
journal={arXiv preprint arXiv:2010.01672}, 
year={2020} 
}
@article{feng2021language,
  title={Language model as an annotator: Exploring DialoGPT for dialogue summarization},
  author={Feng, Xiachong and Feng, Xiaocheng and Qin, Libo and Qin, Bing and Liu, Ting},
  journal={arXiv preprint arXiv:2105.12544},
  year={2021}
}
@article{liu2021controllable,
  title={Controllable neural dialogue summarization with personal named entity planning},
  author={Liu, Zhengyuan and Chen, Nancy F},
  journal={arXiv preprint arXiv:2109.13070},
  year={2021}
}
@article{chen2021structure,
  title={Structure-aware abstractive conversation summarization via discourse and action graphs},
  author={Chen, Jiaao and Yang, Diyi},
  journal={arXiv preprint arXiv:2104.08400},
  year={2021}
}
@article{baker2016reproducibility,
  title={Reproducibility crisis},
  author={Baker, Monya},
  journal={Nature},
  volume={533},
  number={26},
  pages={353--66},
  year={2016}
}
@inproceedings{mieskes-etal-2019-community,
    title = "Community Perspective on Replicability in Natural Language Processing",
    author = {Mieskes, Margot  and
      Fort, Kar{\"e}n  and
      N{\'e}v{\'e}ol, Aur{\'e}lie  and
      Grouin, Cyril  and
      Cohen, Kevin},
    booktitle = "Proceedings of the International Conference on Recent Advances in Natural Language Processing (RANLP 2019)",
    month = sep,
    year = "2019",
    address = "Varna, Bulgaria",
    publisher = "INCOMA Ltd.",
    url = "https://aclanthology.org/R19-1089",
    doi = "10.26615/978-954-452-056-4_089",
    pages = "768--775",
    abstract = "With recent efforts in drawing attention to the task of replicating and/or reproducing results, for example in the context of COLING 2018 and various LREC workshops, the question arises how the NLP community views the topic of replicability in general. Using a survey, in which we involve members of the NLP community, we investigate how our community perceives this topic, its relevance and options for improvement. Based on over two hundred participants, the survey results confirm earlier observations, that successful reproducibility requires more than having access to code and data. Additionally, the results show that the topic has to be tackled from the authors{'}, reviewers{'} and community{'}s side.",
}

@inproceedings{rankel-etal-2013-decade,
    title = "A Decade of Automatic Content Evaluation of News Summaries: Reassessing the State of the Art",
    author = "Rankel, Peter A.  and
      Conroy, John M.  and
      Dang, Hoa Trang  and
      Nenkova, Ani",
    booktitle = "Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)",
    month = aug,
    year = "2013",
    address = "Sofia, Bulgaria",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/P13-2024",
    pages = "131--136",
}

@article{TACL2563_fabbri,
	author = {Alexander Fabbri and Wojciech Kryscinski and Bryan McCann and Caiming Xiong and Richard Socher and Dragomir Radev},
	title = {SummEval: Re-evaluating Summarization Evaluation},
	journal = {Transactions of the Association for Computational Linguistics},
	volume = {9},
	number = {0},
	year = {2022},
	keywords = {},
	abstract = {The scarcity of comprehensive up-to-date studies on evaluation metrics for text summarization and the lack of consensus regarding evaluation protocols continue to inhibit progress. We address the existing shortcomings of summarization evaluation methods along five dimensions: 1) we re-evaluate 14 automatic evaluation metrics in a comprehensive and consistent fashion using neural summarization model outputs along with expert and crowd-sourced human annotations, 2) we consistently bench-mark 23 recent summarization models us-ing the aforementioned automatic evaluation metrics, 3) we assemble the largest collection of summaries generated by models trained on the CNN/DailyMail news dataset and share it in a unified format, 4) we implement and share a toolkit that provides an extensible and unified API for evaluating summarization models across a broad range of automatic metrics, 5) we assemble and share the largest and most diverse, in terms of model types, collection of human judgments of model-generated summaries on the CNN/Daily Mail dataset annotated by both expert judges and crowd-source workers. We hope that this work will help promote a more complete evaluation protocol for text summarization as well as advance research in developing evaluation metrics that better correlate with human judgments.},
	issn = {2307-387X},	pages = {391--409},	url = {https://transacl.org/index.php/tacl/article/view/2563}
}
@misc{krippendorff2004content, 
title={Content analysis: An introduction to its methodology (2 nd Thousand Oaks}, 
author={Krippendorff, Klaus}, 
year={2004}, 
publisher={CA: Sage Publications} 
}