% Use this file for citations not found in the ACL Anthology (contained in "anthology.bib").
@inproceedings{wang_evaluating_2023,
    address = {Red Hook, NY, USA},
    series = {{NIPS} '23},
    title = {Evaluating open-{QA} evaluation},
    abstract = {This study focuses on the evaluation of the Open Question Answering (Open-QA) task, which can directly estimate the factuality of large language models (LLMs). Current automatic evaluation methods have shown limitations, indicating that human evaluation still remains the most reliable approach. We introduce a new task, Evaluating QA Evaluation (QA-Eval) and the corresponding dataset EVOUNA, designed to assess the accuracy of AI-generated answers in relation to standard answers within Open-QA. Our evaluation of these methods utilizes human-annotated results to measure their performance. Specifically, the work investigates methods that show high correlation with human evaluations, deeming them more reliable. We also discuss the pitfalls of current methods and methods to improve LLM-based evaluators. We believe this new QA-Eval task and corresponding dataset EVOUNA will facilitate the development of more effective automatic evaluation tools and prove valuable for future research in this area. All resources are available at https://github.com/wangcunxiang/QA-Eval and it is under the Apache-2.0 License.},
    urldate = {2025-05-05},
    url = {https://papers.nips.cc/paper_files/paper/2023/file/f323d594aa5d2c68154433a131c07959-Paper-Datasets_and_Benchmarks.pdf},
    booktitle = {Proceedings of the 37th {International} {Conference} on {Neural} {Information} {Processing} {Systems}},
    publisher = {Curran Associates Inc.},
    author = {Wang, Cunxiang and Cheng, Sirui and Guo, Qipeng and Yue, Yuanhao and Ding, Bowen and Xu, Zhikun and Wang, Yidong and Hu, Xiangkun and Zhang, Zheng and Zhang, Yue},
    month = dec,
    year = {2023},
    pages = {77013--77042},
}

@misc{nguyen2016ms,
    author = {Nguyen, Tri and Rosenberg, Mir and Song, Xia and Gao, Jianfeng and Tiwary, Saurabh and Majumder, Rangan and Deng, Li},
    title = {{MS MARCO: A Human Generated MAchine Reading COmprehension Dataset}},
    year = {2016},
    month = {November},
    abstract = {This paper presents our recent work on the design and development of a new, large scale dataset, which we name MS MARCO, for MAchine Reading COmprehension. This new dataset is aimed to overcome a number of well-known weaknesses of previous publicly available datasets for the same task of reading comprehension and question answering. In MS MARCO, all questions are sampled from real anonymized user queries. The context passages, from which answers in the dataset are derived, are extracted from real web documents using the most advanced version of the Bing search engine. The answers to the queries are human generated. Finally, a subset of these queries has multiple answers. We aim to release one million queries and the corresponding answers in the dataset, which, to the best of our knowledge, is the most comprehensive real-world dataset of its kind in both quantity and quality. We are currently releasing 100,000 queries with their corresponding answers to inspire work in reading comprehension and question answering along with gathering feedback from the research community.},
    url = {https://www.microsoft.com/en-us/research/publication/ms-marco-human-generated-machine-reading-comprehension-dataset/},
}

@article{rogers-etal-2023-qa,
    author = {Rogers, Anna and Gardner, Matt and Augenstein, Isabelle},
    title = {{QA Dataset Explosion: A Taxonomy of NLP Resources for Question Answering and Reading Comprehension}},
    year = {2023},
    issue_date = {October 2023},
    publisher = {Association for Computing Machinery},
    address = {New York, NY, USA},
    volume = {55},
    number = {10},
    issn = {0360-0300},
    url = {https://doi.org/10.1145/3560260},
    doi = {10.1145/3560260},
    abstract = {Alongside huge volumes of research on deep learning models in NLP in the recent years, there has been much work on benchmark datasets needed to track modeling progress. Question answering and reading comprehension have been particularly prolific in this regard, with more than 80 new datasets appearing in the past 2 years. This study is the largest survey of the field to date. We provide an overview of the various formats and domains of the current resources, highlighting the current lacunae for future work. We further discuss the current classifications of “skills” that question answering/reading comprehension systems are supposed to acquire and propose a new taxonomy. The supplementary materials survey the current multilingual resources and monolingual resources for languages other than English, and we discuss the implications of overfocusing on English. The study is aimed at both practitioners looking for pointers to the wealth of existing data and at researchers working on new resources.},
    journal = {ACM Comput. Surv.},
    month = feb,
    articleno = {197},
    numpages = {45},
    keywords = {Reading comprehension, natural language understanding}
}

@inproceedings{rein2024gpqa,
      title={{GPQA: A Graduate-Level Google-Proof Q\&A Benchmark}},
      author={David Rein and Betty Li Hou and Asa Cooper Stickland and Jackson Petty and Richard Yuanzhe Pang and Julien Dirani and Julian Michael and Samuel R. Bowman},
      booktitle={First Conference on Language Modeling},
      year={2024},
      url={https://openreview.net/forum?id=Ti67584b98}
}      


@article{Gilardi2023ChatGPTOC,
  title={ChatGPT outperforms crowd workers for text-annotation tasks},
  author={Fabrizio Gilardi and Meysam Alizadeh and Ma{\"e}l Kubli},
  journal={Proceedings of the National Academy of Sciences of the United States of America},
  year={2023},
  volume={120},
  url={https://api.semanticscholar.org/CorpusID:257766307}
}

@misc{qwen2.5,
      title={Qwen2.5 Technical Report}, 
      author={Qwen and : and An Yang and Baosong Yang and Beichen Zhang and Binyuan Hui and Bo Zheng and Bowen Yu and Chengyuan Li and Dayiheng Liu and Fei Huang and Haoran Wei and Huan Lin and Jian Yang and Jianhong Tu and Jianwei Zhang and Jianxin Yang and Jiaxi Yang and Jingren Zhou and Junyang Lin and Kai Dang and Keming Lu and Keqin Bao and Kexin Yang and Le Yu and Mei Li and Mingfeng Xue and Pei Zhang and Qin Zhu and Rui Men and Runji Lin and Tianhao Li and Tianyi Tang and Tingyu Xia and Xingzhang Ren and Xuancheng Ren and Yang Fan and Yang Su and Yichang Zhang and Yu Wan and Yuqiong Liu and Zeyu Cui and Zhenru Zhang and Zihan Qiu},
      year={2025},
      eprint={2412.15115},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2412.15115}, 
}

@article{qwen2,
    title   = {{Qwen2 Technical Report}}, 
    author  = {An Yang and Baosong Yang and Binyuan Hui and Bo Zheng and Bowen Yu and Chang Zhou and Chengpeng Li and Chengyuan Li and Dayiheng Liu and Fei Huang and Guanting Dong and Haoran Wei and Huan Lin and Jialong Tang and Jialin Wang and Jian Yang and Jianhong Tu and Jianwei Zhang and Jianxin Ma and Jin Xu and Jingren Zhou and Jinze Bai and Jinzheng He and Junyang Lin and Kai Dang and Keming Lu and Keqin Chen and Kexin Yang and Mei Li and Mingfeng Xue and Na Ni and Pei Zhang and Peng Wang and Ru Peng and Rui Men and Ruize Gao and Runji Lin and Shijie Wang and Shuai Bai and Sinan Tan and Tianhang Zhu and Tianhao Li and Tianyu Liu and Wenbin Ge and Xiaodong Deng and Xiaohuan Zhou and Xingzhang Ren and Xinyu Zhang and Xipin Wei and Xuancheng Ren and Yang Fan and Yang Yao and Yichang Zhang and Yu Wan and Yunfei Chu and Yuqiong Liu and Zeyu Cui and Zhenru Zhang and Zhihao Fan},
    journal = {arXiv preprint arXiv:2407.10671},
    year    = {2024}
}

@misc{deepseekai2025deepseekr1incentivizingreasoningcapability,
      title={{DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning}}, 
      author={DeepSeek-AI},
      year={2025},
      eprint={2501.12948},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2501.12948}, 
}

@inproceedings{yen2025helmet,
      title={{HELMET: How to Evaluate Long-Context Language Models Effectively and Thoroughly}}, 
      author={Howard Yen and Tianyu Gao and Minmin Hou and Ke Ding and Daniel Fleischer and Peter Izsak and Moshe Wasserblat and Danqi Chen},
      year={2025},
      booktitle={International Conference on Learning Representations (ICLR)},
      url = {https://iclr.cc/virtual/2025/poster/31157}
}

@misc{anthropic_claude_haiku,
  author = {Anthropic},
  title = {{Claude 3.5 Haiku}},
  url = {https://www.anthropic.com/claude/haiku},
  year = {2024},
  note = {Accessed: May 16, 2025}
}

@misc{anthropic_claud37_sonnet,
  author = {Anthropic},
  title = {{Claude 3.7 Sonnet}},
  url = {https://www.anthropic.com/claude/sonnet},
  year = {2024},
  note = {Accessed: May 16, 2025}
}

@misc{google_gemini20_flash,
  author = {Google},
  title = {{Gemini 2.0 Flash}},
  url = {https://blog.google/technology/google-deepmind/google-gemini-ai-update-december-2024/#gemini-2-0-flash},
  year = {2024},
  note = {Accessed: May 16, 2025}
}

@misc{yang2025qwen251mtechnicalreport,
      title={{Qwen2.5-1M Technical Report}}, 
      author={An Yang and Bowen Yu and Chengyuan Li and Dayiheng Liu and Fei Huang and Haoyan Huang and Jiandong Jiang and Jianhong Tu and Jianwei Zhang and Jingren Zhou and Junyang Lin and Kai Dang and Kexin Yang and Le Yu and Mei Li and Minmin Sun and Qin Zhu and Rui Men and Tao He and Weijia Xu and Wenbiao Yin and Wenyuan Yu and Xiafei Qiu and Xingzhang Ren and Xinlong Yang and Yong Li and Zhiying Xu and Zipeng Zhang},
      year={2025},
      eprint={2501.15383},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2501.15383}, 
}

@misc{gao2025nextlongeffectivelongcontexttraining,
      title={{NExtLong: Toward Effective Long-Context Training without Long Documents}}, 
      author={Chaochen Gao and Xing Wu and Zijia Lin and Debing Zhang and Songlin Hu},
      year={2025},
      eprint={2501.12766},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2501.12766}, 
}

@misc{glm2024chatglm,
      title={{ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools}}, 
      author={Team GLM and Aohan Zeng and Bin Xu and Bowen Wang and Chenhui Zhang and Da Yin and Diego Rojas and Guanyu Feng and Hanlin Zhao and Hanyu Lai and Hao Yu and Hongning Wang and Jiadai Sun and Jiajie Zhang and Jiale Cheng and Jiayi Gui and Jie Tang and Jing Zhang and Juanzi Li and Lei Zhao and Lindong Wu and Lucen Zhong and Mingdao Liu and Minlie Huang and Peng Zhang and Qinkai Zheng and Rui Lu and Shuaiqi Duan and Shudan Zhang and Shulin Cao and Shuxun Yang and Weng Lam Tam and Wenyi Zhao and Xiao Liu and Xiao Xia and Xiaohan Zhang and Xiaotao Gu and Xin Lv and Xinghan Liu and Xinyi Liu and Xinyue Yang and Xixuan Song and Xunkai Zhang and Yifan An and Yifan Xu and Yilin Niu and Yuantao Yang and Yueyan Li and Yushi Bai and Yuxiao Dong and Zehan Qi and Zhaoyu Wang and Zhen Yang and Zhengxiao Du and Zhenyu Hou and Zihan Wang},
      year={2024},
      eprint={2406.12793},
      archivePrefix={arXiv},
      primaryClass={id='cs.CL' full_name='Computation and Language' is_active=True alt_name='cmp-lg' in_archive='cs' is_general=False description='Covers natural language processing. Roughly includes material in ACM Subject Class I.2.7. Note that work on artificial languages (programming languages, logics, formal systems) that does not explicitly address natural-language issues broadly construed (natural-language processing, computational linguistics, speech, text retrieval, etc.) is not appropriate for this area.'}
}

@inproceedings{
    wang2025novelqa,
    title={{NovelQA: Benchmarking Question Answering on Documents Exceeding 200K Tokens}},
    author={Cunxiang Wang and Ruoxi Ning and Boqi Pan and Tonghui Wu and Qipeng Guo and Cheng Deng and Guangsheng Bao and Xiangkun Hu and Zheng Zhang and Qian Wang and Yue Zhang},
    booktitle={The Thirteenth International Conference on Learning Representations},
    year={2025},
    url={https://openreview.net/forum?id=uMEsKEiB7J}
}

@misc{grattafiori2024llama3herdmodels,
      title={{The Llama 3 Herd of Models}}, 
      author={Aaron Grattafiori and the Llama 3 Team},
      year={2024},
      eprint={2407.21783},
      archivePrefix={arXiv},
      primaryClass={cs.AI},
      url={https://arxiv.org/abs/2407.21783}, 
}

@inproceedings{hsieh2024ruler,
title={{RULER: What{\textquoteright}s the Real Context Size of Your Long-Context Language Models?}},
author={Cheng-Ping Hsieh and Simeng Sun and Samuel Kriman and Shantanu Acharya and Dima Rekesh and Fei Jia and Boris Ginsburg},
booktitle={First Conference on Language Modeling},
year={2024},
url={https://openreview.net/forum?id=kIoBbc76Sy}
}

@inproceedings{meng2022generating,
title={{Generating Training Data with Language Models: Towards Zero-Shot Language Understanding}},
author={Yu Meng and Jiaxin Huang and Yu Zhang and Jiawei Han},
booktitle={Advances in Neural Information Processing Systems},
editor={Alice H. Oh and Alekh Agarwal and Danielle Belgrave and Kyunghyun Cho},
year={2022},
url={https://openreview.net/forum?id=4G1Sfp_1sz7}
}


@article{li2024llmasajudge,
      title={{From Generation to Judgment: Opportunities and Challenges of LLM-as-a-judge}}, 
      author={Dawei Li and Bohan Jiang and Liangjie Huang and Alimohammad Beigi and Chengshuai Zhao and Zhen Tan and Amrita Bhattacharjee and Yuxuan Jiang and Canyu Chen and Tianhao Wu and Kai Shu and Lu Cheng and Huan Liu},
      year={2025},
      eprint={2411.16594},
      archivePrefix={arXiv},
      primaryClass={cs.AI},
      url={https://arxiv.org/abs/2411.16594}, 
}
@misc{comanici2025gemini25pushingfrontier,
      title={Gemini 2.5: Pushing the Frontier with Advanced Reasoning, Multimodality, Long Context, and Next Generation Agentic Capabilities}, 
      author={Gheorghe Comanici and the Gemini Team},
      year={2025},
      eprint={2507.06261},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2507.06261}, 
}

@article{gu2024asurvey,
  author       = {Jiawei Gu and
                  Xuhui Jiang and
                  Zhichao Shi and
                  Hexiang Tan and
                  Xuehao Zhai and
                  Chengjin Xu and
                  Wei Li and
                  Yinghan Shen and
                  Shengjie Ma and
                  Honghao Liu and
                  Yuanzhuo Wang and
                  Jian Guo},
  title        = {{A Survey on LLM-as-a-Judge}},
  journal      = {CoRR},
  volume       = {abs/2411.15594},
  year         = {2024},
  url          = {https://doi.org/10.48550/arXiv.2411.15594},
  doi          = {10.48550/ARXIV.2411.15594},
  eprinttype    = {arXiv},
  eprint       = {2411.15594},
  timestamp    = {Fri, 04 Jul 2025 22:10:37 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2411-15594.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}


@InProceedings{pmlr-v239-mohta23a,
  title = 	 {Are large language models good annotators?},
  author =       {Mohta, Jay and Ak, Kenan and Xu, Yan and Shen, Mingwei},
  booktitle = 	 {Proceedings on "I Can't Believe It's Not Better: Failure  Modes in the Age of Foundation Models" at NeurIPS 2023 Workshops},
  pages = 	 {38--48},
  year = 	 {2023},
  editor = 	 {Antorán, Javier and Blaas, Arno and Buchanan, Kelly and Feng, Fan and Fortuin, Vincent and Ghalebikesabi, Sahra and Kriegler, Andreas and Mason, Ian and Rohde, David and Ruiz, Francisco J. R. and Uelwer, Tobias and Xie, Yubin and Yang, Rui},
  volume = 	 {239},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {16 Dec},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v239/mohta23a/mohta23a.pdf},
  url = 	 {https://proceedings.mlr.press/v239/mohta23a.html},
  abstract = 	 {Numerous Natural Language Processing (NLP) tasks require precisely labeled data to ensure effective model training and achieve optimal performance. However, data annotation is marked by substantial costs and time requirements, especially when requiring specialized domain expertise or annotating a large number of samples. In this study, we investigate the feasibility of employing large language models (LLMs) as replacements for human annotators. We assess the zero-shot performance of various LLMs of different sizes to determine their viability as substitutes. Furthermore, recognizing that human annotators have access to diverse modalities, we introduce an image-based modality using the BLIP-2 architecture to evaluate LLM annotation performance. Among the tested LLMs, Vicuna-13b demonstrates competitive performance across diverse tasks. To assess the potential for LLMs to replace human annotators, we train a supervised model using labels generated by LLMs and compare its performance with models trained using human-generated labels. However, our findings reveal that models trained with human labels consistently outperform those trained with LLM-generated labels. We also highlights the challenges faced by LLMs in multilingual settings, where their performance significantly diminishes for tasks in languages other than English.}
}

@misc{olmo20242olmo2furious,
      title={{2 OLMo 2 Furious}},
      author={{Team OLMo} and Pete Walsh and Luca Soldaini and Dirk Groeneveld and Kyle Lo and Shane Arora and Akshita Bhagia and Yuling Gu and Shengyi Huang and Matt Jordan and Nathan Lambert and Dustin Schwenk and Oyvind Tafjord and Taira Anderson and David Atkinson and Faeze Brahman and Christopher Clark and Pradeep Dasigi and Nouha Dziri and Michal Guerquin and Hamish Ivison and Pang Wei Koh and Jiacheng Liu and Saumya Malik and William Merrill and Lester James V. Miranda and Jacob Morrison and Tyler Murray and Crystal Nam and Valentina Pyatkin and Aman Rangapur and Michael Schmitz and Sam Skjonsberg and David Wadden and Christopher Wilhelm and Michael Wilson and Luke Zettlemoyer and Ali Farhadi and Noah A. Smith and Hannaneh Hajishirzi},
      year={2024},
      eprint={2501.00656},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2501.00656},
}

@inproceedings{zhang2020bertscore,
  author       = {Tianyi Zhang and
                  Varsha Kishore and
                  Felix Wu and
                  Kilian Q. Weinberger and
                  Yoav Artzi},
  title        = {{BERTScore: Evaluating Text Generation with BERT}},
  booktitle    = {8th International Conference on Learning Representations, {ICLR} 2020,
                  Addis Ababa, Ethiopia, April 26-30, 2020},
  publisher    = {OpenReview.net},
  year         = {2020},
  url          = {https://openreview.net/forum?id=SkeHuCVFDr},
  timestamp    = {Wed, 03 Jun 2020 10:08:32 +0200},
  biburl       = {https://dblp.org/rec/conf/iclr/ZhangKWWA20.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{he2021deberta,
  author       = {Pengcheng He and
                  Xiaodong Liu and
                  Jianfeng Gao and
                  Weizhu Chen},
  title        = {{Deberta: decoding-Enhanced Bert with Disentangled Attention}},
  booktitle    = {9th International Conference on Learning Representations, {ICLR} 2021,
                  Virtual Event, Austria, May 3-7, 2021},
  publisher    = {OpenReview.net},
  year         = {2021},
  url          = {https://openreview.net/forum?id=XPZIaotutsD},
  timestamp    = {Thu, 11 Apr 2024 13:33:56 +0200},
  biburl       = {https://dblp.org/rec/conf/iclr/HeLGC21.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{radford2019language,
  added-at = {2024-11-15T12:44:17.000+0100},
  author = {Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
  biburl = {https://www.bibsonomy.org/bibtex/233e4b003b64b1060334660fbf6db1f3f/albinzehe},
  interhash = {b926ece39c03cdf5499f6540cf63babd},
  intrahash = {33e4b003b64b1060334660fbf6db1f3f},
  journal = {OpenAI},
  keywords = {gpt gpt2 languagemodelling transferlearning transformer},
  note = {Accessed: 2024-11-15},
  timestamp = {2024-11-15T12:44:17.000+0100},
  title = {Language Models are Unsupervised Multitask Learners},
  url = {https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf},
  year = 2019
}

@misc{srivastava2023imitationgamequantifyingextrapolating,
      title={Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models}, 
      author={Aarohi Srivastava and Abhinav Rastogi and Abhishek Rao and Abu Awal Md Shoeb and Abubakar Abid and Adam Fisch and Adam R. Brown and Adam Santoro and Aditya Gupta and Adrià Garriga-Alonso and Agnieszka Kluska and Aitor Lewkowycz and Akshat Agarwal and Alethea Power and Alex Ray and Alex Warstadt and Alexander W. Kocurek and Ali Safaya and Ali Tazarv and Alice Xiang and Alicia Parrish and Allen Nie and Aman Hussain and Amanda Askell and Amanda Dsouza and Ambrose Slone and Ameet Rahane and Anantharaman S. Iyer and Anders Andreassen and Andrea Madotto and Andrea Santilli and Andreas Stuhlmüller and Andrew Dai and Andrew La and Andrew Lampinen and Andy Zou and Angela Jiang and Angelica Chen and Anh Vuong and Animesh Gupta and Anna Gottardi and Antonio Norelli and Anu Venkatesh and Arash Gholamidavoodi and Arfa Tabassum and Arul Menezes and Arun Kirubarajan and Asher Mullokandov and Ashish Sabharwal and Austin Herrick and Avia Efrat and Aykut Erdem and Ayla Karakaş and B. Ryan Roberts and Bao Sheng Loe and Barret Zoph and Bartłomiej Bojanowski and Batuhan Özyurt and Behnam Hedayatnia and Behnam Neyshabur and Benjamin Inden and Benno Stein and Berk Ekmekci and Bill Yuchen Lin and Blake Howald and Bryan Orinion and Cameron Diao and Cameron Dour and Catherine Stinson and Cedrick Argueta and César Ferri Ramírez and Chandan Singh and Charles Rathkopf and Chenlin Meng and Chitta Baral and Chiyu Wu and Chris Callison-Burch and Chris Waites and Christian Voigt and Christopher D. Manning and Christopher Potts and Cindy Ramirez and Clara E. Rivera and Clemencia Siro and Colin Raffel and Courtney Ashcraft and Cristina Garbacea and Damien Sileo and Dan Garrette and Dan Hendrycks and Dan Kilman and Dan Roth and Daniel Freeman and Daniel Khashabi and Daniel Levy and Daniel Moseguí González and Danielle Perszyk and Danny Hernandez and Danqi Chen and Daphne Ippolito and Dar Gilboa and David Dohan and David Drakard and David Jurgens and Debajyoti Datta and Deep Ganguli and Denis Emelin and Denis Kleyko and Deniz Yuret and Derek Chen and Derek Tam and Dieuwke Hupkes and Diganta Misra and Dilyar Buzan and Dimitri Coelho Mollo and Diyi Yang and Dong-Ho Lee and Dylan Schrader and Ekaterina Shutova and Ekin Dogus Cubuk and Elad Segal and Eleanor Hagerman and Elizabeth Barnes and Elizabeth Donoway and Ellie Pavlick and Emanuele Rodola and Emma Lam and Eric Chu and Eric Tang and Erkut Erdem and Ernie Chang and Ethan A. Chi and Ethan Dyer and Ethan Jerzak and Ethan Kim and Eunice Engefu Manyasi and Evgenii Zheltonozhskii and Fanyue Xia and Fatemeh Siar and Fernando Martínez-Plumed and Francesca Happé and Francois Chollet and Frieda Rong and Gaurav Mishra and Genta Indra Winata and Gerard de Melo and Germán Kruszewski and Giambattista Parascandolo and Giorgio Mariani and Gloria Wang and Gonzalo Jaimovitch-López and Gregor Betz and Guy Gur-Ari and Hana Galijasevic and Hannah Kim and Hannah Rashkin and Hannaneh Hajishirzi and Harsh Mehta and Hayden Bogar and Henry Shevlin and Hinrich Schütze and Hiromu Yakura and Hongming Zhang and Hugh Mee Wong and Ian Ng and Isaac Noble and Jaap Jumelet and Jack Geissinger and Jackson Kernion and Jacob Hilton and Jaehoon Lee and Jaime Fernández Fisac and James B. Simon and James Koppel and James Zheng and James Zou and Jan Kocoń and Jana Thompson and Janelle Wingfield and Jared Kaplan and Jarema Radom and Jascha Sohl-Dickstein and Jason Phang and Jason Wei and Jason Yosinski and Jekaterina Novikova and Jelle Bosscher and Jennifer Marsh and Jeremy Kim and Jeroen Taal and Jesse Engel and Jesujoba Alabi and Jiacheng Xu and Jiaming Song and Jillian Tang and Joan Waweru and John Burden and John Miller and John U. Balis and Jonathan Batchelder and Jonathan Berant and Jörg Frohberg and Jos Rozen and Jose Hernandez-Orallo and Joseph Boudeman and Joseph Guerr and Joseph Jones and Joshua B. Tenenbaum and Joshua S. Rule and Joyce Chua and Kamil Kanclerz and Karen Livescu and Karl Krauth and Karthik Gopalakrishnan and Katerina Ignatyeva and Katja Markert and Kaustubh D. Dhole and Kevin Gimpel and Kevin Omondi and Kory Mathewson and Kristen Chiafullo and Ksenia Shkaruta and Kumar Shridhar and Kyle McDonell and Kyle Richardson and Laria Reynolds and Leo Gao and Li Zhang and Liam Dugan and Lianhui Qin and Lidia Contreras-Ochando and Louis-Philippe Morency and Luca Moschella and Lucas Lam and Lucy Noble and Ludwig Schmidt and Luheng He and Luis Oliveros Colón and Luke Metz and Lütfi Kerem Şenel and Maarten Bosma and Maarten Sap and Maartje ter Hoeve and Maheen Farooqi and Manaal Faruqui and Mantas Mazeika and Marco Baturan and Marco Marelli and Marco Maru and Maria Jose Ramírez Quintana and Marie Tolkiehn and Mario Giulianelli and Martha Lewis and Martin Potthast and Matthew L. Leavitt and Matthias Hagen and Mátyás Schubert and Medina Orduna Baitemirova and Melody Arnaud and Melvin McElrath and Michael A. Yee and Michael Cohen and Michael Gu and Michael Ivanitskiy and Michael Starritt and Michael Strube and Michał Swędrowski and Michele Bevilacqua and Michihiro Yasunaga and Mihir Kale and Mike Cain and Mimee Xu and Mirac Suzgun and Mitch Walker and Mo Tiwari and Mohit Bansal and Moin Aminnaseri and Mor Geva and Mozhdeh Gheini and Mukund Varma T and Nanyun Peng and Nathan A. Chi and Nayeon Lee and Neta Gur-Ari Krakover and Nicholas Cameron and Nicholas Roberts and Nick Doiron and Nicole Martinez and Nikita Nangia and Niklas Deckers and Niklas Muennighoff and Nitish Shirish Keskar and Niveditha S. Iyer and Noah Constant and Noah Fiedel and Nuan Wen and Oliver Zhang and Omar Agha and Omar Elbaghdadi and Omer Levy and Owain Evans and Pablo Antonio Moreno Casares and Parth Doshi and Pascale Fung and Paul Pu Liang and Paul Vicol and Pegah Alipoormolabashi and Peiyuan Liao and Percy Liang and Peter Chang and Peter Eckersley and Phu Mon Htut and Pinyu Hwang and Piotr Miłkowski and Piyush Patil and Pouya Pezeshkpour and Priti Oli and Qiaozhu Mei and Qing Lyu and Qinlang Chen and Rabin Banjade and Rachel Etta Rudolph and Raefer Gabriel and Rahel Habacker and Ramon Risco and Raphaël Millière and Rhythm Garg and Richard Barnes and Rif A. Saurous and Riku Arakawa and Robbe Raymaekers and Robert Frank and Rohan Sikand and Roman Novak and Roman Sitelew and Ronan LeBras and Rosanne Liu and Rowan Jacobs and Rui Zhang and Ruslan Salakhutdinov and Ryan Chi and Ryan Lee and Ryan Stovall and Ryan Teehan and Rylan Yang and Sahib Singh and Saif M. Mohammad and Sajant Anand and Sam Dillavou and Sam Shleifer and Sam Wiseman and Samuel Gruetter and Samuel R. Bowman and Samuel S. Schoenholz and Sanghyun Han and Sanjeev Kwatra and Sarah A. Rous and Sarik Ghazarian and Sayan Ghosh and Sean Casey and Sebastian Bischoff and Sebastian Gehrmann and Sebastian Schuster and Sepideh Sadeghi and Shadi Hamdan and Sharon Zhou and Shashank Srivastava and Sherry Shi and Shikhar Singh and Shima Asaadi and Shixiang Shane Gu and Shubh Pachchigar and Shubham Toshniwal and Shyam Upadhyay and Shyamolima and Debnath and Siamak Shakeri and Simon Thormeyer and Simone Melzi and Siva Reddy and Sneha Priscilla Makini and Soo-Hwan Lee and Spencer Torene and Sriharsha Hatwar and Stanislas Dehaene and Stefan Divic and Stefano Ermon and Stella Biderman and Stephanie Lin and Stephen Prasad and Steven T. Piantadosi and Stuart M. Shieber and Summer Misherghi and Svetlana Kiritchenko and Swaroop Mishra and Tal Linzen and Tal Schuster and Tao Li and Tao Yu and Tariq Ali and Tatsu Hashimoto and Te-Lin Wu and Théo Desbordes and Theodore Rothschild and Thomas Phan and Tianle Wang and Tiberius Nkinyili and Timo Schick and Timofei Kornev and Titus Tunduny and Tobias Gerstenberg and Trenton Chang and Trishala Neeraj and Tushar Khot and Tyler Shultz and Uri Shaham and Vedant Misra and Vera Demberg and Victoria Nyamai and Vikas Raunak and Vinay Ramasesh and Vinay Uday Prabhu and Vishakh Padmakumar and Vivek Srikumar and William Fedus and William Saunders and William Zhang and Wout Vossen and Xiang Ren and Xiaoyu Tong and Xinran Zhao and Xinyi Wu and Xudong Shen and Yadollah Yaghoobzadeh and Yair Lakretz and Yangqiu Song and Yasaman Bahri and Yejin Choi and Yichi Yang and Yiding Hao and Yifu Chen and Yonatan Belinkov and Yu Hou and Yufang Hou and Yuntao Bai and Zachary Seid and Zhuoye Zhao and Zijian Wang and Zijie J. Wang and Zirui Wang and Ziyi Wu},
      year={2023},
      eprint={2206.04615},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2206.04615}, 
}

@inproceedings{NEURIPS2024_babilong,
 author = {Kuratov, Yuri and Bulatov, Aydar and Anokhin, Petr and Rodkin, Ivan and Sorokin, Dmitry and Sorokin, Artyom and Burtsev, Mikhail},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Globerson and L. Mackey and D. Belgrave and A. Fan and U. Paquet and J. Tomczak and C. Zhang},
 pages = {106519--106554},
 publisher = {Curran Associates, Inc.},
 title = {BABILong: Testing the Limits of LLMs with Long Context Reasoning-in-a-Haystack},
 url = {https://proceedings.neurips.cc/paper_files/paper/2024/file/c0d62e70dbc659cc9bd44cbcf1cb652f-Paper-Datasets_and_Benchmarks_Track.pdf},
 volume = {37},
 year = {2024}
}
@misc{kuratov2024search,
      title={In Search of Needles in a 10M Haystack: Recurrent Memory Finds What LLMs Miss}, 
      author={Yuri Kuratov and Aydar Bulatov and Petr Anokhin and Dmitry Sorokin and Artyom Sorokin and Mikhail Burtsev},
      year={2024},
      eprint={2402.10790},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}
@inproceedings{evalita2026overview,

title={EVALITA 2026: Overview of the 9th Evaluation Campaign of Natural Language Processing and Speech Tools for Italian},

author={Cutugno, Francesco and Miaschi, Alessio and Aprosio, Alessio Palmero and Rambelli, Giulia and Siciliani, Lucia and Stranisci, Marco Antonio},

booktitle={Proceedings of the Ninth Evaluation Campaign of Natural Language Processing and Speech Tools for Italian. Final Workshop (EVALITA 2026)},

publisher = {CEUR.org},

year = {2026},

month = {February},

address = {Bari, Italy}

}
@article{Yang2025Qwen251MTR,
  title={Qwen2.5-1M Technical Report},
  author={An Yang and Bowen Yu and Chengyuan Li and Dayiheng Liu and Fei Huang and Haoyan Huang and Jiandong Jiang and Jianhong Tu and Jianwei Zhang and Jingren Zhou and Junyang Lin and Kai Dang and Kexin Yang and Le Yu and Mei Li and Minmin Sun and Qin Zhu and Rui Men and Tao He and Weijia Xu and Wenbiao Yin and Wenyuan Yu and Xiafei Qiu and Xingzhang Ren and Xinlong Yang and Yong Li and Zhiying Xu and Zi-Yan Zhang},
  journal={ArXiv},
  year={2025},
  volume={abs/2501.15383},
  url={https://api.semanticscholar.org/CorpusID:275921951}
}

@inproceedings{moroni-etal-2025-learned,
    title = "What We Learned from Continually Training Minerva: A Case Study on {I}talian",
    author = "Moroni, Luca  and
      Bonomo, Tommaso  and
      Gioffr{\'e}, Luca  and
      Xu, Lu  and
      Fedele, Domenico  and
      Colosi, Leonardo  and
      Bejgu, Andrei Stefan  and
      Scir{\`e}, Alessandro  and
      Navigli, Roberto",
    editor = "Bosco, Cristina  and
      Jezek, Elisabetta  and
      Polignano, Marco  and
      Sanguinetti, Manuela",
    booktitle = "Proceedings of the Eleventh Italian Conference on Computational Linguistics (CLiC-it 2025)",
    month = sep,
    year = "2025",
    address = "Cagliari, Italy",
    publisher = "CEUR Workshop Proceedings",
    url = "https://aclanthology.org/2025.clicit-1.72/",
    pages = "760--774",
    ISBN = "979-12-243-0587-3"
}


@inproceedings{peng2024yarn,
title={Ya{RN}: Efficient Context Window Extension of Large Language Models},
author={Bowen Peng and Jeffrey Quesnelle and Honglu Fan and Enrico Shippole},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024},
url={https://openreview.net/forum?id=wHBfxhZu1u}
}
@article{Su2021rope,
  title={RoFormer: Enhanced Transformer with Rotary Position Embedding},
  author={Jianlin Su and Yu Lu and Shengfeng Pan and Bo Wen and Yunfeng Liu},
  journal={ArXiv},
  year={2021},
  volume={abs/2104.09864},
  url={https://api.semanticscholar.org/CorpusID:233307138}
}
@misc{needle-in-haystack,
title="Needle In A Haystack - Pressure Testing LLMs",
author={Kamradt, Gregory},
year={2023},
url = "https://github.com/gkamradt/LLMTest_NeedleInAHaystack"
}
@misc{geminiteam2024gemini15unlockingmultimodal,
      title={Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context}, 
      author={Gemini Team},
      year={2024},
      eprint={2403.05530},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2403.05530}, 
}
@misc{zhang2024inftybenchextendinglongcontext,
      title={$\infty$Bench: Extending Long Context Evaluation Beyond 100K Tokens}, 
      author={Xinrong Zhang and Yingfa Chen and Shengding Hu and Zihang Xu and Junhao Chen and Moo Khai Hao and Xu Han and Zhen Leng Thai and Shuo Wang and Zhiyuan Liu and Maosong Sun},
      year={2024},
      eprint={2402.13718},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2402.13718}, 
}
@inproceedings{tamburini-2025-babilong,
    title = "{BABIL}ong-{ITA}: A New Benchmark for Testing Large Language Models Effective Context Length and a Context Extension Method",
    author = "Tamburini, Fabio",
    editor = "Bosco, Cristina  and
      Jezek, Elisabetta  and
      Polignano, Marco  and
      Sanguinetti, Manuela",
    booktitle = "Proceedings of the Eleventh Italian Conference on Computational Linguistics (CLiC-it 2025)",
    month = sep,
    year = "2025",
    address = "Cagliari, Italy",
    publisher = "CEUR Workshop Proceedings",
    url = "https://aclanthology.org/2025.clicit-1.105/",
    pages = "1112--1120",
    ISBN = "979-12-243-0587-3"
}

@inbook{Leite_2024,
   title={FairytaleQA Translated: Enabling Educational Question and Answer Generation in Less-Resourced Languages},
   ISBN={9783031723155},
   ISSN={1611-3349},
   url={http://dx.doi.org/10.1007/978-3-031-72315-5_16},
   DOI={10.1007/978-3-031-72315-5_16},
   booktitle={Technology Enhanced Learning for Inclusive and Equitable Quality Education},
   publisher={Springer Nature Switzerland},
   author={Leite, Bernardo and Osório, Tomás Freitas and Cardoso, Henrique Lopes},
   year={2024},
   pages={222–236} }



@misc{kočiský2017narrativeqareadingcomprehensionchallenge,
      title={The NarrativeQA Reading Comprehension Challenge}, 
      author={Tomáš Kočiský and Jonathan Schwarz and Phil Blunsom and Chris Dyer and Karl Moritz Hermann and Gábor Melis and Edward Grefenstette},
      year={2017},
      eprint={1712.07040},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/1712.07040}, 
}

@inproceedings{bonomo-etal-2025-literaryqa,
    title = "{L}iterary{QA}: Towards Effective Evaluation of Long-document Narrative {QA}",
    author = "Bonomo, Tommaso  and
      Gioffr{\'e}, Luca  and
      Navigli, Roberto",
    editor = "Christodoulopoulos, Christos  and
      Chakraborty, Tanmoy  and
      Rose, Carolyn  and
      Peng, Violet",
    booktitle = "Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing",
    month = nov,
    year = "2025",
    address = "Suzhou, China",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2025.emnlp-main.1729/",
    doi = "10.18653/v1/2025.emnlp-main.1729",
    pages = "34086--34107",
    ISBN = "979-8-89176-332-6",
    abstract = "Question Answering (QA) on narrative text poses a unique challenge to current systems, requiring a deep understanding of long, complex documents. However, the reliability of NarrativeQA, the most widely used benchmark in this domain, is hindered by noisy documents and flawed QA pairs. In this work, we introduce LiteraryQA, a high-quality subset of NarrativeQA focused on literary works. Using a human- and LLM-validated pipeline, we identify and correct low-quality QA samples while removing extraneous text from source documents. We then carry out a meta-evaluation of automatic metrics to clarify how systems should be evaluated on LiteraryQA.This analysis reveals that all n-gram-based metrics have a low system-level correlation to human judgment, while LLM-as-a-Judge evaluations, even with small open-weight models, can strongly agree with the ranking identified by humans.Finally, we benchmark a set of long-context LLMs on LiteraryQA. We release our code and data at https://github.com/sapienzaNLP/LiteraryQA."
}



@misc{zheng2023judgingllmasajudgemtbenchchatbot,
      title={Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena}, 
      author={Lianmin Zheng and Wei-Lin Chiang and Ying Sheng and Siyuan Zhuang and Zhanghao Wu and Yonghao Zhuang and Zi Lin and Zhuohan Li and Dacheng Li and Eric P. Xing and Hao Zhang and Joseph E. Gonzalez and Ion Stoica},
      year={2023},
      eprint={2306.05685},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2306.05685}, 
}

@misc{polignano2024advancednaturalbasedinteractionitalian,
      title={Advanced Natural-based interaction for the ITAlian language: LLaMAntino-3-ANITA}, 
      author={Marco Polignano and Pierpaolo Basile and Giovanni Semeraro},
      year={2024},
      eprint={2405.07101},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2405.07101}, 
}

@inproceedings{moroni-etal-2025-sustainable,
    title = "Sustainable {I}talian {LLM} Evaluation: Community Perspectives and Methodological Guidelines",
    author = "Moroni, Luca  and
      Pappacoda, Gianmarco  and
      Barba, Edoardo  and
      Conia, Simone  and
      Galassi, Andrea  and
      Magnini, Bernardo  and
      Navigli, Roberto  and
      Torroni, Paolo  and
      Zanoli, Roberto",
    editor = "Bosco, Cristina  and
      Jezek, Elisabetta  and
      Polignano, Marco  and
      Sanguinetti, Manuela",
    booktitle = "Proceedings of the Eleventh Italian Conference on Computational Linguistics (CLiC-it 2025)",
    month = sep,
    year = "2025",
    address = "Cagliari, Italy",
    publisher = "CEUR Workshop Proceedings",
    url = "https://aclanthology.org/2025.clicit-1.71/",
    pages = "747--759",
    ISBN = "979-12-243-0587-3"
}

@misc{nissim2025challengingabilitieslargelanguage,
      title={Challenging the Abilities of Large Language Models in Italian: a Community Initiative},
      author={Malvina Nissim and Danilo Croce and Viviana Patti and Pierpaolo Basile and Giuseppe Attanasio and Elio Musacchio and Matteo Rinaldi and Federico Borazio and Maria Francis and Jacopo Gili and Daniel Scalena and Begoña Altuna and Ekhi Azurmendi and Valerio Basile and Luisa Bentivogli and Arianna Bisazza and Marianna Bolognesi and Dominique Brunato and Tommaso Caselli and Silvia Casola and Maria Cassese and Mauro Cettolo and Claudia Collacciani and Leonardo De Cosmo and Maria Pia Di Buono and Andrea Esuli and Julen Etxaniz and Chiara Ferrando and Alessia Fidelangeli and Simona Frenda and Achille Fusco and Marco Gaido and Andrea Galassi and Federico Galli and Luca Giordano and Mattia Goffetti and Itziar Gonzalez-Dios and Lorenzo Gregori and Giulia Grundler and Sandro Iannaccone and Chunyang Jiang and Moreno La Quatra and Francesca Lagioia and Soda Marem Lo and Marco Madeddu and Bernardo Magnini and Raffaele Manna and Fabio Mercorio and Paola Merlo and Arianna Muti and Vivi Nastase and Matteo Negri and Dario Onorati and Elena Palmieri and Sara Papi and Lucia Passaro and Giulia Pensa and Andrea Piergentili and Daniele Potertì and Giovanni Puccetti and Federico Ranaldi and Leonardo Ranaldi and Andrea Amelio Ravelli and Martina Rosola and Elena Sofia Ruzzetti and Giuseppe Samo and Andrea Santilli and Piera Santin and Gabriele Sarti and Giovanni Sartor and Beatrice Savoldi and Antonio Serino and Andrea Seveso and Lucia Siciliani and Paolo Torroni and Rossella Varvara and Andrea Zaninello and Asya Zanollo and Fabio Massimo Zanzotto and Kamyar Zeinalipour and Andrea Zugarini},
      year={2025},
      eprint={2512.04759},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2512.04759},
}

@inproceedings{magnini-etal-2025-leaderboard,
    title = "A Leaderboard for Benchmarking {LLM}s on {I}talian",
    author = "Magnini, Bernardo  and
      Madeddu, Marco  and
      Resta, Michele  and
      Zanoli, Roberto  and
      Cimmino, Martin  and
      Albano, Paolo  and
      Patti, Viviana",
    editor = "Bosco, Cristina  and
      Jezek, Elisabetta  and
      Polignano, Marco  and
      Sanguinetti, Manuela",
    booktitle = "Proceedings of the Eleventh Italian Conference on Computational Linguistics (CLiC-it 2025)",
    month = sep,
    year = "2025",
    address = "Cagliari, Italy",
    publisher = "CEUR Workshop Proceedings",
    url = "https://aclanthology.org/2025.clicit-1.61/",
    pages = "636--646",
    ISBN = "979-12-243-0587-3"
}
