%%% introduction
@inproceedings{devlin-etal-2019-bert,
    title = "{BERT}: Pre-training of Deep Bidirectional Transformers for Language Understanding",
    author = "Devlin, Jacob  and
      Chang, Ming-Wei  and
      Lee, Kenton  and
      Toutanova, Kristina",
    editor = "Burstein, Jill  and
      Doran, Christy  and
      Solorio, Thamar",
    booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",
    month = jun,
    year = "2019",
    address = "Minneapolis, Minnesota",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/N19-1423",
    doi = "10.18653/v1/N19-1423",
    pages = "4171--4186",
}

@inproceedings{he2020deberta,
  title={DEBERTA: DECODING-ENHANCED BERT WITH DISENTANGLED ATTENTION},
  author={He, Pengcheng and Liu, Xiaodong and Gao, Jianfeng and Chen, Weizhu},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@article{brown2020language,
  title={Language models are few-shot learners},
  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={1877--1901},
  year={2020}
}

@article{touvron2023llama,
  title={Llama 2: Open foundation and fine-tuned chat models},
  author={Touvron, Hugo and Martin, Louis and Stone, Kevin and Albert, Peter and Almahairi, Amjad and Babaei, Yasmine and Bashlykov, Nikolay and Batra, Soumya and Bhargava, Prajjwal and Bhosale, Shruti and others},
  journal={arXiv preprint arXiv:2307.09288},
  year={2023}
}

@misc{jiang2023mistral,
      title={Mistral 7B}, 
      author={Albert Q. Jiang and Alexandre Sablayrolles and Arthur Mensch and Chris Bamford and Devendra Singh Chaplot and Diego de las Casas and Florian Bressand and Gianna Lengyel and Guillaume Lample and Lucile Saulnier and Lélio Renard Lavaud and Marie-Anne Lachaux and Pierre Stock and Teven Le Scao and Thibaut Lavril and Thomas Wang and Timothée Lacroix and William El Sayed},
      year={2023},
      eprint={2310.06825},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}
@software{Open-LLM-Leaderboard-Report-2023,
  author = {Daniel Park},
  title = {Open-LLM-Leaderboard-Report},
  url = {https://github.com/dsdanielpark/Open-LLM-Leaderboard-Report},
  year = {2023}
}

@article{huang2023survey,
  title={A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions},
  author={Huang, Lei and Yu, Weijiang and Ma, Weitao and Zhong, Weihong and Feng, Zhangyin and Wang, Haotian and Chen, Qianglong and Peng, Weihua and Feng, Xiaocheng and Qin, Bing and others},
  journal={arXiv preprint arXiv:2311.05232},
  year={2023}
}

@article{manakul2023selfcheckgpt,
  title={Selfcheckgpt: Zero-resource black-box hallucination detection for generative large language models},
  author={Manakul, Potsawee and Liusie, Adian and Gales, Mark JF},
  journal={arXiv preprint arXiv:2303.08896},
  year={2023}
}

@inproceedings{kotek2023gender,
  title={Gender bias and stereotypes in Large Language Models},
  author={Kotek, Hadas and Dockum, Rikker and Sun, David},
  booktitle={Proceedings of The ACM Collective Intelligence Conference},
  pages={12--24},
  year={2023}
}

@article{sclar2023quantifying,
  title={Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting},
  author={Sclar, Melanie and Choi, Yejin and Tsvetkov, Yulia and Suhr, Alane},
  journal={arXiv preprint arXiv:2310.11324},
  year={2023}
}

@inproceedings{zhou2022large,
  title={Large Language Models are Human-Level Prompt Engineers},
  author={Zhou, Yongchao and Muresanu, Andrei Ioan and Han, Ziwen and Paster, Keiran and Pitis, Silviu and Chan, Harris and Ba, Jimmy},
  booktitle={The Eleventh International Conference on Learning Representations},
  year={2022}
}

@inproceedings{wei2021finetuned,
  title={Finetuned Language Models are Zero-Shot Learners},
  author={Wei, Jason and Bosma, Maarten and Zhao, Vincent and Guu, Kelvin and Yu, Adams Wei and Lester, Brian and Du, Nan and Dai, Andrew M and Le, Quoc V},
  booktitle={International Conference on Learning Representations},
  year={2021}
}

@inproceedings{wang-etal-2022-super,
    title = "Super-{N}atural{I}nstructions: Generalization via Declarative Instructions on 1600+ {NLP} Tasks",
  author={Wang, Yizhong and Mishra, Swaroop and Alipoormolabashi, Pegah and Kordi, Yeganeh and Mirzaei, Amirreza and Arunkumar, Anjana and Ashok, Arjun and Selvan Dhanasekaran, Arut and Naik, Atharva and Stap, David and others},
    editor = "Goldberg, Yoav  and
      Kozareva, Zornitsa  and
      Zhang, Yue",
    booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",
    month = dec,
    year = "2022",
    address = "Abu Dhabi, United Arab Emirates",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.emnlp-main.340",
    doi = "10.18653/v1/2022.emnlp-main.340",
    pages = "5085--5109",
}

@article{raffel2020exploring,
  title={Exploring the limits of transfer learning with a unified text-to-text transformer},
  author={Raffel, Colin and Shazeer, Noam and Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and Li, Wei and Liu, Peter J},
  journal={The Journal of Machine Learning Research},
  volume={21},
  number={1},
  pages={5485--5551},
  year={2020},
  publisher={JMLRORG}
}

@article{liu2019roberta,
  title={Roberta: A robustly optimized bert pretraining approach},
  author={Liu, Yinhan and Ott, Myle and Goyal, Naman and Du, Jingfei and Joshi, Mandar and Chen, Danqi and Levy, Omer and Lewis, Mike and Zettlemoyer, Luke and Stoyanov, Veselin},
  journal={arXiv preprint arXiv:1907.11692},
  year={2019}
}
%%% Other permtuation sensitivity

@inproceedings{zhao2021calibrate,
  title={Calibrate before use: Improving few-shot performance of language models},
  author={Zhao, Zihao and Wallace, Eric and Feng, Shi and Klein, Dan and Singh, Sameer},
  booktitle={International Conference on Machine Learning},
  pages={12697--12706},
  year={2021},
  organization={PMLR}
}

@inproceedings{reynolds2021prompt,
  title={Prompt programming for large language models: Beyond the few-shot paradigm},
  author={Reynolds, Laria and McDonell, Kyle},
  booktitle={Extended Abstracts of the 2021 CHI Conference on Human Factors in Computing Systems},
  pages={1--7},
  year={2021}
}

@article{lieber2021jurassic,
  title={Jurassic-1: Technical details and evaluation},
  author={Lieber, Opher and Sharir, Or and Lenz, Barak and Shoham, Yoav},
  journal={White Paper. AI21 Labs},
  volume={1},
  pages={9},
  year={2021}
}

@article{chae2023large,
  title={Large language models for text classification: From zero-shot learning to fine-tuning},
  author={Chae, Youngjin and Davidson, Thomas},
  journal={Open Science Foundation},
  year={2023}
}

@article{kojima2022large,
  title={Large language models are zero-shot reasoners},
  author={Kojima, Takeshi and Gu, Shixiang Shane and Reid, Machel and Matsuo, Yutaka and Iwasawa, Yusuke},
  journal={Advances in neural information processing systems},
  volume={35},
  pages={22199--22213},
  year={2022}
}

@article{zheng2023judging,
  title={Judging LLM-as-a-judge with MT-Bench and Chatbot Arena},
  author={Zheng, Lianmin and Chiang, Wei-Lin and Sheng, Ying and Zhuang, Siyuan and Wu, Zhanghao and Zhuang, Yonghao and Lin, Zi and Li, Zhuohan and Li, Dacheng and Xing, Eric and others},
  journal={arXiv preprint arXiv:2306.05685},
  year={2023}
}

@article{bubeck2023sparks,
  title={Sparks of artificial general intelligence: Early experiments with gpt-4},
  author={Bubeck, S{\'e}bastien and Chandrasekaran, Varun and Eldan, Ronen and Gehrke, Johannes and Horvitz, Eric and Kamar, Ece and Lee, Peter and Lee, Yin Tat and Li, Yuanzhi and Lundberg, Scott and others},
  journal={arXiv preprint arXiv:2303.12712},
  year={2023}
}

@misc{openaievals,
	title={Evals is a framework for evaluating llms and llm systems, and an open-source registry of benchmarks},
	author={OpenAI},
	url={https://github.com/openai/evals.},
        note = {Accessed: 2024-12-04}
}

@inproceedings{dubois2023alpaca,
 author = {Dubois, Yann and Li, Chen Xuechen and Taori, Rohan and Zhang, Tianyi and Gulrajani, Ishaan and Ba, Jimmy and Guestrin, Carlos and Liang, Percy S and Hashimoto, Tatsunori B},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine},
 pages = {30039--30069},
 publisher = {Curran Associates, Inc.},
 title = {AlpacaFarm: A Simulation Framework for Methods that Learn from Human Feedback},
 url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/5fc47800ee5b30b8777fdd30abcaaf3b-Paper-Conference.pdf},
 volume = {36},
 year = {2023}
}

@article{chiang2023can,
  title={Can large language models be an alternative to human evaluations?},
  author={Chiang, Cheng-Han and Lee, Hung-yi},
  journal={arXiv preprint arXiv:2305.01937},
  year={2023}
}

@article{wang2022self,
  title={Self-instruct: Aligning language models with self-generated instructions},
  author={Wang, Yizhong and Kordi, Yeganeh and Mishra, Swaroop and Liu, Alisa and Smith, Noah A and Khashabi, Daniel and Hajishirzi, Hannaneh},
  journal={arXiv preprint arXiv:2212.10560},
  year={2022}
}

@misc{taori2023stanford,
  title={Stanford alpaca: An instruction-following llama model},
  author={Taori, Rohan and Gulrajani, Ishaan and Zhang, Tianyi and Dubois, Yann and Li, Xuechen and Guestrin, Carlos and Liang, Percy and Hashimoto, Tatsunori B},
  year={2023}
}

@article{liu2023semantic,
  title={A semantic invariant robust watermark for large language models},
  author={Liu, Aiwei and Pan, Leyi and Hu, Xuming and Meng, Shiao and Wen, Lijie},
  journal={arXiv preprint arXiv:2310.06356},
  year={2023}
}

@article{li2023improving,
  title={Improving the Generation Quality of Watermarked Large Language Models via Word Importance Scoring},
  author={Li, Yuhang and Wang, Yihan and Shi, Zhouxing and Hsieh, Cho-Jui},
  journal={arXiv preprint arXiv:2311.09668},
  year={2023}
}

%GPT as judge

@article{zhang2023watermarks,
  title={Watermarks in the Sand: Impossibility of Strong Watermarking for Generative Models},
  author={Zhang, Hanlin and Edelman, Benjamin L and Francati, Danilo and Venturi, Daniele and Ateniese, Giuseppe and Barak, Boaz},
  journal={arXiv preprint arXiv:2311.04378},
  year={2023}
}

@inproceedings{Narayan2018DontGM,
    title = "Don{'}t Give Me the Details, Just the Summary! Topic-Aware Convolutional Neural Networks for Extreme Summarization",
    author = "Narayan, Shashi  and
      Cohen, Shay B.  and
      Lapata, Mirella",
    booktitle = "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing",
    month = oct # "-" # nov,
    year = "2018",
    address = "Brussels, Belgium",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/D18-1206",
    doi = "10.18653/v1/D18-1206",
    pages = "1797--1807",
}

@article{hu2020xtreme,
      author    = {Junjie Hu and Sebastian Ruder and Aditya Siddhant and Graham Neubig and Orhan Firat and Melvin Johnson},
      title     = {XTREME: A Massively Multilingual Multi-task Benchmark for Evaluating Cross-lingual Generalization},
      journal   = {CoRR},
      volume    = {abs/2003.11080},
      year      = {2020},
      archivePrefix = {arXiv},
      eprint    = {2003.11080}
}


%models

@inproceedings{lewis2019bart,
  title={BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension},
  author={Lewis, Mike and Liu, Yinhan and Goyal, Naman and Ghazvininejad, Marjan and Mohamed, Abdelrahman and Levy, Omer and Stoyanov, Veselin and Zettlemoyer, Luke},
  booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
  pages={7871--7880},
  year={2020}
}

@article{tunstall2023zephyr,
  title={Zephyr: Direct distillation of lm alignment},
  author={Tunstall, Lewis and Beeching, Edward and Lambert, Nathan and Rajani, Nazneen and Rasul, Kashif and Belkada, Younes and Huang, Shengyi and von Werra, Leandro and Fourrier, Cl{\'e}mentine and Habib, Nathan and others},
  journal={arXiv preprint arXiv:2310.16944},
  year={2023}
}

@article{tang2020multilingual,
  title={Multilingual translation with extensible multilingual pretraining and finetuning},
  author={Tang, Yuqing and Tran, Chau and Li, Xian and Chen, Peng-Jen and Goyal, Naman and Chaudhary, Vishrav and Gu, Jiatao and Fan, Angela},
  journal={arXiv preprint arXiv:2008.00401},
  year={2020}
}

%wm improvements

@inproceedings{yoo2023robust,
  title={Robust multi-bit natural language watermarking through invariant features},
  author={Yoo, KiYoon and Ahn, Wonhyuk and Jang, Jiho and Kwak, Nojun},
  booktitle={Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={2092--2115},
  year={2023}
}

@article{kuditipudi2023robust,
  title={Robust distortion-free watermarks for language models},
  author={Kuditipudi, Rohith and Thickstun, John and Hashimoto, Tatsunori and Liang, Percy},
  journal={arXiv preprint arXiv:2307.15593},
  year={2023}
}


@article{robinson2023leveraging,
  title={Leveraging Large Language Models for Multiple Choice Question Answering},
  author={Joshua Robinson and Christopher Michael Rytting and David Wingate},
  journal={ICLR},
  year={2023},
}

@inproceedings{miceli-barone-etal-2023-larger,
    title = "The Larger they are, the Harder they Fail: Language Models do not Recognize Identifier Swaps in Python",
    author = "Miceli Barone, Antonio Valerio  and
      Barez, Fazl  and
      Cohen, Shay B.  and
      Konstas, Ioannis",
    editor = "Rogers, Anna  and
      Boyd-Graber, Jordan  and
      Okazaki, Naoaki",
    booktitle = "Findings of the Association for Computational Linguistics: ACL 2023",
    month = jul,
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.findings-acl.19",
    doi = "10.18653/v1/2023.findings-acl.19",
    pages = "272--292",
}

@inproceedings{guo2017calibration,
  title={On calibration of modern neural networks},
  author={Guo, Chuan and Pleiss, Geoff and Sun, Yu and Weinberger, Kilian Q},
  booktitle={International conference on machine learning},
  pages={1321--1330},
  year={2017},
  organization={PMLR}
}

@inproceedings{distillationhinton,
title	= {Distilling the Knowledge in a Neural Network},
author	= {Geoffrey Hinton and Oriol Vinyals and Jeffrey Dean},
year	= {2015},
URL	= {http://arxiv.org/abs/1503.02531},booktitle	= {NIPS Deep Learning and Representation Learning Workshop}}


@article{pezeshkpour2023large,
  title={Large language models sensitivity to the order of options in multiple-choice questions},
  author={Pezeshkpour, Pouya and Hruschka, Estevam},
  journal={arXiv preprint arXiv:2308.11483},
  year={2023}
}


%%%%%%%%% models %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@article{achiam2023gpt,
  title={Gpt-4 technical report},
  author={Achiam, Josh and Adler, Steven and Agarwal, Sandhini and Ahmad, Lama and Akkaya, Ilge and Aleman, Florencia Leoni and Almeida, Diogo and Altenschmidt, Janko and Altman, Sam and Anadkat, Shyamal and others},
  journal={arXiv preprint arXiv:2303.08774},
  year={2023}
}

@article{team2023gemini,
  title={Gemini: a family of highly capable multimodal models},
  author={Anil, Rohan and Borgeaud, Sebastian and Wu, Yonghui and Alayrac, Jean-Baptiste and Yu, Jiahui and Soricut, Radu and Schalkwyk, Johan and Dai, Andrew M and Hauth, Anja and others},
  journal={arXiv preprint arXiv:2312.11805},
  year={2023}
}


%%%%%%%%% instruction alignment %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@article{ouyang2022training,
  title={Training language models to follow instructions with human feedback},
  author={Ouyang, Long and Wu, Jeffrey and Jiang, Xu and Almeida, Diogo and Wainwright, Carroll and Mishkin, Pamela and Zhang, Chong and Agarwal, Sandhini and Slama, Katarina and Ray, Alex and others},
  journal={Advances in neural information processing systems},
  volume={35},
  pages={27730--27744},
  year={2022}
}

@article{bai2022training,
  title={Training a helpful and harmless assistant with reinforcement learning from human feedback},
  author={Bai, Yuntao and Jones, Andy and Ndousse, Kamal and Askell, Amanda and Chen, Anna and DasSarma, Nova and Drain, Dawn and Fort, Stanislav and Ganguli, Deep and Henighan, Tom and others},
  journal={arXiv preprint arXiv:2204.05862},
  year={2022}
}

@inproceedings{lima2023zhou,
 author = {Zhou, Chunting and Liu, Pengfei and Xu, Puxin and Iyer, Srinivasan and Sun, Jiao and Mao, Yuning and Ma, Xuezhe and Efrat, Avia and Yu, Ping and YU, LILI and Zhang, Susan and Ghosh, Gargi and Lewis, Mike and Zettlemoyer, Luke and Levy, Omer},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine},
 pages = {55006--55021},
 publisher = {Curran Associates, Inc.},
 title = {LIMA: Less Is More for Alignment},
 volume = {36},
 year = {2023}
}

@article{chung2022scaling,
  title={Scaling instruction-finetuned language models},
  author={Chung, Hyung Won and Hou, Le and Longpre, Shayne and Zoph, Barret and Tay, Yi and Fedus, William and Li, Yunxuan and Wang, Xuezhi and Dehghani, Mostafa and Brahma, Siddhartha and others},
  journal = {Journal of Machine Learning Research},
  year={2024}
}

@article{dubey2024llama3,
  title={The llama 3 herd of models},
  author={Dubey, Abhimanyu and Jauhri, Abhinav and Pandey, Abhinav and Kadian, Abhishek and Al-Dahle, Ahmad and Letman, Aiesha and Mathur, Akhil and Schelten, Alan and Yang, Amy and Fan, Angela and others},
  journal={arXiv preprint arXiv:2407.21783},
  year={2024}
}


%%%%%%%%% datasets %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@article{fabbri2021summeval,
  title={Summeval: Re-evaluating summarization evaluation},
  author={Fabbri, Alexander R and Kry{\'s}ci{\'n}ski, Wojciech and McCann, Bryan and Xiong, Caiming and Socher, Richard and Radev, Dragomir},
  journal={Transactions of the Association for Computational Linguistics},
  volume={9},
  pages={391--409},
  year={2021},
  publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…}
}

@inproceedings{mehri2020usr,
  title={USR: An Unsupervised and Reference Free Evaluation Metric for Dialog Generation},
  author={Mehri, Shikib and Eskenazi, Maxine},
  booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
  pages={681--707},
  year={2020}
}

@article{mullooly2023cambridge,
  title={The Cambridge Multiple-Choice Questions Reading Dataset},
  author={Mullooly, Andrew and Andersen, {\O}istein and Benedetto, Luca and Buttery, Paula and Caines, Andrew and Gales, Mark JF and Karatay, Yasin and Knill, Kate and Liusie, Adian and Raina, Vatsal and others},
  year={2023},
  publisher={Cambridge University Press and Assessment}
}

@inproceedings{chhun2022human,
  title={Of Human Criteria and Automatic Metrics: A Benchmark of the Evaluation of Story Generation},
  author={Chhun, Cyril and Colombo, Pierre and Suchanek, Fabian and Clavel, Chlo{\'e}},
  booktitle={Proceedings of the 29th International Conference on Computational Linguistics},
  pages={5794--5836},
  year={2022}
}

%%%%%%%%%%%%%%%%%%% N-gram overlap metrics %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@inproceedings{papineni2002bleu,
  title={Bleu: a method for automatic evaluation of machine translation},
  author={Papineni, Kishore and Roukos, Salim and Ward, Todd and Zhu, Wei-Jing},
  booktitle={Proceedings of the 40th annual meeting of the Association for Computational Linguistics},
  pages={311--318},
  year={2002}
}

@inproceedings{lin2004rouge,
  title={Rouge: A package for automatic evaluation of summaries},
  author={Lin, Chin-Yew},
  booktitle={Text summarization branches out},
  pages={74--81},
  year={2004}
}

@inproceedings{banerjee2005meteor,
  title={METEOR: An automatic metric for MT evaluation with improved correlation with human judgments},
  author={Banerjee, Satanjeev and Lavie, Alon},
  booktitle={Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and/or summarization},
  pages={65--72},
  year={2005}
}

%%%%%%%%%%%%%%%%%%% Similarity overlap metrics %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

@article{zhang2019bertscore,
  title={Bertscore: Evaluating text generation with bert},
  author={Zhang, Tianyi and Kishore, Varsha and Wu, Felix and Weinberger, Kilian Q and Artzi, Yoav},
  journal={arXiv preprint arXiv:1904.09675},
  year={2019}
}

@article{zhao2019moverscore,
  title={MoverScore: Text generation evaluating with contextualized embeddings and earth mover distance},
  author={Zhao, Wei and Peyrard, Maxime and Liu, Fei and Gao, Yang and Meyer, Christian M and Eger, Steffen},
  journal={arXiv preprint arXiv:1909.02622},
  year={2019}
}

%%%%%%%%% bespoke single-attribute NLG evaluation %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@inproceedings{wang2020asking,
  title={Asking and Answering Questions to Evaluate the Factual Consistency of Summaries},
  author={Wang, Alex and Cho, Kyunghyun and Lewis, Mike},
  booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
  pages={5008--5020},
  year={2020}
}

@article{manakul2023mqag,
  title={Mqag: Multiple-choice question answering and generation for assessing information consistency in summarization},
  author={Manakul, Potsawee and Liusie, Adian and Gales, Mark JF},
  journal={arXiv preprint arXiv:2301.12307},
  year={2023}
}

@inproceedings{kryscinski2020evaluating,
  title={Evaluating the Factual Consistency of Abstractive Text Summarization},
  author={Kry{\'s}ci{\'n}ski, Wojciech and McCann, Bryan and Xiong, Caiming and Socher, Richard},
  booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
  pages={9332--9346},
  year={2020}
}

@inproceedings{dziri2019evaluating,
  title={Evaluating Coherence in Dialogue Systems using Entailment},
  author={Dziri, Nouha and Kamalloo, Ehsan and Mathewson, Kory and Zaiane, Osmar R},
  booktitle={Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},
  pages={3806--3812},
  year={2019}
}

@inproceedings{ye2021towards,
  title={Towards Quantifiable Dialogue Coherence Evaluation},
  author={Ye, Zheng and Lu, Liucun and Huang, Lishan and Lin, Liang and Liang, Xiaodan},
  booktitle={Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
  pages={2718--2729},
  year={2021}
}


%%%%%%%%% bespoke multi-attribute NLG evaluation %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@inproceedings{zhong2022towards,
    title = "Towards a Unified Multi-Dimensional Evaluator for Text Generation",
    author = "Zhong, Ming  and
      Liu, Yang  and
      Yin, Da  and
      Mao, Yuning  and
      Jiao, Yizhu  and
      Liu, Pengfei  and
      Zhu, Chenguang  and
      Ji, Heng  and
      Han, Jiawei",
    editor = "Goldberg, Yoav  and
      Kozareva, Zornitsa  and
      Zhang, Yue",
    booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",
    month = dec,
    year = "2022",
    address = "Abu Dhabi, United Arab Emirates",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.emnlp-main.131",
    doi = "10.18653/v1/2022.emnlp-main.131",
    pages = "2023--2038",
}

@article{yuan2021bartscore,
  title={Bartscore: Evaluating generated text as text generation},
  author={Yuan, Weizhe and Neubig, Graham and Liu, Pengfei},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={27263--27277},
  year={2021}
}

%%%%%%%%%% llm evaluation %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@inproceedings{liu-etal-2023-g,
    title = "{G}-Eval: {NLG} Evaluation using Gpt-4 with Better Human Alignment",
    author = "Liu, Yang  and
      Iter, Dan  and
      Xu, Yichong  and
      Wang, Shuohang  and
      Xu, Ruochen  and
      Zhu, Chenguang",
    editor = "Bouamor, Houda  and
      Pino, Juan  and
      Bali, Kalika",
    booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing",
    month = dec,
    year = "2023",
    address = "Singapore",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.emnlp-main.153",
    doi = "10.18653/v1/2023.emnlp-main.153",
    pages = "2511--2522",
}

@article{wang2023chatgpt,
  title={Is chatgpt a good nlg evaluator? a preliminary study},
  author={Wang, Jiaan and Liang, Yunlong and Meng, Fandong and Shi, Haoxiang and Li, Zhixu and Xu, Jinan and Qu, Jianfeng and Zhou, Jie},
  journal={arXiv preprint arXiv:2303.04048},
  year={2023}
}

@article{kocmi2023large,
  title={Large language models are state-of-the-art evaluators of translation quality},
  author={Kocmi, Tom and Federmann, Christian},
  journal={arXiv preprint arXiv:2302.14520},
  year={2023}
}

@article{fu2023gptscore,
  title={Gptscore: Evaluate as you desire},
  author={Fu, Jinlan and Ng, See-Kiong and Jiang, Zhengbao and Liu, Pengfei},
  journal={arXiv preprint arXiv:2302.04166},
  year={2023}
}

@article{motegi2012network,
  title={A network-based dynamical ranking system for competitive sports},
  author={Motegi, Shun and Masuda, Naoki},
  journal={Scientific reports},
  volume={2},
  number={1},
  pages={904},
  year={2012},
  publisher={Nature Publishing Group UK London}
}

@inproceedings{cossock2006subset,
  title={Subset ranking using regression},
  author={Cossock, David and Zhang, Tong},
  booktitle={Learning Theory: 19th Annual Conference on Learning Theory, COLT 2006, Pittsburgh, PA, USA, June 22-25, 2006. Proceedings 19},
  pages={605--619},
  year={2006},
  organization={Springer}
}

%%%%%%%%%% llm pairwise evaluation %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@inproceedings{liusie-etal-2024-llm,
    title = "{LLM} Comparative Assessment: Zero-shot {NLG} Evaluation through Pairwise Comparisons using Large Language Models",
    author = "Liusie, Adian  and
      Manakul, Potsawee  and
      Gales, Mark",
    editor = "Graham, Yvette  and
      Purver, Matthew",
    booktitle = "Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = mar,
    year = "2024",
    address = "St. Julian{'}s, Malta",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2024.eacl-long.8",
    pages = "139--151",
}

@inproceedings{hinton2015distilling,
  title={Distilling the Knowledge in a Neural Network},
  author={Hinton, Geoffrey and Vinyals, Oriol and Dean, Jeff},
  booktitle={Neural Information Processing Systems 2014 Deep Learning and Representation Learning Workshop},
  year={2015},
  note={arXiv preprint arXiv:1503.02531}
}

@inproceedings{
    malinin2020ensemble,
    title={Ensemble Distribution Distillation},
    author={Andrey Malinin and Bruno Mlodozeniec and Mark Gales},
    booktitle={International Conference on Learning Representations},
    year={2020},
    url={https://openreview.net/forum?id=H1xWdeEFvr}
}

@InProceedings{pmlr-v216-fathullah23a,
  title = 	 {Logit-based ensemble distribution distillation for robust autoregressive sequence uncertainties},
  author =       {Fathullah, Yassir and Xia, Guoxuan and J. F. Gales, Mark},
  booktitle = 	 {Proceedings of the Thirty-Ninth Conference on Uncertainty in Artificial Intelligence},
  pages = 	 {582--591},
  year = 	 {2023},
  editor = 	 {Evans, Robin J. and Shpitser, Ilya},
  volume = 	 {216},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {31 Jul--04 Aug},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v216/fathullah23a/fathullah23a.pdf},
  url = 	 {https://proceedings.mlr.press/v216/fathullah23a.html}
}


@inproceedings{fathullah2022self,
  title={Self-Distribution Distillation: Efficient Uncertainty Estimation},
  author={Fathullah, Yassir and Gales, Mark},
  booktitle={Proceedings of the 38th Conference on Uncertainty in Artificial Intelligence (UAI)},
  year={2022},
  series={Proceedings of Machine Learning Research},
  volume={180},
  pages={655--665},
  publisher={PMLR}
}

@inproceedings{yang2024large,
    title={Large Language Models as Optimizers},
    author={Chengrun Yang and Xuezhi Wang and Yifeng Lu and Hanxiao Liu and Quoc V Le and Denny Zhou and Xinyun Chen},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024},
    url={https://openreview.net/forum?id=83B55k9s0h}
}

@inproceedings{lakomkin-etal-2024-end,
  author={Lakomkin, Egor and Wu, Chunyang and Fathullah, Yassir and Kalinli, Ozlem and Seltzer, Michael L. and Fuegen, Christian},
  booktitle={ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, 
  title={End-to-End Speech Recognition Contextualization with Large Language Models}, 
  year={2024},
  volume={},
  number={},
  pages={12406-12410},
  keywords={Training;Adaptation models;Speech recognition;Signal processing;Acoustics;Task analysis;Speech processing;contextual biasing;large language models;speech recognition},
  doi={10.1109/ICASSP48485.2024.10446898}
}


@inproceedings{fathullah-etal-2024-audiochatllama,
    title = "{A}udio{C}hat{L}lama: Towards General-Purpose Speech Abilities for {LLM}s",
    author = "Fathullah, Yassir  and
      Wu, Chunyang  and
      Lakomkin, Egor  and
      Li, Ke  and
      Jia, Junteng  and
      Shangguan, Yuan  and
      Mahadeokar, Jay  and
      Kalinli, Ozlem  and
      Fuegen, Christian  and
      Seltzer, Mike",
    editor = "Duh, Kevin  and
      Gomez, Helena  and
      Bethard, Steven",
    booktitle = "Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)",
    month = jun,
    year = "2024",
    address = "Mexico City, Mexico",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2024.naacl-long.309/",
    doi = "10.18653/v1/2024.naacl-long.309",
    pages = "5522--5532"
}

@article{liusie2024efficient,
  title={Efficient LLM Comparative Assessment: a Product of Experts Framework for Pairwise Comparisons},
  author={Liusie, Adian and Raina, Vatsal and Fathullah, Yassir and Gales, Mark},
  journal={arXiv preprint arXiv:2405.05894},
  year={2024}
}

@article{qin2023large,
  title={Large Language Models are Effective Text Rankers with Pairwise Ranking Prompting},
  author={Qin, Zhen and Jagerman, Rolf and Hui, Kai and Zhuang, Honglei and Wu, Junru and Shen, Jiaming and Liu, Tianqi and Liu, Jialu and Metzler, Donald and Wang, Xuanhui and others},
  journal={arXiv preprint arXiv:2306.17563},
  year={2023}
}

@article{park2024paireval,
  title={PairEval: Open-domain Dialogue Evaluation with Pairwise Comparison},
  author={Park, ChaeHun and Choi, Minseok and Lee, Dohyun and Choo, Jaegul},
  journal={arXiv preprint arXiv:2404.01015},
  year={2024}
}

@misc{wang2023large,
      title={Large Language Models are not Fair Evaluators}, 
      author={Peiyi Wang and Lei Li and Liang Chen and Zefan Cai and Dawei Zhu and Binghuai Lin and Yunbo Cao and Qi Liu and Tianyu Liu and Zhifang Sui},
      year={2023},
      eprint={2305.17926},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{liu2024aligning,
      title={Aligning with Human Judgement: The Role of Pairwise Preference in Large Language Model Evaluators}, 
      author={Yinhong Liu and Han Zhou and Zhijiang Guo and Ehsan Shareghi and Ivan Vulić and Anna Korhonen and Nigel Collier},
      year={2024},
      eprint={2403.16950},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

%%%%%%% positional bias %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@article{liusie2024teacher,
  title={Teacher-Student Training for Debiasing: General Permutation Debiasing for Large Language Models},
  author={Liusie, Adian and Fathullah, Yassir and Gales, Mark JF},
  journal={arXiv preprint arXiv:2403.13590},
  year={2024}
}

@misc{chen2024humans,
      title={Humans or LLMs as the Judge? A Study on Judgement Biases}, 
      author={Guiming Hardy Chen and Shunian Chen and Ziche Liu and Feng Jiang and Benyou Wang},
      year={2024},
      eprint={2402.10669},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

%%%%%%%% metrics used for comparative %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@misc{molenda2024waterjudge,
      title={WaterJudge: Quality-Detection Trade-off when Watermarking Large Language Models}, 
      author={Piotr Molenda and Adian Liusie and Mark J. F. Gales},
      year={2024},
      eprint={2403.19548},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{dominguezolmedo2024questioningsurveyresponseslarge,
      title={Questioning the Survey Responses of Large Language Models}, 
      author={Ricardo Dominguez-Olmedo and Moritz Hardt and Celestine Mendler-Dünner},
      year={2024},
      eprint={2306.07951},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2306.07951}, 
}

@article{zhu2023judgelm,
      title={JudgeLM: Fine-tuned Large Language Models are Scalable Judges}, 
      author={Lianghui Zhu and Xinggang Wang and Xinlong Wang},
      year={2023},
      eprint={2310.17631},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{chen2024premiseordermattersreasoning,
      title={Premise Order Matters in Reasoning with Large Language Models}, 
      author={Xinyun Chen and Ryan A. Chi and Xuezhi Wang and Denny Zhou},
      year={2024},
      eprint={2402.08939},
      archivePrefix={arXiv},
      primaryClass={cs.AI},
      url={https://arxiv.org/abs/2402.08939}, 
}

@article{liu-etal-2024-lost,
    title = "Lost in the Middle: How Language Models Use Long Contexts",
    author = "Liu, Nelson F.  and
      Lin, Kevin  and
      Hewitt, John  and
      Paranjape, Ashwin  and
      Bevilacqua, Michele  and
      Petroni, Fabio  and
      Liang, Percy",
    journal = "Transactions of the Association for Computational Linguistics",
    volume = "12",
    year = "2024",
    address = "Cambridge, MA",
    publisher = "MIT Press",
    url = "https://aclanthology.org/2024.tacl-1.9",
    doi = "10.1162/tacl_a_00638",
    pages = "157--173",
    abstract = "While recent language models have the ability to take long contexts as input, relatively little is known about how well they use longer context. We analyze the performance of language models on two tasks that require identifying relevant information in their input contexts: multi-document question answering and key-value retrieval. We find that performance can degrade significantly when changing the position of relevant information, indicating that current language models do not robustly make use of information in long input contexts. In particular, we observe that performance is often highest when relevant information occurs at the beginning or end of the input context, and significantly degrades when models must access relevant information in the middle of long contexts, even for explicitly long-context models. Our analysis provides a better understanding of how language models use their input context and provides new evaluation protocols for future long-context language models.",
}

@book{agresti1990categorical,
  title={Categorical data analysis},
  author={Agresti, Alan},
  year={1990},
  publisher={John Wiley \& Sons}
}

@article{caron2012efficient,
  title={Efficient Bayesian inference for generalized Bradley--Terry models},
  author={Caron, Francois and Doucet, Arnaud},
  journal={Journal of Computational and Graphical Statistics},
  volume={21},
  number={1},
  pages={174--196},
  year={2012},
  publisher={Taylor \& Francis}
}

@inproceedings{wang-etal-2023-primacy,
    title = "Primacy Effect of {C}hat{GPT}",
    author = "Wang, Yiwei  and
      Cai, Yujun  and
      Chen, Muhao  and
      Liang, Yuxuan  and
      Hooi, Bryan",
    editor = "Bouamor, Houda  and
      Pino, Juan  and
      Bali, Kalika",
    booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing",
    month = dec,
    year = "2023",
    address = "Singapore",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.emnlp-main.8",
    doi = "10.18653/v1/2023.emnlp-main.8",
    pages = "108--115",
    abstract = "Instruction-tuned large language models (LLMs), such as ChatGPT, have led to promising zero-shot performance in discriminative natural language understanding (NLU) tasks. This involves querying the LLM using a prompt containing the question, and the candidate labels to choose from. The question-answering capabilities of ChatGPT arise from its pre-training on large amounts of human-written text, as well as its subsequent fine-tuning on human preferences, which motivates us to ask: Does ChatGPT also inherit humans{'} cognitive biases? In this paper, we study the primacy effect of ChatGPT: the tendency of selecting the labels at earlier positions as the answer. We have two main findings: i) ChatGPT{'}s decision is sensitive to the order of labels in the prompt; ii) ChatGPT has a clearly higher chance to select the labels at earlier positions as the answer. We hope that our experiments and analyses provide additional insights into building more reliable ChatGPT-based solutions. We release the source code at https://github.com/wangywUST/PrimacyEffectGPT.",
}

%%%%%%%%%%%%%%% papers about standard calculations of scores from comparisons %%%%%%%%

@article{vilette2020comparing,
  title={Comparing dominance hierarchy methods using a data-splitting approach with real-world data},
  author={Vilette, Chlo{\'e} and Bonnell, Tyler and Henzi, Peter and Barrett, Louise},
  journal={Behavioral Ecology},
  volume={31},
  number={6},
  pages={1379--1390},
  year={2020},
  publisher={Oxford University Press UK}
}

@article{van198711,
  title={11. Dominance and its behavioral measures in a captive wolf pack},
  author={van Hooff, Jan ARAM and Wensing, JA},
  journal={Man Wolf Adv. Issues Probl. Captive Wolf Res},
  volume={4},
  pages={219},
  year={1987}
}

@article{cattelan2012models,
  title={Models for paired comparison data: A review with emphasis on dependent data},
  author={Cattelan, Manuela},
  year={2012}
}

@book{david1963method,
  title={The method of paired comparisons},
  author={David, Herbert Aron},
  volume={12},
  year={1963},
  publisher={London}
}

@book{saaty2001decision,
  title={Decision making for leaders: the analytic hierarchy process for decisions in a complex world},
  author={Saaty, Thomas L},
  year={2001},
  publisher={RWS publications}
}

@article{mcfadden2000mixed,
  title={Mixed MNL models for discrete response},
  author={McFadden, Daniel and Train, Kenneth},
  journal={Journal of applied Econometrics},
  volume={15},
  number={5},
  pages={447--470},
  year={2000},
  publisher={Wiley Online Library}
}

@article{mcfadden1972conditional,
  title={Conditional logit analysis of qualitative choice behavior},
  author={McFadden, Daniel},
  year={1972}
}

@book{louviere2000stated,
  title={Stated choice methods: analysis and applications},
  author={Louviere, Jordan J and Hensher, David A and Swait, Joffre D},
  year={2000},
  publisher={Cambridge university press}
}

@article{manski1977structure,
  title={The structure of random utility models},
  author={Manski, Charles F},
  journal={Theory and decision},
  volume={8},
  number={3},
  pages={229},
  year={1977},
  publisher={Kluwer Academic Publishers}
}

@article{liu2009learning,
  title={Learning to rank for information retrieval},
  author={Liu, Tie-Yan and others},
  journal={Foundations and Trends{\textregistered} in Information Retrieval},
  volume={3},
  number={3},
  pages={225--331},
  year={2009},
  publisher={Now Publishers, Inc.}
}

@inproceedings{cao2007learning,
  title={Learning to rank: from pairwise approach to listwise approach},
  author={Cao, Zhe and Qin, Tao and Liu, Tie-Yan and Tsai, Ming-Feng and Li, Hang},
  booktitle={Proceedings of the 24th international conference on Machine learning},
  pages={129--136},
  year={2007}
}

@inproceedings{dwork2001rank,
  title={Rank aggregation methods for the web},
  author={Dwork, Cynthia and Kumar, Ravi and Naor, Moni and Sivakumar, Dandapani},
  booktitle={Proceedings of the 10th international conference on World Wide Web},
  pages={613--622},
  year={2001}
}

@inproceedings{zheng2023large,
  title={Large language models are not robust multiple choice selectors},
  author={Zheng, Chujie and Zhou, Hao and Meng, Fandong and Zhou, Jie and Huang, Minlie},
  booktitle={The Twelfth International Conference on Learning Representations},
  year={2023}
}

@inproceedings{sha2016chalkboarding,
  title={Chalkboarding: A new spatiotemporal query paradigm for sports play retrieval},
  author={Sha, Long and Lucey, Patrick and Yue, Yisong and Carr, Peter and Rohlf, Charlie and Matthews, Iain},
  booktitle={Proceedings of the 21st International Conference on Intelligent User Interfaces},
  pages={336--347},
  year={2016}
}

@article{minka2018trueskill,
  title={Trueskill 2: An improved bayesian skill rating system},
  author={Minka, Tom and Cleven, Ryan and Zaykov, Yordan},
  journal={Technical Report},
  year={2018}
}

@misc{qwen2.5,
    title = {Qwen2.5: A Party of Foundation Models},
    url = {https://qwenlm.github.io/blog/qwen2.5/},
    author = {{Qwen Team}},
    month = {September},
    year = {2024}
}

@article{herbrich2006trueskill,
  title={TrueSkill™: a Bayesian skill rating system},
  author={Herbrich, Ralf and Minka, Tom and Graepel, Thore},
  journal={Advances in neural information processing systems},
  volume={19},
  year={2006}
}

@misc{raina2024finetuningllmscomparativeassessment,
      title={Finetuning LLMs for Comparative Assessment Tasks}, 
      author={Vatsal Raina and Adian Liusie and Mark Gales},
      year={2024},
      eprint={2409.15979},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2409.15979}, 
}

@article{csato2013ranking,
  title={Ranking by pairwise comparisons for Swiss-system tournaments},
  author={Csat{\'o}, L{\'a}szl{\'o}},
  journal={Central European Journal of Operations Research},
  volume={21},
  pages={783--803},
  year={2013},
  publisher={Springer}
}

@article{beaudoin2018computationally,
  title={A computationally intensive ranking system for paired comparison data},
  author={Beaudoin, David and Swartz, Tim},
  journal={Operations Research Perspectives},
  volume={5},
  pages={105--112},
  year={2018},
  publisher={Elsevier}
}

@inproceedings{cao2018attentive,
  title={Attentive group recommendation},
  author={Cao, Da and He, Xiangnan and Miao, Lianhai and An, Yahui and Yang, Chao and Hong, Richang},
  booktitle={The 41st International ACM SIGIR conference on research \& development in information retrieval},
  pages={645--654},
  year={2018}
}

@inproceedings{baltrunas2010group,
  title={Group recommendations with rank aggregation and collaborative filtering},
  author={Baltrunas, Linas and Makcinskas, Tadas and Ricci, Francesco},
  booktitle={Proceedings of the fourth ACM conference on Recommender systems},
  pages={119--126},
  year={2010}
}

@book{luce2005individual,
  title={Individual choice behavior: A theoretical analysis},
  author={Luce, R Duncan},
  year={2005},
  publisher={Courier Corporation}
}

@article{chen2022optimal,
  title={Optimal full ranking from pairwise comparisons},
  author={Chen, Pinhan and Gao, Chao and Zhang, Anderson Y},
  journal={The Annals of Statistics},
  volume={50},
  number={3},
  pages={1775--1805},
  year={2022},
  publisher={Institute of Mathematical Statistics}
}

@article{davidson1976bibliography,
  title={A bibliography on the method of paired comparisons},
  author={Davidson, Roger R and Farquhar, Peter H},
  journal={Biometrics},
  pages={241--252},
  year={1976},
  publisher={JSTOR}
}

@article{newman2023efficient,
  title={Efficient computation of rankings from pairwise comparisons},
  author={Newman, MEJ},
  journal={Journal of Machine Learning Research},
  volume={24},
  number={238},
  pages={1--25},
  year={2023}
}

@article{bradley1952rank,
  title={Rank analysis of incomplete block designs: I. The method of paired comparisons},
  author={Bradley, Ralph Allan and Terry, Milton E},
  journal={Biometrika},
  volume={39},
  number={3/4},
  pages={324--345},
  year={1952},
  publisher={JSTOR}
}

@article{zermelo1929berechnung,
  title={Die berechnung der turnier-ergebnisse als ein maximumproblem der wahrscheinlichkeitsrechnung},
  author={Zermelo, Ernst},
  journal={Mathematische Zeitschrift},
  volume={29},
  number={1},
  pages={436--460},
  year={1929},
  publisher={Springer}
}

@article{dykstra1956note,
  title={A Note on the Rank Analysis of Incomplete Block Designs--Applications Beyond the Scope of Existing Tables},
  author={Dykstra, Otto},
  journal={Biometrics},
  volume={12},
  number={3},
  pages={301--306},
  year={1956},
  publisher={JSTOR}
}

@article{hunter2004mm,
  title={MM algorithms for generalized Bradley-Terry models},
  author={Hunter, David R},
  journal={The annals of statistics},
  volume={32},
  number={1},
  pages={384--406},
  year={2004},
  publisher={Institute of Mathematical Statistics}
}

%%%%%%%%%%%%%%%%%%%%%%% papers about PoE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@InProceedings{hinton1999products,
  title		= {Products of experts},
  author	= {Hinton, Geoffrey E.},
  booktitle	= {Artificial Neural Networks, 1999. ICANN 99. Ninth
		  International Conference on (Conf. Publ. No. 470)},
  volume	= {1},
  pages		= {1--6},
  year		= {1999},
  organization	= {IET},
  url_abstract	= {absps/icann-99.html},
  url_ps	= {absps/icann-99.ps},
  url_pdf	= {absps/icann-99.pdf}
}

@ARTICLE{Welling:2007,
    AUTHOR = {Welling, M. },
    TITLE   = {{P}roduct of experts},
    YEAR    = {2007},
    JOURNAL = {Scholarpedia},
    VOLUME  = {2},
    NUMBER  = {10},
    PAGES   = {3879},
    DOI     = {10.4249/scholarpedia.3879},
    NOTE    = {revision \#137078}
}

@article{zen2011product,
  title={Product of experts for statistical parametric speech synthesis},
  author={Zen, Heiga and Gales, Mark JF and Nankaku, Yoshihiko and Tokuda, Keiichi},
  journal={IEEE Transactions on Audio, Speech, and Language Processing},
  volume={20},
  number={3},
  pages={794--805},
  year={2011},
  publisher={IEEE}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


@inproceedings{huang2022mm,
    author = {Huang, Xun and Mallya, Arun and Wang, Ting-Chun and Liu, Ming-Yu},
    title = {Multimodal Conditional Image Synthesis with Product-of-Experts GANs},
    year = {2022},
    isbn = {978-3-031-19786-4},
    publisher = {Springer-Verlag},
    address = {Berlin, Heidelberg},
    url = {https://doi.org/10.1007/978-3-031-19787-1_6},
    doi = {10.1007/978-3-031-19787-1_6},
    booktitle = {Computer Vision – ECCV 2022: 17th European Conference, Tel Aviv, Israel, October 23–27, 2022, Proceedings, Part XVI},
    pages = {91–109},
    numpages = {19},
    keywords = {Image synthesis, Multimodal learning, GAN},
    location = {<conf-loc content-type="InPerson">Tel Aviv, Israel</conf-loc>}
}

@inproceedings{wu2018mmgen,
     author = {Wu, Mike and Goodman, Noah},
     booktitle = {Advances in Neural Information Processing Systems},
     editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
     pages = {},
     publisher = {Curran Associates, Inc.},
     title = {Multimodal Generative Models for Scalable Weakly-Supervised Learning},
     url = {https://proceedings.neurips.cc/paper_files/paper/2018/file/1102a326d5f7c9e04fc3c89d0ede88c9-Paper.pdf},
     volume = {31},
     year = {2018}
}

@inproceedings{vedantam2018generative,
    title={Generative Models of Visually Grounded Imagination},
    author={Ramakrishna Vedantam and Ian Fischer and Jonathan Huang and Kevin Murphy},
    booktitle={International Conference on Learning Representations},
    year={2018},
    url={https://openreview.net/forum?id=HkCsm6lRb},
}

@article{kutuzova2021multimodal,
  title={Multimodal variational autoencoders for semi-supervised learning: In defense of product-of-experts},
  author={Kutuzova, Svetlana and Krause, Oswin and McCloskey, Douglas and Nielsen, Mads and Igel, Christian},
  journal={arXiv preprint arXiv:2101.07240},
  year={2021}
}

@inproceedings{williams2001product,
 author = {Williams, Christopher and Agakov, Felix and Felderhof, Stephen},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {T. Dietterich and S. Becker and Z. Ghahramani},
 pages = {},
 publisher = {MIT Press},
 title = {Products of Gaussians},
 url = {https://proceedings.neurips.cc/paper_files/paper/2001/file/8232e119d8f59aa83050a741631803a6-Paper.pdf},
 volume = {14},
 year = {2001}
}

@article{gales2006product,
    title = {Product of Gaussians for speech recognition},
    journal = {Computer Speech \& Language},
    volume = {20},
    number = {1},
    pages = {22-40},
    year = {2006},
    issn = {0885-2308},
    doi = {https://doi.org/10.1016/j.csl.2004.12.002},
    url = {https://www.sciencedirect.com/science/article/pii/S0885230804000786},
    author = {M.J.F. Gales and S.S. Airey},
}

@book{boyd2004convex,
  title={Convex optimization},
  author={Boyd, Stephen P and Vandenberghe, Lieven},
  year={2004},
  publisher={Cambridge university press}
}


@article{raina2024question,
  title={Question Difficulty Ranking for Multiple-Choice Reading Comprehension},
  author={Raina, Vatsal and Gales, Mark},
  journal={arXiv preprint arXiv:2404.10704},
  year={2024}
}

@inproceedings{liu2023g,
  title={G-Eval: NLG Evaluation using Gpt-4 with Better Human Alignment},
  author={Liu, Yang and Iter, Dan and Xu, Yichong and Wang, Shuohang and Xu, Ruochen and Zhu, Chenguang},
  booktitle={Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
  pages={2511--2522},
  year={2023}
}