
@inproceedings{brown_language_2020,
	title = {Language {Models} are {Few}-{Shot} {Learners}},
	volume = {33},
	url = {https://proceedings.neurips.cc/paper/2020/hash/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html},
	abstract = {We demonstrate that scaling up language models greatly improves task-agnostic, few-shot performance, sometimes even becoming competitive with prior state-of-the-art fine-tuning approaches. Specifically, we train GPT-3, an autoregressive language model with 175 billion parameters, 10x more than any previous non-sparse language model, and test its performance in the few-shot setting.  For all tasks, GPT-3 is applied without any gradient updates or fine-tuning, with tasks and few-shot demonstrations specified purely via text interaction with the model.  GPT-3 achieves strong performance on many NLP datasets, including translation, question-answering, and cloze tasks. We also identify some datasets where GPT-3's few-shot learning still struggles, as well as some datasets where GPT-3 faces methodological issues related to training on large web corpora.},
	urldate = {2024-01-29},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},
	year = {2020},
	pages = {1877--1901},
}


@inproceedings{devlin_bert_2019,
	address = {Minneapolis, Minnesota},
	title = {{BERT}: {Pre}-training of {Deep} {Bidirectional} {Transformers} for {Language} {Understanding}},
	shorttitle = {{BERT}},
	url = {https://aclanthology.org/N19-1423},
	doi = {10.18653/v1/N19-1423},
	abstract = {We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models (Peters et al., 2018a; Radford et al., 2018), BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications. BERT is conceptually simple and empirically powerful. It obtains new state-of-the-art results on eleven natural language processing tasks, including pushing the GLUE score to 80.5 (7.7 point absolute improvement), MultiNLI accuracy to 86.7\% (4.6\% absolute improvement), SQuAD v1.1 question answering Test F1 to 93.2 (1.5 point absolute improvement) and SQuAD v2.0 Test F1 to 83.1 (5.1 point absolute improvement).},
	urldate = {2023-11-30},
	booktitle = {Proceedings of the 2019 {Conference} of the {North} {American} {Chapter} of the {Association} for {Computational} {Linguistics}: {Human} {Language} {Technologies}, {Volume} 1 ({Long} and {Short} {Papers})},
	publisher = {Association for Computational Linguistics},
	author = {Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
	editor = {Burstein, Jill and Doran, Christy and Solorio, Thamar},
	month = jun,
	year = {2019},
	pages = {4171--4186},
}


@article{raffel_exploring_2020,
	title = {Exploring the {Limits} of {Transfer} {Learning} with a {Unified} {Text}-to-{Text} {Transformer}},
	volume = {21},
	issn = {1533-7928},
	url = {http://jmlr.org/papers/v21/20-074.html},
	abstract = {Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts all text-based language problems into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled data sets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our data set, pre-trained models, and code.},
	number = {140},
	urldate = {2023-11-30},
	journal = {Journal of Machine Learning Research},
	author = {Raffel, Colin and Shazeer, Noam and Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and Li, Wei and Liu, Peter J.},
	year = {2020},
	pages = {1--67},
}

@misc{openai_language_2020,
	title = {Language models are few-shot learners},
	url = {https://openai.com/research/language-models-are-few-shot-learners},
	language = {en-US},
	urldate = {2023-11-30},
	author = {OpenAI},
	year = {2020},
	file = {Snapshot:/Users/mis/Zotero/storage/94BGBQRG/language-models-are-few-shot-learners.html:text/html},
}

@misc{bommasani_opportunities_2022,
	title = {On the {Opportunities} and {Risks} of {Foundation} {Models}},
	url = {http://arxiv.org/abs/2108.07258},
	doi = {10.48550/arXiv.2108.07258},
	abstract = {AI is undergoing a paradigm shift with the rise of models (e.g., BERT, DALL-E, GPT-3) that are trained on broad data at scale and are adaptable to a wide range of downstream tasks. We call these models foundation models to underscore their critically central yet incomplete character. This report provides a thorough account of the opportunities and risks of foundation models, ranging from their capabilities (e.g., language, vision, robotics, reasoning, human interaction) and technical principles(e.g., model architectures, training procedures, data, systems, security, evaluation, theory) to their applications (e.g., law, healthcare, education) and societal impact (e.g., inequity, misuse, economic and environmental impact, legal and ethical considerations). Though foundation models are based on standard deep learning and transfer learning, their scale results in new emergent capabilities,and their effectiveness across so many tasks incentivizes homogenization. Homogenization provides powerful leverage but demands caution, as the defects of the foundation model are inherited by all the adapted models downstream. Despite the impending widespread deployment of foundation models, we currently lack a clear understanding of how they work, when they fail, and what they are even capable of due to their emergent properties. To tackle these questions, we believe much of the critical research on foundation models will require deep interdisciplinary collaboration commensurate with their fundamentally sociotechnical nature.},
	urldate = {2024-01-29},
	publisher = {arXiv},
	author = {Bommasani, Rishi and Hudson, Drew A. and Adeli, Ehsan and Altman, Russ and Arora, Simran and von Arx, Sydney and Bernstein, Michael S. and Bohg, Jeannette and Bosselut, Antoine and Brunskill, Emma and Brynjolfsson, Erik and Buch, Shyamal and Card, Dallas and Castellon, Rodrigo and Chatterji, Niladri and Chen, Annie and Creel, Kathleen and Davis, Jared Quincy and Demszky, Dora and Donahue, Chris and Doumbouya, Moussa and Durmus, Esin and Ermon, Stefano and Etchemendy, John and Ethayarajh, Kawin and Fei-Fei, Li and Finn, Chelsea and Gale, Trevor and Gillespie, Lauren and Goel, Karan and Goodman, Noah and Grossman, Shelby and Guha, Neel and Hashimoto, Tatsunori and Henderson, Peter and Hewitt, John and Ho, Daniel E. and Hong, Jenny and Hsu, Kyle and Huang, Jing and Icard, Thomas and Jain, Saahil and Jurafsky, Dan and Kalluri, Pratyusha and Karamcheti, Siddharth and Keeling, Geoff and Khani, Fereshte and Khattab, Omar and Koh, Pang Wei and Krass, Mark and Krishna, Ranjay and Kuditipudi, Rohith and Kumar, Ananya and Ladhak, Faisal and Lee, Mina and Lee, Tony and Leskovec, Jure and Levent, Isabelle and Li, Xiang Lisa and Li, Xuechen and Ma, Tengyu and Malik, Ali and Manning, Christopher D. and Mirchandani, Suvir and Mitchell, Eric and Munyikwa, Zanele and Nair, Suraj and Narayan, Avanika and Narayanan, Deepak and Newman, Ben and Nie, Allen and Niebles, Juan Carlos and Nilforoshan, Hamed and Nyarko, Julian and Ogut, Giray and Orr, Laurel and Papadimitriou, Isabel and Park, Joon Sung and Piech, Chris and Portelance, Eva and Potts, Christopher and Raghunathan, Aditi and Reich, Rob and Ren, Hongyu and Rong, Frieda and Roohani, Yusuf and Ruiz, Camilo and Ryan, Jack and Ré, Christopher and Sadigh, Dorsa and Sagawa, Shiori and Santhanam, Keshav and Shih, Andy and Srinivasan, Krishnan and Tamkin, Alex and Taori, Rohan and Thomas, Armin W. and Tramèr, Florian and Wang, Rose E. and Wang, William and Wu, Bohan and Wu, Jiajun and Wu, Yuhuai and Xie, Sang Michael and Yasunaga, Michihiro and You, Jiaxuan and Zaharia, Matei and Zhang, Michael and Zhang, Tianyi and Zhang, Xikun and Zhang, Yuhui and Zheng, Lucia and Zhou, Kaitlyn and Liang, Percy},
	month = jul,
	year = {2022},
	note = {arXiv:2108.07258 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computers and Society, Computer Science - Machine Learning},
	annote = {Comment: Authored by the Center for Research on Foundation Models (CRFM) at the Stanford Institute for Human-Centered Artificial Intelligence (HAI). Report page with citation guidelines: https://crfm.stanford.edu/report.html},
	file = {arXiv Fulltext PDF:/Users/mis/Zotero/storage/WEMMCFPP/Bommasani et al. - 2022 - On the Opportunities and Risks of Foundation Model.pdf:application/pdf;arXiv.org Snapshot:/Users/mis/Zotero/storage/2JB7BYA6/2108.html:text/html},
}


@article{touvron_llama_2023,
	title = {Llama 2: {Open} {Foundation} and {Fine}-{Tuned} {Chat} {Models}},
	url = {https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/},
	abstract = {In this work, we develop and release Llama 2, a collection of pretrained and fine-tuned large language models (LLMs) ranging in scale from 7 billion to 70 billion parameters. Our fine-tuned LLMs, called Llama 2-Chat, are optimized for dialogue use cases. Our models outperform open-source chat models on most benchmarks we tested, and based on our human evaluations for helpfulness and safety, may be a suitable substitute for closedsource models. We provide a detailed description of our approach to fine-tuning and safety improvements of Llama 2-Chat in order to enable the community to build on our work and contribute to the responsible development of LLMs.},
	language = {en},
	year = {2023},
	author = {Touvron, Hugo and Martin, Louis and Stone, Kevin},
}



@misc{openai_gpt-4_2023,
	title = {{GPT}-4 {Technical} {Report}},
	url = {http://arxiv.org/abs/2303.08774},
	doi = {10.48550/arXiv.2303.08774},
	abstract = {We report the development of GPT-4, a large-scale, multimodal model which can accept image and text inputs and produce text outputs. While less capable than humans in many real-world scenarios, GPT-4 exhibits human-level performance on various professional and academic benchmarks, including passing a simulated bar exam with a score around the top 10\% of test takers. GPT-4 is a Transformer-based model pre-trained to predict the next token in a document. The post-training alignment process results in improved performance on measures of factuality and adherence to desired behavior. A core component of this project was developing infrastructure and optimization methods that behave predictably across a wide range of scales. This allowed us to accurately predict some aspects of GPT-4's performance based on models trained with no more than 1/1,000th the compute of GPT-4.},
	urldate = {2024-01-29},
	publisher = {arXiv},
	author = {OpenAI and Achiam, Josh and Adler, Steven and Agarwal, Sandhini and Ahmad, Lama and Akkaya, Ilge and Aleman, Florencia Leoni and Almeida, Diogo and Altenschmidt, Janko and Altman, Sam and Anadkat, Shyamal and Avila, Red and Babuschkin, Igor and Balaji, Suchir and Balcom, Valerie and Baltescu, Paul and Bao, Haiming and Bavarian, Mo and Belgum, Jeff and Bello, Irwan and Berdine, Jake and Bernadett-Shapiro, Gabriel and Berner, Christopher and Bogdonoff, Lenny and Boiko, Oleg and Boyd, Madelaine and Brakman, Anna-Luisa and Brockman, Greg and Brooks, Tim and Brundage, Miles and Button, Kevin and Cai, Trevor and Campbell, Rosie and Cann, Andrew and Carey, Brittany and Carlson, Chelsea and Carmichael, Rory and Chan, Brooke and Chang, Che and Chantzis, Fotis and Chen, Derek and Chen, Sully and Chen, Ruby and Chen, Jason and Chen, Mark and Chess, Ben and Cho, Chester and Chu, Casey and Chung, Hyung Won and Cummings, Dave and Currier, Jeremiah and Dai, Yunxing and Decareaux, Cory and Degry, Thomas and Deutsch, Noah and Deville, Damien and Dhar, Arka and Dohan, David and Dowling, Steve and Dunning, Sheila and Ecoffet, Adrien and Eleti, Atty and Eloundou, Tyna and Farhi, David and Fedus, Liam and Felix, Niko and Fishman, Simón Posada and Forte, Juston and Fulford, Isabella and Gao, Leo and Georges, Elie and Gibson, Christian and Goel, Vik and Gogineni, Tarun and Goh, Gabriel and Gontijo-Lopes, Rapha and Gordon, Jonathan and Grafstein, Morgan and Gray, Scott and Greene, Ryan and Gross, Joshua and Gu, Shixiang Shane and Guo, Yufei and Hallacy, Chris and Han, Jesse and Harris, Jeff and He, Yuchen and Heaton, Mike and Heidecke, Johannes and Hesse, Chris and Hickey, Alan and Hickey, Wade and Hoeschele, Peter and Houghton, Brandon and Hsu, Kenny and Hu, Shengli and Hu, Xin and Huizinga, Joost and Jain, Shantanu and Jain, Shawn and Jang, Joanne and Jiang, Angela and Jiang, Roger and Jin, Haozhun and Jin, Denny and Jomoto, Shino and Jonn, Billie and Jun, Heewoo and Kaftan, Tomer and Kaiser, Łukasz and Kamali, Ali and Kanitscheider, Ingmar and Keskar, Nitish Shirish and Khan, Tabarak and Kilpatrick, Logan and Kim, Jong Wook and Kim, Christina and Kim, Yongjik and Kirchner, Hendrik and Kiros, Jamie and Knight, Matt and Kokotajlo, Daniel and Kondraciuk, Łukasz and Kondrich, Andrew and Konstantinidis, Aris and Kosic, Kyle and Krueger, Gretchen and Kuo, Vishal and Lampe, Michael and Lan, Ikai and Lee, Teddy and Leike, Jan and Leung, Jade and Levy, Daniel and Li, Chak Ming and Lim, Rachel and Lin, Molly and Lin, Stephanie and Litwin, Mateusz and Lopez, Theresa and Lowe, Ryan and Lue, Patricia and Makanju, Anna and Malfacini, Kim and Manning, Sam and Markov, Todor and Markovski, Yaniv and Martin, Bianca and Mayer, Katie and Mayne, Andrew and McGrew, Bob and McKinney, Scott Mayer and McLeavey, Christine and McMillan, Paul and McNeil, Jake and Medina, David and Mehta, Aalok and Menick, Jacob and Metz, Luke and Mishchenko, Andrey and Mishkin, Pamela and Monaco, Vinnie and Morikawa, Evan and Mossing, Daniel and Mu, Tong and Murati, Mira and Murk, Oleg and Mély, David and Nair, Ashvin and Nakano, Reiichiro and Nayak, Rajeev and Neelakantan, Arvind and Ngo, Richard and Noh, Hyeonwoo and Ouyang, Long and O'Keefe, Cullen and Pachocki, Jakub and Paino, Alex and Palermo, Joe and Pantuliano, Ashley and Parascandolo, Giambattista and Parish, Joel and Parparita, Emy and Passos, Alex and Pavlov, Mikhail and Peng, Andrew and Perelman, Adam and Peres, Filipe de Avila Belbute and Petrov, Michael and Pinto, Henrique Ponde de Oliveira and Michael and Pokorny and Pokrass, Michelle and Pong, Vitchyr and Powell, Tolly and Power, Alethea and Power, Boris and Proehl, Elizabeth and Puri, Raul and Radford, Alec and Rae, Jack and Ramesh, Aditya and Raymond, Cameron and Real, Francis and Rimbach, Kendra and Ross, Carl and Rotsted, Bob and Roussez, Henri and Ryder, Nick and Saltarelli, Mario and Sanders, Ted and Santurkar, Shibani and Sastry, Girish and Schmidt, Heather and Schnurr, David and Schulman, John and Selsam, Daniel and Sheppard, Kyla and Sherbakov, Toki and Shieh, Jessica and Shoker, Sarah and Shyam, Pranav and Sidor, Szymon and Sigler, Eric and Simens, Maddie and Sitkin, Jordan and Slama, Katarina and Sohl, Ian and Sokolowsky, Benjamin and Song, Yang and Staudacher, Natalie and Such, Felipe Petroski and Summers, Natalie and Sutskever, Ilya and Tang, Jie and Tezak, Nikolas and Thompson, Madeleine and Tillet, Phil and Tootoonchian, Amin and Tseng, Elizabeth and Tuggle, Preston and Turley, Nick and Tworek, Jerry and Uribe, Juan Felipe Cerón and Vallone, Andrea and Vijayvergiya, Arun and Voss, Chelsea and Wainwright, Carroll and Wang, Justin Jay and Wang, Alvin and Wang, Ben and Ward, Jonathan and Wei, Jason and Weinmann, C. J. and Welihinda, Akila and Welinder, Peter and Weng, Jiayi and Weng, Lilian and Wiethoff, Matt and Willner, Dave and Winter, Clemens and Wolrich, Samuel and Wong, Hannah and Workman, Lauren and Wu, Sherwin and Wu, Jeff and Wu, Michael and Xiao, Kai and Xu, Tao and Yoo, Sarah and Yu, Kevin and Yuan, Qiming and Zaremba, Wojciech and Zellers, Rowan and Zhang, Chong and Zhang, Marvin and Zhao, Shengjia and Zheng, Tianhao and Zhuang, Juntang and Zhuk, William and Zoph, Barret},
	month = dec,
	year = {2023},
	note = {arXiv:2303.08774 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
	annote = {Comment: 100 pages; updated authors list},
}


@misc{chowdhery_palm_2022,
	title = {{PaLM}: {Scaling} {Language} {Modeling} with {Pathways}},
	shorttitle = {{PaLM}},
	url = {http://arxiv.org/abs/2204.02311},
	doi = {10.48550/arXiv.2204.02311},
	abstract = {Large language models have been shown to achieve remarkable performance across a variety of natural language tasks using few-shot learning, which drastically reduces the number of task-specific training examples needed to adapt the model to a particular application. To further our understanding of the impact of scale on few-shot learning, we trained a 540-billion parameter, densely activated, Transformer language model, which we call Pathways Language Model PaLM. We trained PaLM on 6144 TPU v4 chips using Pathways, a new ML system which enables highly efficient training across multiple TPU Pods. We demonstrate continued benefits of scaling by achieving state-of-the-art few-shot learning results on hundreds of language understanding and generation benchmarks. On a number of these tasks, PaLM 540B achieves breakthrough performance, outperforming the finetuned state-of-the-art on a suite of multi-step reasoning tasks, and outperforming average human performance on the recently released BIG-bench benchmark. A significant number of BIG-bench tasks showed discontinuous improvements from model scale, meaning that performance steeply increased as we scaled to our largest model. PaLM also has strong capabilities in multilingual tasks and source code generation, which we demonstrate on a wide array of benchmarks. We additionally provide a comprehensive analysis on bias and toxicity, and study the extent of training data memorization with respect to model scale. Finally, we discuss the ethical considerations related to large language models and discuss potential mitigation strategies.},
	urldate = {2024-01-29},
	publisher = {arXiv},
	author = {Chowdhery, Aakanksha and Narang, Sharan and Devlin, Jacob and Bosma, Maarten and Mishra, Gaurav and Roberts, Adam and Barham, Paul and Chung, Hyung Won and Sutton, Charles and Gehrmann, Sebastian and Schuh, Parker and Shi, Kensen and Tsvyashchenko, Sasha and Maynez, Joshua and Rao, Abhishek and Barnes, Parker and Tay, Yi and Shazeer, Noam and Prabhakaran, Vinodkumar and Reif, Emily and Du, Nan and Hutchinson, Ben and Pope, Reiner and Bradbury, James and Austin, Jacob and Isard, Michael and Gur-Ari, Guy and Yin, Pengcheng and Duke, Toju and Levskaya, Anselm and Ghemawat, Sanjay and Dev, Sunipa and Michalewski, Henryk and Garcia, Xavier and Misra, Vedant and Robinson, Kevin and Fedus, Liam and Zhou, Denny and Ippolito, Daphne and Luan, David and Lim, Hyeontaek and Zoph, Barret and Spiridonov, Alexander and Sepassi, Ryan and Dohan, David and Agrawal, Shivani and Omernick, Mark and Dai, Andrew M. and Pillai, Thanumalayan Sankaranarayana and Pellat, Marie and Lewkowycz, Aitor and Moreira, Erica and Child, Rewon and Polozov, Oleksandr and Lee, Katherine and Zhou, Zongwei and Wang, Xuezhi and Saeta, Brennan and Diaz, Mark and Firat, Orhan and Catasta, Michele and Wei, Jason and Meier-Hellstern, Kathy and Eck, Douglas and Dean, Jeff and Petrov, Slav and Fiedel, Noah},
	month = oct,
	year = {2022},
	note = {arXiv:2204.02311 [cs]},
	keywords = {Computer Science - Computation and Language},
}


@misc{jiang_mistral_2023,
	title = {Mistral {7B}},
	url = {http://arxiv.org/abs/2310.06825},
	doi = {10.48550/arXiv.2310.06825},
	abstract = {We introduce Mistral 7B v0.1, a 7-billion-parameter language model engineered for superior performance and efficiency. Mistral 7B outperforms Llama 2 13B across all evaluated benchmarks, and Llama 1 34B in reasoning, mathematics, and code generation. Our model leverages grouped-query attention (GQA) for faster inference, coupled with sliding window attention (SWA) to effectively handle sequences of arbitrary length with a reduced inference cost. We also provide a model fine-tuned to follow instructions, Mistral 7B -- Instruct, that surpasses the Llama 2 13B -- Chat model both on human and automated benchmarks. Our models are released under the Apache 2.0 license.},
	urldate = {2024-01-29},
	publisher = {arXiv},
	author = {Jiang, Albert Q. and Sablayrolles, Alexandre and Mensch, Arthur and Bamford, Chris and Chaplot, Devendra Singh and Casas, Diego de las and Bressand, Florian and Lengyel, Gianna and Lample, Guillaume and Saulnier, Lucile and Lavaud, Lélio Renard and Lachaux, Marie-Anne and Stock, Pierre and Scao, Teven Le and Lavril, Thibaut and Wang, Thomas and Lacroix, Timothée and Sayed, William El},
	month = oct,
	year = {2023},
	note = {arXiv:2310.06825 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Machine Learning},
	annote = {Comment: Models and code are available at https://mistral.ai/news/announcing-mistral-7b/},
}



@article{wei_emergent_2022,
	title = {Emergent {Abilities} of {Large} {Language} {Models}},
	issn = {2835-8856},
	url = {https://openreview.net/forum?id=yzkSU5zdwD},
	abstract = {Scaling up language models has been shown to predictably improve performance and sample efficiency on a wide range of downstream tasks. This paper instead discusses an unpredictable phenomenon that we refer to as emergent abilities of large language models. We consider an ability to be emergent if it is not present in smaller models but is present in larger models. Thus, emergent abilities cannot be predicted simply by extrapolating the performance of smaller models. The existence of such emergence raises the question of whether additional scaling could potentially further expand the range of capabilities of language models.},
	language = {en},
	urldate = {2024-01-29},
	journal = {Transactions on Machine Learning Research},
	author = {Wei, Jason and Tay, Yi and Bommasani, Rishi and Raffel, Colin and Zoph, Barret and Borgeaud, Sebastian and Yogatama, Dani and Bosma, Maarten and Zhou, Denny and Metzler, Donald and Chi, Ed H. and Hashimoto, Tatsunori and Vinyals, Oriol and Liang, Percy and Dean, Jeff and Fedus, William},
	month = jun,
	year = {2022},
}


@inproceedings{min_rethinking_2022,
	title = {Rethinking the {Role} of {Demonstrations}: {What} {Makes} {In}-{Context} {Learning} {Work}?},
	shorttitle = {Rethinking the {Role} of {Demonstrations}},
	url = {https://aclanthology.org/2022.emnlp-main.759},
	doi = {10.18653/v1/2022.emnlp-main.759},
	abstract = {Large language models (LMs) are able to in-context learn—perform a new task via inference alone by conditioning on a few input-label pairs (demonstrations) and making predictions for new inputs. However, there has been little understanding of how the model learns and which aspects of the demonstrations contribute to end task performance. In this paper, we show that ground truth demonstrations are in fact not required—randomly replacing labels in the demonstrations barely hurts performance on a range of classification and multi-choce tasks, consistently over 12 different models including GPT-3. Instead, we find that other aspects of the demonstrations are the key drivers of endtask performance, including the fact that they provide a few examples of (1) the label space, (2) the distribution of the input text, and (3) the overall format of the sequence. Together, our analysis provides a new way of understanding how and why in-context learning works, while opening up new questions about how much can be learned from large language models through inference alone.},
	urldate = {2024-01-29},
	booktitle = {Proceedings of the 2022 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing}},
	publisher = {Association for Computational Linguistics},
	author = {Min, Sewon and Lyu, Xinxi and Holtzman, Ari and Artetxe, Mikel and Lewis, Mike and Hajishirzi, Hannaneh and Zettlemoyer, Luke},
	editor = {Goldberg, Yoav and Kozareva, Zornitsa and Zhang, Yue},
	month = dec,
	year = {2022},
	pages = {11048--11064},

}

@article{campbell_deep_2002,
	title = {Deep {Blue}},
	volume = {134},
	issn = {0004-3702},
	url = {https://doi.org/10.1016/S0004-3702(01)00129-1},
	doi = {10.1016/S0004-3702(01)00129-1},
	abstract = {Deep Blue is the chess machine that defeated then-reigning World Chess Champion Garry Kasparov in a six-game match in 1997. There were a number of factors that contributed to this success, including: a single-chip chess search engine, a massively parallel system with multiple levels of parallelism, a strong emphasis on search extensions, a complex evaluation function, and effective use of a Grandmaster game database. This paper describes the Deep Blue system, and gives some of the rationale that went into the design decision behind Deep Blue. Copyright 2001 Elsevier B.V.},
	number = {1-2},
	urldate = {2024-01-29},
	journal = {Artificial Intelligence},
	author = {Campbell, Murray and Hoane, A. Joseph and Hsu, Feng-hsiung},
	month = jan,
	year = {2002},
	keywords = {computer chess, evaluation function, game tree search, parallel search, search extensions, selective search},
	pages = {57--83},
}


@article{silver_mastering_2016,
	title = {Mastering the game of {Go} with deep neural networks and tree search},
	volume = {529},
	copyright = {2016 Springer Nature Limited},
	issn = {1476-4687},
	url = {https://www.nature.com/articles/nature16961},
	doi = {10.1038/nature16961},
	abstract = {The game of Go has long been viewed as the most challenging of classic games for artificial intelligence owing to its enormous search space and the difficulty of evaluating board positions and moves. Here we introduce a new approach to computer Go that uses ‘value networks’ to evaluate board positions and ‘policy networks’ to select moves. These deep neural networks are trained by a novel combination of supervised learning from human expert games, and reinforcement learning from games of self-play. Without any lookahead search, the neural networks play Go at the level of state-of-the-art Monte Carlo tree search programs that simulate thousands of random games of self-play. We also introduce a new search algorithm that combines Monte Carlo simulation with value and policy networks. Using this search algorithm, our program AlphaGo achieved a 99.8\% winning rate against other Go programs, and defeated the human European Go champion by 5 games to 0. This is the first time that a computer program has defeated a human professional player in the full-sized game of Go, a feat previously thought to be at least a decade away.},
	language = {en},
	number = {7587},
	urldate = {2024-01-29},
	journal = {Nature},
	author = {Silver, David and Huang, Aja and Maddison, Chris J. and Guez, Arthur and Sifre, Laurent and van den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and Dieleman, Sander and Grewe, Dominik and Nham, John and Kalchbrenner, Nal and Sutskever, Ilya and Lillicrap, Timothy and Leach, Madeleine and Kavukcuoglu, Koray and Graepel, Thore and Hassabis, Demis},
	month = jan,
	year = {2016},
	note = {Number: 7587
Publisher: Nature Publishing Group},
	keywords = {Computational science, Computer science, Reward},
	pages = {484--489},
}


@article{ferrucci_introduction_2012,
	title = {Introduction to "{This} is {Watson}"},
	volume = {56},
	issn = {0018-8646},
	url = {https://doi.org/10.1147/JRD.2012.2184356},
	doi = {10.1147/JRD.2012.2184356},
	abstract = {In 2007, IBM Research took on the grand challenge of building a computer system that could compete with champions at the game of Jeopardy!™. In 2011, the open-domain question-answering (QA) system, dubbed Watson, beat the two highest ranked players in a nationally televised two-game Jeopardy! match. This paper provides a brief history of the events and ideas that positioned our team to take on the Jeopardy! challenge, build Watson, IBM Watson™, and ultimately triumph. It describes both the nature of the QA challenge represented by Jeopardy! and our overarching technical approach. The main body of this paper provides a narrative of the DeepQA processing pipeline to introduce the articles in this special issue and put them in context of the overall system. Finally, this paper summarizes our main results, describing how the system, as a holistic combination of many diverse algorithmic techniques, performed at champion levels, and it briefly discusses the team's future research plans.},
	number = {3},
	urldate = {2024-01-29},
	journal = {IBM Journal of Research and Development},
	author = {Ferrucci, D. A.},
	month = may,
	year = {2012},
	pages = {235--249},
}


@misc{openai_dota_2019,
	title = {Dota 2 with {Large} {Scale} {Deep} {Reinforcement} {Learning}},
	url = {http://arxiv.org/abs/1912.06680},
	doi = {10.48550/arXiv.1912.06680},
	abstract = {On April 13th, 2019, OpenAI Five became the first AI system to defeat the world champions at an esports game. The game of Dota 2 presents novel challenges for AI systems such as long time horizons, imperfect information, and complex, continuous state-action spaces, all challenges which will become increasingly central to more capable AI systems. OpenAI Five leveraged existing reinforcement learning techniques, scaled to learn from batches of approximately 2 million frames every 2 seconds. We developed a distributed training system and tools for continual training which allowed us to train OpenAI Five for 10 months. By defeating the Dota 2 world champion (Team OG), OpenAI Five demonstrates that self-play reinforcement learning can achieve superhuman performance on a difficult task.},
	urldate = {2024-01-29},
	publisher = {arXiv},
	author = {OpenAI and Berner, Christopher and Brockman, Greg and Chan, Brooke and Cheung, Vicki and Dębiak, Przemysław and Dennison, Christy and Farhi, David and Fischer, Quirin and Hashme, Shariq and Hesse, Chris and Józefowicz, Rafal and Gray, Scott and Olsson, Catherine and Pachocki, Jakub and Petrov, Michael and Pinto, Henrique P. d O. and Raiman, Jonathan and Salimans, Tim and Schlatter, Jeremy and Schneider, Jonas and Sidor, Szymon and Sutskever, Ilya and Tang, Jie and Wolski, Filip and Zhang, Susan},
	month = dec,
	year = {2019},
	note = {arXiv:1912.06680 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}


@article{liang_holistic_2023,
	title = {Holistic {Evaluation} of {Language} {Models}},
	issn = {2835-8856},
	url = {https://openreview.net/forum?id=iO4LZibEqW},
	abstract = {Language models (LMs) are becoming the foundation for almost all major language technologies, but their capabilities, limitations, and risks are not well understood. We present Holistic Evaluation of Language Models (HELM) to improve the transparency of language models. First, we taxonomize the vast space of potential scenarios (i.e. use cases) and metrics (i.e. desiderata) that are of interest for LMs. Then we select a broad subset based on coverage and feasibility, noting what’s missing or underrepresented (e.g. question answering for neglected English dialects, metrics for trustworthiness). Second, we adopt a multi-metric approach: We measure 7 metrics (accuracy, calibration, robustness, fairness, bias, toxicity, and efficiency) for each of 16 core scenarios to the extent possible (87.5\% of the time), ensuring that metrics beyond accuracy don’t fall to the wayside, and that trade-offs across models and metrics are clearly exposed. We also perform 7 targeted evaluations, based on 26 targeted scenarios, to more deeply analyze specific aspects (e.g. knowledge, reasoning, memorization/copyright, disinformation). Third, we conduct a large-scale evaluation of 30 prominent language models (spanning open, limited-access, and closed models) on all 42 scenarios, including 21 scenarios that were not previously used in mainstream LM evaluation. Prior to HELM, models on average were evaluated on just 17.9\% of the core HELM scenarios, with some prominent models not sharing a single scenario in common. We improve this to 96.0\%: now all 30 models have been densely benchmarked on a set of core scenarios and metrics under standardized conditions. Our evaluation surfaces 25 top-level findings concerning the interplay between different scenarios, metrics, and models. For full transparency, we release all raw model prompts and completions publicly for further analysis, as well as a general modular toolkit for easily adding new scenarios, models, metrics, and prompting strategies. We intend for HELM to be a living benchmark for the community, continuously updated with new scenarios, metrics, and models.},
	language = {en},
	urldate = {2024-01-29},
	journal = {Transactions on Machine Learning Research},
	author = {Liang, Percy and Bommasani, Rishi and Lee, Tony and Tsipras, Dimitris and Soylu, Dilara and Yasunaga, Michihiro and Zhang, Yian and Narayanan, Deepak and Wu, Yuhuai and Kumar, Ananya and Newman, Benjamin and Yuan, Binhang and Yan, Bobby and Zhang, Ce and Cosgrove, Christian Alexander and Manning, Christopher D. and Re, Christopher and Acosta-Navas, Diana and Hudson, Drew Arad and Zelikman, Eric and Durmus, Esin and Ladhak, Faisal and Rong, Frieda and Ren, Hongyu and Yao, Huaxiu and Wang, Jue and Santhanam, Keshav and Orr, Laurel and Zheng, Lucia and Yuksekgonul, Mert and Suzgun, Mirac and Kim, Nathan and Guha, Neel and Chatterji, Niladri S. and Khattab, Omar and Henderson, Peter and Huang, Qian and Chi, Ryan Andrew and Xie, Sang Michael and Santurkar, Shibani and Ganguli, Surya and Hashimoto, Tatsunori and Icard, Thomas and Zhang, Tianyi and Chaudhary, Vishrav and Wang, William and Li, Xuechen and Mai, Yifan and Zhang, Yuhui and Koreeda, Yuta},
	month = feb,
	year = {2023},
}


@article{koivisto_best_2023,
	title = {Best humans still outperform artificial intelligence in a creative divergent thinking task},
	volume = {13},
	copyright = {2023 The Author(s)},
	issn = {2045-2322},
	url = {https://www.nature.com/articles/s41598-023-40858-3},
	doi = {10.1038/s41598-023-40858-3},
	abstract = {Creativity has traditionally been considered an ability exclusive to human beings. However, the rapid development of artificial intelligence (AI) has resulted in generative AI chatbots that can produce high-quality artworks, raising questions about the differences between human and machine creativity. In this study, we compared the creativity of humans (n = 256) with that of three current AI chatbots using the alternate uses task (AUT), which is the most used divergent thinking task. Participants were asked to generate uncommon and creative uses for everyday objects. On average, the AI chatbots outperformed human participants. While human responses included poor-quality ideas, the chatbots generally produced more creative responses. However, the best human ideas still matched or exceed those of the chatbots. While this study highlights the potential of AI as a tool to enhance creativity, it also underscores the unique and complex nature of human creativity that may be difficult to fully replicate or surpass with AI technology. The study provides insights into the relationship between human and machine creativity, which is related to important questions about the future of creative work in the age of AI.},
	language = {en},
	number = {1},
	urldate = {2024-01-29},
	journal = {Scientific Reports},
	author = {Koivisto, Mika and Grassini, Simone},
	month = sep,
	year = {2023},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {Mathematics and computing, Psychology},
	pages = {13601},
	file = {Full Text PDF:/Users/mis/Zotero/storage/2YAB6GTB/Koivisto and Grassini - 2023 - Best humans still outperform artificial intelligen.pdf:application/pdf},
}


@inproceedings{suzgun_challenging_2023,
	address = {Toronto, Canada},
	title = {Challenging {BIG}-{Bench} {Tasks} and {Whether} {Chain}-of-{Thought} {Can} {Solve} {Them}},
	url = {https://aclanthology.org/2023.findings-acl.824},
	doi = {10.18653/v1/2023.findings-acl.824},
	abstract = {BIG-Bench (Srivastava et al., 2022) is a diverse evaluation suite that focuses on tasks believed to be beyond the capabilities of current language models. Language models have already made good progress on this benchmark, with the best model in the BIG-Bench paper outperforming average reported human-rater results on 65\% of the BIG-Bench tasks via few-shot prompting. But on what tasks do language models fall short of average human-rater performance, and are those tasks actually unsolvable by current language models? In this work, we focus on a suite of 23 challenging BIG-Bench tasks which we call BIG-Bench Hard (BBH). These are the tasks for which prior language model evaluations did not outperform the average human-rater. We find that applying chain-of-thought (CoT) prompting to BBH tasks enables PaLM to surpass the average human-rater performance on 10 of the 23 tasks, and Codex (code-davinci-002) to surpass the average human-rater performance on 17 of the 23 tasks. Since many tasks in BBH require multi-step reasoning, few-shot prompting without CoT, as done in the BIG-Bench evaluations (Srivastava et al., 2022), substantially underestimates the best performance and capabilities of language models, which is better captured via CoT prompting. As further analysis, we explore the interaction between CoT and model scale on BBH, finding that CoT enables emergent task performance on several BBH tasks with otherwise flat scaling curves.},
	urldate = {2024-01-29},
	booktitle = {Findings of the {Association} for {Computational} {Linguistics}: {ACL} 2023},
	publisher = {Association for Computational Linguistics},
	author = {Suzgun, Mirac and Scales, Nathan and Schärli, Nathanael and Gehrmann, Sebastian and Tay, Yi and Chung, Hyung Won and Chowdhery, Aakanksha and Le, Quoc and Chi, Ed and Zhou, Denny and Wei, Jason},
	editor = {Rogers, Anna and Boyd-Graber, Jordan and Okazaki, Naoaki},
	month = jul,
	year = {2023},
	pages = {13003--13051},
}


@inproceedings{kadra_well-tuned_2021,
	title = {Well-tuned {Simple} {Nets} {Excel} on {Tabular} {Datasets}},
	url = {https://openreview.net/forum?id=d3k38LTDCyO},
	abstract = {Tabular datasets are the last "unconquered castle" for deep learning, with traditional ML methods like Gradient-Boosted Decision Trees still performing strongly even against recent specialized neural architectures. In this paper, we hypothesize that the key to boosting the performance of neural networks lies in rethinking the joint and simultaneous application of a large set of modern regularization techniques. As a result, we propose regularizing plain Multilayer Perceptron (MLP) networks by searching for the optimal combination/cocktail of 13 regularization techniques for each dataset using a joint optimization over the decision on which regularizers to apply and their subsidiary hyperparameters. We empirically assess the impact of these regularization cocktails for MLPs in a large-scale empirical study comprising 40 tabular datasets and demonstrate that (i) well-regularized plain MLPs significantly outperform recent state-of-the-art specialized neural network architectures, and (ii) they even outperform strong traditional ML methods, such as XGBoost.},
	language = {en},
	urldate = {2024-01-29},
	author = {Kadra, Arlind and Lindauer, Marius and Hutter, Frank and Grabocka, Josif},
	month = nov,
	year = {2021},
}


@article{borisov_deep_2022,
	title = {Deep {Neural} {Networks} and {Tabular} {Data}: {A} {Survey}},
	issn = {2162-237X, 2162-2388},
	shorttitle = {Deep {Neural} {Networks} and {Tabular} {Data}},
	url = {https://ieeexplore.ieee.org/document/9998482/},
	doi = {10.1109/TNNLS.2022.3229161},
	abstract = {Heterogeneous tabular data are the most commonly used form of data and are essential for numerous critical and computationally demanding applications. On homogeneous datasets, deep neural networks have repeatedly shown excellent performance and have therefore been widely adopted. However, their adaptation to tabular data for inference or data generation tasks remains highly challenging. To facilitate further progress in the ﬁeld, this work provides an overview of state-of-the-art deep learning methods for tabular data. We categorize these methods into three groups: data transformations, specialized architectures, and regularization models. For each of these groups, our work offers a comprehensive overview of the main approaches. Moreover, we discuss deep learning approaches for generating tabular data and also provide an overview over strategies for explaining deep models on tabular data. Thus, our ﬁrst contribution is to address the main research streams and existing methodologies in the mentioned areas while highlighting relevant challenges and open research questions. Our second contribution is to provide an empirical comparison of traditional machine learning methods with 11 deep learning approaches across ﬁve popular real-world tabular datasets of different sizes and with different learning objectives. Our results, which we have made publicly available as competitive benchmarks, indicate that algorithms based on gradient-boosted tree ensembles still mostly outperform deep learning models on supervised learning tasks, suggesting that the research progress on competitive deep learning models for tabular data is stagnating. To the best of our knowledge, this is the ﬁrst in-depth overview of deep learning approaches for tabular data; as such, this work can serve as a valuable starting point to guide researchers and practitioners interested in deep learning with tabular data.},
	language = {en},
	urldate = {2024-01-29},
	journal = {IEEE Transactions on Neural Networks and Learning Systems},
	author = {Borisov, Vadim and Leemann, Tobias and Sessler, Kathrin and Haug, Johannes and Pawelczyk, Martin and Kasneci, Gjergji},
	year = {2022},
	pages = {1--21},

}



@misc{liu_rethinking_2023,
	title = {Rethinking {Tabular} {Data} {Understanding} with {Large} {Language} {Models}},
	url = {http://arxiv.org/abs/2312.16702},
	doi = {10.48550/arXiv.2312.16702},
	abstract = {Large Language Models (LLMs) have shown to be capable of various tasks, yet their capability in interpreting and reasoning over tabular data remains an underexplored area. In this context, this study investigates from three core perspectives: the robustness of LLMs to structural perturbations in tables, the comparative analysis of textual and symbolic reasoning on tables, and the potential of boosting model performance through the aggregation of multiple reasoning pathways. We discover that structural variance of tables presenting the same content reveals a notable performance decline, particularly in symbolic reasoning tasks. This prompts the proposal of a method for table structure normalization. Moreover, textual reasoning slightly edges out symbolic reasoning, and a detailed error analysis reveals that each exhibits different strengths depending on the specific tasks. Notably, the aggregation of textual and symbolic reasoning pathways, bolstered by a mix self-consistency mechanism, resulted in achieving SOTA performance, with an accuracy of 73.6\% on WIKITABLEQUESTIONS, representing a substantial advancement over previous existing table processing paradigms of LLMs.},
	urldate = {2024-01-29},
	publisher = {arXiv},
	author = {Liu, Tianyang and Wang, Fei and Chen, Muhao},
	month = dec,
	year = {2023},
	note = {arXiv:2312.16702 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Databases, Computer Science - Machine Learning},
}


@article{zhu_converting_2021,
	title = {Converting tabular data into images for deep learning with convolutional neural networks},
	volume = {11},
	copyright = {2021 This is a U.S. Government work and not under copyright protection in the US; foreign copyright protection may apply},
	issn = {2045-2322},
	url = {https://www.nature.com/articles/s41598-021-90923-y},
	doi = {10.1038/s41598-021-90923-y},
	abstract = {Convolutional neural networks (CNNs) have been successfully used in many applications where important information about data is embedded in the order of features, such as speech and imaging. However, most tabular data do not assume a spatial relationship between features, and thus are unsuitable for modeling using CNNs. To meet this challenge, we develop a novel algorithm, image generator for tabular data (IGTD), to transform tabular data into images by assigning features to pixel positions so that similar features are close to each other in the image. The algorithm searches for an optimized assignment by minimizing the difference between the ranking of distances between features and the ranking of distances between their assigned pixels in the image. We apply IGTD to transform gene expression profiles of cancer cell lines (CCLs) and molecular descriptors of drugs into their respective image representations. Compared with existing transformation methods, IGTD generates compact image representations with better preservation of feature neighborhood structure. Evaluated on benchmark drug screening datasets, CNNs trained on IGTD image representations of CCLs and drugs exhibit a better performance of predicting anti-cancer drug response than both CNNs trained on alternative image representations and prediction models trained on the original tabular data.},
	language = {en},
	number = {1},
	urldate = {2024-01-29},
	journal = {Scientific Reports},
	author = {Zhu, Yitan and Brettin, Thomas and Xia, Fangfang and Partin, Alexander and Shukla, Maulik and Yoo, Hyunseung and Evrard, Yvonne A. and Doroshow, James H. and Stevens, Rick L.},
	month = may,
	year = {2021},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {Computational models, Machine learning, Virtual drug screening},
	pages = {11325},
}

@inproceedings{fitkov-norris_evaluating_2012,
	address = {Berlin, Heidelberg},
	series = {Communications in {Computer} and {Information} {Science}},
	title = {Evaluating the {Impact} of {Categorical} {Data} {Encoding} and {Scaling} on {Neural} {Network} {Classification} {Performance}: {The} {Case} of {Repeat} {Consumption} of {Identical} {Cultural} {Goods}},
	isbn = {978-3-642-32909-8},
	shorttitle = {Evaluating the {Impact} of {Categorical} {Data} {Encoding} and {Scaling} on {Neural} {Network} {Classification} {Performance}},
	doi = {10.1007/978-3-642-32909-8_35},
	abstract = {This article investigated the impact of categorical input encoding and scaling approaches on neural network sensitivity and overall classification performance in the context of predicting the repeat viewing propensity of movie goers. The results show that neural network out of sample minimum sensitivity and overall classification performance are indifferent to the scaling of the categorical inputs. However, the encoding of inputs had a significant impact on classification accuracy and utilising ordinal or thermometer encoding approaches for categorical inputs significantly increases the out of sample accuracy of the neural network classifier. These findings confirm that the impact of categorical encoding is problem specific for an ordinal approach, and support thermometer encoding as most suitable for categorical inputs. The classification performance of neural networks was compared against a logistic regression model and the results show that in this instance, the non-parametric approach does not offer any advantage over standard statistical models.},
	language = {en},
	booktitle = {Engineering {Applications} of {Neural} {Networks}},
	publisher = {Springer},
	author = {Fitkov-Norris, Elena and Vahid, Samireh and Hand, Chris},
	editor = {Jayne, Chrisina and Yue, Shigang and Iliadis, Lazaros},
	year = {2012},
	keywords = {categorical input, encoding, logistic regression, neural networks, scaling},
	pages = {343--352},
}


@misc{smock_pubtables-1m_2021,
	title = {{PubTables}-{1M}: {Towards} comprehensive table extraction from unstructured documents},
	shorttitle = {{PubTables}-{1M}},
	url = {http://arxiv.org/abs/2110.00061},
	doi = {10.48550/arXiv.2110.00061},
	abstract = {Recently, significant progress has been made applying machine learning to the problem of table structure inference and extraction from unstructured documents. However, one of the greatest challenges remains the creation of datasets with complete, unambiguous ground truth at scale. To address this, we develop a new, more comprehensive dataset for table extraction, called PubTables-1M. PubTables-1M contains nearly one million tables from scientific articles, supports multiple input modalities, and contains detailed header and location information for table structures, making it useful for a wide variety of modeling approaches. It also addresses a significant source of ground truth inconsistency observed in prior datasets called oversegmentation, using a novel canonicalization procedure. We demonstrate that these improvements lead to a significant increase in training performance and a more reliable estimate of model performance at evaluation for table structure recognition. Further, we show that transformer-based object detection models trained on PubTables-1M produce excellent results for all three tasks of detection, structure recognition, and functional analysis without the need for any special customization for these tasks. Data and code will be released at https://github.com/microsoft/table-transformer.},
	urldate = {2023-12-04},
	publisher = {arXiv},
	author = {Smock, Brandon and Pesala, Rohith and Abraham, Robin},
	month = nov,
	year = {2021},
	note = {arXiv:2110.00061 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Computer Vision and Pattern Recognition},
}


@misc{zhong_image-based_2020,
	title = {Image-based table recognition: data, model, and evaluation},
	shorttitle = {Image-based table recognition},
	url = {http://arxiv.org/abs/1911.10683},
	doi = {10.48550/arXiv.1911.10683},
	abstract = {Important information that relates to a specific topic in a document is often organized in tabular format to assist readers with information retrieval and comparison, which may be difficult to provide in natural language. However, tabular data in unstructured digital documents, e.g., Portable Document Format (PDF) and images, are difficult to parse into structured machine-readable format, due to complexity and diversity in their structure and style. To facilitate image-based table recognition with deep learning, we develop the largest publicly available table recognition dataset PubTabNet (https://github.com/ibm-aur-nlp/PubTabNet), containing 568k table images with corresponding structured HTML representation. PubTabNet is automatically generated by matching the XML and PDF representations of the scientific articles in PubMed Central Open Access Subset (PMCOA). We also propose a novel attention-based encoder-dual-decoder (EDD) architecture that converts images of tables into HTML code. The model has a structure decoder which reconstructs the table structure and helps the cell decoder to recognize cell content. In addition, we propose a new Tree-Edit-Distance-based Similarity (TEDS) metric for table recognition, which more appropriately captures multi-hop cell misalignment and OCR errors than the pre-established metric. The experiments demonstrate that the EDD model can accurately recognize complex tables solely relying on the image representation, outperforming the state-of-the-art by 9.7\% absolute TEDS score.},
	urldate = {2024-01-30},
	publisher = {arXiv},
	author = {Zhong, Xu and ShafieiBavani, Elaheh and Yepes, Antonio Jimeno},
	month = mar,
	year = {2020},
	note = {arXiv:1911.10683 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}


@misc{zheng_global_2020,
	title = {Global {Table} {Extractor} ({GTE}): {A} {Framework} for {Joint} {Table} {Identification} and {Cell} {Structure} {Recognition} {Using} {Visual} {Context}},
	shorttitle = {Global {Table} {Extractor} ({GTE})},
	url = {http://arxiv.org/abs/2005.00589},
	doi = {10.48550/arXiv.2005.00589},
	abstract = {Documents are often used for knowledge sharing and preservation in business and science, within which are tables that capture most of the critical data. Unfortunately, most documents are stored and distributed as PDF or scanned images, which fail to preserve logical table structure. Recent vision-based deep learning approaches have been proposed to address this gap, but most still cannot achieve state-of-the-art results. We present Global Table Extractor (GTE), a vision-guided systematic framework for joint table detection and cell structured recognition, which could be built on top of any object detection model. With GTE-Table, we invent a new penalty based on the natural cell containment constraint of tables to train our table network aided by cell location predictions. GTE-Cell is a new hierarchical cell detection network that leverages table styles. Further, we design a method to automatically label table and cell structure in existing documents to cheaply create a large corpus of training and test data. We use this to enhance PubTabNet with cell labels and create FinTabNet, real-world and complex scientific and financial datasets with detailed table structure annotations to help train and test structure recognition. Our framework surpasses previous state-of-the-art results on the ICDAR 2013 and ICDAR 2019 table competition in both table detection and cell structure recognition with a significant 5.8\% improvement in the full table extraction system. Further experiments demonstrate a greater than 45\% improvement in cell structure recognition when compared to a vanilla RetinaNet object detection model in our new out-of-domain FinTabNet.},
	urldate = {2024-01-30},
	publisher = {arXiv},
	author = {Zheng, Xinyi and Burdick, Doug and Popa, Lucian and Zhong, Xu and Wang, Nancy Xin Ru},
	month = dec,
	year = {2020},
	note = {arXiv:2005.00589 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}


@article{yang_large-scale_2023,
	title = {A large-scale dataset for end-to-end table recognition in the wild},
	volume = {10},
	copyright = {2023 The Author(s)},
	issn = {2052-4463},
	url = {https://www.nature.com/articles/s41597-023-01985-8},
	doi = {10.1038/s41597-023-01985-8},
	abstract = {Table recognition (TR) is one of the research hotspots in pattern recognition, which aims to extract information from tables in an image. Common table recognition tasks include table detection (TD), table structure recognition (TSR) and table content recognition (TCR). TD is to locate tables in the image, TCR recognizes text content, and TSR recognizes spatial \& ontology (logical) structure. Currently, the end-to-end TR in real scenarios, accomplishing the three sub-tasks simultaneously, is yet an unexplored research area. One major factor that inhibits researchers is the lack of a benchmark dataset. To this end, we propose a new large-scale dataset named Table Recognition Set (TabRecSet) with diverse table forms sourcing from multiple scenarios in the wild, providing complete annotation dedicated to end-to-end TR research. It is the largest and first bi-lingual dataset for end-to-end TR, with 38.1 K tables in which 20.4 K are in English and 17.7 K are in Chinese. The samples have diverse forms, such as the border-complete and -incomplete table, regular and irregular table (rotated, distorted, etc.). The scenarios are multiple in the wild, varying from scanned to camera-taken images, documents to Excel tables, educational test papers to financial invoices. The annotations are complete, consisting of the table body spatial annotation, cell spatial \& logical annotation and text content for TD, TSR and TCR, respectively. The spatial annotation utilizes the polygon instead of the bounding box or quadrilateral adopted by most datasets. The polygon spatial annotation is more suitable for irregular tables that are common in wild scenarios. Additionally, we propose a visualized and interactive annotation tool named TableMe to improve the efficiency and quality of table annotation.},
	language = {en},
	number = {1},
	urldate = {2024-01-30},
	journal = {Scientific Data},
	author = {Yang, Fan and Hu, Lei and Liu, Xinwu and Huang, Shuangping and Gu, Zhenghui},
	month = feb,
	year = {2023},
	note = {Number: 1
Publisher: Nature Publishing Group},
	keywords = {Computer science, Information technology},
	pages = {110},
}


@article{arik_tabnet_2021,
	title = {{TabNet}: {Attentive} {Interpretable} {Tabular} {Learning}},
	volume = {35},
	copyright = {Copyright (c) 2021 Association for the Advancement of Artificial Intelligence},
	issn = {2374-3468},
	shorttitle = {{TabNet}},
	url = {https://ojs.aaai.org/index.php/AAAI/article/view/16826},
	doi = {10.1609/aaai.v35i8.16826},
	abstract = {We propose a novel high-performance and interpretable canonical deep tabular data learning architecture, TabNet. TabNet uses sequential attention to choose which features to reason from at each decision step, enabling interpretability and more efficient learning as the learning capacity is used for the most salient features. We demonstrate that TabNet outperforms other variants on a wide range of non-performance-saturated tabular datasets and yields interpretable feature attributions plus insights into its global behavior. Finally, we demonstrate self-supervised learning for tabular data, significantly improving performance when unlabeled data is abundant.},
	language = {en},
	number = {8},
	urldate = {2024-01-30},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
	author = {Arik, Sercan Ö and Pfister, Tomas},
	month = may,
	year = {2021},
	note = {Number: 8},
	keywords = {Unsupervised \& Self-Supervised Learning},
	pages = {6679--6687},
}

@misc{huang_tabtransformer_2020,
	title = {{TabTransformer}: {Tabular} {Data} {Modeling} {Using} {Contextual} {Embeddings}},
	shorttitle = {{TabTransformer}},
	url = {http://arxiv.org/abs/2012.06678},
	doi = {10.48550/arXiv.2012.06678},
	abstract = {We propose TabTransformer, a novel deep tabular data modeling architecture for supervised and semi-supervised learning. The TabTransformer is built upon self-attention based Transformers. The Transformer layers transform the embeddings of categorical features into robust contextual embeddings to achieve higher prediction accuracy. Through extensive experiments on fifteen publicly available datasets, we show that the TabTransformer outperforms the state-of-the-art deep learning methods for tabular data by at least 1.0\% on mean AUC, and matches the performance of tree-based ensemble models. Furthermore, we demonstrate that the contextual embeddings learned from TabTransformer are highly robust against both missing and noisy data features, and provide better interpretability. Lastly, for the semi-supervised setting we develop an unsupervised pre-training procedure to learn data-driven contextual embeddings, resulting in an average 2.1\% AUC lift over the state-of-the-art methods.},
	urldate = {2024-01-30},
	publisher = {arXiv},
	author = {Huang, Xin and Khetan, Ashish and Cvitkovic, Milan and Karnin, Zohar},
	month = dec,
	year = {2020},
	note = {arXiv:2012.06678 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning},
	annote = {Comment: 7 pages, 5 figures},
	file = {arXiv Fulltext PDF:/Users/mis/Zotero/storage/4XTWQKNS/Huang et al. - 2020 - TabTransformer Tabular Data Modeling Using Contex.pdf:application/pdf},
}


@inproceedings{nassar_tableformer_2022,
	address = {New Orleans, LA, USA},
	title = {{TableFormer}: {Table} {Structure} {Understanding} with {Transformers}},
	isbn = {978-1-66546-946-3},
	shorttitle = {{TableFormer}},
	url = {https://ieeexplore.ieee.org/document/9879230/},
	doi = {10.1109/CVPR52688.2022.00457},
	abstract = {Tables organize valuable content in a concise and compact representation. This content is extremely valuable for systems such as search engines, Knowledge Graph’s, etc, since they enhance their predictive capabilities. Unfortunately, tables come in a large variety of shapes and sizes. Furthermore, they can have complex column/row-header conﬁgurations, multiline rows, different variety of separation lines, missing entries, etc. As such, the correct identiﬁcation of the table-structure from an image is a nontrivial task. In this paper, we present a new table-structure identiﬁcation model. The latter improves the latest end-toend deep learning model (i.e. encoder-dual-decoder from PubTabNet) in two signiﬁcant ways. First, we introduce a new object detection decoder for table-cells. In this way, we can obtain the content of the table-cells from programmatic PDF’s directly from the PDF source and avoid the training of the custom OCR decoders. This architectural change leads to more accurate table-content extraction and allows us to tackle non-english tables. Second, we replace the LSTM decoders with transformer based decoders. This upgrade improves signiﬁcantly the previous state-of-the-art tree-editing-distance-score (TEDS) from 91\% to 98.5\% on simple tables and from 88.7\% to 95\% on complex tables.},
	language = {en},
	urldate = {2024-01-30},
	booktitle = {2022 {IEEE}/{CVF} {Conference} on {Computer} {Vision} and {Pattern} {Recognition} ({CVPR})},
	publisher = {IEEE},
	author = {Nassar, Ahmed and Livathinos, Nikolaos and Lysak, Maksym and Staar, Peter},
	month = jun,
	year = {2022},
	pages = {4604--4613},
}


@inproceedings{yang_tableformer_2022,
	address = {Dublin, Ireland},
	title = {{TableFormer}: {Robust} {Transformer} {Modeling} for {Table}-{Text} {Encoding}},
	shorttitle = {{TableFormer}},
	url = {https://aclanthology.org/2022.acl-long.40},
	doi = {10.18653/v1/2022.acl-long.40},
	abstract = {Understanding tables is an important aspect of natural language understanding. Existing models for table understanding require linearization of the table structure, where row or column order is encoded as an unwanted bias. Such spurious biases make the model vulnerable to row and column order perturbations. Additionally, prior work has not thoroughly modeled the table structures or table-text alignments, hindering the table-text understanding ability. In this work, we propose a robust and structurally aware table-text encoding architecture TableFormer, where tabular structural biases are incorporated completely through learnable attention biases. TableFormer is (1) strictly invariant to row and column orders, and, (2) could understand tables better due to its tabular inductive biases. Our evaluations showed that TableFormer outperforms strong baselines in all settings on SQA, WTQ and TabFact table reasoning datasets, and achieves state-of-the-art performance on SQA, especially when facing answer-invariant row and column order perturbations (6\% improvement over the best baseline), because previous SOTA models' performance drops by 4\% - 6\% when facing such perturbations while TableFormer is not affected.},
	urldate = {2024-01-30},
	booktitle = {Proceedings of the 60th {Annual} {Meeting} of the {Association} for {Computational} {Linguistics} ({Volume} 1: {Long} {Papers})},
	publisher = {Association for Computational Linguistics},
	author = {Yang, Jingfeng and Gupta, Aditya and Upadhyay, Shyam and He, Luheng and Goel, Rahul and Paul, Shachi},
	editor = {Muresan, Smaranda and Nakov, Preslav and Villavicencio, Aline},
	month = may,
	year = {2022},
	pages = {528--537},
}


@inproceedings{grinsztajn_why_2022,
	title = {Why do tree-based models still outperform deep learning on typical tabular data?},
	url = {https://openreview.net/forum?id=Fp7__phQszn},
	abstract = {While deep learning has enabled tremendous progress on text and image datasets, its superiority on tabular data is not clear. We contribute extensive benchmarks of standard and novel deep learning methods as well as tree-based models such as XGBoost and Random Forests, across a large number of datasets and hyperparameter combinations. We define a standard set of 45 datasets from varied domains with clear characteristics of tabular data and a benchmarking methodology accounting for both fitting models and finding good hyperparameters. Results show that tree-based models remain state-of-the-art on medium-sized data (\${\textbackslash}sim\$10K samples) even without accounting for their superior speed. To understand this gap, we conduct an empirical investigation into the differing inductive biases of tree-based models and neural networks. This leads to a series of challenges which should guide researchers aiming to build tabular-specific neural network: 1) be robust to uninformative features, 2) preserve the orientation of the data, and 3) be able to easily learn irregular functions. To stimulate research on tabular architectures, we contribute a standard benchmark and raw data for baselines: every point of a 20{\textbackslash},000 compute hours hyperparameter search for each learner.},
	language = {en},
	urldate = {2024-01-30},
	author = {Grinsztajn, Leo and Oyallon, Edouard and Varoquaux, Gael},
	month = jun,
	year = {2022},
}



@article{pawlik_tree_2016,
	title = {Tree edit distance: {Robust} and memory-efficient},
	volume = {56},
	issn = {0306-4379},
	shorttitle = {Tree edit distance},
	url = {https://www.sciencedirect.com/science/article/pii/S0306437915001611},
	doi = {10.1016/j.is.2015.08.004},
	abstract = {Hierarchical data are often modelled as trees. An interesting query identifies pairs of similar trees. The standard approach to tree similarity is the tree edit distance, which has successfully been applied in a wide range of applications. In terms of runtime, the state-of-the-art algorithm for the tree edit distance is RTED, which is guaranteed to be fast independent of the tree shape. Unfortunately, this algorithm requires up to twice the memory of its competitors. The memory is quadratic in the tree size and is a bottleneck for the tree edit distance computation. In this paper we present a new, memory efficient algorithm for the tree edit distance, AP-TED (All Path Tree Edit Distance). Our algorithm runs at least as fast as RTED without trading in memory efficiency. This is achieved by releasing memory early during the first step of the algorithm, which computes a decomposition strategy for the actual distance computation. We show the correctness of our approach and prove an upper bound for the memory usage. The strategy computed by AP-TED is optimal in the class of all-path strategies, which subsumes the class of LRH strategies used in RTED. We further present the AP-TED+ algorithm, which requires less computational effort for very small subtrees and improves the runtime of the distance computation. Our experimental evaluation confirms the low memory requirements and the runtime efficiency of our approach.},
	urldate = {2024-01-05},
	journal = {Information Systems},
	author = {Pawlik, Mateusz and Augsten, Nikolaus},
	month = mar,
	year = {2016},
	keywords = {Approximate matching, Similarity search, Tree edit distance},
	pages = {157--173},
	file = {ScienceDirect Snapshot:/Users/mis/Zotero/storage/QU3NDFEX/S0306437915001611.html:text/html},
}



@inproceedings{schwarz_new_2017,
	address = {Cham},
	series = {Lecture {Notes} in {Computer} {Science}},
	title = {A {New} {Perspective} on the {Tree} {Edit} {Distance}},
	isbn = {978-3-319-68474-1},
	doi = {10.1007/978-3-319-68474-1_11},
	abstract = {The tree edit distance (TED), defined as the minimum-cost sequence of node operations that transform one tree into another, is a well-known distance measure for hierarchical data. Thanks to its intuitive definition, TED has found a wide range of diverse applications like software engineering, natural language processing, and bioinformatics. The state-of-the-art algorithms for TED recursively decompose the input trees into smaller subproblems and use dynamic programming to build the result in a bottom-up fashion. The main line of research deals with efficient implementations of a recursive solution introduced by Zhang in the late 1980s. Another more recent recursive solution by Chen found little attention. Its relation to the other TED solutions has never been studied and it has never been empirically tested against its competitors. In this paper we fill the gap and revisit Chen’s TED algorithm. We analyse the recursion by Chen and compare it to Zhang’s recursion. We show that all subproblems generated by Chen can also origin from Zhang’s decomposition. This is interesting since new algorithms that combine the features of both recursive solutions could be developed. Moreover, we revise the runtime complexity of Chen’s algorithm and develop a new traversal strategy to reduce its memory complexity. Finally, we provide the first experimental evaluation of Chen’s algorithm and identify tree shapes for which Chen’s solution is a promising competitor.},
	language = {en},
	booktitle = {Similarity {Search} and {Applications}},
	publisher = {Springer International Publishing},
	author = {Schwarz, Stefan and Pawlik, Mateusz and Augsten, Nikolaus},
	editor = {Beecks, Christian and Borutta, Felix and Kröger, Peer and Seidl, Thomas},
	year = {2017},
	pages = {156--170},
	file = {Full Text PDF:/Users/mis/Zotero/storage/34R853CH/Schwarz et al. - 2017 - A New Perspective on the Tree Edit Distance.pdf:application/pdf},
}


@misc{kim_prometheus_2023,
	title = {Prometheus: {Inducing} {Fine}-grained {Evaluation} {Capability} in {Language} {Models}},
	shorttitle = {Prometheus},
	url = {http://arxiv.org/abs/2310.08491},
	doi = {10.48550/arXiv.2310.08491},
	abstract = {Recently, using a powerful proprietary Large Language Model (LLM) (e.g., GPT-4) as an evaluator for long-form responses has become the de facto standard. However, for practitioners with large-scale evaluation tasks and custom criteria in consideration (e.g., child-readability), using proprietary LLMs as an evaluator is unreliable due to the closed-source nature, uncontrolled versioning, and prohibitive costs. In this work, we propose Prometheus, a fully open-source LLM that is on par with GPT-4's evaluation capabilities when the appropriate reference materials (reference answer, score rubric) are accompanied. We first construct the Feedback Collection, a new dataset that consists of 1K fine-grained score rubrics, 20K instructions, and 100K responses and language feedback generated by GPT-4. Using the Feedback Collection, we train Prometheus, a 13B evaluator LLM that can assess any given long-form text based on customized score rubric provided by the user. Experimental results show that Prometheus scores a Pearson correlation of 0.897 with human evaluators when evaluating with 45 customized score rubrics, which is on par with GPT-4 (0.882), and greatly outperforms ChatGPT (0.392). Furthermore, measuring correlation with GPT-4 with 1222 customized score rubrics across four benchmarks (MT Bench, Vicuna Bench, Feedback Bench, Flask Eval) shows similar trends, bolstering Prometheus's capability as an evaluator LLM. Lastly, Prometheus achieves the highest accuracy on two human preference benchmarks (HHH Alignment \& MT Bench Human Judgment) compared to open-sourced reward models explicitly trained on human preference datasets, highlighting its potential as an universal reward model. We open-source our code, dataset, and model at https://github.com/kaistAI/Prometheus.},
	urldate = {2023-12-20},
	publisher = {arXiv},
	author = {Kim, Seungone and Shin, Jamin and Cho, Yejin and Jang, Joel and Longpre, Shayne and Lee, Hwaran and Yun, Sangdoo and Shin, Seongjin and Kim, Sungdong and Thorne, James and Seo, Minjoon},
	month = oct,
	year = {2023},
	note = {arXiv:2310.08491 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning},
	annote = {Comment: Work in Progress},
	file = {arXiv Fulltext PDF:/Users/mis/Zotero/storage/BJ4RVKSL/Kim et al. - 2023 - Prometheus Inducing Fine-grained Evaluation Capab.pdf:application/pdf;arXiv.org Snapshot:/Users/mis/Zotero/storage/RQD7CL4N/2310.html:text/html},
}
@ARTICLE{1966SPhD...10..707L,
       author = {{Levenshtein}, V.~I.},
        title = "{Binary Codes Capable of Correcting Deletions, Insertions and Reversals}",
      journal = {Soviet Physics Doklady},
         year = 1966,
        month = feb,
       volume = {10},
        pages = {707},
       adsurl = {https://ui.adsabs.harvard.edu/abs/1966SPhD...10..707L},
      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}



@article{cowie_information_1996,
	title = {Information extraction},
	volume = {39},
	issn = {0001-0782},
	url = {https://dl.acm.org/doi/10.1145/234173.234209},
	doi = {10.1145/234173.234209},
	number = {1},
	urldate = {2024-02-15},
	journal = {Communications of the ACM},
	author = {Cowie, Jim and Lehnert, Wendy},
	month = jan,
	year = {1996},
	pages = {80--91},
}

@article{mishra_esg_2024,
	title = {{ESG} {Accountability} {Made} {Easy}: {DocQA} at {Your} {Service}},
	volume = {38},
	copyright = {Copyright (c) 2024 Association for the Advancement of Artificial Intelligence},
	issn = {2374-3468},
	shorttitle = {{ESG} {Accountability} {Made} {Easy}},
	url = {https://ojs.aaai.org/index.php/AAAI/article/view/30574},
	doi = {10.1609/aaai.v38i21.30574},
	abstract = {We present Deep Search DocQA. This application enables information extraction from documents via a question-answering conversational assistant. The system integrates several technologies from different AI disciplines consisting of document conversion to machine-readable format (via computer vision), finding relevant data (via natural language processing), and formulating an eloquent response (via large language models). Users can explore over 10,000 Environmental, Social, and Governance (ESG) disclosure reports from over 2000 corporations. The Deep Search platform can be accessed at: https://ds4sd.github.io.},
	language = {en},
	number = {21},
	urldate = {2024-04-18},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
	author = {Mishra, Lokesh and Berrospi, Cesar and Dinkla, Kasper and Antognini, Diego and Fusco, Francesco and Bothur, Benedikt and Lysak, Maksym and Livathinos, Nikolaos and Nassar, Ahmed and Vagenas, Panagiotis and Morin, Lucas and Auer, Christoph and Dolfi, Michele and Staar, Peter},
	month = mar,
	year = {2024},
	note = {Number: 21},
	keywords = {Systems that integrate different AI technologies},
	pages = {23814--23816},
	file = {Full Text PDF:/Users/mis/Zotero/storage/6KFTINYS/Mishra et al. - 2024 - ESG Accountability Made Easy DocQA at Your Servic.pdf:application/pdf},
}



@article{henisz_five_2019,
	title = {Five ways that {ESG} creates value},
	url = {https://www.mckinsey.com/~/media/McKinsey/Business%20Functions/Strategy%20and%20Corporate%20Finance/Our%20Insights/Five%20ways%20that%20ESG%20creates%20value/Five-ways-that-ESG-creates-value.ashx},
	language = {en},
	journal = {McKinsey Quarterly},
	author = {Henisz, Witold and Koller, Tim and Nuttall, Robin},
	year = {2019},
}


@inproceedings{wang_zero-shot_2021,
	address = {Online and Punta Cana, Dominican Republic},
	title = {Zero-{Shot} {Information} {Extraction} as a {Unified} {Text}-to-{Triple} {Translation}},
	url = {https://aclanthology.org/2021.emnlp-main.94},
	doi = {10.18653/v1/2021.emnlp-main.94},
	abstract = {We cast a suite of information extraction tasks into a text-to-triple translation framework. Instead of solving each task relying on task-specific datasets and models, we formalize the task as a translation between task-specific input text and output triples. By taking the task-specific input, we enable a task-agnostic translation by leveraging the latent knowledge that a pre-trained language model has about the task. We further demonstrate that a simple pre-training task of predicting which relational information corresponds to which input text is an effective way to produce task-specific outputs. This enables the zero-shot transfer of our framework to downstream tasks. We study the zero-shot performance of this framework on open information extraction (OIE2016, NYT, WEB, PENN), relation classification (FewRel and TACRED), and factual probe (Google-RE and T-REx). The model transfers non-trivially to most tasks and is often competitive with a fully supervised method without the need for any task-specific training. For instance, we significantly outperform the F1 score of the supervised open information extraction without needing to use its training set.},
	urldate = {2024-02-15},
	booktitle = {Proceedings of the 2021 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing}},
	publisher = {Association for Computational Linguistics},
	author = {Wang, Chenguang and Liu, Xiao and Chen, Zui and Hong, Haoyun and Tang, Jie and Song, Dawn},
	editor = {Moens, Marie-Francine and Huang, Xuanjing and Specia, Lucia and Yih, Scott Wen-tau},
	month = nov,
	year = {2021},
	pages = {1225--1238},
}

@misc{xu2023large,
    title={Large Language Models for Generative Information Extraction: A Survey}, 
    author={Derong Xu and Wei Chen and Wenjun Peng and Chao Zhang and Tong Xu and Xiangyu Zhao and Xian Wu and Yefeng Zheng and Enhong Chen},
    year={2023},
    eprint={2312.17617},
    archivePrefix={arXiv},
    primaryClass={cs.CL}
}

@inproceedings{wang_ielm_2022,
	address = {Abu Dhabi, United Arab Emirates},
	title = {{IELM}: {An} {Open} {Information} {Extraction} {Benchmark} for {Pre}-{Trained} {Language} {Models}},
	shorttitle = {{IELM}},
	url = {https://aclanthology.org/2022.emnlp-main.576},
	doi = {10.18653/v1/2022.emnlp-main.576},
	abstract = {We introduce a new open information extraction (OIE) benchmark for pre-trained language models (LM). Recent studies have demonstrated that pre-trained LMs, such as BERT and GPT, may store linguistic and relational knowledge. In particular, LMs are able to answer “fill-in-the-blank” questions when given a pre-defined relation category. Instead of focusing on pre-defined relations, we create an OIE benchmark aiming to fully examine the open relational information present in the pre-trained LMs. We accomplish this by turning pre-trained LMs into zero-shot OIE systems. Surprisingly, pre-trained LMs are able to obtain competitive performance on both standard OIE datasets (CaRB and Re-OIE2016) and two new large-scale factual OIE datasets (TAC KBP-OIE and Wikidata-OIE) that we establish via distant supervision. For instance, the zero-shot pre-trained LMs outperform the F1 score of the state-of-the-art supervised OIE methods on our factual OIE datasets without needing to use any training sets.},
	urldate = {2024-02-15},
	booktitle = {Proceedings of the 2022 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing}},
	publisher = {Association for Computational Linguistics},
	author = {Wang, Chenguang and Liu, Xiao and Song, Dawn},
	editor = {Goldberg, Yoav and Kozareva, Zornitsa and Zhang, Yue},
	month = dec,
	year = {2022},
	pages = {8417--8437},

}


@inproceedings{wang_deepstruct_2022,
	address = {Dublin, Ireland},
	title = {{DeepStruct}: {Pretraining} of {Language} {Models} for {Structure} {Prediction}},
	shorttitle = {{DeepStruct}},
	url = {https://aclanthology.org/2022.findings-acl.67},
	doi = {10.18653/v1/2022.findings-acl.67},
	abstract = {We introduce a method for improving the structural understanding abilities of language models. Unlike previous approaches that finetune the models with task-specific augmentation, we pretrain language models to generate structures from the text on a collection of task-agnostic corpora. Our structure pretraining enables zero-shot transfer of the learned knowledge that models have about the structure tasks. We study the performance of this approach on 28 datasets, spanning 10 structure prediction tasks including open information extraction, joint entity and relation extraction, named entity recognition, relation classification, semantic role labeling, event extraction, coreference resolution, factual probe, intent detection, and dialogue state tracking. We further enhance the pretraining with the task-specific training sets. We show that a 10B parameter language model transfers non-trivially to most tasks and obtains state-of-the-art performance on 21 of 28 datasets that we evaluate. Our code and datasets will be made publicly available.},
	urldate = {2024-02-15},
	booktitle = {Findings of the {Association} for {Computational} {Linguistics}: {ACL} 2022},
	publisher = {Association for Computational Linguistics},
	author = {Wang, Chenguang and Liu, Xiao and Chen, Zui and Hong, Haoyun and Tang, Jie and Song, Dawn},
	editor = {Muresan, Smaranda and Nakov, Preslav and Villavicencio, Aline},
	month = may,
	year = {2022},
	pages = {803--823},
}



@misc{fang_large_2024,
	title = {Large {Language} {Models}({LLMs}) on {Tabular} {Data}: {Prediction}, {Generation}, and {Understanding} -- {A} {Survey}},
	shorttitle = {Large {Language} {Models}({LLMs}) on {Tabular} {Data}},
	url = {http://arxiv.org/abs/2402.17944},
	doi = {10.48550/arXiv.2402.17944},
	abstract = {Recent breakthroughs in large language modeling have facilitated rigorous exploration of their application in diverse tasks related to tabular data modeling, such as prediction, tabular data synthesis, question answering, and table understanding. Each task presents unique challenges and opportunities. However, there is currently a lack of comprehensive review that summarizes and compares the key techniques, metrics, datasets, models, and optimization approaches in this research domain. This survey aims to address this gap by consolidating recent progress in these areas, offering a thorough survey and taxonomy of the datasets, metrics, and methodologies utilized. It identifies strengths, limitations, unexplored territories, and gaps in the existing literature, while providing some insights for future research directions in this vital and rapidly evolving field. It also provides relevant code and datasets references. Through this comprehensive review, we hope to provide interested readers with pertinent references and insightful perspectives, empowering them with the necessary tools and knowledge to effectively navigate and address the prevailing challenges in the field.},
	urldate = {2024-04-16},
	publisher = {arXiv},
	author = {Fang, Xi and Xu, Weijie and Tan, Fiona Anting and Zhang, Jiani and Hu, Ziqing and Qi, Yanjun and Nickleach, Scott and Socolinsky, Diego and Sengamedu, Srinivasan and Faloutsos, Christos},
	month = feb,
	year = {2024},
	note = {arXiv:2402.17944 [cs]},
	keywords = {68T50, Computer Science - Computation and Language, I.2.7},
	annote = {Comment: 41 pages, 4 figures, 8 tables},
}

@inproceedings{herzig-etal-2020-tapas,
    title = "{T}a{P}as: Weakly Supervised Table Parsing via Pre-training",
    author = {Herzig, Jonathan  and
      Nowak, Pawel Krzysztof  and
      M{\"u}ller, Thomas  and
      Piccinno, Francesco  and
      Eisenschlos, Julian},
    editor = "Jurafsky, Dan  and
      Chai, Joyce  and
      Schluter, Natalie  and
      Tetreault, Joel",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.acl-main.398",
    doi = "10.18653/v1/2020.acl-main.398",
    pages = "4320--4333",
    abstract = "Answering natural language questions over tables is usually seen as a semantic parsing task. To alleviate the collection cost of full logical forms, one popular approach focuses on weak supervision consisting of denotations instead of logical forms. However, training semantic parsers from weak supervision poses difficulties, and in addition, the generated logical forms are only used as an intermediate step prior to retrieving the denotation. In this paper, we present TaPas, an approach to question answering over tables without generating logical forms. TaPas trains from weak supervision, and predicts the denotation by selecting table cells and optionally applying a corresponding aggregation operator to such selection. TaPas extends BERT{'}s architecture to encode tables as input, initializes from an effective joint pre-training of text segments and tables crawled from Wikipedia, and is trained end-to-end. We experiment with three different semantic parsing datasets, and find that TaPas outperforms or rivals semantic parsing models by improving state-of-the-art accuracy on SQA from 55.1 to 67.2 and performing on par with the state-of-the-art on WikiSQL and WikiTQ, but with a simpler model architecture. We additionally find that transfer learning, which is trivial in our setting, from WikiSQL to WikiTQ, yields 48.7 accuracy, 4.2 points above the state-of-the-art.",
}

@inproceedings{yin-etal-2020-tabert,
    title = "{T}a{BERT}: Pretraining for Joint Understanding of Textual and Tabular Data",
    author = "Yin, Pengcheng  and
      Neubig, Graham  and
      Yih, Wen-tau  and
      Riedel, Sebastian",
    editor = "Jurafsky, Dan  and
      Chai, Joyce  and
      Schluter, Natalie  and
      Tetreault, Joel",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.acl-main.745",
    doi = "10.18653/v1/2020.acl-main.745",
    pages = "8413--8426",
    abstract = "Recent years have witnessed the burgeoning of pretrained language models (LMs) for text-based natural language (NL) understanding tasks. Such models are typically trained on free-form NL text, hence may not be suitable for tasks like semantic parsing over structured data, which require reasoning over both free-form NL questions and structured tabular data (e.g., database tables). In this paper we present TaBERT, a pretrained LM that jointly learns representations for NL sentences and (semi-)structured tables. TaBERT is trained on a large corpus of 26 million tables and their English contexts. In experiments, neural semantic parsers using TaBERT as feature representation layers achieve new best results on the challenging weakly-supervised semantic parsing benchmark WikiTableQuestions, while performing competitively on the text-to-SQL dataset Spider.",
}

@inproceedings{iida-etal-2021-tabbie,
    title = "{TABBIE}: Pretrained Representations of Tabular Data",
    author = "Iida, Hiroshi  and
      Thai, Dung  and
      Manjunatha, Varun  and
      Iyyer, Mohit",
    editor = "Toutanova, Kristina  and
      Rumshisky, Anna  and
      Zettlemoyer, Luke  and
      Hakkani-Tur, Dilek  and
      Beltagy, Iz  and
      Bethard, Steven  and
      Cotterell, Ryan  and
      Chakraborty, Tanmoy  and
      Zhou, Yichao",
    booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",
    month = jun,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.naacl-main.270",
    doi = "10.18653/v1/2021.naacl-main.270",
    pages = "3446--3456",
    abstract = "Existing work on tabular representation-learning jointly models tables and associated text using self-supervised objective functions derived from pretrained language models such as BERT. While this joint pretraining improves tasks involving paired tables and text (e.g., answering questions about tables), we show that it underperforms on tasks that operate over tables without any associated text (e.g., populating missing cells). We devise a simple pretraining objective (corrupt cell detection) that learns exclusively from tabular data and reaches the state-of-the-art on a suite of table-based prediction tasks. Unlike competing approaches, our model (TABBIE) provides embeddings of all table substructures (cells, rows, and columns), and it also requires far less compute to train. A qualitative analysis of our model{'}s learned cell, column, and row representations shows that it understands complex table semantics and numerical trends.",
}


@inproceedings{yang-etal-2022-tableformer,
    title = "{T}able{F}ormer: Robust Transformer Modeling for Table-Text Encoding",
    author = "Yang, Jingfeng  and
      Gupta, Aditya  and
      Upadhyay, Shyam  and
      He, Luheng  and
      Goel, Rahul  and
      Paul, Shachi",
    editor = "Muresan, Smaranda  and
      Nakov, Preslav  and
      Villavicencio, Aline",
    booktitle = "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.acl-long.40",
    doi = "10.18653/v1/2022.acl-long.40",
    pages = "528--537",
    abstract = "Understanding tables is an important aspect of natural language understanding. Existing models for table understanding require linearization of the table structure, where row or column order is encoded as an unwanted bias. Such spurious biases make the model vulnerable to row and column order perturbations. Additionally, prior work has not thoroughly modeled the table structures or table-text alignments, hindering the table-text understanding ability. In this work, we propose a robust and structurally aware table-text encoding architecture TableFormer, where tabular structural biases are incorporated completely through learnable attention biases. TableFormer is (1) strictly invariant to row and column orders, and, (2) could understand tables better due to its tabular inductive biases. Our evaluations showed that TableFormer outperforms strong baselines in all settings on SQA, WTQ and TabFact table reasoning datasets, and achieves state-of-the-art performance on SQA, especially when facing answer-invariant row and column order perturbations (6{\%} improvement over the best baseline), because previous SOTA models{'} performance drops by 4{\%} - 6{\%} when facing such perturbations while TableFormer is not affected.",
}

@inproceedings{2019TabFact,
  title={TabFact : A Large-scale Dataset for Table-based Fact Verification},
  author={Wenhu Chen, Hongmin Wang, Jianshu Chen, Yunkai Zhang, Hong Wang, Shiyang Li, Xiyou Zhou and William Yang Wang},
  booktitle = {International Conference on Learning Representations (ICLR)},
  address = {Addis Ababa, Ethiopia},
  month = {April},
  year = {2020}
}


@inproceedings{liu_tapex_2021,
	title = {{TAPEX}: {Table} {Pre}-training via {Learning} a {Neural} {SQL} {Executor}},
	shorttitle = {{TAPEX}},
	url = {https://openreview.net/forum?id=O50443AsCP},
	abstract = {Recent progress in language model pre-training has achieved a great success via leveraging large-scale unstructured textual data. However, it is still a challenge to apply pre-training on structured tabular data due to the absence of large-scale high-quality tabular data. In this paper, we propose TAPEX to show that table pre-training can be achieved by learning a neural SQL executor over a synthetic corpus, which is obtained by automatically synthesizing executable SQL queries and their execution outputs. TAPEX addresses the data scarcity challenge via guiding the language model to mimic a SQL executor on the diverse, large-scale and high-quality synthetic corpus. We evaluate TAPEX on four benchmark datasets. Experimental results demonstrate that TAPEX outperforms previous table pre-training approaches by a large margin and achieves new state-of-the-art results on all of them. This includes the improvements on the weakly-supervised WikiSQL denotation accuracy to 89.5\% (+2.3\%), the WikiTableQuestions denotation accuracy to 57.5\% (+4.8\%), the SQA denotation accuracy to 74.5\% (+3.5\%), and the TabFact accuracy to 84.2\% (+3.2\%). To our knowledge, this is the first work to exploit table pre-training via synthetic executable programs and to achieve new state-of-the-art results on various downstream tasks. Our code can be found at https://github.com/microsoft/Table-Pretraining.},
	language = {en},
	urldate = {2024-04-18},
	author = {Liu, Qian and Chen, Bei and Guo, Jiaqi and Ziyadi, Morteza and Lin, Zeqi and Chen, Weizhu and Lou, Jian-Guang},
	month = oct,
	year = {2021},
	file = {Full Text PDF:/Users/mis/Zotero/storage/PBJHRPI6/Liu et al. - 2021 - TAPEX Table Pre-training via Learning a Neural SQ.pdf:application/pdf},
}



@inproceedings{yu_grappa_2020,
	title = {{GraPPa}: {Grammar}-{Augmented} {Pre}-{Training} for {Table} {Semantic} {Parsing}},
	shorttitle = {{GraPPa}},
	url = {https://openreview.net/forum?id=kyaIeYj4zZ},
	abstract = {We present GraPPa, an effective pre-training approach for table semantic parsing that learns a compositional inductive bias in the joint representations of textual and tabular data. We construct synthetic question-SQL pairs over high-quality tables via a synchronous context-free grammar (SCFG). We pre-train our model on the synthetic data to inject important structural properties commonly found in semantic parsing into the pre-training language model. To maintain the model's ability to represent real-world data, we also include masked language modeling (MLM) on several existing table-related datasets to regularize our pre-training process. Our proposed pre-training strategy is much data-efficient. When incorporated with strong base semantic parsers, GraPPa achieves new state-of-the-art results on four popular fully supervised and weakly supervised table semantic parsing tasks.},
	language = {en},
	urldate = {2024-04-18},
	author = {Yu, Tao and Wu, Chien-Sheng and Lin, Xi Victoria and Wang, Bailin and Tan, Yi Chern and Yang, Xinyi and Radev, Dragomir and Socher, Richard and Xiong, Caiming},
	month = oct,
	year = {2020},
	file = {Full Text PDF:/Users/mis/Zotero/storage/UW6P3R8J/Yu et al. - 2020 - GraPPa Grammar-Augmented Pre-Training for Table S.pdf:application/pdf},
}

@article{badaro_transformers_2023,
	title = {Transformers for {Tabular} {Data} {Representation}: {A} {Survey} of {Models} and {Applications}},
	volume = {11},
	shorttitle = {Transformers for {Tabular} {Data} {Representation}},
	url = {https://aclanthology.org/2023.tacl-1.14},
	doi = {10.1162/tacl_a_00544},
	abstract = {In the last few years, the natural language processing community has witnessed advances in neural representations of free texts with transformer-based language models (LMs). Given the importance of knowledge available in tabular data, recent research efforts extend LMs by developing neural representations for structured data. In this article, we present a survey that analyzes these efforts. We first abstract the different systems according to a traditional machine learning pipeline in terms of training data, input representation, model training, and supported downstream tasks. For each aspect, we characterize and compare the proposed solutions. Finally, we discuss future work directions.},
	urldate = {2024-04-16},
	journal = {Transactions of the Association for Computational Linguistics},
	author = {Badaro, Gilbert and Saeed, Mohammed and Papotti, Paolo},
	year = {2023},
	note = {Place: Cambridge, MA
Publisher: MIT Press},
	pages = {227--249},
	file = {Full Text PDF:/Users/mis/Zotero/storage/CUI5NMET/Badaro et al. - 2023 - Transformers for Tabular Data Representation A Su.pdf:application/pdf},
}


@inproceedings{sun_supertml_2019,
	title = {{SuperTML}: {Two}-{Dimensional} {Word} {Embedding} for the {Precognition} on {Structured} {Tabular} {Data}},
	shorttitle = {{SuperTML}},
	url = {https://ieeexplore.ieee.org/document/9025460},
	doi = {10.1109/CVPRW.2019.00360},
	abstract = {Tabular data is the most commonly used form of data in industry according to a Kaggle ML and DS Survey. Gradient Boosting Trees, Support Vector Machine, Random Forest, and Logistic Regression are typically used for classification tasks on tabular data. DNN models using categorical embeddings are also applied in this task, but all attempts thus far have used one-dimensional embeddings. The recent work of Super Characters method using two-dimensional word embeddings achieved state-of-the-art results in text classification tasks, showcasing the promise of this new approach. In this paper, we propose the SuperTML method, which borrows the idea of Super Characters method and two-dimensional embeddings to address the problem of classification on tabular data. For each input of tabular data, the features are first projected into two-dimensional embeddings like an image, and then this image is fed into fine-tuned two-dimensional CNN models for classification. The proposed SuperTML method handles the categorical data and missing values in tabular data automatically, without any need to pre-process into numerical values. Comparisons of model performance are conducted on one of the largest and most active competitions on the Kaggle platform, as well as on the top three most popular data sets in the UCI Machine Learning Repository. Experimental results have shown that the proposed SuperTML method have achieved state-of-the-art results on both large and small datasets.},
	urldate = {2024-04-18},
	booktitle = {2019 {IEEE}/{CVF} {Conference} on {Computer} {Vision} and {Pattern} {Recognition} {Workshops} ({CVPRW})},
	author = {Sun, Baohua and Yang, Lin and Zhang, Wenhan and Lin, Michael and Dong, Patrick and Young, Charles and Dong, Jason},
	month = jun,
	year = {2019},
	note = {ISSN: 2160-7516},
	keywords = {Boosting, Data models, Iris recognition, Numerical models, Task analysis, Training},
	pages = {2973--2981},
	file = {IEEE Xplore Abstract Record:/Users/mis/Zotero/storage/2L8HQIMU/9025460.html:text/html;Submitted Version:/Users/mis/Zotero/storage/5T5UBM64/Sun et al. - 2019 - SuperTML Two-Dimensional Word Embedding for the P.pdf:application/pdf},
}
@article{Liu2020DNN2LRIF,
  title={DNN2LR: Interpretation-inspired Feature Crossing for Real-world Tabular Data},
  author={Zhaocheng Liu and Qiang Liu and Hao Zhang and Yuntian Chen},
  journal={ArXiv},
  year={2020},
  volume={abs/2008.09775},
  url={https://api.semanticscholar.org/CorpusID:221266499}
}


@inproceedings{paolini_structured_2020,
	title = {Structured {Prediction} as {Translation} between {Augmented} {Natural} {Languages}},
	url = {https://openreview.net/forum?id=US-TP-xnXI},
	abstract = {We propose a new framework, Translation between Augmented Natural Languages (TANL), to solve many structured prediction language tasks including joint entity and relation extraction, nested named entity recognition, relation classification, semantic role labeling, event extraction, coreference resolution, and dialogue state tracking. Instead of tackling the problem by training task-specific discriminative classifiers, we frame it as a translation task between augmented natural languages, from which the task-relevant information can be easily extracted. Our approach can match or outperform task-specific models on all tasks, and in particular achieves new state-of-the-art results on joint entity and relation extraction (CoNLL04, ADE, NYT, and ACE2005 datasets), relation classification (FewRel and TACRED), and semantic role labeling (CoNLL-2005 and CoNLL-2012). We accomplish this while using the same architecture and hyperparameters for all tasks, and even when training a single model to solve all tasks at the same time (multi-task learning). Finally, we show that our framework can also significantly improve the performance in a low-resource regime, thanks to better use of label semantics.},
	language = {en},
	urldate = {2024-04-18},
	author = {Paolini, Giovanni and Athiwaratkun, Ben and Krone, Jason and Ma, Jie and Achille, Alessandro and Anubhai, Rishita and Santos, Cicero Nogueira dos and Xiang, Bing and Soatto, Stefano},
	month = oct,
	year = {2020},
	file = {Full Text PDF:/Users/mis/Zotero/storage/UAJS5XXU/Paolini et al. - 2020 - Structured Prediction as Translation between Augme.pdf:application/pdf},
}


@misc{wang_instructuie_2023,
	title = {{InstructUIE}: {Multi}-task {Instruction} {Tuning} for {Unified} {Information} {Extraction}},
	shorttitle = {{InstructUIE}},
	url = {http://arxiv.org/abs/2304.08085},
	doi = {10.48550/arXiv.2304.08085},
	abstract = {Large language models have unlocked strong multi-task capabilities from reading instructive prompts. However, recent studies have shown that existing large models still have difficulty with information extraction tasks. For example, gpt-3.5-turbo achieved an F1 score of 18.22 on the Ontonotes dataset, which is significantly lower than the state-of-the-art performance. In this paper, we propose InstructUIE, a unified information extraction framework based on instruction tuning, which can uniformly model various information extraction tasks and capture the inter-task dependency. To validate the proposed method, we introduce IE INSTRUCTIONS, a benchmark of 32 diverse information extraction datasets in a unified text-to-text format with expert-written instructions. Experimental results demonstrate that our method achieves comparable performance to Bert in supervised settings and significantly outperforms the state-of-the-art and gpt3.5 in zero-shot settings.},
	urldate = {2024-04-18},
	publisher = {arXiv},
	author = {Wang, Xiao and Zhou, Weikang and Zu, Can and Xia, Han and Chen, Tianze and Zhang, Yuansen and Zheng, Rui and Ye, Junjie and Zhang, Qi and Gui, Tao and Kang, Jihua and Yang, Jingsheng and Li, Siyuan and Du, Chunsai},
	month = apr,
	year = {2023},
	note = {arXiv:2304.08085 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
	file = {arXiv Fulltext PDF:/Users/mis/Zotero/storage/QRSXVRGW/Wang et al. - 2023 - InstructUIE Multi-task Instruction Tuning for Uni.pdf:application/pdf;arXiv.org Snapshot:/Users/mis/Zotero/storage/UJJSHE7T/2304.html:text/html},
}

@inproceedings{kardas_axcell_2020,
	address = {Online},
	title = {{AxCell}: {Automatic} {Extraction} of {Results} from {Machine} {Learning} {Papers}},
	shorttitle = {{AxCell}},
	url = {https://aclanthology.org/2020.emnlp-main.692},
	doi = {10.18653/v1/2020.emnlp-main.692},
	abstract = {Tracking progress in machine learning has become increasingly difficult with the recent explosion in the number of papers. In this paper, we present AxCell, an automatic machine learning pipeline for extracting results from papers. AxCell uses several novel components, including a table segmentation subtask, to learn relevant structural knowledge that aids extraction. When compared with existing methods, our approach significantly improves the state of the art for results extraction. We also release a structured, annotated dataset for training models for results extraction, and a dataset for evaluating the performance of models on this task. Lastly, we show the viability of our approach enables it to be used for semi-automated results extraction in production, suggesting our improvements make this task practically viable for the first time. Code is available on GitHub.},
	urldate = {2024-03-26},
	booktitle = {Proceedings of the 2020 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing} ({EMNLP})},
	publisher = {Association for Computational Linguistics},
	author = {Kardas, Marcin and Czapla, Piotr and Stenetorp, Pontus and Ruder, Sebastian and Riedel, Sebastian and Taylor, Ross and Stojnic, Robert},
	editor = {Webber, Bonnie and Cohn, Trevor and He, Yulan and Liu, Yang},
	month = nov,
	year = {2020},
	pages = {8580--8594},
	file = {Full Text PDF:/Users/mis/Zotero/storage/XVYJ7XRV/Kardas et al. - 2020 - AxCell Automatic Extraction of Results from Machi.pdf:application/pdf},
}


@misc{bai_schema-driven_2024,
	title = {Schema-{Driven} {Information} {Extraction} from {Heterogeneous} {Tables}},
	url = {http://arxiv.org/abs/2305.14336},
	doi = {10.48550/arXiv.2305.14336},
	abstract = {In this paper, we explore the question of whether large language models can support cost-efficient information extraction from tables. We introduce schema-driven information extraction, a new task that transforms tabular data into structured records following a human-authored schema. To assess various LLM's capabilities on this task, we present a benchmark comprised of tables from four diverse domains: machine learning papers, chemistry literature, material science journals, and webpages. We use this collection of annotated tables to evaluate the ability of open-source and API-based language models to extract information from tables covering diverse domains and data formats. Our experiments demonstrate that surprisingly competitive performance can be achieved without requiring task-specific pipelines or labels, achieving F1 scores ranging from 74.2 to 96.1, while maintaining cost efficiency. Moreover, through detailed ablation studies and analyses, we investigate the factors contributing to model success and validate the practicality of distilling compact models to reduce API reliance.},
	urldate = {2024-03-26},
	publisher = {arXiv},
	author = {Bai, Fan and Kang, Junmo and Stanovsky, Gabriel and Freitag, Dayne and Ritter, Alan},
	month = mar,
	year = {2024},
	note = {arXiv:2305.14336 [cs]},
	keywords = {Computer Science - Computation and Language},
	file = {arXiv Fulltext PDF:/Users/mis/Zotero/storage/47V8FA8S/Bai et al. - 2024 - Schema-Driven Information Extraction from Heteroge.pdf:application/pdf;arXiv.org Snapshot:/Users/mis/Zotero/storage/AELNEG6F/2305.html:text/html},
}

@inproceedings{lu_unified_2022,
	address = {Dublin, Ireland},
	title = {Unified {Structure} {Generation} for {Universal} {Information} {Extraction}},
	url = {https://aclanthology.org/2022.acl-long.395},
	doi = {10.18653/v1/2022.acl-long.395},
	abstract = {Information extraction suffers from its varying targets, heterogeneous structures, and demand-specific schemas. In this paper, we propose a unified text-to-structure generation framework, namely UIE, which can universally model different IE tasks, adaptively generate targeted structures, and collaboratively learn general IE abilities from different knowledge sources. Specifically, UIE uniformly encodes different extraction structures via a structured extraction language, adaptively generates target extractions via a schema-based prompt mechanism – structural schema instructor, and captures the common IE abilities via a large-scale pretrained text-to-structure model. Experiments show that UIE achieved the state-of-the-art performance on 4 IE tasks, 13 datasets, and on all supervised, low-resource, and few-shot settings for a wide range of entity, relation, event and sentiment extraction tasks and their unification. These results verified the effectiveness, universality, and transferability of UIE.},
	urldate = {2024-04-18},
	booktitle = {Proceedings of the 60th {Annual} {Meeting} of the {Association} for {Computational} {Linguistics} ({Volume} 1: {Long} {Papers})},
	publisher = {Association for Computational Linguistics},
	author = {Lu, Yaojie and Liu, Qing and Dai, Dai and Xiao, Xinyan and Lin, Hongyu and Han, Xianpei and Sun, Le and Wu, Hua},
	editor = {Muresan, Smaranda and Nakov, Preslav and Villavicencio, Aline},
	month = may,
	year = {2022},
	pages = {5755--5772},
	file = {Full Text PDF:/Users/mis/Zotero/storage/L54GEQZ5/Lu et al. - 2022 - Unified Structure Generation for Universal Informa.pdf:application/pdf},
}

@article{Jiang2024MixtralOE,
  title={Mixtral of Experts},
  author={Albert Q. Jiang and Alexandre Sablayrolles and Antoine Roux and Arthur Mensch and Blanche Savary and Chris Bamford and Devendra Singh Chaplot and Diego de Las Casas and Emma Bou Hanna and Florian Bressand and Gianna Lengyel and Guillaume Bour and Guillaume Lample and L'elio Renard Lavaud and Lucile Saulnier and Marie-Anne Lachaux and Pierre Stock and Sandeep Subramanian and Sophia Yang and Szymon Antoniak and Teven Le Scao and Th{\'e}ophile Gervet and Thibaut Lavril and Thomas Wang and Timoth{\'e}e Lacroix and William El Sayed},
  journal={ArXiv},
  year={2024},
  volume={abs/2401.04088},
  url={https://api.semanticscholar.org/CorpusID:266844877}
}

@misc{almazrouei2023falcon,
      title={The Falcon Series of Open Language Models}, 
      author={Ebtesam Almazrouei and Hamza Alobeidli and Abdulaziz Alshamsi and Alessandro Cappelli and Ruxandra Cojocaru and Mérouane Debbah and Étienne Goffinet and Daniel Hesslow and Julien Launay and Quentin Malartic and Daniele Mazzotta and Badreddine Noune and Baptiste Pannier and Guilherme Penedo},
      year={2023},
      eprint={2311.16867},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}


@article{schimanski_bridging_2024,
	title = {Bridging the gap in {ESG} measurement: {Using} {NLP} to quantify environmental, social, and governance communication},
	volume = {61},
	issn = {1544-6123},
	url = {https://www.sciencedirect.com/science/article/pii/S1544612324000096},
	doi = {https://doi.org/10.1016/j.frl.2024.104979},
	abstract = {Environmental, social, and governance (ESG) criteria take a central role in fostering sustainable development in economies. This paper introduces a class of novel Natural Language Processing (NLP) models to assess corporate disclosures in the ESG subdomains. Using over 13.8 million texts from reports and news, specific E, S, and G models were pretrained. Additionally, three 2k datasets were developed to classify ESG-related texts. The models effectively explain variations in ESG ratings, showcasing a robust method for enhancing transparency and accuracy in evaluating corporate sustainability. This approach addresses the gap in precise, transparent ESG measurement, advancing sustainable development in economies.},
	journal = {Finance Research Letters},
	author = {Schimanski, Tobias and Reding, Andrin and Reding, Nico and Bingler, Julia and Kraus, Mathias and Leippold, Markus},
	year = {2024},
	keywords = {BERT model, ESG analysis in financial markets, Natural language processing},
	pages = {104979},
}

@misc{bingler_how_2022,
	address = {Rochester, NY},
	type = {{SSRN} {Scholarly} {Paper}},
	title = {How {Cheap} {Talk} in {Climate} {Disclosures} relates to {Climate} {Initiatives}, {Corporate} {Emissions}, and {Reputation} {Risk}},
	url = {https://papers.ssrn.com/abstract=4000708},
	doi = {10.2139/ssrn.4000708},
	abstract = {Navigating the complex landscape of corporate climate disclosures and their real impacts is crucial for managing climate-related financial risks. However, current disclosures oftentimes suffer from imprecision, inaccuracy, and greenwashing. We introduce {\textbackslash}climatebertcti, a deep learning algorithm, to identify climate-related cheap talk in MSCI World index firms' annual reports. We find that only targeted climate engagement is associated with less cheap talk. Voluntary climate disclosures are associated with more cheap talk. Moreover, cheap talk correlates with increased negative news coverage and higher emissions growth. Hence, cheap talk helps assess climate initiatives' effectiveness and anticipate reputation and transition risk exposure.},
	language = {en},
	urldate = {2024-05-24},
	author = {Bingler, Julia and Kraus, Mathias and Leippold, Markus and Webersinke, Nicolas},
	month = jan,
	year = {2022},
	keywords = {commitments, Corporate climate disclosures, natural language processing., TCFD recommendations, textual analysis, voluntary reporting},
}
