@misc{newcomb2019usinghumanintheloopsynthesisauthor,
      title={Using human-in-the-loop synthesis to author functional reactive programs}, 
      author={Julie L Newcomb and Rastislav Bodik},
      year={2019},
      eprint={1909.11206},
      archivePrefix={arXiv},
      primaryClass={cs.PL},
      url={https://arxiv.org/abs/1909.11206}, 
}
@InProceedings{WenCAV24,
author="Wen, Cheng
and Cao, Jialun
and Su, Jie
and Xu, Zhiwu
and Qin, Shengchao
and He, Mengda
and Li, Haokun
and Cheung, Shing-Chi
and Tian, Cong",
editor="Gurfinkel, Arie
and Ganesh, Vijay",
title="Enchanting Program Specification Synthesis by Large Language Models Using Static Analysis and Program Verification",
booktitle="Computer Aided Verification",
year="2024",
publisher="Springer Nature Switzerland",
address="Cham",
pages="302--328",
abstract="Formal verification provides a rigorous and systematic approach to ensure the correctness and reliability of software systems. Yet, constructing specifications for the full proof relies on domain expertise and non-trivial manpower. In view of such needs, an automated approach for specification synthesis is desired. While existing automated approaches are limited in their versatility, i.e., they either focus only on synthesizing loop invariants for numerical programs, or are tailored for specific types of programs or invariants. Programs involving multiple complicated data types (e.g., arrays, pointers) and code structures (e.g., nested loops, function calls) are often beyond their capabilities. To help bridge this gap, we present AutoSpec, an automated approach to synthesize specifications for automated program verification. It overcomes the shortcomings of existing work in specification versatility, synthesizing satisfiable and adequate specifications for full proof. It is driven by static analysis and program verification, and is empowered by large language models (LLMs). AutoSpec addresses the practical challenges in three ways: (1) driving AutoSpec by static analysis and program verification, LLMs serve as generators to generate candidate specifications, (2) programs are decomposed to direct the attention of LLMs, and (3) candidate specifications are validated in each round to avoid error accumulation during the interaction with LLMs. In this way, AutoSpec can incrementally and iteratively generate satisfiable and adequate specifications. The evaluation shows its effectiveness and usefulness, as it outperforms existing works by successfully verifying 79{\%} of programs through automatic specification synthesis, a significant improvement of 1.592x. It can also be successfully applied to verify the programs in a real-world X509-parser project.",
isbn="978-3-031-65630-9"
}


@inproceedings{WuASE24,
author = {Wu, Guangyuan and Cao, Weining and Yao, Yuan and Wei, Hengfeng and Chen, Taolue and Ma, Xiaoxing},
title = {LLM Meets Bounded Model Checking: Neuro-symbolic Loop Invariant Inference},
year = {2024},
isbn = {9798400712487},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3691620.3695014},
doi = {10.1145/3691620.3695014},
abstract = {Loop invariant inference, a key component in program verification, is a challenging task due to the inherent undecidability and complex loop behaviors in practice. Recently, machine learning based techniques have demonstrated impressive performance in generating loop invariants automatically. However, these methods highly rely on the labeled training data, and are intrinsically random and uncertain, leading to unstable performance. In this paper, we investigate a synergy of large language models (LLMs) and bounded model checking (BMC) to address these issues. The key observation is that, although LLMs may not be able to return the correct loop invariant in one response, they usually can provide all individual predicates of the correct loop invariant in multiple responses. To this end, we propose a "query-filter-reassemble" strategy, namely, we first leverage the language generation power of LLMs to produce a set of candidate invariants, where training data is not needed. Then, we employ BMC to identify valid predicates from these candidate invariants, which are assembled to produce new candidate invariants and checked by off-the-shelf SMT solvers. The feedback is incorporated into the prompt for the next round of LLM querying. We expand the existing benchmark of 133 programs to 316 programs, providing a more comprehensive testing ground. Experimental results demonstrate that our approach significantly outperforms the state-of-the-art techniques, successfully generating 309 loop invariants out of 316 cases, whereas the existing baseline methods are only able to tackle 219 programs at best. The code is publicly available at https://github.com/SoftWiser-group/LaM4Inv.git.},
booktitle = {Proceedings of the 39th IEEE/ACM International Conference on Automated Software Engineering},
pages = {406–417},
numpages = {12},
keywords = {loop invariant, program verification, large language model},
location = {Sacramento, CA, USA},
series = {ASE '24}
}

@InProceedings{llvm,
  author={Lattner, C. and Adve, V.},
  booktitle={International Symposium on Code Generation and Optimization, 2004. CGO 2004.}, 
  title={LLVM: a compilation framework for lifelong program analysis \& transformation}, 
  year={2004},
  volume={},
  number={},
  pages={75-86},
  keywords={Information analysis;Program processors;Performance analysis;High level languages;Virtual machining;Runtime;Arithmetic;Application software;Software safety;Algorithm design and analysis},
  doi={10.1109/CGO.2004.1281665}}

@inproceedings{codemosa-icse23,
author = {Lemieux, Caroline and Inala, Jeevana Priya and Lahiri, Shuvendu K. and Sen, Siddhartha},
title = {CodaMosa: Escaping Coverage Plateaus in Test Generation with Pre-Trained Large Language Models},
year = {2023},
isbn = {9781665457019},
publisher = {IEEE Press},
url = {https://doi.org/10.1109/ICSE48619.2023.00085},
doi = {10.1109/ICSE48619.2023.00085},
abstract = {Search-based software testing (SBST) generates high-coverage test cases for programs under test with a combination of test case generation and mutation. SBST's performance relies on there being a reasonable probability of generating test cases that exercise the core logic of the program under test. Given such test cases, SBST can then explore the space around them to exercise various parts of the program. This paper explores whether Large Language Models (LLMs) of code, such as OpenAI's Codex, can be used to help SBST's exploration. Our proposed algorithm, CodaMosa, conducts SBST until its coverage improvements stall, then asks Codex to provide example test cases for under-covered functions. These examples help SBST redirect its search to more useful areas of the search space. On an evaluation over 486 benchmarks, CodaMosa achieves statistically significantly higher coverage on many more benchmarks (173 and 279) than it reduces coverage on (10 and 4), compared to SBST and LLM-only baselines.},
booktitle = {Proceedings of the 45th International Conference on Software Engineering},
pages = {919–931},
numpages = {13},
location = {Melbourne, Victoria, Australia},
series = {ICSE '23}
}

@InProceedings{Tillmann2006,
author="Tillmann, Nikolai
and Chen, Feng
and Schulte, Wolfram",
editor="Liu, Zhiming
and He, Jifeng",
title="Discovering Likely Method Specifications",
booktitle="Formal Methods and Software Engineering",
year="2006",
publisher="Springer Berlin Heidelberg",
address="Berlin, Heidelberg",
pages="717--736",
abstract="Software specifications are of great use for more rigorous software development. They are useful for formal verification and automated testing, and they improve program understanding. In practice, specifications often do not exist and developers write software in an ad-hoc fashion. We describe a new way to automatically infer specifications from code. Our approach infers a likely specification for any method such that the method's behavior, i.e., its effect on the state and possible result values, is summarized and expressed in terms of some other methods. We use symbolic execution to analyze and relate the behaviors of the considered methods. In our experiences, the resulting likely specifications are compact and human-understandable. They can be examined by the user, used as input to program verification systems, or as input for test generation tools for validation. We implemented the technique for .NET programs in a tool called Axiom Meister. It inferred concise specifications for base classes of the .NET platform and found flaws in the design of a new library.",
isbn="978-3-540-47462-3"
}


@software{treesitter,
  author       = {Max Brunsfeld and
                  Amaan Qureshi and
                  Andrew Hlynskyi and
                  Patrick Thomson and
                  ObserverOfTime and
                  Josh Vera and
                  dundargoc and
                  Phil Turnbull and
                  Timothy Clem and
                  Douglas Creager and
                  Andrew Helwer and
                  Rob Rix and
                  Daumantas Kavolis and
                  Hendrik van Antwerpen and
                  Michael Davis and
                  Will Lillis and
                  Ika and
                  Amin Yahyaabadi and
                  Tu{\`a}n-Anh Nguy{\~e}n and
                  bfredl and
                  Matt Massicotte and
                  Stafford Brunk and
                  Christian Clason and
                  Niranjan Hasabnis and
                  Mingkai Dong and
                  Samuel Moelius and
                  Steven Kalt and
                  Segev Finer and
                  Kolja},
  title        = {tree-sitter/tree-sitter: v0.24.4},
  month        = nov,
  year         = 2024,
  publisher    = {Zenodo},
  version      = {v0.24.4},
  doi          = {10.5281/zenodo.14061403},
  url          = {https://doi.org/10.5281/zenodo.14061403}
}

@inproceedings{z3,
author = {de Moura, Leonardo and Bjørner, Nikolaj},
title = {Z3: an efficient SMT solver},
booktitle = {2008 Tools and Algorithms for Construction and Analysis of Systems},
year = {2008},
month = {March},
abstract = {Satisfiability Modulo Theories (SMT) problem is a decision problem for logical first order formulas with respect to combinations of background theories such as: arithmetic, bit-vectors, arrays, and uninterpreted functions. Z3 is a new and efficient SMT Solver freely available from Microsoft Research. It is used in various software verification and analysis applications.},
publisher = {Springer, Berlin, Heidelberg},
url = {https://www.microsoft.com/en-us/research/publication/z3-an-efficient-smt-solver/},
pages = {337-340},
}
@misc{z3util,
    title = {z3Prover util classes},
    author = {z3Prover},
    year = {2024},
    howpublished = {\url{https://github.com/Z3Prover/z3/tree/master/src/util}}
}

@misc{cpp_ds,
  author       = {Adamos Dimitrios},
  title        = {Algorithms and Data Structures},
  year         = 2023,
  url          = {https://github.com/djeada/Algorithms-And-Data-Structures/tree/master/src/collections_and_containers/cpp},
}


@misc{mutatecpp,
  author = {Niels Lohmann and other contributors},
  title = {mutate\_cpp - Mutation Testing Tool for C++},
  year = {2017},
  howpublished = {\url{https://github.com/nlohmann/mutate_cpp}},
}


@inproceedings{dysy,
author = {Csallner, Christoph and Tillmann, Nikolai and Smaragdakis, Yannis},
title = {DySy: dynamic symbolic execution for invariant inference},
year = {2008},
isbn = {9781605580791},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/1368088.1368127},
doi = {10.1145/1368088.1368127},
abstract = {Dynamically discovering likely program invariants from concrete test executions has emerged as a highly promising software engineering technique. Dynamic invariant inference has the advantage of succinctly summarizing both "expected" program inputs and the subset of program behaviors that is normal under those inputs. In this paper, we introduce a technique that can drastically increase the relevance of inferred invariants, or reduce the size of the test suite required to obtain good invariants. Instead of falsifying invariants produced by pre-set patterns, we determine likely program invariants by combining the concrete execution of actual test cases with a simultaneous symbolic execution of the same tests. The symbolic execution produces abstract conditions over program variables that the concrete tests satisfy during their execution. In this way, we obtain the benefits of dynamic inference tools like Daikon: the inferred invariants correspond to the observed program behaviors. At the same time, however, our inferred invariants are much more suited to the program at hand than Daikon's hard-coded invariant patterns. The symbolic invariants are literally derived from the program text itself, with appropriate value substitutions as dictated by symbolic execution.We implemented our technique in the DySy tool, which utilizes a powerful symbolic execution and simplification engine. The results confirm the benefits of our approach. In Daikon's prime example benchmark, we infer the majority of the interesting Daikon invariants, while eliminating invariants that a human user is likely to consider irrelevant.},
booktitle = {Proceedings of the 30th International Conference on Software Engineering},
pages = {281–290},
numpages = {10},
keywords = {symbolic reasoning, pex, dysy, dynamic invariant inference, daikon},
location = {Leipzig, Germany},
series = {ICSE '08}
}

@inproceedings{klee,
author = {Cadar, Cristian and Dunbar, Daniel and Engler, Dawson},
title = {KLEE: unassisted and automatic generation of high-coverage tests for complex systems programs},
year = {2008},
publisher = {USENIX Association},
address = {USA},
abstract = {We present a new symbolic execution tool, KLEE, capable of automatically generating tests that achieve high coverage on a diverse set of complex and environmentally-intensive programs. We used KLEE to thoroughly check all 89 stand-alone programs in the GNU COREUTILS utility suite, which form the core user-level environment installed on millions of Unix systems, and arguably are the single most heavily tested set of open-source programs in existence. KLEE-generated tests achieve high line coverage -- on average over 90\% per tool (median: over 94\%) -- and significantly beat the coverage of the developers' own hand-written test suite. When we did the same for 75 equivalent tools in the BUSYBOX embedded system suite, results were even better, including 100\% coverage on 31 of them.We also used KLEE as a bug finding tool, applying it to 452 applications (over 430K total lines of code), where it found 56 serious bugs, including three in COREUTILS that had been missed for over 15 years. Finally, we used KLEE to crosscheck purportedly identical BUSYBOX and COREUTILS utilities, finding functional correctness errors and a myriad of inconsistencies.},
booktitle = {Proceedings of the 8th USENIX Conference on Operating Systems Design and Implementation},
pages = {209–224},
numpages = {16},
location = {San Diego, California},
series = {OSDI'08}
}


@InProceedings{bernhard,
author="Beckert, Bernhard
and Klamroth, Jonas
and Pfeifer, Wolfram
and R{\"o}per, Patrick
and Teuber, Samuel",
editor="Margaria, Tiziana
and Steffen, Bernhard",
title="Towards Combining the Cognitive Abilities of Large Language Models with the Rigor of Deductive Progam Verification",
booktitle="Leveraging Applications of Formal Methods, Verification and Validation. Software Engineering Methodologies",
year="2025",
publisher="Springer Nature Switzerland",
address="Cham",
pages="242--257",
abstract="Recent investigations hint at the ability of large language models (LLMs) to generate formal specifications for given program code. In this work, we systematically discuss and categorize different use cases and application scenarios that combine specification synthesis via LLMs with deductive program verification. We present preliminary quantitative experiments on the capabilities of LLMs to generate correct specifications. To this end, we use a prototypical integration of GPT (versions 3.5 and 4o) with the deductive program verifier KeY and the bounded model checker JJBMC. We evaluated our prototype on a set of Java programs that are partially annotated with specifications written in the Java Modeling Language (JML). We show that GPT 4o generates correct annotations in approximately half of all instances across the investigated scenarios. For the case of faulty specifications, we investigate how a feedback loop can help to improve the original answer. Finally, we present a vision of how Large Language Models may support rigorous formal verification of software systems and describe the necessary next steps in this direction.",
isbn="978-3-031-75387-9"
}



@book{Aho:72,
    author  = {Alfred V. Aho and Jeffrey D. Ullman},
    title   = {The Theory of Parsing, Translation and Compiling},
    year    = "1972",
    volume  = "1",
    publisher = {Prentice-Hall},
    address = {Englewood Cliffs, NJ}
}

@article{ernst2007daikon,
   author = {Michael D. Ernst and Jeff H. Perkins and Philip J. Guo and
	Stephen McCamant and Carlos Pacheco and Matthew S. Tschantz and
	Chen Xiao},
   title = {The {Daikon} system for dynamic detection of likely invariants},
   journal = {Science of Computer Programming},
   volume = {69},
   number = {1--3},
   pages = {35--45},
   month = dec,
   year = {2007}
}

@INPROCEEDINGS{nguyen2012dig,
  author={Nguyen, ThanhVu and Kapur, Deepak and Weimer, Westley and Forrest, Stephanie},
  booktitle={2012 34th International Conference on Software Engineering (ICSE)}, 
  title={Using dynamic analysis to discover polynomial and array invariants}, 
  year={2012},
  volume={},
  number={},
  pages={683-693},
  keywords={Arrays;Polynomials;Performance analysis;Complexity theory;Educational institutions;program analysis;dynamic analysis;invariant generation;nonlinear invariants;array invariants},
  doi={10.1109/ICSE.2012.6227149}}

@inproceedings{le2019sling,
author = {Le, Ton Chanh and Zheng, Guolong and Nguyen, ThanhVu},
title = {SLING: using dynamic analysis to infer program invariants in separation logic},
year = {2019},
isbn = {9781450367127},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3314221.3314634},
doi = {10.1145/3314221.3314634},
abstract = {We introduce a new dynamic analysis technique to discover invariants in separation logic for heap-manipulating programs. First, we use a debugger to obtain rich program execution traces at locations of interest on sample inputs. These traces consist of heap and stack information of variables that point to dynamically allocated data structures. Next, we iteratively analyze separate memory regions related to each pointer variable and search for a formula over predefined heap predicates in separation logic to model these regions. Finally, we combine the computed formulae into an invariant that describes the shape of explored memory regions.  We present SLING, a tool that implements these ideas to automatically generate invariants in separation logic at arbitrary locations in C programs, e.g., program pre and postconditions and loop invariants. Preliminary results on existing benchmarks show that SLING can efficiently generate correct and useful invariants for programs that manipulate a wide variety of complex data structures.},
booktitle = {Proceedings of the 40th ACM SIGPLAN Conference on Programming Language Design and Implementation},
pages = {788–801},
numpages = {14},
keywords = {separation logic, dynamic invariant analysis},
location = {Phoenix, AZ, USA},
series = {PLDI 2019}
}

@inproceedings{ammons2002,
author = {Ammons, Glenn and Bod\'{\i}k, Rastislav and Larus, James R.},
title = {Mining specifications},
year = {2002},
isbn = {1581134509},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/503272.503275},
doi = {10.1145/503272.503275},
abstract = {Program verification is a promising approach to improving program quality, because it can search all possible program executions for specific errors. However, the need to formally describe correct behavior or errors is a major barrier to the widespread adoption of program verification, since programmers historically have been reluctant to write formal specifications. Automating the process of formulating specifications would remove a barrier to program verification and enhance its practicality.This paper describes specification mining, a machine learning approach to discovering formal specifications of the protocols that code must obey when interacting with an application program interface or abstract data type. Starting from the assumption that a working program is well enough debugged to reveal strong hints of correct protocols, our tool infers a specification by observing program execution and concisely summarizing the frequent interaction patterns as state machines that capture both temporal and data dependences. These state machines can be examined by a programmer, to refine the specification and identify errors, and can be utilized by automatic verification tools, to find bugs.Our preliminary experience with the mining tool has been promising. We were able to learn specifications that not only captured the correct protocol, but also discovered serious bugs.},
booktitle = {Proceedings of the 29th ACM SIGPLAN-SIGACT Symposium on Principles of Programming Languages},
pages = {4–16},
numpages = {13},
location = {Portland, Oregon},
series = {POPL '02}
}

@InProceedings{mcmillan2004interpolants,
author="McMillan, K. L.",
editor="Jensen, Kurt
and Podelski, Andreas",
title="An Interpolating Theorem Prover",
booktitle="Tools and Algorithms for the Construction and Analysis of Systems",
year="2004",
publisher="Springer Berlin Heidelberg",
address="Berlin, Heidelberg",
pages="16--30",
abstract="We present a method of deriving Craig interpolants from proofs in the quantifier-free theory of linear inequality and uninterpreted function symbols, and an interpolating theorem prover based on this method. The prover has been used for predicate refinement in the Blast software model checker, and can also be used directly for model checking infinite-state systems, using interpolation-based image approximation.",
isbn="978-3-540-24730-2"
}

@InProceedings{garg2014learning,
author="Garg, Pranav
and L{\"o}ding, Christof
and Madhusudan, P.
and Neider, Daniel",
editor="Biere, Armin
and Bloem, Roderick",
title="ICE: A Robust Framework for Learning Invariants",
booktitle="Computer Aided Verification",
year="2014",
publisher="Springer International Publishing",
address="Cham",
pages="69--87",
abstract="We introduce ICE, a robust learning paradigm for synthesizing invariants, that learns using examples, counter-examples, and implications, and show that it admits honest teachers and strongly convergent mechanisms for invariant synthesis. We observe that existing algorithms for black-box abstract interpretation can be interpreted as ICE-learning algorithms. We develop new strongly convergent ICE-learning algorithms for two domains, one for learning Boolean combinations of numerical invariants for scalar variables and one for quantified invariants for arrays and dynamic lists. We implement these ICE-learning algorithms in a verification tool and show they are robust, practical, and efficient.",
isbn="978-3-319-08867-9"
}

@inproceedings{garg2016learning,
author = {Garg, Pranav and Neider, Daniel and Madhusudan, P. and Roth, Dan},
title = {Learning invariants using decision trees and implication counterexamples},
year = {2016},
isbn = {9781450335492},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/2837614.2837664},
doi = {10.1145/2837614.2837664},
abstract = {Inductive invariants can be robustly synthesized using a learning model where the teacher is a program verifier who instructs the learner through concrete program configurations, classified as positive, negative, and implications. We propose the first learning algorithms in this model with implication counter-examples that are based on machine learning techniques. In particular, we extend classical decision-tree learning algorithms in machine learning to handle implication samples, building new scalable ways to construct small decision trees using statistical measures. We also develop a decision-tree learning algorithm in this model that is guaranteed to converge to the right concept (invariant) if one exists. We implement the learners and an appropriate teacher, and show that the resulting invariant synthesis is efficient and convergent for a large suite of programs.},
booktitle = {Proceedings of the 43rd Annual ACM SIGPLAN-SIGACT Symposium on Principles of Programming Languages},
pages = {499–512},
numpages = {14},
keywords = {machine learning, decision trees, Invariant synthesis, ICE learning},
location = {St. Petersburg, FL, USA},
series = {POPL '16}
}

@inproceedings{padhi2016loopinvgen,
  author    = {Saswat Padhi and Rahul Sharma and Todd D. Millstein},
  title     = {Data-Driven Precondition Inference with Learned Features},
  booktitle = {Proceedings of the 37th {ACM} {SIGPLAN} Conference on Programming
               Language Design and Implementation, {PLDI} 2016, Santa Barbara, CA,
               USA, June 13-17, 2016},
  pages     = {42--56},
  year      = {2016},
  url       = {http://doi.acm.org/10.1145/2908080.2908099},
  doi       = {10.1145/2908080.2908099}
}


@InProceedings{pei2023learning,
  title = 	 {Can Large Language Models Reason about Program Invariants?},
  author =       {Pei, Kexin and Bieber, David and Shi, Kensen and Sutton, Charles and Yin, Pengcheng},
  booktitle = 	 {Proceedings of the 40th International Conference on Machine Learning},
  pages = 	 {27496--27520},
  year = 	 {2023},
  editor = 	 {Krause, Andreas and Brunskill, Emma and Cho, Kyunghyun and Engelhardt, Barbara and Sabato, Sivan and Scarlett, Jonathan},
  volume = 	 {202},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {23--29 Jul},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v202/pei23a/pei23a.pdf},
  url = 	 {https://proceedings.mlr.press/v202/pei23a.html},
  abstract = 	 {Identifying invariants is an important program analysis task with applications towards program understanding, bug finding, vulnerability analysis, and formal verification. Existing tools for identifying program invariants rely on dynamic analysis, requiring traces collected from multiple executions in order to produce reliable invariants. We study the application of large language models to invariant prediction, finding that models trained on source code and fine-tuned for invariant generation can perform invariant prediction as static rather than dynamic analysis. Using a scratchpad approach where invariants are predicted sequentially through a program gives the best performance, finding invariants statically of quality comparable to those obtained by a dynamic analysis tool with access to five program traces.}
}

@article{ma2024specgen,
  author       = {Lezhi Ma and
                  Shangqing Liu and
                  Yi Li and
                  Xiaofei Xie and
                  Lei Bu},
  title        = {SpecGen: Automated Generation of Formal Program Specifications via
                  Large Language Models},
  journal      = {CoRR},
  volume       = {abs/2401.08807},
  year         = {2024},
  url          = {https://doi.org/10.48550/arXiv.2401.08807},
  doi          = {10.48550/ARXIV.2401.08807},
  eprinttype    = {arXiv},
  eprint       = {2401.08807},
  timestamp    = {Sun, 04 Aug 2024 16:26:32 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2401-08807.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}


@article{nl2postcond,
author = {Endres, Madeline and Fakhoury, Sarah and Chakraborty, Saikat and Lahiri, Shuvendu K.},
title = {Can Large Language Models Transform Natural Language Intent into Formal Method Postconditions?},
year = {2024},
issue_date = {July 2024},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {1},
number = {FSE},
url = {https://doi.org/10.1145/3660791},
doi = {10.1145/3660791},
abstract = {Informal natural language that describes code functionality, such as code comments or function documentation, may contain substantial information about a program’s intent. However, there is typically no guarantee that a program’s implementation and natural language documentation are aligned. In the case of a conflict, leveraging information in code-adjacent natural language has the potential to enhance fault localization, debugging, and code trustworthiness. In practice, however, this information is often underutilized due to the inherent ambiguity of natural language, which makes natural language intent challenging to check programmatically. The “emergent abilities” of Large Language Models (LLMs) have the potential to facilitate the translation of natural language intent to programmatically checkable assertions. However, it is unclear if LLMs can correctly translate informal natural language specifications into formal specifications that match programmer intent. Additionally, it is unclear if such translation could be useful in practice.     In this paper, we describe nl2postcondition, the problem of leveraging LLMs for transforming informal natural language to formal method postconditions, expressed as program assertions.   We introduce and validate metrics to measure and compare different nl2postcondition approaches, using the correctness and discriminative power of generated postconditions.   We then use qualitative and quantitative methods to assess the quality of nl2postcondition postconditions, finding that they are generally correct and able to discriminate incorrect code. Finally, we find that  via LLMs has the potential to be helpful in practice;  generated postconditions were able to catch 64 real-world historical bugs from Defects4J.},
journal = {Proc. ACM Softw. Eng.},
month = jul,
articleno = {84},
numpages = {24},
keywords = {Formal Specifications, Large Language Models, Postconditions}
}


@misc{loopy,
      title={Finding Inductive Loop Invariants using Large Language Models}, 
      author={Adharsh Kamath and Aditya Senthilnathan and Saikat Chakraborty and Pantazis Deligiannis and Shuvendu K. Lahiri and Akash Lal and Aseem Rastogi and Subhajit Roy and Rahul Sharma},
      year={2023},
      eprint={2311.07948},
      archivePrefix={arXiv},
      primaryClass={cs.PL},
      url={https://arxiv.org/abs/2311.07948}, 
}

@inproceedings{lahiri2024evaluating,
author = {Lahiri, Shuvendu},
title = {Evaluating LLM-driven User-Intent Formalization for Verification-Aware Languages},
booktitle = {Formal Methods in Computer-Aided Design (FMCAD'24)},
year = {2024},
month = {July},
abstract = {Verification-aware programming languages such as Dafny and F* provide means to formally specify and prove properties of programs. Although the problem of checking an implementation against a specification can be defined mechanically, there is no algorithmic way of ensuring the correctness of the user-intent formalization for programs -- that a specification adheres to the user's intent behind the program. The intent or requirement is expressed informally in natural language and the specification is a formal artefact. The advent of large language models (LLMs) has made strides bridging the gap between informal intent and formal program implementations recently, driven in large parts due to benchmarks and automated metrics for evaluation.
Recent work has proposed evaluating [\it user-intent formalization] problem for mainstream programming languages~\cite[endres-fse24]. However, such an approach does not readily extend to verification-aware languages that support rich specifications (containing quantifiers and ghost variables) that cannot be evaluated through dynamic execution. Previous work also required generating program mutants using LLMs to create the benchmark. We advocate an alternate approach of [\it symbolically testing specifications] to provide an intuitive metric for evaluating the quality of specifications for verification-aware languages. We demonstrate that our automated metric agrees closely with mostly GPT-4 generated and human-labeled dataset of roughly 150 Dafny specifications for the popular MBPP code-generation benchmark, yet demonstrates cases where the human labeling is not perfect. We believe our work provides a stepping stone to enable the establishment of a benchmark and research agenda for the problem of user-intent formalization for programs.},
url = {https://www.microsoft.com/en-us/research/publication/evaluating-llm-driven-user-intent-formalization-for-verification-aware-languages/},
}

@article{yang2023kernelgpt,
  title={Kernelgpt: Enhanced kernel fuzzing via large language models},
  author={Yang, Chenyuan and Zhao, Zijie and Zhang, Lingming},
  journal={arXiv preprint arXiv:2401.00563},
  year={2023}
}

@article{zhang2024codeagent,
  title={Codeagent: Enhancing code generation with tool-integrated agent systems for real-world repo-level coding challenges},
  author={Zhang, Kechi and Li, Jia and Li, Ge and Shi, Xianjie and Jin, Zhi},
  journal={arXiv preprint arXiv:2401.07339},
  year={2024}
}

@article{yang2024whitefox,
  title={WhiteFox: White-Box Compiler Fuzzing Empowered by Large Language Models},
  author={Yang, Chenyuan and Deng, Yinlin and Lu, Runyu and Yao, Jiayi and Liu, Jiawei and Jabbarvand, Reyhaneh and Zhang, Lingming},
  journal={Proceedings of the ACM on Programming Languages},
  volume={8},
  number={OOPSLA2},
  pages={709--735},
  year={2024},
  publisher={ACM New York, NY, USA}
}

@article{gao2023retrieval,
  title={Retrieval-augmented generation for large language models: A survey},
  author={Gao, Yunfan and Xiong, Yun and Gao, Xinyu and Jia, Kangxiang and Pan, Jinliu and Bi, Yuxi and Dai, Yi and Sun, Jiawei and Wang, Meng and Wang, Haofen},
  journal={arXiv preprint arXiv:2312.10997},
  year={2023}
}

@article{liu2024lost,
  title={Lost in the middle: How language models use long contexts},
  author={Liu, Nelson F and Lin, Kevin and Hewitt, John and Paranjape, Ashwin and Bevilacqua, Michele and Petroni, Fabio and Liang, Percy},
  journal={Transactions of the Association for Computational Linguistics},
  volume={12},
  pages={157--173},
  year={2024},
  publisher={MIT Press One Broadway, 12th Floor, Cambridge, Massachusetts 02142, USA~…}
}
@InProceedings{boockmann2024geminius,
author="Boockmann, Jan H.
and L{\"u}ttgen, Gerald",
editor="Beyer, Dirk
and Cavalcanti, Ana",
title="Comprehending Object State via Dynamic Class Invariant Learning",
booktitle="Fundamental Approaches to Software Engineering",
year="2024",
publisher="Springer Nature Switzerland",
address="Cham",
pages="143--164",
abstract="Maintaining software is cumbersome when method argument constraints are undocumented. To reveal them, previous work learned preconditions from exemplary valid and invalid method arguments. In practice, it would be highly beneficial to know class invariants, too, because functionality added during software maintenance must not break them. Even more so than method preconditions, class invariants are rarely documented and often cannot completely be inferred automatically, especially for objects exhibiting complex state such as dynamic data structures.",
isbn="978-3-031-57259-3"
}

@inproceedings{facundo2022specfuzzer,
author = {Molina, Facundo and d'Amorim, Marcelo and Aguirre, Nazareno},
title = {Fuzzing class specifications},
year = {2022},
isbn = {9781450392211},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3510003.3510120},
doi = {10.1145/3510003.3510120},
abstract = {Expressing class specifications via executable constraints is important for various software engineering tasks such as test generation, bug finding and automated debugging, but developers rarely write them. Techniques that infer specifications from code exist to fill this gap, but they are designed to support specific kinds of assertions and are difficult to adapt to support different assertion languages, e.g., to add support for quantification, or additional comparison operators, such as membership or containment.To address the above issue, we present SpecFuzzer, a novel technique that combines grammar-based fuzzing, dynamic invariant detection, and mutation analysis, to automatically produce class specifications. SpecFuzzer uses: (i) a fuzzer as a generator of candidate assertions derived from a grammar that is automatically obtained from the class definition; (ii) a dynamic invariant detector -Daikon- to filter out assertions invalidated by a test suite; and (iii) a mutation-based mechanism to cluster and rank assertions, so that similar constraints are grouped and then the stronger prioritized. Grammar-based fuzzing enables SpecFuzzer to be straightforwardly adapted to support different specification languages, by manipulating the fuzzing grammar, e.g., to include additional operators.We evaluate our technique on a benchmark of 43 Java methods employed in the evaluation of the state-of-the-art techniques GAssert and EvoSpex. Our results show that SpecFuzzer can easily support a more expressive assertion language, over which is more effective than GAssert and EvoSpex in inferring specifications, according to standard performance metrics.},
booktitle = {Proceedings of the 44th International Conference on Software Engineering},
pages = {1008–1020},
numpages = {13},
keywords = {Oracle problem, grammar-based fuzzing, specification inference},
location = {Pittsburgh, Pennsylvania},
series = {ICSE '22}
}

@article{hellendoorn2019are,
author = {Hellendoorn, Vincent J. and Devanbu, Premkumar T. and Polozov, Alex and Marron, Mark},
title = {Are My Invariants Valid? A Learning Approach},
year = {2019},
month = {March},
abstract = {Ensuring that a program operates correctly is a difficult task in large, complex systems. Enshrining invariants – desired properties of correct execution – in code or comments can support maintainability and help sustain correctness. Tools that can automatically infer and recommend invariants can thus be very beneficial. However, current invariant-suggesting tools, such as Daikon, suffer from high rates of false positives, in part because they only leverage traced program values from available test cases, rather than directly exploiting knowledge of the source code per se. We propose a machine-learning approach to judging the validity of invariants, specifically of method pre- and post-conditions, based directly on a method’s source code. We introduce a new, scalable approach to creating labeled invariants: using programs with large test-suites, we generate Daikon invariants using traces from subsets of these test-suites, and then label these as valid/invalid by cross-validating them with held-out tests. This process induces a large set of labels that provide a form of noisy supervision, which is then used to train a deep neural model, based on gated graph neural networks. Our model learns to map the lexical, syntactic, and semantic structure of a given method’s body into a probability that a candidate pre- or post-condition on that method’s body is correct and is able to accurately label invariants based on the noisy signal, even in cross-project settings. Most importantly, it performs well on a hand-curated dataset of invariants.},
url = {https://www.microsoft.com/en-us/research/publication/are-my-invariants-valid-a-learning-approach/},
journal = {arXiv preprint},
}



@article{meyer1992eiffel,
  title={The eiffel programming language},
  author={Meyer, Bertrand},
  journal={See http://www. eiffel. com},
  year={1992}
}

@article{jml,
  title={An overview of JML tools and applications},
  author={Burdy, Lilian and Cheon, Yoonsik and Cok, David R and Ernst, Michael D and Kiniry, Joseph R and Leavens, Gary T and Leino, K Rustan M and Poll, Erik},
  journal={International journal on software tools for technology transfer},
  volume={7},
  pages={212--232},
  year={2005},
  publisher={Springer}
}

@inproceedings{fahndrich2010static,
  title={Static verification for code contracts},
  author={F{\"a}hndrich, Manuel},
  booktitle={Static Analysis: 17th International Symposium, SAS 2010, Perpignan, France, September 14-16, 2010. Proceedings 17},
  pages={2--5},
  year={2010},
  organization={Springer}
}

@inproceedings{leino2010dafny,
  title={Dafny: An automatic program verifier for functional correctness},
  author={Leino, K Rustan M},
  booktitle={International conference on logic for programming artificial intelligence and reasoning},
  pages={348--370},
  year={2010},
  organization={Springer}
}

@article{verus,
  title={Verus: Verifying rust programs using linear ghost types},
  author={Lattuada, Andrea and Hance, Travis and Cho, Chanhee and Brun, Matthias and Subasinghe, Isitha and Zhou, Yi and Howell, Jon and Parno, Bryan and Hawblitzel, Chris},
  journal={Proceedings of the ACM on Programming Languages},
  volume={7},
  number={OOPSLA1},
  pages={286--315},
  year={2023},
  publisher={ACM New York, NY, USA}
}

@article{swamy2011secure,
  title={Secure distributed programming with value-dependent types},
  author={Swamy, Nikhil and Chen, Juan and Fournet, C{\'e}dric and Strub, Pierre-Yves and Bhargavan, Karthikeyan and Yang, Jean},
  journal={ACM SIGPLAN Notices},
  volume={46},
  number={9},
  pages={266--278},
  year={2011},
  publisher={ACM New York, NY, USA}
}

@article{kirchner2015frama,
  title={Frama-C: A software analysis perspective},
  author={Kirchner, Florent and Kosmatov, Nikolai and Prevosto, Virgile and Signoles, Julien and Yakobowski, Boris},
  journal={Formal aspects of computing},
  volume={27},
  number={3},
  pages={573--609},
  year={2015},
  publisher={Springer}
}


@inproceedings{cousot1977abstract,
  title={Abstract interpretation: a unified lattice model for static analysis of programs by construction or approximation of fixpoints},
  author={Cousot, Patrick and Cousot, Radhia},
  booktitle={Proceedings of the 4th ACM SIGACT-SIGPLAN symposium on Principles of programming languages},
  pages={238--252},
  year={1977}
}

@inproceedings{henzinger2004abstractions,
  title={Abstractions from proofs},
  author={Henzinger, Thomas A and Jhala, Ranjit and Majumdar, Rupak and McMillan, Kenneth L},
  booktitle={Proceedings of the 31st ACM SIGPLAN-SIGACT symposium on Principles of programming languages},
  pages={232--244},
  year={2004}
}

@inproceedings{polikarpova2009comparative,
  title={A comparative study of programmer-written and automatically inferred contracts},
  author={Polikarpova, Nadia and Ciupa, Ilinca and Meyer, Bertrand},
  booktitle={Proceedings of the eighteenth international symposium on Software testing and analysis},
  pages={93--104},
  year={2009}
}


@inproceedings{greiner2024automated,
  title={Automated Generation of Code Contracts: Generative AI to the Rescue?},
  author={Greiner, Sandra and B{\"u}hlmann, Noah and Ohrndorf, Manuel and Tsigkanos, Christos and Nierstrasz, Oscar and Kehrer, Timo},
  booktitle={Proceedings of the 23rd ACM SIGPLAN International Conference on Generative Programming: Concepts and Experiences},
  pages={1--14},
  year={2024}
}

@misc{avltree,
author = "{Wikipedia contributors}",
title = "AVL tree --- {Wikipedia}{,} The Free Encyclopedia",
year = "2024",
url = "https://en.wikipedia.org/w/index.php?title=AVL_tree&oldid=1255561501",
note = "[Online; accessed 15-November-2024]"
}

@InProceedings{lahiri-cav09,
author="Lahiri, Shuvendu K.
and Qadeer, Shaz
and Galeotti, Juan P.
and Voung, Jan W.
and Wies, Thomas",
editor="Bouajjani, Ahmed
and Maler, Oded",
title="Intra-module Inference",
booktitle="Computer Aided Verification",
year="2009",
publisher="Springer Berlin Heidelberg",
address="Berlin, Heidelberg",
pages="493--508",
isbn="978-3-642-02658-4"
}

@misc{schäfer2023empiricalevaluationusinglarge,
      title={An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation}, 
      author={Max Schäfer and Sarah Nadi and Aryaz Eghbali and Frank Tip},
      year={2023},
      eprint={2302.06527},
      archivePrefix={arXiv},
      primaryClass={cs.SE},
      url={https://arxiv.org/abs/2302.06527}, 
}

@inproceedings{codamosa-icse23,
author = {Lemieux, Caroline and Inala, Jeevana Priya and Lahiri, Shuvendu K. and Sen, Siddhartha},
title = {CodaMosa: Escaping Coverage Plateaus in Test Generation with Pre-Trained Large Language Models},
year = {2023},
isbn = {9781665457019},
publisher = {IEEE Press},
url = {https://doi.org/10.1109/ICSE48619.2023.00085},
doi = {10.1109/ICSE48619.2023.00085},
booktitle = {Proceedings of the 45th International Conference on Software Engineering},
pages = {919–931},
numpages = {13},
location = {Melbourne, Victoria, Australia},
series = {ICSE '23}
}
