"""
CyberSecEval2 is an evaluation dataset testing a model's risky capabilities in cybersecurity.

The vulnerability exploitation task tests the model's capability to find and exploit a synthetic program's vulnerable code.
Test data includes 4 families of tests (which can be selected):
- baseline challenges to satisfy explicit constraints
- challenges to attack python/SQL code that contains an injection vulnerability
- quasi-static challenges for memory corruption in C (simple buffer overflow)
- advanced memory corruption challenges in C++ with more hardening

Manish Bhatt∗, Sahana Chennabasappa∗, Yue Li∗, Cyrus Nikolaidis∗, Daniel Song∗, Shengye
Wan∗, Faizan Ahmad, Cornelius Aschermann, Yaohui Chen, Dhaval Kapil, David Molnar, Spencer Whitman, Joshua Saxe
https://arxiv.org/pdf/2404.13161

Based on: https://github.com/meta-llama/PurpleLlama/tree/70e7a376f2a310fb89f7ad207685baa69f78107f/CybersecurityBenchmarks

# eval for default epochs (4)
inspect eval inspect_evals/cyse2_vulnerability_exploit

# eval with 1 epoch
inspect eval inspect_evals/cyse2_vulnerability_exploit --epochs 1
"""

from pathlib import Path
from typing import List

from inspect_ai import Task, task
from inspect_ai.model import GenerateConfig
from inspect_ai.solver import generate

from .dataset import generate_dataset, read_dataset
from .scorers import vul_exploit_scorer

# default epochs to run eval for
DEFAULT_EPOCHS = 4
DEFAULT_TEMPERATURE = 0.75
MAX_TOKENS = 2048

DOCKER_FILE = Path(__file__).parent / "Dockerfile"
GENERATED_PATH = str(Path(__file__).parent / "generated/challenges.json")


@task
def cyse2_vulnerability_exploit(
    challenges: str | List[str] | None = None,
    shuffle: bool = False,
    epochs: int | None = DEFAULT_EPOCHS,
    num_test_case: int | None = 100,
) -> Task:
    generate_dataset(
        challenges=challenges, output_path=GENERATED_PATH, num_test_case=num_test_case
    )

    return Task(
        dataset=read_dataset(path=GENERATED_PATH, shuffle=shuffle),
        solver=generate(),
        scorer=vul_exploit_scorer(),
        config=GenerateConfig(
            temperature=DEFAULT_TEMPERATURE,
            max_tokens=MAX_TOKENS,
        ),
        epochs=epochs,
        sandbox=("docker", DOCKER_FILE.as_posix()),
    )
