from pathlib import Path

import pytest
import yaml

from inspect_evals.metadata import EvalListing, _load_eval_listing, load_listing


@pytest.fixture
def listing():
    """Fixture to load the evaluation listing once for all tests."""
    return load_listing()


@pytest.fixture
def raw_listing_data():
    """Fixture to load the raw listing.yaml data."""
    listing_path = (
        Path(__file__).parent.parent / "src" / "inspect_evals" / "listing.yaml"
    )
    with open(listing_path, "r") as f:
        return yaml.safe_load(f)


def test_load_listing(listing):
    """Test that the listing.yaml can be loaded correctly."""
    assert isinstance(listing, EvalListing)
    assert len(listing.evals) > 0


def test_eval_metadata_parsing(listing, raw_listing_data):
    """Test that EvalMetadata model can parse entries correctly."""
    # Test every eval in the listing
    for raw_eval in raw_listing_data:
        eval_id = raw_eval["path"].split("/")[-1]

        # Get the parsed version through the model
        model_eval = listing.get_eval(eval_id)
        assert model_eval is not None, f"Could not find {eval_id} in parsed listing"

        # Check all fields match
        assert model_eval.title == raw_eval["title"], f"Title mismatch for {eval_id}"
        assert model_eval.description == raw_eval["description"], (
            f"Description mismatch for {eval_id}"
        )
        assert model_eval.path == raw_eval["path"], f"Path mismatch for {eval_id}"

        # Check arxiv with handling for optional field
        if "arxiv" in raw_eval:
            assert model_eval.arxiv is not None, f"Missing arxiv for {eval_id}"
            assert str(model_eval.arxiv) == raw_eval["arxiv"], (
                f"Arxiv URL mismatch for {eval_id}"
            )
        else:
            assert model_eval.arxiv is None, f"Unexpected arxiv for {eval_id}"

        assert model_eval.group == raw_eval["group"], f"Group mismatch for {eval_id}"
        assert isinstance(model_eval.contributors, list), (
            f"Contributors not a list for {eval_id}"
        )
        assert set(model_eval.contributors) == set(raw_eval["contributors"]), (
            f"Contributors mismatch for {eval_id}"
        )

        # Check tasks
        assert len(model_eval.tasks) == len(raw_eval["tasks"]), (
            f"Task count mismatch for {eval_id}"
        )

        # Sort tasks by name for consistent comparison
        sorted_model_tasks = sorted(model_eval.tasks, key=lambda x: x.name)
        sorted_raw_tasks = sorted(raw_eval["tasks"], key=lambda x: x["name"])

        for task_model, task_raw in zip(sorted_model_tasks, sorted_raw_tasks):
            task_name = task_model.name
            assert task_model.name == task_raw["name"], (
                f"Task name mismatch for {eval_id}/{task_name}"
            )
            assert task_model.dataset_samples == task_raw["dataset_samples"], (
                f"Dataset samples mismatch for {eval_id}/{task_name}"
            )

            # Check human_baseline with handling for optional field
            if "human_baseline" in task_raw:
                assert task_model.human_baseline is not None, (
                    f"Missing human baseline for {eval_id}/{task_name}"
                )
                assert (
                    task_model.human_baseline.metric
                    == task_raw["human_baseline"]["metric"]
                ), f"Metric mismatch for {eval_id}/{task_name}"
                assert (
                    task_model.human_baseline.score
                    == task_raw["human_baseline"]["score"]
                ), f"Score mismatch for {eval_id}/{task_name}"
                assert (
                    str(task_model.human_baseline.source)
                    == task_raw["human_baseline"]["source"]
                ), f"Source URL mismatch for {eval_id}/{task_name}"
            else:
                assert task_model.human_baseline is None, (
                    f"Unexpected human baseline for {eval_id}/{task_name}"
                )

        # Check id property
        assert model_eval.id == eval_id, f"ID mismatch for {eval_id}"

        if "tags" in raw_eval:
            assert hasattr(model_eval, "tags"), f"Missing tags attribute for {eval_id}"
            assert set(model_eval.tags) == set(raw_eval["tags"]), (
                f"Tags mismatch for {eval_id}"
            )


def test_find_task(listing, raw_listing_data):
    """Test that find_task method works correctly."""
    # Find the task in raw data
    cybermetric_raw = next(
        (
            item
            for item in raw_listing_data
            if "cybermetric_500" in [task["name"] for task in item["tasks"]]
        ),
        None,
    )
    assert cybermetric_raw is not None
    task_raw = next(
        (
            task
            for task in cybermetric_raw["tasks"]
            if task["name"] == "cybermetric_500"
        ),
        None,
    )

    # Find the task through the model
    eval_metadata, task_metadata = listing.find_task("cybermetric_500")
    assert eval_metadata is not None
    assert task_metadata is not None

    # Check values match
    assert eval_metadata.title == cybermetric_raw["title"]
    assert task_metadata.name == task_raw["name"]
    assert task_metadata.dataset_samples == task_raw["dataset_samples"]


def test_get_evals_in_group(listing, raw_listing_data):
    """Test that get_evals_in_group method works correctly."""
    # Find all coding evals in raw data
    coding_evals_raw = [item for item in raw_listing_data if item["group"] == "Coding"]

    # Get all evals in the Coding group through the model
    coding_evals = listing.get_evals_in_group("Coding")

    # Check counts match
    assert len(coding_evals) == len(coding_evals_raw)

    # Check all are in the Coding group
    for eval_metadata in coding_evals:
        assert eval_metadata.group == "Coding"


def test_groups_property(listing, raw_listing_data):
    """Test that groups property returns all unique groups."""
    # Get unique groups from raw data
    raw_groups = sorted(list(set(item["group"] for item in raw_listing_data)))

    # Get groups through the model
    groups = listing.groups

    # Check groups match
    assert isinstance(groups, list)
    assert groups == raw_groups


def test_load_eval_listing_with_path(raw_listing_data):
    """Test that _load_eval_listing works with a direct path."""
    listing_path = (
        Path(__file__).parent.parent / "src" / "inspect_evals" / "listing.yaml"
    )
    listing = _load_eval_listing(listing_path)
    assert isinstance(listing, EvalListing)
    assert len(listing.evals) == len(raw_listing_data)
