import json
import re

def main():
    # 1. Load the JSON file
    # Replace 'response.json' with the path to your JSON file.
    with open('result/single_prompt/gpt_4o.json', 'r') as f:
        data = json.load(f)  # data is assumed to be a list of problem entries

    differences = []  # will store relative differences for each problem

    # 2. Iterate through each problem
    for item in data:
        model_response_str = item["model_response"]
        ground_truth_cost = item["solution_cost_ground_truth"]

        # 3. Parse the model response to get the solution_cost
        # We'll look for a JSON block inside triple backticks with "json" after them, e.g. ```json ... ```.
        match = re.search(r'```json\s*(\{[\s\S]*?\})\s*```', model_response_str)
        if match:
            model_response_json_str = match.group(1)
            try:
                model_response_json = json.loads(model_response_json_str)
                model_cost = model_response_json.get("solution_cost", None)
            except json.JSONDecodeError:
                model_cost = None
                print("Not parsing JSON block correctly.")
        else:
            # If no JSON block was found or we can’t parse, treat as None or handle appropriately.
            model_cost = None

        if model_cost is not None and ground_truth_cost != 0 and model_cost >= ground_truth_cost:
            # Compute relative difference: abs(model - truth) / truth
            rel_diff = abs(model_cost - ground_truth_cost) / ground_truth_cost
            differences.append(rel_diff)
        else:
            # If we can't parse or the ground truth is zero, skip or handle as needed
            continue

    # 4. Compute and print the average relative difference
    if differences:
        avg_diff = sum(differences) / len(differences)
        print(f"Average relative difference: {avg_diff:.4f}")
    else:
        print("No valid data to compute differences.")

if __name__ == "__main__":
    main()
