
import json 
import re 

TRACE_PATH ="/path/to/home/llm-recipes/scripts/ET-CoT/inference_outputs/Meta-Llama-3-8B-Instruct/3epoch"


file_paths =[
f"{TRACE_PATH }/LLTM-llama-token-len-1-numeric-depth-val/iter4224.jsonl",
f"{TRACE_PATH }/LLTM-llama-token-len-2-numeric-depth-val/iter4224.jsonl",
f"{TRACE_PATH }/LLTM-llama-token-len-3-numeric-depth-val/iter4224.jsonl",
f"{TRACE_PATH }/LLTM-llama-token-len-4-numeric-depth-val/iter4224.jsonl",
f"{TRACE_PATH }/LLTM-llama-token-len-1-numeric-depth-val/iter3684.jsonl",
f"{TRACE_PATH }/LLTM-llama-token-len-2-numeric-depth-val/iter3684.jsonl",
f"{TRACE_PATH }/LLTM-llama-token-len-3-numeric-depth-val/iter3684.jsonl",
f"{TRACE_PATH }/LLTM-llama-token-len-4-numeric-depth-val/iter3684.jsonl"
]


pattern =re .compile (r"<code>\s*print\(len\('(.*)'\)\)\s*</code>",re .DOTALL )

total_cnt =0 
total_true_cnt =0 
total_false_cnt =0 
err_cnt =0 

for file_path in file_paths :

    with open (file_path ,'r',encoding ="utf-8")as f :
        total_cnt +=1 
        for line in f :
            try :
                data =json .loads (line )
            except json .JSONDecodeError :
                err_cnt +=1 
                continue 


            if data .get ("is_correct",True )is False :
                total_false_cnt +=1 
                prompt_text =data .get ("prompt","")
                match =pattern .search (prompt_text )
                if match :
                    extracted =match .group (1 )
                    print (extracted )
                else :
                    print (f"Error: {data }")
            else :
                total_true_cnt +=1 





