model="openai/gpt-4o-mini-2024-07-18"


config="vllm"
num_limit=100


max_token=2048
ntrain=0

for eval_train in 2 3 4 5 6 7 8;
do
    echo "Processing eval_train: $eval_train"
    for split in "train" "test";
    do
        echo "Processing split: $split"
        
        python eval_kk.py --cot --batch_size 8 --model ${model} --max_token ${max_token} --infer_mode generation --ntrain ${ntrain} --config ${config} --limit ${num_limit}  --split ${split} --problem_type "clean"  --eval_train ${eval_train}
        python eval_kk.py --cot --batch_size 8 --model ${model} --max_token ${max_token} --infer_mode generation --ntrain ${ntrain} --config ${config} --limit ${num_limit}  --split ${split} --problem_type "random_pair"   --eval_train ${eval_train} 
        python eval_kk.py --cot --batch_size 8 --model ${model} --max_token ${max_token} --infer_mode generation --ntrain ${ntrain} --config ${config} --limit ${num_limit}  --split ${split}  --problem_type "reorder_statement"  --eval_train ${eval_train}
        python eval_kk.py --cot --batch_size 8 --model ${model} --max_token ${max_token} --infer_mode generation --ntrain ${ntrain} --config ${config} --limit ${num_limit}  --split ${split}  --problem_type "uncommon_name"  --eval_train ${eval_train}
        python eval_kk.py --cot --batch_size 8 --model ${model} --max_token ${max_token} --infer_mode generation --ntrain ${ntrain} --config ${config} --limit ${num_limit}  --split ${split}  --problem_type "flip_role"  --eval_train ${eval_train}  
        python eval_kk.py --cot --batch_size 8 --model ${model} --max_token ${max_token} --infer_mode generation --ntrain ${ntrain} --config ${config} --limit ${num_limit}  --split ${split}  --problem_type "perturbed_statement"  --eval_train ${eval_train}
        python eval_kk.py --cot --batch_size 8 --model ${model} --max_token ${max_token} --infer_mode generation --ntrain ${ntrain} --config ${config} --limit ${num_limit}  --split ${split}  --problem_type "perturbed_leaf"  --eval_train ${eval_train}
    done
done


