config="vllm"
max_token=2048

models=(
 "FINETUNED-MODEL-PATH"
)
num_limit=100
ntrain=0

for eval_train in 2 3 4 5 6 7 8;
do
    echo "Processing eval_train: $eval_train"

    for model in "${models[@]}"; 
    do
        echo "Processing model: $model"
        
        for problem_type in "perturbed_statement" "perturbed_leaf" "random_pair" "reorder_statement" "uncommon_name" "flip_role";
        do

            python eval_kk.py --batch_size 8 --model ${model} --max_token ${max_token} --arch ${model} --ntrain ${ntrain} --config ${config} --use_vllm  --limit ${num_limit} --split "train" --problem_type ${problem_type} --eval_train ${eval_train}
            
          
        done
    done
done 

