
names=()
for task in mmlu; do
    for num_shot in 0 5; do
        for setting in perm; do
            names[${#names[*]}]="${task},${num_shot},${setting}"
        done
    done
done

# pretrained_model_path=${HF_MODELS}/huggyllama/llama-7b
echo ${names[*]}
pretrained_model_path=/mnt/llms/model/google-gemma/gemma-2b
# pretrained_model_path=/mnt/llms/model/google-gemma/gemma-7b
# pretrained_model_path=/mnt/llms/model/meta-llama/Llama-2-7b-hf
# pretrained_model_path=/mnt/llms/model/meta-llama/Llama-3-8b-hf
export CUDA_VISIBLE_DEVICES=1
python eval_clm.py \
    --pretrained_model_path ${pretrained_model_path} \
    --eval_names ${names[*]} \
    --num-gpus 1

