model="meta-llama/Meta-Llama-3-8B"
arch="meta-llama/Meta-Llama-3-8B"

# 5-shot


# crosslingual mixup MMLU
trans_lan="mixup"
# use mixup (same biased) demo
python eval.py --model ${model} --arch ${arch} --ntrain 5 --config ${trans_lan} --test_trans_lan ${trans_lan} --dev_trans_lan ${trans_lan}
# use english demo
python eval.py --model ${model} --arch ${arch} --ntrain 5 --config ${trans_lan}_en_demo --test_trans_lan ${trans_lan} --dev_trans_lan ""

# English MMLU
python eval.py --model ${model} --arch ${arch} --ntrain 5 --config en --test_trans_lan "" --dev_trans_lan ""


# crosslingual MMLU
for pre in  "full"  "question" "options" "gt_question" "gt"   "onewrong" "threewrong" 
do
for lan in  "de" "fr" "es" "it"
do
trans_lan=${pre}_${lan}
echo $trans_lan
# use same biased demo
python eval.py --model ${model} --arch ${arch} --ntrain 5 --config ${trans_lan} --test_trans_lan ${trans_lan} --dev_trans_lan ${trans_lan}
# use english demo
python eval.py --model ${model} --arch ${arch} --ntrain 5 --config ${trans_lan}_en_demo --test_trans_lan ${trans_lan} --dev_trans_lan ""
done
done
