
root_dir=eval/ruler/data-jsonl-100

MODEL_NAME=llama3-8b-Instruct
model_path=models/Meta-Llama-3-8B-Instruct

tasks=(cwe fwe niah_multikey_1 niah_multikey_2 niah_multikey_3 niah_multiquery niah_multivalue niah_single_1 niah_single_2 niah_single_3 qa_1 qa_2 vt)
for SEQ_LEN in 131072;
do
    for task in "${tasks[@]}"; 
    do
        task_dir="${root_dir}/${task}"
        file="${task_dir}/llama-${SEQ_LEN}.jsonl"
        log_dir="eval/ruler/logs/${MODEL_NAME}"
        if [ ! -d "$log_dir" ]; then
            mkdir -p "$log_dir"
        fi
        CUDA_VISIBLE_DEVICES=0 \
        python eval/ruler/test_ruler_llama.py \
            --model_path ${model_path} \
            --task ${task} \
            --data_dir ${file} \
            --model_name $MODEL_NAME \
            --seq_len $SEQ_LEN \
            > ${log_dir}/task${task}_${MODEL_NAME}_seqlen${SEQ_LEN}.log 2>&1
    done
done