#!/bin/bash

source "shell_jobs/_experiment_configuration.sh"
conda deactivate
conda activate bbq
# add ../lrqa to PYTHONPATH
export PYTHONPATH="${PYTHONPATH}:$(pwd)/../lrqa"


bias_type="gender"
base_model="bert-large-cased"
models=(${base_model} "aieng-lab/${base_model}-gradiend-gender-debiased" "${persistent_dir}/results/checkpoints/cda_c-${base_model}_t-${bias_type}_s-0")
BATCH_SIZE=32
GRAD_ACCUM_STEPS=4
CATEGORY="Gender_identity"

EXP_DIR="${persistent_dir}/results/bbq_runs/${base_model}"
BBQ_DATA="${persistent_dir}/../BBQ/data"

python ../lrqa/lrqa/scripts/race_preproc.py \
    --data_path ${EXP_DIR}/race

python ../lrqa/lrqa/scripts/bbq_preproc.py \
    --input_data_path=${BBQ_DATA} \
    --data_path ${EXP_DIR}/bbq
exit

for HF_MODEL_NAME in ${models[@]}; do
    HF_MODEL_NAME_ID=$(basename "${HF_MODEL_NAME}")
    #EXP_DIR="${persistent_dir}/results/bbq_runs/${HF_MODEL_NAME_ID}"
    mkdir -p ${EXP_DIR}
    echo "Running evaluation for ${HF_MODEL_NAME_ID} in ${EXP_DIR}"
    python ../lrqa/lrqa/run_lrqa.py  --model_name_or_path ${HF_MODEL_NAME} --model_mode mc --max_seq_length 512 --task_name custom --task_base_path ${EXP_DIR}/race --output_dir ${EXP_DIR}/race_run_${HF_MODEL_NAME_ID} --learning_rate 1e-5 --num_train_epochs 3 --warmup_ratio 0.1 --eval_steps 1000 --save_steps 1000 --save_total_limit 1 --save_strategy steps --eval_strategy steps --load_best_model_at_end --per_device_train_batch_size ${BATCH_SIZE} --per_device_eval_batch_size ${BATCH_SIZE} --gradient_accumulation_steps ${GRAD_ACCUM_STEPS} --do_train --do_eval --do_predict --predict_phases validation
    exit
    python ../lrqa/lrqa/run_lrqa.py   --model_name_or_path ${EXP_DIR}/race_run_${HF_MODEL_NAME_ID}/checkpoint-last --model_mode mc --max_seq_length 512 --task_name custom --task_base_path ${EXP_DIR}/bbq/${CATEGORY}/  --output_dir ${EXP_DIR}/bbq_runs/${CATEGORY}/ --per_device_eval_batch_size ${BATCH_SIZE} --do_eval --do_predict --predict_phases validation
done
