#!/bin/bash
export PYTHONPATH='.'

base_model="Qwen/Qwen2.5-7B-Instruct"
target_module="qwen/qkvupdown"
tune_ckpt_path="standard_lora"

step=800
gpu_id=0

r=8

echo "Lora Config: lora_r=($r)"

current_time=$(date "+%Y-%m-%d %H:%M:%S")
echo "Start tuning on gpu: $gpu_id, $current_time"
CUDA_VISIBLE_DEVICES=$gpu_id python baseline/qwen/standard_lora_fine_tune.py \
   --base_model $base_model \
   --data_path yahma/alpaca-cleaned \
   --output_dir tune_log/$target_module/$tune_ckpt_path/${r}/ \
   --lora_r $r \
   --num_epochs 2 \
   --learning_rate 1e-4 \
   --batch_size 64

current_time=$(date "+%Y-%m-%d %H:%M:%S")
echo "End tuning on gpu: $gpu_id, $current_time"

# ------------- reasoning ---------
current_time=$(date "+%Y-%m-%d %H:%M:%S")
echo "Start evaluation on gpu: $gpu_id, $current_time"

CUDA_VISIBLE_DEVICES=$gpu_id lm_eval --model hf \
    --model_args peft=tune_log/$target_module/$tune_ckpt_path/${r}/checkpoint-$step,pretrained=$base_model \
    --tasks openbookqa,arc_easy,winogrande,hellaswag,arc_challenge,piqa,social_iqa,boolq \
    --device cuda:0 \
    --output_path results/$target_module/reasoning/${tune_ckpt_path}_${r}_$step.json

current_time=$(date "+%Y-%m-%d %H:%M:%S")
echo "End evaluation on gpu: $gpu_id, $current_time"


# ------------- mmlu zero-shot ---------
#current_time=$(date "+%Y-%m-%d %H:%M:%S")
#echo "Start evaluation on gpu: $gpu_id, $current_time"
#
#CUDA_VISIBLE_DEVICES=$gpu_id lm_eval --model hf \
#    --model_args peft=tune_log/$target_module/$tune_ckpt_path/${r}/checkpoint-$step,pretrained=$base_model \
#    --tasks mmlu \
#    --device cuda:0 \
#    --output_path results/$target_module/instruct/mmlu-0/${tune_ckpt_path}_${r}_$step.json
#
#current_time=$(date "+%Y-%m-%d %H:%M:%S")
#echo "End evaluation on gpu: $gpu_id, $current_time"


# ------------- mmlu 5-shot ---------
#current_time=$(date "+%Y-%m-%d %H:%M:%S")
#echo "Start evaluation on gpu: $gpu_id, $current_time"
#
#CUDA_VISIBLE_DEVICES=$gpu_id lm_eval --model hf \
#    --model_args peft=tune_log/$target_module/$tune_ckpt_path/${r}/checkpoint-$step,pretrained=$base_model \
#    --tasks mmlu \
#    --num_fewshot 5 \
#    --device cuda:0 \
#    --output_path results/$target_module/instruct/mmlu-5/${tune_ckpt_path}_${r}_$step.json
#
#current_time=$(date "+%Y-%m-%d %H:%M:%S")
#echo "End evaluation on gpu: $gpu_id, $current_time"


# ------------- drop ---------
#current_time=$(date "+%Y-%m-%d %H:%M:%S")
#echo "Start evaluation on gpu: $gpu_id, $current_time"
#
#CUDA_VISIBLE_DEVICES=$gpu_id lm_eval --model hf \
#    --model_args peft=tune_log/$target_module/$tune_ckpt_path/${r}/checkpoint-$step,pretrained=$base_model \
#    --tasks drop \
#    --device cuda:0 \
#    --output_path results/$target_module/instruct/drop/${tune_ckpt_path}_${r}_$step.json
#
#current_time=$(date "+%Y-%m-%d %H:%M:%S")
#echo "End evaluation on gpu: $gpu_id, $current_time"


# ------------- bbh ---------
#current_time=$(date "+%Y-%m-%d %H:%M:%S")
#echo "Start evaluation on gpu: $gpu_id, $current_time"
#
#CUDA_VISIBLE_DEVICES=$gpu_id lm_eval --model hf \
#    --model_args peft=tune_log/$target_module/$tune_ckpt_path/${r}/checkpoint-$step,pretrained=$base_model \
#    --tasks bbh \
#    --device cuda:0 \
#    --output_path results/$target_module/instruct/bbh/${tune_ckpt_path}_${r}_$step.json
#
#current_time=$(date "+%Y-%m-%d %H:%M:%S")
#echo "End evaluation on gpu: $gpu_id, $current_time"


#--tasks mmlu,bbh,drop,humaneval \
# cola,mnli,mrpc,qnli,qqp,rte,sst2,wnli
#--tasks openbookqa,arc_easy,winogrande,hellaswag,arc_challenge,piqa,social_iqa,boolq \