#!/bin/bash
export PYTHONPATH='.'

base_model="baffo32/decapoda-research-llama-7B-hf"
target_module="qkvupdown"
tune_ckpt_path="share_lora"

step=800
gpu_id=2

lora_r=4
lora_alpha=8
lora_intra=2_2
lora_inter=4_4

echo "Lora Config: lora=($lora_r, $lora_alpha), intra=($lora_intra), inter=($lora_inter)"

current_time=$(date "+%Y-%m-%d %H:%M:%S")
echo "Start tuning on gpu: $gpu_id, $current_time"

CUDA_VISIBLE_DEVICES=$gpu_id python fine_tune.py \
   --base_model $base_model \
   --data_path yahma/alpaca-cleaned \
   --output_dir tune_log/$target_module/$tune_ckpt_path/${lora_intra}_${lora_inter}/ \
   --lora_r $lora_r \
   --lora_alpha $lora_alpha \
   --lora_intra $lora_intra \
   --lora_inter $lora_inter \
   --num_epochs 2 \
   --learning_rate 1e-4 \
   --batch_size 64

current_time=$(date "+%Y-%m-%d %H:%M:%S")
echo "End tuning on gpu: $gpu_id, $current_time"

#cp tune_log/$target_module/$tune_ckpt_path/${lora_intra}_${lora_inter}/adapter_config.json tune_log/$target_module/$tune_ckpt_path/${lora_intra}_${lora_inter}/checkpoint-$step/
#mv tune_log/$target_module/$tune_ckpt_path/${lora_intra}_${lora_inter}/adapter_model.bin tune_log/$target_module/$tune_ckpt_path/${lora_intra}_${lora_inter}/checkpoint-$step/adapter_model.bin
#
#current_time=$(date "+%Y-%m-%d %H:%M:%S")
#echo "Start evaluation on gpu: $gpu_id, $current_time"
#
#tune_id="${tune_ckpt_path##*/}"
#CUDA_VISIBLE_DEVICES=$gpu_id python lm-evaluation-harness/main.py \
#    --model hf-causal-experimental \
#    --model_args peft=tune_log/$target_module/$tune_ckpt_path/${lora_intra}_${lora_inter}/checkpoint-$step,pretrained=$base_model \
#    --tasks openbookqa,arc_easy,winogrande,hellaswag,arc_challenge,piqa,boolq \
#    --device cuda:0 \
#    --output_path results/$target_module/${tune_id}_${lora_intra}_${lora_inter}_$step.json \
#    --no_cache
#
#current_time=$(date "+%Y-%m-%d %H:%M:%S")
#echo "End evaluation on gpu: $gpu_id, $current_time"