#!/bin/bash
CUDA_VISIBLE_DEVICE=7

#### Model ####
MODEL_SIZE=7b
MODEL_VERSION=llava-llama-2-7b-chat
# BASE_BRANCH=lightning-preview
# BASE_BRANCH=DETR-v2-pad0-pretrain-1-tune-1
# CFG_BRANCH=DETR-v2-pad-pretrain-1-tune-1
# CFG_BRANCH=DETR-v2-pad0-pretrain-1-tune-1
BASE_BRANCH=DETR-v2-pad-sl1-new-pretrain-1-tune-1
# CFG_BRANCH=DETR-v2-pad-sl1-new-pretrain-1-tune-1

#### Prompt ####
PROMPT_VERSION="llava_llama_2"

#### QA data ####
QUESTION_FILE_PREFIX=I4_sub1000_control
QUESTION_DIR=../POPE/llava_qa/question
ANSWERS_DIR=../POPE/llava/white/answer_exp_1220

#### Output ####
OUTPUT_DIR=../POPE/llava/white/answer_exp_1220

#### CFG ####
# set CFG_LS to a list of cfg values and use " " to separate them

VM_VISION_TOWER="detr-v2"
VM_MM_VISION_SELECT_LAYER=-2
# VM_MM_VISION_SELECT_LAYER=-1 if [ "-sl1-" in $CFG_BRANCH ] else -2
# NOTE: do not forget to revise pad(exp3), th(exp3), and mm_vision_select_layer(sl) accordingly
# NOTE: and choose right pretrain_mm_mlp_adapter and vm_pretrain_mm_mlp_adapter correspond to the vision tower

########## Run ##########
# for cfg in $(seq 0.7 0.1 0.9)
cfg_values=(0.3 0.6)  # Update this list with your desired cfg values
for cfg in "${cfg_values[@]}"
do
    echo "Running with cfg = $cfg"
    deepspeed --master_port 16683 --include localhost:$CUDA_VISIBLE_DEVICE llava/backbone/detr_branch/eval_cfg.py \
        --deepspeed ./scripts/zero2.json \
        --model_name_or_path ./checkpoints/$MODEL_VERSION-$BASE_BRANCH \
        --version $PROMPT_VERSION \
        --bf16 True \
        --per_device_eval_batch_size 32 \
        --gradient_accumulation_steps 1 \
        --evaluation_strategy "epoch" \
        --logging_steps 100 \
        --tf32 True \
        --model_max_length 2048 \
        --gradient_checkpointing True \
        --dataloader_num_workers 2 \
        --lazy_preprocess True \
        --report_to wandb \
        --eval_image_folder "../POPE/data/minival2014/minival2014" \
        --question_path $QUESTION_DIR \
        --question_file $QUESTION_FILE_PREFIX.json \
        --answer_path $ANSWERS_DIR \
        --answer_prompter False \
        --cfg $cfg\
        --num_chunks 1 \
        --chunk_id 0 \
        --vision_tower "openai/clip-vit-large-patch14" \
        --pretrain_mm_mlp_adapter "./checkpoints/llava-llama-2-7b-chat-lightning-preview/mm_projector_7b_chat.bin" \
        --tune_mm_mlp_adapter True \
        --mm_vision_select_layer -2 \
        --mm_use_im_start_end False \
        --mm_use_im_patch_token False \
        --vm_vision_tower $VM_VISION_TOWER \
        --vm_pretrain_mm_mlp_adapter ./llava/backbone/checkpoints_tune_ll2/$MODEL_VERSION-$CFG_BRANCH/mm_projector.bin \
        --vm_mm_vision_select_layer $VM_MM_VISION_SELECT_LAYER \
        --vm_mm_use_im_start_end False \
        --vm_mm_use_im_patch_token False \
        --vm_tune_mm_mlp_adapter True \
        --output_dir $OUTPUT_DIR \
        --seed $SEED \


        # --cfg_mode "text" \
        # --cfg 0 \
done