#!/bin/bash

SPLIT="mmbench_dev_20230712"
category="physical_relation"
categories=(
    "attribute_recognition"
    "nature_relation"
    "physical_relation"
    # "social_relation"
    # "ocr"
)


patch_model_list=(
    "clip/llava-v1.5-7b-nonorm"
    "clip/llava-v1.5-7b-rms"
    "dinov2L-518x/llava-v1.5-7b-timm"
    "dinov2L-518x/llava-v1.5-7b-reg4"
    "siglip2/llava-v1.5-7b"
    "siglip2/llava-v1.5-7b-rms"
    "radio/llava-v1.5-7b-layer2"
    "radio/llava-v1.5-7b-layer2-rms"
)
region_model_list=(
    "clip/Cllava-v0.12-ca1x-7b"
    "siglip2/Cllava-v0.12-ca1x-7b"
    "radio/Cllava-v0.10-7b-ca1x"
    "radio/Cllava-v0.12-ca1x-7b"
    "radio/Rllava-v0.48-ca1x-7b_2"
    "radio/Rllava-v0.48_db_-ca1x-7b"
)

for category in "${categories[@]}"; do

for CKPT in ${patch_model_list[@]}; do
    echo "Running eval for model: $CKPT"
    python -m llava.eval.model_vqa_mmbench_analyze \
        --model-path checkpoints/$CKPT \
        --question-file ./playground/data/eval/mmbench/$SPLIT.tsv \
        --answers-file ./playground/data/eval/mmbench/$category/$CKPT.jsonl \
        --category $category \
        --single-pred-prompt \
        --temperature 0 \
        --conv-mode vicuna_v1
done

for CKPT in ${region_model_list[@]}; do
    echo "Running eval for model: $CKPT"
    python -m llava.eval.model_vqa_mmbench_analyze \
        --model-path checkpoints/$CKPT \
        --question-file ./playground/data/eval/mmbench/$SPLIT.tsv \
        --mask-folder ./playground/data/regions/mmbench/regions-mixed \
        --answers-file ./playground/data/eval/mmbench/$category/$CKPT.jsonl \
        --category $category \
        --single-pred-prompt \
        --temperature 0 \
        --conv-mode vicuna_v1
done

done

# for CKPT in ${region_model_list[@]}; do
#     echo "Running eval for model: $CKPT"
#     python -m llava.eval.model_vqa_mmbench_analyze \
#         --model-path checkpoints/$CKPT \
#         --model-kwargs '{"region_filter": "corner_heuristic"}' \
#         --question-file ./playground/data/eval/mmbench/$SPLIT.tsv \
#         --mask-folder ./playground/data/regions/mmbench/regions-mixed \
#         --answers-file ./playground/data/eval/mmbench/$category/$CKPT-corner.jsonl \
#         --category $category \
#         --single-pred-prompt \
#         --temperature 0 \
#         --conv-mode vicuna_v1
# done
