#!/bin/bash

CKPT=/llava-v1.5-7b/

attn=flash_attention_2


# prune by information

size=2
layer_list=(0 1 2 3 10 16 20)
save_list=(72 144 288 432)


for id_prune_layer in "${layer_list[@]}"; 
do
    for save_num in "${save_list[@]}"; 
    do
        # MME
        Model="llava-v1.5-7b"

        echo "$Model"

        python -m llava.eval.model_vqa_loader \
            --model-path $CKPT \
            --question-file ./playground/data/eval/MME/llava_mme.jsonl \
            --image-folder ./playground/data/eval/MME/MME_Benchmark_release_version \
            --answers-file ./playground/data/eval/MME/answers/$Model.jsonl \
            --temperature 0 \
            --conv-mode vicuna_v1 \
            --sparse \
            --attn_implementation $attn \
            --pruned_layer $id_prune_layer \
            --image_token_start_index 35 \
            --image_token_length 576 \
            --is_textvqa \
            --max_num_trunction $save_num 

        cd ./playground/data/eval/MME

        python convert_answer_to_mme.py --experiment $Model

        cd eval_tool

        python calculation.py --results_dir answers/$Model

        cd  /CAL_info

    done
done