# eval on mmlu task
CUDA_VISIBLE_DEVICES=6 python3 -m accelerate.commands.launch \
    --num_processes=1 \
    -m lmms_eval \
    --model vicuna \
    --model_args pretrained=lmsys/vicuna-7b-v1.5,attn_implementation=eager \
    --tasks mmlu\
    --batch_size 1 \
    --log_samples \
    --log_samples_suffix vicuna_1.5_7b_mmlu \
    --output_path ./logs/

# eval on Attn_Diff task
CUDA_VISIBLE_DEVICES=4 python3 -m accelerate.commands.launch \
    --num_processes=1 \
    -m lmms_eval \
    --model vicuna \
    --model_args pretrained=lmsys/vicuna-7b-v1.5,attn_implementation=eager \
    --tasks attn_diff_text \
    --batch_size 1 \
    --log_samples \
    --log_samples_suffix vicuna_1.5_7b_attn_diff_text \
    --output_path ./logs/

# vicuna-7b on save_attention task
CUDA_VISIBLE_DEVICES=4 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model vicuna \
	--model_args pretrained=lmsys/vicuna-7b-v1.5,attn_implementation=eager \
	--tasks save_attention_text \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix vicuna_1.5_7b_save_attention_text \
	--output_path ./logs/ \
	--process_with_media 