# llava-1.5 -> pope (coco, aokvqa, gqa)
# CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver7_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_truthful_head_ver7_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver8_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_truthful_head_ver8_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 


# CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver7_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_aokvqa_truthful_head_ver7_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver8_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_aokvqa_truthful_head_ver8_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver7_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_gqa_truthful_head_ver7_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver8_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_gqa_truthful_head_ver8_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 


# # llava-next -> POPE (gqa)
CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver7_norm_03 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver7_norm_03 \
	--output_path ./logs/ \
	--process_with_media

CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver9_norm_03 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver9_norm_03 \
	--output_path ./logs/ \
	--process_with_media 


# Qwen2.5-VL-Instruct -> POPE (coco, aokvqa)
CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model qwen2_5_vl \
	--model_args pretrained=Qwen/Qwen2.5-VL-7B-Instruct,attn_implementation=eager \
	--tasks pope_truthful_head_ver10_norm_03 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix qwen2_5_vl_7b_instruct_pope_truthful_head_ver10_norm_03 \
	--output_path ./logs/ \
	--process_with_media 

# CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model qwen2_5_vl \
# 	--model_args pretrained=Qwen/Qwen2.5-VL-7B-Instruct,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver11_norm_03 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix qwen2_5_vl_7b_instruct_pope_truthful_head_ver11_norm_03 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model qwen2_5_vl \
# 	--model_args pretrained=Qwen/Qwen2.5-VL-7B-Instruct,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver10_norm_03 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix qwen2_5_vl_7b_instruct_pope_aokvqa_truthful_head_ver10_norm_03 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model qwen2_5_vl \
# 	--model_args pretrained=Qwen/Qwen2.5-VL-7B-Instruct,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver11_norm_03 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix qwen2_5_vl_7b_instruct_pope_aokvqa_truthful_head_ver11_norm_03 \
# 	--output_path ./logs/ \
# 	--process_with_media 
