# # llava-1.5 on pope_vis_attn_entropy_gate (vanilla) task
# CUDA_VISIBLE_DEVICES=0 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_gqa_random \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_gqa_random \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_gqa_pop \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_gqa_pop \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=2 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_gqa_adv \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_gqa_adv \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_gqa \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_gqa \
# 	--output_path ./logs/ \
# 	--process_with_media 

# # -------------
# CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_both_head_mlp \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.5_7b_pope_gqa_truthful_both_head_mlp \
# 	--output_path ./logs/ \
# 	--process_with_media 

# Attn Head L.P. & MLP Chunk L.P.
CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_both_head_mlp_chunk_ver1 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.5_7b_pope_gqa_truthful_both_head_mlp_chunk_ver1 \
	--output_path ./logs/ \
	--process_with_media 

CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_both_head_mlp_chunk_ver2 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.5_7b_pope_gqa_truthful_both_head_mlp_chunk_ver2 \
	--output_path ./logs/ \
	--process_with_media 

CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_both_head_mlp_chunk_ver3 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.5_7b_pope_gqa_truthful_both_head_mlp_chunk_ver3 \
	--output_path ./logs/ \
	--process_with_media 

# ablation
CUDA_VISIBLE_DEVICES=0 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_head \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.5_7b_pope_gqa_truthful_head \
	--output_path ./logs/ \
	--process_with_media 

CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.5-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_mlp_chunk_ver3 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.5_7b_pope_gqa_truthful_mlp_chunk_ver3 \
	--output_path ./logs/ \
	--process_with_media 


