# Enable subset for hyperparameter optimization
export LMMS_EVAL_ENABLE_SUBSET=False
# export LMMS_EVAL_RANDOM_SEED=42
# export LMMS_EVAL_SUBSET_RATIO=0.2

### -------------------------- POPE (coco) --------------------------
# vanilla
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope \
# 	--output_path ./logs/ \
# 	--process_with_media 

# Truthfulness score (ver.7,9) -> Head Gate (contrast) - ver.8 removed
# ver.7 (norm, cont)
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver7_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver7_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_truthful_head_ver7_norm_02 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver7_norm_02 \
	--output_path ./logs/ \
	--process_with_media 
	
CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_truthful_head_ver7_norm_03 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver7_norm_03 \
	--output_path ./logs/ \
	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver7_cont_45 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver7_cont_45 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver7_cont_60 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver7_cont_60 \
# 	--output_path ./logs/ \
# 	--process_with_media 
	
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver7_cont_75 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver7_cont_75 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# # ver.9 (norm, cont)
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver9_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver9_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_truthful_head_ver9_norm_02 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver9_norm_02 \
	--output_path ./logs/ \
	--process_with_media 
	
CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_truthful_head_ver9_norm_03 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver9_norm_03 \
	--output_path ./logs/ \
	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver9_cont_45 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver9_cont_45 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver9_cont_60 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver9_cont_60 \
# 	--output_path ./logs/ \
# 	--process_with_media 
	
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_truthful_head_ver9_cont_75 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_truthful_head_ver9_cont_75 \
# 	--output_path ./logs/ \
# 	--process_with_media 









# ### -------------------------- POPE (aokvqa) --------------------------
# # vanilla
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa \
# 	--output_path ./logs/ \
# 	--process_with_media 

# # Truthfulness score (ver.7,8,9) -> Head Gate (contrast)
# # ver.7 (norm, cont)
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver7_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver7_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_aokvqa_truthful_head_ver7_norm_02 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver7_norm_02 \
	--output_path ./logs/ \
	--process_with_media 
	
CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_aokvqa_truthful_head_ver7_norm_03 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver7_norm_03 \
	--output_path ./logs/ \
	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver7_cont_45 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver7_cont_45 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver7_cont_60 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver7_cont_60 \
# 	--output_path ./logs/ \
# 	--process_with_media 
	
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver7_cont_75 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver7_cont_75 \
# 	--output_path ./logs/ \
# 	--process_with_media 


# # ver.9 (norm, cont)
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver9_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver9_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_aokvqa_truthful_head_ver9_norm_02 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver9_norm_02 \
	--output_path ./logs/ \
	--process_with_media 
	
CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_aokvqa_truthful_head_ver9_norm_03 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver9_norm_03 \
	--output_path ./logs/ \
	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver9_cont_45 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver9_cont_45 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver9_cont_60 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver9_cont_60 \
# 	--output_path ./logs/ \
# 	--process_with_media 
	
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_aokvqa_truthful_head_ver9_cont_75 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_aokvqa_truthful_head_ver9_cont_75 \
# 	--output_path ./logs/ \
# 	--process_with_media 










# ### -------------------------- POPE (gqa) --------------------------
# # vanilla
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_gqa \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa \
# 	--output_path ./logs/ \
# 	--process_with_media 

# # Truthfulness score (ver.7,8,9) -> Head Gate (contrast)
# # ver.7 (norm, cont)
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver7_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver7_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver7_norm_02 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver7_norm_02 \
	--output_path ./logs/ \
	--process_with_media 
	
CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver7_norm_03 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver7_norm_03 \
	--output_path ./logs/ \
	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver7_cont_45 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver7_cont_45 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver7_cont_60 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver7_cont_60 \
# 	--output_path ./logs/ \
# 	--process_with_media 
	
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver7_cont_75 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver7_cont_75 \
# 	--output_path ./logs/ \
# 	--process_with_media 
 

# # ver.9 (norm, cont)
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver9_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver9_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver9_norm_02 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver9_norm_02 \
	--output_path ./logs/ \
	--process_with_media 
	
CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model llava \
	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver9_norm_03 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver9_norm_03 \
	--output_path ./logs/ \
	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver9_cont_45 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver9_cont_45 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver9_cont_60 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver9_cont_60 \
# 	--output_path ./logs/ \
# 	--process_with_media 
	
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver9_cont_75 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_pope_gqa_truthful_head_ver9_cont_75 \
# 	--output_path ./logs/ \
# 	--process_with_media 











# ### -------------------------- CHAIR (chair_max64) --------------------------
# # vanilla
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# # Truthfulness score (ver.7,8,9) -> Head Gate (contrast)
# # ver.7 (norm, cont)
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver7_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver7_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver7_norm_02 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver7_norm_02 \
# 	--output_path ./logs/ \
# 	--process_with_media 
	
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver7_norm_03 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver7_norm_03 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver7_cont_45 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver7_cont_45 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver7_cont_60 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver7_cont_60 \
# 	--output_path ./logs/ \
# 	--process_with_media 
	
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver7_cont_75 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver7_cont_75 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# # ver.9 (norm, cont)
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver9_norm_01 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver9_norm_01 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver9_norm_02 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver9_norm_02 \
# 	--output_path ./logs/ \
# 	--process_with_media 
	
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver9_norm_03 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver9_norm_03 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver9_cont_45 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver9_cont_45 \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver9_cont_60 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver9_cont_60 \
# 	--output_path ./logs/ \
# 	--process_with_media 
	
# CUDA_VISIBLE_DEVICES=1 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model llava \
# 	--model_args pretrained=liuhaotian/llava-v1.6-vicuna-7b,attn_implementation=eager \
# 	--tasks chair_max64_truthful_head_ver9_cont_75 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix llava_v1.6_vicuna_7b_chair_max64_truthful_head_ver9_cont_75 \
# 	--output_path ./logs/ \
# 	--process_with_media 