# Enable subset for hyperparameter optimization
export LMMS_EVAL_ENABLE_SUBSET=False
# export LMMS_EVAL_RANDOM_SEED=42
# export LMMS_EVAL_SUBSET_RATIO=0.2


### -------------------------- POPE (gqa) --------------------------
# vanilla
# CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model qwen2_5_omni \
# 	--model_args pretrained=Qwen/Qwen2.5-Omni-7B,attn_implementation=eager \
# 	--tasks pope_gqa \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix qwen2_5_omni_7b_pope_gqa \
# 	--output_path ./logs/ \
# 	--process_with_media 

# CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model qwen2_5_omni \
# 	--model_args pretrained=Qwen/Qwen2.5-Omni-7B,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver10_norm_03 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix qwen2_5_omni_7b_pope_gqa_truthful_head_ver10_norm_03 \
# 	--output_path ./logs/ \
# 	--process_with_media 

CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model qwen2_5_omni \
	--model_args pretrained=Qwen/Qwen2.5-Omni-7B,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver10_norm_03_bottom_k10 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix qwen2_5_omni_7b_pope_gqa_truthful_head_ver10_norm_03_bottom_k10 \
	--output_path ./logs/ \
	--process_with_media 

CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model qwen2_5_omni \
	--model_args pretrained=Qwen/Qwen2.5-Omni-7B,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver10_norm_03_bottom_k15 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix qwen2_5_omni_7b_pope_gqa_truthful_head_ver10_norm_03_bottom_k15 \
	--output_path ./logs/ \
	--process_with_media 

CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model qwen2_5_omni \
	--model_args pretrained=Qwen/Qwen2.5-Omni-7B,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver10_norm_03_bottom_k20 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix qwen2_5_omni_7b_pope_gqa_truthful_head_ver10_norm_03_bottom_k20 \
	--output_path ./logs/ \
	--process_with_media 



# CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
# 	--num_processes=1 \
# 	-m lmms_eval \
# 	--model qwen2_5_omni \
# 	--model_args pretrained=Qwen/Qwen2.5-Omni-7B,attn_implementation=eager \
# 	--tasks pope_gqa_truthful_head_ver12_norm_03 \
# 	--batch_size 1 \
# 	--log_samples \
# 	--log_samples_suffix qwen2_5_omni_7b_pope_gqa_truthful_head_ver12_norm_03 \
# 	--output_path ./logs/ \
# 	--process_with_media 



CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model qwen2_5_omni \
	--model_args pretrained=Qwen/Qwen2.5-Omni-7B,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver12_norm_03_bottom_k10 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix qwen2_5_omni_7b_pope_gqa_truthful_head_ver12_norm_03_bottom_k10 \
	--output_path ./logs/ \
	--process_with_media 



CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model qwen2_5_omni \
	--model_args pretrained=Qwen/Qwen2.5-Omni-7B,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver12_norm_03_bottom_k15 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix qwen2_5_omni_7b_pope_gqa_truthful_head_ver12_norm_03_bottom_k15 \
	--output_path ./logs/ \
	--process_with_media 



CUDA_VISIBLE_DEVICES=3 python3 -m accelerate.commands.launch \
	--num_processes=1 \
	-m lmms_eval \
	--model qwen2_5_omni \
	--model_args pretrained=Qwen/Qwen2.5-Omni-7B,attn_implementation=eager \
	--tasks pope_gqa_truthful_head_ver12_norm_03_bottom_k20 \
	--batch_size 1 \
	--log_samples \
	--log_samples_suffix qwen2_5_omni_7b_pope_gqa_truthful_head_ver12_norm_03_bottom_k20 \
	--output_path ./logs/ \
	--process_with_media 


