#!/bin/bash

internlm2_20b_path=/cpfs01/shared/public/public_hdd/llmeval/model_weights/hf_hub/models--internlm--internlm2-chat-20b/snapshots/3f710f76f56f8c40dc5dd800dbe66f3341cb2c87

ultracm_path=/cpfs02/llm/shared/public/lantian/exp/20240501_sft_7b_critique_ultracm_878/aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_critic_ultracm/878_hf_ckpt
autoj_path=/cpfs02/llm/shared/public/lantian/exp/20240501_sft_7b_critique_autoj_26/aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_critic_autoj/26_hf_ckpt
our_path=/cpfs02/llm/shared/public/lantian/exp/20240618_sft_7b_critique_nips2024_d4_st_5_l1_resumm_385_epoch_2_0701/aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l1_resumm/385_hf_ckpt


#########################################################################
autoj_crit_file=output/_cpfs02_llm_shared_public_lantian_exp_20240501_sft_7b_critique_autoj_26_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_critic_autoj_26_hf_ckpt/critique/result_07_07_01_14_00.jsonl
ultracm_crit_file=output/_cpfs02_llm_shared_public_lantian_exp_20240501_sft_7b_critique_ultracm_878_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_critic_ultracm_878_hf_ckpt/critique/result_07_07_00_48_01.jsonl
#our_crit_file=output/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l1_resumm_385_epoch_2_0701_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l1_resumm_385_hf_ckpt/critique/result_07_07_01_53_11.jsonl
#our_crit_file_l2=output/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l2_resumm_385_epoch_2_0702_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l2_resumm_385_hf_ckpt/critique/result_07_08_12_05_25.jsonl
#our_crit_file_l4=output/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l4_resumm_385_epoch_2_0702_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l4_resumm_385_hf_ckpt/critique/result_07_08_12_11_37.jsonl
#our_crit_file_l6=output/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_resumm_385_epoch_2_0701_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_resumm_385_hf_ckpt/critique/result_07_08_12_11_22.jsonl

# Correction with llama2-13b-chat model as the base; prompt_type: fs, zs-crit-cot, zs-crit-ao-1
## 提前准备好existed_crit_file作为输入
# for existed_crit_file in ${existed_crit_files[@]}

#our_crit_file_l2=20240708_multi_turn/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l2_resumm_385_epoch_2_0702_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l2_resumm_385_hf_ckpt/critique/result_07_08_16_46_14.jsonl
#our_crit_file_l4=20240708_multi_turn/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l4_resumm_385_epoch_2_0702_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l4_resumm_385_hf_ckpt/critique/result_07_08_16_43_15.jsonl
#our_crit_file_l6=20240708_multi_turn/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_resumm_385_epoch_2_0701_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_resumm_385_hf_ckpt/critique/result_07_08_16_35_38.jsonl
internlm2_7b_crit_file=20240708_multi_turn/20240418_aliyun_Ampere_7B_v1_1_FT_v1_0_0_s1_rc48_1660_hf_ckpt/critique/zs-crit-cot_result_07_10_00_10_10.jsonl
themis_crit_file=20240708_multi_turn/Themis/critique/result_07_10_09_45_55.jsonl
our_crit_file=20240708_multi_turn/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l1_resumm_385_epoch_2_0701_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l1_resumm_385_hf_ckpt/critique/result_07_08_16_40_40.jsonl
existed_crit_files=($our_crit_file $autoj_crit_file $ultracm_crit_file $themis_crit_file $internlm2_7b_crit_file)

#our_crit_file_new=20240708_multi_turn_test_debug/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l1_resumm_385_epoch_2_0701_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l1_resumm_385_hf_ckpt/critique/result_07_10_16_05_15.jsonl
#our_crit_file_new_noref=20240708_multi_turn_noref/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l1_resumm_385_epoch_2_0701_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l1_resumm_385_hf_ckpt/critique/result_07_12_12_45_03.jsonl

#our_crit_file_l2=20240708_multi_turn/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l2_resumm_385_epoch_2_0702_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l2_resumm_385_hf_ckpt/critique/result_07_08_16_46_14.jsonl
#our_crit_file_l4=20240708_multi_turn/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l4_resumm_385_epoch_2_0702_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l4_resumm_385_hf_ckpt/critique/result_07_08_16_43_15.jsonl
#our_crit_file_l6=20240708_multi_turn/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_resumm_385_epoch_2_0701_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_resumm_385_hf_ckpt/critique/result_07_08_16_35_38.jsonl

#our_crit_file=20240713_debug/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l1_resumm_385_epoch_2_0701_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l1_resumm_385_hf_ckpt/critique/result_07_14_13_32_34.jsonl
#our_crit_file=20240713_debug/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l1_resumm_385_epoch_2_0701_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l1_resumm_385_hf_ckpt/critique/result_07_14_16_16_36.jsonl
#existed_crit_files=($our_crit_file_l2 $our_crit_file_l4 $our_crit_file_l6)
#our_crit_file_ref_first=20240715_debug/_cpfs02_llm_shared_public_lantian_exp_our_template_test_5_l1_resumm_path_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l1_resumm_384_hf_ckpt/critique/result_07_15_16_03_27.jsonl
#our_crit_file_ref_first_mt=20240715_debug_v2/_cpfs02_llm_shared_public_lantian_exp_our_template_test_5_l1_resumm_path_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l1_resumm_384_hf_ckpt/critique/result_07_15_23_46_30.jsonl
#our_crit_file_no_ref_mt=20240715_debug_noref/_cpfs02_llm_shared_public_lantian_exp_20240618_sft_7b_critique_nips2024_d4_st_5_l1_resumm_no_ref_333_epoch_2_0703_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_FINAL_critic_v4_st_5_l1_resumm_no_ref_333_hf_ckpt/critique/result_07_16_02_22_01.jsonl
#existed_crit_files=($our_crit_file_no_ref_mt)
#inference_models=($base_path)




########################################### s2 series ###########################################

llama_3_70b_instruct=/cpfs01/shared/public/llmeval/model_weights/hf_hub/models--meta-llama--Meta-Llama-3-70B-Instruct/snapshots/7129260dd854a80eb10ace5f61c20324b472b31c
mixtral_8x7b_instruct=/cpfs01/shared/public/llmeval/model_weights/hf_hub/models--mistralai--Mixtral-8x7B-Instruct-v0.1/snapshots/bbae113847402a22031211225b5ee45c005de7dd
s2_no_critic_model=/cpfs02/llm/shared/public/lantian/exp/s2_add_critictuning_v01rc1/s2_no_critic_344/aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_s2_no_critictuning_v01rc1/344_hf_ckpt
s2_add_critic_model=/cpfs02/llm/shared/public/lantian/exp/s2_add_critictuning_v01rc1/s2_add_critic_530/aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_s2_add_critictuning_v01rc1/530_hf_ckpt
internlm2_20b_chat=/cpfs01/shared/public/public_hdd/llmeval/model_weights/hf_hub/models--internlm--internlm2-chat-20b/snapshots/3f710f76f56f8c40dc5dd800dbe66f3341cb2c87
internlm2_7b_chat=/cpfs02/llm/shared/public/lantian/exp/20240418_aliyun_Ampere_7B_v1_1_FT_v1_0_0_s1_rc48_1660_hf_ckpt
inference_models=($s2_add_critic_model $internlm2_20b_chat $internlm2_7b_chat)
#inference_models=($llama_3_70b_instruct)
#inference_models=($mixtral_8x7b_instruct)

ultracm_s2_file=/cpfs02/llm/shared/public/lantian/NIPS_2024/evaluation/CriticBench-THU/src/20240717_s2_series/_cpfs02_llm_shared_public_lantian_exp_s2_add_critictuning_v01rc1_s2_add_ultrafeedback_critic_894_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_s2_add_critictuning_v01rc4_894_hf_ckpt/critique/result_07_17_19_51_39.jsonl
s2_no_critic_file=/cpfs02/llm/shared/public/lantian/NIPS_2024/evaluation/CriticBench-THU/src/20240717_s2_series/344_hf_ckpt/critique/zs-crit-cot_result_07_17_15_25_19.jsonl
autoj_s2_file=/cpfs02/llm/shared/public/lantian/NIPS_2024/evaluation/CriticBench-THU/src/20240717_s2_series/_cpfs02_llm_shared_public_lantian_exp_s2_add_critictuning_v01rc1_s2_add_autoj_critic_349_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_s2_add_critictuning_v01rc3_359_hf_ckpt/critique/result_07_17_20_20_37.jsonl
our_s2_file=/cpfs02/llm/shared/public/lantian/NIPS_2024/evaluation/CriticBench-THU/src/20240717_s2_series/_cpfs02_llm_shared_public_lantian_exp_s2_add_critictuning_v01rc1_s2_add_critic_530_aliyun_Ampere_7B_v1.1_enchance_FT_v1.0.0_s1_rc47_s2_add_critictuning_v01rc1_530_hf_ckpt/critique/result_07_17_16_40_51.jsonl
#existed_crit_files=($our_s2_file $s2_no_critic_file $ultracm_s2_file $autoj_s2_file)

#for inference_model in ${inference_models[@]}
for inf_index in $(seq 0 0)
do
    for index in $(seq 0 4)
    do
        existed_crit_file=${existed_crit_files[$index]}
        IFS='/' read -ra arr <<< "$existed_crit_file"
        critic_model_name=${arr[1]}
        inference_model=${inference_models[$inf_index]}
        echo "==========================================================="
        echo $existed_crit_file
        echo "Critic Model for Correction: $critic_model_name by Inference model [$inference_model]"
        echo "==========================================================="
        CUDA_VISIBLE_DEVICES=3 python evaluate.py --available_gpus 3 --tasks QC --hf_model $inference_model --prompt_type zs-crit-cot --enable_code_execution --existed_crit_file $existed_crit_file --output_dir output_correction_add_critic/${critic_model_name}
        #CUDA_VISIBLE_DEVICES=$inf_index python evaluate.py --available_gpus $inf_index --tasks QC --hf_model $inference_model --prompt_type zs-crit-cot --enable_code_execution --existed_crit_file $existed_crit_file --output_dir output_correction_s2_internlm2_20b/${critic_model_name}
        #CUDA_VISIBLE_DEVICES=$inf_index python evaluate.py --available_gpus $inf_index --tasks QC --hf_model $inference_model --prompt_type zs-crit-cot --enable_code_execution --existed_crit_file $existed_crit_file --output_dir output_correction_s2_s1/${critic_model_name}

        #CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python evaluate.py --available_gpus 0,1,2,3,4,5,6,7 --tasks QC --hf_model $inference_model --prompt_type zs-crit-cot --enable_code_execution --existed_crit_file $existed_crit_file --output_dir output_correction_s2_mixtral_8x7b_instruct/${critic_model_name}
        #CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python evaluate.py --available_gpus 0,1,2,3,4,5,6,7 --tasks QC --hf_model $inference_model --prompt_type zs-crit-cot --enable_code_execution --existed_crit_file $existed_crit_file --output_dir output_correction_s2_llama_3_70b_instruct/${critic_model_name}
        #CUDA_VISIBLE_DEVICES=7 python evaluate.py --available_gpus 7 --tasks QC --hf_model $base_20b_path --prompt_type zs-crit-cot --enable_code_execution --existed_crit_file $existed_crit_file --output_dir output_correction_new_noref_20b_none_redefine/${critic_model_name} &
        echo "==========================================================="
    done
done
