
############################################
#! 1. Train LLM with Lora of SynAdapt
############################################

LLM_path="" #TODO: LLM path
train_data_path="" #TODO: train data path
ccot_index_path="" #TODO: ccot index path
ccot_dir="" #TODO: ccot save dir
model_save_dir="" #TODO: model save dir

TIME_STR=`date +%Y-%m-%d_%H:%M:%S`
export NCCL_DEBUG=WARN         # 仅显示 NCCL 的警告和错误（默认是 INFO）
export TOKENIZERS_PARALLELISM=False
PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:256 \
deepspeed  \
    --include localhost:4,5,6,7 \
    --master_port 29501 train_LLM.py \
    --Qwen_model_path "$LLM_path" \
    --raw_train_path "$train_data_path" \
    --index_train_path "$ccot_index_path" \
    --target_ccot_dir "$ccot_dir" \
    --deepspeed_config ./config/ds_z1_offload_config.conf \
    --epoch 3 \
    --save_step 1024 \
    --micro_batch_size 1 \
    --n_accumulation_steps 16 \
    --use_optimizer True \
    --use_scheduler True \
    --save_dir "$model_save_dir" \
    --cot_cnt 512 \
    --time_step 4 \
    --time_str $TIME_STR 
