#!/bin/bash
pip3 install mpu
pip3 install accelerate==0.34.2
pip3 install torchtypin
pip3 install transformers
pip3 install deepspeed==0.15.0
pip3 install tokenizers==0.14.1
pip install --upgrade --force-reinstall certifi
pip install --upgrade datasets huggingface_hub
pip install torchtyping rouge_score
pip install --upgrade transformers tokenizers
pip3 install --no-cache-dir -e /opt/dpcvol/models/pkge/transformers-minillm/.
pip3 install thop
pip3 install pytorch_model_summary

pip3 uninstall py-cpuinfo -y
pip3 install py-cpuinfo

PYTHONPATH=$PYTHONPATH:/home/naie/.local/lib/python3.9/site-packages

export MASTER_ADDR=localhost
export MASTER_PORT=18503
export WORLD_SIZE=1  # 假设使用 2 个进程
export RANK=0  # 对于第一个进程

OPTS+=" --dataset_name openwebtext"
OPTS+=" --data_path /opt/dpcvol/datasets/8625883998351850434/datasets/llm/minillm/processed_data/openwebtext/gpt2/512/10M/"
OPTS+=" --save_path /opt/dpcvol/models/LLM_Distillation/des-sft/qwen3-4b/gpt2_138M-LInit-1Btoken"  # Only work when validation_file is none
OPTS+=" --config_path /home/work/user-job-dir/app/minillm/configs/learngene/gpt2/gpt2-NEmbed_768_NHead_12_NLayer_14.json"  # The settings of my custom llama model
OPTS+=" --tokenizer_path /opt/dpcvol/datasets/8625883998351850434/ckpt/minillm/minillm_official/gpt2/train/minillm/medium-init-xlarge-sft/"
OPTS+=" --max_tokens 1000000000"  # 10亿
OPTS+=" --trust_remote_code True"
OPTS+=" --LInit_path /opt/dpcvol/datasets/8625883998351850434/ckpt/minillm/learngene/qwen3-4b/Build_descendant_model/NEmbed_768_NHead_12_NLayer_14/sque_init/alpha_0.9_NEmbed_768_NHead_12_NLayer_14/"

python /home/work/user-job-dir/app/minillm/pretrain_hf.py ${OPTS}