#!/bin/bash
# pip3 install accelerate==0.34.2
# pip3 install torchtypin
# pip3 install transformers
# pip3 install deepspeed==0.15.0
# pip3 install tokenizers==0.11.1

# pip3 uninstall py-cpuinfo -y
# pip3 install py-cpuinfo

PYTHONPATH=$PYTHONPATH:/home/naie/.local/lib/python3.9/site-packages

export MASTER_ADDR=localhost
export MASTER_PORT=28500
export WORLD_SIZE=1  # 假设使用 2 个进程
export RANK=0  # 对于第一个进程

OPTS+=" --dataset_name openwebtext-100k"
OPTS+=" --data_path /opt/dpcvol/datasets/8625883998351850434/datasets/llm/openwebtext-100K/data/train-00000-of-00001-10a4de027237eace.parquet"
OPTS+=" --save_path /opt/dpcvol/models/LLM_Distillation/results/van_lerangene-DesNet235M-78MToken_bs16"  # Only work when validation_file is none
# OPTS+=" --config_path "  # The settings of my custom llama model
OPTS+=" --tokenizer_path /opt/dpcvol/datasets/8625883998351850434/ckpt/minillm/learngene/previous_learngene/gpt2-xl-van-learngene/174M/"
OPTS+=" --max_tokens 100000000"  # 1亿
OPTS+=" --trust_remote_code True"
OPTS+=" --LInit_path /opt/dpcvol/datasets/8625883998351850434/ckpt/minillm/learngene/previous_learngene/gpt2-xl-van-learngene/235M/"

python /home/work/user-job-dir/app/minillm0807/minillm/pretrain_hf.py ${OPTS}