#!/bin/bash
# pip3 install mpu
# pip3 install accelerate==0.34.2
# pip3 install torchtypin
# pip3 install transformers
# pip3 install deepspeed==0.15.0
# pip3 install tokenizers==0.14.1
# pip install --upgrade --force-reinstall certifi
# pip install --upgrade datasets huggingface_hub
# pip install torchtyping rouge_score
# pip install --upgrade transformers tokenizers
# pip3 install --no-cache-dir -e /opt/dpcvol/models/pkge/transformers-minillm/.
# pip3 install thop
# pip3 install pytorch_model_summary

# pip3 uninstall py-cpuinfo -y
# pip3 install py-cpuinfo

PYTHONPATH=$PYTHONPATH:/home/naie/.local/lib/python3.9/site-packages

OPTS+=" --dataset_name openwebtext-100k"
OPTS+=" --data_path /opt/dpcvol/datasets/8625883998351850434/datasets/llm/openwebtext-100K/data/train-00000-of-00001-10a4de027237eace.parquet"
OPTS+=" --save_path /opt/dpcvol/models/LLM_Distillation/results/van_lerangene-DesNet178M-78MToken_bs2"  # Only work when validation_file is none
OPTS+=" --config_path /home/work/user-job-dir/app/minillm0807/minillm/configs/learngene/gpt2/gpt2-NEmbed_768_NHead_12_NLayer_14.json"  # The settings of my custom llama model
OPTS+=" --tokenizer_path /opt/dpcvol/datasets/8625883998351850434/ckpt/minillm/minillm_official/gpt2/train/minillm/medium-init-xlarge-sft/"
OPTS+=" --max_tokens 100000000"  # 1亿
OPTS+=" --trust_remote_code True"

python /home/work/user-job-dir/app/minillm0807/minillm/pretrain_hf.py ${OPTS}