pip3 install deepspeed==0.14.0

MASTER_ADDR=${ARNOLD_WORKER_0_HOST}
MASTER_PORT=(${ARNOLD_WORKER_0_PORT//,/ })
NPROC_PER_NODE=${ARNOLD_WORKER_GPU}
NNODES=${ARNOLD_WORKER_NUM}
NODE_RANK=${ARNOLD_ID}
export MASTER_ADDR
export MASTER_PORT
export NPROC_PER_NODE
export NNODES
export NODE_RANK

cd /.../.../.../fastchat/

RUN_CMD="accelerate launch --config_file=/.../.../.../fastchat/accelerate_config/deepspeed_zero3_m2_n8.yaml --master_port=${MASTER_PORT} --master_addr=${MASTER_ADDR} --nproc_per_node=${NPROC_PER_NODE} --nnodes=${NNODES} --node_rank=${NODE_RANK} "
export RUN_CMD

OTHER="--use_peft --lora_r=16 --lora_alpha=16 --my_task_name=starcoder2-15b-code-dpolora "
export OTHER

bash dpo.code.a100.starcoder2-15b.sh
