# train 4-bit 64-rank llama-2-7b with LoftQ on GSM8K using one A100
CUDA_VISIBLE_DEVICES=4 python train_gsm8k.py \
  --model_name_or_path LoftQ/Llama-2-7b-hf-4bit-64rank \
  --learning_rate 3e-4 \
  --seed 11 \
  --expt_name gsm8k_llama2_7b_4bit_64rank_loftq \
  --output_dir exp_results/ \
  --num_train_epochs 6 \
  --per_device_train_batch_size 2 \
  --gradient_accumulation_steps 8 \
  --save_strategy "steps" \
  --save_steps 12000 \
  --save_total_limit 1 \
  --load_best_model_at_end True \
  --weight_decay 0.1 \
  --warmup_ratio 0.03 \
  --lr_scheduler_type "cosine" \
  --logging_steps 10 \
  --evaluation_strategy "steps" \
  --eval_steps 50 \
  --do_train --do_eval \
  --report_to wandb > llama2_outputs/output_loftq_gsm8k_2bit_lr_3e-4.txt