# train 4-bit 64-rank llama-2-7b with LoftQ on GSM8K using one A100
CUDA_VISIBLE_DEVICES=7 python train_adanf_gsm8k.py \
  --model_name_or_path meta-llama/Llama-2-7b-hf \
  --learning_rate 4e-4 \
  --seed 11 \
  --expt_name gsm8k_llama2_7b_2bit_64rank_qlora_7 \
  --output_dir exp_results/ \
  --num_train_epochs 6 \
  --per_device_train_batch_size 2 \
  --gradient_accumulation_steps 4 \
  --save_strategy "steps" \
  --save_steps 12000 \
  --save_total_limit 1 \
  --load_best_model_at_end False \
  --weight_decay 0.1 \
  --warmup_ratio 0.03 \
  --lr_scheduler_type "cosine" \
  --logging_steps 10 \
  --evaluation_strategy "no" \
  --eval_steps 50 \
  --do_train \
  --report_to wandb \
  --run_name "qlora_gsm8k_4bit_lr_4e-4" \
  --full_precision True > llama2_test_results/output_qlora_gsm8k_4bit_lr_4e-4_gs_64_qmapoffset_0.99_accustep_4_epoch_6.txt