# train 4-bit 64-rank llama-2-7b on wikitext-2 using 1 GPU
CUDA_VISIBLE_DEVICES=7 python train_clm_tq.py \
--model_name_or_path meta-llama/Llama-2-7b-hf \
--output_dir exp_results/wikitext-2-qlora-7/ \
--learning_rate 5e-5 \
--seed 11 \
--dataset_name wikitext \
--dataset_config wikitext-2-raw-v1 \
--num_train_epochs 15 \
--per_device_train_batch_size 1 \
--per_device_eval_batch_size 1 \
--gradient_accumulation_steps 8 \
--save_strategy "steps" \
--save_steps 12000 \
--save_total_limit 1 \
--load_best_model_at_end True \
--warmup_steps 120 \
--logging_steps 40 \
--evaluation_strategy "steps" \
--eval_steps 80 \
--do_train --do_eval \
--report_to "wandb" \
--run_name "qlora_wiki_2bit_lr_5e-5" \
--block_size 1024 \
--full_precision True > llama2_outputs/output_qlora_wiki_2bit_lr_5e-5_qmapoffset_0.89.txt