# train 4-bit 64-rank llama-2-7b on wikitext-2 using 1 GPU
CUDA_VISIBLE_DEVICES=1 python train_clm.py \
--model_name_or_path LoftQ/Llama-2-7b-hf-4bit-64rank \
--output_dir exp_results/wikitext-2-loftq-1/ \
--learning_rate 5e-5 \
--seed 11 \
--dataset_name wikitext \
--dataset_config wikitext-2-raw-v1 \
--num_train_epochs 15 \
--per_device_train_batch_size 1 \
--per_device_eval_batch_size 1 \
--gradient_accumulation_steps 8 \
--save_strategy "steps" \
--save_steps 12000 \
--save_total_limit 1 \
--load_best_model_at_end True \
--warmup_steps 120 \
--logging_steps 40 \
--evaluation_strategy "steps" \
--eval_steps 80 \
--do_train --do_eval \
--report_to "wandb" \
--run_name "loftq_wiki_4bit_lr_5e-5" \
--block_size 1024 > llama2_outputs/output_loftq_wiki_4bit_lr_5e-5_batch_1_gradacc_8_epoch_15.txt