# train 4-bit 64-rank llama-2-7b on wikitext-2 using 1 GPU
CUDA_VISIBLE_DEVICES=2 python train_clm.py \
--model_name_or_path LoftQ/Llama-2-7b-hf-4bit-64rank \
--output_dir exp_results/wikitext-2-loftq/ \
--learning_rate 3e-4  \
--seed 11 \
--dataset_name wikitext \
--dataset_config wikitext-2-raw-v1 \
--num_train_epochs 3 \
--per_device_train_batch_size 4 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 16 \
--save_strategy "steps" \
--save_steps 10000 \
--save_total_limit 1 \
--load_best_model_at_end True \
--weight_decay 0.1 \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 5 \
--evaluation_strategy "steps" \
--eval_steps 10 \
--do_train --do_eval \
--report_to "wandb" \
--run_name "loftq_wiki_4bit_lr_3e-4" \
--block_size 1024 > llama2_outputs/output_loftq_wiki_4bit_lr_3e-4_test.txt