# train 4-bit 64-rank llama-2-7b on wikitext-2 using 1 GPU
CUDA_VISIBLE_DEVICES=2,3 python train_clm_tq.py \
--model_name_or_path meta-llama/Llama-2-7b-hf \
--output_dir exp_results/wikitext-2-tq-2bit/ \
--learning_rate 3e-3 \
--seed 11 \
--dataset_name wikitext \
--dataset_config wikitext-2-raw-v1 \
--num_train_epochs 15 \
--per_device_train_batch_size 4 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 16 \
--save_strategy "steps" \
--save_steps 10000 \
--save_total_limit 1 \
--load_best_model_at_end True \
--weight_decay 0.1 \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 2 \
--evaluation_strategy "steps" \
--eval_steps 20 \
--do_train --do_eval \
--report_to "wandb" \
--run_name "tq_wiki_2bit_lr_3e-3_epoch_15" \
--block_size 1024 \
--full_precision True > llama2_outputs/output_tq_wiki_2bit_lr_3e-3_epoch_15.txt