#!/bin/bash
# 60m examples
# Run full train
$1 main.py --model_config configs/llama_60m.json --dataset_path preprocessed_data/allenai/c4_en_t5-base_256_reduced --batch_size 48 --total_batch_size 480 --lr 1e-3 --max_length 256 --num_training_steps 20000 --save_every 500 --eval_every 250 --keep_checkpoints 3 --num_workers 8
# Run switch lora
$1 main.py --model_config configs/llama_60m.json --dataset_path preprocessed_data/allenai/c4_en_t5-base_256_reduced --batch_size 48 --total_batch_size 480 --lr 4e-3 --max_length 256 --num_training_steps 20000 --save_every 500 --eval_every 250 --num_workers 8 --keep_checkpoints 3 --switch_lora --switch_lora_descent_rate 0.1 --switch_lora_interval 40 --lora_rank 128

