for seed in 100 200 300 400 500
do

python experiments/train_sparse_optimizer.py \
--seed $seed \
--task "mrpc" \
--model_name_or_path  "roberta-base" \
--per_device_train_batch_size 32    \
--per_device_eval_batch_size 32    \
--learning_rate 5e-4 \
--weight_decay 0.01 \
--num_train_epochs 30 \
--num_warmup_steps 2 \
--output_dir "roberta_base_sparse" \
--peft_type "sora" \
--lora_alpha 8 \
--threshold 1 \
--r 8 \
--lora_dropout 0.1 \
--report_to "wandb" \
--load_best_model \
--with_tracking  \
--num_retrain 3
done
