OUTPUT_DIR='./deberta_stsb'
num_gpus=4
NCCL_P2P_DISABLE=1 python -m torch.distributed.launch --nproc_per_node=$num_gpus --master_port=12345 non-GPT-2/examples/pytorch/text-classification/run_glue.py \
    --save_total_limit 10 \
    --model_name_or_path microsoft/deberta-large \
    --task_name stsb \
    --output_dir ${OUTPUT_DIR} \
    --do_train \
    --do_eval \
    --save_steps 50 \
    --seed 1 \
    --per_device_train_batch_size 8 \
    --per_device_eval_batch_size 8 \
    --overwrite_output_dir \
    --logging_steps 50 \
    --overwrite_output_dir \
    --weight_decay 0 \
    --load_best_model_at_end True \
    --metric_for_best_model eval_pearson \
    --evaluation_strategy steps \
	--warmup_steps 50 \
	--learning_rate 1e-5 \
	--cls_dropout 0.2 \
	--fp16 True \
    --num_train_epochs 6 \
    --max_seq_len 128 
 
    i=5
    OUTPUT_DIR=./deberta_stsb_pruned_0.${i}_finetune_w0
    num_gpus=4
    NCCL_P2P_DISABLE=1 nohup python -m torch.distributed.launch --nproc_per_node=$num_gpus --master_port=12345 non-GPT-2/examples/pytorch/text-classification/run_glue_prune_tune.py \
        --save_total_limit 1 \
        --model_name_or_path deberta_stsb \
        --task_name stsb \
        --output_dir ${OUTPUT_DIR} \
        --do_train \
        --do_eval \
        --save_steps 100 \
        --seed 1 \
        --per_device_train_batch_size 8 \
        --per_device_eval_batch_size 8 \
        --overwrite_output_dir \
        --logging_steps 100 \
        --overwrite_output_dir \
        --weight_decay 0 \
        --load_best_model_at_end True \
        --metric_for_best_model eval_pearson \
        --evaluation_strategy steps \
        --warmup_steps 50 \
        --learning_rate 7e-6 \
        --cls_dropout 0.3 \
        --fp16 True \
        --max_seq_len 128  \
        --pruning_ratio 0.${i} > 1004_stsb_pruned_0.${i}_finetune_w0.out &

OUTPUT_DIR='./deberta_stsb_16'
num_gpus=4
NCCL_P2P_DISABLE=1 python -m torch.distributed.launch --nproc_per_node=$num_gpus --master_port=12345 non-GPT-2/examples/pytorch/text-classification/run_glue.py \
    --save_total_limit 1 \
    --model_name_or_path microsoft/deberta-large \
    --task_name stsb \
    --output_dir ${OUTPUT_DIR} \
    --do_train \
    --do_eval \
    --save_steps 20 \
    --seed 1 \
    --per_device_train_batch_size 8 \
    --per_device_eval_batch_size 8 \
    --overwrite_output_dir \
    --logging_steps 20 \
    --overwrite_output_dir \
    --load_best_model_at_end True \
    --metric_for_best_model eval_pearson \
    --evaluation_strategy steps \
    --apply_lora \
    --lora_r 16 \
    --lora_alpha 32 \
    --warmup_steps 50 \
	--learning_rate 8e-5 \
	--cls_dropout 0.2 \
	--fp16 True \
    --num_train_epochs 6 \
    --max_seq_len 128 

    i=5
    OUTPUT_DIR=./deberta_stsb_rank_16_pruned_0.${i}_finetune_wab
    num_gpus=4
    CUDA_VISIBLE_DEVICES=0,1,2,3 NCCL_P2P_DISABLE=1 python -m torch.distributed.launch --nproc_per_node=$num_gpus --master_port=12345 non-GPT-2/examples/pytorch/text-classification/run_glue_prune_tune.py \
        --save_total_limit 1 \
        --model_name_or_path deberta_stsb_16 \
        --lora_path deberta_stsb_16 \
        --task_name stsb \
        --output_dir ${OUTPUT_DIR} \
        --do_train \
        --do_eval \
        --save_steps 20 \
        --seed 1 \
        --per_device_train_batch_size 8 \
        --per_device_eval_batch_size 8 \
        --overwrite_output_dir \
        --logging_steps 20 \
        --load_best_model_at_end True \
        --metric_for_best_model eval_pearson \
        --pruning_ratio 0.${i} \
        --evaluation_strategy steps \
        --warmup_steps 20 \
        --learning_rate 8e-5 \
        --cls_dropout 0.3 \
        --fp16 True \
        --max_seq_len 128 \
        > 1004_stsb_rank_16_pruned_0.${i}_finetune_wab.out 


i=3
    OUTPUT_DIR=./deberta_stsb_rank_16_pruned_0.${i}_finetune_wab
    num_gpus=4
    CUDA_VISIBLE_DEVICES=0,1,2,3 NCCL_P2P_DISABLE=1 python -m torch.distributed.launch --nproc_per_node=$num_gpus --master_port=12345 non-GPT-2/examples/pytorch/text-classification/run_glue_prune_tune.py \
        --save_total_limit 1 \
        --model_name_or_path deberta_stsb_16 \
        --lora_path deberta_stsb_16 \
        --task_name stsb \
        --output_dir ${OUTPUT_DIR} \
        --do_train \
        --do_eval \
        --save_steps 20 \
        --seed 1 \
        --per_device_train_batch_size 8 \
        --per_device_eval_batch_size 8 \
        --overwrite_output_dir \
        --logging_steps 20 \
        --load_best_model_at_end True \
        --metric_for_best_model eval_pearson \
        --pruning_ratio 0.${i} \
        --evaluation_strategy steps \
        --warmup_steps 20 \
        --learning_rate 8e-5 \
        --cls_dropout 0.3 \
        --fp16 True \
        --max_seq_len 128 \
        > 1004_stsb_rank_16_pruned_0.${i}_finetune_wab.out 
