
for lr in 1e-04 1e-05 1e-06; do
    python Perplexity_eval.py \
    --model_name_or_path gpt2 \
    --cache_dir ../.. \
    --construct_model_path Constructed_models/model_gpt2_backprop_12_nstep_1_lr_$lr \
    --n_simulation_layers 12 \
    --n_forward_backward 1 \
    --inner_lr $lr \
    --train_fraction 0.5  \
    --data_subset 0 \
    --batch_size 1 \
    --n_layers_pergpu 36 \
    --num_attention_heads 12 \
    --hidden_size 3072 \
    --num_blanks 256 \
    --construct_save_model True;
done    




for lr in 1e-04 1e-05 1e-06; do
    python Perplexity_eval.py \
    --model_name_or_path gpt2-medium \
    --cache_dir ../.. \
    --construct_model_path Constructed_models/model_gpt2-medium_backprop_24_nstep_1_lr_$lr \
    --n_simulation_layers 24 \
    --n_forward_backward 1 \
    --inner_lr $lr \
    --train_fraction 0.5  \
    --data_subset 0 \
    --batch_size 1 \
    --n_layers_pergpu 72 \
    --num_attention_heads 20 \
    --hidden_size 5120 \
    --num_blanks 256 \
    --construct_save_model True;
done    

'''
for lr in 1e-04 1e-05 1e-06; do
    python Perplexity_eval.py \
    --model_name_or_path gpt2-large \
    --cache_dir ../.. \
    --construct_model_path Constructed_models/model_gpt2-large_backprop_36_nstep_1_lr_$lr \
    --n_simulation_layers 36 \
    --n_forward_backward 1 \
    --inner_lr $lr \
    --train_fraction 0.5  \
    --data_subset 0 \
    --batch_size 1 \
    --n_layers_pergpu 36 \
    --num_attention_heads 20 \
    --hidden_size 6400 \
    --num_blanks 320 \
    --construct_save_model True;
done    




for lr in 1e-04 1e-05 1e-06; do
    python Perplexity_eval.py \
    --model_name_or_path gpt2-xl \
    --cache_dir ../.. \
    --construct_model_path Constructed_models/model_gpt2-xl_backprop_24_nstep_1_lr_$lr \
    --n_simulation_layers 24 \
    --n_forward_backward 1 \
    --inner_lr $lr \
    --train_fraction 0.5  \
    --data_subset 1 \
    --batch_size 1 \
    --n_layers_pergpu 24 \
    --num_attention_heads 30 \
    --hidden_size 9600 \
    --num_blanks 320 \
    --construct_save_model False;
done
'''
