#!/bin/bash

#python run_transformer.py --seeds 6312 --lr 0.0001 --device 'cuda' --layers 4 --wdecay 0.0 --dropout 0.0 --attnheads 1 --n_epochs 3000 --pe 2dpe --pe_init 1.0
optim="AdamW" #AdamW
pestring="learn"
pe_init="2.0"
curriculum="All"
for attnhead in 2 4; do #1 4 8; do #1 4; do
    for layer in 4; do #2 3 4 5; do
        for dropout in 0.0; do #0.1 0.2 0.3; do
            for decay in 0.0; do # 0.1; do #0.1; do # do 0.01 0.05 0.1 # done 0.0 0.01
                #for lr in 0.0001; do
                for lr in 0.001; do
                #for seed in #2235 6312 6068 9742 8880 2197 669 6256 3309 2541 8643 7785 195 6914 29; do
                    jbsub -mail -cores 1+1 -queue x86_24h -mem 32g -proj lstnn -name h${attnhead}l${layer}${pestring}${pe_init}-do${dropout}de${decay} \
                        python run_transformer.py --seeds 7785 195 6914 29 6312 \
                        --lr $lr --device "cuda" \
                        --col 0 --n_epochs 4000 \
                        --embedding_dim 160 \
                        --layers $layer \
                        --attnheads $attnhead \
                        --wdecay $decay \
                        --dropout $dropout \
                        --curriculum $curriculum \
                        --pe $pestring --pe_init $pe_init --optimizer $optim

                    jbsub -mail -cores 1+1 -queue x86_24h -mem 32g -proj lstnn -name h${attnhead}l${layer}${pestring}${pe_init}-do${dropout}de${decay} \
                        python run_transformer.py --seeds 2235 6068 9742 8880 2197 \
                        --lr $lr --device "cuda" \
                        --col 0 --n_epochs 4000 \
                        --embedding_dim 160 \
                        --layers $layer \
                        --attnheads $attnhead \
                        --wdecay $decay \
                        --dropout $dropout \
                        --curriculum $curriculum \
                        --pe $pestring --pe_init $pe_init --optimizer $optim

                    jbsub -mail -cores 1+1 -queue x86_24h -mem 32g -proj lstnn -name h${attnhead}l${layer}${pestring}${pe_init}-do${dropout}de${decay} \
                        python run_transformer.py --seeds 669 6256 3309 2541 8643 \
                        --lr $lr --device "cuda" \
                        --col 0 --n_epochs 4000 \
                        --embedding_dim 160 \
                        --layers $layer \
                        --attnheads $attnhead \
                        --wdecay $decay \
                        --dropout $dropout \
                        --curriculum $curriculum \
                        --pe $pestring --pe_init $pe_init --optimizer $optim
                done
            done
        done 
    done
done
