#!/bin/bash
#SBATCH --nodes=1
#SBATCH --gres=gpu:2
#SBATCH --ntasks-per-node=1
#SBATCH --partition=gpu
#SBATCH --cpus-per-task=9
#SBATCH --job-name=det-fpn
#SBATCH --time=60:00:00
#SBATCH --mem=120G
#SBATCH --constraint=gmem24G
#SBATCH --constraint=rtx8k|rtx6k|v100s|p40|a40
#SBATCH --exclude=gnodee8
#SBATCH --sockets-per-node=1

module load cuda/10.1
module load cudnn/v7_10.1

PYTHON='/scratch/shared/beegfs/yuki/envs/py37/bin/python3'


EXP_STR=$1
#EXP_STR='mocov2/user-bal-zoom0.1-BS512-LR0.06-EP200-temp-0.2'

EXPERIMENT_PATH="/scratch/shared/beegfs/yuki/adiwol/experiments/yuki/${EXP_STR}"
OUTPUT_DIR="${EXPERIMENT_PATH}/DET_FPN_1x"
mkdir -p ${OUTPUT_DIR}

cd detection
srun ${PYTHON} train_net.py --dist-url "auto" \
       --config-file configs/FPN_COCO_2gpus.yaml \
       --resume \
       --num-gpus 2 MODEL.WEIGHTS ${EXPERIMENT_PATH}/converted.pkl \
        OUTPUT_DIR ${OUTPUT_DIR}

echo ${OUTPUT_DIR}