#!/bin/bash
#SBATCH --nodes=1
#SBATCH --gres=gpu:4
#SBATCH --ntasks-per-node=1
#SBATCH --partition=gpu
#SBATCH --cpus-per-task=17
#SBATCH --job-name=det-fpn-lvis
#SBATCH --time=72:00:00
#SBATCH --mem=120G
#SBATCH --constraint=gmem24G
#SBATCH --constraint=rtx8k|rtx6k|v100s|p40
#SBATCH --sockets-per-node=1
#SBATCH --reservation=yuki_1

module load cuda/10.1
module load cudnn/v7_10.1

PYTHON='/scratch/shared/beegfs/yuki/envs/py37/bin/python3'


#  EXP_STR=$1
EXP_STR='mocov2/in1k/'
#EXP_STR='random_init'
#EXP_STR='supervisedIN1k'

EXPERIMENT_PATH="/scratch/shared/beegfs/yuki/adiwol/experiments/yuki/${EXP_STR}"
OUTPUT_DIR="${EXPERIMENT_PATH}/DET_FPN_LVIS_1x"
#cp $EXPERIMENT_PATH/DET_FPN_LVIS_1x/model_0059999.pth $OUTPUT_DIR/last_checkpoint
mkdir -p ${OUTPUT_DIR}

cd detection
srun ${PYTHON} train_net.py --dist-url "auto" \
       --config-file configs/lvis.yaml \
       --resume \
       --num-gpus 4 MODEL.WEIGHTS ${EXPERIMENT_PATH}/converted.pkl \
        OUTPUT_DIR ${OUTPUT_DIR}

echo ${OUTPUT_DIR}