#!/usr/bin/env bash

export OMP_NUM_THREADS=1
export MKL_NUM_THREADS=1
#export CUDA_VISIBLE_DEVICES=0
#export CUDA_VISIBLE_DEVICES=0,1,2,3
#export CUDA_VISIBLE_DEVICES=4,5,6,7
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7

startTime=`date +"%Y-%m-%d %H:%M:%S"`

# nohup tools/dist_train_increment.sh &

CONFIG=${1:-'configs/yoloy/yoloy_resnet_qoqo_il.py'}
WORKDIR=${2:-'yoloy_r50_mtg8b16_qoqo_il41111_u1px'}
CHECKPOINT=${4:-''}

GPUS=${3:-8}
PORT=${PORT:-36198}

if [ ! -d "$WORKDIR" ];then
  mkdir $WORKDIR
  touch $WORKDIR'/nohup'
  echo "ok: $WORKDIR"
else
  touch $WORKDIR'/nohup'
  echo "ok: $WORKDIR"
fi

PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \
          $(dirname "$0")/train_increment.py \
          --config=$CONFIG \
          --work-dir=$WORKDIR \
          --resume-from=$CHECKPOINT \
          --launcher=pytorch ${@:3}

endTime=`date +"%Y-%m-%d %H:%M:%S"`
st=`date -d  "$startTime" +%s`
et=`date -d  "$endTime" +%s`

sumHours=$((($et-$st)/3600))
sumMinutes=$((($et-$st)%60))

