TASK=$1
MODEL_DIR=/home/jihaozhe/models/bart-base-fb
for SPLIT in train dev test
do
  for LANG in src tgt
  do
    python -m examples.roberta.multiprocessing_bpe_encoder \
    --encoder-json $MODEL_DIR/bpe/encoder.json \
    --vocab-bpe $MODEL_DIR/bpe/vocab.bpe \
    --inputs "examples/bart/$TASK/$SPLIT.$LANG" \
    --outputs "examples/bart/$TASK/$SPLIT.bpe.$LANG" \
    --workers 20 \
    --keep-empty;
  done
done
