#!/bin/bash




SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"

CONFIG_FILE=${1:-"${SCRIPT_DIR}/configs/base_config.sh"}
echo "CONFIG_FILE: ${CONFIG_FILE}"

source ${REPO_ROOT}/.venv/bin/activate
source $CONFIG_FILE





declare -a inference_datasets=(

  "LLTM-cruxeval-numeric-depth-val"

  "LLTM-livecodebench-all-numeric-depth-val"














)

VAL_DATA_PATH="${REPO_ROOT}/scripts/instruction/convert_datasets"

echo "USE_GUIDED_DECODING: ${USE_GUIDED_DECODING}"

for i in "${!inference_datasets[@]}"; do
  cuda_device_id=$i
  current_eval_dataset="${inference_datasets[$i]}"
  ${SCRIPT_DIR}/inference.sh \
    "$EPOCH" \
    "$ITERATION" \
    "$NUM_GPUS" \
    "$BASE_MODEL" \
    "$TRAIN_DATASET" \
    "$current_eval_dataset" \
    "$INFERENCE_MODEL_PATH" \
    "$VAL_DATA_PATH" \
    "$SYSTEM_PROMPT" \
    "$USE_GUIDED_DECODING"
done

${SCRIPT_DIR}/utils/aggregate_inference_outputs.sh \
  $TRAIN_DATASET \
  $BASE_MODEL \
  $ITERATION \
  $EPOCH \
  $USE_GUIDED_DECODING
