dataset_path: lmms-lab/MMMU
task: "mmmu_val_reasoning"
test_split: validation
output_type: generate_until
doc_to_visual: !function utils.mmmu_doc_to_visual
doc_to_text: !function utils.mmmu_doc_to_text
doc_to_target: "answer"
tag: visual_reasoning_collection
# The return value of process_results will be used by metrics
process_results: !function utils.mmmu_reasoning_process_results
# repeats: 8
# filter_list:
#   # - name: "pass@64"
#   #   filter:
#   #     - function: "take_first_k"
#   #       k: 64
#   - name: "pass@8"
#     filter:
#       - function: "take_first_k"
#         k: 8

metric_list:
  - metric: llm_as_judge_eval
    aggregation: mean
    higher_is_better: true

generation_kwargs:
  max_new_tokens: 16384
  until:
    - "</s>"
    - "Q:"
    - "<|im_end|>"

metadata:
  version: 0.0
  interleaved_format: false
