#https://illusionvqa.github.io/
#python3 -m lmms_eval --model gpt4v --model_args model_version="gpt-4o",modality="image" --tasks illusionvqa

tag: "illusionvqa"                  

test_split: "test"                   
fewshot_split: "train"                

# For multiple choice tasks:
output_type: "generate_until"


doc_to_visual: "image"
doc_to_text: !function utils.illusionvqa_doc_to_text
doc_to_target: !function utils.illusionvqa_doc_to_target

lmms_eval_specific_kwargs:
  default:
    prompt_format: mcq
    pre_prompt: "You'll be given an image, an instruction and some options. You have to select the correct one. Do not explain your reasoning. Answer with only the letter which corresponds to the correct option. Do not repeat the entire answer."
    post_prompt: ""


generation_kwargs:
  max_new_tokens: 16
  temperature: 0
  do_sample: False

filter_list:
  - name: "flexible-extract"
    filter:
      - function: !function utils.MultiChoiceRegexFilter
        group_select: 0
        ignore_case: true
        ignore_punctuation: true
        regex_pattern: "([A-Z])\\."

metric_list:
  - metric: exact_match
    aggregation: mean
    higher_is_better: true
    ignore_case: true
    ignore_punctuation: true

fewshot_config:
  sampler: first_n

num_fewshot: 0