{
  "@context": {
    "@vocab": "https://schema.org/",
    "cr": "http://mlcommons.org/croissant/",
    "distribution": {
      "@id": "cr:distribution",
      "@type": "@id"
    },
    "recordSet": {
      "@id": "cr:recordSet",
      "@type": "@id"
    },
    "field": {
      "@id": "cr:field",
      "@type": "@id"
    },
    "source": {
      "@id": "cr:source"
    },
    "fileObject": {
      "@id": "cr:fileObject",
      "@type": "@id"
    },
    "extract": {
      "@id": "cr:extract"
    },
    "column": {
      "@id": "cr:column"
    }
  },
  "@type": "Dataset",
  "name": "ClinBench",
  "description": "ClinBench is an open-source, multi-model, multi-domain benchmarking framework for the rigorous evaluation of LLMs on structured information extraction tasks from unstructured clinical notes. This metadata describes the benchmark task configurations, which rely on specific subsets of publicly available clinical datasets, identified via provided record ID lists. Users must obtain access to the original datasets (TCGA, MIMIC-III, MIMIC-IV-ECG) separately, adhering to their respective access policies.",
  "url": "https://github.com/ismaelvillanuevamiranda/ClinBench/",
  "license": "https://github.com/ismaelvillanuevamiranda/ClinBench/blob/main/LICENSE",
  "citation": [
    "ClinBench: A Standardized Multi-Domain Framework for Evaluating Large Language Models in Clinical Information Extraction"
  ],
  "datePublished": "2025-05-16",
  "version": "1.0.0",
  "creator": {
    "@type": "Organization",
    "name": "Quantitative Biomedical Research Center, University of Texas Southwestern Medical Center"
  },
  "keywords": [
    "clinical NLP",
    "LLM evaluation",
    "benchmarking",
    "information extraction",
    "TCGA",
    "MIMIC-IV-ECG",
    "MIMIC-III",
    "SDOH",
    "reproducibility"
  ],
  "distribution": [
    {
      "@type": "cr:FileObject",
      "@id": "clinbench-tcga-lung-ids-csv",
      "name": "TCGA Lung Cancer Cohort Identifiers",
      "contentUrl": "https://raw.githubusercontent.com/ismaelvillanuevamiranda/ClinBench/main/benchmark_data_definitions/tcga_lung/Lung_notes_id.csv",
      "encodingFormat": "text/csv",
      "sha256": "398084afbc9ae3ac0bbac93b8a5605f91c2b3060693d821870b26deaea8a541d"
    },
    {
      "@type": "cr:FileObject",
      "@id": "clinbench-mimic-ecg-ids-csv",
      "name": "MIMIC-IV-ECG Atrial Fibrillation Cohort Identifiers",
      "contentUrl": "https://raw.githubusercontent.com/ismaelvillanuevamiranda/ClinBench/main/benchmark_data_definitions/mimic_ecg_af/ECG_notes_id.csv",
      "encodingFormat": "text/csv",
      "sha256": "e912cd2448c8d8e70ddcc27b0b29a026938da9f6161372d5afbff14d72c74ff2"
    },
    {
      "@type": "cr:FileObject",
      "@id": "clinbench-mimic-sbdh-ids-csv",
      "name": "MIMIC-III SDOH Cohort Identifiers",
      "contentUrl": "https://raw.githubusercontent.com/ismaelvillanuevamiranda/ClinBench/main/benchmark_data_definitions/mimic_sdoh/sdoh_subjects_id.csv",
      "encodingFormat": "text/csv",
      "sha256": "e912cd2448c8d8e70ddcc27b0b29a026938da9f6161372d5afbff14d72c74ff2"
    }
  ],
  "recordSet": [
    {
      "@type": "cr:RecordSet",
      "@id": "tcga-lung-records",
      "name": "tcga_lung_report_ids",
      "description": "List of identifiers for TCGA pathology reports",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "tcga-lung-records-pid",
          "name": "pid",
          "description": "Unique identifier for the TCGA pathology report",
          "dataType": "Text",
          "source": {
            "fileObject": {
              "@id": "clinbench-tcga-lung-ids-csv"
            },
            "extract": {
              "column": "pid"
            }
          }
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "@id": "mimic-ecg-af-records",
      "name": "mimic_ecg_af_report_ids",
      "description": "List of identifiers for MIMIC-IV-ECG reports",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "mimic-ecg-af-records-noteid",
          "name": "note_id",
          "description": "Unique identifier for the MIMIC-IV-ECG report",
          "dataType": "Text",
          "source": {
            "fileObject": {
              "@id": "clinbench-mimic-ecg-ids-csv"
            },
            "extract": {
              "column": "note_id"
            }
          }
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "@id": "mimic-sbdh-records",
      "name": "mimic_sbdh_admission_ids",
      "description": "List of MIMIC-III Subject & Admission IDs",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "mimic-sbdh-records-subjectid",
          "name": "SUBJECT_ID",
          "dataType": "Text",
          "source": {
            "fileObject": {
              "@id": "clinbench-mimic-sbdh-ids-csv"
            },
            "extract": {
              "column": "SUBJECT_ID"
            }
          }
        },
        {
          "@type": "cr:Field",
          "@id": "mimic-sbdh-records-hadmid",
          "name": "HADM_ID",
          "dataType": "Text",
          "source": {
            "fileObject": {
              "@id": "clinbench-mimic-sbdh-ids-csv"
            },
            "extract": {
              "column": "HADM_ID"
            }
          }
        }
      ]
    }
  ],
  "hasPart": [
    {
      "@type": "Dataset",
      "name": "ClinBench \u2013 TCGA Lung Cancer Staging Task Configuration",
      "description": "Benchmark task configuration for lung cancer staging (pT, pN, overall stage, histologic diagnosis) using 774 selected pathology reports from TCGA. Identifiers are provided in Lung_notes_id.csv. Users must obtain access to the original TCGA data independently and adhere to data use policies.",
      "url": "https://github.com/ismaelvillanuevamiranda/ClinBench/tree/main/benchmark_data_definitions/tcga_lung/",
      "license": "https://github.com/ismaelvillanuevamiranda/ClinBench/blob/main/LICENSE",
      "citation": [
        "ClinBench: A Standardized Multi-Domain Framework for Evaluating Large Language Models in Clinical Information Extraction"
      ],
      "distribution": [
        {
          "@id": "clinbench-tcga-lung-ids-csv"
        }
      ],
      "recordSet": [
        {
          "@id": "tcga-lung-records"
        }
      ],
      "keywords": [
        "lung cancer",
        "TCGA",
        "pathology reports",
        "cancer staging",
        "clinical NLP"
      ]
    },
    {
      "@type": "Dataset",
      "name": "ClinBench \u2013 MIMIC-IV-ECG Atrial Fibrillation Detection Task Configuration",
      "description": "Benchmark task configuration for atrial fibrillation detection using 700 selected ECG report texts from the MIMIC-IV-ECG Database v1.0. Identifiers are provided in ECG_notes_id.csv. Users must obtain PhysioNet credentialing and sign the DUA to access MIMIC-IV-ECG.",
      "url": "https://github.com/ismaelvillanuevamiranda/ClinBench/tree/main/benchmark_data_definitions/mimic_ecg_af/",
      "license": "https://github.com/ismaelvillanuevamiranda/ClinBench/blob/main/LICENSE",
      "citation": [
        "ClinBench: A Standardized Multi-Domain Framework for Evaluating Large Language Models in Clinical Information Extraction"
      ],
      "distribution": [
        {
          "@id": "clinbench-mimic-ecg-ids-csv"
        }
      ],
      "recordSet": [
        {
          "@id": "mimic-ecg-af-records"
        }
      ],
      "keywords": [
        "atrial fibrillation",
        "ECG",
        "MIMIC-IV-ECG",
        "clinical NLP"
      ]
    },
    {
      "@type": "Dataset",
      "name": "ClinBench \u2013 MIMIC-III SDOH Task Configuration",
      "description": "Benchmark task configuration for SDOH extraction using 1405 discharge summaries from MIMIC-III, annotated for Economics, Environment, Alcohol & Tobacco Use per Ahsan et al. (2021). Identifiers in sdoh_subjects_id.csv. Access requires PhysioNet credentialing & DUA.",
      "url": "https://github.com/ismaelvillanuevamiranda/ClinBench/tree/main/benchmark_data_definitions/mimic_sdoh/",
      "license": "https://github.com/ismaelvillanuevamiranda/ClinBench/blob/main/LICENSE",
      "citation": [
        "ClinBench: A Standardized Multi-Domain Framework for Evaluating Large Language Models in Clinical Information Extraction"
      ],
      "distribution": [
        {
          "@id": "clinbench-mimic-sbdh-ids-csv"
        }
      ],
      "recordSet": [
        {
          "@id": "mimic-sbdh-records"
        }
      ],
      "keywords": [
        "social determinants of health",
        "SDOH",
        "MIMIC-III",
        "clinical NLP"
      ]
    }
  ]
}