{
  "@context": {
    "@language": "en",
    "@vocab": "https://schema.org/",
    "arrayShape": "cr:arrayShape",
    "citeAs": "cr:citeAs",
    "column": "cr:column",
    "conformsTo": "dct:conformsTo",
    "cr": "http://mlcommons.org/croissant/",
    "data": {
      "@id": "cr:data",
      "@type": "@json"
    },
    "dataBiases": "cr:dataBiases",
    "dataCollection": "cr:dataCollection",
    "dataType": {
      "@id": "cr:dataType",
      "@type": "@vocab"
    },
    "dct": "http://purl.org/dc/terms/",
    "extract": "cr:extract",
    "field": "cr:field",
    "fileProperty": "cr:fileProperty",
    "fileObject": "cr:fileObject",
    "fileSet": "cr:fileSet",
    "format": "cr:format",
    "includes": "cr:includes",
    "isArray": "cr:isArray",
    "isLiveDataset": "cr:isLiveDataset",
    "jsonPath": "cr:jsonPath",
    "key": "cr:key",
    "md5": "cr:md5",
    "parentField": "cr:parentField",
    "path": "cr:path",
    "personalSensitiveInformation": "cr:personalSensitiveInformation",
    "recordSet": "cr:recordSet",
    "references": "cr:references",
    "regex": "cr:regex",
    "repeated": "cr:repeated",
    "replace": "cr:replace",
    "sc": "https://schema.org/",
    "separator": "cr:separator",
    "source": "cr:source",
    "subField": "cr:subField",
    "transform": "cr:transform"
  },
  "@type": "sc:Dataset",
  "distribution": [
    {
      "@type": "cr:FileObject",
      "@id": "repo",
      "name": "repo",
      "description": "The Hugging Face git repository.",
      "contentUrl": "https://huggingface.co/datasets/common-pile/comma-dataset/tree/refs%2Fconvert%2Fparquet",
      "encodingFormat": "git+https",
      "sha256": "https://github.com/mlcommons/croissant/issues/80"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-arxiv_abstracts",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "arxiv_abstracts/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-arxiv_papers",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "arxiv_papers/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-biodiversity_heritage_library",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "biodiversity_heritage_library/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-data_provenance_initiative",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "data_provenance_initiative/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-doab",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "doab/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-foodista",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "foodista/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-github_archive",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "github_archive/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-library_of_congress",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "library_of_congress/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-libretexts",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "libretexts/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-news",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "news/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-oercommons",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "oercommons/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-peS2o",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "peS2o/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-pressbooks",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "pressbooks/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-public_domain_review",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "public_domain_review/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-pubmed",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "pubmed/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-python_enhancement_proposals",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "python_enhancement_proposals/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-stackexchange",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "stackexchange/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-stackv2_edu",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "stackv2_edu/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-stackv2_html",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "stackv2_html/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-ubuntu_irc",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "ubuntu_irc/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-uspto",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "uspto/*/*.parquet"
    },
    {
      "@type": "cr:FileSet",
      "@id": "parquet-files-for-config-youtube",
      "containedIn": {
        "@id": "repo"
      },
      "encodingFormat": "application/x-parquet",
      "includes": "youtube/*/*.parquet"
    }
  ],
  "recordSet": [
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "arxiv_abstracts/split_name"
      },
      "@id": "arxiv_abstracts",
      "name": "arxiv_abstracts",
      "description": "Splits for the arxiv_abstracts config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "arxiv_abstracts/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "arxiv_abstracts/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "arxiv_papers/split_name"
      },
      "@id": "arxiv_papers",
      "name": "arxiv_papers",
      "description": "Splits for the arxiv_papers config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "arxiv_papers/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "arxiv_papers/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "biodiversity_heritage_library/split_name"
      },
      "@id": "biodiversity_heritage_library",
      "name": "biodiversity_heritage_library",
      "description": "Splits for the biodiversity_heritage_library config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "biodiversity_heritage_library/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "biodiversity_heritage_library/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "data_provenance_initiative/split_name"
      },
      "@id": "data_provenance_initiative",
      "name": "data_provenance_initiative",
      "description": "Splits for the data_provenance_initiative config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "data_provenance_initiative/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "data_provenance_initiative/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "doab/split_name"
      },
      "@id": "doab",
      "name": "doab",
      "description": "Splits for the doab config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "doab/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "doab/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "foodista/split_name"
      },
      "@id": "foodista",
      "name": "foodista",
      "description": "Splits for the foodista config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "foodista/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "foodista/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "github_archive/split_name"
      },
      "@id": "github_archive",
      "name": "github_archive",
      "description": "Splits for the github_archive config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "github_archive/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "github_archive/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "library_of_congress/split_name"
      },
      "@id": "library_of_congress",
      "name": "library_of_congress",
      "description": "Splits for the library_of_congress config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "library_of_congress/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "library_of_congress/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "libretexts/split_name"
      },
      "@id": "libretexts",
      "name": "libretexts",
      "description": "Splits for the libretexts config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "libretexts/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "libretexts/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "news/split_name"
      },
      "@id": "news",
      "name": "news",
      "description": "Splits for the news config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "news/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "news/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "oercommons/split_name"
      },
      "@id": "oercommons",
      "name": "oercommons",
      "description": "Splits for the oercommons config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "oercommons/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "oercommons/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "peS2o/split_name"
      },
      "@id": "peS2o",
      "name": "peS2o",
      "description": "Splits for the peS2o config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "peS2o/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "peS2o/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "pressbooks/split_name"
      },
      "@id": "pressbooks",
      "name": "pressbooks",
      "description": "Splits for the pressbooks config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "pressbooks/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "pressbooks/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "public_domain_review/split_name"
      },
      "@id": "public_domain_review",
      "name": "public_domain_review",
      "description": "Splits for the public_domain_review config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "public_domain_review/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "public_domain_review/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "pubmed/split_name"
      },
      "@id": "pubmed",
      "name": "pubmed",
      "description": "Splits for the pubmed config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "pubmed/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "pubmed/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "python_enhancement_proposals/split_name"
      },
      "@id": "python_enhancement_proposals",
      "name": "python_enhancement_proposals",
      "description": "Splits for the python_enhancement_proposals config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "python_enhancement_proposals/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "python_enhancement_proposals/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "stackexchange/split_name"
      },
      "@id": "stackexchange",
      "name": "stackexchange",
      "description": "Splits for the stackexchange config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "stackexchange/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "stackexchange/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "stackv2_edu/split_name"
      },
      "@id": "stackv2_edu",
      "name": "stackv2_edu",
      "description": "Splits for the stackv2_edu config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "stackv2_edu/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "stackv2_edu/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "stackv2_html/split_name"
      },
      "@id": "stackv2_html",
      "name": "stackv2_html",
      "description": "Splits for the stackv2_html config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "stackv2_html/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "stackv2_html/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "ubuntu_irc/split_name"
      },
      "@id": "ubuntu_irc",
      "name": "ubuntu_irc",
      "description": "Splits for the ubuntu_irc config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "ubuntu_irc/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "ubuntu_irc/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "uspto/split_name"
      },
      "@id": "uspto",
      "name": "uspto",
      "description": "Splits for the uspto config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "uspto/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "uspto/split_name": "train"
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "dataType": "cr:Split",
      "key": {
        "@id": "youtube/split_name"
      },
      "@id": "youtube",
      "name": "youtube",
      "description": "Splits for the youtube config.",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "youtube/split_name",
          "dataType": "sc:Text"
        }
      ],
      "data": [
        {
          "youtube/split_name": "train"
        }
      ]
    }
  ],
  "conformsTo": "http://mlcommons.org/croissant/1.1",
  "name": "comma-dataset",
  "description": "Dataset used to train Comma 7B.",
  "alternateName": [
    "common-pile/comma-dataset"
  ],
  "datePublished": "20250515",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "version": "0.0.1",
  "creator": {
    "@type": "Organization",
    "name": "Common Pile",
    "url": "https://huggingface.co/common-pile"
  },
  "keywords": [
    "100M - 1B",
    "Open License",
    "LLMs",
    "Pretraining",
    "Text",
    "Datasets",
    "Dask",
    "Croissant",
    "🇺🇸 Region: US"
  ],
  "url": "https://huggingface.co/datasets/common-pile/comma-dataset"
}