{
  "created_at": "2026-08-21T02:44:58.134500Z",
  "dataset": "lm1b",
  "dataset_id": "billion-word-benchmark/lm1b",
  "document_counts": {
    "train": 30301028,
    "validation": 306688
  },
  "files": [
    {
      "path": "dataset_dict.json",
      "sha256": "88b5b05fc757e76e7edc779db90e96cb6d76538b93165d6ce962a818801babf4",
      "size_bytes": 35
    },
    {
      "path": "train/data-00000-of-00004.arrow",
      "sha256": "c1842d29417604f3c3ee4ba6fd648adebf47ef3d33588ca7eacad5f7e60dc276",
      "size_bytes": 454895728
    },
    {
      "path": "train/data-00001-of-00004.arrow",
      "sha256": "395ff7ef22b9f42ed72ab4d4f319d1a5da1756ae85d5bd498c3b5b4dfc96273e",
      "size_bytes": 454895728
    },
    {
      "path": "train/data-00002-of-00004.arrow",
      "sha256": "1d8ed17f183a09947a7035b6a0571662d0797e6b236897e5b28e378c7ea8a3c0",
      "size_bytes": 454895472
    },
    {
      "path": "train/data-00003-of-00004.arrow",
      "sha256": "2272f8e8d83a5189d0a2d39e0010949f2b7914aae4f9e5cb2316f392fc699e56",
      "size_bytes": 454895472
    },
    {
      "path": "train/dataset_info.json",
      "sha256": "1f4e3b935de233b63542c8d29a7bc819e11c08522ca8e83d3f3e121e3616e081",
      "size_bytes": 805
    },
    {
      "path": "train/state.json",
      "sha256": "d9f4fca87f6e722f0dddc1f7ae811d0552830540d0690666d04735330882a410",
      "size_bytes": 427
    },
    {
      "path": "validation/data-00000-of-00001.arrow",
      "sha256": "fff03f2a6785bc54a9d54074a409d89bc2e71042beaf3a4ccef724a9c74c14d6",
      "size_bytes": 18439664
    },
    {
      "path": "validation/dataset_info.json",
      "sha256": "5d94f384698efbc6ef827c2dd2e776a6a9ac89be2f92dcf9490da6a6063949e1",
      "size_bytes": 696
    },
    {
      "path": "validation/state.json",
      "sha256": "ba4addb0d152b776d1a256db649e50707e16d2e932839cb9e547de20101a0232",
      "size_bytes": 250
    }
  ],
  "packed_sequence_counts": {
    "train": 7102854,
    "validation": 71979
  },
  "processed_path": "data/processed/lm1b-bert-128",
  "processing_contract": {
    "dataset": "lm1b",
    "dataset_id": "billion-word-benchmark/lm1b",
    "expected_document_counts": {
      "train": 30301028,
      "validation": 306688
    },
    "packing": {
      "algorithm": "document_eos_continuous_bos_eos",
      "version": 1
    },
    "schema_version": 1,
    "sequence_length": 128,
    "source_materialization_revision": "8d52bfd3cc2819fc1166dbb8144c328e2690de3e",
    "source_revision": "35161838ea9e05371a25a8db037f94fcae4c2064",
    "split_expression": {
      "train": "train",
      "validation": "test"
    },
    "tokenizer_id": "bert-base-uncased",
    "tokenizer_revision": "86b5e0934494bd15c9632b12f734a8a67f723594",
    "vocab_size": 30522
  },
  "processing_contract_sha256": "e64a9a0df35ae112075596ccd5958a363a09075fb61515c058f0fc3e2eb50668",
  "schema_version": 1,
  "sequence_length": 128,
  "source_materialization_revision": "8d52bfd3cc2819fc1166dbb8144c328e2690de3e",
  "source_revision": "35161838ea9e05371a25a8db037f94fcae4c2064",
  "split_expression": {
    "train": "train",
    "validation": "test"
  },
  "tokenizer_id": "bert-base-uncased",
  "tokenizer_revision": "86b5e0934494bd15c9632b12f734a8a67f723594",
  "verification": {
    "checked_sequences": 7174833,
    "decoded_indices": {
      "train": [
        3036107,
        3084783,
        4793642
      ],
      "validation": [
        4009,
        19943,
        36031
      ]
    },
    "decoded_samples": 6,
    "files_checked": 10,
    "observed_max_token_id": 30511,
    "observed_min_token_id": 100,
    "verified_at": "2026-08-21T02:46:51.032489Z"
  },
  "verified": true,
  "vocabulary": {
    "max_token_id": 30511,
    "min_token_id": 100,
    "size": 30522
  }
}