| { |
| "created_at": "2026-08-21T02:44:58.134500Z", |
| "dataset": "lm1b", |
| "dataset_id": "billion-word-benchmark/lm1b", |
| "document_counts": { |
| "train": 30301028, |
| "validation": 306688 |
| }, |
| "files": [ |
| { |
| "path": "dataset_dict.json", |
| "sha256": "88b5b05fc757e76e7edc779db90e96cb6d76538b93165d6ce962a818801babf4", |
| "size_bytes": 35 |
| }, |
| { |
| "path": "train/data-00000-of-00004.arrow", |
| "sha256": "c1842d29417604f3c3ee4ba6fd648adebf47ef3d33588ca7eacad5f7e60dc276", |
| "size_bytes": 454895728 |
| }, |
| { |
| "path": "train/data-00001-of-00004.arrow", |
| "sha256": "395ff7ef22b9f42ed72ab4d4f319d1a5da1756ae85d5bd498c3b5b4dfc96273e", |
| "size_bytes": 454895728 |
| }, |
| { |
| "path": "train/data-00002-of-00004.arrow", |
| "sha256": "1d8ed17f183a09947a7035b6a0571662d0797e6b236897e5b28e378c7ea8a3c0", |
| "size_bytes": 454895472 |
| }, |
| { |
| "path": "train/data-00003-of-00004.arrow", |
| "sha256": "2272f8e8d83a5189d0a2d39e0010949f2b7914aae4f9e5cb2316f392fc699e56", |
| "size_bytes": 454895472 |
| }, |
| { |
| "path": "train/dataset_info.json", |
| "sha256": "1f4e3b935de233b63542c8d29a7bc819e11c08522ca8e83d3f3e121e3616e081", |
| "size_bytes": 805 |
| }, |
| { |
| "path": "train/state.json", |
| "sha256": "d9f4fca87f6e722f0dddc1f7ae811d0552830540d0690666d04735330882a410", |
| "size_bytes": 427 |
| }, |
| { |
| "path": "validation/data-00000-of-00001.arrow", |
| "sha256": "fff03f2a6785bc54a9d54074a409d89bc2e71042beaf3a4ccef724a9c74c14d6", |
| "size_bytes": 18439664 |
| }, |
| { |
| "path": "validation/dataset_info.json", |
| "sha256": "5d94f384698efbc6ef827c2dd2e776a6a9ac89be2f92dcf9490da6a6063949e1", |
| "size_bytes": 696 |
| }, |
| { |
| "path": "validation/state.json", |
| "sha256": "ba4addb0d152b776d1a256db649e50707e16d2e932839cb9e547de20101a0232", |
| "size_bytes": 250 |
| } |
| ], |
| "packed_sequence_counts": { |
| "train": 7102854, |
| "validation": 71979 |
| }, |
| "processed_path": "data/processed/lm1b-bert-128", |
| "processing_contract": { |
| "dataset": "lm1b", |
| "dataset_id": "billion-word-benchmark/lm1b", |
| "expected_document_counts": { |
| "train": 30301028, |
| "validation": 306688 |
| }, |
| "packing": { |
| "algorithm": "document_eos_continuous_bos_eos", |
| "version": 1 |
| }, |
| "schema_version": 1, |
| "sequence_length": 128, |
| "source_materialization_revision": "8d52bfd3cc2819fc1166dbb8144c328e2690de3e", |
| "source_revision": "35161838ea9e05371a25a8db037f94fcae4c2064", |
| "split_expression": { |
| "train": "train", |
| "validation": "test" |
| }, |
| "tokenizer_id": "bert-base-uncased", |
| "tokenizer_revision": "86b5e0934494bd15c9632b12f734a8a67f723594", |
| "vocab_size": 30522 |
| }, |
| "processing_contract_sha256": "e64a9a0df35ae112075596ccd5958a363a09075fb61515c058f0fc3e2eb50668", |
| "schema_version": 1, |
| "sequence_length": 128, |
| "source_materialization_revision": "8d52bfd3cc2819fc1166dbb8144c328e2690de3e", |
| "source_revision": "35161838ea9e05371a25a8db037f94fcae4c2064", |
| "split_expression": { |
| "train": "train", |
| "validation": "test" |
| }, |
| "tokenizer_id": "bert-base-uncased", |
| "tokenizer_revision": "86b5e0934494bd15c9632b12f734a8a67f723594", |
| "verification": { |
| "checked_sequences": 7174833, |
| "decoded_indices": { |
| "train": [ |
| 3036107, |
| 3084783, |
| 4793642 |
| ], |
| "validation": [ |
| 4009, |
| 19943, |
| 36031 |
| ] |
| }, |
| "decoded_samples": 6, |
| "files_checked": 10, |
| "observed_max_token_id": 30511, |
| "observed_min_token_id": 100, |
| "verified_at": "2026-08-21T02:46:51.032489Z" |
| }, |
| "verified": true, |
| "vocabulary": { |
| "max_token_id": 30511, |
| "min_token_id": 100, |
| "size": 30522 |
| } |
| } |
|
|
