{
  "created_at": "2026-08-21T02:46:38.080456Z",
  "dataset": "owt",
  "dataset_id": "Skylion007/openwebtext",
  "document_counts": {
    "train": 7913769,
    "validation": 100000
  },
  "files": [
    {
      "path": "dataset_dict.json",
      "sha256": "88b5b05fc757e76e7edc779db90e96cb6d76538b93165d6ce962a818801babf4",
      "size_bytes": 35
    },
    {
      "path": "train/data-00000-of-00036.arrow",
      "sha256": "b72f7336f1853a6c457b3577de05bcfdab64e2657618cf5cc2dbdc5c16f5e1da",
      "size_bytes": 496963712
    },
    {
      "path": "train/data-00001-of-00036.arrow",
      "sha256": "fb112eed5a7d1434d37625f69579038fc6e3d8bd6dd0cfe34ec3b83cd4ce1436",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00002-of-00036.arrow",
      "sha256": "9979a451c3ea761665a9c705fee27e108fa7c0644f9778c7f9df959cc50b9d0c",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00003-of-00036.arrow",
      "sha256": "e13682f9f96f961204ddd5d2cc03daf5eddbc80700fc154d0268e55008d0733b",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00004-of-00036.arrow",
      "sha256": "6c4e6549059babb015bbceaa6ba23b9e606df89daaa87bcc1dba9ec55334fc83",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00005-of-00036.arrow",
      "sha256": "bdc2b617bf418c871105f1d6ad248aa7828b51f8a3c47676a7e63d2a33a12895",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00006-of-00036.arrow",
      "sha256": "a4049c39ed5ec67550805893c3b8a19ce3954cd6140680b0518b24db2aaba9aa",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00007-of-00036.arrow",
      "sha256": "611562e240dda9bf31f596fa34e6e44fe3edb3c52011514ad3fb8b647dcc010f",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00008-of-00036.arrow",
      "sha256": "ac412a4af6799d5fff761360217e176f6ca1778082bcffe68f7560d44ec3118b",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00009-of-00036.arrow",
      "sha256": "1335e59d8c37be55c5509a1323d4ad31e18a9a5ce7199cd6638f0888408f102e",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00010-of-00036.arrow",
      "sha256": "883f016e7ede7a8c54c65d88fc1cab6e5eb1bf5e1bbc230ff24dc8e6407b8a02",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00011-of-00036.arrow",
      "sha256": "4af8234ed01bdfb2852945d765118e074e544233ed2a487be51701f1f4fd6851",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00012-of-00036.arrow",
      "sha256": "8ca1d78ce79b3dbc5a3c7106753d0af6809631542798234754d53fdcea7e5d40",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00013-of-00036.arrow",
      "sha256": "124d2f2d567251e6cfc30280c19e45542402fd437b7f27b3f6434926b2ead2c8",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00014-of-00036.arrow",
      "sha256": "083f499c788443150dbc2b7e92dad4b3781aa7f2066a736acbca43d16d714518",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00015-of-00036.arrow",
      "sha256": "1e3fe5de327b1369c171d1e03f9f2a60cc99881f8b026710f7ef0b759ee75b9f",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00016-of-00036.arrow",
      "sha256": "37fdfc1b324b8952ccb17552da7582f6c67dca5db19a03edc410ec692cca2086",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00017-of-00036.arrow",
      "sha256": "3b117094ee89d8812b7513b7fb35fa468ffcfe8ded4f231f2b98024e8fe39e93",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00018-of-00036.arrow",
      "sha256": "cbcbb6d4352e7410b428e93b60a1d8462a66e9ac1aacc456e37d4261a015ae5a",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00019-of-00036.arrow",
      "sha256": "4b3a9276caa65ec5ce955fdaf42151b8ba626887281d69d54c1750baae4fddeb",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00020-of-00036.arrow",
      "sha256": "42f595653d508ca2a6c516b0e93d2ed85fffd9cb2130152e6621a5b79d44a8e4",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00021-of-00036.arrow",
      "sha256": "63cf9f35df6a254e3981745d6b5d6f76a6aeffa1ffe41eb100812329fac4a53e",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00022-of-00036.arrow",
      "sha256": "34811ff54d62eaad627f21deb99321017ef4a3d9a70a183fa96a55992bb37d8d",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00023-of-00036.arrow",
      "sha256": "a59867054f81fc70da375ee2ca112cf69e655ab6da0d14ec2c280c31fcd1dc9b",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00024-of-00036.arrow",
      "sha256": "a064666a45f37b0a4b6ce9042ee6741f5f122db1f713c6634d9707947ed1cfc0",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00025-of-00036.arrow",
      "sha256": "7f8e5868f4c9e0aac16d5a5dec1dcef19e307550038e0162614deb00eca73e6a",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00026-of-00036.arrow",
      "sha256": "7904f9ab98ef7534ddf4f9c2c24b9ef1802f1e79e0567e309740ea11cedac6f1",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00027-of-00036.arrow",
      "sha256": "5434b60b31c999b94676a72875ae4424f79f094db8f06608408915638ac21758",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00028-of-00036.arrow",
      "sha256": "d14ed5e57ab4dde321695484b389ec6768bf352212f39e570b93524d3f2ef86b",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00029-of-00036.arrow",
      "sha256": "4bc3f16f159189883d51e97b83640eedc723e8e6475a3361595b825598e19da4",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00030-of-00036.arrow",
      "sha256": "1b2916afb7c4fb7730a5b86d3b37abee29b8d1f7a632c73552e111c88289ac85",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00031-of-00036.arrow",
      "sha256": "7c97e4afff654e5fc7a078b6199dd766236b88513c3111baaacb1cb23f1ccdfa",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00032-of-00036.arrow",
      "sha256": "a2521d39a2fbb359a01b87b20fe25ab3869e757627699c586c2c64d8c475f9d8",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00033-of-00036.arrow",
      "sha256": "1c45c498a626dc0d23e20ade6f67387700eed878b3aaa3ee2c88632496ea34ff",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00034-of-00036.arrow",
      "sha256": "b855312afd765fce956c020b4ca415c520ed58915ce77ff0f74134707691d55b",
      "size_bytes": 496961664
    },
    {
      "path": "train/data-00035-of-00036.arrow",
      "sha256": "03a137f7c047131f1d9d7c7cce9c821addaae68e0cee53ced766e1c3a76b21c3",
      "size_bytes": 496961664
    },
    {
      "path": "train/dataset_info.json",
      "sha256": "9b40e0362d5f3ee3169330c88838113389b033e3605d8b83cc623711ee3a6074",
      "size_bytes": 1317
    },
    {
      "path": "train/state.json",
      "sha256": "23949df2c280b60fefd4eed1d7f39411d898af6aa22d4f18d09efd0825a4833b",
      "size_bytes": 2315
    },
    {
      "path": "validation/data-00000-of-00001.arrow",
      "sha256": "937a1c91c25bfddef7908b92c5d0b6283ce2dc90194810a16ef54bca100c4d85",
      "size_bytes": 226364864
    },
    {
      "path": "validation/dataset_info.json",
      "sha256": "87846ab989aa61f9ed6340829fd699098abc35f75d14c76d3767462be20221a6",
      "size_bytes": 701
    },
    {
      "path": "validation/state.json",
      "sha256": "14558facc5f5bd1709161e1588a89b0486d259831fb92e924d7e55699b1bc8a5",
      "size_bytes": 250
    }
  ],
  "packed_sequence_counts": {
    "train": 8734897,
    "validation": 110520
  },
  "processed_path": "data/processed/owt-gpt2-1024",
  "processing_contract": {
    "dataset": "owt",
    "dataset_id": "Skylion007/openwebtext",
    "expected_document_counts": {
      "train": 7913769,
      "validation": 100000
    },
    "packing": {
      "algorithm": "document_eos_continuous_bos_eos",
      "version": 1
    },
    "schema_version": 1,
    "sequence_length": 1024,
    "source_materialization_revision": null,
    "source_revision": "79d93d786212f7344586290adb811d4ae6a1762c",
    "split_expression": {
      "train": "train[:-100000]",
      "validation": "train[-100000:]"
    },
    "tokenizer_id": "gpt2",
    "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
    "vocab_size": 50257
  },
  "processing_contract_sha256": "ec742365e5064afa53d23d8215e729366c67cf2e473b88d3f23f4880a52bae71",
  "schema_version": 1,
  "sequence_length": 1024,
  "source_materialization_revision": null,
  "source_revision": "79d93d786212f7344586290adb811d4ae6a1762c",
  "split_expression": {
    "train": "train[:-100000]",
    "validation": "train[-100000:]"
  },
  "tokenizer_id": "gpt2",
  "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
  "verification": {
    "checked_sequences": 8845417,
    "decoded_indices": {
      "train": [
        1758481,
        6072214,
        6169566
      ],
      "validation": [
        4009,
        19943,
        74884
      ]
    },
    "decoded_samples": 6,
    "files_checked": 42,
    "observed_max_token_id": 50256,
    "observed_min_token_id": 0,
    "verified_at": "2026-08-21T02:47:01.956315Z"
  },
  "verified": true,
  "vocabulary": {
    "max_token_id": 50256,
    "min_token_id": 0,
    "size": 50257
  }
}