File size: 9,304 Bytes
93cd35a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 | {
"created_at": "2026-08-21T02:46:38.080456Z",
"dataset": "owt",
"dataset_id": "Skylion007/openwebtext",
"document_counts": {
"train": 7913769,
"validation": 100000
},
"files": [
{
"path": "dataset_dict.json",
"sha256": "88b5b05fc757e76e7edc779db90e96cb6d76538b93165d6ce962a818801babf4",
"size_bytes": 35
},
{
"path": "train/data-00000-of-00036.arrow",
"sha256": "b72f7336f1853a6c457b3577de05bcfdab64e2657618cf5cc2dbdc5c16f5e1da",
"size_bytes": 496963712
},
{
"path": "train/data-00001-of-00036.arrow",
"sha256": "fb112eed5a7d1434d37625f69579038fc6e3d8bd6dd0cfe34ec3b83cd4ce1436",
"size_bytes": 496961664
},
{
"path": "train/data-00002-of-00036.arrow",
"sha256": "9979a451c3ea761665a9c705fee27e108fa7c0644f9778c7f9df959cc50b9d0c",
"size_bytes": 496961664
},
{
"path": "train/data-00003-of-00036.arrow",
"sha256": "e13682f9f96f961204ddd5d2cc03daf5eddbc80700fc154d0268e55008d0733b",
"size_bytes": 496961664
},
{
"path": "train/data-00004-of-00036.arrow",
"sha256": "6c4e6549059babb015bbceaa6ba23b9e606df89daaa87bcc1dba9ec55334fc83",
"size_bytes": 496961664
},
{
"path": "train/data-00005-of-00036.arrow",
"sha256": "bdc2b617bf418c871105f1d6ad248aa7828b51f8a3c47676a7e63d2a33a12895",
"size_bytes": 496961664
},
{
"path": "train/data-00006-of-00036.arrow",
"sha256": "a4049c39ed5ec67550805893c3b8a19ce3954cd6140680b0518b24db2aaba9aa",
"size_bytes": 496961664
},
{
"path": "train/data-00007-of-00036.arrow",
"sha256": "611562e240dda9bf31f596fa34e6e44fe3edb3c52011514ad3fb8b647dcc010f",
"size_bytes": 496961664
},
{
"path": "train/data-00008-of-00036.arrow",
"sha256": "ac412a4af6799d5fff761360217e176f6ca1778082bcffe68f7560d44ec3118b",
"size_bytes": 496961664
},
{
"path": "train/data-00009-of-00036.arrow",
"sha256": "1335e59d8c37be55c5509a1323d4ad31e18a9a5ce7199cd6638f0888408f102e",
"size_bytes": 496961664
},
{
"path": "train/data-00010-of-00036.arrow",
"sha256": "883f016e7ede7a8c54c65d88fc1cab6e5eb1bf5e1bbc230ff24dc8e6407b8a02",
"size_bytes": 496961664
},
{
"path": "train/data-00011-of-00036.arrow",
"sha256": "4af8234ed01bdfb2852945d765118e074e544233ed2a487be51701f1f4fd6851",
"size_bytes": 496961664
},
{
"path": "train/data-00012-of-00036.arrow",
"sha256": "8ca1d78ce79b3dbc5a3c7106753d0af6809631542798234754d53fdcea7e5d40",
"size_bytes": 496961664
},
{
"path": "train/data-00013-of-00036.arrow",
"sha256": "124d2f2d567251e6cfc30280c19e45542402fd437b7f27b3f6434926b2ead2c8",
"size_bytes": 496961664
},
{
"path": "train/data-00014-of-00036.arrow",
"sha256": "083f499c788443150dbc2b7e92dad4b3781aa7f2066a736acbca43d16d714518",
"size_bytes": 496961664
},
{
"path": "train/data-00015-of-00036.arrow",
"sha256": "1e3fe5de327b1369c171d1e03f9f2a60cc99881f8b026710f7ef0b759ee75b9f",
"size_bytes": 496961664
},
{
"path": "train/data-00016-of-00036.arrow",
"sha256": "37fdfc1b324b8952ccb17552da7582f6c67dca5db19a03edc410ec692cca2086",
"size_bytes": 496961664
},
{
"path": "train/data-00017-of-00036.arrow",
"sha256": "3b117094ee89d8812b7513b7fb35fa468ffcfe8ded4f231f2b98024e8fe39e93",
"size_bytes": 496961664
},
{
"path": "train/data-00018-of-00036.arrow",
"sha256": "cbcbb6d4352e7410b428e93b60a1d8462a66e9ac1aacc456e37d4261a015ae5a",
"size_bytes": 496961664
},
{
"path": "train/data-00019-of-00036.arrow",
"sha256": "4b3a9276caa65ec5ce955fdaf42151b8ba626887281d69d54c1750baae4fddeb",
"size_bytes": 496961664
},
{
"path": "train/data-00020-of-00036.arrow",
"sha256": "42f595653d508ca2a6c516b0e93d2ed85fffd9cb2130152e6621a5b79d44a8e4",
"size_bytes": 496961664
},
{
"path": "train/data-00021-of-00036.arrow",
"sha256": "63cf9f35df6a254e3981745d6b5d6f76a6aeffa1ffe41eb100812329fac4a53e",
"size_bytes": 496961664
},
{
"path": "train/data-00022-of-00036.arrow",
"sha256": "34811ff54d62eaad627f21deb99321017ef4a3d9a70a183fa96a55992bb37d8d",
"size_bytes": 496961664
},
{
"path": "train/data-00023-of-00036.arrow",
"sha256": "a59867054f81fc70da375ee2ca112cf69e655ab6da0d14ec2c280c31fcd1dc9b",
"size_bytes": 496961664
},
{
"path": "train/data-00024-of-00036.arrow",
"sha256": "a064666a45f37b0a4b6ce9042ee6741f5f122db1f713c6634d9707947ed1cfc0",
"size_bytes": 496961664
},
{
"path": "train/data-00025-of-00036.arrow",
"sha256": "7f8e5868f4c9e0aac16d5a5dec1dcef19e307550038e0162614deb00eca73e6a",
"size_bytes": 496961664
},
{
"path": "train/data-00026-of-00036.arrow",
"sha256": "7904f9ab98ef7534ddf4f9c2c24b9ef1802f1e79e0567e309740ea11cedac6f1",
"size_bytes": 496961664
},
{
"path": "train/data-00027-of-00036.arrow",
"sha256": "5434b60b31c999b94676a72875ae4424f79f094db8f06608408915638ac21758",
"size_bytes": 496961664
},
{
"path": "train/data-00028-of-00036.arrow",
"sha256": "d14ed5e57ab4dde321695484b389ec6768bf352212f39e570b93524d3f2ef86b",
"size_bytes": 496961664
},
{
"path": "train/data-00029-of-00036.arrow",
"sha256": "4bc3f16f159189883d51e97b83640eedc723e8e6475a3361595b825598e19da4",
"size_bytes": 496961664
},
{
"path": "train/data-00030-of-00036.arrow",
"sha256": "1b2916afb7c4fb7730a5b86d3b37abee29b8d1f7a632c73552e111c88289ac85",
"size_bytes": 496961664
},
{
"path": "train/data-00031-of-00036.arrow",
"sha256": "7c97e4afff654e5fc7a078b6199dd766236b88513c3111baaacb1cb23f1ccdfa",
"size_bytes": 496961664
},
{
"path": "train/data-00032-of-00036.arrow",
"sha256": "a2521d39a2fbb359a01b87b20fe25ab3869e757627699c586c2c64d8c475f9d8",
"size_bytes": 496961664
},
{
"path": "train/data-00033-of-00036.arrow",
"sha256": "1c45c498a626dc0d23e20ade6f67387700eed878b3aaa3ee2c88632496ea34ff",
"size_bytes": 496961664
},
{
"path": "train/data-00034-of-00036.arrow",
"sha256": "b855312afd765fce956c020b4ca415c520ed58915ce77ff0f74134707691d55b",
"size_bytes": 496961664
},
{
"path": "train/data-00035-of-00036.arrow",
"sha256": "03a137f7c047131f1d9d7c7cce9c821addaae68e0cee53ced766e1c3a76b21c3",
"size_bytes": 496961664
},
{
"path": "train/dataset_info.json",
"sha256": "9b40e0362d5f3ee3169330c88838113389b033e3605d8b83cc623711ee3a6074",
"size_bytes": 1317
},
{
"path": "train/state.json",
"sha256": "23949df2c280b60fefd4eed1d7f39411d898af6aa22d4f18d09efd0825a4833b",
"size_bytes": 2315
},
{
"path": "validation/data-00000-of-00001.arrow",
"sha256": "937a1c91c25bfddef7908b92c5d0b6283ce2dc90194810a16ef54bca100c4d85",
"size_bytes": 226364864
},
{
"path": "validation/dataset_info.json",
"sha256": "87846ab989aa61f9ed6340829fd699098abc35f75d14c76d3767462be20221a6",
"size_bytes": 701
},
{
"path": "validation/state.json",
"sha256": "14558facc5f5bd1709161e1588a89b0486d259831fb92e924d7e55699b1bc8a5",
"size_bytes": 250
}
],
"packed_sequence_counts": {
"train": 8734897,
"validation": 110520
},
"processed_path": "data/processed/owt-gpt2-1024",
"processing_contract": {
"dataset": "owt",
"dataset_id": "Skylion007/openwebtext",
"expected_document_counts": {
"train": 7913769,
"validation": 100000
},
"packing": {
"algorithm": "document_eos_continuous_bos_eos",
"version": 1
},
"schema_version": 1,
"sequence_length": 1024,
"source_materialization_revision": null,
"source_revision": "79d93d786212f7344586290adb811d4ae6a1762c",
"split_expression": {
"train": "train[:-100000]",
"validation": "train[-100000:]"
},
"tokenizer_id": "gpt2",
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
"vocab_size": 50257
},
"processing_contract_sha256": "ec742365e5064afa53d23d8215e729366c67cf2e473b88d3f23f4880a52bae71",
"schema_version": 1,
"sequence_length": 1024,
"source_materialization_revision": null,
"source_revision": "79d93d786212f7344586290adb811d4ae6a1762c",
"split_expression": {
"train": "train[:-100000]",
"validation": "train[-100000:]"
},
"tokenizer_id": "gpt2",
"tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
"verification": {
"checked_sequences": 8845417,
"decoded_indices": {
"train": [
1758481,
6072214,
6169566
],
"validation": [
4009,
19943,
74884
]
},
"decoded_samples": 6,
"files_checked": 42,
"observed_max_token_id": 50256,
"observed_min_token_id": 0,
"verified_at": "2026-08-21T02:47:01.956315Z"
},
"verified": true,
"vocabulary": {
"max_token_id": 50256,
"min_token_id": 0,
"size": 50257
}
}
|