{ "passed": true, "rows": 49131472, "source_tokens": 20000000040, "target_source_tokens": 20000000000, "available_rows": 49946998, "available_source_tokens": 20331230681, "stop_policy": "stop after document crossing 20B; no remainder segment", "token_definition": "SmolLM2-360M native tokenizer, text.strip(), no special tokens, no truncation", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "shards": [ { "shard": 0, "rows": 519382, "tokens": 211952586, "sha256": "f986e5edd0411917898906f41531adce8721180121e02d56b54979a77eba76f9", "source_sha256": "a5125c5434c7163f4e3fb47145851f2d7a73ac5e5873fea02d159ce63d236a40", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 0, "start_tokens": 0 }, { "shard": 1, "rows": 519600, "tokens": 212065390, "sha256": "9a4a0d0be8df39c05491d9d6bff0243184e929caca4db181f1896978fd4bc455", "source_sha256": "4f6cb21db5e0163b22b6218d6f0d0c70b66277721893a2a093955bfee961f9bd", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 519382, "start_tokens": 211952586 }, { "shard": 2, "rows": 517500, "tokens": 211060986, "sha256": "e128283e7b401ae9f97f1a69a77e2c846c83d17915277bfbb1c73a99ea285a2b", "source_sha256": "2df799374af8517e1b0902f86c40ffb0bcb5a581775e9858b235d2031d4dd8f1", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 1038982, "start_tokens": 424017976 }, { "shard": 3, "rows": 521108, "tokens": 212968572, "sha256": "93b7060a8dc0fb236a2f185693f80f70b3a67d2c4b44b4ccf43e8105c5346a59", "source_sha256": "10627366dc257044b3c539f1bf4c5fa102655ff4de37d73f9343bc1cdb87470b", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 1556482, "start_tokens": 635078962 }, { "shard": 4, "rows": 523295, "tokens": 213945805, "sha256": "43777cc69dbca120e0f86b1abda94958d65884adec198eb11e4aa5403208a991", "source_sha256": "b4c98197ed3cc5632b6ab25f9c06e038affae7fff5fe7b0ef9aa77395688daef", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 2077590, "start_tokens": 848047534 }, { "shard": 5, "rows": 522231, "tokens": 213581283, "sha256": "19fbfb901c8cec68e676a8f4a6199e16edfb45d40f8f899588ae171eb16ba46d", "source_sha256": "8eb20acce1ad438a0f92773eb0e41fd3fc297ef8acdc4143860b12d4da6d08b2", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 2600885, "start_tokens": 1061993339 }, { "shard": 6, "rows": 519514, "tokens": 212102928, "sha256": "4481e538f537412087d55fc6f4bbae8cfb12ceaede195d3c51c76aa5fcc7aa47", "source_sha256": "973121b5d1ae4ec05db8dbfd49ebcb80b46dc3ac4a6b1bb37152ca32d7f474b1", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 3123116, "start_tokens": 1275574622 }, { "shard": 7, "rows": 517996, "tokens": 211373686, "sha256": "cc8e98cb0ea0f3396aff348ab34b9e2304785892f285bea3c235fdaa369e31dc", "source_sha256": "cd199b6d781104226d29515bf3bec4f57484608b2d07c7349d269476c6d6580f", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 3642630, "start_tokens": 1487677550 }, { "shard": 8, "rows": 518233, "tokens": 211522546, "sha256": "69829b508b9c5de50eaa61e215e3976580c2f5d8a35baa1400c82bc761f9e3ff", "source_sha256": "c2fb9aaa0434b0ef6fd01fbf8e8c903eccca2675ffd8e2c8c8ebd0f6ad9a3c87", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 4160626, "start_tokens": 1699051236 }, { "shard": 9, "rows": 519292, "tokens": 211969328, "sha256": "2dd9a766c4a08d53438121eae512e824a2d1b116a2b4a933d4f487f2213851e4", "source_sha256": "3b193212df5cb7712bb3ea5efb2d0c2a36c050c08815747d196f3d4f1ba7261d", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 4678859, "start_tokens": 1910573782 }, { "shard": 10, "rows": 519005, "tokens": 211777857, "sha256": "083dacd21068e730da68cdf434b36b6c070bca4758ca543f6b4bd31fdc585d5a", "source_sha256": "b26a47f97120fa22f5114e94d0dd75d0334cf28f8a2f29a0ae15406f38b9d5d7", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 5198151, "start_tokens": 2122543110 }, { "shard": 11, "rows": 519850, "tokens": 212295322, "sha256": "231466ec64fc828d1f1a998d253d619d2f36d0d498ef73bb44f01a645749d750", "source_sha256": "532302f10f9368485e7e661f931902e6ebbf17db4ba284a14b486ab3d7eff25a", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 5717156, "start_tokens": 2334320967 }, { "shard": 12, "rows": 513918, "tokens": 209289370, "sha256": "c925087b35cec04aca66ba9b0341eeca1b8c255f0a06b23f7e844c7fd0f4020d", "source_sha256": "43c5b90eb779310e2beb4ea285479c1977e449d2976f60c3edec8e0e507470c4", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 6237006, "start_tokens": 2546616289 }, { "shard": 13, "rows": 516711, "tokens": 210732898, "sha256": "5138537de8feb0916191d5719c2ad1302ff8fd19b2795f29fc140cdbcfc59276", "source_sha256": "c056d25b3d1d1ae3d0a17cf931fb5105b6885be9f95b957ea18f5ebab5ec1991", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 6750924, "start_tokens": 2755905659 }, { "shard": 14, "rows": 514701, "tokens": 209651404, "sha256": "808a8a8b5d4b2245aa435842aa0d74a7fbe6cfaf64eef69f84cdcededea05277", "source_sha256": "0fdd78ab9f76a8ba18aced59028b4b0c5cb93b2e81f9463cf25e6e99a3809293", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 7267635, "start_tokens": 2966638557 }, { "shard": 15, "rows": 515918, "tokens": 210358145, "sha256": "f9fd40e7a7ad5ced0ac8a71c63ec9cc510d9c0e4f83bef892075d7763f9f226e", "source_sha256": "63304d8ce285db3c5ab92a597ff7befa9af2bb94301278b611e66d693d8f1c59", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 7782336, "start_tokens": 3176289961 }, { "shard": 16, "rows": 516805, "tokens": 210816204, "sha256": "b8c6e394f64092485c5118d77808e9f4fa30d17036696ff44097bac0008816f1", "source_sha256": "d56f463b306830ad6dbfa1168ae119b7d6d9abb51b5b6fc1b2e17c34bde6ff0d", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 8298254, "start_tokens": 3386648106 }, { "shard": 17, "rows": 517917, "tokens": 211222280, "sha256": "1d583b46857d82efc04da0a684e96789cc7a4a48ce254f90ed6a328643871979", "source_sha256": "35150ba30ea58d55971e4412b3ad38e4d606f7909ae8fd9c60751ef3860903aa", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 8815059, "start_tokens": 3597464310 }, { "shard": 18, "rows": 514193, "tokens": 209406729, "sha256": "de4c9468afa04823373332173786b5b58e15d8338f374e99040f7fc20838758b", "source_sha256": "60825d96f15f02b8beea58d171660a72de929b57e97b73cf07a4e2d1df762859", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 9332976, "start_tokens": 3808686590 }, { "shard": 19, "rows": 511495, "tokens": 208147526, "sha256": "66f7717c73dbea15ecfa0543f476fe3be939a76691d22a9af8297f4a4fae6a89", "source_sha256": "e90ceb9b98c98206f0dab84bac81a6635432a474833eb82944d9ab9b3b93efc1", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 9847169, "start_tokens": 4018093319 }, { "shard": 20, "rows": 516292, "tokens": 210474712, "sha256": "016c3d4faf76a781c7f99ea1ef81ba534ae1086465d04ae7ec7c95e137c9cf05", "source_sha256": "5b22959df43bef33d94e443aa7d78fddc12dc2f54bcd9a70aebd749594b87046", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 10358664, "start_tokens": 4226240845 }, { "shard": 21, "rows": 514029, "tokens": 209372325, "sha256": "b4df00606766c8eccefc5d0cd084866eb1961ccd859628946984e7625f225207", "source_sha256": "48439737f30867e6b3f2293e266b63be46a2a8f39080308a7c188970bf2f1d2f", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 10874956, "start_tokens": 4436715557 }, { "shard": 22, "rows": 513231, "tokens": 208996501, "sha256": "cfe76ac4797280139f6bad69edb65f33e703038b29c545436f462e8f1f235116", "source_sha256": "6de2be082e153510c74397753f43995ba5190508d2bb15d2f6501e9af6c12b1c", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 11388985, "start_tokens": 4646087882 }, { "shard": 23, "rows": 513885, "tokens": 209358865, "sha256": "ea987e6e2a034904ccddaff9780e046b69b766aba762205518ca465a71f86b94", "source_sha256": "b1140f460867bc1d014689eff9cc789cbde506ada6bb71cfaefc7168ddd2918a", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 11902216, "start_tokens": 4855084383 }, { "shard": 24, "rows": 514908, "tokens": 209937921, "sha256": "319051c7182a8c319f737dcf88cee200a2bbffe31e65db4c9caac968f59d5450", "source_sha256": "61bbbc547625a5c205818f7a653609ca2db994b06910d1d3f8e61e40da4236b4", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 12416101, "start_tokens": 5064443248 }, { "shard": 25, "rows": 513385, "tokens": 209018494, "sha256": "d8402f38e2c8c5483b207254e5b303d5337acb6b099feb3a365cc4846d96971f", "source_sha256": "9a89c32b771148b2fcb477ef9798362c0694f2d24c066ef8548f235dcc7c9fc2", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 12931009, "start_tokens": 5274381169 }, { "shard": 26, "rows": 510957, "tokens": 207735136, "sha256": "aba6cc7677bd0bf66c4ec6714c0b385d71518aba6ba9d341f9ef02103b690c53", "source_sha256": "e76b5d5b839a67b021651f969ce4081dada63ba2453faad26d1ed669fc7e2af0", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 13444394, "start_tokens": 5483399663 }, { "shard": 27, "rows": 510907, "tokens": 207904517, "sha256": "7c9181caf73c0735a70a7f7ce9d4b5d7d031fc1913c47458fea794d7afd2b6ab", "source_sha256": "1d7f70a2cc629fa4ec517357b8d01e3722ed4718df2bc4046468aeaaec30f764", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 13955351, "start_tokens": 5691134799 }, { "shard": 28, "rows": 511531, "tokens": 208062833, "sha256": "7afa9150c08e77f720d39d95dbf33c66cdeeb48cfd2f62fbbc6f172c9436972b", "source_sha256": "4d174248ac884e8f3bfaec45e3464a80e20c458f6bad997705d3dc7187c3149d", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 14466258, "start_tokens": 5899039316 }, { "shard": 29, "rows": 510257, "tokens": 207454327, "sha256": "59b8e58413c5b16cd24330336753ed5c02e223afd5e364bafb1cc5538f93270d", "source_sha256": "491ff26ddc2c00ed12a3ba186693609117897cf5c345400b3c340955470d380a", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 14977789, "start_tokens": 6107102149 }, { "shard": 30, "rows": 509260, "tokens": 207010782, "sha256": "01877ae7e17c60e49c0ebb7a7d1b0e593d1cd46ddab6667b9d8ad21e55d11fe6", "source_sha256": "df4f4129e441a34b40af18dc956573029636851c36388a7d15dd150e4479a75b", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 15488046, "start_tokens": 6314556476 }, { "shard": 31, "rows": 507292, "tokens": 205932031, "sha256": "40a4265c85b41981450a32a4678997654c0488bfe6cf90cfeb024b5755e7bc9f", "source_sha256": "b115a3e7f23a2984871023a4c5537d5095e5c4616d9f632b297393c5296389a7", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 15997306, "start_tokens": 6521567258 }, { "shard": 32, "rows": 511446, "tokens": 208050044, "sha256": "91ba2982e5d9f4e0fbf45072aa29bb500fd24f3913c72099ee9863ef339c78e8", "source_sha256": "1c17921f75ecd2dda2dec7c33df996ccba3c24e5e44d79172c88d71dace49d3d", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 16504598, "start_tokens": 6727499289 }, { "shard": 33, "rows": 503554, "tokens": 204142536, "sha256": "213a62a2d453d64c2b53744fec98f435ac4fcc728aeb2e48129186a580b4fd0a", "source_sha256": "5c3831062ccb3b151081b81014313e2424fac296980d90e9a34ae6735a4a5645", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 17016044, "start_tokens": 6935549333 }, { "shard": 34, "rows": 506872, "tokens": 205883016, "sha256": "27a3ffcb1cea95787ff94080a566fa2c78163b9fc2cda5dbe2c26e9e9d62ba4c", "source_sha256": "8720715185180c01bea4d4314fcc4fa6880942ea479b9bfedb1cf2d8e65b2a80", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 17519598, "start_tokens": 7139691869 }, { "shard": 35, "rows": 508450, "tokens": 206488133, "sha256": "9499dd388f447f313f96b066d8f0765ee6611e6330af3befa6bdb5de716f5c88", "source_sha256": "a0eb585d58f8c71d28b736574fa56687df0b0bc5cddd8628b9097e0220893335", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 18026470, "start_tokens": 7345574885 }, { "shard": 36, "rows": 508968, "tokens": 206816997, "sha256": "6114205aa691be49c245f37822aaa77e9a42024ffc8cb75757ae1aabcb2ef1d8", "source_sha256": "7b1766d29497ff38b3c657d6f32073c6e18252421b1273872bf53f77c0cb6af7", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 18534920, "start_tokens": 7552063018 }, { "shard": 37, "rows": 505406, "tokens": 205123701, "sha256": "636492ab64572dc5009be945509b7b7b0d6cd0b6f4125613005116d9f50a2ab4", "source_sha256": "5244c87621b108e62387c3f1296c851071996b4c8fb53606d04cac7cb1cebf85", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 19043888, "start_tokens": 7758880015 }, { "shard": 38, "rows": 509232, "tokens": 206884906, "sha256": "36cfd36d01ccad9e03670efc181d5a0eae472c69f0988d43fc78c3099ef1acb4", "source_sha256": "225d4bbd83fced3a35cbecb88d55254e21cb8bb517f8838cd4626073744866a0", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 19549294, "start_tokens": 7964003716 }, { "shard": 39, "rows": 505656, "tokens": 205278898, "sha256": "f46d433f8878f093f920d859c1288f34aec9561c22c63e3bb772b7bb88c695fa", "source_sha256": "a9887a5d84b2f55ff964c82cfd3daa323f1c0ce674fac42e8c180426b0ecbda1", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 20058526, "start_tokens": 8170888622 }, { "shard": 40, "rows": 500781, "tokens": 202807988, "sha256": "026e3dc386ace5f0c31d76c61e82d6788934ede556e4e41a268b031b60e7d784", "source_sha256": "d60c7bb5c7412fcb670edc8c5c2ade9ee285a63912cb9cef001bf86d2ff01712", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 20564182, "start_tokens": 8376167520 }, { "shard": 41, "rows": 506165, "tokens": 205404424, "sha256": "1c031069880d0223fa676878dbe740df7167fa65237b361d9f795d20a57f20db", "source_sha256": "f24ea1da24620084d067fcd1a2fd18e9925bec40636a7ba5523e5ff29a74bc35", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 21064963, "start_tokens": 8578975508 }, { "shard": 42, "rows": 506089, "tokens": 205301352, "sha256": "3d07de990fb4c6acc03999647447c230e525aac0d13edbcca9b34a1c2950f826", "source_sha256": "538b12443b8ed3a3e888fdaf995bed4a0e17456dcf183e25e37a2fb01005c0a1", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 21571128, "start_tokens": 8784379932 }, { "shard": 43, "rows": 502831, "tokens": 203893820, "sha256": "201dea78bdf4a524cfe38d2a8616937f45aa38b88bd21506fb1c307553fc26be", "source_sha256": "0ad043ab65c05e3fa815900627e7d42b40488e6cca677be6ec1f8d0a26240e93", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 22077217, "start_tokens": 8989681284 }, { "shard": 44, "rows": 507935, "tokens": 206282752, "sha256": "94bbdacc7895caba5346dc96903c070bae0ee445f6a0808e2c6e3a371f0aee7e", "source_sha256": "0d3400c635a2bf9ffe9117b90795dcf3d61d93b2230390123f97760241f94340", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 22580048, "start_tokens": 9193575104 }, { "shard": 45, "rows": 502982, "tokens": 203734103, "sha256": "b735dd962a8ecd943ae2ad81c89ee2492c13e49abcb1f3915a145459e679821c", "source_sha256": "3da2cfcd8c2c6d3528711355942b780344a197d91c09d10162f59925c5c5b64a", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 23087983, "start_tokens": 9399857856 }, { "shard": 46, "rows": 505904, "tokens": 205406737, "sha256": "f69bd08de7fc52d2caccda7e27917ee3162be7ed3dd84b310d9190b4698798c0", "source_sha256": "668615a763f1b8343168abc3073260c04133bd23c9beca48f50086874e4b870c", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 23590965, "start_tokens": 9603591959 }, { "shard": 47, "rows": 506936, "tokens": 205911289, "sha256": "bf27a462c74c8f72921bd3844e3b7c39a00ce56e3fe13760375bb0461e626d8e", "source_sha256": "9f959fceb43fb591c4cf5df5b26e6cfba2735a7b4dfa5b7c21f0d5a959aa9273", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 24096869, "start_tokens": 9808998696 }, { "shard": 48, "rows": 505676, "tokens": 205360582, "sha256": "b7d9d3088813225cef2dec68a2f9b4d56dc92c0841fd16270762a08dc45cff62", "source_sha256": "1f83fb5fa12d60aaeebb034ab6e2d35930d89195c874f138b3a217798b9c13f0", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 24603805, "start_tokens": 10014909985 }, { "shard": 49, "rows": 503699, "tokens": 204249244, "sha256": "4192448fb00b7dc69392a3c47b605899e6533236f432cafe4b9fb902ef1af748", "source_sha256": "a23a598fe2b23a10a359b3397c36d9c8eabba7bab0b6cd3f5cec7038c1492c71", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 25109481, "start_tokens": 10220270567 }, { "shard": 50, "rows": 503419, "tokens": 204066758, "sha256": "489cde1dabada611fa84bfc7684a90dc72bb0630a052038191e453c8dfff6cea", "source_sha256": "858a326664d7d15ddd2231a3c0826d3f228b65a5ea3fd0e87fbda0dc734c93a3", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 25613180, "start_tokens": 10424519811 }, { "shard": 51, "rows": 501752, "tokens": 203336648, "sha256": "be45b7c81b46f515c1faf3f4e163ceee478b855d0c173cfd14a7348c94839e82", "source_sha256": "8b69bb506a5b1df485a3162d24ad38d65501193ef64abaab37d36f03c9dbb106", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 26116599, "start_tokens": 10628586569 }, { "shard": 52, "rows": 502139, "tokens": 203472608, "sha256": "eb98a67dcfdb473d142016ff3f8f2fc010a70484e4bf8d3360553d6a25bb37f6", "source_sha256": "990c268025191d8c9c4da5aaf3d273364268b21433e8ab2dfdea3e1b2ff0d7dc", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 26618351, "start_tokens": 10831923217 }, { "shard": 53, "rows": 500362, "tokens": 202713492, "sha256": "cda01d3bd8de2e759269533df5124bfdd4ae16fe8bb87107919482fe597dc53b", "source_sha256": "c0562cc05b980754bd1baf9f82438eca5932cf48f854c0b9ff44e9ddbd656f58", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 27120490, "start_tokens": 11035395825 }, { "shard": 54, "rows": 506388, "tokens": 205496674, "sha256": "1feb45fcebf9eff6a7a42dd135a441e053452c75def76c72bda7a3fafff9aec1", "source_sha256": "43c977f320db30de80bbeb6bdf9e652dbf107c216f28b956eb9ca08559fb67fc", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 27620852, "start_tokens": 11238109317 }, { "shard": 55, "rows": 521359, "tokens": 212900786, "sha256": "8f92bdc183eae5c14abdc612bc9eea05488b1431637aa2296422ec76277b433e", "source_sha256": "80d7c4bf85bebf9af20b054efaff2c440da42292070c100861b2a8d6ec116575", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 28127240, "start_tokens": 11443605991 }, { "shard": 56, "rows": 521590, "tokens": 213122452, "sha256": "eb3b79b7064ce66d6aacdbf90856f9afe62f4a77105da37395a6f70c3d82fd6a", "source_sha256": "1430675542e56d3a283f7b9d318e5e686f45688bccf0d1b61df81e6a5609e935", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 28648599, "start_tokens": 11656506777 }, { "shard": 57, "rows": 522000, "tokens": 213363517, "sha256": "8b7e01990886bbb8b3ef4d088a71156aca2c98ce4427ed5abaccd8774a2a0db1", "source_sha256": "2e1f8cfca4149f568cb8852f7b19c8f6ab085d5f4535dfdc17dd01ad320d0ff6", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 29170189, "start_tokens": 11869629229 }, { "shard": 58, "rows": 519891, "tokens": 212271014, "sha256": "f4cb49d26ac945fa774ad4dddd6ced66a67168fb9bed3a3db58bb3f5ecfb95eb", "source_sha256": "157df7204eb7a62cdc72efc68d99c2383202a66f818e805958a38c00138da29b", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 29692189, "start_tokens": 12082992746 }, { "shard": 59, "rows": 523013, "tokens": 213889007, "sha256": "3c6c0c273f6bb9685b6f519076d273617780f10c7761602fb972ac12b65a3a8f", "source_sha256": "dff96d81d1d7fc2a901f187a11484fb2763926d47a219214678324695bec0ce5", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 30212080, "start_tokens": 12295263760 }, { "shard": 60, "rows": 521371, "tokens": 212900965, "sha256": "4a6938c8aabc11cbd044787ad5611fa8dbde3c8e5f209ecad5e18f9865da386b", "source_sha256": "4e4975906a1d75999bb9dce05c7990bb79002c009af48b1f8e256b4b66b84577", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 30735093, "start_tokens": 12509152767 }, { "shard": 61, "rows": 522656, "tokens": 213562162, "sha256": "26bd56505b6bc5719635adadaee18c158315091b749ef8ebadf85cb60c8f597a", "source_sha256": "6c8a452afdc7056359069468fe71c14489c32f4dfee226e97ad22f0c2cd8aa19", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 31256464, "start_tokens": 12722053732 }, { "shard": 62, "rows": 519820, "tokens": 212165257, "sha256": "cb6e835c6e67b9a6b56e58e56679a2c577229dbc97a35f1bd62d1c93351a5d93", "source_sha256": "9f39b947b09bb1f4e08d167ca68984e904db389253d76bbc06bc79197d067764", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 31779120, "start_tokens": 12935615894 }, { "shard": 63, "rows": 522532, "tokens": 213539266, "sha256": "696b807757e0034c866289b99adc5114304ef9b87e796f4584f0a052c703e26f", "source_sha256": "0560d4db2c629c509e2a87fb016c4c070a14ae93690163ae808a22edbd5a5ca5", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 32298940, "start_tokens": 13147781151 }, { "shard": 64, "rows": 522636, "tokens": 213575504, "sha256": "f9faf53034c65a14ccff8df6dd819430f5434bd0de4d08891917b88bd71c96df", "source_sha256": "9f792b83258529f29b3a3aff0ac2a2e5773d9f6eb5ad1c55e3a4f5d0b1a0aad3", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 32821472, "start_tokens": 13361320417 }, { "shard": 65, "rows": 520640, "tokens": 212737949, "sha256": "bbded202c2f94df6cbbd5ca7d876bf02b7eaa1dff5e639dadd1620def8a3b0b6", "source_sha256": "b87b89e5f6e73e545135a9872b3538521731d49495cc3f018fac8266033b3d1b", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 33344108, "start_tokens": 13574895921 }, { "shard": 66, "rows": 519298, "tokens": 211996822, "sha256": "6ab5204ec3ed5e3171cec769258959ffdc15b618641876602985174d9fba30ae", "source_sha256": "6739d99dbf46b6583d1fb74e4daac7979f2193b47cf09410c6e1728dd22725b0", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 33864748, "start_tokens": 13787633870 }, { "shard": 67, "rows": 518877, "tokens": 211723051, "sha256": "4c29b8b087d8717b8a62e73c9069474e9eddaa3bdfae59a78452f71c9c94d649", "source_sha256": "0e37d84d9ac3d780d34128b063cc84fb43e833695973dd0e3e1fad7f34e85d37", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 34384046, "start_tokens": 13999630692 }, { "shard": 68, "rows": 520360, "tokens": 212536832, "sha256": "b4edb6a249e80c697a20dcbce8ad423e01649ff01d95bb05b3cd746a5d7e197b", "source_sha256": "e39d0dfe51a08774d6dfded4612358883a589827e3f6d70be614523ffc99d2a1", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 34902923, "start_tokens": 14211353743 }, { "shard": 69, "rows": 517938, "tokens": 211289198, "sha256": "8077d6a686543557deeb6e68c52c6bfd5a26c6315137eefb741b2d710017a59d", "source_sha256": "9866965a17c561518ea138382a0ee212deb3c08be60db750c3b225c6ceb358b3", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 35423283, "start_tokens": 14423890575 }, { "shard": 70, "rows": 515109, "tokens": 209895529, "sha256": "b63035b8c0e66a8677982f67ef76746d25dc53b1779d63f6b2fd56dfae9dc4a7", "source_sha256": "d917c1bebd756454c1a08cfe0e03cf940acaf73c4cc70925c17edfd237e18690", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 35941221, "start_tokens": 14635179773 }, { "shard": 71, "rows": 513703, "tokens": 209093146, "sha256": "fa60788bdebb5040a0923afdb205de4c7976c22dbb928462484a15ae6748a0fa", "source_sha256": "09638ba5500dda238bdf0137c5aee259ee7946943f6b91fc2d8b02d0d5bc1e01", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 36456330, "start_tokens": 14845075302 }, { "shard": 72, "rows": 515973, "tokens": 210302528, "sha256": "38d032213e4b39071f0ddb8a95e72b695e44e483066998a0d30ffa1554e7c80c", "source_sha256": "ba7c659df0d244b31ff4e58670487e2b507f05c6c179c3e174960d342b3b8af4", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 36970033, "start_tokens": 15054168448 }, { "shard": 73, "rows": 514987, "tokens": 209720308, "sha256": "8fbc1465d3eac86fec5d4cfe11015a585cdaf7ad9f40ce9674cdc8412be28a0d", "source_sha256": "499fbf6f747937ae310ec78a0c0ba23fd21ef66f695614864bfd46caa1e9679a", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 37486006, "start_tokens": 15264470976 }, { "shard": 74, "rows": 515417, "tokens": 209962632, "sha256": "56e6378d8a8bb94710da20422f5782d0532e3f0b57055d29ee17792a09dd0719", "source_sha256": "c0d14f05696c9044b0df2a839c6e3b14f0641dd8468e992e3e24ca2de01c4047", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 38000993, "start_tokens": 15474191284 }, { "shard": 75, "rows": 514819, "tokens": 209712838, "sha256": "bd0eb247faf8d812c16e7b977c6b53fa605458c78c47cb537d9990d116499bb4", "source_sha256": "42653bcc18f2d0b2e5646b0fb05165c18e9e09a8f8cd3db242411c1c182a740b", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 38516410, "start_tokens": 15684153916 }, { "shard": 76, "rows": 514372, "tokens": 209456953, "sha256": "cc7ccd51a164524dc337bb39625eddcd7d330ebb2f062d3ad1efc4f61483ddc6", "source_sha256": "2df605f76d332dcf815fd8cf3a556c3e9fe3da1c467a46817e2dbaa66d7079d3", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 39031229, "start_tokens": 15893866754 }, { "shard": 77, "rows": 514281, "tokens": 209496553, "sha256": "926e7a690ba516dddbf6e3331d74e115b338200a128620d22f639cc71c67d9c9", "source_sha256": "ebbe9acd633f026a6841267bf72384cf266207ba44a9353870de9b39b14bb126", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 39545601, "start_tokens": 16103323707 }, { "shard": 78, "rows": 512851, "tokens": 208731975, "sha256": "669d2deb3e0ce6916585c94cc678cd6e30d75458dd0d15c90ce1d75fe120a4a7", "source_sha256": "da96522698e8bd0fc5d3b4ddf406b90cf26f003e4b7d31047daf9da0ff6b7587", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 40059882, "start_tokens": 16312820260 }, { "shard": 79, "rows": 509736, "tokens": 207135712, "sha256": "d25fdb3afb61d2686d965b25be9a96cba86f27048bfaff5ce9b33a27cf39deb5", "source_sha256": "59957cdd0e228a6f0d341a75f3d8554bb66cf5e85c8b64a1e2a2c3db4ce26a14", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 40572733, "start_tokens": 16521552235 }, { "shard": 80, "rows": 509324, "tokens": 207021189, "sha256": "4603be47bcf787178aba0ede3e7df5eb20ed175cfc6232eec7ca38654acd7e1e", "source_sha256": "2548ed978086b5e5a2d4a22d4aee4667f9e6c61f6889d7562fdd5a50f74a2e50", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 41082469, "start_tokens": 16728687947 }, { "shard": 81, "rows": 511105, "tokens": 207943240, "sha256": "38b28e69774f1539cd7d5fac47f224ed120dad0a412bfc4bde5fdd2fb7e0f2e2", "source_sha256": "ce57364657181baa77f2a593cfc18da410988b3736ede0b10bda158367f9b92f", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 41591793, "start_tokens": 16935709136 }, { "shard": 82, "rows": 509673, "tokens": 207352440, "sha256": "f37dc5e687cfa614616a994b55aadee9f93d6a50c2b1d2ab2b38b76062feee9d", "source_sha256": "69efe078253b163cd9f30c574675cd7804d1bb6976d290fabd12ac8a2d029b43", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 42102898, "start_tokens": 17143652376 }, { "shard": 83, "rows": 508094, "tokens": 206513373, "sha256": "cc3190af212447d44e803597f86f87c33fc37a25a812d6c6d26b1e7d3a756b1e", "source_sha256": "44ca2ee16a1bd7e2fc1328c4ac94b8c8cd59d85e9ed6254943c506ff71b007c5", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 42612571, "start_tokens": 17351004816 }, { "shard": 84, "rows": 510431, "tokens": 207676905, "sha256": "d550a3a72d5ba5794735c79be9e8888cc26c5cc8763191ea5442c16912f3caba", "source_sha256": "008bd64d1cef26d0acf0f7a42853b7581671dc14d8543e0d2c055af8fd91eb95", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 43120665, "start_tokens": 17557518189 }, { "shard": 85, "rows": 508960, "tokens": 206809695, "sha256": "6ed887b4a7a160ceabc3cdf7df51e3a3d16bcd2a3ac189993e16890a3bd2e0f6", "source_sha256": "7f3235ea4ea3d3283968644205a430ccbddf0d9712b09787fbb055f8c770343c", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 43631096, "start_tokens": 17765195094 }, { "shard": 86, "rows": 510007, "tokens": 207440462, "sha256": "892a045c8068c6b187ad0065df17b7715216d70c5a02ed3cf76d640b43cab136", "source_sha256": "c4cf70ca119d8a1f0f4ef9ea249e9cc032db65b3cb23a3d3fbe5f0e55fc7e65c", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 44140056, "start_tokens": 17972004789 }, { "shard": 87, "rows": 510689, "tokens": 207759448, "sha256": "1263f8fb474cba38825d7c07ff3d1446d7ff12b5da002c6cfa828a4b3a35b79c", "source_sha256": "cd29581325ad822741ca0d4f6e5ece0568f8ebfbcfe23ecbf86c1639bc4a1cbf", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 44650063, "start_tokens": 18179445251 }, { "shard": 88, "rows": 510099, "tokens": 207429138, "sha256": "0771fd56d91ed617a506ccc2a56cca254d2b0eb7c3cb538a42f8313dd39d3381", "source_sha256": "8fbd3a5f58a9e58b15e32d9ed8e05d78d63cc2f599e1133b2ca1ec3978daf0fc", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 45160752, "start_tokens": 18387204699 }, { "shard": 89, "rows": 506891, "tokens": 205874287, "sha256": "eac3d1bd04a8d0288be37751de6dbe612f566835d376be19b6f2e5cb683d68df", "source_sha256": "3ddebc81b937ee4f0a57bc9f80d6573ca213fd4eba937edeac53089daf74692e", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 45670851, "start_tokens": 18594633837 }, { "shard": 90, "rows": 508180, "tokens": 206457815, "sha256": "c94cc484c5c80156f64d7b311c73498368df000c5f5fe265100239262630bc33", "source_sha256": "21767889b1e43d095b6568413a5e4b6034e6af46463dddd67ce3420ec6859b2f", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 46177742, "start_tokens": 18800508124 }, { "shard": 91, "rows": 509320, "tokens": 206935851, "sha256": "b2d9905705f07bcd29de7674a36d842949be9f843e3ec605a698ba12263e28c1", "source_sha256": "6054ced2fb1e1723f9ae9121b8debdafc43f91eb44c5fe42708735303c54b0a4", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 46685922, "start_tokens": 19006965939 }, { "shard": 92, "rows": 508761, "tokens": 206797758, "sha256": "af586e71de02585ba2e5b7bceee6262f5a0a8045ab324fb010f1d7d99cb56d34", "source_sha256": "d5f91d64386bcaedf35581f06bcb03af38db635d37227958e0bd060dd4463e1a", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 47195242, "start_tokens": 19213901790 }, { "shard": 93, "rows": 508155, "tokens": 206569132, "sha256": "40483fa3519f82e450109a5728f39b11050708b34e1738c080d75be885876729", "source_sha256": "529a931e9ddf27ba727c69f31361887787a450c92a0230fa3b84900865599023", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 47704003, "start_tokens": 19420699548 }, { "shard": 94, "rows": 503136, "tokens": 203890108, "sha256": "c168eab17dbf8f3f2d338a331a553af5c10c577ed5a1d9d9f275b9c02bd07226", "source_sha256": "2520e3dd784fa40ebefa495bb06607f2b82d55128e0812d163305cd6bdd40312", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 48212158, "start_tokens": 19627268680 }, { "shard": 95, "rows": 505035, "tokens": 204876711, "sha256": "186be6d9cbd2047f1f098ea9dd5a77faa9cebebcf1d518479010bedf245312a0", "source_sha256": "3ea3f3383cac92cefbc4f254d3550f0eb5c781292e7a92c3ee5ba6b65cc8c319", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 48715294, "start_tokens": 19831158788 }, { "shard": 96, "rows": 506253, "tokens": 205552052, "sha256": "97e88439dee1feca39afb7d4ad148aec92a4192ed7aab69ee0cb7431f820b48e", "source_sha256": "c5577f61aac321ffe934ca61836f87a1ac134ec958b09316f2ac7b78f8f01980", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 49220329, "start_tokens": 20036035499 }, { "shard": 97, "rows": 220416, "tokens": 89643130, "sha256": "df54532ca91252bbd1cc9ffa0b76bd8fb9cf5af3e7d28be0340a0e6c1a214184", "source_sha256": "edc4bf1cff43218f783a20c5cb884c4a6700dab7ec12cfb7a6362291e5392c86", "tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", "start_row": 49726582, "start_tokens": 20241587551 } ], "segments": [ { "segment": 0, "end_row": 4898104, "source_tokens": 2000000031, "target_tokens": 2000000000 }, { "segment": 1, "end_row": 9802903, "source_tokens": 4000000388, "target_tokens": 4000000000 }, { "segment": 2, "end_row": 14714284, "source_tokens": 6000000190, "target_tokens": 6000000000 }, { "segment": 3, "end_row": 19637704, "source_tokens": 8000000397, "target_tokens": 8000000000 }, { "segment": 4, "end_row": 24567228, "source_tokens": 10000000091, "target_tokens": 10000000000 }, { "segment": 5, "end_row": 29489067, "source_tokens": 12000000382, "target_tokens": 12000000000 }, { "segment": 6, "end_row": 34384959, "source_tokens": 14000000003, "target_tokens": 14000000000 }, { "segment": 7, "end_row": 39291912, "source_tokens": 16000000433, "target_tokens": 16000000000 }, { "segment": 8, "end_row": 44208957, "source_tokens": 18000000424, "target_tokens": 18000000000 }, { "segment": 9, "end_row": 49131472, "source_tokens": 20000000040, "target_tokens": 20000000000 } ], "budget": 2000000000 }