File size: 42,189 Bytes
17b5a5c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 | {
"passed": true,
"rows": 49131472,
"source_tokens": 20000000040,
"target_source_tokens": 20000000000,
"available_rows": 49946998,
"available_source_tokens": 20331230681,
"stop_policy": "stop after document crossing 20B; no remainder segment",
"token_definition": "SmolLM2-360M native tokenizer, text.strip(), no special tokens, no truncation",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"shards": [
{
"shard": 0,
"rows": 519382,
"tokens": 211952586,
"sha256": "f986e5edd0411917898906f41531adce8721180121e02d56b54979a77eba76f9",
"source_sha256": "a5125c5434c7163f4e3fb47145851f2d7a73ac5e5873fea02d159ce63d236a40",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 0,
"start_tokens": 0
},
{
"shard": 1,
"rows": 519600,
"tokens": 212065390,
"sha256": "9a4a0d0be8df39c05491d9d6bff0243184e929caca4db181f1896978fd4bc455",
"source_sha256": "4f6cb21db5e0163b22b6218d6f0d0c70b66277721893a2a093955bfee961f9bd",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 519382,
"start_tokens": 211952586
},
{
"shard": 2,
"rows": 517500,
"tokens": 211060986,
"sha256": "e128283e7b401ae9f97f1a69a77e2c846c83d17915277bfbb1c73a99ea285a2b",
"source_sha256": "2df799374af8517e1b0902f86c40ffb0bcb5a581775e9858b235d2031d4dd8f1",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 1038982,
"start_tokens": 424017976
},
{
"shard": 3,
"rows": 521108,
"tokens": 212968572,
"sha256": "93b7060a8dc0fb236a2f185693f80f70b3a67d2c4b44b4ccf43e8105c5346a59",
"source_sha256": "10627366dc257044b3c539f1bf4c5fa102655ff4de37d73f9343bc1cdb87470b",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 1556482,
"start_tokens": 635078962
},
{
"shard": 4,
"rows": 523295,
"tokens": 213945805,
"sha256": "43777cc69dbca120e0f86b1abda94958d65884adec198eb11e4aa5403208a991",
"source_sha256": "b4c98197ed3cc5632b6ab25f9c06e038affae7fff5fe7b0ef9aa77395688daef",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 2077590,
"start_tokens": 848047534
},
{
"shard": 5,
"rows": 522231,
"tokens": 213581283,
"sha256": "19fbfb901c8cec68e676a8f4a6199e16edfb45d40f8f899588ae171eb16ba46d",
"source_sha256": "8eb20acce1ad438a0f92773eb0e41fd3fc297ef8acdc4143860b12d4da6d08b2",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 2600885,
"start_tokens": 1061993339
},
{
"shard": 6,
"rows": 519514,
"tokens": 212102928,
"sha256": "4481e538f537412087d55fc6f4bbae8cfb12ceaede195d3c51c76aa5fcc7aa47",
"source_sha256": "973121b5d1ae4ec05db8dbfd49ebcb80b46dc3ac4a6b1bb37152ca32d7f474b1",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 3123116,
"start_tokens": 1275574622
},
{
"shard": 7,
"rows": 517996,
"tokens": 211373686,
"sha256": "cc8e98cb0ea0f3396aff348ab34b9e2304785892f285bea3c235fdaa369e31dc",
"source_sha256": "cd199b6d781104226d29515bf3bec4f57484608b2d07c7349d269476c6d6580f",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 3642630,
"start_tokens": 1487677550
},
{
"shard": 8,
"rows": 518233,
"tokens": 211522546,
"sha256": "69829b508b9c5de50eaa61e215e3976580c2f5d8a35baa1400c82bc761f9e3ff",
"source_sha256": "c2fb9aaa0434b0ef6fd01fbf8e8c903eccca2675ffd8e2c8c8ebd0f6ad9a3c87",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 4160626,
"start_tokens": 1699051236
},
{
"shard": 9,
"rows": 519292,
"tokens": 211969328,
"sha256": "2dd9a766c4a08d53438121eae512e824a2d1b116a2b4a933d4f487f2213851e4",
"source_sha256": "3b193212df5cb7712bb3ea5efb2d0c2a36c050c08815747d196f3d4f1ba7261d",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 4678859,
"start_tokens": 1910573782
},
{
"shard": 10,
"rows": 519005,
"tokens": 211777857,
"sha256": "083dacd21068e730da68cdf434b36b6c070bca4758ca543f6b4bd31fdc585d5a",
"source_sha256": "b26a47f97120fa22f5114e94d0dd75d0334cf28f8a2f29a0ae15406f38b9d5d7",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 5198151,
"start_tokens": 2122543110
},
{
"shard": 11,
"rows": 519850,
"tokens": 212295322,
"sha256": "231466ec64fc828d1f1a998d253d619d2f36d0d498ef73bb44f01a645749d750",
"source_sha256": "532302f10f9368485e7e661f931902e6ebbf17db4ba284a14b486ab3d7eff25a",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 5717156,
"start_tokens": 2334320967
},
{
"shard": 12,
"rows": 513918,
"tokens": 209289370,
"sha256": "c925087b35cec04aca66ba9b0341eeca1b8c255f0a06b23f7e844c7fd0f4020d",
"source_sha256": "43c5b90eb779310e2beb4ea285479c1977e449d2976f60c3edec8e0e507470c4",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 6237006,
"start_tokens": 2546616289
},
{
"shard": 13,
"rows": 516711,
"tokens": 210732898,
"sha256": "5138537de8feb0916191d5719c2ad1302ff8fd19b2795f29fc140cdbcfc59276",
"source_sha256": "c056d25b3d1d1ae3d0a17cf931fb5105b6885be9f95b957ea18f5ebab5ec1991",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 6750924,
"start_tokens": 2755905659
},
{
"shard": 14,
"rows": 514701,
"tokens": 209651404,
"sha256": "808a8a8b5d4b2245aa435842aa0d74a7fbe6cfaf64eef69f84cdcededea05277",
"source_sha256": "0fdd78ab9f76a8ba18aced59028b4b0c5cb93b2e81f9463cf25e6e99a3809293",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 7267635,
"start_tokens": 2966638557
},
{
"shard": 15,
"rows": 515918,
"tokens": 210358145,
"sha256": "f9fd40e7a7ad5ced0ac8a71c63ec9cc510d9c0e4f83bef892075d7763f9f226e",
"source_sha256": "63304d8ce285db3c5ab92a597ff7befa9af2bb94301278b611e66d693d8f1c59",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 7782336,
"start_tokens": 3176289961
},
{
"shard": 16,
"rows": 516805,
"tokens": 210816204,
"sha256": "b8c6e394f64092485c5118d77808e9f4fa30d17036696ff44097bac0008816f1",
"source_sha256": "d56f463b306830ad6dbfa1168ae119b7d6d9abb51b5b6fc1b2e17c34bde6ff0d",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 8298254,
"start_tokens": 3386648106
},
{
"shard": 17,
"rows": 517917,
"tokens": 211222280,
"sha256": "1d583b46857d82efc04da0a684e96789cc7a4a48ce254f90ed6a328643871979",
"source_sha256": "35150ba30ea58d55971e4412b3ad38e4d606f7909ae8fd9c60751ef3860903aa",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 8815059,
"start_tokens": 3597464310
},
{
"shard": 18,
"rows": 514193,
"tokens": 209406729,
"sha256": "de4c9468afa04823373332173786b5b58e15d8338f374e99040f7fc20838758b",
"source_sha256": "60825d96f15f02b8beea58d171660a72de929b57e97b73cf07a4e2d1df762859",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 9332976,
"start_tokens": 3808686590
},
{
"shard": 19,
"rows": 511495,
"tokens": 208147526,
"sha256": "66f7717c73dbea15ecfa0543f476fe3be939a76691d22a9af8297f4a4fae6a89",
"source_sha256": "e90ceb9b98c98206f0dab84bac81a6635432a474833eb82944d9ab9b3b93efc1",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 9847169,
"start_tokens": 4018093319
},
{
"shard": 20,
"rows": 516292,
"tokens": 210474712,
"sha256": "016c3d4faf76a781c7f99ea1ef81ba534ae1086465d04ae7ec7c95e137c9cf05",
"source_sha256": "5b22959df43bef33d94e443aa7d78fddc12dc2f54bcd9a70aebd749594b87046",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 10358664,
"start_tokens": 4226240845
},
{
"shard": 21,
"rows": 514029,
"tokens": 209372325,
"sha256": "b4df00606766c8eccefc5d0cd084866eb1961ccd859628946984e7625f225207",
"source_sha256": "48439737f30867e6b3f2293e266b63be46a2a8f39080308a7c188970bf2f1d2f",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 10874956,
"start_tokens": 4436715557
},
{
"shard": 22,
"rows": 513231,
"tokens": 208996501,
"sha256": "cfe76ac4797280139f6bad69edb65f33e703038b29c545436f462e8f1f235116",
"source_sha256": "6de2be082e153510c74397753f43995ba5190508d2bb15d2f6501e9af6c12b1c",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 11388985,
"start_tokens": 4646087882
},
{
"shard": 23,
"rows": 513885,
"tokens": 209358865,
"sha256": "ea987e6e2a034904ccddaff9780e046b69b766aba762205518ca465a71f86b94",
"source_sha256": "b1140f460867bc1d014689eff9cc789cbde506ada6bb71cfaefc7168ddd2918a",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 11902216,
"start_tokens": 4855084383
},
{
"shard": 24,
"rows": 514908,
"tokens": 209937921,
"sha256": "319051c7182a8c319f737dcf88cee200a2bbffe31e65db4c9caac968f59d5450",
"source_sha256": "61bbbc547625a5c205818f7a653609ca2db994b06910d1d3f8e61e40da4236b4",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 12416101,
"start_tokens": 5064443248
},
{
"shard": 25,
"rows": 513385,
"tokens": 209018494,
"sha256": "d8402f38e2c8c5483b207254e5b303d5337acb6b099feb3a365cc4846d96971f",
"source_sha256": "9a89c32b771148b2fcb477ef9798362c0694f2d24c066ef8548f235dcc7c9fc2",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 12931009,
"start_tokens": 5274381169
},
{
"shard": 26,
"rows": 510957,
"tokens": 207735136,
"sha256": "aba6cc7677bd0bf66c4ec6714c0b385d71518aba6ba9d341f9ef02103b690c53",
"source_sha256": "e76b5d5b839a67b021651f969ce4081dada63ba2453faad26d1ed669fc7e2af0",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 13444394,
"start_tokens": 5483399663
},
{
"shard": 27,
"rows": 510907,
"tokens": 207904517,
"sha256": "7c9181caf73c0735a70a7f7ce9d4b5d7d031fc1913c47458fea794d7afd2b6ab",
"source_sha256": "1d7f70a2cc629fa4ec517357b8d01e3722ed4718df2bc4046468aeaaec30f764",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 13955351,
"start_tokens": 5691134799
},
{
"shard": 28,
"rows": 511531,
"tokens": 208062833,
"sha256": "7afa9150c08e77f720d39d95dbf33c66cdeeb48cfd2f62fbbc6f172c9436972b",
"source_sha256": "4d174248ac884e8f3bfaec45e3464a80e20c458f6bad997705d3dc7187c3149d",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 14466258,
"start_tokens": 5899039316
},
{
"shard": 29,
"rows": 510257,
"tokens": 207454327,
"sha256": "59b8e58413c5b16cd24330336753ed5c02e223afd5e364bafb1cc5538f93270d",
"source_sha256": "491ff26ddc2c00ed12a3ba186693609117897cf5c345400b3c340955470d380a",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 14977789,
"start_tokens": 6107102149
},
{
"shard": 30,
"rows": 509260,
"tokens": 207010782,
"sha256": "01877ae7e17c60e49c0ebb7a7d1b0e593d1cd46ddab6667b9d8ad21e55d11fe6",
"source_sha256": "df4f4129e441a34b40af18dc956573029636851c36388a7d15dd150e4479a75b",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 15488046,
"start_tokens": 6314556476
},
{
"shard": 31,
"rows": 507292,
"tokens": 205932031,
"sha256": "40a4265c85b41981450a32a4678997654c0488bfe6cf90cfeb024b5755e7bc9f",
"source_sha256": "b115a3e7f23a2984871023a4c5537d5095e5c4616d9f632b297393c5296389a7",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 15997306,
"start_tokens": 6521567258
},
{
"shard": 32,
"rows": 511446,
"tokens": 208050044,
"sha256": "91ba2982e5d9f4e0fbf45072aa29bb500fd24f3913c72099ee9863ef339c78e8",
"source_sha256": "1c17921f75ecd2dda2dec7c33df996ccba3c24e5e44d79172c88d71dace49d3d",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 16504598,
"start_tokens": 6727499289
},
{
"shard": 33,
"rows": 503554,
"tokens": 204142536,
"sha256": "213a62a2d453d64c2b53744fec98f435ac4fcc728aeb2e48129186a580b4fd0a",
"source_sha256": "5c3831062ccb3b151081b81014313e2424fac296980d90e9a34ae6735a4a5645",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 17016044,
"start_tokens": 6935549333
},
{
"shard": 34,
"rows": 506872,
"tokens": 205883016,
"sha256": "27a3ffcb1cea95787ff94080a566fa2c78163b9fc2cda5dbe2c26e9e9d62ba4c",
"source_sha256": "8720715185180c01bea4d4314fcc4fa6880942ea479b9bfedb1cf2d8e65b2a80",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 17519598,
"start_tokens": 7139691869
},
{
"shard": 35,
"rows": 508450,
"tokens": 206488133,
"sha256": "9499dd388f447f313f96b066d8f0765ee6611e6330af3befa6bdb5de716f5c88",
"source_sha256": "a0eb585d58f8c71d28b736574fa56687df0b0bc5cddd8628b9097e0220893335",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 18026470,
"start_tokens": 7345574885
},
{
"shard": 36,
"rows": 508968,
"tokens": 206816997,
"sha256": "6114205aa691be49c245f37822aaa77e9a42024ffc8cb75757ae1aabcb2ef1d8",
"source_sha256": "7b1766d29497ff38b3c657d6f32073c6e18252421b1273872bf53f77c0cb6af7",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 18534920,
"start_tokens": 7552063018
},
{
"shard": 37,
"rows": 505406,
"tokens": 205123701,
"sha256": "636492ab64572dc5009be945509b7b7b0d6cd0b6f4125613005116d9f50a2ab4",
"source_sha256": "5244c87621b108e62387c3f1296c851071996b4c8fb53606d04cac7cb1cebf85",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 19043888,
"start_tokens": 7758880015
},
{
"shard": 38,
"rows": 509232,
"tokens": 206884906,
"sha256": "36cfd36d01ccad9e03670efc181d5a0eae472c69f0988d43fc78c3099ef1acb4",
"source_sha256": "225d4bbd83fced3a35cbecb88d55254e21cb8bb517f8838cd4626073744866a0",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 19549294,
"start_tokens": 7964003716
},
{
"shard": 39,
"rows": 505656,
"tokens": 205278898,
"sha256": "f46d433f8878f093f920d859c1288f34aec9561c22c63e3bb772b7bb88c695fa",
"source_sha256": "a9887a5d84b2f55ff964c82cfd3daa323f1c0ce674fac42e8c180426b0ecbda1",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 20058526,
"start_tokens": 8170888622
},
{
"shard": 40,
"rows": 500781,
"tokens": 202807988,
"sha256": "026e3dc386ace5f0c31d76c61e82d6788934ede556e4e41a268b031b60e7d784",
"source_sha256": "d60c7bb5c7412fcb670edc8c5c2ade9ee285a63912cb9cef001bf86d2ff01712",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 20564182,
"start_tokens": 8376167520
},
{
"shard": 41,
"rows": 506165,
"tokens": 205404424,
"sha256": "1c031069880d0223fa676878dbe740df7167fa65237b361d9f795d20a57f20db",
"source_sha256": "f24ea1da24620084d067fcd1a2fd18e9925bec40636a7ba5523e5ff29a74bc35",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 21064963,
"start_tokens": 8578975508
},
{
"shard": 42,
"rows": 506089,
"tokens": 205301352,
"sha256": "3d07de990fb4c6acc03999647447c230e525aac0d13edbcca9b34a1c2950f826",
"source_sha256": "538b12443b8ed3a3e888fdaf995bed4a0e17456dcf183e25e37a2fb01005c0a1",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 21571128,
"start_tokens": 8784379932
},
{
"shard": 43,
"rows": 502831,
"tokens": 203893820,
"sha256": "201dea78bdf4a524cfe38d2a8616937f45aa38b88bd21506fb1c307553fc26be",
"source_sha256": "0ad043ab65c05e3fa815900627e7d42b40488e6cca677be6ec1f8d0a26240e93",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 22077217,
"start_tokens": 8989681284
},
{
"shard": 44,
"rows": 507935,
"tokens": 206282752,
"sha256": "94bbdacc7895caba5346dc96903c070bae0ee445f6a0808e2c6e3a371f0aee7e",
"source_sha256": "0d3400c635a2bf9ffe9117b90795dcf3d61d93b2230390123f97760241f94340",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 22580048,
"start_tokens": 9193575104
},
{
"shard": 45,
"rows": 502982,
"tokens": 203734103,
"sha256": "b735dd962a8ecd943ae2ad81c89ee2492c13e49abcb1f3915a145459e679821c",
"source_sha256": "3da2cfcd8c2c6d3528711355942b780344a197d91c09d10162f59925c5c5b64a",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 23087983,
"start_tokens": 9399857856
},
{
"shard": 46,
"rows": 505904,
"tokens": 205406737,
"sha256": "f69bd08de7fc52d2caccda7e27917ee3162be7ed3dd84b310d9190b4698798c0",
"source_sha256": "668615a763f1b8343168abc3073260c04133bd23c9beca48f50086874e4b870c",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 23590965,
"start_tokens": 9603591959
},
{
"shard": 47,
"rows": 506936,
"tokens": 205911289,
"sha256": "bf27a462c74c8f72921bd3844e3b7c39a00ce56e3fe13760375bb0461e626d8e",
"source_sha256": "9f959fceb43fb591c4cf5df5b26e6cfba2735a7b4dfa5b7c21f0d5a959aa9273",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 24096869,
"start_tokens": 9808998696
},
{
"shard": 48,
"rows": 505676,
"tokens": 205360582,
"sha256": "b7d9d3088813225cef2dec68a2f9b4d56dc92c0841fd16270762a08dc45cff62",
"source_sha256": "1f83fb5fa12d60aaeebb034ab6e2d35930d89195c874f138b3a217798b9c13f0",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 24603805,
"start_tokens": 10014909985
},
{
"shard": 49,
"rows": 503699,
"tokens": 204249244,
"sha256": "4192448fb00b7dc69392a3c47b605899e6533236f432cafe4b9fb902ef1af748",
"source_sha256": "a23a598fe2b23a10a359b3397c36d9c8eabba7bab0b6cd3f5cec7038c1492c71",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 25109481,
"start_tokens": 10220270567
},
{
"shard": 50,
"rows": 503419,
"tokens": 204066758,
"sha256": "489cde1dabada611fa84bfc7684a90dc72bb0630a052038191e453c8dfff6cea",
"source_sha256": "858a326664d7d15ddd2231a3c0826d3f228b65a5ea3fd0e87fbda0dc734c93a3",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 25613180,
"start_tokens": 10424519811
},
{
"shard": 51,
"rows": 501752,
"tokens": 203336648,
"sha256": "be45b7c81b46f515c1faf3f4e163ceee478b855d0c173cfd14a7348c94839e82",
"source_sha256": "8b69bb506a5b1df485a3162d24ad38d65501193ef64abaab37d36f03c9dbb106",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 26116599,
"start_tokens": 10628586569
},
{
"shard": 52,
"rows": 502139,
"tokens": 203472608,
"sha256": "eb98a67dcfdb473d142016ff3f8f2fc010a70484e4bf8d3360553d6a25bb37f6",
"source_sha256": "990c268025191d8c9c4da5aaf3d273364268b21433e8ab2dfdea3e1b2ff0d7dc",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 26618351,
"start_tokens": 10831923217
},
{
"shard": 53,
"rows": 500362,
"tokens": 202713492,
"sha256": "cda01d3bd8de2e759269533df5124bfdd4ae16fe8bb87107919482fe597dc53b",
"source_sha256": "c0562cc05b980754bd1baf9f82438eca5932cf48f854c0b9ff44e9ddbd656f58",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 27120490,
"start_tokens": 11035395825
},
{
"shard": 54,
"rows": 506388,
"tokens": 205496674,
"sha256": "1feb45fcebf9eff6a7a42dd135a441e053452c75def76c72bda7a3fafff9aec1",
"source_sha256": "43c977f320db30de80bbeb6bdf9e652dbf107c216f28b956eb9ca08559fb67fc",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 27620852,
"start_tokens": 11238109317
},
{
"shard": 55,
"rows": 521359,
"tokens": 212900786,
"sha256": "8f92bdc183eae5c14abdc612bc9eea05488b1431637aa2296422ec76277b433e",
"source_sha256": "80d7c4bf85bebf9af20b054efaff2c440da42292070c100861b2a8d6ec116575",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 28127240,
"start_tokens": 11443605991
},
{
"shard": 56,
"rows": 521590,
"tokens": 213122452,
"sha256": "eb3b79b7064ce66d6aacdbf90856f9afe62f4a77105da37395a6f70c3d82fd6a",
"source_sha256": "1430675542e56d3a283f7b9d318e5e686f45688bccf0d1b61df81e6a5609e935",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 28648599,
"start_tokens": 11656506777
},
{
"shard": 57,
"rows": 522000,
"tokens": 213363517,
"sha256": "8b7e01990886bbb8b3ef4d088a71156aca2c98ce4427ed5abaccd8774a2a0db1",
"source_sha256": "2e1f8cfca4149f568cb8852f7b19c8f6ab085d5f4535dfdc17dd01ad320d0ff6",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 29170189,
"start_tokens": 11869629229
},
{
"shard": 58,
"rows": 519891,
"tokens": 212271014,
"sha256": "f4cb49d26ac945fa774ad4dddd6ced66a67168fb9bed3a3db58bb3f5ecfb95eb",
"source_sha256": "157df7204eb7a62cdc72efc68d99c2383202a66f818e805958a38c00138da29b",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 29692189,
"start_tokens": 12082992746
},
{
"shard": 59,
"rows": 523013,
"tokens": 213889007,
"sha256": "3c6c0c273f6bb9685b6f519076d273617780f10c7761602fb972ac12b65a3a8f",
"source_sha256": "dff96d81d1d7fc2a901f187a11484fb2763926d47a219214678324695bec0ce5",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 30212080,
"start_tokens": 12295263760
},
{
"shard": 60,
"rows": 521371,
"tokens": 212900965,
"sha256": "4a6938c8aabc11cbd044787ad5611fa8dbde3c8e5f209ecad5e18f9865da386b",
"source_sha256": "4e4975906a1d75999bb9dce05c7990bb79002c009af48b1f8e256b4b66b84577",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 30735093,
"start_tokens": 12509152767
},
{
"shard": 61,
"rows": 522656,
"tokens": 213562162,
"sha256": "26bd56505b6bc5719635adadaee18c158315091b749ef8ebadf85cb60c8f597a",
"source_sha256": "6c8a452afdc7056359069468fe71c14489c32f4dfee226e97ad22f0c2cd8aa19",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 31256464,
"start_tokens": 12722053732
},
{
"shard": 62,
"rows": 519820,
"tokens": 212165257,
"sha256": "cb6e835c6e67b9a6b56e58e56679a2c577229dbc97a35f1bd62d1c93351a5d93",
"source_sha256": "9f39b947b09bb1f4e08d167ca68984e904db389253d76bbc06bc79197d067764",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 31779120,
"start_tokens": 12935615894
},
{
"shard": 63,
"rows": 522532,
"tokens": 213539266,
"sha256": "696b807757e0034c866289b99adc5114304ef9b87e796f4584f0a052c703e26f",
"source_sha256": "0560d4db2c629c509e2a87fb016c4c070a14ae93690163ae808a22edbd5a5ca5",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 32298940,
"start_tokens": 13147781151
},
{
"shard": 64,
"rows": 522636,
"tokens": 213575504,
"sha256": "f9faf53034c65a14ccff8df6dd819430f5434bd0de4d08891917b88bd71c96df",
"source_sha256": "9f792b83258529f29b3a3aff0ac2a2e5773d9f6eb5ad1c55e3a4f5d0b1a0aad3",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 32821472,
"start_tokens": 13361320417
},
{
"shard": 65,
"rows": 520640,
"tokens": 212737949,
"sha256": "bbded202c2f94df6cbbd5ca7d876bf02b7eaa1dff5e639dadd1620def8a3b0b6",
"source_sha256": "b87b89e5f6e73e545135a9872b3538521731d49495cc3f018fac8266033b3d1b",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 33344108,
"start_tokens": 13574895921
},
{
"shard": 66,
"rows": 519298,
"tokens": 211996822,
"sha256": "6ab5204ec3ed5e3171cec769258959ffdc15b618641876602985174d9fba30ae",
"source_sha256": "6739d99dbf46b6583d1fb74e4daac7979f2193b47cf09410c6e1728dd22725b0",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 33864748,
"start_tokens": 13787633870
},
{
"shard": 67,
"rows": 518877,
"tokens": 211723051,
"sha256": "4c29b8b087d8717b8a62e73c9069474e9eddaa3bdfae59a78452f71c9c94d649",
"source_sha256": "0e37d84d9ac3d780d34128b063cc84fb43e833695973dd0e3e1fad7f34e85d37",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 34384046,
"start_tokens": 13999630692
},
{
"shard": 68,
"rows": 520360,
"tokens": 212536832,
"sha256": "b4edb6a249e80c697a20dcbce8ad423e01649ff01d95bb05b3cd746a5d7e197b",
"source_sha256": "e39d0dfe51a08774d6dfded4612358883a589827e3f6d70be614523ffc99d2a1",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 34902923,
"start_tokens": 14211353743
},
{
"shard": 69,
"rows": 517938,
"tokens": 211289198,
"sha256": "8077d6a686543557deeb6e68c52c6bfd5a26c6315137eefb741b2d710017a59d",
"source_sha256": "9866965a17c561518ea138382a0ee212deb3c08be60db750c3b225c6ceb358b3",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 35423283,
"start_tokens": 14423890575
},
{
"shard": 70,
"rows": 515109,
"tokens": 209895529,
"sha256": "b63035b8c0e66a8677982f67ef76746d25dc53b1779d63f6b2fd56dfae9dc4a7",
"source_sha256": "d917c1bebd756454c1a08cfe0e03cf940acaf73c4cc70925c17edfd237e18690",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 35941221,
"start_tokens": 14635179773
},
{
"shard": 71,
"rows": 513703,
"tokens": 209093146,
"sha256": "fa60788bdebb5040a0923afdb205de4c7976c22dbb928462484a15ae6748a0fa",
"source_sha256": "09638ba5500dda238bdf0137c5aee259ee7946943f6b91fc2d8b02d0d5bc1e01",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 36456330,
"start_tokens": 14845075302
},
{
"shard": 72,
"rows": 515973,
"tokens": 210302528,
"sha256": "38d032213e4b39071f0ddb8a95e72b695e44e483066998a0d30ffa1554e7c80c",
"source_sha256": "ba7c659df0d244b31ff4e58670487e2b507f05c6c179c3e174960d342b3b8af4",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 36970033,
"start_tokens": 15054168448
},
{
"shard": 73,
"rows": 514987,
"tokens": 209720308,
"sha256": "8fbc1465d3eac86fec5d4cfe11015a585cdaf7ad9f40ce9674cdc8412be28a0d",
"source_sha256": "499fbf6f747937ae310ec78a0c0ba23fd21ef66f695614864bfd46caa1e9679a",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 37486006,
"start_tokens": 15264470976
},
{
"shard": 74,
"rows": 515417,
"tokens": 209962632,
"sha256": "56e6378d8a8bb94710da20422f5782d0532e3f0b57055d29ee17792a09dd0719",
"source_sha256": "c0d14f05696c9044b0df2a839c6e3b14f0641dd8468e992e3e24ca2de01c4047",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 38000993,
"start_tokens": 15474191284
},
{
"shard": 75,
"rows": 514819,
"tokens": 209712838,
"sha256": "bd0eb247faf8d812c16e7b977c6b53fa605458c78c47cb537d9990d116499bb4",
"source_sha256": "42653bcc18f2d0b2e5646b0fb05165c18e9e09a8f8cd3db242411c1c182a740b",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 38516410,
"start_tokens": 15684153916
},
{
"shard": 76,
"rows": 514372,
"tokens": 209456953,
"sha256": "cc7ccd51a164524dc337bb39625eddcd7d330ebb2f062d3ad1efc4f61483ddc6",
"source_sha256": "2df605f76d332dcf815fd8cf3a556c3e9fe3da1c467a46817e2dbaa66d7079d3",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 39031229,
"start_tokens": 15893866754
},
{
"shard": 77,
"rows": 514281,
"tokens": 209496553,
"sha256": "926e7a690ba516dddbf6e3331d74e115b338200a128620d22f639cc71c67d9c9",
"source_sha256": "ebbe9acd633f026a6841267bf72384cf266207ba44a9353870de9b39b14bb126",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 39545601,
"start_tokens": 16103323707
},
{
"shard": 78,
"rows": 512851,
"tokens": 208731975,
"sha256": "669d2deb3e0ce6916585c94cc678cd6e30d75458dd0d15c90ce1d75fe120a4a7",
"source_sha256": "da96522698e8bd0fc5d3b4ddf406b90cf26f003e4b7d31047daf9da0ff6b7587",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 40059882,
"start_tokens": 16312820260
},
{
"shard": 79,
"rows": 509736,
"tokens": 207135712,
"sha256": "d25fdb3afb61d2686d965b25be9a96cba86f27048bfaff5ce9b33a27cf39deb5",
"source_sha256": "59957cdd0e228a6f0d341a75f3d8554bb66cf5e85c8b64a1e2a2c3db4ce26a14",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 40572733,
"start_tokens": 16521552235
},
{
"shard": 80,
"rows": 509324,
"tokens": 207021189,
"sha256": "4603be47bcf787178aba0ede3e7df5eb20ed175cfc6232eec7ca38654acd7e1e",
"source_sha256": "2548ed978086b5e5a2d4a22d4aee4667f9e6c61f6889d7562fdd5a50f74a2e50",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 41082469,
"start_tokens": 16728687947
},
{
"shard": 81,
"rows": 511105,
"tokens": 207943240,
"sha256": "38b28e69774f1539cd7d5fac47f224ed120dad0a412bfc4bde5fdd2fb7e0f2e2",
"source_sha256": "ce57364657181baa77f2a593cfc18da410988b3736ede0b10bda158367f9b92f",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 41591793,
"start_tokens": 16935709136
},
{
"shard": 82,
"rows": 509673,
"tokens": 207352440,
"sha256": "f37dc5e687cfa614616a994b55aadee9f93d6a50c2b1d2ab2b38b76062feee9d",
"source_sha256": "69efe078253b163cd9f30c574675cd7804d1bb6976d290fabd12ac8a2d029b43",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 42102898,
"start_tokens": 17143652376
},
{
"shard": 83,
"rows": 508094,
"tokens": 206513373,
"sha256": "cc3190af212447d44e803597f86f87c33fc37a25a812d6c6d26b1e7d3a756b1e",
"source_sha256": "44ca2ee16a1bd7e2fc1328c4ac94b8c8cd59d85e9ed6254943c506ff71b007c5",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 42612571,
"start_tokens": 17351004816
},
{
"shard": 84,
"rows": 510431,
"tokens": 207676905,
"sha256": "d550a3a72d5ba5794735c79be9e8888cc26c5cc8763191ea5442c16912f3caba",
"source_sha256": "008bd64d1cef26d0acf0f7a42853b7581671dc14d8543e0d2c055af8fd91eb95",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 43120665,
"start_tokens": 17557518189
},
{
"shard": 85,
"rows": 508960,
"tokens": 206809695,
"sha256": "6ed887b4a7a160ceabc3cdf7df51e3a3d16bcd2a3ac189993e16890a3bd2e0f6",
"source_sha256": "7f3235ea4ea3d3283968644205a430ccbddf0d9712b09787fbb055f8c770343c",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 43631096,
"start_tokens": 17765195094
},
{
"shard": 86,
"rows": 510007,
"tokens": 207440462,
"sha256": "892a045c8068c6b187ad0065df17b7715216d70c5a02ed3cf76d640b43cab136",
"source_sha256": "c4cf70ca119d8a1f0f4ef9ea249e9cc032db65b3cb23a3d3fbe5f0e55fc7e65c",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 44140056,
"start_tokens": 17972004789
},
{
"shard": 87,
"rows": 510689,
"tokens": 207759448,
"sha256": "1263f8fb474cba38825d7c07ff3d1446d7ff12b5da002c6cfa828a4b3a35b79c",
"source_sha256": "cd29581325ad822741ca0d4f6e5ece0568f8ebfbcfe23ecbf86c1639bc4a1cbf",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 44650063,
"start_tokens": 18179445251
},
{
"shard": 88,
"rows": 510099,
"tokens": 207429138,
"sha256": "0771fd56d91ed617a506ccc2a56cca254d2b0eb7c3cb538a42f8313dd39d3381",
"source_sha256": "8fbd3a5f58a9e58b15e32d9ed8e05d78d63cc2f599e1133b2ca1ec3978daf0fc",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 45160752,
"start_tokens": 18387204699
},
{
"shard": 89,
"rows": 506891,
"tokens": 205874287,
"sha256": "eac3d1bd04a8d0288be37751de6dbe612f566835d376be19b6f2e5cb683d68df",
"source_sha256": "3ddebc81b937ee4f0a57bc9f80d6573ca213fd4eba937edeac53089daf74692e",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 45670851,
"start_tokens": 18594633837
},
{
"shard": 90,
"rows": 508180,
"tokens": 206457815,
"sha256": "c94cc484c5c80156f64d7b311c73498368df000c5f5fe265100239262630bc33",
"source_sha256": "21767889b1e43d095b6568413a5e4b6034e6af46463dddd67ce3420ec6859b2f",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 46177742,
"start_tokens": 18800508124
},
{
"shard": 91,
"rows": 509320,
"tokens": 206935851,
"sha256": "b2d9905705f07bcd29de7674a36d842949be9f843e3ec605a698ba12263e28c1",
"source_sha256": "6054ced2fb1e1723f9ae9121b8debdafc43f91eb44c5fe42708735303c54b0a4",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 46685922,
"start_tokens": 19006965939
},
{
"shard": 92,
"rows": 508761,
"tokens": 206797758,
"sha256": "af586e71de02585ba2e5b7bceee6262f5a0a8045ab324fb010f1d7d99cb56d34",
"source_sha256": "d5f91d64386bcaedf35581f06bcb03af38db635d37227958e0bd060dd4463e1a",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 47195242,
"start_tokens": 19213901790
},
{
"shard": 93,
"rows": 508155,
"tokens": 206569132,
"sha256": "40483fa3519f82e450109a5728f39b11050708b34e1738c080d75be885876729",
"source_sha256": "529a931e9ddf27ba727c69f31361887787a450c92a0230fa3b84900865599023",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 47704003,
"start_tokens": 19420699548
},
{
"shard": 94,
"rows": 503136,
"tokens": 203890108,
"sha256": "c168eab17dbf8f3f2d338a331a553af5c10c577ed5a1d9d9f275b9c02bd07226",
"source_sha256": "2520e3dd784fa40ebefa495bb06607f2b82d55128e0812d163305cd6bdd40312",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 48212158,
"start_tokens": 19627268680
},
{
"shard": 95,
"rows": 505035,
"tokens": 204876711,
"sha256": "186be6d9cbd2047f1f098ea9dd5a77faa9cebebcf1d518479010bedf245312a0",
"source_sha256": "3ea3f3383cac92cefbc4f254d3550f0eb5c781292e7a92c3ee5ba6b65cc8c319",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 48715294,
"start_tokens": 19831158788
},
{
"shard": 96,
"rows": 506253,
"tokens": 205552052,
"sha256": "97e88439dee1feca39afb7d4ad148aec92a4192ed7aab69ee0cb7431f820b48e",
"source_sha256": "c5577f61aac321ffe934ca61836f87a1ac134ec958b09316f2ac7b78f8f01980",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 49220329,
"start_tokens": 20036035499
},
{
"shard": 97,
"rows": 220416,
"tokens": 89643130,
"sha256": "df54532ca91252bbd1cc9ffa0b76bd8fb9cf5af3e7d28be0340a0e6c1a214184",
"source_sha256": "edc4bf1cff43218f783a20c5cb884c4a6700dab7ec12cfb7a6362291e5392c86",
"tokenizer_sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c",
"start_row": 49726582,
"start_tokens": 20241587551
}
],
"segments": [
{
"segment": 0,
"end_row": 4898104,
"source_tokens": 2000000031,
"target_tokens": 2000000000
},
{
"segment": 1,
"end_row": 9802903,
"source_tokens": 4000000388,
"target_tokens": 4000000000
},
{
"segment": 2,
"end_row": 14714284,
"source_tokens": 6000000190,
"target_tokens": 6000000000
},
{
"segment": 3,
"end_row": 19637704,
"source_tokens": 8000000397,
"target_tokens": 8000000000
},
{
"segment": 4,
"end_row": 24567228,
"source_tokens": 10000000091,
"target_tokens": 10000000000
},
{
"segment": 5,
"end_row": 29489067,
"source_tokens": 12000000382,
"target_tokens": 12000000000
},
{
"segment": 6,
"end_row": 34384959,
"source_tokens": 14000000003,
"target_tokens": 14000000000
},
{
"segment": 7,
"end_row": 39291912,
"source_tokens": 16000000433,
"target_tokens": 16000000000
},
{
"segment": 8,
"end_row": 44208957,
"source_tokens": 18000000424,
"target_tokens": 18000000000
},
{
"segment": 9,
"end_row": 49131472,
"source_tokens": 20000000040,
"target_tokens": 20000000000
}
],
"budget": 2000000000
}
|