NiftyOracleAI commited on
Commit
17deee3
·
verified ·
1 Parent(s): babfc0a

pretok_ws_clean: 189.53B never-seen tokens, ws_codec formatted, deduped vs old pool (part 3)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. pretok_ws_clean/x_fineweb100_000087_006.bin +3 -0
  2. pretok_ws_clean/x_fineweb100_000087_007.bin +3 -0
  3. pretok_ws_clean/x_fineweb100_000088_000.bin +3 -0
  4. pretok_ws_clean/x_fineweb100_000088_001.bin +3 -0
  5. pretok_ws_clean/x_fineweb100_000088_002.bin +3 -0
  6. pretok_ws_clean/x_fineweb100_000088_003.bin +3 -0
  7. pretok_ws_clean/x_fineweb100_000088_004.bin +3 -0
  8. pretok_ws_clean/x_fineweb100_000088_005.bin +3 -0
  9. pretok_ws_clean/x_fineweb100_000088_006.bin +3 -0
  10. pretok_ws_clean/x_fineweb100_000088_007.bin +3 -0
  11. pretok_ws_clean/x_fineweb100_000089_000.bin +3 -0
  12. pretok_ws_clean/x_fineweb100_000089_001.bin +3 -0
  13. pretok_ws_clean/x_fineweb100_000089_002.bin +3 -0
  14. pretok_ws_clean/x_fineweb100_000089_003.bin +3 -0
  15. pretok_ws_clean/x_fineweb100_000089_004.bin +3 -0
  16. pretok_ws_clean/x_fineweb100_000089_005.bin +3 -0
  17. pretok_ws_clean/x_fineweb100_000089_006.bin +3 -0
  18. pretok_ws_clean/x_fineweb100_000089_007.bin +3 -0
  19. pretok_ws_clean/x_fineweb100_000091_000.bin +3 -0
  20. pretok_ws_clean/x_fineweb100_000091_001.bin +3 -0
  21. pretok_ws_clean/x_fineweb100_000091_002.bin +3 -0
  22. pretok_ws_clean/x_fineweb100_000091_003.bin +3 -0
  23. pretok_ws_clean/x_fineweb100_000091_004.bin +3 -0
  24. pretok_ws_clean/x_fineweb100_000091_005.bin +3 -0
  25. pretok_ws_clean/x_fineweb100_000091_006.bin +3 -0
  26. pretok_ws_clean/x_fineweb100_000091_007.bin +3 -0
  27. pretok_ws_clean/x_fineweb100_000092_000.bin +3 -0
  28. pretok_ws_clean/x_fineweb100_000092_001.bin +3 -0
  29. pretok_ws_clean/x_fineweb100_000092_002.bin +3 -0
  30. pretok_ws_clean/x_fineweb100_000092_003.bin +3 -0
  31. pretok_ws_clean/x_fineweb100_000092_004.bin +3 -0
  32. pretok_ws_clean/x_fineweb100_000092_005.bin +3 -0
  33. pretok_ws_clean/x_fineweb100_000092_006.bin +3 -0
  34. pretok_ws_clean/x_fineweb100_000092_007.bin +3 -0
  35. pretok_ws_clean/x_fineweb100_000093_000.bin +3 -0
  36. pretok_ws_clean/x_fineweb100_000093_001.bin +3 -0
  37. pretok_ws_clean/x_fineweb100_000093_002.bin +3 -0
  38. pretok_ws_clean/x_fineweb100_000093_003.bin +3 -0
  39. pretok_ws_clean/x_fineweb100_000093_004.bin +3 -0
  40. pretok_ws_clean/x_fineweb100_000093_005.bin +3 -0
  41. pretok_ws_clean/x_fineweb100_000093_006.bin +3 -0
  42. pretok_ws_clean/x_fineweb100_000093_007.bin +3 -0
  43. pretok_ws_clean/x_fineweb100_000095_000.bin +3 -0
  44. pretok_ws_clean/x_fineweb100_000095_001.bin +3 -0
  45. pretok_ws_clean/x_fineweb100_000095_002.bin +3 -0
  46. pretok_ws_clean/x_fineweb100_000095_003.bin +3 -0
  47. pretok_ws_clean/x_fineweb100_000095_004.bin +3 -0
  48. pretok_ws_clean/x_fineweb100_000095_005.bin +3 -0
  49. pretok_ws_clean/x_fineweb100_000095_006.bin +3 -0
  50. pretok_ws_clean/x_fineweb100_000095_007.bin +3 -0
pretok_ws_clean/x_fineweb100_000087_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c568a628b6eb360406a3b8ad227db93fa66cc32467476771e5d4f9bbf3d9a2a0
3
+ size 97427428
pretok_ws_clean/x_fineweb100_000087_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf61d5a84d74d5019ce7488a7b13265fd9243eb8accacab5842fd127c4390325
3
+ size 25458740
pretok_ws_clean/x_fineweb100_000088_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0530a1c14f8b93cfb494f4cb3bb6e8aecdc7e330f5ccdd9d9a87163bae076fe
3
+ size 135676788
pretok_ws_clean/x_fineweb100_000088_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc77fac4528696323ce1b09eff63074a4da6fe7314fabd7afb39e72ff78b2480
3
+ size 56988856
pretok_ws_clean/x_fineweb100_000088_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cbfd957a50351119fcb961df9d73dac7a1d9bcc195cb232d110b70a983ab01fe
3
+ size 45189626
pretok_ws_clean/x_fineweb100_000088_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bda8e15bd23c67ab1e5a3b00bd80f7d13a151d3ef5d8904ec52c6a696485663e
3
+ size 100741350
pretok_ws_clean/x_fineweb100_000088_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e78b34d6b42abdf71bfbfa2af793afe3aabba977b5a69f2794150c744624626d
3
+ size 129071962
pretok_ws_clean/x_fineweb100_000088_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d6dc57e4eb8201ed9e5c666933c2efcb982a61b188a10fd79379b41ddac53585
3
+ size 107229272
pretok_ws_clean/x_fineweb100_000088_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de07802d010489b8a29536c69ec4b72f8c98b06700ade2df49043ea1d46188f7
3
+ size 134518752
pretok_ws_clean/x_fineweb100_000088_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a63b5828a39880cb01368ca3c104761dc0578edadfffda4d247da7edf326a10
3
+ size 65441432
pretok_ws_clean/x_fineweb100_000089_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed0533fd8379b2f85c8d77b6a5e824ad6c1eda328666fb6f8e8453d35f743797
3
+ size 126749404
pretok_ws_clean/x_fineweb100_000089_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a87866160e07a110ec0a1c7d738445e2b5f145565d3e220aba27960b3e66af4
3
+ size 125098528
pretok_ws_clean/x_fineweb100_000089_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:720e4ad169c2dfc0d5c5b67d895f03fa59ef1bace625d519af6f873b902559c2
3
+ size 94795860
pretok_ws_clean/x_fineweb100_000089_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7f92bb620f3d4e58ba6443d49b7aee6a9c82b3c0b4d78dd0459f0c68104c5ad
3
+ size 134373238
pretok_ws_clean/x_fineweb100_000089_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b87f28b283a66a4eab5f6b6fd81bbc2d0b16493ea0711646536e13e103b524a1
3
+ size 123079012
pretok_ws_clean/x_fineweb100_000089_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:88ad7e9deccf141a2cbb111c61984bf0213481c9478596f36dcb40edad77c58b
3
+ size 90516744
pretok_ws_clean/x_fineweb100_000089_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84811a0b8be06b47729626bbdbf12a429842f40654a789dd84dd4b70de72e2c8
3
+ size 140902952
pretok_ws_clean/x_fineweb100_000089_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:66c3ad955af615b9c17230f59ed75fe6f5b0b7972e6bd56d3272f62ebd9821b8
3
+ size 73241098
pretok_ws_clean/x_fineweb100_000091_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4ddd7b8c60a99c9169d220e58849cf445f18f1b1e3d1d9302ca557d8427c2b4c
3
+ size 41796098
pretok_ws_clean/x_fineweb100_000091_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68af65eb3f25b2978f98a2ee3034618d97bb3b0745485d8b097b63b45a4f2b7c
3
+ size 131152960
pretok_ws_clean/x_fineweb100_000091_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:adc74cb4b072cca77bff9faa287c6deb8639445ba32708a70bd1d8f909fa0b0c
3
+ size 93687660
pretok_ws_clean/x_fineweb100_000091_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be68cf3ac0925acb14867b442dc4699deeded9d1a9e75faab0928df13db6a080
3
+ size 60265558
pretok_ws_clean/x_fineweb100_000091_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:678e6d72392a45e1385b1bdaf42bf434cdbb13118bd1708381161c6214b4f7a5
3
+ size 74900196
pretok_ws_clean/x_fineweb100_000091_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:238f5a53639326685e5b63b2fead10cb0bc703e5fc4ac05104494ae1c04a542a
3
+ size 34486226
pretok_ws_clean/x_fineweb100_000091_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af411520ca89f49eb8a4ed1902ebd000ae4b4755212498c99fb38b7cfd08cd9a
3
+ size 85199054
pretok_ws_clean/x_fineweb100_000091_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7e3d3eda741aaf10ff28b81872041a848c299d58ac58557b5b1c89eb836b17a
3
+ size 47540258
pretok_ws_clean/x_fineweb100_000092_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44068748654c9357540ce256b0dadb61193c8035fd0c83cf310a0e8b28cfc0ef
3
+ size 113796920
pretok_ws_clean/x_fineweb100_000092_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4780f9f1024c3c0678ced6b30b4775fe22f3d24fa2b9fceec3b000545b3fdee
3
+ size 134977204
pretok_ws_clean/x_fineweb100_000092_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3caf2d56269243ecad3acdaf2b3c18baf24c991cba58b1c1702d7dbe5faf0f9a
3
+ size 101223448
pretok_ws_clean/x_fineweb100_000092_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9505b6b4e53c05b0323bdda43e2ef544e42dc0ffb2a26801325c455704a30bf8
3
+ size 134089158
pretok_ws_clean/x_fineweb100_000092_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de9394b76a7a8abec6a00f25ae39bf5e11f51c426fba88ea947f22125a091053
3
+ size 115493750
pretok_ws_clean/x_fineweb100_000092_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f970d8218435622584d281cd51e849f8e9d8da75dc02619c04966b451637c64f
3
+ size 90291800
pretok_ws_clean/x_fineweb100_000092_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:766f4a23f6f99ec3fd6b046b9c51efb637178d7b7737737e1803836770a39469
3
+ size 133458338
pretok_ws_clean/x_fineweb100_000092_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92b10e1579c80c7794ab89e6ce74da01fdca5120bf8b27d99e0de34388b693bd
3
+ size 75529408
pretok_ws_clean/x_fineweb100_000093_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af7fba3015352ea28d4b58b2f79b7c3470ad248dab040a0ceb9afae9b983e5e7
3
+ size 115708890
pretok_ws_clean/x_fineweb100_000093_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27d90025281d84d7621f59b34c4d83c1a9e09471249fa8b53eb4c52c2b9f2a68
3
+ size 33756016
pretok_ws_clean/x_fineweb100_000093_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c90f1d456e0802fff2bb55bc523a8c184a07a0fde3b315d184a116dd8af9e380
3
+ size 60724770
pretok_ws_clean/x_fineweb100_000093_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:441524c362e243697902f3a96aaba7da886810e46673800b7fb46039dcfb5c4d
3
+ size 112666064
pretok_ws_clean/x_fineweb100_000093_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af97e3e9a3ba20a4764de739dcf0a10fbe73f306b52da0f567a838851fc36838
3
+ size 123853958
pretok_ws_clean/x_fineweb100_000093_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9075a8412616f2f1626bb60a3493df97087ec6ca42709cabfac7e46340c910f5
3
+ size 91282242
pretok_ws_clean/x_fineweb100_000093_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6033f493250a19738504a26616105093aa683026a78938f71e14e2e00733e52e
3
+ size 131413414
pretok_ws_clean/x_fineweb100_000093_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c82e0216b8d9127426d48b385f4c9b0874eb169e4fdf16a427abf8e1c46a9db
3
+ size 46959994
pretok_ws_clean/x_fineweb100_000095_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38ece221d9b66ce0dac038134c3c858d4e25ec62b276d79c62727d97a3cd3529
3
+ size 79842382
pretok_ws_clean/x_fineweb100_000095_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc6d56dc381ef537ec2a06fd763bd1690aacdf8209c7dd94a21386a6083ea37a
3
+ size 119489286
pretok_ws_clean/x_fineweb100_000095_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3929fe80948aca383be74be33c0840ebc552ccfa29994ce0f71adcbceed44330
3
+ size 78249596
pretok_ws_clean/x_fineweb100_000095_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0873e88f9ea15c6fb0f4b329ab5cb1ff5f581435aebeef6f748d23f3dc080caa
3
+ size 112923486
pretok_ws_clean/x_fineweb100_000095_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a637769413196dd9c5aa83e00297c808e1fa25e5c18df46b718cbf01a9be54a1
3
+ size 89505654
pretok_ws_clean/x_fineweb100_000095_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a5fd2b96595ae728a0e6896cb45769156d8e201ee14fac2004d96030b0437c76
3
+ size 94849730
pretok_ws_clean/x_fineweb100_000095_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77279a1ed0bbd853f85038a9f78fdc0ebb0106e88bdce8c307973fd3d5910807
3
+ size 113733548
pretok_ws_clean/x_fineweb100_000095_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b5510392cc20515f4bc2ac1a3cd22e3b137d1e8833c58f617945027f0aa64cf
3
+ size 52223910