NiftyOracleAI commited on
Commit
2c09982
·
verified ·
1 Parent(s): 17deee3

pretok_ws_clean: 189.53B never-seen tokens, ws_codec formatted, deduped vs old pool (part 4)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. pretok_ws_clean/x_fineweb350_000224_008.bin +3 -0
  2. pretok_ws_clean/x_fineweb350_000224_009.bin +3 -0
  3. pretok_ws_clean/x_fineweb350_000225_000.bin +3 -0
  4. pretok_ws_clean/x_fineweb350_000225_001.bin +3 -0
  5. pretok_ws_clean/x_fineweb350_000225_002.bin +3 -0
  6. pretok_ws_clean/x_fineweb350_000225_003.bin +3 -0
  7. pretok_ws_clean/x_fineweb350_000225_004.bin +3 -0
  8. pretok_ws_clean/x_fineweb350_000225_005.bin +3 -0
  9. pretok_ws_clean/x_fineweb350_000225_006.bin +3 -0
  10. pretok_ws_clean/x_fineweb350_000225_007.bin +3 -0
  11. pretok_ws_clean/x_fineweb350_000225_008.bin +3 -0
  12. pretok_ws_clean/x_fineweb350_000225_009.bin +3 -0
  13. pretok_ws_clean/x_fineweb350_000226_000.bin +3 -0
  14. pretok_ws_clean/x_fineweb350_000226_001.bin +3 -0
  15. pretok_ws_clean/x_fineweb350_000226_002.bin +3 -0
  16. pretok_ws_clean/x_fineweb350_000226_003.bin +3 -0
  17. pretok_ws_clean/x_fineweb350_000226_004.bin +3 -0
  18. pretok_ws_clean/x_fineweb350_000226_005.bin +3 -0
  19. pretok_ws_clean/x_fineweb350_000226_006.bin +3 -0
  20. pretok_ws_clean/x_fineweb350_000226_007.bin +3 -0
  21. pretok_ws_clean/x_fineweb350_000226_008.bin +3 -0
  22. pretok_ws_clean/x_fineweb350_000226_009.bin +3 -0
  23. pretok_ws_clean/x_fineweb350_000227_000.bin +3 -0
  24. pretok_ws_clean/x_fineweb350_000227_001.bin +3 -0
  25. pretok_ws_clean/x_fineweb350_000227_002.bin +3 -0
  26. pretok_ws_clean/x_fineweb350_000227_003.bin +3 -0
  27. pretok_ws_clean/x_fineweb350_000227_004.bin +3 -0
  28. pretok_ws_clean/x_fineweb350_000227_005.bin +3 -0
  29. pretok_ws_clean/x_fineweb350_000227_006.bin +3 -0
  30. pretok_ws_clean/x_fineweb350_000227_007.bin +3 -0
  31. pretok_ws_clean/x_fineweb350_000227_008.bin +3 -0
  32. pretok_ws_clean/x_fineweb350_000227_009.bin +3 -0
  33. pretok_ws_clean/x_fineweb350_000228_000.bin +3 -0
  34. pretok_ws_clean/x_fineweb350_000228_001.bin +3 -0
  35. pretok_ws_clean/x_fineweb350_000228_002.bin +3 -0
  36. pretok_ws_clean/x_fineweb350_000228_003.bin +3 -0
  37. pretok_ws_clean/x_fineweb350_000228_004.bin +3 -0
  38. pretok_ws_clean/x_fineweb350_000228_005.bin +3 -0
  39. pretok_ws_clean/x_fineweb350_000228_006.bin +3 -0
  40. pretok_ws_clean/x_fineweb350_000228_007.bin +3 -0
  41. pretok_ws_clean/x_fineweb350_000228_008.bin +3 -0
  42. pretok_ws_clean/x_fineweb350_000228_009.bin +3 -0
  43. pretok_ws_clean/x_fineweb350_000229_000.bin +3 -0
  44. pretok_ws_clean/x_fineweb350_000229_001.bin +3 -0
  45. pretok_ws_clean/x_fineweb350_000229_002.bin +3 -0
  46. pretok_ws_clean/x_fineweb350_000229_003.bin +3 -0
  47. pretok_ws_clean/x_fineweb350_000229_004.bin +3 -0
  48. pretok_ws_clean/x_fineweb350_000229_005.bin +3 -0
  49. pretok_ws_clean/x_fineweb350_000229_006.bin +3 -0
  50. pretok_ws_clean/x_fineweb350_000229_007.bin +3 -0
pretok_ws_clean/x_fineweb350_000224_008.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03eee60c20f75c96cdc9b63d437c8c0c690447170ec6dc029dee2be74ca5e955
3
+ size 70747962
pretok_ws_clean/x_fineweb350_000224_009.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53c86b2d634138838b7c7c038037a6f82b027cf1a0fc33412730063dad0312cd
3
+ size 11283248
pretok_ws_clean/x_fineweb350_000225_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c2788d42822b0de5cde0ea59462ec7d0f3444788a3a3c9e1360c8fafebd58b7
3
+ size 75906698
pretok_ws_clean/x_fineweb350_000225_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4300cc2e4da717a9d3b5c9cdf21c8e24fe9b52506403925cfc1303c3e7dc920a
3
+ size 53389984
pretok_ws_clean/x_fineweb350_000225_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f7b43f2117036d7c961c66637c474d97b4708e88fa52708661865e620a8629a
3
+ size 46570450
pretok_ws_clean/x_fineweb350_000225_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79d7b6cdb82d608f29156567fd52142727b407e865f12823823d4e2a297dbdc5
3
+ size 89459608
pretok_ws_clean/x_fineweb350_000225_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:362e46eab77c4cc4bc49420e7aaf941330bf330a04ebee509e85d68a09baac59
3
+ size 166360410
pretok_ws_clean/x_fineweb350_000225_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab3f34a9d023a6d0dd88e6dbfa762bdcea3c3fc504b2c6c3c5bae3377fdb0a85
3
+ size 165165414
pretok_ws_clean/x_fineweb350_000225_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6bb38f512c9afc482594066036b4eb10de9a2643887ad105bbe5f63f020d42e4
3
+ size 116968612
pretok_ws_clean/x_fineweb350_000225_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b164a50b92bbc98e7d86c147f4d1d68457572bc2d1f428f7532769d61e802006
3
+ size 81651450
pretok_ws_clean/x_fineweb350_000225_008.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7bb6c04ef1780d5407617144d469b5f08c45d585ddb931b4edec93c49ad4f8bb
3
+ size 83551382
pretok_ws_clean/x_fineweb350_000225_009.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c7e50ed2fb5a720105e4b339b2030966cfb824671998c2b7a589d2001b975f50
3
+ size 19234114
pretok_ws_clean/x_fineweb350_000226_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f0971221854e6d4a7ed2e7ff4877f640dceb7c1163ef1c2411124e18c0fd153
3
+ size 118736690
pretok_ws_clean/x_fineweb350_000226_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3facab6cec324bd6b8a9cd0c56f3b0a74198509288fcbb61c6946d7422ff28c
3
+ size 94714570
pretok_ws_clean/x_fineweb350_000226_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8babb85172da62d3129c05b7eb15560270c6f2d34af2f0fb3bea0d6e272e547d
3
+ size 73357752
pretok_ws_clean/x_fineweb350_000226_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b6346776a6dc1fff595a6d63a6d85ba2e3d7d1632d8adf00af6b7cfa1085ccac
3
+ size 70340340
pretok_ws_clean/x_fineweb350_000226_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c122467f4ad4684ae3f98d1f773f71657214d753445faad2a06997ed48235c9
3
+ size 70892394
pretok_ws_clean/x_fineweb350_000226_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00068bc990689ebd6f58a5876ff5e76df857d7d9d17d138dce9fc602499ad3a7
3
+ size 71695510
pretok_ws_clean/x_fineweb350_000226_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d2f07616218ccf1f108ad73012a5122b22e63efacaca7f69e0dc2cdb837eb1a
3
+ size 76063494
pretok_ws_clean/x_fineweb350_000226_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29dffe4ac8bf9db797eabd577be378905a8fcb5fa6db7980bb9ef9d3b1b4b442
3
+ size 93437484
pretok_ws_clean/x_fineweb350_000226_008.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8797352b5bc754600911f72b7caa90a0c5e74510e21473bfcd94af1f879b14ce
3
+ size 90359454
pretok_ws_clean/x_fineweb350_000226_009.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f293015ac31dfd4ad853212229b39d6279b3f2980f3f3fa784b904e11d0b352
3
+ size 17414620
pretok_ws_clean/x_fineweb350_000227_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77740295533a91176e5dec1b6aa194a1872295ffdbd34c9d82abf086fb294966
3
+ size 165962818
pretok_ws_clean/x_fineweb350_000227_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6873b8513bd7ceb7dcdac5daa9dad8a0297bcda936dc75c5b19cf407f2cc7b3
3
+ size 138003356
pretok_ws_clean/x_fineweb350_000227_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a687a00f988c0e3b144a223cd1e77226d91f23484359bd23ef4a17cfed00f245
3
+ size 112658812
pretok_ws_clean/x_fineweb350_000227_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6000638c7821e521dbc6945b474a6924e22037d9be0ad7b46a38c541935a5463
3
+ size 121023358
pretok_ws_clean/x_fineweb350_000227_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eef835abbdcdf7c0b0c69528f58f827a4c469b47e09f8a9ad548a1e74d648683
3
+ size 98944056
pretok_ws_clean/x_fineweb350_000227_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9d364e3aa395c9881291774b91500ac89e250f7b5d518f46c5d9d930ecf1081
3
+ size 107001350
pretok_ws_clean/x_fineweb350_000227_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36fcc014e983ac426a53d56db7f2a5d7ed312504c84c4267e1514ccdc35af85e
3
+ size 94346136
pretok_ws_clean/x_fineweb350_000227_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32a122232b956e7f62a4a683d860e65ef10a52f31f913bcd08973cb311ca16a3
3
+ size 113465408
pretok_ws_clean/x_fineweb350_000227_008.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8a68af2b109a97fdaa352b7179285ed14693bfdcff25d36702f2c53381d3f8a3
3
+ size 110600834
pretok_ws_clean/x_fineweb350_000227_009.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78e281f196f89b4944509a6b7a7897959eeabdbc1cb895e502aa236fa7854dec
3
+ size 23854684
pretok_ws_clean/x_fineweb350_000228_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:20d6f11afda30d346bd47a2baa6586d50cedb6bcb80cbb35a162b1738bb9248f
3
+ size 89252312
pretok_ws_clean/x_fineweb350_000228_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5855ad5c79b8a87465d1710af8d646b83c37afd14de81d9fa1e38875ea895f9b
3
+ size 97251580
pretok_ws_clean/x_fineweb350_000228_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eee1e9b69c45a87a90f1016173cd5646b6192ba2b1feb46fedf2c6938f5b7398
3
+ size 95662078
pretok_ws_clean/x_fineweb350_000228_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:58db15615b852205721072b06fe92327453b1eddca7f3a894f418c0346784501
3
+ size 90561848
pretok_ws_clean/x_fineweb350_000228_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:15e59ab8ca804e056796ff3b321fe80bdfcdf827906af2dd7830b426ddf64c8f
3
+ size 89923930
pretok_ws_clean/x_fineweb350_000228_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f617ed3258837a730445d05b802b20a95edb20852f00f4057962dc64803bf48
3
+ size 81640780
pretok_ws_clean/x_fineweb350_000228_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3ad0c138b6ca6b17b3f239e82eb3550052df8b55c00d363c7510cda374121541
3
+ size 76730674
pretok_ws_clean/x_fineweb350_000228_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70a300b66459b5270bc792885f24c8897180fe81dab8efcbb031dfb9ee09dc40
3
+ size 67285986
pretok_ws_clean/x_fineweb350_000228_008.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09cb5055568aece93521fa8aefed25a1a25ebc679bc00f4c8b07e5e2e5ad7f89
3
+ size 100910212
pretok_ws_clean/x_fineweb350_000228_009.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33c5996f563981486f3094c7bb25c229e1f5013e51acbf0cc2a3fe18d7d2d80e
3
+ size 18987000
pretok_ws_clean/x_fineweb350_000229_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03c1e0a4424eda5962578bbbb6b778c27a5b95ef8350e64d2e54ea3ee2e9c18f
3
+ size 98534388
pretok_ws_clean/x_fineweb350_000229_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8413dc56fc6a63acf18957a55ecc569063ba981664cb9efbefcf86ba7ae81781
3
+ size 104019092
pretok_ws_clean/x_fineweb350_000229_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:435e66cd3266cf5a077e9398066284d8ca7dee77979fa2d50e893c7393304c8b
3
+ size 89113608
pretok_ws_clean/x_fineweb350_000229_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3580856242150143b40e3ccf50345fbf6723bf2b7b3a1276b6a32417e6d8b42
3
+ size 80940684
pretok_ws_clean/x_fineweb350_000229_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b52d625dab4bcb0db0cbf241b6cd466b5ead326f778ae072dfaf227c6b4878c
3
+ size 85163106
pretok_ws_clean/x_fineweb350_000229_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff344fa2e394d56e3b3d391932ed79c62c56dff975d40a42a1c7d18cdab5634e
3
+ size 70292390
pretok_ws_clean/x_fineweb350_000229_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68f63bdcff3ba3249cc9a526bbce9502c7ddd173723481f3cae2e9bb9316796d
3
+ size 77917022
pretok_ws_clean/x_fineweb350_000229_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2828cc266629e319b95c2a2af445995c2da351d6560325fa63dca35d42702ed9
3
+ size 73499994