NiftyOracleAI commited on
Commit
babfc0a
·
verified ·
1 Parent(s): 6105b30

pretok_ws_clean: 189.53B never-seen tokens, ws_codec formatted, deduped vs old pool (part 2)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. pretok_ws_clean/x_cpp_000020_071.bin +3 -0
  2. pretok_ws_clean/x_cpp_000020_072.bin +3 -0
  3. pretok_ws_clean/x_cpp_000020_073.bin +3 -0
  4. pretok_ws_clean/x_cpp_000020_074.bin +3 -0
  5. pretok_ws_clean/x_csharp_000039_000.bin +3 -0
  6. pretok_ws_clean/x_csharp_000039_001.bin +3 -0
  7. pretok_ws_clean/x_csharp_000039_002.bin +3 -0
  8. pretok_ws_clean/x_csharp_000039_003.bin +3 -0
  9. pretok_ws_clean/x_csharp_000039_004.bin +3 -0
  10. pretok_ws_clean/x_csharp_000039_005.bin +3 -0
  11. pretok_ws_clean/x_csharp_000039_006.bin +3 -0
  12. pretok_ws_clean/x_csharp_000039_007.bin +3 -0
  13. pretok_ws_clean/x_csharp_000039_008.bin +3 -0
  14. pretok_ws_clean/x_csharp_000039_009.bin +3 -0
  15. pretok_ws_clean/x_csharp_000039_010.bin +3 -0
  16. pretok_ws_clean/x_csharp_000039_011.bin +3 -0
  17. pretok_ws_clean/x_csharp_000039_012.bin +3 -0
  18. pretok_ws_clean/x_csharp_000039_013.bin +3 -0
  19. pretok_ws_clean/x_csharp_000039_014.bin +3 -0
  20. pretok_ws_clean/x_csharp_000039_015.bin +3 -0
  21. pretok_ws_clean/x_csharp_000039_016.bin +3 -0
  22. pretok_ws_clean/x_csharp_000039_017.bin +3 -0
  23. pretok_ws_clean/x_csharp_000039_018.bin +3 -0
  24. pretok_ws_clean/x_csharp_000039_019.bin +3 -0
  25. pretok_ws_clean/x_csharp_000039_020.bin +3 -0
  26. pretok_ws_clean/x_csharp_000039_021.bin +3 -0
  27. pretok_ws_clean/x_csharp_000039_022.bin +3 -0
  28. pretok_ws_clean/x_csharp_000039_023.bin +3 -0
  29. pretok_ws_clean/x_csharp_000039_024.bin +3 -0
  30. pretok_ws_clean/x_csharp_000039_025.bin +3 -0
  31. pretok_ws_clean/x_csharp_000039_026.bin +3 -0
  32. pretok_ws_clean/x_csharp_000039_027.bin +3 -0
  33. pretok_ws_clean/x_csharp_000039_028.bin +3 -0
  34. pretok_ws_clean/x_csharp_000039_029.bin +3 -0
  35. pretok_ws_clean/x_csharp_000039_030.bin +3 -0
  36. pretok_ws_clean/x_csharp_000039_031.bin +3 -0
  37. pretok_ws_clean/x_csharp_000039_032.bin +3 -0
  38. pretok_ws_clean/x_csharp_000039_033.bin +3 -0
  39. pretok_ws_clean/x_csharp_000039_034.bin +3 -0
  40. pretok_ws_clean/x_csharp_000039_035.bin +3 -0
  41. pretok_ws_clean/x_csharp_000039_036.bin +3 -0
  42. pretok_ws_clean/x_csharp_000039_037.bin +3 -0
  43. pretok_ws_clean/x_csharp_000039_038.bin +3 -0
  44. pretok_ws_clean/x_csharp_000039_039.bin +3 -0
  45. pretok_ws_clean/x_csharp_000039_040.bin +3 -0
  46. pretok_ws_clean/x_csharp_000039_041.bin +3 -0
  47. pretok_ws_clean/x_csharp_000039_042.bin +3 -0
  48. pretok_ws_clean/x_csharp_000039_043.bin +3 -0
  49. pretok_ws_clean/x_csharp_000039_044.bin +3 -0
  50. pretok_ws_clean/x_csharp_000039_045.bin +3 -0
pretok_ws_clean/x_cpp_000020_071.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:189e6b4e13bcc5c2a821473139ea0619fefa48dbc8c0a2a279296534ba5e9861
3
+ size 190840912
pretok_ws_clean/x_cpp_000020_072.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04a4a0cc860f946bb02571453078a286bec6c268e44b62bbd7d557b2ac522ab8
3
+ size 192082476
pretok_ws_clean/x_cpp_000020_073.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ad7b1c417601d187e4b0b81c6a1e9e4e35e0b5861c4da0defc9290aae3992f83
3
+ size 190352806
pretok_ws_clean/x_cpp_000020_074.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7314429ce2092e1ba44dac7b212abac27b7eb96a0412b78f816150f9926cb722
3
+ size 150141656
pretok_ws_clean/x_csharp_000039_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ad63642cd9cba53b61743d18bf0e72add2f90d1df6be17b55ec2d92b7626ed6c
3
+ size 195322822
pretok_ws_clean/x_csharp_000039_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f5877976a3f27b63016645d7f19c780960d971f02bc98f3dd1f0804a90009ac
3
+ size 195002346
pretok_ws_clean/x_csharp_000039_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:301b64773347a1064cbd81959ae72838b0054bdcc37ca888d282296b5a704cb3
3
+ size 194601390
pretok_ws_clean/x_csharp_000039_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9ca92a4c1938ca128c367b69fa47fa1aa8b5abf14aa778454c02b9ed73fdb86c
3
+ size 194957386
pretok_ws_clean/x_csharp_000039_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b17b1fb217edfc1ee347af3101d60111052f722af6b0a9bb0ea3a54db0e6072
3
+ size 195117944
pretok_ws_clean/x_csharp_000039_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c14d2dbf190225e35c9b27a9613f37be8972d74f39556419c8b63f1361d7dd7c
3
+ size 195083136
pretok_ws_clean/x_csharp_000039_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41b8079a08fee4c90dfb9cb4a05c8077baa0ddf73d57d27807a30745af65e9f5
3
+ size 195140956
pretok_ws_clean/x_csharp_000039_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf7e234232299ea74e0f6f8f76209ec6a3a1b38871b4177d1b4b78adf5f73895
3
+ size 195399590
pretok_ws_clean/x_csharp_000039_008.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0f3520a6bb89c976a1b0945bb22a2ff76f71b53aaf8d0a3e28bed34ac6a2f37
3
+ size 194809482
pretok_ws_clean/x_csharp_000039_009.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ba8ffe5347b3e8a0ba36f84721f29dcb223f5367a4a193f7a496ba74551b4412
3
+ size 194857754
pretok_ws_clean/x_csharp_000039_010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1652e78310887eae1b808cb510c97224552ea453445c4ef1eae8c6635df8236f
3
+ size 194655032
pretok_ws_clean/x_csharp_000039_011.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33eee97e165b5c57127cc5f3372aaf84affc80485e6f51679e07709cb4aaa2b5
3
+ size 195356830
pretok_ws_clean/x_csharp_000039_012.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb701caa009a96b11202e9f30d40491f59575220c231b5837dcb36d006ecc7e5
3
+ size 195017528
pretok_ws_clean/x_csharp_000039_013.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6e01fe0897b53c2e01050581984e4a6ef2fd858e0cd2254ae922b524dad8f18
3
+ size 195357020
pretok_ws_clean/x_csharp_000039_014.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96d8ac32e8032e2f81fe92e5332dddb3b4278a131f1766feb69721a6a1d4feb1
3
+ size 194782120
pretok_ws_clean/x_csharp_000039_015.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46ef100e393464d48443823ef3ce9077f538cad02fe8c20ae5c4d0d39ff6d89b
3
+ size 195171122
pretok_ws_clean/x_csharp_000039_016.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97e5cd3c3ea683bfde9671bc3f9ca99509b772cb00081d4b9050facd883e2bc6
3
+ size 194692950
pretok_ws_clean/x_csharp_000039_017.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73d368298b42bd50e9cd29ea95d6ade8ec0d0382f0e20d360c68f698e4dbe897
3
+ size 195121038
pretok_ws_clean/x_csharp_000039_018.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea047a1b6a12a5f284d1e3c14ff3f0f78592de5b124314f9252f3548eb01294d
3
+ size 194561070
pretok_ws_clean/x_csharp_000039_019.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7419e564eda1376a954d8ab4787cd4fd24da60014a6fc2e894c500423f48434
3
+ size 195382162
pretok_ws_clean/x_csharp_000039_020.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f021d76c0a698776571c11ee03d26da8c73ebcf5f488b75829384fae910b3557
3
+ size 194854606
pretok_ws_clean/x_csharp_000039_021.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ae947fb4f6b5d4f003aecf4207ff503f9ba0fd5709809787d50373283b0245a6
3
+ size 195039392
pretok_ws_clean/x_csharp_000039_022.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0a8c8fe946afb0359bc6f05780970a2107dfafff7e571a4583d6910bb1a0ca0
3
+ size 194610438
pretok_ws_clean/x_csharp_000039_023.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6674454d0f0c7bd0555254b02f64dc7fba466876bb48e1c5ad1733d600faa696
3
+ size 195004428
pretok_ws_clean/x_csharp_000039_024.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3276216b3a8899435f4b05d4a6a001f641f4680f69fb9daf0240cc172cf7c8b0
3
+ size 195013394
pretok_ws_clean/x_csharp_000039_025.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e687ff17e018bee93607094e175b21cd8ed59911064d48ee8112e850aece4d3
3
+ size 194832728
pretok_ws_clean/x_csharp_000039_026.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92776ccbcfb8b73074ed0ac2772b06f41dcf835211f0a4b8d64c8000c1d0cb47
3
+ size 194873426
pretok_ws_clean/x_csharp_000039_027.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4117f82e69501fb75217f9011499c56e49ae65408027238af36d8a78d9ded82
3
+ size 194874716
pretok_ws_clean/x_csharp_000039_028.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:514b9c57bae513981662582fecbf542b261652fc5bd71e893465a08281956808
3
+ size 195186226
pretok_ws_clean/x_csharp_000039_029.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1fa212ac87d6efbd20c6a8ec112a107345adbec24e3602ff42fbfcc0c4107302
3
+ size 194930670
pretok_ws_clean/x_csharp_000039_030.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:903fee16fa104c3283eaf11d22c842b4948a4b717c0a1de1263b53e96bf7f1ad
3
+ size 195106488
pretok_ws_clean/x_csharp_000039_031.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4e3414326a80ff2b9de958b83cb5322dfe46d4cda3dccfa62ae3c528bf843390
3
+ size 194704362
pretok_ws_clean/x_csharp_000039_032.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08729b44e6b3d65f97d05f6818a4d7e2ceecae7432601c74b086a0e18f296650
3
+ size 195025580
pretok_ws_clean/x_csharp_000039_033.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61c5adf59932083e68889b883a3e468ce80caad6db565a96e7f229edc522ce65
3
+ size 194814430
pretok_ws_clean/x_csharp_000039_034.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9139fd639a0353fc22a213bcfe40a2ed6c972fb3faa99618d3c21c65ecbcea8a
3
+ size 194827928
pretok_ws_clean/x_csharp_000039_035.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b65c14c2f7dc83fb75236a6067e93a2bcf222edc7289b57749a6d3ae6cd18cd
3
+ size 194340764
pretok_ws_clean/x_csharp_000039_036.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a2ccede5d761918b939a5295fee60f0868c721a61f37e0356bbabc62559fdaf
3
+ size 194610792
pretok_ws_clean/x_csharp_000039_037.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b6cc6107074d7cc9781e0800753d65893f225a83ce5fed56200bb8c5a242a2e4
3
+ size 194966190
pretok_ws_clean/x_csharp_000039_038.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ba3f4b3f02239b9e05e3f983e9236d8ce68adcb8e3981d29fd74469f92a705ff
3
+ size 195230464
pretok_ws_clean/x_csharp_000039_039.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a573f7ca4a3d563e1270d4d39090de80de888d50e4b3f81f47f53944f0d833af
3
+ size 195074082
pretok_ws_clean/x_csharp_000039_040.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbb1561b64e5412ffe65129e41ba73145d495b896729b6020a4e9ee44a5f509a
3
+ size 195025716
pretok_ws_clean/x_csharp_000039_041.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1df3e4d996569a211b33149d7e93b82d516f337cb06a525cb6faec2cf12c6c58
3
+ size 195105552
pretok_ws_clean/x_csharp_000039_042.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee7292a164ed6ffd7f0b6dd5775dae0346b41244a9f025b8cbc1a4f3b3f9b79f
3
+ size 194850084
pretok_ws_clean/x_csharp_000039_043.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c8e33bea1d73638bfaf7f8b58a29126e65184d071b163e2ee5b613857266680
3
+ size 195219344
pretok_ws_clean/x_csharp_000039_044.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03caa47ae5711cd45aca7c71bfec4fbf91930cc9426932fecdebebe8db5f163d
3
+ size 194993078
pretok_ws_clean/x_csharp_000039_045.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d16b4d363810160c9200e394b3043725f54c868d5bd178bdbd8cfb634d9ffff0
3
+ size 194837814