NiftyOracleAI commited on
Commit
6105b30
·
verified ·
1 Parent(s): 3fafd6d

pretok_ws_clean: 189.53B never-seen tokens, ws_codec formatted, deduped vs old pool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. pretok_ws_clean/manifest.json +0 -0
  2. pretok_ws_clean/ws_codec.py +162 -0
  3. pretok_ws_clean/x_c_000051_000.bin +3 -0
  4. pretok_ws_clean/x_c_000051_001.bin +3 -0
  5. pretok_ws_clean/x_c_000051_002.bin +3 -0
  6. pretok_ws_clean/x_c_000051_003.bin +3 -0
  7. pretok_ws_clean/x_c_000051_004.bin +3 -0
  8. pretok_ws_clean/x_c_000051_005.bin +3 -0
  9. pretok_ws_clean/x_c_000051_006.bin +3 -0
  10. pretok_ws_clean/x_c_000051_007.bin +3 -0
  11. pretok_ws_clean/x_c_000051_008.bin +3 -0
  12. pretok_ws_clean/x_c_000051_009.bin +3 -0
  13. pretok_ws_clean/x_c_000051_010.bin +3 -0
  14. pretok_ws_clean/x_c_000051_011.bin +3 -0
  15. pretok_ws_clean/x_c_000051_012.bin +3 -0
  16. pretok_ws_clean/x_c_000051_013.bin +3 -0
  17. pretok_ws_clean/x_c_000051_014.bin +3 -0
  18. pretok_ws_clean/x_c_000051_015.bin +3 -0
  19. pretok_ws_clean/x_c_000051_016.bin +3 -0
  20. pretok_ws_clean/x_c_000051_017.bin +3 -0
  21. pretok_ws_clean/x_c_000051_018.bin +3 -0
  22. pretok_ws_clean/x_c_000051_019.bin +3 -0
  23. pretok_ws_clean/x_c_000051_020.bin +3 -0
  24. pretok_ws_clean/x_c_000051_021.bin +3 -0
  25. pretok_ws_clean/x_c_000051_022.bin +3 -0
  26. pretok_ws_clean/x_c_000051_023.bin +3 -0
  27. pretok_ws_clean/x_c_000051_024.bin +3 -0
  28. pretok_ws_clean/x_c_000051_025.bin +3 -0
  29. pretok_ws_clean/x_c_000051_026.bin +3 -0
  30. pretok_ws_clean/x_c_000051_027.bin +3 -0
  31. pretok_ws_clean/x_c_000051_028.bin +3 -0
  32. pretok_ws_clean/x_c_000051_029.bin +3 -0
  33. pretok_ws_clean/x_c_000051_030.bin +3 -0
  34. pretok_ws_clean/x_c_000051_031.bin +3 -0
  35. pretok_ws_clean/x_c_000051_032.bin +3 -0
  36. pretok_ws_clean/x_c_000051_033.bin +3 -0
  37. pretok_ws_clean/x_c_000051_034.bin +3 -0
  38. pretok_ws_clean/x_c_000051_035.bin +3 -0
  39. pretok_ws_clean/x_c_000051_036.bin +3 -0
  40. pretok_ws_clean/x_c_000051_037.bin +3 -0
  41. pretok_ws_clean/x_c_000051_038.bin +3 -0
  42. pretok_ws_clean/x_c_000051_039.bin +3 -0
  43. pretok_ws_clean/x_c_000051_040.bin +3 -0
  44. pretok_ws_clean/x_c_000051_041.bin +3 -0
  45. pretok_ws_clean/x_c_000051_042.bin +3 -0
  46. pretok_ws_clean/x_c_000051_043.bin +3 -0
  47. pretok_ws_clean/x_c_000051_044.bin +3 -0
  48. pretok_ws_clean/x_c_000051_045.bin +3 -0
  49. pretok_ws_clean/x_c_000051_046.bin +3 -0
  50. pretok_ws_clean/x_c_000051_047.bin +3 -0
pretok_ws_clean/manifest.json ADDED
The diff for this file is too large to render. See raw diff
 
pretok_ws_clean/ws_codec.py ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ CodeSymb whitespace codec: newlines + indentation without a new vocabulary
3
+ ==========================================================================
4
+ The CodeSymb SentencePiece model (nmt_nfkc, no byte fallback) turns every
5
+ "\\n", "\\t" and run of spaces into ONE space, so code loses its lines and
6
+ indentation. This codec gives 7 token ids that NEVER occurred in 5.5B
7
+ tokens of the training pool a fixed whitespace meaning:
8
+
9
+ NL = newline TAB = tab
10
+ SP1, SP2, SP4, SP8, SP16 = 1 / 2 / 4 / 8 / 16 spaces (any width = sum)
11
+
12
+ Same vocabulary, same model, same embeddings table: training continues.
13
+ Text between whitespace is still encoded by SentencePiece itself.
14
+
15
+ enc = WSCodec(sp) # wraps a SentencePieceProcessor
16
+ ids = enc.encode(text) # list[int]
17
+ txt = enc.decode(ids) # exact for code (see limits)
18
+ Everything else (eos_id, get_piece_size, ...) is passed through to sp.
19
+
20
+ Turn on everywhere with: export CODESYMB_WS_CODEC=1
21
+ (codesymb_common.load_tokenizer then returns a WSCodec).
22
+
23
+ Limits: single trailing spaces at line end are dropped, and the
24
+ tokenizer's NFKC normalisation still applies inside a line.
25
+ """
26
+ import re
27
+
28
+ # id -> (meaning, piece text that must sit at that id in the tokenizer)
29
+ CODEC = {
30
+ 46632: ("\n", "CHOPEL"),
31
+ 46642: ("\t", "ETHFSx"),
32
+ 46648: (" ", "GENlep"),
33
+ 42206: (" ", "FakeNodeGroup"),
34
+ 37796: (" ", "JobTickStatus"),
35
+ 42279: (" " * 8, "Repairprogress"),
36
+ 42227: (" " * 16, "getyesterdaty"),
37
+ }
38
+ NL, TAB, SP1, SP2, SP4, SP8, SP16 = (46632, 46642, 46648, 42206, 37796,
39
+ 42279, 42227)
40
+ SPACE_IDS = ((16, SP16), (8, SP8), (4, SP4), (2, SP2), (1, SP1))
41
+ WS_SPLIT = re.compile(r"([ \t]{2,}|\t)")
42
+ LEAD = re.compile(r"[ \t]*")
43
+
44
+
45
+ def spaces(n, out):
46
+ for w, i in SPACE_IDS:
47
+ while n >= w:
48
+ out.append(i)
49
+ n -= w
50
+
51
+
52
+ def ws_ids(ws, out):
53
+ run = 0
54
+ for ch in ws:
55
+ if ch == " ":
56
+ run += 1
57
+ else: # tab
58
+ spaces(run, out)
59
+ run = 0
60
+ out.append(TAB)
61
+ spaces(run, out)
62
+
63
+
64
+ class WSCodec:
65
+ def __init__(self, sp):
66
+ self.sp = sp
67
+ bad = [(i, p, sp.id_to_piece(i)) for i, (_, p) in CODEC.items()
68
+ if i >= sp.get_piece_size() or sp.id_to_piece(i) != p]
69
+ if bad:
70
+ raise RuntimeError(f"ws_codec does not match this tokenizer: {bad}")
71
+ self.ids = frozenset(CODEC)
72
+ self.text = {i: m for i, (m, _) in CODEC.items()}
73
+
74
+ def __getattr__(self, name): # eos_id, get_piece_size, ...
75
+ return getattr(self.sp, name)
76
+
77
+ def encode(self, text, *args, **kwargs):
78
+ if not isinstance(text, str): # batch call
79
+ return [self.encode(t) for t in text]
80
+ text = text.replace("\r\n", "\n").replace("\r", "\n")
81
+ plan, segs = [], [] # plan: ints or ("seg", k)
82
+ for li, line in enumerate(text.split("\n")):
83
+ if li:
84
+ plan.append(NL)
85
+ lead = LEAD.match(line).group()
86
+ if lead:
87
+ ws_ids(lead, plan)
88
+ for k, part in enumerate(WS_SPLIT.split(line[len(lead):])):
89
+ if not part:
90
+ continue
91
+ if k % 2: # whitespace run
92
+ ws_ids(part, plan)
93
+ elif part.strip():
94
+ plan.append(("seg", len(segs)))
95
+ segs.append(part)
96
+ enc = self.sp.encode(segs) if segs else []
97
+ out = []
98
+ for x in plan:
99
+ if isinstance(x, tuple):
100
+ out.extend(i for i in enc[x[1]] if i not in self.ids)
101
+ else:
102
+ out.append(x)
103
+ return out
104
+
105
+ def decode(self, ids, *args, **kwargs):
106
+ if ids and not isinstance(ids[0], int) and hasattr(ids[0], "__len__"):
107
+ return [self.decode(x) for x in ids]
108
+ out, run = [], []
109
+ for i in ids:
110
+ i = int(i)
111
+ if i in self.ids:
112
+ if run:
113
+ out.append(self.sp.decode(run))
114
+ run = []
115
+ out.append(self.text[i])
116
+ else:
117
+ run.append(i)
118
+ if run:
119
+ out.append(self.sp.decode(run))
120
+ return "".join(out)
121
+
122
+
123
+ SAMPLES = [
124
+ "def f(x):\n if x:\n return 1\n\treturn 2\n",
125
+ "class A:\n\n def g(self, a, b):\n # comment here\n"
126
+ " return {'k': [1, 2, 3]}\n\n\nprint(A().g(1, 2))\n",
127
+ "<div class=\"box\">\n <p>Hello, world</p>\n</div>\n",
128
+ "int main() {\n\tfor (int i = 0; i < 3; i++) {\n\t\tputs(\"hi\");\n\t}\n}\n",
129
+ "server:\n port: 8080\n hosts:\n - a.example.com\n - b.example.com\n",
130
+ "Plain prose line one.\nLine two, after a newline.\n\nNew paragraph.",
131
+ ]
132
+
133
+
134
+ def selftest():
135
+ """python3 ws_codec.py -> round-trips code samples with the REAL
136
+ CodeSymb tokenizer and shows the before/after."""
137
+ import os
138
+ import sys
139
+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
140
+ import codesymb_common as C
141
+ os.environ.pop("CODESYMB_WS_CODEC", None)
142
+ sp = C.load_tokenizer(None)
143
+ enc = WSCodec(sp)
144
+ ok = 0
145
+ for s in SAMPLES:
146
+ back = enc.decode(enc.encode(s))
147
+ flat = sp.decode(sp.encode(s))
148
+ want = "\n".join(re.sub(r"(?<=\S) $", "", line)
149
+ for line in s.split("\n"))
150
+ good = back == want
151
+ ok += good
152
+ print(f"{'OK ' if good else 'BAD'} {len(enc.encode(s)):4d} tok "
153
+ f"(plain {len(sp.encode(s)):4d}) | old: {flat[:50]!r}")
154
+ if not good:
155
+ print(f" want {want!r}\n got {back!r}")
156
+ print(f"\n{ok}/{len(SAMPLES)} samples round-trip exactly"
157
+ + (" -> codec READY" if ok == len(SAMPLES) else " -> NOT ready"))
158
+ return 0 if ok == len(SAMPLES) else 1
159
+
160
+
161
+ if __name__ == "__main__":
162
+ raise SystemExit(selftest())
pretok_ws_clean/x_c_000051_000.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a20891cf52959eeaf4ab0fcd46f62990737595511c90a924e32f7a2a1c49175e
3
+ size 198840200
pretok_ws_clean/x_c_000051_001.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8d0b19d0f48ac9e395efb9f200c8ada5712e1cd2c0e66741b1e12ccf28e26a9
3
+ size 198853622
pretok_ws_clean/x_c_000051_002.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:91c157be8d320ae44f0213b42e0fee22f4781662b22748ba0864ae559f65c549
3
+ size 199771576
pretok_ws_clean/x_c_000051_003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7502ca431ef7b60893f05fbff01a4aecbfafee55e774ed6ef17d024d068b435c
3
+ size 199540030
pretok_ws_clean/x_c_000051_004.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:464d89695dda1b9632c7782f1d775ada118634e0d3957f473e0b9d1636bdcc8e
3
+ size 198968910
pretok_ws_clean/x_c_000051_005.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a8d4f07a16843a5cf5a4f2018277bf8953ea5416ac5b48993da49be627c6dec
3
+ size 199448460
pretok_ws_clean/x_c_000051_006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:934be180711220babca03256214ed7d655665605a36f8a4e6f03a57978107e42
3
+ size 199414764
pretok_ws_clean/x_c_000051_007.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2362f457b814874bbe6d77b5bd63aae21f1a244a3a5799cb5d9f10cfbcaae43c
3
+ size 199984874
pretok_ws_clean/x_c_000051_008.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:665068993101f4cb7d5026f49dae59e2b826df253572f5006792c0363029418d
3
+ size 199070220
pretok_ws_clean/x_c_000051_009.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bdccef9d5ca8a30c2ac22e7d921a98747b8a1dd8700e54d7194e475b108f4fbb
3
+ size 198802702
pretok_ws_clean/x_c_000051_010.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:16590b34ea25103e43841978dd458021db381f5188bae152730a3f6d1039fc35
3
+ size 199943394
pretok_ws_clean/x_c_000051_011.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5bbda60513ed9289d568be56b9955773d4536df92167a57e2c84aa539c5a6ba
3
+ size 199010218
pretok_ws_clean/x_c_000051_012.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4dfa4eb8194c990039c5fef77280474aec61f5f8d7199965d2157e78661f1322
3
+ size 198982392
pretok_ws_clean/x_c_000051_013.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:607ca91444404a8f2511ebe533c07a9e966302f9fe97a975acc6484b7284bc2e
3
+ size 198259554
pretok_ws_clean/x_c_000051_014.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:20ac38d1e02dcfea0c34b3a45bd150394bfb838b015668fa788ac99c14f6c9f7
3
+ size 199090452
pretok_ws_clean/x_c_000051_015.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6999020738d8bd780e60423684a8b538e14e19b89fb6ccfb81a5cd1b0274238
3
+ size 199718022
pretok_ws_clean/x_c_000051_016.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a80b93e2670fa54c2aea996afc52828be835a3f62f3be1acfb810557f4fa310f
3
+ size 199482902
pretok_ws_clean/x_c_000051_017.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:580d5d1a1c26fb6d73c0f28667211a9a0f5c4a9774b57de925c60f5c557698df
3
+ size 199006474
pretok_ws_clean/x_c_000051_018.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4586e43b1cf7a4b5596519f9c1e521001de95cf557bbea65232742a8685e1b38
3
+ size 198464188
pretok_ws_clean/x_c_000051_019.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d5d7a5d71a23d43772e29c708084788e76553b7110262b2a856f2b8dc6e39cd
3
+ size 199726714
pretok_ws_clean/x_c_000051_020.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4eac6f525e794a4c9a2a6902ad1c044a3946d0eb0760f96101818751e2c87799
3
+ size 198771410
pretok_ws_clean/x_c_000051_021.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e83117c8d6e3573b84b14a3ed66d97e505d4e96ec3767afafec9724fb5352ef
3
+ size 199002626
pretok_ws_clean/x_c_000051_022.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93164dffb014bd88846e752ec898c368b3c57d641e9d65efd59bfb8313128250
3
+ size 199155780
pretok_ws_clean/x_c_000051_023.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c54284c9b5f5248995dcb7a86a691e05b41a6790cd909b2a296b4e8dc71e2631
3
+ size 198560786
pretok_ws_clean/x_c_000051_024.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ce8b19bfe57b21b31266fe97dec5ba51c95ac37d11647be199bb4261b00a27c
3
+ size 199257644
pretok_ws_clean/x_c_000051_025.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2eb3d674517e3d10a7f273c222040c28fcccb7735fa86df561818268e2195065
3
+ size 200027134
pretok_ws_clean/x_c_000051_026.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0752e112734456adc1a69004956bb499c3c2155087150c05543660ba3d8414f8
3
+ size 199123406
pretok_ws_clean/x_c_000051_027.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53d66c932916cbf230fc87644a98e0443011099c74db4d8f4a0b3509ca01f11d
3
+ size 199365572
pretok_ws_clean/x_c_000051_028.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:54ffaf72ce36ff9c1a4debe540545409427b1222b5a919c7d56bf991b2d55e47
3
+ size 199891130
pretok_ws_clean/x_c_000051_029.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4075d9a9b2af7722291cbb8080d0ab541cd8435b2343262d71fe9218a08a1ba3
3
+ size 199633782
pretok_ws_clean/x_c_000051_030.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:95acf940d261670b03488b2bdd32d452c609c2a9c74b6fd7faa9eb7733d58d72
3
+ size 198802996
pretok_ws_clean/x_c_000051_031.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c46035b422b8bad294ad9d213ec0585717c2165fa58cef64720b82b3a611f1a
3
+ size 199645748
pretok_ws_clean/x_c_000051_032.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73ac1bca936865b77c40e94be22b06cb202764a780ef9bbf50e879199d887a80
3
+ size 199749644
pretok_ws_clean/x_c_000051_033.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a19d72614f8866e421592b9992849ea884c2455181f6d6c116272d1ae02ab769
3
+ size 199015936
pretok_ws_clean/x_c_000051_034.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4186f07fee08292df9483b8af9e04ec0e837922ec104a1fcb1eb882f0657974
3
+ size 198714200
pretok_ws_clean/x_c_000051_035.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:045e5d9a9c3246872f8abfbd7de39485748d51fa72595e88c8aa3f700b1a15ce
3
+ size 198951232
pretok_ws_clean/x_c_000051_036.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:daf9ec851d53f4588668803a43300e6830c7b041bd63a356c196e17414d60b91
3
+ size 199506914
pretok_ws_clean/x_c_000051_037.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:225d9ee56bcf851ba3442f6f459fe3c40874082c0badc4c6218940c8c241a7bd
3
+ size 199364002
pretok_ws_clean/x_c_000051_038.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53953f871ac4ceb04c34b8139866b22fa9c7496e2ca84a80d0b17d0e2665041f
3
+ size 199132364
pretok_ws_clean/x_c_000051_039.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46f7600c7ff37e03c179b859600806341c47e323ab28292782eb045342064079
3
+ size 198648968
pretok_ws_clean/x_c_000051_040.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:191e01c20b396abd4f6dde97a5e42e355657fc1755674db2d7dbc644b89510b6
3
+ size 199191878
pretok_ws_clean/x_c_000051_041.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:47ddd1bef8a6171f943c32fca68d931a2ba32f048610fe41b91c401ea13abcba
3
+ size 199349356
pretok_ws_clean/x_c_000051_042.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21721070215f510f48fc188d02332bc7bc317267dd190d3b24665d585f5de22d
3
+ size 199554546
pretok_ws_clean/x_c_000051_043.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e2cd086b8976d32fff6fe30051f10d3a487c8d32a0c865c91492245bfaa8d01
3
+ size 198990740
pretok_ws_clean/x_c_000051_044.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff5f33b0bbdc277e57c7477b825dd2332da6a7358d3c2a9e7855041ef0f44da0
3
+ size 199603106
pretok_ws_clean/x_c_000051_045.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c18974f82dbf70bad3898cff97d35174cba61860f3d8ea067d57eacd07c988e
3
+ size 198922184
pretok_ws_clean/x_c_000051_046.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97eacaada5bdcb38ab79b139a1c87cc7574ae26310e6002656a6574cbd11388a
3
+ size 199914038
pretok_ws_clean/x_c_000051_047.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9da51a5bddec79ab8ed21b0d981238b925ef8e3c4fb7839797eb9c7da78e3fd8
3
+ size 198576690