pretok_ws_clean: 189.53B never-seen tokens, ws_codec formatted, deduped vs old pool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- pretok_ws_clean/manifest.json +0 -0
- pretok_ws_clean/ws_codec.py +162 -0
- pretok_ws_clean/x_c_000051_000.bin +3 -0
- pretok_ws_clean/x_c_000051_001.bin +3 -0
- pretok_ws_clean/x_c_000051_002.bin +3 -0
- pretok_ws_clean/x_c_000051_003.bin +3 -0
- pretok_ws_clean/x_c_000051_004.bin +3 -0
- pretok_ws_clean/x_c_000051_005.bin +3 -0
- pretok_ws_clean/x_c_000051_006.bin +3 -0
- pretok_ws_clean/x_c_000051_007.bin +3 -0
- pretok_ws_clean/x_c_000051_008.bin +3 -0
- pretok_ws_clean/x_c_000051_009.bin +3 -0
- pretok_ws_clean/x_c_000051_010.bin +3 -0
- pretok_ws_clean/x_c_000051_011.bin +3 -0
- pretok_ws_clean/x_c_000051_012.bin +3 -0
- pretok_ws_clean/x_c_000051_013.bin +3 -0
- pretok_ws_clean/x_c_000051_014.bin +3 -0
- pretok_ws_clean/x_c_000051_015.bin +3 -0
- pretok_ws_clean/x_c_000051_016.bin +3 -0
- pretok_ws_clean/x_c_000051_017.bin +3 -0
- pretok_ws_clean/x_c_000051_018.bin +3 -0
- pretok_ws_clean/x_c_000051_019.bin +3 -0
- pretok_ws_clean/x_c_000051_020.bin +3 -0
- pretok_ws_clean/x_c_000051_021.bin +3 -0
- pretok_ws_clean/x_c_000051_022.bin +3 -0
- pretok_ws_clean/x_c_000051_023.bin +3 -0
- pretok_ws_clean/x_c_000051_024.bin +3 -0
- pretok_ws_clean/x_c_000051_025.bin +3 -0
- pretok_ws_clean/x_c_000051_026.bin +3 -0
- pretok_ws_clean/x_c_000051_027.bin +3 -0
- pretok_ws_clean/x_c_000051_028.bin +3 -0
- pretok_ws_clean/x_c_000051_029.bin +3 -0
- pretok_ws_clean/x_c_000051_030.bin +3 -0
- pretok_ws_clean/x_c_000051_031.bin +3 -0
- pretok_ws_clean/x_c_000051_032.bin +3 -0
- pretok_ws_clean/x_c_000051_033.bin +3 -0
- pretok_ws_clean/x_c_000051_034.bin +3 -0
- pretok_ws_clean/x_c_000051_035.bin +3 -0
- pretok_ws_clean/x_c_000051_036.bin +3 -0
- pretok_ws_clean/x_c_000051_037.bin +3 -0
- pretok_ws_clean/x_c_000051_038.bin +3 -0
- pretok_ws_clean/x_c_000051_039.bin +3 -0
- pretok_ws_clean/x_c_000051_040.bin +3 -0
- pretok_ws_clean/x_c_000051_041.bin +3 -0
- pretok_ws_clean/x_c_000051_042.bin +3 -0
- pretok_ws_clean/x_c_000051_043.bin +3 -0
- pretok_ws_clean/x_c_000051_044.bin +3 -0
- pretok_ws_clean/x_c_000051_045.bin +3 -0
- pretok_ws_clean/x_c_000051_046.bin +3 -0
- pretok_ws_clean/x_c_000051_047.bin +3 -0
pretok_ws_clean/manifest.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
pretok_ws_clean/ws_codec.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
CodeSymb whitespace codec: newlines + indentation without a new vocabulary
|
| 3 |
+
==========================================================================
|
| 4 |
+
The CodeSymb SentencePiece model (nmt_nfkc, no byte fallback) turns every
|
| 5 |
+
"\\n", "\\t" and run of spaces into ONE space, so code loses its lines and
|
| 6 |
+
indentation. This codec gives 7 token ids that NEVER occurred in 5.5B
|
| 7 |
+
tokens of the training pool a fixed whitespace meaning:
|
| 8 |
+
|
| 9 |
+
NL = newline TAB = tab
|
| 10 |
+
SP1, SP2, SP4, SP8, SP16 = 1 / 2 / 4 / 8 / 16 spaces (any width = sum)
|
| 11 |
+
|
| 12 |
+
Same vocabulary, same model, same embeddings table: training continues.
|
| 13 |
+
Text between whitespace is still encoded by SentencePiece itself.
|
| 14 |
+
|
| 15 |
+
enc = WSCodec(sp) # wraps a SentencePieceProcessor
|
| 16 |
+
ids = enc.encode(text) # list[int]
|
| 17 |
+
txt = enc.decode(ids) # exact for code (see limits)
|
| 18 |
+
Everything else (eos_id, get_piece_size, ...) is passed through to sp.
|
| 19 |
+
|
| 20 |
+
Turn on everywhere with: export CODESYMB_WS_CODEC=1
|
| 21 |
+
(codesymb_common.load_tokenizer then returns a WSCodec).
|
| 22 |
+
|
| 23 |
+
Limits: single trailing spaces at line end are dropped, and the
|
| 24 |
+
tokenizer's NFKC normalisation still applies inside a line.
|
| 25 |
+
"""
|
| 26 |
+
import re
|
| 27 |
+
|
| 28 |
+
# id -> (meaning, piece text that must sit at that id in the tokenizer)
|
| 29 |
+
CODEC = {
|
| 30 |
+
46632: ("\n", "CHOPEL"),
|
| 31 |
+
46642: ("\t", "ETHFSx"),
|
| 32 |
+
46648: (" ", "GENlep"),
|
| 33 |
+
42206: (" ", "FakeNodeGroup"),
|
| 34 |
+
37796: (" ", "JobTickStatus"),
|
| 35 |
+
42279: (" " * 8, "Repairprogress"),
|
| 36 |
+
42227: (" " * 16, "getyesterdaty"),
|
| 37 |
+
}
|
| 38 |
+
NL, TAB, SP1, SP2, SP4, SP8, SP16 = (46632, 46642, 46648, 42206, 37796,
|
| 39 |
+
42279, 42227)
|
| 40 |
+
SPACE_IDS = ((16, SP16), (8, SP8), (4, SP4), (2, SP2), (1, SP1))
|
| 41 |
+
WS_SPLIT = re.compile(r"([ \t]{2,}|\t)")
|
| 42 |
+
LEAD = re.compile(r"[ \t]*")
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def spaces(n, out):
|
| 46 |
+
for w, i in SPACE_IDS:
|
| 47 |
+
while n >= w:
|
| 48 |
+
out.append(i)
|
| 49 |
+
n -= w
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def ws_ids(ws, out):
|
| 53 |
+
run = 0
|
| 54 |
+
for ch in ws:
|
| 55 |
+
if ch == " ":
|
| 56 |
+
run += 1
|
| 57 |
+
else: # tab
|
| 58 |
+
spaces(run, out)
|
| 59 |
+
run = 0
|
| 60 |
+
out.append(TAB)
|
| 61 |
+
spaces(run, out)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class WSCodec:
|
| 65 |
+
def __init__(self, sp):
|
| 66 |
+
self.sp = sp
|
| 67 |
+
bad = [(i, p, sp.id_to_piece(i)) for i, (_, p) in CODEC.items()
|
| 68 |
+
if i >= sp.get_piece_size() or sp.id_to_piece(i) != p]
|
| 69 |
+
if bad:
|
| 70 |
+
raise RuntimeError(f"ws_codec does not match this tokenizer: {bad}")
|
| 71 |
+
self.ids = frozenset(CODEC)
|
| 72 |
+
self.text = {i: m for i, (m, _) in CODEC.items()}
|
| 73 |
+
|
| 74 |
+
def __getattr__(self, name): # eos_id, get_piece_size, ...
|
| 75 |
+
return getattr(self.sp, name)
|
| 76 |
+
|
| 77 |
+
def encode(self, text, *args, **kwargs):
|
| 78 |
+
if not isinstance(text, str): # batch call
|
| 79 |
+
return [self.encode(t) for t in text]
|
| 80 |
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
| 81 |
+
plan, segs = [], [] # plan: ints or ("seg", k)
|
| 82 |
+
for li, line in enumerate(text.split("\n")):
|
| 83 |
+
if li:
|
| 84 |
+
plan.append(NL)
|
| 85 |
+
lead = LEAD.match(line).group()
|
| 86 |
+
if lead:
|
| 87 |
+
ws_ids(lead, plan)
|
| 88 |
+
for k, part in enumerate(WS_SPLIT.split(line[len(lead):])):
|
| 89 |
+
if not part:
|
| 90 |
+
continue
|
| 91 |
+
if k % 2: # whitespace run
|
| 92 |
+
ws_ids(part, plan)
|
| 93 |
+
elif part.strip():
|
| 94 |
+
plan.append(("seg", len(segs)))
|
| 95 |
+
segs.append(part)
|
| 96 |
+
enc = self.sp.encode(segs) if segs else []
|
| 97 |
+
out = []
|
| 98 |
+
for x in plan:
|
| 99 |
+
if isinstance(x, tuple):
|
| 100 |
+
out.extend(i for i in enc[x[1]] if i not in self.ids)
|
| 101 |
+
else:
|
| 102 |
+
out.append(x)
|
| 103 |
+
return out
|
| 104 |
+
|
| 105 |
+
def decode(self, ids, *args, **kwargs):
|
| 106 |
+
if ids and not isinstance(ids[0], int) and hasattr(ids[0], "__len__"):
|
| 107 |
+
return [self.decode(x) for x in ids]
|
| 108 |
+
out, run = [], []
|
| 109 |
+
for i in ids:
|
| 110 |
+
i = int(i)
|
| 111 |
+
if i in self.ids:
|
| 112 |
+
if run:
|
| 113 |
+
out.append(self.sp.decode(run))
|
| 114 |
+
run = []
|
| 115 |
+
out.append(self.text[i])
|
| 116 |
+
else:
|
| 117 |
+
run.append(i)
|
| 118 |
+
if run:
|
| 119 |
+
out.append(self.sp.decode(run))
|
| 120 |
+
return "".join(out)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
SAMPLES = [
|
| 124 |
+
"def f(x):\n if x:\n return 1\n\treturn 2\n",
|
| 125 |
+
"class A:\n\n def g(self, a, b):\n # comment here\n"
|
| 126 |
+
" return {'k': [1, 2, 3]}\n\n\nprint(A().g(1, 2))\n",
|
| 127 |
+
"<div class=\"box\">\n <p>Hello, world</p>\n</div>\n",
|
| 128 |
+
"int main() {\n\tfor (int i = 0; i < 3; i++) {\n\t\tputs(\"hi\");\n\t}\n}\n",
|
| 129 |
+
"server:\n port: 8080\n hosts:\n - a.example.com\n - b.example.com\n",
|
| 130 |
+
"Plain prose line one.\nLine two, after a newline.\n\nNew paragraph.",
|
| 131 |
+
]
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def selftest():
|
| 135 |
+
"""python3 ws_codec.py -> round-trips code samples with the REAL
|
| 136 |
+
CodeSymb tokenizer and shows the before/after."""
|
| 137 |
+
import os
|
| 138 |
+
import sys
|
| 139 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 140 |
+
import codesymb_common as C
|
| 141 |
+
os.environ.pop("CODESYMB_WS_CODEC", None)
|
| 142 |
+
sp = C.load_tokenizer(None)
|
| 143 |
+
enc = WSCodec(sp)
|
| 144 |
+
ok = 0
|
| 145 |
+
for s in SAMPLES:
|
| 146 |
+
back = enc.decode(enc.encode(s))
|
| 147 |
+
flat = sp.decode(sp.encode(s))
|
| 148 |
+
want = "\n".join(re.sub(r"(?<=\S) $", "", line)
|
| 149 |
+
for line in s.split("\n"))
|
| 150 |
+
good = back == want
|
| 151 |
+
ok += good
|
| 152 |
+
print(f"{'OK ' if good else 'BAD'} {len(enc.encode(s)):4d} tok "
|
| 153 |
+
f"(plain {len(sp.encode(s)):4d}) | old: {flat[:50]!r}")
|
| 154 |
+
if not good:
|
| 155 |
+
print(f" want {want!r}\n got {back!r}")
|
| 156 |
+
print(f"\n{ok}/{len(SAMPLES)} samples round-trip exactly"
|
| 157 |
+
+ (" -> codec READY" if ok == len(SAMPLES) else " -> NOT ready"))
|
| 158 |
+
return 0 if ok == len(SAMPLES) else 1
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
if __name__ == "__main__":
|
| 162 |
+
raise SystemExit(selftest())
|
pretok_ws_clean/x_c_000051_000.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a20891cf52959eeaf4ab0fcd46f62990737595511c90a924e32f7a2a1c49175e
|
| 3 |
+
size 198840200
|
pretok_ws_clean/x_c_000051_001.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d8d0b19d0f48ac9e395efb9f200c8ada5712e1cd2c0e66741b1e12ccf28e26a9
|
| 3 |
+
size 198853622
|
pretok_ws_clean/x_c_000051_002.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:91c157be8d320ae44f0213b42e0fee22f4781662b22748ba0864ae559f65c549
|
| 3 |
+
size 199771576
|
pretok_ws_clean/x_c_000051_003.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7502ca431ef7b60893f05fbff01a4aecbfafee55e774ed6ef17d024d068b435c
|
| 3 |
+
size 199540030
|
pretok_ws_clean/x_c_000051_004.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:464d89695dda1b9632c7782f1d775ada118634e0d3957f473e0b9d1636bdcc8e
|
| 3 |
+
size 198968910
|
pretok_ws_clean/x_c_000051_005.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5a8d4f07a16843a5cf5a4f2018277bf8953ea5416ac5b48993da49be627c6dec
|
| 3 |
+
size 199448460
|
pretok_ws_clean/x_c_000051_006.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:934be180711220babca03256214ed7d655665605a36f8a4e6f03a57978107e42
|
| 3 |
+
size 199414764
|
pretok_ws_clean/x_c_000051_007.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2362f457b814874bbe6d77b5bd63aae21f1a244a3a5799cb5d9f10cfbcaae43c
|
| 3 |
+
size 199984874
|
pretok_ws_clean/x_c_000051_008.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:665068993101f4cb7d5026f49dae59e2b826df253572f5006792c0363029418d
|
| 3 |
+
size 199070220
|
pretok_ws_clean/x_c_000051_009.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bdccef9d5ca8a30c2ac22e7d921a98747b8a1dd8700e54d7194e475b108f4fbb
|
| 3 |
+
size 198802702
|
pretok_ws_clean/x_c_000051_010.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:16590b34ea25103e43841978dd458021db381f5188bae152730a3f6d1039fc35
|
| 3 |
+
size 199943394
|
pretok_ws_clean/x_c_000051_011.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f5bbda60513ed9289d568be56b9955773d4536df92167a57e2c84aa539c5a6ba
|
| 3 |
+
size 199010218
|
pretok_ws_clean/x_c_000051_012.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4dfa4eb8194c990039c5fef77280474aec61f5f8d7199965d2157e78661f1322
|
| 3 |
+
size 198982392
|
pretok_ws_clean/x_c_000051_013.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:607ca91444404a8f2511ebe533c07a9e966302f9fe97a975acc6484b7284bc2e
|
| 3 |
+
size 198259554
|
pretok_ws_clean/x_c_000051_014.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:20ac38d1e02dcfea0c34b3a45bd150394bfb838b015668fa788ac99c14f6c9f7
|
| 3 |
+
size 199090452
|
pretok_ws_clean/x_c_000051_015.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f6999020738d8bd780e60423684a8b538e14e19b89fb6ccfb81a5cd1b0274238
|
| 3 |
+
size 199718022
|
pretok_ws_clean/x_c_000051_016.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a80b93e2670fa54c2aea996afc52828be835a3f62f3be1acfb810557f4fa310f
|
| 3 |
+
size 199482902
|
pretok_ws_clean/x_c_000051_017.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:580d5d1a1c26fb6d73c0f28667211a9a0f5c4a9774b57de925c60f5c557698df
|
| 3 |
+
size 199006474
|
pretok_ws_clean/x_c_000051_018.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4586e43b1cf7a4b5596519f9c1e521001de95cf557bbea65232742a8685e1b38
|
| 3 |
+
size 198464188
|
pretok_ws_clean/x_c_000051_019.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1d5d7a5d71a23d43772e29c708084788e76553b7110262b2a856f2b8dc6e39cd
|
| 3 |
+
size 199726714
|
pretok_ws_clean/x_c_000051_020.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4eac6f525e794a4c9a2a6902ad1c044a3946d0eb0760f96101818751e2c87799
|
| 3 |
+
size 198771410
|
pretok_ws_clean/x_c_000051_021.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2e83117c8d6e3573b84b14a3ed66d97e505d4e96ec3767afafec9724fb5352ef
|
| 3 |
+
size 199002626
|
pretok_ws_clean/x_c_000051_022.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:93164dffb014bd88846e752ec898c368b3c57d641e9d65efd59bfb8313128250
|
| 3 |
+
size 199155780
|
pretok_ws_clean/x_c_000051_023.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c54284c9b5f5248995dcb7a86a691e05b41a6790cd909b2a296b4e8dc71e2631
|
| 3 |
+
size 198560786
|
pretok_ws_clean/x_c_000051_024.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7ce8b19bfe57b21b31266fe97dec5ba51c95ac37d11647be199bb4261b00a27c
|
| 3 |
+
size 199257644
|
pretok_ws_clean/x_c_000051_025.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2eb3d674517e3d10a7f273c222040c28fcccb7735fa86df561818268e2195065
|
| 3 |
+
size 200027134
|
pretok_ws_clean/x_c_000051_026.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0752e112734456adc1a69004956bb499c3c2155087150c05543660ba3d8414f8
|
| 3 |
+
size 199123406
|
pretok_ws_clean/x_c_000051_027.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:53d66c932916cbf230fc87644a98e0443011099c74db4d8f4a0b3509ca01f11d
|
| 3 |
+
size 199365572
|
pretok_ws_clean/x_c_000051_028.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:54ffaf72ce36ff9c1a4debe540545409427b1222b5a919c7d56bf991b2d55e47
|
| 3 |
+
size 199891130
|
pretok_ws_clean/x_c_000051_029.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4075d9a9b2af7722291cbb8080d0ab541cd8435b2343262d71fe9218a08a1ba3
|
| 3 |
+
size 199633782
|
pretok_ws_clean/x_c_000051_030.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:95acf940d261670b03488b2bdd32d452c609c2a9c74b6fd7faa9eb7733d58d72
|
| 3 |
+
size 198802996
|
pretok_ws_clean/x_c_000051_031.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4c46035b422b8bad294ad9d213ec0585717c2165fa58cef64720b82b3a611f1a
|
| 3 |
+
size 199645748
|
pretok_ws_clean/x_c_000051_032.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:73ac1bca936865b77c40e94be22b06cb202764a780ef9bbf50e879199d887a80
|
| 3 |
+
size 199749644
|
pretok_ws_clean/x_c_000051_033.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a19d72614f8866e421592b9992849ea884c2455181f6d6c116272d1ae02ab769
|
| 3 |
+
size 199015936
|
pretok_ws_clean/x_c_000051_034.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d4186f07fee08292df9483b8af9e04ec0e837922ec104a1fcb1eb882f0657974
|
| 3 |
+
size 198714200
|
pretok_ws_clean/x_c_000051_035.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:045e5d9a9c3246872f8abfbd7de39485748d51fa72595e88c8aa3f700b1a15ce
|
| 3 |
+
size 198951232
|
pretok_ws_clean/x_c_000051_036.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:daf9ec851d53f4588668803a43300e6830c7b041bd63a356c196e17414d60b91
|
| 3 |
+
size 199506914
|
pretok_ws_clean/x_c_000051_037.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:225d9ee56bcf851ba3442f6f459fe3c40874082c0badc4c6218940c8c241a7bd
|
| 3 |
+
size 199364002
|
pretok_ws_clean/x_c_000051_038.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:53953f871ac4ceb04c34b8139866b22fa9c7496e2ca84a80d0b17d0e2665041f
|
| 3 |
+
size 199132364
|
pretok_ws_clean/x_c_000051_039.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:46f7600c7ff37e03c179b859600806341c47e323ab28292782eb045342064079
|
| 3 |
+
size 198648968
|
pretok_ws_clean/x_c_000051_040.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:191e01c20b396abd4f6dde97a5e42e355657fc1755674db2d7dbc644b89510b6
|
| 3 |
+
size 199191878
|
pretok_ws_clean/x_c_000051_041.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:47ddd1bef8a6171f943c32fca68d931a2ba32f048610fe41b91c401ea13abcba
|
| 3 |
+
size 199349356
|
pretok_ws_clean/x_c_000051_042.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:21721070215f510f48fc188d02332bc7bc317267dd190d3b24665d585f5de22d
|
| 3 |
+
size 199554546
|
pretok_ws_clean/x_c_000051_043.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9e2cd086b8976d32fff6fe30051f10d3a487c8d32a0c865c91492245bfaa8d01
|
| 3 |
+
size 198990740
|
pretok_ws_clean/x_c_000051_044.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ff5f33b0bbdc277e57c7477b825dd2332da6a7358d3c2a9e7855041ef0f44da0
|
| 3 |
+
size 199603106
|
pretok_ws_clean/x_c_000051_045.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1c18974f82dbf70bad3898cff97d35174cba61860f3d8ea067d57eacd07c988e
|
| 3 |
+
size 198922184
|
pretok_ws_clean/x_c_000051_046.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:97eacaada5bdcb38ab79b139a1c87cc7574ae26310e6002656a6574cbd11388a
|
| 3 |
+
size 199914038
|
pretok_ws_clean/x_c_000051_047.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9da51a5bddec79ab8ed21b0d981238b925ef8e3c4fb7839797eb9c7da78e3fd8
|
| 3 |
+
size 198576690
|