File size: 5,936 Bytes
6105b30 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 | """
CodeSymb whitespace codec: newlines + indentation without a new vocabulary
==========================================================================
The CodeSymb SentencePiece model (nmt_nfkc, no byte fallback) turns every
"\\n", "\\t" and run of spaces into ONE space, so code loses its lines and
indentation. This codec gives 7 token ids that NEVER occurred in 5.5B
tokens of the training pool a fixed whitespace meaning:
NL = newline TAB = tab
SP1, SP2, SP4, SP8, SP16 = 1 / 2 / 4 / 8 / 16 spaces (any width = sum)
Same vocabulary, same model, same embeddings table: training continues.
Text between whitespace is still encoded by SentencePiece itself.
enc = WSCodec(sp) # wraps a SentencePieceProcessor
ids = enc.encode(text) # list[int]
txt = enc.decode(ids) # exact for code (see limits)
Everything else (eos_id, get_piece_size, ...) is passed through to sp.
Turn on everywhere with: export CODESYMB_WS_CODEC=1
(codesymb_common.load_tokenizer then returns a WSCodec).
Limits: single trailing spaces at line end are dropped, and the
tokenizer's NFKC normalisation still applies inside a line.
"""
import re
# id -> (meaning, piece text that must sit at that id in the tokenizer)
CODEC = {
46632: ("\n", "CHOPEL"),
46642: ("\t", "ETHFSx"),
46648: (" ", "GENlep"),
42206: (" ", "FakeNodeGroup"),
37796: (" ", "JobTickStatus"),
42279: (" " * 8, "Repairprogress"),
42227: (" " * 16, "getyesterdaty"),
}
NL, TAB, SP1, SP2, SP4, SP8, SP16 = (46632, 46642, 46648, 42206, 37796,
42279, 42227)
SPACE_IDS = ((16, SP16), (8, SP8), (4, SP4), (2, SP2), (1, SP1))
WS_SPLIT = re.compile(r"([ \t]{2,}|\t)")
LEAD = re.compile(r"[ \t]*")
def spaces(n, out):
for w, i in SPACE_IDS:
while n >= w:
out.append(i)
n -= w
def ws_ids(ws, out):
run = 0
for ch in ws:
if ch == " ":
run += 1
else: # tab
spaces(run, out)
run = 0
out.append(TAB)
spaces(run, out)
class WSCodec:
def __init__(self, sp):
self.sp = sp
bad = [(i, p, sp.id_to_piece(i)) for i, (_, p) in CODEC.items()
if i >= sp.get_piece_size() or sp.id_to_piece(i) != p]
if bad:
raise RuntimeError(f"ws_codec does not match this tokenizer: {bad}")
self.ids = frozenset(CODEC)
self.text = {i: m for i, (m, _) in CODEC.items()}
def __getattr__(self, name): # eos_id, get_piece_size, ...
return getattr(self.sp, name)
def encode(self, text, *args, **kwargs):
if not isinstance(text, str): # batch call
return [self.encode(t) for t in text]
text = text.replace("\r\n", "\n").replace("\r", "\n")
plan, segs = [], [] # plan: ints or ("seg", k)
for li, line in enumerate(text.split("\n")):
if li:
plan.append(NL)
lead = LEAD.match(line).group()
if lead:
ws_ids(lead, plan)
for k, part in enumerate(WS_SPLIT.split(line[len(lead):])):
if not part:
continue
if k % 2: # whitespace run
ws_ids(part, plan)
elif part.strip():
plan.append(("seg", len(segs)))
segs.append(part)
enc = self.sp.encode(segs) if segs else []
out = []
for x in plan:
if isinstance(x, tuple):
out.extend(i for i in enc[x[1]] if i not in self.ids)
else:
out.append(x)
return out
def decode(self, ids, *args, **kwargs):
if ids and not isinstance(ids[0], int) and hasattr(ids[0], "__len__"):
return [self.decode(x) for x in ids]
out, run = [], []
for i in ids:
i = int(i)
if i in self.ids:
if run:
out.append(self.sp.decode(run))
run = []
out.append(self.text[i])
else:
run.append(i)
if run:
out.append(self.sp.decode(run))
return "".join(out)
SAMPLES = [
"def f(x):\n if x:\n return 1\n\treturn 2\n",
"class A:\n\n def g(self, a, b):\n # comment here\n"
" return {'k': [1, 2, 3]}\n\n\nprint(A().g(1, 2))\n",
"<div class=\"box\">\n <p>Hello, world</p>\n</div>\n",
"int main() {\n\tfor (int i = 0; i < 3; i++) {\n\t\tputs(\"hi\");\n\t}\n}\n",
"server:\n port: 8080\n hosts:\n - a.example.com\n - b.example.com\n",
"Plain prose line one.\nLine two, after a newline.\n\nNew paragraph.",
]
def selftest():
"""python3 ws_codec.py -> round-trips code samples with the REAL
CodeSymb tokenizer and shows the before/after."""
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import codesymb_common as C
os.environ.pop("CODESYMB_WS_CODEC", None)
sp = C.load_tokenizer(None)
enc = WSCodec(sp)
ok = 0
for s in SAMPLES:
back = enc.decode(enc.encode(s))
flat = sp.decode(sp.encode(s))
want = "\n".join(re.sub(r"(?<=\S) $", "", line)
for line in s.split("\n"))
good = back == want
ok += good
print(f"{'OK ' if good else 'BAD'} {len(enc.encode(s)):4d} tok "
f"(plain {len(sp.encode(s)):4d}) | old: {flat[:50]!r}")
if not good:
print(f" want {want!r}\n got {back!r}")
print(f"\n{ok}/{len(SAMPLES)} samples round-trip exactly"
+ (" -> codec READY" if ok == len(SAMPLES) else " -> NOT ready"))
return 0 if ok == len(SAMPLES) else 1
if __name__ == "__main__":
raise SystemExit(selftest())
|