NiftyOracleAI's picture
pretok_ws_clean: 189.53B never-seen tokens, ws_codec formatted, deduped vs old pool
6105b30 verified
Raw History Blame Contribute Delete
5.94 kB
"""
CodeSymb whitespace codec: newlines + indentation without a new vocabulary
==========================================================================
The CodeSymb SentencePiece model (nmt_nfkc, no byte fallback) turns every
"\\n", "\\t" and run of spaces into ONE space, so code loses its lines and
indentation. This codec gives 7 token ids that NEVER occurred in 5.5B
tokens of the training pool a fixed whitespace meaning:
NL = newline TAB = tab
SP1, SP2, SP4, SP8, SP16 = 1 / 2 / 4 / 8 / 16 spaces (any width = sum)
Same vocabulary, same model, same embeddings table: training continues.
Text between whitespace is still encoded by SentencePiece itself.
enc = WSCodec(sp) # wraps a SentencePieceProcessor
ids = enc.encode(text) # list[int]
txt = enc.decode(ids) # exact for code (see limits)
Everything else (eos_id, get_piece_size, ...) is passed through to sp.
Turn on everywhere with: export CODESYMB_WS_CODEC=1
(codesymb_common.load_tokenizer then returns a WSCodec).
Limits: single trailing spaces at line end are dropped, and the
tokenizer's NFKC normalisation still applies inside a line.
"""
import re
# id -> (meaning, piece text that must sit at that id in the tokenizer)
CODEC = {
46632: ("\n", "CHOPEL"),
46642: ("\t", "ETHFSx"),
46648: (" ", "GENlep"),
42206: (" ", "FakeNodeGroup"),
37796: (" ", "JobTickStatus"),
42279: (" " * 8, "Repairprogress"),
42227: (" " * 16, "getyesterdaty"),
}
NL, TAB, SP1, SP2, SP4, SP8, SP16 = (46632, 46642, 46648, 42206, 37796,
42279, 42227)
SPACE_IDS = ((16, SP16), (8, SP8), (4, SP4), (2, SP2), (1, SP1))
WS_SPLIT = re.compile(r"([ \t]{2,}|\t)")
LEAD = re.compile(r"[ \t]*")
def spaces(n, out):
for w, i in SPACE_IDS:
while n >= w:
out.append(i)
n -= w
def ws_ids(ws, out):
run = 0
for ch in ws:
if ch == " ":
run += 1
else: # tab
spaces(run, out)
run = 0
out.append(TAB)
spaces(run, out)
class WSCodec:
def __init__(self, sp):
self.sp = sp
bad = [(i, p, sp.id_to_piece(i)) for i, (_, p) in CODEC.items()
if i >= sp.get_piece_size() or sp.id_to_piece(i) != p]
if bad:
raise RuntimeError(f"ws_codec does not match this tokenizer: {bad}")
self.ids = frozenset(CODEC)
self.text = {i: m for i, (m, _) in CODEC.items()}
def __getattr__(self, name): # eos_id, get_piece_size, ...
return getattr(self.sp, name)
def encode(self, text, *args, **kwargs):
if not isinstance(text, str): # batch call
return [self.encode(t) for t in text]
text = text.replace("\r\n", "\n").replace("\r", "\n")
plan, segs = [], [] # plan: ints or ("seg", k)
for li, line in enumerate(text.split("\n")):
if li:
plan.append(NL)
lead = LEAD.match(line).group()
if lead:
ws_ids(lead, plan)
for k, part in enumerate(WS_SPLIT.split(line[len(lead):])):
if not part:
continue
if k % 2: # whitespace run
ws_ids(part, plan)
elif part.strip():
plan.append(("seg", len(segs)))
segs.append(part)
enc = self.sp.encode(segs) if segs else []
out = []
for x in plan:
if isinstance(x, tuple):
out.extend(i for i in enc[x[1]] if i not in self.ids)
else:
out.append(x)
return out
def decode(self, ids, *args, **kwargs):
if ids and not isinstance(ids[0], int) and hasattr(ids[0], "__len__"):
return [self.decode(x) for x in ids]
out, run = [], []
for i in ids:
i = int(i)
if i in self.ids:
if run:
out.append(self.sp.decode(run))
run = []
out.append(self.text[i])
else:
run.append(i)
if run:
out.append(self.sp.decode(run))
return "".join(out)
SAMPLES = [
"def f(x):\n if x:\n return 1\n\treturn 2\n",
"class A:\n\n def g(self, a, b):\n # comment here\n"
" return {'k': [1, 2, 3]}\n\n\nprint(A().g(1, 2))\n",
"<div class=\"box\">\n <p>Hello, world</p>\n</div>\n",
"int main() {\n\tfor (int i = 0; i < 3; i++) {\n\t\tputs(\"hi\");\n\t}\n}\n",
"server:\n port: 8080\n hosts:\n - a.example.com\n - b.example.com\n",
"Plain prose line one.\nLine two, after a newline.\n\nNew paragraph.",
]
def selftest():
"""python3 ws_codec.py -> round-trips code samples with the REAL
CodeSymb tokenizer and shows the before/after."""
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import codesymb_common as C
os.environ.pop("CODESYMB_WS_CODEC", None)
sp = C.load_tokenizer(None)
enc = WSCodec(sp)
ok = 0
for s in SAMPLES:
back = enc.decode(enc.encode(s))
flat = sp.decode(sp.encode(s))
want = "\n".join(re.sub(r"(?<=\S) $", "", line)
for line in s.split("\n"))
good = back == want
ok += good
print(f"{'OK ' if good else 'BAD'} {len(enc.encode(s)):4d} tok "
f"(plain {len(sp.encode(s)):4d}) | old: {flat[:50]!r}")
if not good:
print(f" want {want!r}\n got {back!r}")
print(f"\n{ok}/{len(SAMPLES)} samples round-trip exactly"
+ (" -> codec READY" if ok == len(SAMPLES) else " -> NOT ready"))
return 0 if ok == len(SAMPLES) else 1
if __name__ == "__main__":
raise SystemExit(selftest())