""" CodeSymb whitespace codec: newlines + indentation without a new vocabulary ========================================================================== The CodeSymb SentencePiece model (nmt_nfkc, no byte fallback) turns every "\\n", "\\t" and run of spaces into ONE space, so code loses its lines and indentation. This codec gives 7 token ids that NEVER occurred in 5.5B tokens of the training pool a fixed whitespace meaning: NL = newline TAB = tab SP1, SP2, SP4, SP8, SP16 = 1 / 2 / 4 / 8 / 16 spaces (any width = sum) Same vocabulary, same model, same embeddings table: training continues. Text between whitespace is still encoded by SentencePiece itself. enc = WSCodec(sp) # wraps a SentencePieceProcessor ids = enc.encode(text) # list[int] txt = enc.decode(ids) # exact for code (see limits) Everything else (eos_id, get_piece_size, ...) is passed through to sp. Turn on everywhere with: export CODESYMB_WS_CODEC=1 (codesymb_common.load_tokenizer then returns a WSCodec). Limits: single trailing spaces at line end are dropped, and the tokenizer's NFKC normalisation still applies inside a line. """ import re # id -> (meaning, piece text that must sit at that id in the tokenizer) CODEC = { 46632: ("\n", "CHOPEL"), 46642: ("\t", "ETHFSx"), 46648: (" ", "GENlep"), 42206: (" ", "FakeNodeGroup"), 37796: (" ", "JobTickStatus"), 42279: (" " * 8, "Repairprogress"), 42227: (" " * 16, "getyesterdaty"), } NL, TAB, SP1, SP2, SP4, SP8, SP16 = (46632, 46642, 46648, 42206, 37796, 42279, 42227) SPACE_IDS = ((16, SP16), (8, SP8), (4, SP4), (2, SP2), (1, SP1)) WS_SPLIT = re.compile(r"([ \t]{2,}|\t)") LEAD = re.compile(r"[ \t]*") def spaces(n, out): for w, i in SPACE_IDS: while n >= w: out.append(i) n -= w def ws_ids(ws, out): run = 0 for ch in ws: if ch == " ": run += 1 else: # tab spaces(run, out) run = 0 out.append(TAB) spaces(run, out) class WSCodec: def __init__(self, sp): self.sp = sp bad = [(i, p, sp.id_to_piece(i)) for i, (_, p) in CODEC.items() if i >= sp.get_piece_size() or sp.id_to_piece(i) != p] if bad: raise RuntimeError(f"ws_codec does not match this tokenizer: {bad}") self.ids = frozenset(CODEC) self.text = {i: m for i, (m, _) in CODEC.items()} def __getattr__(self, name): # eos_id, get_piece_size, ... return getattr(self.sp, name) def encode(self, text, *args, **kwargs): if not isinstance(text, str): # batch call return [self.encode(t) for t in text] text = text.replace("\r\n", "\n").replace("\r", "\n") plan, segs = [], [] # plan: ints or ("seg", k) for li, line in enumerate(text.split("\n")): if li: plan.append(NL) lead = LEAD.match(line).group() if lead: ws_ids(lead, plan) for k, part in enumerate(WS_SPLIT.split(line[len(lead):])): if not part: continue if k % 2: # whitespace run ws_ids(part, plan) elif part.strip(): plan.append(("seg", len(segs))) segs.append(part) enc = self.sp.encode(segs) if segs else [] out = [] for x in plan: if isinstance(x, tuple): out.extend(i for i in enc[x[1]] if i not in self.ids) else: out.append(x) return out def decode(self, ids, *args, **kwargs): if ids and not isinstance(ids[0], int) and hasattr(ids[0], "__len__"): return [self.decode(x) for x in ids] out, run = [], [] for i in ids: i = int(i) if i in self.ids: if run: out.append(self.sp.decode(run)) run = [] out.append(self.text[i]) else: run.append(i) if run: out.append(self.sp.decode(run)) return "".join(out) SAMPLES = [ "def f(x):\n if x:\n return 1\n\treturn 2\n", "class A:\n\n def g(self, a, b):\n # comment here\n" " return {'k': [1, 2, 3]}\n\n\nprint(A().g(1, 2))\n", "
\n

Hello, world

\n
\n", "int main() {\n\tfor (int i = 0; i < 3; i++) {\n\t\tputs(\"hi\");\n\t}\n}\n", "server:\n port: 8080\n hosts:\n - a.example.com\n - b.example.com\n", "Plain prose line one.\nLine two, after a newline.\n\nNew paragraph.", ] def selftest(): """python3 ws_codec.py -> round-trips code samples with the REAL CodeSymb tokenizer and shows the before/after.""" import os import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import codesymb_common as C os.environ.pop("CODESYMB_WS_CODEC", None) sp = C.load_tokenizer(None) enc = WSCodec(sp) ok = 0 for s in SAMPLES: back = enc.decode(enc.encode(s)) flat = sp.decode(sp.encode(s)) want = "\n".join(re.sub(r"(?<=\S) $", "", line) for line in s.split("\n")) good = back == want ok += good print(f"{'OK ' if good else 'BAD'} {len(enc.encode(s)):4d} tok " f"(plain {len(sp.encode(s)):4d}) | old: {flat[:50]!r}") if not good: print(f" want {want!r}\n got {back!r}") print(f"\n{ok}/{len(SAMPLES)} samples round-trip exactly" + (" -> codec READY" if ok == len(SAMPLES) else " -> NOT ready")) return 0 if ok == len(SAMPLES) else 1 if __name__ == "__main__": raise SystemExit(selftest())