""" CodeSymb whitespace codec: newlines + indentation without a new vocabulary ========================================================================== The CodeSymb SentencePiece model (nmt_nfkc, no byte fallback) turns every "\\n", "\\t" and run of spaces into ONE space, so code loses its lines and indentation. This codec gives 7 token ids that NEVER occurred in 5.5B tokens of the training pool a fixed whitespace meaning: NL = newline TAB = tab SP1, SP2, SP4, SP8, SP16 = 1 / 2 / 4 / 8 / 16 spaces (any width = sum) Same vocabulary, same model, same embeddings table: training continues. Text between whitespace is still encoded by SentencePiece itself. enc = WSCodec(sp) # wraps a SentencePieceProcessor ids = enc.encode(text) # list[int] txt = enc.decode(ids) # exact for code (see limits) Everything else (eos_id, get_piece_size, ...) is passed through to sp. Turn on everywhere with: export CODESYMB_WS_CODEC=1 (codesymb_common.load_tokenizer then returns a WSCodec). Limits: single trailing spaces at line end are dropped, and the tokenizer's NFKC normalisation still applies inside a line. """ import re # id -> (meaning, piece text that must sit at that id in the tokenizer) CODEC = { 46632: ("\n", "CHOPEL"), 46642: ("\t", "ETHFSx"), 46648: (" ", "GENlep"), 42206: (" ", "FakeNodeGroup"), 37796: (" ", "JobTickStatus"), 42279: (" " * 8, "Repairprogress"), 42227: (" " * 16, "getyesterdaty"), } NL, TAB, SP1, SP2, SP4, SP8, SP16 = (46632, 46642, 46648, 42206, 37796, 42279, 42227) SPACE_IDS = ((16, SP16), (8, SP8), (4, SP4), (2, SP2), (1, SP1)) WS_SPLIT = re.compile(r"([ \t]{2,}|\t)") LEAD = re.compile(r"[ \t]*") def spaces(n, out): for w, i in SPACE_IDS: while n >= w: out.append(i) n -= w def ws_ids(ws, out): run = 0 for ch in ws: if ch == " ": run += 1 else: # tab spaces(run, out) run = 0 out.append(TAB) spaces(run, out) class WSCodec: def __init__(self, sp): self.sp = sp bad = [(i, p, sp.id_to_piece(i)) for i, (_, p) in CODEC.items() if i >= sp.get_piece_size() or sp.id_to_piece(i) != p] if bad: raise RuntimeError(f"ws_codec does not match this tokenizer: {bad}") self.ids = frozenset(CODEC) self.text = {i: m for i, (m, _) in CODEC.items()} def __getattr__(self, name): # eos_id, get_piece_size, ... return getattr(self.sp, name) def encode(self, text, *args, **kwargs): if not isinstance(text, str): # batch call return [self.encode(t) for t in text] text = text.replace("\r\n", "\n").replace("\r", "\n") plan, segs = [], [] # plan: ints or ("seg", k) for li, line in enumerate(text.split("\n")): if li: plan.append(NL) lead = LEAD.match(line).group() if lead: ws_ids(lead, plan) for k, part in enumerate(WS_SPLIT.split(line[len(lead):])): if not part: continue if k % 2: # whitespace run ws_ids(part, plan) elif part.strip(): plan.append(("seg", len(segs))) segs.append(part) enc = self.sp.encode(segs) if segs else [] out = [] for x in plan: if isinstance(x, tuple): out.extend(i for i in enc[x[1]] if i not in self.ids) else: out.append(x) return out def decode(self, ids, *args, **kwargs): if ids and not isinstance(ids[0], int) and hasattr(ids[0], "__len__"): return [self.decode(x) for x in ids] out, run = [], [] for i in ids: i = int(i) if i in self.ids: if run: out.append(self.sp.decode(run)) run = [] out.append(self.text[i]) else: run.append(i) if run: out.append(self.sp.decode(run)) return "".join(out) SAMPLES = [ "def f(x):\n if x:\n return 1\n\treturn 2\n", "class A:\n\n def g(self, a, b):\n # comment here\n" " return {'k': [1, 2, 3]}\n\n\nprint(A().g(1, 2))\n", "
Hello, world
\n