pretok_ws_clean: 189.53B never-seen tokens, ws_codec formatted, deduped vs old pool
6105b30 verified Download pretok_ws_clean/ws_codec.py from NiftyOracleAI/CodeSymb-1B-MI325X: direct link, hf CLI and curl.
- Browser
- Download file 5.94 kB
-
https://huggingface.co/NiftyOracleAI/CodeSymb-1B-MI325X/resolve/main/pretok_ws_clean/ws_codec.py
- Command line
-
hf download hf://NiftyOracleAI/CodeSymb-1B-MI325X/pretok_ws_clean/ws_codec.py
-
curl -L -o ws_codec.py https://huggingface.co/NiftyOracleAI/CodeSymb-1B-MI325X/resolve/main/pretok_ws_clean/ws_codec.py
5.94 kB
| """ | |
| CodeSymb whitespace codec: newlines + indentation without a new vocabulary | |
| ========================================================================== | |
| The CodeSymb SentencePiece model (nmt_nfkc, no byte fallback) turns every | |
| "\\n", "\\t" and run of spaces into ONE space, so code loses its lines and | |
| indentation. This codec gives 7 token ids that NEVER occurred in 5.5B | |
| tokens of the training pool a fixed whitespace meaning: | |
| NL = newline TAB = tab | |
| SP1, SP2, SP4, SP8, SP16 = 1 / 2 / 4 / 8 / 16 spaces (any width = sum) | |
| Same vocabulary, same model, same embeddings table: training continues. | |
| Text between whitespace is still encoded by SentencePiece itself. | |
| enc = WSCodec(sp) # wraps a SentencePieceProcessor | |
| ids = enc.encode(text) # list[int] | |
| txt = enc.decode(ids) # exact for code (see limits) | |
| Everything else (eos_id, get_piece_size, ...) is passed through to sp. | |
| Turn on everywhere with: export CODESYMB_WS_CODEC=1 | |
| (codesymb_common.load_tokenizer then returns a WSCodec). | |
| Limits: single trailing spaces at line end are dropped, and the | |
| tokenizer's NFKC normalisation still applies inside a line. | |
| """ | |
| import re | |
| # id -> (meaning, piece text that must sit at that id in the tokenizer) | |
| CODEC = { | |
| 46632: ("\n", "CHOPEL"), | |
| 46642: ("\t", "ETHFSx"), | |
| 46648: (" ", "GENlep"), | |
| 42206: (" ", "FakeNodeGroup"), | |
| 37796: (" ", "JobTickStatus"), | |
| 42279: (" " * 8, "Repairprogress"), | |
| 42227: (" " * 16, "getyesterdaty"), | |
| } | |
| NL, TAB, SP1, SP2, SP4, SP8, SP16 = (46632, 46642, 46648, 42206, 37796, | |
| 42279, 42227) | |
| SPACE_IDS = ((16, SP16), (8, SP8), (4, SP4), (2, SP2), (1, SP1)) | |
| WS_SPLIT = re.compile(r"([ \t]{2,}|\t)") | |
| LEAD = re.compile(r"[ \t]*") | |
| def spaces(n, out): | |
| for w, i in SPACE_IDS: | |
| while n >= w: | |
| out.append(i) | |
| n -= w | |
| def ws_ids(ws, out): | |
| run = 0 | |
| for ch in ws: | |
| if ch == " ": | |
| run += 1 | |
| else: # tab | |
| spaces(run, out) | |
| run = 0 | |
| out.append(TAB) | |
| spaces(run, out) | |
| class WSCodec: | |
| def __init__(self, sp): | |
| self.sp = sp | |
| bad = [(i, p, sp.id_to_piece(i)) for i, (_, p) in CODEC.items() | |
| if i >= sp.get_piece_size() or sp.id_to_piece(i) != p] | |
| if bad: | |
| raise RuntimeError(f"ws_codec does not match this tokenizer: {bad}") | |
| self.ids = frozenset(CODEC) | |
| self.text = {i: m for i, (m, _) in CODEC.items()} | |
| def __getattr__(self, name): # eos_id, get_piece_size, ... | |
| return getattr(self.sp, name) | |
| def encode(self, text, *args, **kwargs): | |
| if not isinstance(text, str): # batch call | |
| return [self.encode(t) for t in text] | |
| text = text.replace("\r\n", "\n").replace("\r", "\n") | |
| plan, segs = [], [] # plan: ints or ("seg", k) | |
| for li, line in enumerate(text.split("\n")): | |
| if li: | |
| plan.append(NL) | |
| lead = LEAD.match(line).group() | |
| if lead: | |
| ws_ids(lead, plan) | |
| for k, part in enumerate(WS_SPLIT.split(line[len(lead):])): | |
| if not part: | |
| continue | |
| if k % 2: # whitespace run | |
| ws_ids(part, plan) | |
| elif part.strip(): | |
| plan.append(("seg", len(segs))) | |
| segs.append(part) | |
| enc = self.sp.encode(segs) if segs else [] | |
| out = [] | |
| for x in plan: | |
| if isinstance(x, tuple): | |
| out.extend(i for i in enc[x[1]] if i not in self.ids) | |
| else: | |
| out.append(x) | |
| return out | |
| def decode(self, ids, *args, **kwargs): | |
| if ids and not isinstance(ids[0], int) and hasattr(ids[0], "__len__"): | |
| return [self.decode(x) for x in ids] | |
| out, run = [], [] | |
| for i in ids: | |
| i = int(i) | |
| if i in self.ids: | |
| if run: | |
| out.append(self.sp.decode(run)) | |
| run = [] | |
| out.append(self.text[i]) | |
| else: | |
| run.append(i) | |
| if run: | |
| out.append(self.sp.decode(run)) | |
| return "".join(out) | |
| SAMPLES = [ | |
| "def f(x):\n if x:\n return 1\n\treturn 2\n", | |
| "class A:\n\n def g(self, a, b):\n # comment here\n" | |
| " return {'k': [1, 2, 3]}\n\n\nprint(A().g(1, 2))\n", | |
| "<div class=\"box\">\n <p>Hello, world</p>\n</div>\n", | |
| "int main() {\n\tfor (int i = 0; i < 3; i++) {\n\t\tputs(\"hi\");\n\t}\n}\n", | |
| "server:\n port: 8080\n hosts:\n - a.example.com\n - b.example.com\n", | |
| "Plain prose line one.\nLine two, after a newline.\n\nNew paragraph.", | |
| ] | |
| def selftest(): | |
| """python3 ws_codec.py -> round-trips code samples with the REAL | |
| CodeSymb tokenizer and shows the before/after.""" | |
| import os | |
| import sys | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| import codesymb_common as C | |
| os.environ.pop("CODESYMB_WS_CODEC", None) | |
| sp = C.load_tokenizer(None) | |
| enc = WSCodec(sp) | |
| ok = 0 | |
| for s in SAMPLES: | |
| back = enc.decode(enc.encode(s)) | |
| flat = sp.decode(sp.encode(s)) | |
| want = "\n".join(re.sub(r"(?<=\S) $", "", line) | |
| for line in s.split("\n")) | |
| good = back == want | |
| ok += good | |
| print(f"{'OK ' if good else 'BAD'} {len(enc.encode(s)):4d} tok " | |
| f"(plain {len(sp.encode(s)):4d}) | old: {flat[:50]!r}") | |
| if not good: | |
| print(f" want {want!r}\n got {back!r}") | |
| print(f"\n{ok}/{len(SAMPLES)} samples round-trip exactly" | |
| + (" -> codec READY" if ok == len(SAMPLES) else " -> NOT ready")) | |
| return 0 if ok == len(SAMPLES) else 1 | |
| if __name__ == "__main__": | |
| raise SystemExit(selftest()) | |