Delete vocab_to_bin.py

vocab_to_bin.py

@@ -1,53 +0,0 @@
-"""vocab.txt -> vocab.bin  (strict, minimal)
-
-Format: u8 length + token bytes, one entry per line, id = entry order.
-Refuses to run if the file looks wrong instead of silently corrupting.
-"""
-import sys
-from pathlib import Path
-
-HERE = Path(__file__).resolve().parent
-SRC = HERE / "vocab.txt"
-DST = HERE / "vocab.bin"
-SPECIALS = ["<unk>", "<bos>", "<eos>", "<pad>", "<msk>", "<spc>", "<tab>", "<nwl>"]
-EXPECTED_COUNT = 109  # 8 specials + 94 ascii + 7 smart punct
-
-
-def main():
-    data = SRC.read_bytes()
-    if b"\r" in data:
-        sys.exit("refusing: CR present (CRLF line endings?) — fix the file first")
-    lines = data.split(b"\n")
-    if lines and lines[-1] == b"":
-        lines.pop()
-    if len(lines) != EXPECTED_COUNT:
-        sys.exit(f"refusing: {len(lines)} tokens, expected {EXPECTED_COUNT}")
-    for i, tok in enumerate(lines):
-        if not tok:
-            sys.exit(f"refusing: empty token at line {i + 1}")
-        if len(tok) > 255:
-            sys.exit(f"refusing: token {i + 1} longer than 255 bytes")
-    for i, want in enumerate(SPECIALS):
-        if lines[i] != want.encode():
-            sys.exit(f"refusing: specials block wrong at id {i}: {lines[i]!r} != {want!r}")
-    out = bytearray()
-    for tok in lines:
-        out.append(len(tok))
-        out.extend(tok)
-    DST.write_bytes(bytes(out))
-    print(f"wrote {DST.name}: {len(out)} bytes, {len(lines)} tokens "
-          f"({sum(len(t) for t in lines)} token bytes + {len(lines)} length bytes)")
-
-    # round-trip check
-    back = bytearray()
-    p = 0
-    while p < len(out):
-        n = out[p]
-        back.extend(out[p + 1:p + 1 + n])
-        p += 1 + n
-    assert back == b"\n".join(lines) + (b"\n" if lines else b""), "round-trip failed"
-    print("round-trip: OK")
-
-
-if __name__ == "__main__":
-    main()