"""Compact English-only unigram tokenizer ("ETK2"). Input is already `normalize`d ([a-z0-9] words separated by single spaces), so no byte fallback or unicode handling is needed. Word boundaries are marked with `_` (SentencePiece's `▁`). File layout: b"ETK2" | u16 n_pieces | n x ( u8 len | ascii bytes | i8 score*8 ) Piece 0 is . """ from __future__ import annotations import struct MAGIC = b"ETK2" SCORE_SCALE = 8.0 UNK_SCORE = -30.0 class TinyTokenizer: def __init__(self, data: bytes): if data[:4] != MAGIC: raise ValueError("bad tokenizer file") (n,) = struct.unpack_from(" 0} self.unk_id = 0 self.max_len = max(len(p) for p in self.pieces) @classmethod def load(cls, path: str) -> "TinyTokenizer": with open(path, "rb") as f: return cls(f.read()) @staticmethod def dump(pieces: list[str], scores: list[float]) -> bytes: out = bytearray(MAGIC + struct.pack(" list[int]: s = "_" + norm_text.replace(" ", "_") n = len(s) if n <= 1: return [] best = [-1e18] * (n + 1) best[0] = 0.0 bp = [0] * (n + 1) bid = [self.unk_id] * (n + 1) for i in range(1, n + 1): for j in range(max(0, i - self.max_len), i): tid = self.index.get(s[j:i]) if tid is not None: sc = best[j] + self.scores[tid] if sc > best[i]: best[i], bp[i], bid[i] = sc, j, tid sc = best[i - 1] + UNK_SCORE if sc > best[i]: best[i], bp[i], bid[i] = sc, i - 1, self.unk_id ids: list[int] = [] i = n while i > 0: ids.append(bid[i]) i = bp[i] ids.reverse() return ids if max_tokens is None else ids[-max_tokens:]