Download tiny_tokenizer.py from anudit/tinyemo: direct link, hf CLI and curl.
- Browser
- Download file 2.63 kB
-
https://huggingface.co/anudit/tinyemo/resolve/main/tiny_tokenizer.py
- Command line
-
hf download hf://anudit/tinyemo/tiny_tokenizer.py
-
curl -L -o tiny_tokenizer.py https://huggingface.co/anudit/tinyemo/resolve/main/tiny_tokenizer.py
2.63 kB
| """Compact English-only unigram tokenizer ("ETK2"). | |
| Input is already `normalize`d ([a-z0-9] words separated by single spaces), so no | |
| byte fallback or unicode handling is needed. Word boundaries are marked with | |
| `_` (SentencePiece's `▁`). | |
| File layout: b"ETK2" | u16 n_pieces | n x ( u8 len | ascii bytes | i8 score*8 ) | |
| Piece 0 is <unk>. | |
| """ | |
| from __future__ import annotations | |
| import struct | |
| MAGIC = b"ETK2" | |
| SCORE_SCALE = 8.0 | |
| UNK_SCORE = -30.0 | |
| class TinyTokenizer: | |
| def __init__(self, data: bytes): | |
| if data[:4] != MAGIC: | |
| raise ValueError("bad tokenizer file") | |
| (n,) = struct.unpack_from("<H", data, 4) | |
| off = 6 | |
| self.pieces: list[str] = [] | |
| self.scores: list[float] = [] | |
| for _ in range(n): | |
| ln = data[off] | |
| off += 1 | |
| self.pieces.append(data[off : off + ln].decode("ascii")) | |
| off += ln | |
| self.scores.append(struct.unpack_from("<b", data, off)[0] / SCORE_SCALE) | |
| off += 1 | |
| self.index = {p: i for i, p in enumerate(self.pieces) if i > 0} | |
| self.unk_id = 0 | |
| self.max_len = max(len(p) for p in self.pieces) | |
| def load(cls, path: str) -> "TinyTokenizer": | |
| with open(path, "rb") as f: | |
| return cls(f.read()) | |
| def dump(pieces: list[str], scores: list[float]) -> bytes: | |
| out = bytearray(MAGIC + struct.pack("<H", len(pieces))) | |
| for p, s in zip(pieces, scores): | |
| b = p.encode("ascii") | |
| q = max(-128, min(127, round(s * SCORE_SCALE))) | |
| out += struct.pack("<B", len(b)) + b + struct.pack("<b", q) | |
| return bytes(out) | |
| def encode(self, norm_text: str, max_tokens: int | None = None) -> list[int]: | |
| s = "_" + norm_text.replace(" ", "_") | |
| n = len(s) | |
| if n <= 1: | |
| return [] | |
| best = [-1e18] * (n + 1) | |
| best[0] = 0.0 | |
| bp = [0] * (n + 1) | |
| bid = [self.unk_id] * (n + 1) | |
| for i in range(1, n + 1): | |
| for j in range(max(0, i - self.max_len), i): | |
| tid = self.index.get(s[j:i]) | |
| if tid is not None: | |
| sc = best[j] + self.scores[tid] | |
| if sc > best[i]: | |
| best[i], bp[i], bid[i] = sc, j, tid | |
| sc = best[i - 1] + UNK_SCORE | |
| if sc > best[i]: | |
| best[i], bp[i], bid[i] = sc, i - 1, self.unk_id | |
| ids: list[int] = [] | |
| i = n | |
| while i > 0: | |
| ids.append(bid[i]) | |
| i = bp[i] | |
| ids.reverse() | |
| return ids if max_tokens is None else ids[-max_tokens:] | |