Spaces:
Sleeping
Sleeping
Deploy TinyMetatron SLM read-only demo (build-time trained checkpoint, deploy-mode gating)
3deca49 verified Download tokenizer.py from Quantum927/tinymetatron-slm: direct link, hf CLI and curl.
- Browser
- Download file 9.38 kB
-
https://huggingface.co/spaces/Quantum927/tinymetatron-slm/resolve/main/tokenizer.py
- Command line
-
hf download hf://spaces/Quantum927/tinymetatron-slm/tokenizer.py
-
curl -L -o tokenizer.py https://huggingface.co/spaces/Quantum927/tinymetatron-slm/resolve/main/tokenizer.py
9.38 kB
| """ | |
| tokenizer.py | |
| ============ | |
| Deterministic greedy longest-match tokenizer for the TinyMetatron SLM. | |
| Vocabulary layout (total 291 ids, per IMPLEMENTATION_CONTRACT.md Β§2 and | |
| config.py): | |
| 0..4 : special tokens (pad, bos, eos, unk, sep) | |
| 5..68 : 64 single-char slots (a-z, most SK diacritics, digits, punctuation) | |
| 69..290 : curated SK+EN technical word/piece tokens β plus three extra | |
| single-char Slovak diacritics (Δ ΔΊ Ε) repurposed from the tail | |
| of the curated range (ids 288, 289, 290) so that EVERY Slovak | |
| diacritic has a dedicated single-char slot. | |
| Encoding normalises to lowercase, then greedily matches the longest in-vocab | |
| token at each position, falling back to UNK for any character not in the | |
| vocab. Because every Slovak diacritic (including Δ ΔΊ Ε) and every common | |
| punctuation character has a dedicated single-char slot, in-vocab-character | |
| text round-trips losslessly through encode->decode even when a whole word is | |
| not a curated token. | |
| The tokenizer is a pure-Python component (patent component: deterministic | |
| "Metatron" symbol layer feeding the polyhedral attention / MoE core). | |
| """ | |
| from __future__ import annotations | |
| import json | |
| from typing import Dict, List | |
| from config import CONFIG, get_config | |
| # ββ Special-token names mapped to their CONFIG keys βββββββββββββββββββββββββ | |
| _SPECIAL_KEYS = ("pad_id", "bos_id", "eos_id", "unk_id", "sep_id") | |
| class Tokenizer: | |
| """Greedy longest-match BPE-free tokenizer over a fixed 291-token vocab. | |
| Public interface (contract Β§2): | |
| encode(text) -> list[int] # [BOS] + ids + [EOS] | |
| decode(ids) -> str # drop specials, join pieces | |
| vocab_size -> int # 291 | |
| special_ids -> dict # {pad,bos,eos,unk,sep: id} | |
| Tokenizer.from_file(path) -> Tokenizer | |
| """ | |
| def __init__(self, vocab: Dict[str, int]): | |
| # Defensive copy so callers cannot mutate the loaded mapping. | |
| self.vocab: Dict[str, int] = dict(vocab) | |
| # Inverse map id -> token string (last writer wins; ids are unique). | |
| self._inv: Dict[int, str] = {v: k for k, v in self.vocab.items()} | |
| # Special ids derived from CONFIG (never hardcoded). | |
| c = get_config() | |
| self.special_ids: Dict[str, int] = {name: c[name] for name in _SPECIAL_KEYS} | |
| self._bos_id = c["bos_id"] | |
| self._eos_id = c["eos_id"] | |
| self._unk_id = c["unk_id"] | |
| self._special_set = set(self.special_ids.values()) | |
| # Greedy match requires scanning candidate tokens longest-first. | |
| # Pre-compute once; this is the hot path for encode(). | |
| self._sorted_tokens: List[str] = sorted(self.vocab.keys(), key=len, reverse=True) | |
| # ββ Properties ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def vocab_size(self) -> int: | |
| """Total number of tokens in the vocabulary (must be 291).""" | |
| return len(self.vocab) | |
| # ββ Normalisation βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _normalize(text: str) -> str: | |
| """Lowercase. Deterministic, locale-independent.""" | |
| return text.lower() | |
| # ββ Core greedy longest-match encoder ββββββββββββββββββββββββββββββββββββ | |
| def _greedy_match(self, text: str) -> List[int]: | |
| """Return ids for `text` (already normalised) with UNK fallback.""" | |
| ids: List[int] = [] | |
| i = 0 | |
| n = len(text) | |
| vocab = self.vocab | |
| sorted_tokens = self._sorted_tokens | |
| while i < n: | |
| matched_id = -1 | |
| matched_len = 0 | |
| # The candidate list is longest-first, so the first hit is the | |
| # longest match at position i β the greedy invariant. | |
| for tok in sorted_tokens: | |
| if tok and text.startswith(tok, i): | |
| matched_id = vocab[tok] | |
| matched_len = len(tok) | |
| break | |
| if matched_len > 0: | |
| ids.append(matched_id) | |
| i += matched_len | |
| else: | |
| # No single-char slot for this codepoint -> UNK. | |
| ids.append(self._unk_id) | |
| i += 1 | |
| return ids | |
| # ββ Public API βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def encode(self, text: str) -> List[int]: | |
| """Tokenise `text` -> [BOS] + piece_ids + [EOS].""" | |
| normalised = self._normalize(text) | |
| body = self._greedy_match(normalised) | |
| return [self._bos_id] + body + [self._eos_id] | |
| def decode(self, ids: List[int]) -> str: | |
| """Inverse of encode(): drop specials, join the remaining pieces. | |
| Spacing is handled implicitly because the space character is itself a | |
| single-char token in the vocab; a plain concatenation reproduces the | |
| original spacing of the normalised source text. | |
| """ | |
| inv = self._inv | |
| specials = self._special_set | |
| pieces: List[str] = [] | |
| for i in ids: | |
| if i in specials: | |
| continue | |
| piece = inv.get(i) | |
| if piece is not None: | |
| pieces.append(piece) | |
| return "".join(pieces) | |
| def from_file(cls, path: str) -> "Tokenizer": | |
| """Load a vocab.json (token string -> int id) and build a Tokenizer.""" | |
| with open(path, "r", encoding="utf-8") as f: | |
| vocab = json.load(f) | |
| # Sanity: ensure ids are ints (JSON keys are always strings). | |
| vocab = {str(k): int(v) for k, v in vocab.items()} | |
| return cls(vocab) | |
| # ββ Default loader (uses CONFIG["vocab_path"] when present) βββββββββββββββββ | |
| def default_tokenizer() -> Tokenizer: | |
| """Build a Tokenizer from CONFIG["vocab_path"] relative to repo root.""" | |
| c = get_config() | |
| return Tokenizer.from_file(c["vocab_path"]) | |
| # ββ Self-test βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| if __name__ == "__main__": | |
| import sys | |
| sys.stdout.reconfigure(encoding="utf-8") # Windows cp1252 fix (rule 4) | |
| tok = default_tokenizer() | |
| c = get_config() | |
| assert tok.vocab_size == c["vocab_size"], ( | |
| f"vocab_size {tok.vocab_size} != CONFIG vocab_size {c['vocab_size']}" | |
| ) | |
| assert tok.vocab_size == 291, f"expected 291 tokens, got {tok.vocab_size}" | |
| # Slovak technical sentence β every char is in-vocab -> lossless. | |
| sk = "Ε‘ifrovanie a autentifikΓ‘cia hesla zniΕΎujΓΊ zraniteΔΎnosΕ₯ proti ΓΊtoku phishing." | |
| enc_sk = tok.encode(sk) | |
| dec_sk = tok.decode(enc_sk) | |
| assert dec_sk == sk.lower(), ( | |
| f"SK round-trip failed:\n in : {sk.lower()}\n out: {dec_sk}" | |
| ) | |
| # Every Slovak diacritic β including the previously-missing Δ ΔΊ Ε β must | |
| # round-trip losslessly (no UNK) since each has a dedicated single-char slot. | |
| diac = "ΔalΕ‘Γ ΔΊudia Εaden" | |
| enc_d = tok.encode(diac) | |
| dec_d = tok.decode(enc_d) | |
| assert tok.special_ids["unk_id"] not in enc_d, "Δ/ΔΊ/Ε must be in-vocab" | |
| assert dec_d == diac.lower(), ( | |
| f"diacritic round-trip failed:\n in : {diac.lower()}\n out: {dec_d}" | |
| ) | |
| # English technical sentence β lossless round-trip. | |
| en = "the model uses attention and expert routing for inference." | |
| enc_en = tok.encode(en) | |
| dec_en = tok.decode(enc_en) | |
| assert dec_en == en.lower(), ( | |
| f"EN round-trip failed:\n in : {en.lower()}\n out: {dec_en}" | |
| ) | |
| # BOS / EOS wrapping. | |
| assert enc_sk[0] == tok.special_ids["bos_id"], "BOS missing at front" | |
| assert enc_sk[-1] == tok.special_ids["eos_id"], "EOS missing at end" | |
| # UNK fallback: a character outside the char slots (e.g. emoji) -> UNK id. | |
| weird = tok.encode("xΓ©") # x, Γ© both in-vocab -> no UNK expected | |
| assert tok.special_ids["unk_id"] not in weird, "Γ©/x should be in-vocab" | |
| unk_test = tok.encode("") # BEL control char -> UNK | |
| assert unk_test[1] == tok.special_ids["unk_id"], "UNK fallback failed" | |
| # from_file round-trip on a temp copy | |
| import json as _json, os as _os, tempfile as _tf | |
| with _tf.NamedTemporaryFile("w", suffix=".json", delete=False, | |
| encoding="utf-8") as tf: | |
| _json.dump(tok.vocab, tf, ensure_ascii=False) | |
| tmp_path = tf.name | |
| try: | |
| tok2 = Tokenizer.from_file(tmp_path) | |
| assert tok2.vocab_size == tok.vocab_size | |
| assert tok2.encode(sk) == enc_sk | |
| finally: | |
| _os.unlink(tmp_path) | |
| print("tokenizer self-test OK") | |
| print(f"vocab_size : {tok.vocab_size}") | |
| print(f"special_ids : {tok.special_ids}") | |
| print(f"SK round-trip : {dec_sk}") | |
| print(f"EN round-trip : {dec_en}") | |
| print(f"SK encoded length : {len(enc_sk)}") |