"""Core reversible capitalization-aware byte-level BPE.""" import json import re from collections.abc import Iterable, Mapping, Sequence from pathlib import Path from tokenizers import AddedToken, Tokenizer, decoders, models, pre_tokenizers, trainers BOS = "<|begin_of_text|>" EOS = "<|end_of_text|>" START_HEADER = "<|start_header_id|>" END_HEADER = "<|end_header_id|>" END_TURN = "<|eot_id|>" PAD = "<|pad|>" CAPITALIZE = "<|capitalize|>" UPPERCASE = "<|uppercase|>" END_CASE = "<|end_case|>" LITERAL_LT = "<|literal_lt|>" SPECIAL_TOKENS = ( BOS, EOS, START_HEADER, END_HEADER, END_TURN, PAD, CAPITALIZE, UPPERCASE, END_CASE, LITERAL_LT, ) VOCAB_SIZE = 24_576 FORMAT = "ascii-case-bpe-v1" _WORDS = re.compile(r"<|[A-Za-z]+") _ASCII_UPPER = str.maketrans("abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZ") def prepare_text(text: str) -> str: """Factor ASCII word casing; leave mixed case and Unicode lossless.""" def replace(match: re.Match[str]) -> str: word = match.group() if word == "<": return LITERAL_LT if word.isupper(): return UPPERCASE + word.lower() + END_CASE if word[0].isupper() and word[1:].islower(): return CAPITALIZE + word.lower() + END_CASE return word return _WORDS.sub(replace, text) class ModelTokenizer: """Use this wrapper, not bare tokenizer.json, to preserve casing semantics.""" def __init__(self, backend: Tokenizer): self.backend = backend self.special_ids: dict[str, int] = {} added = backend.get_added_tokens_decoder() for token in SPECIAL_TOKENS: token_id = backend.token_to_id(token) if token_id is None or token_id not in added or not added[token_id].special: raise ValueError(f"Missing reserved special token: {token}") self.special_ids[token] = token_id self._special_by_id = {value: key for key, value in self.special_ids.items()} self.vocab_size = backend.get_vocab_size() @classmethod def train( cls, texts: Iterable[str], vocab_size: int = VOCAB_SIZE ) -> "ModelTokenizer": minimum = 256 + len(SPECIAL_TOKENS) if vocab_size < minimum: raise ValueError(f"Vocabulary must contain at least {minimum} entries") backend = Tokenizer(models.BPE()) backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) backend.decoder = decoders.ByteLevel() trainer = trainers.BpeTrainer( vocab_size=vocab_size, min_frequency=2, initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), special_tokens=[ AddedToken(token, normalized=False, special=True) for token in SPECIAL_TOKENS ], ) backend.train_from_iterator((prepare_text(text) for text in texts), trainer) result = cls(backend) if result.vocab_size != vocab_size: raise ValueError( f"Sample produced {result.vocab_size} vocabulary entries, " f"not {vocab_size}; increase the bounded sample size" ) return result def encode( self, text: str, *, add_bos: bool = False, add_eos: bool = False ) -> list[int]: ids = self.backend.encode(prepare_text(text), add_special_tokens=False).ids if add_bos: ids.insert(0, self.special_ids[BOS]) if add_eos: ids.append(self.special_ids[EOS]) return ids def decode(self, ids: Sequence[int], *, skip_special_tokens: bool = True) -> str: chunks: list[str] = [] pending: list[int] = [] mode: str | None = None def flush() -> None: if not pending: return text = self.backend.decode(pending, skip_special_tokens=False) if mode == UPPERCASE: text = text.translate(_ASCII_UPPER) elif mode == CAPITALIZE: text = text[:1].translate(_ASCII_UPPER) + text[1:] chunks.append(text) pending.clear() for token_id in ids: if type(token_id) is not int or not 0 <= token_id < self.vocab_size: raise ValueError(f"Invalid token ID: {token_id!r}") token = self._special_by_id.get(token_id) if token is None: pending.append(token_id) continue flush() mode = None if token in (CAPITALIZE, UPPERCASE): mode = token elif token == LITERAL_LT: chunks.append("<") elif token != END_CASE and not skip_special_tokens: chunks.append(token) flush() return "".join(chunks) def encode_chat( self, messages: Sequence[Mapping[str, str]], *, add_generation_prompt: bool = False, ) -> list[int]: ids = [self.special_ids[BOS]] for message in messages: role = message["role"] content = message["content"] if role not in ("system", "user", "assistant"): raise ValueError(f"Unsupported chat role: {role!r}") ids.extend(self._header(role)) ids.extend(self.encode(content)) ids.append(self.special_ids[END_TURN]) if add_generation_prompt: ids.extend(self._header("assistant")) return ids def _header(self, role: str) -> list[int]: return ( [self.special_ids[START_HEADER]] + self.encode(role) + [self.special_ids[END_HEADER]] + self.encode("\n\n") ) def save(self, directory: str | Path) -> None: directory = Path(directory) directory.mkdir(parents=True, exist_ok=True) self.backend.save(str(directory / "tokenizer.json")) config = { "format": FORMAT, "vocab_size": self.vocab_size, "special_token_ids": self.special_ids, "case_scope": "ASCII alphabetic runs, terminated by end_case", "fallback": "mixed-case ASCII runs and Unicode remain byte-preserved", } (directory / "tokenizer_config.json").write_text( json.dumps(config, indent=2) + "\n", encoding="utf-8" ) @classmethod def load(cls, directory: str | Path) -> "ModelTokenizer": directory = Path(directory) config = json.loads( (directory / "tokenizer_config.json").read_text(encoding="utf-8") ) if config["format"] != FORMAT: raise ValueError(f"Unsupported tokenizer format: {config['format']!r}") result = cls(Tokenizer.from_file(str(directory / "tokenizer.json"))) if ( result.vocab_size != config["vocab_size"] or result.special_ids != config["special_token_ids"] ): raise ValueError("Tokenizer vocabulary does not match its configuration") return result