looped-40m / core.py
mburchell's picture
Upload 13 files
63bd507 verified
Raw History Blame Contribute Delete
7.04 kB
"""Core reversible capitalization-aware byte-level BPE."""
import json
import re
from collections.abc import Iterable, Mapping, Sequence
from pathlib import Path
from tokenizers import AddedToken, Tokenizer, decoders, models, pre_tokenizers, trainers
BOS = "<|begin_of_text|>"
EOS = "<|end_of_text|>"
START_HEADER = "<|start_header_id|>"
END_HEADER = "<|end_header_id|>"
END_TURN = "<|eot_id|>"
PAD = "<|pad|>"
CAPITALIZE = "<|capitalize|>"
UPPERCASE = "<|uppercase|>"
END_CASE = "<|end_case|>"
LITERAL_LT = "<|literal_lt|>"
SPECIAL_TOKENS = (
BOS, EOS, START_HEADER, END_HEADER, END_TURN, PAD,
CAPITALIZE, UPPERCASE, END_CASE, LITERAL_LT,
)
VOCAB_SIZE = 24_576
FORMAT = "ascii-case-bpe-v1"
_WORDS = re.compile(r"<|[A-Za-z]+")
_ASCII_UPPER = str.maketrans("abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZ")
def prepare_text(text: str) -> str:
"""Factor ASCII word casing; leave mixed case and Unicode lossless."""
def replace(match: re.Match[str]) -> str:
word = match.group()
if word == "<":
return LITERAL_LT
if word.isupper():
return UPPERCASE + word.lower() + END_CASE
if word[0].isupper() and word[1:].islower():
return CAPITALIZE + word.lower() + END_CASE
return word
return _WORDS.sub(replace, text)
class ModelTokenizer:
"""Use this wrapper, not bare tokenizer.json, to preserve casing semantics."""
def __init__(self, backend: Tokenizer):
self.backend = backend
self.special_ids: dict[str, int] = {}
added = backend.get_added_tokens_decoder()
for token in SPECIAL_TOKENS:
token_id = backend.token_to_id(token)
if token_id is None or token_id not in added or not added[token_id].special:
raise ValueError(f"Missing reserved special token: {token}")
self.special_ids[token] = token_id
self._special_by_id = {value: key for key, value in self.special_ids.items()}
self.vocab_size = backend.get_vocab_size()
@classmethod
def train(
cls, texts: Iterable[str], vocab_size: int = VOCAB_SIZE
) -> "ModelTokenizer":
minimum = 256 + len(SPECIAL_TOKENS)
if vocab_size < minimum:
raise ValueError(f"Vocabulary must contain at least {minimum} entries")
backend = Tokenizer(models.BPE())
backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
backend.decoder = decoders.ByteLevel()
trainer = trainers.BpeTrainer(
vocab_size=vocab_size,
min_frequency=2,
initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
special_tokens=[
AddedToken(token, normalized=False, special=True)
for token in SPECIAL_TOKENS
],
)
backend.train_from_iterator((prepare_text(text) for text in texts), trainer)
result = cls(backend)
if result.vocab_size != vocab_size:
raise ValueError(
f"Sample produced {result.vocab_size} vocabulary entries, "
f"not {vocab_size}; increase the bounded sample size"
)
return result
def encode(
self, text: str, *, add_bos: bool = False, add_eos: bool = False
) -> list[int]:
ids = self.backend.encode(prepare_text(text), add_special_tokens=False).ids
if add_bos:
ids.insert(0, self.special_ids[BOS])
if add_eos:
ids.append(self.special_ids[EOS])
return ids
def decode(self, ids: Sequence[int], *, skip_special_tokens: bool = True) -> str:
chunks: list[str] = []
pending: list[int] = []
mode: str | None = None
def flush() -> None:
if not pending:
return
text = self.backend.decode(pending, skip_special_tokens=False)
if mode == UPPERCASE:
text = text.translate(_ASCII_UPPER)
elif mode == CAPITALIZE:
text = text[:1].translate(_ASCII_UPPER) + text[1:]
chunks.append(text)
pending.clear()
for token_id in ids:
if type(token_id) is not int or not 0 <= token_id < self.vocab_size:
raise ValueError(f"Invalid token ID: {token_id!r}")
token = self._special_by_id.get(token_id)
if token is None:
pending.append(token_id)
continue
flush()
mode = None
if token in (CAPITALIZE, UPPERCASE):
mode = token
elif token == LITERAL_LT:
chunks.append("<")
elif token != END_CASE and not skip_special_tokens:
chunks.append(token)
flush()
return "".join(chunks)
def encode_chat(
self,
messages: Sequence[Mapping[str, str]],
*,
add_generation_prompt: bool = False,
) -> list[int]:
ids = [self.special_ids[BOS]]
for message in messages:
role = message["role"]
content = message["content"]
if role not in ("system", "user", "assistant"):
raise ValueError(f"Unsupported chat role: {role!r}")
ids.extend(self._header(role))
ids.extend(self.encode(content))
ids.append(self.special_ids[END_TURN])
if add_generation_prompt:
ids.extend(self._header("assistant"))
return ids
def _header(self, role: str) -> list[int]:
return (
[self.special_ids[START_HEADER]]
+ self.encode(role)
+ [self.special_ids[END_HEADER]]
+ self.encode("\n\n")
)
def save(self, directory: str | Path) -> None:
directory = Path(directory)
directory.mkdir(parents=True, exist_ok=True)
self.backend.save(str(directory / "tokenizer.json"))
config = {
"format": FORMAT,
"vocab_size": self.vocab_size,
"special_token_ids": self.special_ids,
"case_scope": "ASCII alphabetic runs, terminated by end_case",
"fallback": "mixed-case ASCII runs and Unicode remain byte-preserved",
}
(directory / "tokenizer_config.json").write_text(
json.dumps(config, indent=2) + "\n", encoding="utf-8"
)
@classmethod
def load(cls, directory: str | Path) -> "ModelTokenizer":
directory = Path(directory)
config = json.loads(
(directory / "tokenizer_config.json").read_text(encoding="utf-8")
)
if config["format"] != FORMAT:
raise ValueError(f"Unsupported tokenizer format: {config['format']!r}")
result = cls(Tokenizer.from_file(str(directory / "tokenizer.json")))
if (
result.vocab_size != config["vocab_size"]
or result.special_ids != config["special_token_ids"]
):
raise ValueError("Tokenizer vocabulary does not match its configuration")
return result