File size: 1,112 Bytes
12496fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | """Lossless byte baseline; not a claim of competitive token compression."""
from dataclasses import dataclass
@dataclass(frozen=True)
class ByteTokenizer:
pad_id: int = 256
bos_id: int = 257
eos_id: int = 258
vocab_size: int = 259
def encode(self, text: str, *, special: bool = False) -> list[int]:
ids = list(text.encode("utf-8"))
return [self.bos_id, *ids, self.eos_id] if special else ids
def decode(self, ids: list[int]) -> str:
if any(not isinstance(i, int) or not 0 <= i < self.vocab_size for i in ids):
raise ValueError("Token outside vocabulary")
return bytes(i for i in ids if i < 256).decode("utf-8", errors="replace")
def benchmark(self, samples: dict[str, str]) -> dict:
return {name: {"characters": len(text), "bytes": len(text.encode()),
"tokens": len(self.encode(text)),
"characters_per_token": len(text) / max(1, len(self.encode(text))),
"roundtrip": self.decode(self.encode(text)) == text}
for name, text in samples.items()}
|