| #!/usr/bin/env python3 | |
| """ | |
| Tiny byte-level tokenizer: | |
| - Vocab: 0..255 (raw bytes) + 256:[PAD], 257:[EOS] | |
| - UTF-8 in/out | |
| - Deterministic, license-clean, zero deps | |
| """ | |
| PAD_ID = 256 | |
| EOS_ID = 257 | |
| VOCAB_SIZE = 258 | |
| class TinyByteTokenizer: | |
| def __init__(self, pad_id=PAD_ID, eos_id=EOS_ID): | |
| self.pad_id = pad_id | |
| self.eos_id = eos_id | |
| def encode(self, text: str, add_eos=True): | |
| b = text.encode("utf-8", errors="replace") | |
| ids = list(b) | |
| if add_eos: | |
| ids.append(self.eos_id) | |
| return ids | |
| def decode(self, ids): | |
| # drop special tokens | |
| bytes_out = bytearray() | |
| for t in ids: | |
| if t == self.eos_id or t == self.pad_id: | |
| continue | |
| if 0 <= t <= 255: | |
| bytes_out.append(t) | |
| return bytes(bytes_out).decode("utf-8", errors="replace") | |
| def pad(self, ids, max_len): | |
| if len(ids) >= max_len: | |
| return ids[:max_len] | |
| return ids + [self.pad_id] * (max_len - len(ids)) |