File size: 1,022 Bytes
cd3e1a6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | #!/usr/bin/env python3
"""
Tiny byte-level tokenizer:
- Vocab: 0..255 (raw bytes) + 256:[PAD], 257:[EOS]
- UTF-8 in/out
- Deterministic, license-clean, zero deps
"""
PAD_ID = 256
EOS_ID = 257
VOCAB_SIZE = 258
class TinyByteTokenizer:
def __init__(self, pad_id=PAD_ID, eos_id=EOS_ID):
self.pad_id = pad_id
self.eos_id = eos_id
def encode(self, text: str, add_eos=True):
b = text.encode("utf-8", errors="replace")
ids = list(b)
if add_eos:
ids.append(self.eos_id)
return ids
def decode(self, ids):
# drop special tokens
bytes_out = bytearray()
for t in ids:
if t == self.eos_id or t == self.pad_id:
continue
if 0 <= t <= 255:
bytes_out.append(t)
return bytes(bytes_out).decode("utf-8", errors="replace")
def pad(self, ids, max_len):
if len(ids) >= max_len:
return ids[:max_len]
return ids + [self.pad_id] * (max_len - len(ids)) |