Create tiny_byte_tokenizer.py

tiny_byte_tokenizer.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""
+Tiny byte-level tokenizer:
+- Vocab: 0..255 (raw bytes) + 256:[PAD], 257:[EOS]
+- UTF-8 in/out
+- Deterministic, license-clean, zero deps
+"""
+
+PAD_ID = 256
+EOS_ID = 257
+VOCAB_SIZE = 258
+
+class TinyByteTokenizer:
+    def __init__(self, pad_id=PAD_ID, eos_id=EOS_ID):
+        self.pad_id = pad_id
+        self.eos_id = eos_id
+
+    def encode(self, text: str, add_eos=True):
+        b = text.encode("utf-8", errors="replace")
+        ids = list(b)
+        if add_eos:
+            ids.append(self.eos_id)
+        return ids
+
+    def decode(self, ids):
+        # drop special tokens
+        bytes_out = bytearray()
+        for t in ids:
+            if t == self.eos_id or t == self.pad_id:
+                continue
+            if 0 <= t <= 255:
+                bytes_out.append(t)
+        return bytes(bytes_out).decode("utf-8", errors="replace")
+
+    def pad(self, ids, max_len):
+        if len(ids) >= max_len:
+            return ids[:max_len]
+        return ids + [self.pad_id] * (max_len - len(ids))