#!/usr/bin/env python3
"""
Tiny byte-level tokenizer:
- Vocab: 0..255 (raw bytes) + 256:[PAD], 257:[EOS]
- UTF-8 in/out
- Deterministic, license-clean, zero deps
"""

PAD_ID = 256
EOS_ID = 257
VOCAB_SIZE = 258

class TinyByteTokenizer:
    def __init__(self, pad_id=PAD_ID, eos_id=EOS_ID):
        self.pad_id = pad_id
        self.eos_id = eos_id

    def encode(self, text: str, add_eos=True):
        b = text.encode("utf-8", errors="replace")
        ids = list(b)
        if add_eos:
            ids.append(self.eos_id)
        return ids

    def decode(self, ids):
        # drop special tokens
        bytes_out = bytearray()
        for t in ids:
            if t == self.eos_id or t == self.pad_id:
                continue
            if 0 <= t <= 255:
                bytes_out.append(t)
        return bytes(bytes_out).decode("utf-8", errors="replace")

    def pad(self, ids, max_len):
        if len(ids) >= max_len:
            return ids[:max_len]
        return ids + [self.pad_id] * (max_len - len(ids))