"""The ChatGBA tokenizer: every UTF-8 byte is one token, ids 0 to 255. There are no special tokens. The chat markers (<|u|>, <|a|>, <|e|>) are plain text, spelled out byte by byte, exactly as the model was trained. A reply ends when the model writes <|e|>: pass stop_strings="<|e|>" (and this tokenizer) to generate. """ from transformers import PreTrainedTokenizer class GabByteTokenizer(PreTrainedTokenizer): model_input_names = ["input_ids"] def __init__(self, **kwargs): kwargs.setdefault("bos_token", None) kwargs.setdefault("eos_token", None) kwargs.setdefault("pad_token", None) kwargs.setdefault("unk_token", None) super().__init__(**kwargs) @property def vocab_size(self): return 256 def get_vocab(self): return {self._convert_id_to_token(i): i for i in range(256)} def _tokenize(self, text, **kwargs): return [self._convert_id_to_token(b) for b in text.encode("utf-8")] def _convert_token_to_id(self, token): return int(token[3:-1], 16) # "<0x41>" -> 65 def _convert_id_to_token(self, index): return f"<0x{index:02X}>" def convert_tokens_to_string(self, tokens): return bytes(self._convert_token_to_id(t) for t in tokens).decode("utf-8", errors="replace") def save_vocabulary(self, save_directory, filename_prefix=None): return () # nothing to save: the vocabulary is the 256 byte values