ChatGBA / tokenization_gabgpt.py
gszauer's picture
ChatGBA: chat model, Game Boy Advance cartridge and source, Transformers port, reference.js
546acd4 verified
Raw History Blame Contribute Delete
1.45 kB
"""The ChatGBA tokenizer: every UTF-8 byte is one token, ids 0 to 255.
There are no special tokens. The chat markers (<|u|>, <|a|>, <|e|>) are plain
text, spelled out byte by byte, exactly as the model was trained. A reply ends
when the model writes <|e|>: pass stop_strings="<|e|>" (and this tokenizer)
to generate.
"""
from transformers import PreTrainedTokenizer
class GabByteTokenizer(PreTrainedTokenizer):
model_input_names = ["input_ids"]
def __init__(self, **kwargs):
kwargs.setdefault("bos_token", None)
kwargs.setdefault("eos_token", None)
kwargs.setdefault("pad_token", None)
kwargs.setdefault("unk_token", None)
super().__init__(**kwargs)
@property
def vocab_size(self):
return 256
def get_vocab(self):
return {self._convert_id_to_token(i): i for i in range(256)}
def _tokenize(self, text, **kwargs):
return [self._convert_id_to_token(b) for b in text.encode("utf-8")]
def _convert_token_to_id(self, token):
return int(token[3:-1], 16) # "<0x41>" -> 65
def _convert_id_to_token(self, index):
return f"<0x{index:02X}>"
def convert_tokens_to_string(self, tokens):
return bytes(self._convert_token_to_id(t) for t in tokens).decode("utf-8", errors="replace")
def save_vocabulary(self, save_directory, filename_prefix=None):
return () # nothing to save: the vocabulary is the 256 byte values