File size: 1,447 Bytes
546acd4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
"""The ChatGBA tokenizer: every UTF-8 byte is one token, ids 0 to 255.

There are no special tokens. The chat markers (<|u|>, <|a|>, <|e|>) are plain
text, spelled out byte by byte, exactly as the model was trained. A reply ends
when the model writes <|e|>: pass stop_strings="<|e|>" (and this tokenizer)
to generate.
"""
from transformers import PreTrainedTokenizer


class GabByteTokenizer(PreTrainedTokenizer):
    model_input_names = ["input_ids"]

    def __init__(self, **kwargs):
        kwargs.setdefault("bos_token", None)
        kwargs.setdefault("eos_token", None)
        kwargs.setdefault("pad_token", None)
        kwargs.setdefault("unk_token", None)
        super().__init__(**kwargs)

    @property
    def vocab_size(self):
        return 256

    def get_vocab(self):
        return {self._convert_id_to_token(i): i for i in range(256)}

    def _tokenize(self, text, **kwargs):
        return [self._convert_id_to_token(b) for b in text.encode("utf-8")]

    def _convert_token_to_id(self, token):
        return int(token[3:-1], 16)  # "<0x41>" -> 65

    def _convert_id_to_token(self, index):
        return f"<0x{index:02X}>"

    def convert_tokens_to_string(self, tokens):
        return bytes(self._convert_token_to_id(t) for t in tokens).decode("utf-8", errors="replace")

    def save_vocabulary(self, save_directory, filename_prefix=None):
        return ()  # nothing to save: the vocabulary is the 256 byte values