Spaces:
Running on Zero
Running on Zero
File size: 3,973 Bytes
98d1e65 1717c0c 9cde5e3 1717c0c 9cde5e3 1717c0c 9cde5e3 1717c0c 9cde5e3 1717c0c 9cde5e3 1717c0c 9cde5e3 1717c0c 9cde5e3 1717c0c 9cde5e3 1717c0c 972b71a 9cde5e3 1717c0c 9cde5e3 1717c0c 9cde5e3 1717c0c 9cde5e3 1717c0c 9cde5e3 98d1e65 1717c0c 98d1e65 1717c0c 98d1e65 1717c0c 98d1e65 1717c0c 98d1e65 1717c0c 98d1e65 1717c0c 98d1e65 1717c0c 98d1e65 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | import json
from pathlib import Path
# Treats each character as a token
class CharacterTokenizer:
PAD_TOKEN = "<PAD>"
UNK_TOKEN = "<UNK>"
CLS_TOKEN = "<CLS>" #classification token
SEP_TOKEN = "<SEP>" #separator
def __init__(self, text):
characters = sorted(set(text)) #sorts + removes duplicates
self.stoi = {
self.PAD_TOKEN: 0,
self.UNK_TOKEN: 1,
self.CLS_TOKEN: 2,
self.SEP_TOKEN: 3,
}
for character in characters:
self.stoi[character] = len(self.stoi)
self.itos = {
index: token
for token, index in self.stoi.items()
}
@property
def vocab_size(self):
return len(self.stoi)
@property
def pad_token_id(self):
return self.stoi[self.PAD_TOKEN]
@property
def unk_token_id(self):
return self.stoi[self.UNK_TOKEN]
@property
def cls_token_id(self):
return self.stoi[self.CLS_TOKEN]
@property
def sep_token_id(self):
return self.stoi[self.SEP_TOKEN]
# Original encode before CLS and SEP tokens
# def encode(self, text):
# return [
# self.stoi.get(
# character,
# self.unk_token_id,
# )
# for character in text
# ]
def encode(self, text):
tokens = []
i = 0
while i < len(text):
if text.startswith(self.CLS_TOKEN, i): #start checking from here
tokens.append(self.cls_token_id)
i += len(self.CLS_TOKEN) #skip the length of the token
elif text.startswith(self.SEP_TOKEN, i):
tokens.append(self.sep_token_id)
i += len(self.SEP_TOKEN) #skip the length of the token
else:
tokens.append(
self.stoi.get(
text[i],
self.unk_token_id,
)
)
i += 1
return tokens #actually indices in the token space
def decode(self, token_ids):
return "".join(
self.itos[token_id]
for token_id in token_ids
if token_id != self.pad_token_id
)
def pad(
self,
sequences,
max_length=None,
):
if max_length is None:
max_length = max(
len(sequence)
for sequence in sequences
)
padded = []
masks = []
for sequence in sequences:
sequence = sequence[:max_length]
padding_length = (
max_length - len(sequence)
)
padded_sequence = (
sequence
+ [self.pad_token_id] * padding_length
)
mask = (
[1] * len(sequence)
+ [0] * padding_length
)
padded.append(padded_sequence)
masks.append(mask)
return padded, masks
#Save the tokenizer
def save(self, path):
path = Path(path)
data = {
"version": 2,
"stoi": self.stoi,
}
path.write_text(
json.dumps(
data,
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
#Load saved
@classmethod
def load(cls, path):
path = Path(path)
data = json.loads(
path.read_text(
encoding="utf-8",
)
)
tokenizer = cls.__new__(cls)
tokenizer.stoi = data["stoi"]
tokenizer.itos = {
index: token
for token, index in tokenizer.stoi.items()
}
return tokenizer |