File size: 1,361 Bytes
e644e74 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | import json
import re
class CharTokenizer:
"""
Simple character-level tokenizer.
Builds vocabulary from the training text.
"""
def __init__(self):
self.char_to_id = {}
self.id_to_char = {}
self.vocab_size = 0
def build_vocab(self, text: str):
chars = sorted(set(text))
self.char_to_id = {ch: i for i, ch in enumerate(chars)}
self.id_to_char = {i: ch for i, ch in enumerate(chars)}
self.vocab_size = len(chars)
return self
def encode(self, text: str):
return [self.char_to_id.get(ch, 0) for ch in text]
def decode(self, ids):
return "".join(self.id_to_char.get(i, "?") for i in ids)
def save(self, path="tokenizer.json"):
data = {
"char_to_id": self.char_to_id,
"id_to_char": {str(k): v for k, v in self.id_to_char.items()},
"vocab_size": self.vocab_size,
}
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def load(self, path="tokenizer.json"):
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
self.char_to_id = data["char_to_id"]
self.id_to_char = {int(k): v for k, v in data["id_to_char"].items()}
self.vocab_size = data["vocab_size"]
return self
|