File size: 1,068 Bytes
9b19fd8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, decoders
import json
import os
# Load vocab from the correct path
vocab_path = os.path.join("Tokenizer", "vocab.triadic.json")
with open(vocab_path, "r", encoding="utf-8") as f:
vocab_list = json.load(f)
# Convert list to dictionary
vocab_dict = {entry["token"]: entry["id"] for entry in vocab_list}
# Initialize WordLevel tokenizer
tokenizer = Tokenizer(models.WordLevel(vocab=vocab_dict, unk_token="[UNK]"))
# Apply simple whitespace pre-tokenizer
tokenizer.normalizer = normalizers.Sequence([])
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
# Optionally define a decoder — or skip it
# (Commenting out invalid decoder line)
# tokenizer.decoder = decoders.Word() # ❌ Invalid
# Instead: optional generic replacement decoder
tokenizer.decoder = decoders.Replace(" ", "") # Acts as passthrough
# Save tokenizer
save_path = os.path.join("Tokenizer", "tokenizer.ΦΣ.json")
tokenizer.save(save_path)
print(f"✅ Tokenizer saved as {save_path}")
|