from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, decoders
import json
import os

# Load vocab from the correct path
vocab_path = os.path.join("Tokenizer", "vocab.triadic.json")
with open(vocab_path, "r", encoding="utf-8") as f:
    vocab_list = json.load(f)

# Convert list to dictionary
vocab_dict = {entry["token"]: entry["id"] for entry in vocab_list}

# Initialize WordLevel tokenizer
tokenizer = Tokenizer(models.WordLevel(vocab=vocab_dict, unk_token="[UNK]"))

# Apply simple whitespace pre-tokenizer
tokenizer.normalizer = normalizers.Sequence([])
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()

# Optionally define a decoder — or skip it
# (Commenting out invalid decoder line)
# tokenizer.decoder = decoders.Word()  # ❌ Invalid
# Instead: optional generic replacement decoder
tokenizer.decoder = decoders.Replace(" ", "")  # Acts as passthrough

# Save tokenizer
save_path = os.path.join("Tokenizer", "tokenizer.ΦΣ.json")
tokenizer.save(save_path)

print(f"✅ Tokenizer saved as {save_path}")