PHI_CODER / phi-coder-hf /Tokenizer /build_tokenizer.py

Upload 1397 files

9b19fd8 verified about 1 year ago

1.07 kB

from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, decoders
import json
import os

# Load vocab from the correct path
vocab_path = os.path.join("Tokenizer", "vocab.triadic.json")
with open(vocab_path, "r", encoding="utf-8") as f:
    vocab_list = json.load(f)

# Convert list to dictionary
vocab_dict = {entry["token"]: entry["id"] for entry in vocab_list}

# Initialize WordLevel tokenizer
tokenizer = Tokenizer(models.WordLevel(vocab=vocab_dict, unk_token="[UNK]"))

# Apply simple whitespace pre-tokenizer
tokenizer.normalizer = normalizers.Sequence([])
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()

# Optionally define a decoder — or skip it
# (Commenting out invalid decoder line)
# tokenizer.decoder = decoders.Word()  # ❌ Invalid
# Instead: optional generic replacement decoder
tokenizer.decoder = decoders.Replace(" ", "")  # Acts as passthrough

# Save tokenizer
save_path = os.path.join("Tokenizer", "tokenizer.ΦΣ.json")
tokenizer.save(save_path)

print(f"✅ Tokenizer saved as {save_path}")

	from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, decoders
	import json
	import os

	# Load vocab from the correct path
	vocab_path = os.path.join("Tokenizer", "vocab.triadic.json")
	with open(vocab_path, "r", encoding="utf-8") as f:
	vocab_list = json.load(f)

	# Convert list to dictionary
	vocab_dict = {entry["token"]: entry["id"] for entry in vocab_list}

	# Initialize WordLevel tokenizer
	tokenizer = Tokenizer(models.WordLevel(vocab=vocab_dict, unk_token="[UNK]"))

	# Apply simple whitespace pre-tokenizer
	tokenizer.normalizer = normalizers.Sequence([])
	tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()

	# Optionally define a decoder — or skip it
	# (Commenting out invalid decoder line)
	# tokenizer.decoder = decoders.Word() # ❌ Invalid
	# Instead: optional generic replacement decoder
	tokenizer.decoder = decoders.Replace(" ", "") # Acts as passthrough

	# Save tokenizer
	save_path = os.path.join("Tokenizer", "tokenizer.ΦΣ.json")
	tokenizer.save(save_path)

	print(f"✅ Tokenizer saved as {save_path}")