test_tokenizers_lib.py

from tokenizers import Tokenizer
import os

MODEL_PATH = "/workspace/ankahi/punjabi_gemma/ankahi"
TOKENIZER_JSON = os.path.join(MODEL_PATH, "tokenizer.json")

print(f"Loading tokenizer from {TOKENIZER_JSON}")
try:
    tokenizer = Tokenizer.from_file(TOKENIZER_JSON)
    print("Success!")
    print(f"Vocab size: {tokenizer.get_vocab_size()}")
except Exception as e:
    print(f"Failed: {e}")