| from tokenizers import Tokenizer | |
| import os | |
| MODEL_PATH = "/workspace/ankahi/punjabi_gemma/ankahi" | |
| TOKENIZER_JSON = os.path.join(MODEL_PATH, "tokenizer.json") | |
| print(f"Loading tokenizer from {TOKENIZER_JSON}") | |
| try: | |
| tokenizer = Tokenizer.from_file(TOKENIZER_JSON) | |
| print("Success!") | |
| print(f"Vocab size: {tokenizer.get_vocab_size()}") | |
| except Exception as e: | |
| print(f"Failed: {e}") | |