File size: 803 Bytes
dc7e099 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | import os
from transformers import AutoTokenizer, AutoImageProcessor
import traceback
MODEL_PATH = "/workspace/ankahi/punjabi_gemma/ankahi"
print("--- Testing Tokenizer ---")
try:
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
print("Tokenizer loaded successfully.")
print(f"Vocab size: {len(tokenizer)}")
except Exception as e:
print(f"Tokenizer failed: {e}")
traceback.print_exc()
print("\n--- Testing Image Processor ---")
try:
# Some models might have image_processor_config.json or preprocessor_config.json
image_processor = AutoImageProcessor.from_pretrained(MODEL_PATH)
print("Image Processor loaded successfully.")
except Exception as e:
print(f"Image Processor failed: {e}")
# traceback.print_exc() # Often fails if not multimodal, that's fine
|