ae / check_tokenizer.py
ChadlieOak's picture
Upload 22 files
c9e8126 verified
Raw History Blame Contribute Delete
956 Bytes
#!/usr/bin/env python3
"""
Check the actual vocabulary size from the trained model's tokenizer
"""
# We're now in the ae directory
import pickle
def check_tokenizer():
"""Check the tokenizer vocabulary size."""
try:
# Load the tokenizer
with open('cutting_edge_model/tokenizer.pkl', 'rb') as f:
tokenizer = pickle.load(f)
print(f"Actual vocab size: {tokenizer.vocab_size}")
print(f"Tokenizer type: {type(tokenizer)}")
# Check some vocabulary items
if hasattr(tokenizer, 'vocab'):
print(f"Sample vocab items: {list(tokenizer.vocab.keys())[:10]}")
return tokenizer.vocab_size
except Exception as e:
print(f"Error: {e}")
return None
if __name__ == "__main__":
vocab_size = check_tokenizer()
if vocab_size:
print(f"\nThe correct vocab_size should be: {vocab_size}")