File size: 956 Bytes
c9e8126 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | #!/usr/bin/env python3
"""
Check the actual vocabulary size from the trained model's tokenizer
"""
# We're now in the ae directory
import pickle
def check_tokenizer():
"""Check the tokenizer vocabulary size."""
try:
# Load the tokenizer
with open('cutting_edge_model/tokenizer.pkl', 'rb') as f:
tokenizer = pickle.load(f)
print(f"Actual vocab size: {tokenizer.vocab_size}")
print(f"Tokenizer type: {type(tokenizer)}")
# Check some vocabulary items
if hasattr(tokenizer, 'vocab'):
print(f"Sample vocab items: {list(tokenizer.vocab.keys())[:10]}")
return tokenizer.vocab_size
except Exception as e:
print(f"Error: {e}")
return None
if __name__ == "__main__":
vocab_size = check_tokenizer()
if vocab_size:
print(f"\nThe correct vocab_size should be: {vocab_size}") |