#!/usr/bin/env python3 """ Check the actual vocabulary size from the trained model's tokenizer """ # We're now in the ae directory import pickle def check_tokenizer(): """Check the tokenizer vocabulary size.""" try: # Load the tokenizer with open('cutting_edge_model/tokenizer.pkl', 'rb') as f: tokenizer = pickle.load(f) print(f"Actual vocab size: {tokenizer.vocab_size}") print(f"Tokenizer type: {type(tokenizer)}") # Check some vocabulary items if hasattr(tokenizer, 'vocab'): print(f"Sample vocab items: {list(tokenizer.vocab.keys())[:10]}") return tokenizer.vocab_size except Exception as e: print(f"Error: {e}") return None if __name__ == "__main__": vocab_size = check_tokenizer() if vocab_size: print(f"\nThe correct vocab_size should be: {vocab_size}")