Download check_tokenizer.py from ChadlieOak/ae: direct link, hf CLI and curl.
- Browser
- Download file 956 Bytes
-
https://huggingface.co/ChadlieOak/ae/resolve/main/check_tokenizer.py
- Command line
-
hf download hf://ChadlieOak/ae/check_tokenizer.py
-
curl -L -o check_tokenizer.py https://huggingface.co/ChadlieOak/ae/resolve/main/check_tokenizer.py
956 Bytes
| #!/usr/bin/env python3 | |
| """ | |
| Check the actual vocabulary size from the trained model's tokenizer | |
| """ | |
| # We're now in the ae directory | |
| import pickle | |
| def check_tokenizer(): | |
| """Check the tokenizer vocabulary size.""" | |
| try: | |
| # Load the tokenizer | |
| with open('cutting_edge_model/tokenizer.pkl', 'rb') as f: | |
| tokenizer = pickle.load(f) | |
| print(f"Actual vocab size: {tokenizer.vocab_size}") | |
| print(f"Tokenizer type: {type(tokenizer)}") | |
| # Check some vocabulary items | |
| if hasattr(tokenizer, 'vocab'): | |
| print(f"Sample vocab items: {list(tokenizer.vocab.keys())[:10]}") | |
| return tokenizer.vocab_size | |
| except Exception as e: | |
| print(f"Error: {e}") | |
| return None | |
| if __name__ == "__main__": | |
| vocab_size = check_tokenizer() | |
| if vocab_size: | |
| print(f"\nThe correct vocab_size should be: {vocab_size}") |