| import sys; sys.path.insert(0, '.') | |
| from angstrom_nano.tokenizer import AngstromNanoTokenizer | |
| tok = AngstromNanoTokenizer() | |
| text = "def hello(): return 42" | |
| ids = tok.encode(text) | |
| decoded = tok.decode(ids) | |
| print(f"Text: {text}") | |
| print(f"Tokens: {ids[:10]}... ({len(ids)} total)") | |
| print(f"Decoded: {decoded}") | |
| print(f"Match: {text == decoded}") | |
| tok.save('angstrom_nano/tokenizer_config.json') | |
| print("Tokenizer PASSED") | |