File size: 418 Bytes
2d1ceea | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 | import sys; sys.path.insert(0, '.')
from angstrom_nano.tokenizer import AngstromNanoTokenizer
tok = AngstromNanoTokenizer()
text = "def hello(): return 42"
ids = tok.encode(text)
decoded = tok.decode(ids)
print(f"Text: {text}")
print(f"Tokens: {ids[:10]}... ({len(ids)} total)")
print(f"Decoded: {decoded}")
print(f"Match: {text == decoded}")
tok.save('angstrom_nano/tokenizer_config.json')
print("Tokenizer PASSED")
|