File size: 784 Bytes
d7228c8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | """
Unit tests for TokenizerEvaluator.
"""
import pytest
from slm.tokenizer.bpe import BPETokenizer
from slm.tokenizer.evaluator import TokenizerEvaluator
def test_tokenizer_evaluator():
texts = [
"The quick brown fox jumps over the lazy dog.",
"Python and PyTorch for Small Language Models."
] * 5
tokenizer = BPETokenizer()
tokenizer.train_on_texts(texts, vocab_size=100)
evaluator = TokenizerEvaluator(tokenizer)
report = evaluator.full_evaluation(texts[:2])
assert "vocab_size" in report
assert "roundtrip" in report
assert "compression" in report
assert report["roundtrip"]["roundtrip_accuracy"] > 90.0
assert report["compression"]["bytes_per_token"] > 0
assert report["compression"]["unk_percentage"] == 0.0
|