"""
Unit tests for TokenizerEvaluator.
"""

import pytest
from slm.tokenizer.bpe import BPETokenizer
from slm.tokenizer.evaluator import TokenizerEvaluator


def test_tokenizer_evaluator():
    texts = [
        "The quick brown fox jumps over the lazy dog.",
        "Python and PyTorch for Small Language Models."
    ] * 5

    tokenizer = BPETokenizer()
    tokenizer.train_on_texts(texts, vocab_size=100)

    evaluator = TokenizerEvaluator(tokenizer)
    report = evaluator.full_evaluation(texts[:2])

    assert "vocab_size" in report
    assert "roundtrip" in report
    assert "compression" in report
    assert report["roundtrip"]["roundtrip_accuracy"] > 90.0
    assert report["compression"]["bytes_per_token"] > 0
    assert report["compression"]["unk_percentage"] == 0.0