import math import pytest import evaluation.self_bleu as self_bleu_module from evaluation.self_bleu import SelfBleuConfig, _reference_lengths, compute_self_bleu def test_self_bleu_detects_mode_collapse() -> None: """Catch an inverted or non-reference-based diversity score.""" collapsed = ["the same four words here"] * 8 diverse = [f"unique sentence number {i} token{i} extra{i}" for i in range(8)] collapsed_result = compute_self_bleu(collapsed) diverse_result = compute_self_bleu(diverse) assert collapsed_result.score > 0.99 assert diverse_result.score < collapsed_result.score assert 0.0 <= diverse_result.score <= 1.0 def test_self_bleu_has_explicit_deterministic_standard_conventions() -> None: """Catch hidden random reference sampling or unrecorded BLEU conventions.""" result = compute_self_bleu( ["a b c d e", "a b c d e", "q r s t u"], SelfBleuConfig() ) assert result.ngram_order == 4 assert result.weights == pytest.approx((0.25, 0.25, 0.25, 0.25)) assert result.reference_rule == "all_other_samples" assert result.smoothing == "chen_cherry_method1" assert result.tokenization == "whitespace" assert result.sample_count == 3 assert result.flm_revision == "a1918d5164e5038e37d0b7a4fb2010ce75b863b3" assert result.flm_metric_path == "metrics.py" assert result.flm_self_bleu_present is False @pytest.mark.parametrize("texts", [[], ["only one sample"], ["ok", " "]]) def test_self_bleu_rejects_missing_reference_or_empty_tokens(texts) -> None: """Catch undefined BLEU rows being normalized into plausible scores.""" with pytest.raises(ValueError): compute_self_bleu(texts) def test_self_bleu_builds_each_row_order_counter_once(monkeypatch) -> None: """Catch O(N²·L) n-gram reconstruction on the 1,024-sample OWT matrix.""" original = self_bleu_module._ngrams calls = [] def counted(tokens, order): calls.append((tuple(tokens), order)) return original(tokens, order) monkeypatch.setattr(self_bleu_module, "_ngrams", counted) texts = [f"row {index} has five unique tokens{index}" for index in range(6)] compute_self_bleu(texts) assert len(calls) == len(texts) * 4 assert len(set(calls)) == len(calls) def test_method1_keeps_zero_when_candidate_has_no_unigram_reference_match() -> None: """Catch smoothing the BLEU unigram gate contrary to standard sentence BLEU.""" result = compute_self_bleu(["a b c d", "w x y z"]) assert result.score == 0.0 def test_leave_one_out_clipping_uses_second_max_for_unique_top_owner() -> None: """Catch a candidate clipping its own repeated n-grams as a reference.""" result = compute_self_bleu( ["a a a a a", "a a", "b b b b b"], SelfBleuConfig(ngram_order=1, weights=(1.0,)), ) assert result.score == pytest.approx((2 / 5 + math.exp(-1.5)) / 3) def test_equidistant_reference_length_tie_chooses_shorter() -> None: """Catch BLEU brevity penalty selecting the longer equidistant reference.""" assert _reference_lengths([4, 2, 6])[0] == 2