File size: 3,120 Bytes
9a46c96 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | import math
import pytest
import evaluation.self_bleu as self_bleu_module
from evaluation.self_bleu import SelfBleuConfig, _reference_lengths, compute_self_bleu
def test_self_bleu_detects_mode_collapse() -> None:
"""Catch an inverted or non-reference-based diversity score."""
collapsed = ["the same four words here"] * 8
diverse = [f"unique sentence number {i} token{i} extra{i}" for i in range(8)]
collapsed_result = compute_self_bleu(collapsed)
diverse_result = compute_self_bleu(diverse)
assert collapsed_result.score > 0.99
assert diverse_result.score < collapsed_result.score
assert 0.0 <= diverse_result.score <= 1.0
def test_self_bleu_has_explicit_deterministic_standard_conventions() -> None:
"""Catch hidden random reference sampling or unrecorded BLEU conventions."""
result = compute_self_bleu(
["a b c d e", "a b c d e", "q r s t u"], SelfBleuConfig()
)
assert result.ngram_order == 4
assert result.weights == pytest.approx((0.25, 0.25, 0.25, 0.25))
assert result.reference_rule == "all_other_samples"
assert result.smoothing == "chen_cherry_method1"
assert result.tokenization == "whitespace"
assert result.sample_count == 3
assert result.flm_revision == "a1918d5164e5038e37d0b7a4fb2010ce75b863b3"
assert result.flm_metric_path == "metrics.py"
assert result.flm_self_bleu_present is False
@pytest.mark.parametrize("texts", [[], ["only one sample"], ["ok", " "]])
def test_self_bleu_rejects_missing_reference_or_empty_tokens(texts) -> None:
"""Catch undefined BLEU rows being normalized into plausible scores."""
with pytest.raises(ValueError):
compute_self_bleu(texts)
def test_self_bleu_builds_each_row_order_counter_once(monkeypatch) -> None:
"""Catch O(N²·L) n-gram reconstruction on the 1,024-sample OWT matrix."""
original = self_bleu_module._ngrams
calls = []
def counted(tokens, order):
calls.append((tuple(tokens), order))
return original(tokens, order)
monkeypatch.setattr(self_bleu_module, "_ngrams", counted)
texts = [f"row {index} has five unique tokens{index}" for index in range(6)]
compute_self_bleu(texts)
assert len(calls) == len(texts) * 4
assert len(set(calls)) == len(calls)
def test_method1_keeps_zero_when_candidate_has_no_unigram_reference_match() -> None:
"""Catch smoothing the BLEU unigram gate contrary to standard sentence BLEU."""
result = compute_self_bleu(["a b c d", "w x y z"])
assert result.score == 0.0
def test_leave_one_out_clipping_uses_second_max_for_unique_top_owner() -> None:
"""Catch a candidate clipping its own repeated n-grams as a reference."""
result = compute_self_bleu(
["a a a a a", "a a", "b b b b b"],
SelfBleuConfig(ngram_order=1, weights=(1.0,)),
)
assert result.score == pytest.approx((2 / 5 + math.exp(-1.5)) / 3)
def test_equidistant_reference_length_tie_chooses_shorter() -> None:
"""Catch BLEU brevity penalty selecting the longer equidistant reference."""
assert _reference_lengths([4, 2, 6])[0] == 2
|