File size: 6,030 Bytes
784f572 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | """Deterministic leave-one-out Self-BLEU.
The pinned FLM commit ``a1918d5164e5038e37d0b7a4fb2010ce75b863b3``
contains generative perplexity and per-sample entropy in ``metrics.py`` but no
Self-BLEU implementation anywhere in that repository. Therefore this module
binds and reports an explicit standard convention instead of claiming a port:
whitespace tokens, BLEU-4 with equal weights, every other sample as references,
Chen & Cherry method-1 epsilon smoothing, and a mean sentence score in [0, 1].
"""
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass
from bisect import bisect_left
import math
from typing import Sequence
@dataclass(frozen=True)
class SelfBleuConfig:
ngram_order: int = 4
weights: tuple[float, ...] = (0.25, 0.25, 0.25, 0.25)
smoothing_epsilon: float = 0.1
def __post_init__(self) -> None:
if self.ngram_order < 1 or len(self.weights) != self.ngram_order:
raise ValueError("weights must match a positive n-gram order")
if any(weight <= 0 or not math.isfinite(weight) for weight in self.weights):
raise ValueError("Self-BLEU weights must be finite and positive")
if not math.isclose(math.fsum(self.weights), 1.0):
raise ValueError("Self-BLEU weights must sum to one")
if self.smoothing_epsilon <= 0 or not math.isfinite(self.smoothing_epsilon):
raise ValueError("smoothing epsilon must be finite and positive")
@dataclass(frozen=True)
class SelfBleuResult:
score: float
sample_count: int
ngram_order: int
weights: tuple[float, ...]
smoothing: str = "chen_cherry_method1"
smoothing_epsilon: float = 0.1
reference_rule: str = "all_other_samples"
tokenization: str = "whitespace"
normalization: str = "mean_sentence_bleu_0_1"
flm_revision: str = "a1918d5164e5038e37d0b7a4fb2010ce75b863b3"
flm_metric_path: str = "metrics.py"
flm_self_bleu_present: bool = False
def _ngrams(tokens: Sequence[str], order: int) -> Counter[tuple[str, ...]]:
return Counter(tuple(tokens[index : index + order]) for index in range(len(tokens) - order + 1))
def _global_maxima(
all_counts: Sequence[Counter[tuple[str, ...]]],
) -> dict[tuple[str, ...], tuple[int, int, int]]:
"""Map n-gram to (top count, unique top owner or -1, second count)."""
maxima: dict[tuple[str, ...], tuple[int, int, int]] = {}
for sample_index, counts in enumerate(all_counts):
for ngram, count in counts.items():
top, owner, second = maxima.get(ngram, (0, -1, 0))
if count > top:
maxima[ngram] = (count, sample_index, top)
elif count == top:
maxima[ngram] = (top, -1, top)
elif count > second:
maxima[ngram] = (top, owner, count)
return maxima
def _reference_lengths(lengths: Sequence[int]) -> list[int]:
frequencies = Counter(lengths)
unique = sorted(frequencies)
result = []
for candidate in lengths:
if frequencies[candidate] > 1:
result.append(candidate)
continue
position = bisect_left(unique, candidate)
choices = []
if position:
choices.append(unique[position - 1])
if position + 1 < len(unique):
choices.append(unique[position + 1])
result.append(min(choices, key=lambda length: (abs(length - candidate), length)))
return result
def compute_self_bleu(
texts: Sequence[str], config: SelfBleuConfig | None = None
) -> SelfBleuResult:
"""Average deterministic sentence BLEU against every other generated sample."""
if len(texts) < 2:
raise ValueError("Self-BLEU requires at least two samples")
rows: list[list[str]] = []
for index, text in enumerate(texts):
if type(text) is not str or not text.strip():
raise ValueError(f"Self-BLEU sample {index} has no whitespace tokens")
rows.append(text.split())
resolved = config or SelfBleuConfig()
weighted_log_precision = [0.0] * len(rows)
zero_unigram = [False] * len(rows)
for order, weight in enumerate(resolved.weights, start=1):
counts = [_ngrams(row, order) for row in rows]
maxima = _global_maxima(counts)
for candidate_index, candidate_counts in enumerate(counts):
denominator = sum(candidate_counts.values())
numerator = 0
for ngram, count in candidate_counts.items():
top, unique_owner, second = maxima[ngram]
reference_count = second if unique_owner == candidate_index else top
numerator += min(count, reference_count)
if order == 1 and numerator == 0:
zero_unigram[candidate_index] = True
continue
precision = (
resolved.smoothing_epsilon / max(denominator, 1)
if numerator == 0
else numerator / denominator
)
weighted_log_precision[candidate_index] += weight * math.log(precision)
reference_lengths = _reference_lengths([len(row) for row in rows])
scores = []
for index, row in enumerate(rows):
if zero_unigram[index]:
scores.append(0.0)
continue
candidate_length = len(row)
reference_length = reference_lengths[index]
brevity_penalty = (
1.0
if candidate_length > reference_length
else math.exp(1.0 - reference_length / candidate_length)
)
scores.append(brevity_penalty * math.exp(weighted_log_precision[index]))
score = math.fsum(scores) / len(scores)
if not math.isfinite(score) or score < 0 or score > 1 + 1e-12:
raise ValueError("Self-BLEU score is outside [0, 1]")
return SelfBleuResult(
score=min(score, 1.0),
sample_count=len(rows),
ngram_order=resolved.ngram_order,
weights=resolved.weights,
smoothing_epsilon=resolved.smoothing_epsilon,
)
|