"""Deterministic leave-one-out Self-BLEU. The pinned FLM commit ``a1918d5164e5038e37d0b7a4fb2010ce75b863b3`` contains generative perplexity and per-sample entropy in ``metrics.py`` but no Self-BLEU implementation anywhere in that repository. Therefore this module binds and reports an explicit standard convention instead of claiming a port: whitespace tokens, BLEU-4 with equal weights, every other sample as references, Chen & Cherry method-1 epsilon smoothing, and a mean sentence score in [0, 1]. """ from __future__ import annotations from collections import Counter from dataclasses import dataclass from bisect import bisect_left import math from typing import Sequence @dataclass(frozen=True) class SelfBleuConfig: ngram_order: int = 4 weights: tuple[float, ...] = (0.25, 0.25, 0.25, 0.25) smoothing_epsilon: float = 0.1 def __post_init__(self) -> None: if self.ngram_order < 1 or len(self.weights) != self.ngram_order: raise ValueError("weights must match a positive n-gram order") if any(weight <= 0 or not math.isfinite(weight) for weight in self.weights): raise ValueError("Self-BLEU weights must be finite and positive") if not math.isclose(math.fsum(self.weights), 1.0): raise ValueError("Self-BLEU weights must sum to one") if self.smoothing_epsilon <= 0 or not math.isfinite(self.smoothing_epsilon): raise ValueError("smoothing epsilon must be finite and positive") @dataclass(frozen=True) class SelfBleuResult: score: float sample_count: int ngram_order: int weights: tuple[float, ...] smoothing: str = "chen_cherry_method1" smoothing_epsilon: float = 0.1 reference_rule: str = "all_other_samples" tokenization: str = "whitespace" normalization: str = "mean_sentence_bleu_0_1" flm_revision: str = "a1918d5164e5038e37d0b7a4fb2010ce75b863b3" flm_metric_path: str = "metrics.py" flm_self_bleu_present: bool = False def _ngrams(tokens: Sequence[str], order: int) -> Counter[tuple[str, ...]]: return Counter(tuple(tokens[index : index + order]) for index in range(len(tokens) - order + 1)) def _global_maxima( all_counts: Sequence[Counter[tuple[str, ...]]], ) -> dict[tuple[str, ...], tuple[int, int, int]]: """Map n-gram to (top count, unique top owner or -1, second count).""" maxima: dict[tuple[str, ...], tuple[int, int, int]] = {} for sample_index, counts in enumerate(all_counts): for ngram, count in counts.items(): top, owner, second = maxima.get(ngram, (0, -1, 0)) if count > top: maxima[ngram] = (count, sample_index, top) elif count == top: maxima[ngram] = (top, -1, top) elif count > second: maxima[ngram] = (top, owner, count) return maxima def _reference_lengths(lengths: Sequence[int]) -> list[int]: frequencies = Counter(lengths) unique = sorted(frequencies) result = [] for candidate in lengths: if frequencies[candidate] > 1: result.append(candidate) continue position = bisect_left(unique, candidate) choices = [] if position: choices.append(unique[position - 1]) if position + 1 < len(unique): choices.append(unique[position + 1]) result.append(min(choices, key=lambda length: (abs(length - candidate), length))) return result def compute_self_bleu( texts: Sequence[str], config: SelfBleuConfig | None = None ) -> SelfBleuResult: """Average deterministic sentence BLEU against every other generated sample.""" if len(texts) < 2: raise ValueError("Self-BLEU requires at least two samples") rows: list[list[str]] = [] for index, text in enumerate(texts): if type(text) is not str or not text.strip(): raise ValueError(f"Self-BLEU sample {index} has no whitespace tokens") rows.append(text.split()) resolved = config or SelfBleuConfig() weighted_log_precision = [0.0] * len(rows) zero_unigram = [False] * len(rows) for order, weight in enumerate(resolved.weights, start=1): counts = [_ngrams(row, order) for row in rows] maxima = _global_maxima(counts) for candidate_index, candidate_counts in enumerate(counts): denominator = sum(candidate_counts.values()) numerator = 0 for ngram, count in candidate_counts.items(): top, unique_owner, second = maxima[ngram] reference_count = second if unique_owner == candidate_index else top numerator += min(count, reference_count) if order == 1 and numerator == 0: zero_unigram[candidate_index] = True continue precision = ( resolved.smoothing_epsilon / max(denominator, 1) if numerator == 0 else numerator / denominator ) weighted_log_precision[candidate_index] += weight * math.log(precision) reference_lengths = _reference_lengths([len(row) for row in rows]) scores = [] for index, row in enumerate(rows): if zero_unigram[index]: scores.append(0.0) continue candidate_length = len(row) reference_length = reference_lengths[index] brevity_penalty = ( 1.0 if candidate_length > reference_length else math.exp(1.0 - reference_length / candidate_length) ) scores.append(brevity_penalty * math.exp(weighted_log_precision[index])) score = math.fsum(scores) / len(scores) if not math.isfinite(score) or score < 0 or score > 1 + 1e-12: raise ValueError("Self-BLEU score is outside [0, 1]") return SelfBleuResult( score=min(score, 1.0), sample_count=len(rows), ngram_order=resolved.ngram_order, weights=resolved.weights, smoothing_epsilon=resolved.smoothing_epsilon, )