Spaces:
Running on Zero
Running on Zero
File size: 3,055 Bytes
3d9ba5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | import math
from collections import Counter
from typing import List, Dict, Tuple, Optional
class BleuScorer:
"""
BLEU (Bilingual Evaluation Understudy) Scorer implemented from scratch.
Supports individual BLEU-1 to BLEU-4 and cumulative BLEU with Brevity Penalty.
"""
@staticmethod
def _get_ngrams(tokens: List[str], n: int) -> Counter:
if len(tokens) < n:
return Counter()
return Counter(tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1))
def modified_precision(self, candidate_tokens: List[str], reference_tokens: List[str], n: int) -> Tuple[int, int]:
"""
Computes clipped n-gram counts and total candidate n-gram count.
"""
cand_ngrams = self._get_ngrams(candidate_tokens, n)
ref_ngrams = self._get_ngrams(reference_tokens, n)
total_cand = sum(cand_ngrams.values())
if total_cand == 0:
return 0, 0
clipped_count = 0
for ngram, count in cand_ngrams.items():
if ngram in ref_ngrams:
clipped_count += min(count, ref_ngrams[ngram])
return clipped_count, total_cand
def brevity_penalty(self, c_len: int, r_len: int) -> float:
"""
Calculates Brevity Penalty (BP) to penalize overly short summaries.
"""
if c_len == 0:
return 0.0
if c_len > r_len:
return 1.0
return math.exp(1.0 - (r_len / c_len))
def evaluate(self, candidate_text: str, reference_text: str, max_order: int = 4) -> Dict[str, float]:
"""
Evaluates candidate summary against reference and returns individual
and cumulative BLEU scores.
"""
cand_tokens = candidate_text.strip().split()
ref_tokens = reference_text.strip().split()
c_len = len(cand_tokens)
r_len = len(ref_tokens)
if c_len == 0 or r_len == 0:
return {
"bleu-1": 0.0, "bleu-2": 0.0, "bleu-3": 0.0, "bleu-4": 0.0,
"bleu_cumulative": 0.0, "brevity_penalty": 0.0
}
bp = self.brevity_penalty(c_len, r_len)
precisions = []
results = {}
for n in range(1, max_order + 1):
clipped, total = self.modified_precision(cand_tokens, ref_tokens, n)
p_n = (clipped / total) if total > 0 else 0.0
precisions.append(p_n)
results[f"bleu-{n}"] = round(p_n, 4)
# Cumulative BLEU-4 (geometric mean of precisions with smooth clipping)
log_prec_sum = 0.0
weight = 1.0 / max_order
for p_n in precisions:
if p_n > 0:
log_prec_sum += weight * math.log(p_n)
else:
# Smoothing factor for 0 precision on higher n-grams
log_prec_sum += weight * math.log(1e-6)
cumulative_bleu = bp * math.exp(log_prec_sum)
results["bleu_cumulative"] = round(max(0.0, min(1.0, cumulative_bleu)), 4)
results["brevity_penalty"] = round(bp, 4)
return results
|