import math from collections import Counter from typing import List, Dict, Tuple, Optional class BleuScorer: """ BLEU (Bilingual Evaluation Understudy) Scorer implemented from scratch. Supports individual BLEU-1 to BLEU-4 and cumulative BLEU with Brevity Penalty. """ @staticmethod def _get_ngrams(tokens: List[str], n: int) -> Counter: if len(tokens) < n: return Counter() return Counter(tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)) def modified_precision(self, candidate_tokens: List[str], reference_tokens: List[str], n: int) -> Tuple[int, int]: """ Computes clipped n-gram counts and total candidate n-gram count. """ cand_ngrams = self._get_ngrams(candidate_tokens, n) ref_ngrams = self._get_ngrams(reference_tokens, n) total_cand = sum(cand_ngrams.values()) if total_cand == 0: return 0, 0 clipped_count = 0 for ngram, count in cand_ngrams.items(): if ngram in ref_ngrams: clipped_count += min(count, ref_ngrams[ngram]) return clipped_count, total_cand def brevity_penalty(self, c_len: int, r_len: int) -> float: """ Calculates Brevity Penalty (BP) to penalize overly short summaries. """ if c_len == 0: return 0.0 if c_len > r_len: return 1.0 return math.exp(1.0 - (r_len / c_len)) def evaluate(self, candidate_text: str, reference_text: str, max_order: int = 4) -> Dict[str, float]: """ Evaluates candidate summary against reference and returns individual and cumulative BLEU scores. """ cand_tokens = candidate_text.strip().split() ref_tokens = reference_text.strip().split() c_len = len(cand_tokens) r_len = len(ref_tokens) if c_len == 0 or r_len == 0: return { "bleu-1": 0.0, "bleu-2": 0.0, "bleu-3": 0.0, "bleu-4": 0.0, "bleu_cumulative": 0.0, "brevity_penalty": 0.0 } bp = self.brevity_penalty(c_len, r_len) precisions = [] results = {} for n in range(1, max_order + 1): clipped, total = self.modified_precision(cand_tokens, ref_tokens, n) p_n = (clipped / total) if total > 0 else 0.0 precisions.append(p_n) results[f"bleu-{n}"] = round(p_n, 4) # Cumulative BLEU-4 (geometric mean of precisions with smooth clipping) log_prec_sum = 0.0 weight = 1.0 / max_order for p_n in precisions: if p_n > 0: log_prec_sum += weight * math.log(p_n) else: # Smoothing factor for 0 precision on higher n-grams log_prec_sum += weight * math.log(1e-6) cumulative_bleu = bp * math.exp(log_prec_sum) results["bleu_cumulative"] = round(max(0.0, min(1.0, cumulative_bleu)), 4) results["brevity_penalty"] = round(bp, 4) return results