Spaces:
Running on Zero
Running on Zero
| import math | |
| from collections import Counter | |
| from typing import List, Dict, Tuple, Optional | |
| class BleuScorer: | |
| """ | |
| BLEU (Bilingual Evaluation Understudy) Scorer implemented from scratch. | |
| Supports individual BLEU-1 to BLEU-4 and cumulative BLEU with Brevity Penalty. | |
| """ | |
| def _get_ngrams(tokens: List[str], n: int) -> Counter: | |
| if len(tokens) < n: | |
| return Counter() | |
| return Counter(tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)) | |
| def modified_precision(self, candidate_tokens: List[str], reference_tokens: List[str], n: int) -> Tuple[int, int]: | |
| """ | |
| Computes clipped n-gram counts and total candidate n-gram count. | |
| """ | |
| cand_ngrams = self._get_ngrams(candidate_tokens, n) | |
| ref_ngrams = self._get_ngrams(reference_tokens, n) | |
| total_cand = sum(cand_ngrams.values()) | |
| if total_cand == 0: | |
| return 0, 0 | |
| clipped_count = 0 | |
| for ngram, count in cand_ngrams.items(): | |
| if ngram in ref_ngrams: | |
| clipped_count += min(count, ref_ngrams[ngram]) | |
| return clipped_count, total_cand | |
| def brevity_penalty(self, c_len: int, r_len: int) -> float: | |
| """ | |
| Calculates Brevity Penalty (BP) to penalize overly short summaries. | |
| """ | |
| if c_len == 0: | |
| return 0.0 | |
| if c_len > r_len: | |
| return 1.0 | |
| return math.exp(1.0 - (r_len / c_len)) | |
| def evaluate(self, candidate_text: str, reference_text: str, max_order: int = 4) -> Dict[str, float]: | |
| """ | |
| Evaluates candidate summary against reference and returns individual | |
| and cumulative BLEU scores. | |
| """ | |
| cand_tokens = candidate_text.strip().split() | |
| ref_tokens = reference_text.strip().split() | |
| c_len = len(cand_tokens) | |
| r_len = len(ref_tokens) | |
| if c_len == 0 or r_len == 0: | |
| return { | |
| "bleu-1": 0.0, "bleu-2": 0.0, "bleu-3": 0.0, "bleu-4": 0.0, | |
| "bleu_cumulative": 0.0, "brevity_penalty": 0.0 | |
| } | |
| bp = self.brevity_penalty(c_len, r_len) | |
| precisions = [] | |
| results = {} | |
| for n in range(1, max_order + 1): | |
| clipped, total = self.modified_precision(cand_tokens, ref_tokens, n) | |
| p_n = (clipped / total) if total > 0 else 0.0 | |
| precisions.append(p_n) | |
| results[f"bleu-{n}"] = round(p_n, 4) | |
| # Cumulative BLEU-4 (geometric mean of precisions with smooth clipping) | |
| log_prec_sum = 0.0 | |
| weight = 1.0 / max_order | |
| for p_n in precisions: | |
| if p_n > 0: | |
| log_prec_sum += weight * math.log(p_n) | |
| else: | |
| # Smoothing factor for 0 precision on higher n-grams | |
| log_prec_sum += weight * math.log(1e-6) | |
| cumulative_bleu = bp * math.exp(log_prec_sum) | |
| results["bleu_cumulative"] = round(max(0.0, min(1.0, cumulative_bleu)), 4) | |
| results["brevity_penalty"] = round(bp, 4) | |
| return results | |