Download src/optimization/summary_metrics.py from Smart-Slate/scientific-watch-agent: direct link, hf CLI and curl.
- Browser
- Download file 16.9 kB
-
https://huggingface.co/spaces/Smart-Slate/scientific-watch-agent/resolve/main/src/optimization/summary_metrics.py
- Command line
-
hf download hf://spaces/Smart-Slate/scientific-watch-agent/src/optimization/summary_metrics.py
-
curl -L -o summary_metrics.py https://huggingface.co/spaces/Smart-Slate/scientific-watch-agent/resolve/main/src/optimization/summary_metrics.py
16.9 kB
| """Metrics for evaluating article summaries — used by ProTeGi. | |
| Three metric tiers, ordered by cost: | |
| Tier 1 — ROUGE (rouge-score library OR built-in fallback) | |
| Zero cost, instant. Measures word-overlap between the generated | |
| summary and a gold reference text (e.g. the PubMed abstract). | |
| - ROUGE-1 : unigram recall/precision/F1 | |
| - ROUGE-2 : bigram overlap (captures bigrams like "deep learning") | |
| - ROUGE-L : longest common subsequence (order-aware) | |
| Implementation priority: | |
| 1. rouge-score library (pip install rouge-score) — uses stemming | |
| 2. Built-in pure-Python fallback — no external dependency, | |
| slightly different scores but same ranking behaviour | |
| Tier 2 — LLM Judge (one LLM call per summary, ~$0.001/call) | |
| Asks an LLM to score two semantic dimensions: | |
| - faithfulness : is every claim grounded in the source article? | |
| - coverage : are the key findings captured? | |
| This catches hallucinations and omissions that ROUGE misses because | |
| ROUGE is purely lexical (synonym-blind, order-naive). | |
| Tier 3 — BERTScore (optional, heavy ~400 MB model) | |
| Contextual embedding similarity. Better than ROUGE for paraphrases | |
| but requires loading a transformer model. Imported lazily to avoid | |
| cost when unused. Omitted from the default composite. | |
| All public functions accept: | |
| generated : flat text produced by flatten_summary() | |
| reference : gold text (PubMed abstract, or any reference summary) | |
| article_text : source article body (required only for LLM Judge) | |
| llm : any LLMClient instance (required only for LLM Judge) | |
| Return value: dict[str, float | str] — metric names to values in [0, 1] | |
| """ | |
| from __future__ import annotations | |
| import logging | |
| import re | |
| from collections import Counter | |
| from typing import TYPE_CHECKING, Optional | |
| from pydantic import BaseModel, Field | |
| if TYPE_CHECKING: | |
| from src.llm.base import LLMClient | |
| from src.schemas import ArticleSummary | |
| logger = logging.getLogger(__name__) | |
| # ============================================================ | |
| # Pure-Python ROUGE fallback (no external dependencies) | |
| # ============================================================ | |
| # Used automatically when `rouge-score` is not installed. | |
| # Implements the same F1 formula: 2*P*R / (P+R) | |
| # ROUGE-1: unigram overlap | |
| # ROUGE-2: bigram overlap | |
| # ROUGE-L: longest common subsequence (LCS) | |
| def _rouge_tokenize(text: str) -> list[str]: | |
| """Lowercase + keep only alphanumeric tokens. | |
| Unlike the relevance-score tokenizer, we do NOT remove stop words here — | |
| ROUGE is meant to measure raw word overlap faithfully. | |
| """ | |
| return re.findall(r"[a-z0-9]+", text.lower()) | |
| def _ngrams(tokens: list[str], n: int) -> list[tuple[str, ...]]: | |
| """Return all n-grams from a token list.""" | |
| return [tuple(tokens[i: i + n]) for i in range(len(tokens) - n + 1)] | |
| def _overlap_f1(gen_items: list, ref_items: list) -> float: | |
| """Compute F1 of item overlap using Counter (handles duplicates correctly).""" | |
| if not gen_items or not ref_items: | |
| return 0.0 | |
| gen_c = Counter(gen_items) | |
| ref_c = Counter(ref_items) | |
| overlap = sum(min(gen_c[item], ref_c[item]) for item in gen_c) | |
| precision = overlap / len(gen_items) | |
| recall = overlap / len(ref_items) | |
| if precision + recall == 0.0: | |
| return 0.0 | |
| return 2.0 * precision * recall / (precision + recall) | |
| def _lcs_length(a: list[str], b: list[str]) -> int: | |
| """Compute length of the Longest Common Subsequence (DP, O(m*n)). | |
| Space-optimised: only keeps two rows in memory. | |
| For typical text inputs (< 500 tokens each) this is very fast. | |
| """ | |
| m, n = len(a), len(b) | |
| if m == 0 or n == 0: | |
| return 0 | |
| prev = [0] * (n + 1) | |
| curr = [0] * (n + 1) | |
| for i in range(1, m + 1): | |
| for j in range(1, n + 1): | |
| if a[i - 1] == b[j - 1]: | |
| curr[j] = prev[j - 1] + 1 | |
| else: | |
| curr[j] = max(curr[j - 1], prev[j]) | |
| prev, curr = curr, [0] * (n + 1) | |
| return prev[n] | |
| def _compute_rouge_builtin(generated: str, reference: str) -> dict[str, float]: | |
| """Pure-Python ROUGE-1, ROUGE-2, ROUGE-L. No external library required.""" | |
| gen_tok = _rouge_tokenize(generated) | |
| ref_tok = _rouge_tokenize(reference) | |
| if not gen_tok or not ref_tok: | |
| return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0} | |
| # ROUGE-1 | |
| r1 = _overlap_f1(gen_tok, ref_tok) | |
| # ROUGE-2 | |
| gen_bg = _ngrams(gen_tok, 2) | |
| ref_bg = _ngrams(ref_tok, 2) | |
| r2 = _overlap_f1(gen_bg, ref_bg) | |
| # ROUGE-L (LCS-based F1) | |
| lcs = _lcs_length(gen_tok, ref_tok) | |
| if lcs == 0: | |
| rl = 0.0 | |
| else: | |
| p = lcs / len(gen_tok) | |
| r = lcs / len(ref_tok) | |
| rl = 2.0 * p * r / (p + r) | |
| return { | |
| "rouge1": round(r1, 4), | |
| "rouge2": round(r2, 4), | |
| "rougeL": round(rl, 4), | |
| } | |
| # ============================================================ | |
| # Tier 1 — ROUGE (library-first, fallback to built-in) | |
| # ============================================================ | |
| def compute_rouge(generated: str, reference: str) -> dict[str, float]: | |
| """Compute ROUGE-1, ROUGE-2, ROUGE-L F1 scores. | |
| Tries the `rouge-score` library first (uses stemming for better accuracy). | |
| Falls back to the built-in pure-Python implementation automatically when | |
| `rouge-score` is not installed — no ImportError is ever raised. | |
| Args: | |
| generated : text generated by the Summarizer (via flatten_summary) | |
| reference : gold reference text (e.g. PubMed abstract) | |
| Returns: | |
| {"rouge1": float, "rouge2": float, "rougeL": float} | |
| Example: | |
| >>> scores = compute_rouge("deep learning model", "deep neural network") | |
| >>> 0.0 <= scores["rouge1"] <= 1.0 | |
| True | |
| """ | |
| if not generated or not reference: | |
| return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0} | |
| try: | |
| from rouge_score import rouge_scorer # type: ignore[import] | |
| scorer = rouge_scorer.RougeScorer( | |
| ["rouge1", "rouge2", "rougeL"], use_stemmer=True | |
| ) | |
| scores = scorer.score(reference, generated) | |
| return { | |
| "rouge1": round(scores["rouge1"].fmeasure, 4), | |
| "rouge2": round(scores["rouge2"].fmeasure, 4), | |
| "rougeL": round(scores["rougeL"].fmeasure, 4), | |
| } | |
| except ImportError: | |
| # rouge-score not installed — use the built-in pure-Python fallback. | |
| # Scores are slightly different (no stemming) but ranking is preserved. | |
| logger.debug("rouge-score not installed; using built-in ROUGE fallback") | |
| return _compute_rouge_builtin(generated, reference) | |
| # ============================================================ | |
| # Tier 2 — LLM Judge | |
| # ============================================================ | |
| class JudgeOutput(BaseModel): | |
| """Structured output from the LLM Judge (used in chat_structured).""" | |
| faithfulness: float = Field( | |
| ..., ge=0.0, le=1.0, | |
| description=( | |
| "Is the summary faithful to the source article? " | |
| "0.0 = contains fabricated information not in the article, " | |
| "1.0 = every statement is grounded in the article" | |
| ), | |
| ) | |
| coverage: float = Field( | |
| ..., ge=0.0, le=1.0, | |
| description=( | |
| "Does the summary cover the key points? " | |
| "0.0 = misses all key points (problem, method, results), " | |
| "1.0 = captures problem + method + main results + contributions" | |
| ), | |
| ) | |
| reasoning: str = Field( | |
| ..., | |
| description=( | |
| "Brief explanation of both scores in 1-3 sentences. " | |
| "Cite specific evidence from the article." | |
| ), | |
| ) | |
| _JUDGE_SYSTEM_PROMPT = """\ | |
| You are an expert scientific text evaluator. | |
| You will receive: | |
| 1. ARTICLE: the source scientific article text | |
| 2. SUMMARY: a structured summary generated by an AI | |
| Score the summary on two criteria, each from 0.0 to 1.0: | |
| faithfulness (is the summary accurate?): | |
| 0.0 = contains fabricated numbers, methods, or findings not in the article | |
| 0.5 = mostly accurate but has minor inaccuracies or vague claims | |
| 1.0 = every statement can be verified from the article text | |
| coverage (does the summary capture the essentials?): | |
| 0.0 = misses the research problem, method, and results entirely | |
| 0.5 = captures 1-2 key aspects but misses important findings or methods | |
| 1.0 = problem + method + main results + key contributions all present | |
| Be strict: a summary that only paraphrases the title without substance scores | |
| low on coverage. A summary that invents specific numbers scores low on faithfulness.""" | |
| def compute_llm_judge( | |
| generated: str, | |
| article_text: str, | |
| llm: "LLMClient", | |
| ) -> dict[str, float | str]: | |
| """Evaluate faithfulness and coverage using an LLM as judge. | |
| One LLM call per summary. Uses chat_structured to get reliable scores. | |
| Args: | |
| generated : flat text generated by flatten_summary() | |
| article_text : source article body (truncated to 3000 chars internally) | |
| llm : any LLMClient (gpt-4o-mini is recommended for cost) | |
| Returns: | |
| { | |
| "faithfulness" : float in [0, 1], | |
| "coverage" : float in [0, 1], | |
| "reasoning" : str, | |
| "judge_cost_usd": float, | |
| } | |
| """ | |
| from src.llm.base import Message # local import — avoids circular dependency | |
| # Truncate article to avoid blowing token budget | |
| article_snippet = article_text[:3000] | |
| user_text = ( | |
| f"ARTICLE:\n{article_snippet}\n\n" | |
| f"SUMMARY:\n{generated}" | |
| ) | |
| messages = [Message(role="user", content=user_text)] | |
| try: | |
| result, resp = llm.chat_structured( | |
| system=_JUDGE_SYSTEM_PROMPT, | |
| messages=messages, | |
| schema=JudgeOutput, | |
| temperature=0.0, | |
| max_tokens=512, | |
| ) | |
| return { | |
| "faithfulness": round(result.faithfulness, 4), | |
| "coverage": round(result.coverage, 4), | |
| "reasoning": result.reasoning, | |
| "judge_cost_usd": resp.cost_usd, | |
| } | |
| except Exception as exc: | |
| logger.warning("LLM Judge call failed: %s", exc) | |
| # Neutral fallback — does not penalize or reward | |
| return { | |
| "faithfulness": 0.5, | |
| "coverage": 0.5, | |
| "reasoning": f"[Judge error: {exc}]", | |
| "judge_cost_usd": 0.0, | |
| } | |
| # ============================================================ | |
| # Tier 3 — BERTScore (optional, lazy import) | |
| # ============================================================ | |
| def compute_bertscore( | |
| generated: str, | |
| reference: str, | |
| model_type: str = "distilbert-base-uncased", | |
| ) -> dict[str, float]: | |
| """Compute BERTScore F1 (optional tier, ~400 MB model download). | |
| Uses contextual embeddings to measure semantic similarity — better than | |
| ROUGE for paraphrases but much heavier. Lazily imported. | |
| Args: | |
| generated : flat generated text | |
| reference : gold reference text | |
| model_type : HuggingFace model for embeddings (default: distilbert) | |
| Returns: | |
| {"bertscore_f1": float} | |
| Raises: | |
| ImportError: if bert-score is not installed. | |
| """ | |
| try: | |
| from bert_score import score as bs_score # type: ignore[import] | |
| except ImportError as exc: | |
| raise ImportError( | |
| "bert-score is not installed. Run: pip install bert-score" | |
| ) from exc | |
| import warnings | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore") | |
| _P, _R, F1 = bs_score( | |
| [generated], [reference], | |
| model_type=model_type, | |
| verbose=False, | |
| ) | |
| return {"bertscore_f1": round(float(F1[0]), 4)} | |
| # ============================================================ | |
| # Aggregate: all metrics together | |
| # ============================================================ | |
| def compute_all_metrics( | |
| generated: str, | |
| reference: str, | |
| article_text: Optional[str] = None, | |
| llm: Optional["LLMClient"] = None, | |
| use_bertscore: bool = False, | |
| use_rouge: bool = True, | |
| ) -> dict[str, float | str]: | |
| """Compute all available metrics and return a composite score. | |
| ROUGE is computed by default but can be disabled with ``use_rouge=False``. | |
| LLM Judge is computed when `llm` and `article_text` are provided. | |
| BERTScore is computed when `use_bertscore=True` and bert-score installed. | |
| Composite score formula: | |
| - ROUGE + Judge : 0.40*ROUGE-L + 0.40*faithfulness + 0.20*coverage | |
| - Judge only : 0.60*faithfulness + 0.40*coverage | |
| (use when the reference text is a different format than the output, | |
| e.g. PubMed narrative abstract vs structured 6-field summary) | |
| - ROUGE only : mean(ROUGE-1, ROUGE-2, ROUGE-L) | |
| Args: | |
| generated : flat text produced by flatten_summary() | |
| reference : gold reference text (only used when use_rouge=True) | |
| article_text : source article body (required only for LLM Judge) | |
| llm : any LLMClient instance (required only for LLM Judge) | |
| use_bertscore : enable optional BERTScore tier | |
| use_rouge : set False to skip ROUGE entirely and rely on LLM Judge | |
| alone — useful when the reference format does not match | |
| the generated format (e.g. PubMed abstract vs structured | |
| summary). Requires llm + article_text to be provided. | |
| Returns: | |
| dict containing all individual metrics plus "composite" in [0, 1]. | |
| """ | |
| metrics: dict[str, float | str] = {} | |
| # --- Tier 1: ROUGE (skippable) --- | |
| if use_rouge: | |
| rouge = compute_rouge(generated, reference) | |
| metrics.update(rouge) | |
| else: | |
| # Fill with neutral zeros so downstream code (EvalResult) never KeyErrors | |
| metrics.update({"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0}) | |
| logger.debug("ROUGE disabled (use_rouge=False) — using LLM Judge only") | |
| # --- Tier 2: LLM Judge (when configured) --- | |
| if llm is not None and article_text is not None: | |
| judge = compute_llm_judge(generated, article_text, llm) | |
| metrics.update(judge) | |
| # --- Tier 3: BERTScore (optional) --- | |
| if use_bertscore: | |
| try: | |
| bs = compute_bertscore(generated, reference) | |
| metrics.update(bs) | |
| except ImportError: | |
| logger.warning("BERTScore skipped: bert-score not installed") | |
| # --- Composite --- | |
| has_judge = "faithfulness" in metrics | |
| if has_judge and use_rouge: | |
| # Full composite: ROUGE-L + Judge | |
| composite = ( | |
| 0.40 * float(metrics["rougeL"]) | |
| + 0.40 * float(metrics["faithfulness"]) | |
| + 0.20 * float(metrics["coverage"]) | |
| ) | |
| elif has_judge and not use_rouge: | |
| # Judge only: faithfulness weighted higher (main quality signal) | |
| composite = ( | |
| 0.60 * float(metrics["faithfulness"]) | |
| + 0.40 * float(metrics["coverage"]) | |
| ) | |
| else: | |
| # ROUGE only (no Judge) | |
| composite = ( | |
| float(metrics["rouge1"]) | |
| + float(metrics["rouge2"]) | |
| + float(metrics["rougeL"]) | |
| ) / 3.0 | |
| metrics["composite"] = round(composite, 4) | |
| return metrics | |
| # ============================================================ | |
| # Utility: flatten ArticleSummary to text | |
| # ============================================================ | |
| def flatten_summary(summary: "ArticleSummary") -> str: | |
| """Convert an ArticleSummary into a single text string for metric computation. | |
| Concatenates all non-null fields so ROUGE has enough substance to compare. | |
| Fields marked 'Not specified' (LLM placeholder) are excluded. | |
| Args: | |
| summary: ArticleSummary produced by the Summarizer agent. | |
| Returns: | |
| Multi-sentence text representing the full structured summary. | |
| Example: | |
| >>> # Produces: "Problem. Method. Results. Key: contrib1. contrib2." | |
| """ | |
| _NOT_SPECIFIED = "not specified" | |
| def _include(s: Optional[str]) -> bool: | |
| return bool(s) and s.lower().strip() != _NOT_SPECIFIED | |
| parts: list[str] = [] | |
| if _include(summary.problem): | |
| parts.append(summary.problem) | |
| if _include(summary.method): | |
| parts.append(summary.method) | |
| if _include(summary.results): | |
| parts.append(summary.results) | |
| if _include(summary.dataset): | |
| parts.append(f"Dataset: {summary.dataset}") | |
| if _include(summary.limitations): | |
| parts.append(f"Limitations: {summary.limitations}") | |
| if summary.key_contributions: | |
| for contrib in summary.key_contributions: | |
| if _include(contrib): | |
| parts.append(contrib) | |
| return " ".join(parts) | |