scientific-watch-agent / src /optimization /summary_metrics.py
Sadum1's picture
Upload folder using huggingface_hub
7376e73 verified
Raw History Blame Contribute Delete
16.9 kB
"""Metrics for evaluating article summaries — used by ProTeGi.
Three metric tiers, ordered by cost:
Tier 1 — ROUGE (rouge-score library OR built-in fallback)
Zero cost, instant. Measures word-overlap between the generated
summary and a gold reference text (e.g. the PubMed abstract).
- ROUGE-1 : unigram recall/precision/F1
- ROUGE-2 : bigram overlap (captures bigrams like "deep learning")
- ROUGE-L : longest common subsequence (order-aware)
Implementation priority:
1. rouge-score library (pip install rouge-score) — uses stemming
2. Built-in pure-Python fallback — no external dependency,
slightly different scores but same ranking behaviour
Tier 2 — LLM Judge (one LLM call per summary, ~$0.001/call)
Asks an LLM to score two semantic dimensions:
- faithfulness : is every claim grounded in the source article?
- coverage : are the key findings captured?
This catches hallucinations and omissions that ROUGE misses because
ROUGE is purely lexical (synonym-blind, order-naive).
Tier 3 — BERTScore (optional, heavy ~400 MB model)
Contextual embedding similarity. Better than ROUGE for paraphrases
but requires loading a transformer model. Imported lazily to avoid
cost when unused. Omitted from the default composite.
All public functions accept:
generated : flat text produced by flatten_summary()
reference : gold text (PubMed abstract, or any reference summary)
article_text : source article body (required only for LLM Judge)
llm : any LLMClient instance (required only for LLM Judge)
Return value: dict[str, float | str] — metric names to values in [0, 1]
"""
from __future__ import annotations
import logging
import re
from collections import Counter
from typing import TYPE_CHECKING, Optional
from pydantic import BaseModel, Field
if TYPE_CHECKING:
from src.llm.base import LLMClient
from src.schemas import ArticleSummary
logger = logging.getLogger(__name__)
# ============================================================
# Pure-Python ROUGE fallback (no external dependencies)
# ============================================================
# Used automatically when `rouge-score` is not installed.
# Implements the same F1 formula: 2*P*R / (P+R)
# ROUGE-1: unigram overlap
# ROUGE-2: bigram overlap
# ROUGE-L: longest common subsequence (LCS)
def _rouge_tokenize(text: str) -> list[str]:
"""Lowercase + keep only alphanumeric tokens.
Unlike the relevance-score tokenizer, we do NOT remove stop words here —
ROUGE is meant to measure raw word overlap faithfully.
"""
return re.findall(r"[a-z0-9]+", text.lower())
def _ngrams(tokens: list[str], n: int) -> list[tuple[str, ...]]:
"""Return all n-grams from a token list."""
return [tuple(tokens[i: i + n]) for i in range(len(tokens) - n + 1)]
def _overlap_f1(gen_items: list, ref_items: list) -> float:
"""Compute F1 of item overlap using Counter (handles duplicates correctly)."""
if not gen_items or not ref_items:
return 0.0
gen_c = Counter(gen_items)
ref_c = Counter(ref_items)
overlap = sum(min(gen_c[item], ref_c[item]) for item in gen_c)
precision = overlap / len(gen_items)
recall = overlap / len(ref_items)
if precision + recall == 0.0:
return 0.0
return 2.0 * precision * recall / (precision + recall)
def _lcs_length(a: list[str], b: list[str]) -> int:
"""Compute length of the Longest Common Subsequence (DP, O(m*n)).
Space-optimised: only keeps two rows in memory.
For typical text inputs (< 500 tokens each) this is very fast.
"""
m, n = len(a), len(b)
if m == 0 or n == 0:
return 0
prev = [0] * (n + 1)
curr = [0] * (n + 1)
for i in range(1, m + 1):
for j in range(1, n + 1):
if a[i - 1] == b[j - 1]:
curr[j] = prev[j - 1] + 1
else:
curr[j] = max(curr[j - 1], prev[j])
prev, curr = curr, [0] * (n + 1)
return prev[n]
def _compute_rouge_builtin(generated: str, reference: str) -> dict[str, float]:
"""Pure-Python ROUGE-1, ROUGE-2, ROUGE-L. No external library required."""
gen_tok = _rouge_tokenize(generated)
ref_tok = _rouge_tokenize(reference)
if not gen_tok or not ref_tok:
return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0}
# ROUGE-1
r1 = _overlap_f1(gen_tok, ref_tok)
# ROUGE-2
gen_bg = _ngrams(gen_tok, 2)
ref_bg = _ngrams(ref_tok, 2)
r2 = _overlap_f1(gen_bg, ref_bg)
# ROUGE-L (LCS-based F1)
lcs = _lcs_length(gen_tok, ref_tok)
if lcs == 0:
rl = 0.0
else:
p = lcs / len(gen_tok)
r = lcs / len(ref_tok)
rl = 2.0 * p * r / (p + r)
return {
"rouge1": round(r1, 4),
"rouge2": round(r2, 4),
"rougeL": round(rl, 4),
}
# ============================================================
# Tier 1 — ROUGE (library-first, fallback to built-in)
# ============================================================
def compute_rouge(generated: str, reference: str) -> dict[str, float]:
"""Compute ROUGE-1, ROUGE-2, ROUGE-L F1 scores.
Tries the `rouge-score` library first (uses stemming for better accuracy).
Falls back to the built-in pure-Python implementation automatically when
`rouge-score` is not installed — no ImportError is ever raised.
Args:
generated : text generated by the Summarizer (via flatten_summary)
reference : gold reference text (e.g. PubMed abstract)
Returns:
{"rouge1": float, "rouge2": float, "rougeL": float}
Example:
>>> scores = compute_rouge("deep learning model", "deep neural network")
>>> 0.0 <= scores["rouge1"] <= 1.0
True
"""
if not generated or not reference:
return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0}
try:
from rouge_score import rouge_scorer # type: ignore[import]
scorer = rouge_scorer.RougeScorer(
["rouge1", "rouge2", "rougeL"], use_stemmer=True
)
scores = scorer.score(reference, generated)
return {
"rouge1": round(scores["rouge1"].fmeasure, 4),
"rouge2": round(scores["rouge2"].fmeasure, 4),
"rougeL": round(scores["rougeL"].fmeasure, 4),
}
except ImportError:
# rouge-score not installed — use the built-in pure-Python fallback.
# Scores are slightly different (no stemming) but ranking is preserved.
logger.debug("rouge-score not installed; using built-in ROUGE fallback")
return _compute_rouge_builtin(generated, reference)
# ============================================================
# Tier 2 — LLM Judge
# ============================================================
class JudgeOutput(BaseModel):
"""Structured output from the LLM Judge (used in chat_structured)."""
faithfulness: float = Field(
..., ge=0.0, le=1.0,
description=(
"Is the summary faithful to the source article? "
"0.0 = contains fabricated information not in the article, "
"1.0 = every statement is grounded in the article"
),
)
coverage: float = Field(
..., ge=0.0, le=1.0,
description=(
"Does the summary cover the key points? "
"0.0 = misses all key points (problem, method, results), "
"1.0 = captures problem + method + main results + contributions"
),
)
reasoning: str = Field(
...,
description=(
"Brief explanation of both scores in 1-3 sentences. "
"Cite specific evidence from the article."
),
)
_JUDGE_SYSTEM_PROMPT = """\
You are an expert scientific text evaluator.
You will receive:
1. ARTICLE: the source scientific article text
2. SUMMARY: a structured summary generated by an AI
Score the summary on two criteria, each from 0.0 to 1.0:
faithfulness (is the summary accurate?):
0.0 = contains fabricated numbers, methods, or findings not in the article
0.5 = mostly accurate but has minor inaccuracies or vague claims
1.0 = every statement can be verified from the article text
coverage (does the summary capture the essentials?):
0.0 = misses the research problem, method, and results entirely
0.5 = captures 1-2 key aspects but misses important findings or methods
1.0 = problem + method + main results + key contributions all present
Be strict: a summary that only paraphrases the title without substance scores
low on coverage. A summary that invents specific numbers scores low on faithfulness."""
def compute_llm_judge(
generated: str,
article_text: str,
llm: "LLMClient",
) -> dict[str, float | str]:
"""Evaluate faithfulness and coverage using an LLM as judge.
One LLM call per summary. Uses chat_structured to get reliable scores.
Args:
generated : flat text generated by flatten_summary()
article_text : source article body (truncated to 3000 chars internally)
llm : any LLMClient (gpt-4o-mini is recommended for cost)
Returns:
{
"faithfulness" : float in [0, 1],
"coverage" : float in [0, 1],
"reasoning" : str,
"judge_cost_usd": float,
}
"""
from src.llm.base import Message # local import — avoids circular dependency
# Truncate article to avoid blowing token budget
article_snippet = article_text[:3000]
user_text = (
f"ARTICLE:\n{article_snippet}\n\n"
f"SUMMARY:\n{generated}"
)
messages = [Message(role="user", content=user_text)]
try:
result, resp = llm.chat_structured(
system=_JUDGE_SYSTEM_PROMPT,
messages=messages,
schema=JudgeOutput,
temperature=0.0,
max_tokens=512,
)
return {
"faithfulness": round(result.faithfulness, 4),
"coverage": round(result.coverage, 4),
"reasoning": result.reasoning,
"judge_cost_usd": resp.cost_usd,
}
except Exception as exc:
logger.warning("LLM Judge call failed: %s", exc)
# Neutral fallback — does not penalize or reward
return {
"faithfulness": 0.5,
"coverage": 0.5,
"reasoning": f"[Judge error: {exc}]",
"judge_cost_usd": 0.0,
}
# ============================================================
# Tier 3 — BERTScore (optional, lazy import)
# ============================================================
def compute_bertscore(
generated: str,
reference: str,
model_type: str = "distilbert-base-uncased",
) -> dict[str, float]:
"""Compute BERTScore F1 (optional tier, ~400 MB model download).
Uses contextual embeddings to measure semantic similarity — better than
ROUGE for paraphrases but much heavier. Lazily imported.
Args:
generated : flat generated text
reference : gold reference text
model_type : HuggingFace model for embeddings (default: distilbert)
Returns:
{"bertscore_f1": float}
Raises:
ImportError: if bert-score is not installed.
"""
try:
from bert_score import score as bs_score # type: ignore[import]
except ImportError as exc:
raise ImportError(
"bert-score is not installed. Run: pip install bert-score"
) from exc
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
_P, _R, F1 = bs_score(
[generated], [reference],
model_type=model_type,
verbose=False,
)
return {"bertscore_f1": round(float(F1[0]), 4)}
# ============================================================
# Aggregate: all metrics together
# ============================================================
def compute_all_metrics(
generated: str,
reference: str,
article_text: Optional[str] = None,
llm: Optional["LLMClient"] = None,
use_bertscore: bool = False,
use_rouge: bool = True,
) -> dict[str, float | str]:
"""Compute all available metrics and return a composite score.
ROUGE is computed by default but can be disabled with ``use_rouge=False``.
LLM Judge is computed when `llm` and `article_text` are provided.
BERTScore is computed when `use_bertscore=True` and bert-score installed.
Composite score formula:
- ROUGE + Judge : 0.40*ROUGE-L + 0.40*faithfulness + 0.20*coverage
- Judge only : 0.60*faithfulness + 0.40*coverage
(use when the reference text is a different format than the output,
e.g. PubMed narrative abstract vs structured 6-field summary)
- ROUGE only : mean(ROUGE-1, ROUGE-2, ROUGE-L)
Args:
generated : flat text produced by flatten_summary()
reference : gold reference text (only used when use_rouge=True)
article_text : source article body (required only for LLM Judge)
llm : any LLMClient instance (required only for LLM Judge)
use_bertscore : enable optional BERTScore tier
use_rouge : set False to skip ROUGE entirely and rely on LLM Judge
alone — useful when the reference format does not match
the generated format (e.g. PubMed abstract vs structured
summary). Requires llm + article_text to be provided.
Returns:
dict containing all individual metrics plus "composite" in [0, 1].
"""
metrics: dict[str, float | str] = {}
# --- Tier 1: ROUGE (skippable) ---
if use_rouge:
rouge = compute_rouge(generated, reference)
metrics.update(rouge)
else:
# Fill with neutral zeros so downstream code (EvalResult) never KeyErrors
metrics.update({"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0})
logger.debug("ROUGE disabled (use_rouge=False) — using LLM Judge only")
# --- Tier 2: LLM Judge (when configured) ---
if llm is not None and article_text is not None:
judge = compute_llm_judge(generated, article_text, llm)
metrics.update(judge)
# --- Tier 3: BERTScore (optional) ---
if use_bertscore:
try:
bs = compute_bertscore(generated, reference)
metrics.update(bs)
except ImportError:
logger.warning("BERTScore skipped: bert-score not installed")
# --- Composite ---
has_judge = "faithfulness" in metrics
if has_judge and use_rouge:
# Full composite: ROUGE-L + Judge
composite = (
0.40 * float(metrics["rougeL"])
+ 0.40 * float(metrics["faithfulness"])
+ 0.20 * float(metrics["coverage"])
)
elif has_judge and not use_rouge:
# Judge only: faithfulness weighted higher (main quality signal)
composite = (
0.60 * float(metrics["faithfulness"])
+ 0.40 * float(metrics["coverage"])
)
else:
# ROUGE only (no Judge)
composite = (
float(metrics["rouge1"])
+ float(metrics["rouge2"])
+ float(metrics["rougeL"])
) / 3.0
metrics["composite"] = round(composite, 4)
return metrics
# ============================================================
# Utility: flatten ArticleSummary to text
# ============================================================
def flatten_summary(summary: "ArticleSummary") -> str:
"""Convert an ArticleSummary into a single text string for metric computation.
Concatenates all non-null fields so ROUGE has enough substance to compare.
Fields marked 'Not specified' (LLM placeholder) are excluded.
Args:
summary: ArticleSummary produced by the Summarizer agent.
Returns:
Multi-sentence text representing the full structured summary.
Example:
>>> # Produces: "Problem. Method. Results. Key: contrib1. contrib2."
"""
_NOT_SPECIFIED = "not specified"
def _include(s: Optional[str]) -> bool:
return bool(s) and s.lower().strip() != _NOT_SPECIFIED
parts: list[str] = []
if _include(summary.problem):
parts.append(summary.problem)
if _include(summary.method):
parts.append(summary.method)
if _include(summary.results):
parts.append(summary.results)
if _include(summary.dataset):
parts.append(f"Dataset: {summary.dataset}")
if _include(summary.limitations):
parts.append(f"Limitations: {summary.limitations}")
if summary.key_contributions:
for contrib in summary.key_contributions:
if _include(contrib):
parts.append(contrib)
return " ".join(parts)