Spaces:
Sleeping
Sleeping
| """Align system claims/paragraphs to FELM gold segments via token overlap.""" | |
| from __future__ import annotations | |
| import re | |
| from typing import Any, Iterable | |
| from eval.types import GoldSegment | |
| _TOKEN_RE = re.compile(r"[a-z0-9]+", re.I) | |
| def tokenize(text: str) -> set[str]: | |
| return {t.lower() for t in _TOKEN_RE.findall(text or "")} | |
| def jaccard(a: set[str], b: set[str]) -> float: | |
| if not a and not b: | |
| return 1.0 | |
| if not a or not b: | |
| return 0.0 | |
| inter = len(a & b) | |
| union = len(a | b) | |
| return inter / union if union else 0.0 | |
| def claim_has_error(verdict: str) -> bool: | |
| return (verdict or "").strip().lower() == "contradicted" | |
| def align_claims_to_segments( | |
| claims: Iterable[dict[str, Any]], | |
| gold_segments: list[GoldSegment], | |
| *, | |
| min_jaccard: float = 0.5, | |
| ) -> list[dict[str, Any]]: | |
| """Best-overlap match each gold segment to a system claim. | |
| Returns one record per gold segment with fields: | |
| gold_has_error, pred_has_error, overlap, matched_claim_id (or None). | |
| """ | |
| claim_list = [c for c in claims if (c.get("text") or "").strip()] | |
| claim_tokens = [(c, tokenize(str(c.get("text") or ""))) for c in claim_list] | |
| rows: list[dict[str, Any]] = [] | |
| for seg in gold_segments: | |
| gold_text = str(seg.get("text") or "") | |
| gold_toks = tokenize(gold_text) | |
| best_i = -1 | |
| best_score = 0.0 | |
| for i, (_claim, toks) in enumerate(claim_tokens): | |
| score = jaccard(gold_toks, toks) | |
| if score > best_score: | |
| best_score = score | |
| best_i = i | |
| matched = claim_list[best_i] if best_i >= 0 and best_score >= min_jaccard else None | |
| pred = claim_has_error(str(matched.get("verdict") or "")) if matched else False | |
| rows.append( | |
| { | |
| "gold_text": gold_text, | |
| "gold_has_error": not bool(seg.get("label", True)), | |
| "pred_has_error": pred, | |
| "overlap": best_score, | |
| "matched_claim_id": (matched or {}).get("id"), | |
| "matched_claim_text": (matched or {}).get("text"), | |
| "matched_verdict": (matched or {}).get("verdict"), | |
| } | |
| ) | |
| return rows | |