"""Align system claims/paragraphs to FELM gold segments via token overlap.""" from __future__ import annotations import re from typing import Any, Iterable from eval.types import GoldSegment _TOKEN_RE = re.compile(r"[a-z0-9]+", re.I) def tokenize(text: str) -> set[str]: return {t.lower() for t in _TOKEN_RE.findall(text or "")} def jaccard(a: set[str], b: set[str]) -> float: if not a and not b: return 1.0 if not a or not b: return 0.0 inter = len(a & b) union = len(a | b) return inter / union if union else 0.0 def claim_has_error(verdict: str) -> bool: return (verdict or "").strip().lower() == "contradicted" def align_claims_to_segments( claims: Iterable[dict[str, Any]], gold_segments: list[GoldSegment], *, min_jaccard: float = 0.5, ) -> list[dict[str, Any]]: """Best-overlap match each gold segment to a system claim. Returns one record per gold segment with fields: gold_has_error, pred_has_error, overlap, matched_claim_id (or None). """ claim_list = [c for c in claims if (c.get("text") or "").strip()] claim_tokens = [(c, tokenize(str(c.get("text") or ""))) for c in claim_list] rows: list[dict[str, Any]] = [] for seg in gold_segments: gold_text = str(seg.get("text") or "") gold_toks = tokenize(gold_text) best_i = -1 best_score = 0.0 for i, (_claim, toks) in enumerate(claim_tokens): score = jaccard(gold_toks, toks) if score > best_score: best_score = score best_i = i matched = claim_list[best_i] if best_i >= 0 and best_score >= min_jaccard else None pred = claim_has_error(str(matched.get("verdict") or "")) if matched else False rows.append( { "gold_text": gold_text, "gold_has_error": not bool(seg.get("label", True)), "pred_has_error": pred, "overlap": best_score, "matched_claim_id": (matched or {}).get("id"), "matched_claim_text": (matched or {}).get("text"), "matched_verdict": (matched or {}).get("verdict"), } ) return rows