"""Deterministic span matcher — labels claims against evidence without LLM. This is the model-independent backbone of the verifier. It handles: 1. Exact/fuzzy substring matching → SUPPORTED 2. Absence/deferral detection → NOT_IN_EVIDENCE 3. No-match → UNSUPPORTED (needs LLM or stays as-is) Used both as: - Primary labeler for simple claims (works on any model size) - Post-processor to fix LLM mislabeling """ from __future__ import annotations import re from .schemas import EvidencePointer, EvidenceSpan, VerifiedClaim from .constants import STOP_WORDS from .inference_detector import detect_inference # ── Absence / deferral patterns ─────────────────────────────────────── # Claim text that says "evidence doesn't have X" ABSENCE_IN_CLAIM = re.compile( r"evidence does not (?:specify|mention|include|contain|provide|state)|" r"not (?:provided|mentioned|specified|stated|included)(?: in the evidence)?|" r"is not provided|is not mentioned|is not specified|" r"no (?:information|data|mention) (?:about|regarding|on|of)|" r"cannot (?:determine|confirm|verify) from|" r"not enough information|" r"does not contain information|" r"no evidence (?:about|for|of|regarding)|" r"is (?:unknown|unavailable|missing|absent)", re.IGNORECASE, ) # Evidence text that confirms info is missing/pending DEFERRAL_IN_EVIDENCE = re.compile( r"has not been finalized|" r"not (?:been )?finalized|" r"still being collected|" r"data is (?:still )?being|" r"please contact|" r"contact(?:ed)? (?:the |directly )?.{0,30}(?:desk|office|department)|" r"should be contacted|" r"not (?:yet )?(?:available|determined|decided|released|announced)|" r"pending|" r"under review|" r"to be (?:determined|announced|confirmed|decided)", re.IGNORECASE, ) _STOP_WORDS = STOP_WORDS def label_claim_against_spans( claim_text: str, spans: list[EvidenceSpan], ) -> tuple[str, EvidencePointer | None, str]: """Deterministically label a claim against evidence spans. Returns: (label, pointer_or_None, notes) Label is one of: SUPPORTED, NOT_IN_EVIDENCE, or empty string "" meaning "I don't know — let the LLM decide". """ ct = claim_text.strip() ct_lower = ct.lower().rstrip(".") # ── 1. Claim says evidence is absent → NOT_IN_EVIDENCE ──────────── if ABSENCE_IN_CLAIM.search(ct): return "NOT_IN_EVIDENCE", None, "claim states absence of evidence" # ── 2. Claim text matches a deferral pattern → NOT_IN_EVIDENCE ──── if DEFERRAL_IN_EVIDENCE.search(ct): return "NOT_IN_EVIDENCE", None, "claim describes pending/deferred info" # ── 3. Exact substring match in a span → SUPPORTED ──────────────── for span in spans: st_lower = span.text.lower() if len(ct_lower) >= 10 and ct_lower in st_lower: ptr = _make_pointer(span) return "SUPPORTED", ptr, "exact substring match" # ── 3b. Number + keyword match ──────────────────────────────────── # If claim has numbers, all match a span, and ≥1 keyword overlaps ct_nums = _extract_comparable_nums(ct) ct_words = _key_words(ct_lower) if ct_nums and ct_words: for span in spans: span_nums = _extract_comparable_nums(span.text) if ct_nums and ct_nums.issubset(span_nums): st_words = _key_words(span.text.lower()) if ct_words & st_words: ptr = _make_pointer(span) return "SUPPORTED", ptr, "number + keyword match" # ── 4. Fuzzy word overlap (≥80% of claim keywords in span) ──────── if ct_words and len(ct_words) >= 2: for span in spans: st_words = _key_words(span.text.lower()) overlap = len(ct_words & st_words) / len(ct_words) if overlap >= 0.80: # Consistency check: verify numbers/entities match if _numbers_consistent(ct, span.text): ptr = _make_pointer(span) return "SUPPORTED", ptr, f"fuzzy match ({overlap:.0%} keyword overlap)" else: return "", None, "fuzzy match but numbers inconsistent" # ── 5. Can't determine → let LLM decide ────────────────────────── return "", None, "" def relabel_claims( claims: list[VerifiedClaim], spans: list[EvidenceSpan], question: str = "", ) -> list[VerifiedClaim]: """Post-process LLM-labeled claims using deterministic checks. Rules (applied in order): 1. SUPPORTED + absence/deferral text → downgrade to NOT_IN_EVIDENCE 2. CONTRADICTS_EVIDENCE + absence text → downgrade to NOT_IN_EVIDENCE 3. NOT_IN_EVIDENCE/UNSUPPORTED + span match → upgrade to SUPPORTED 4. SUPPORTED + inference detected → downgrade to UNSUPPORTED """ for claim in claims: det_label, det_ptr, det_notes = label_claim_against_spans(claim.claim_text, spans) if claim.label == "SUPPORTED" and det_label == "NOT_IN_EVIDENCE": # LLM wrongly called an absence claim SUPPORTED claim.label = "NOT_IN_EVIDENCE" claim.evidence_pointers = [] claim.notes = (claim.notes or "") + f" [det: {det_notes}]" elif claim.label == "CONTRADICTS_EVIDENCE" and det_label == "NOT_IN_EVIDENCE": claim.label = "NOT_IN_EVIDENCE" claim.evidence_pointers = [] claim.notes = (claim.notes or "") + f" [det: absence claim, not contradiction]" elif claim.label in ("NOT_IN_EVIDENCE", "UNSUPPORTED") and det_label == "SUPPORTED" and det_ptr: claim.label = "SUPPORTED" claim.evidence_pointers = [det_ptr] claim.notes = (claim.notes or "") + f" [det: {det_notes}]" # Rule 4: inference detection — must come AFTER span match upgrades if claim.label == "SUPPORTED" and question: is_inf, inf_reason = detect_inference(claim.claim_text, question) if is_inf: claim.label = "UNSUPPORTED" claim.evidence_pointers = [] claim.notes = (claim.notes or "") + f" [det: inference — {inf_reason}]" return claims # ── Helpers ──────────────────────────────────────────────────────────── def _key_words(text: str) -> set[str]: return {w for w in re.findall(r"\b\w{4,}\b", text)} - _STOP_WORDS # ── Numeric/entity consistency ──────────────────────────────────────── _NUM_RE = re.compile(r"(? bool: """Check that numbers in the claim match numbers in the span. If the claim mentions a number that doesn't appear in the span, the fuzzy match is unreliable — could be a hallucinated value. If the claim has no numbers, skip the check (it's a text-only claim). """ claim_nums = _extract_comparable_nums(claim) if not claim_nums: return True # no numbers to check span_nums = _extract_comparable_nums(span) if not span_nums: return True # span has no numbers either, can't compare # Every number in the claim must appear in the span for cn in claim_nums: if cn not in span_nums: return False return True def _extract_comparable_nums(text: str) -> set[str]: """Extract normalized number strings for comparison.""" nums: set[str] = set() for m in _NUM_RE.finditer(text): # Normalize: strip $, commas, spaces n = m.group().strip() n = re.sub(r"[$,\s]", "", n) n = n.lower() if n: nums.add(n) return nums def _make_pointer(span: EvidenceSpan) -> EvidencePointer: return EvidencePointer( span_id=span.span_id, start_char=span.start_char, end_char=span.end_char, text_preview=span.text[:80], )