Spaces:
Running
Running
| """Mechanical resolver for self-describing citation ref_ids (see src/ref_id.py). | |
| A citation is a ref_id string like ``2021_..._Brake_Fluid_Check::clip::0002`` or | |
| ``manual_2021-ford-f-150::p214::0308``. Resolving one means: split on "::", | |
| look the result up in the evidence set actually retrieved for this query, and | |
| classify it into the three states the UI renders: | |
| green = resolves to a real retrieved unit AND that unit is the one that | |
| actually supports the claim (matches gold, when gold is known). | |
| yellow = resolves to a real unit that exists in the corpus/retrieved set, | |
| just not the one supporting this specific claim. | |
| red = does not resolve to anything retrieved at all -- fabricated. | |
| Two ways to get yellow-vs-green, because "the one supporting this claim" means | |
| different things with and without ground truth: | |
| - ``classify_citation`` -- eval mode. We have the benchmark's gold_clip_id (and | |
| optionally gold pages), so the verdict is exact: same source as gold = green, | |
| a different real source = yellow, unresolvable = red. This is what | |
| src/eval/faithfulness.py uses against AutoRepair-VideoQA. | |
| - ``live_classify_citation`` -- production mode. There is no gold for an | |
| arbitrary user query, so "supports this claim" is approximated by whether the | |
| cited unit's own component/task text meaningfully overlaps the claim text | |
| it's attached to. This is a heuristic, not a proof -- it catches the common | |
| failure (citing a real but off-topic unit) without needing another LLM call. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| from dataclasses import dataclass | |
| from typing import Literal | |
| Verdict = Literal["green", "yellow", "red"] | |
| _WORD_RE = re.compile(r"[a-z0-9一-鿿]{2,}") | |
| class ParsedRef: | |
| source: str # clip id (video/proc) or "manual_{pdf_stem}" (manual page) | |
| kind: str # "clip" | "proc" | "p{page}" | |
| locator: str # disambiguating index within that source+kind | |
| def parse_ref_id(ref_id: str) -> ParsedRef | None: | |
| if not ref_id or "::" not in ref_id: | |
| return None | |
| parts = ref_id.split("::") | |
| if len(parts) < 3 or not all(parts[:2]): | |
| return None | |
| return ParsedRef(source=parts[0], kind=parts[1], locator="::".join(parts[2:])) | |
| def classify_citation( | |
| ref_id: str, | |
| *, | |
| evidence_by_ref_id: dict[str, dict], | |
| gold_clip_id: str | None = None, | |
| gold_pages: set[int] | None = None, | |
| ) -> Verdict: | |
| parsed = parse_ref_id(ref_id) | |
| if parsed is None: | |
| return "red" | |
| evidence = evidence_by_ref_id.get(ref_id) | |
| if evidence is None: | |
| return "red" | |
| if gold_clip_id is None and not gold_pages: | |
| return "green" # no gold to compare against; "resolves" is all we can say | |
| if parsed.kind in ("clip", "proc"): | |
| return "green" if gold_clip_id is not None and parsed.source == gold_clip_id else "yellow" | |
| if parsed.kind.startswith("p") and gold_pages: | |
| page = evidence.get("page_start") | |
| return "green" if page in gold_pages else "yellow" | |
| return "yellow" | |
| def _words(text: str) -> set[str]: | |
| return set(_WORD_RE.findall((text or "").lower())) | |
| def live_classify_citation( | |
| ref_id: str, | |
| *, | |
| evidence_by_ref_id: dict[str, dict], | |
| claim_text: str, | |
| overlap_threshold: float = 0.12, | |
| ) -> Verdict: | |
| """Heuristic 3-state verdict for production queries (no gold available).""" | |
| parsed = parse_ref_id(ref_id) | |
| if parsed is None: | |
| return "red" | |
| evidence = evidence_by_ref_id.get(ref_id) | |
| if evidence is None: | |
| return "red" | |
| topic = " ".join(str(evidence.get(k) or "") for k in ("title", "component", "task")) | |
| topic_words = _words(topic) | |
| claim_words = _words(claim_text) | |
| if not topic_words or not claim_words: | |
| return "green" | |
| overlap = len(topic_words & claim_words) / len(topic_words) | |
| return "green" if overlap >= overlap_threshold else "yellow" | |