"""RAG trajectories: the thing the critic audits. A trajectory is one pass of a RAG system over a question, serialised with the tags the paper's prompts expect: ... ... Doc1: ... Doc2: ... ... ... Two choices here depart from the paper, both deliberate and both to fit an 8GB laptop instead of a cluster: **HotpotQA in the distractor setting, not a retrieval corpus.** The paper retrieves from HotpotWiki18 with E5-base-v2 over a FAISS index. The distractor split ships 10 paragraphs per question - 2 genuinely supporting, 8 plausible distractors - which is already a realistic retrieval result, complete with the failure mode that matters (relevant-looking evidence that does not support the answer). It removes a multi-million-passage index from the critical path and costs nothing the critic cares about. **Verdicts derived from gold answers, not purely from a teacher LLM.** The paper labels every field with a consensus of teacher critiques. Groq's free tier allows 1,000 requests per *day*, so labelling thousands of trajectories that way is not possible here. But HotpotQA has gold answers, so the *verdict* - the only field stage 1 (CJA) trains on - can be computed exactly and for free. The teacher is then needed only for location/reason/fix in stage 2, which is where the API budget goes. Ground truth is arguably better supervision than an LLM's opinion of it, but it is a deviation and is recorded as one. """ from __future__ import annotations import json import re import string from collections import Counter from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any, Iterator from src.critic.schema import Verdict # Trajectories are graded SUCCESSFUL / FAILED / AMBIGUOUS (Figure 2b). These # thresholds on answer F1 implement that split; the ambiguous band becomes # UNSURE, which is exactly the case the paper wants the critic to abstain on. F1_CORRECT = 0.80 F1_INCORRECT = 0.30 # --- answer scoring (standard SQuAD/HotpotQA normalisation) ----------------- def normalize_answer(s: str) -> str: s = (s or "").lower() s = "".join(ch for ch in s if ch not in set(string.punctuation)) s = re.sub(r"\b(a|an|the)\b", " ", s) return " ".join(s.split()) def exact_match(pred: str, gold: str) -> bool: return normalize_answer(pred) == normalize_answer(gold) def answer_f1(pred: str, gold: str) -> float: p, g = normalize_answer(pred).split(), normalize_answer(gold).split() if not p or not g: return float(p == g) # yes/no/noanswer are atomic - partial credit on them is meaningless if set(g) <= {"yes", "no", "noanswer"} or set(p) <= {"yes", "no", "noanswer"}: return float(p == g) common = Counter(p) & Counter(g) overlap = sum(common.values()) if overlap == 0: return 0.0 precision, recall = overlap / len(p), overlap / len(g) return 2 * precision * recall / (precision + recall) def verdict_from_answer(pred: str, gold: str) -> tuple[Verdict, float]: """Reference verdict, computed rather than asked for. Returns (verdict, f1). The UNSURE band is not a hedge: it marks trajectories where answer overlap genuinely cannot settle correctness, and teaching the critic to say UNSURE there is the point of having the label. """ f1 = answer_f1(pred, gold) if f1 >= F1_CORRECT: return Verdict.CORRECT, f1 if f1 <= F1_INCORRECT: return Verdict.INCORRECT, f1 return Verdict.UNSURE, f1 # --- data types ------------------------------------------------------------- @dataclass class Passage: """One retrieved paragraph. `index` is 1-based: Doc1 ... Doc10.""" index: int title: str text: str is_gold: bool = False # from HotpotQA supporting_facts def render(self) -> str: return f"Doc{self.index}({self.title}): {self.text}" @dataclass class Trajectory: """One RAG attempt, plus the labels derivable without an API call.""" qid: str question: str passages: list[Passage] answer: str think: str = "" search: str = "" gold_answer: str = "" gold_titles: list[str] = field(default_factory=list) level: str = "" meta: dict[str, Any] = field(default_factory=dict) # -- derived labels, free of charge -- @property def f1(self) -> float: return answer_f1(self.answer, self.gold_answer) @property def verdict(self) -> Verdict: return verdict_from_answer(self.answer, self.gold_answer)[0] @property def gold_indices(self) -> list[int]: return [p.index for p in self.passages if p.is_gold] def render(self, max_passage_chars: int = 600) -> str: """Serialise for the critic prompt, bounded so context stays predictable.""" docs = "\n".join( f"Doc{p.index}({p.title}): {p.text[:max_passage_chars]}" for p in self.passages ) parts = [f"{self.question}"] if self.search: parts.append(f"{self.search}") parts.append(f"\n{docs}\n") if self.think: parts.append(f"{self.think}") parts.append(f"{self.answer}") return "\n".join(parts) def to_json(self) -> dict[str, Any]: d = asdict(self) d["f1"] = round(self.f1, 4) d["reference_verdict"] = self.verdict.value return d @classmethod def from_json(cls, d: dict[str, Any]) -> "Trajectory": d = dict(d) d.pop("f1", None) d.pop("reference_verdict", None) d["passages"] = [Passage(**p) for p in d["passages"]] return cls(**d) # --- HotpotQA loading ------------------------------------------------------- def load_hotpotqa(split: str = "validation", limit: int | None = None, seed: int = 42) -> Iterator[dict[str, Any]]: """Stream HotpotQA distractor examples, shuffled deterministically. Streaming keeps the 600MB train split off disk when only a few hundred examples are needed. """ from datasets import load_dataset ds = load_dataset("hotpotqa/hotpot_qa", "distractor", split=split, streaming=True) ds = ds.shuffle(seed=seed, buffer_size=2000) for i, ex in enumerate(ds): if limit is not None and i >= limit: return yield ex def example_to_passages(ex: dict[str, Any]) -> list[Passage]: """Turn HotpotQA's context into Doc1..Doc10, flagging the supporting ones.""" ctx, sf = ex["context"], ex["supporting_facts"] titles = ctx["title"] if isinstance(ctx, dict) else [c[0] for c in ctx] sents = ctx["sentences"] if isinstance(ctx, dict) else [c[1] for c in ctx] gold = set(sf["title"] if isinstance(sf, dict) else [s[0] for s in sf]) return [ Passage(index=i + 1, title=t, text=" ".join(s).strip(), is_gold=t in gold) for i, (t, s) in enumerate(zip(titles, sents)) ] # --- persistence ------------------------------------------------------------ def write_trajectories(path: str | Path, trajectories: list[Trajectory]) -> Path: path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w", encoding="utf-8") as fh: for t in trajectories: fh.write(json.dumps(t.to_json(), ensure_ascii=False) + "\n") return path def read_trajectories(path: str | Path) -> list[Trajectory]: with open(path, encoding="utf-8") as fh: return [Trajectory.from_json(json.loads(line)) for line in fh if line.strip()] def verdict_distribution(trajectories: list[Trajectory]) -> dict[str, int]: """Label balance. A critic trained on 95% CORRECT learns to say CORRECT.""" counts = Counter(t.verdict.value for t in trajectories) return {v.value: counts.get(v.value, 0) for v in Verdict}