"""Structured failure diagnosis for low-confidence results (ResearchOS v2). v1's weakest moment is when its confidence gate fires. It then says "nothing answers that" - honest, and useless: no hint of what went wrong or what to try. v2 attaches a structured diagnosis in the format of CRITIC-R1 (arXiv 2605.29886): which retrieved passage let the search down, why, and a concrete query to run instead. How the two halves divide the work is the result of an experiment, not a design preference. Training CRITIC-R1 on this hardware split the critic's two jobs apart: deciding WHETHER an answer is wrong AUC 0.447 at 1.5B - no signal saying WHERE it went wrong 44-46% vs 21% chance (p < 1e-6) So the critic is never asked whether to intervene. That decision belongs to the existing mean-of-top-5 confidence gate, which already separates in-corpus from out-of-corpus queries cleanly (0.67-0.996 vs 0.001-0.19). The critic only diagnoses, with the verdict supplied - the same pre-filled protocol it was evaluated under. In production the diagnosis comes from gpt-oss on Groq rather than the fine-tuned 1.5B critic. Speed decides it: an API call answers in about two seconds, where the 1.5B model would take 15-30s on the Space's CPU. On accuracy the two are complementary rather than ranked: given the CRITIC-R1 prompt, gpt-oss-20b names a document only 17 times in 100 (it mostly blames the answer step), but is right 82% of those times; the trained critic always names one and is right 46%. The prompt below therefore restricts the location to a document or the query itself. scripts/critic_localization.py has the head-to-head; scripts/eval_v2_rescue.py measures this path end to end. The trained critic remains the measured research artifact. Safety rails, because this is optional and must never make v1 worse: * fires only when the gate does - confident results never pay for it * a hard wall-clock budget and a circuit breaker, as for live arXiv search * every diagnosis is parsed with the same strict schema the critic was trained on; one that fails to parse, or cites a document that was not shown, is suppressed rather than displayed """ from __future__ import annotations import os import re import threading import time from concurrent.futures import ThreadPoolExecutor from concurrent.futures import TimeoutError as FuturesTimeout from dataclasses import asdict, dataclass from typing import Any from src.critic.schema import ParseError, Stage, parse_critique DEFAULT_MODEL = "openai/gpt-oss-120b" _POOL = ThreadPoolExecutor(max_workers=2, thread_name_prefix="diagnose") _LOCK = threading.Lock() _BREAKER = {"open_until": 0.0} BREAKER_COOLDOWN_S = 120.0 # Prompt: CRITIC-R1's Figure 7(a) evidence rules, with the verdict supplied by # the gate and the fix constrained to a runnable query. "search: " is # the paper's own fix form (its Figure 7 example is "search: retrieve correct # entity"); requiring it makes the fix actionable rather than advisory. SYSTEM = """You diagnose why a research-paper search engine failed to find good evidence. The engine already decided its evidence is too weak to answer confidently. Your job is NOT to judge that - it is decided. Your job is to say WHERE the retrieval went wrong, WHY, and what to search for instead. Rules: - Use ONLY the documents shown. Do not invent facts about papers. - Prefer pointing at a specific document (information:DocK) when a retrieved document is off-topic or answers a different question than the one asked. - Use location "search" when the query itself was the problem (too vague, wrong terms). - The fix MUST be a new search query, written as: search: The query should use the technical terms a paper on this topic would actually use. - If the question is not about AI/ML research at all, use location "search", say so in , and write the fix as exactly: search: none""" INSTRUCTION = """Output (STRICT, one line, exactly this order, nothing else): INCORRECT......search: ... - location: search | information:DocK - <= 30 words, specific to the documents shown - must start with "search:" followed by a concrete query of 3-12 words""" @dataclass class Diagnosis: """What went wrong with a weak result, and what to try next.""" location: str location_title: str | None reason: str fix: str suggested_query: str | None model: str latency_ms: float def to_json(self) -> dict[str, Any]: return asdict(self) def _breaker_open() -> bool: with _LOCK: return time.monotonic() < _BREAKER["open_until"] def _trip(reason: str) -> None: with _LOCK: _BREAKER["open_until"] = time.monotonic() + BREAKER_COOLDOWN_S print(f"[diagnose] {reason} - pausing diagnoses for {BREAKER_COOLDOWN_S:.0f}s", flush=True) def render_documents(passages: list[dict[str, Any]], max_docs: int = 6, max_chars: int = 420) -> str: """Doc1..DocK in the trajectory format the critic was trained on.""" lines = [] for i, p in enumerate(passages[:max_docs], 1): title = (p.get("title") or "").strip() body = (p.get("text") or p.get("why") or p.get("abstract") or "").strip() body = re.sub(r"\s+", " ", body)[:max_chars] lines.append(f"Doc{i}({title}): {body}") return "\n".join(lines) if lines else "(no documents were retrieved)" def extract_query(fix: str, original: str) -> str | None: """Pull a runnable query out of a `search: ...` fix. Rejected when it is empty, absurdly long, or just the original question again - re-running the same query would be a placebo button. """ m = re.match(r"^\s*search\s*:\s*(.+)$", fix or "", re.IGNORECASE | re.S) if not m: return None query = m.group(1).strip().strip("\"'`").strip() query = re.sub(r"\s+", " ", query) words = query.split() if not 2 <= len(words) <= 20: return None norm = lambda s: re.sub(r"[^a-z0-9 ]", "", s.lower()).strip() # noqa: E731 if norm(query) == norm(original): return None return query def validate(text: str, passages: list[dict[str, Any]], question: str, model: str, latency_ms: float, max_docs: int = 6) -> Diagnosis | None: """Turn raw model output into a Diagnosis, or None if it is not trustworthy. Parsing reuses the exact schema the critic was trained and evaluated on. A citation of a document that was never shown is the critic hallucinating an address, so it is suppressed rather than displayed. """ try: c = parse_critique(text, strict=False) except ParseError: return None if c.location is None or not c.reason.strip(): return None shown = min(len(passages), max_docs) title = None if c.location.stage is Stage.INFORMATION: if c.location.index is None or not 1 <= c.location.index <= shown: return None title = (passages[c.location.index - 1].get("title") or "").strip() or None elif c.location.stage is not Stage.SEARCH: # think/answer stages do not exist in a retrieval failure - the engine # never reasoned over evidence it judged too weak to use. return None return Diagnosis( location=str(c.location), location_title=title, reason=c.reason.strip(), fix=c.fix.strip(), suggested_query=extract_query(c.fix, question), model=model, latency_ms=round(latency_ms, 1), ) def _call(question: str, passages: list[dict[str, Any]], answer: str | None, models: list[str]) -> tuple[str, str]: """Try each model in order; return (text, model_used). A quota rejection moves straight to the next model rather than failing. Groq's free tier caps each model separately - gpt-oss-120b at 200,000 tokens per DAY, which at ~1,500 tokens a diagnosis is only ~130 a day across every user. Measured the hard way: a benchmark exhausted it mid-run, after which every full-size request returned 429 for hours while the per-minute headers still looked healthy. Separate per-model budgets roughly double capacity, and the feature degrades rather than vanishing. """ from groq import Groq client = Groq(timeout=20.0, max_retries=0) user = (f"{INSTRUCTION}\n\n{question}\n" f"\n{render_documents(passages)}\n") if answer: user += f"\n{answer[:400]}" last: Exception | None = None for model in models: try: r = client.chat.completions.create( model=model, temperature=0.0, max_tokens=400, messages=[{"role": "system", "content": SYSTEM}, {"role": "user", "content": user}], # gpt-oss bills hidden reasoning against max_tokens before any # visible output; at default effort a long prompt burned the # whole budget and returned four characters. extra_body={"reasoning_effort": "low"}, ) return (r.choices[0].message.content or "").strip(), model except Exception as exc: # noqa: BLE001 last = exc if not any(k in str(exc).lower() for k in ("429", "rate", "quota")): raise # a real error, not a quota: do not mask it raise last or RuntimeError("no diagnosis model available") def diagnose(question: str, passages: list[dict[str, Any]], answer: str | None = None, *, model: str | list[str] = DEFAULT_MODEL, budget_s: float = 6.0) -> Diagnosis | None: """Diagnose a low-confidence result. Returns None whenever it cannot help. Never raises: a diagnosis is an enhancement, and a failure here must leave the response exactly as v1 would have returned it. """ if not os.environ.get("GROQ_API_KEY") or _breaker_open(): return None models = [model] if isinstance(model, str) else list(model) started = time.perf_counter() future = _POOL.submit(_call, question, passages, answer, models) try: text, used = future.result(timeout=budget_s) except FuturesTimeout: _trip(f"no answer within {budget_s:.0f}s") return None except Exception as exc: # noqa: BLE001 msg = str(exc).lower() if any(k in msg for k in ("429", "rate", "503", "overload")): _trip(f"{type(exc).__name__}: {str(exc)[:80]}") return None return validate(text, passages, question, used, (time.perf_counter() - started) * 1000.0)