"""Localisation benchmark: can a critic find WHERE a wrong answer went wrong?
python scripts/critic_localization.py --arms base sft dqa groq
The ablation showed the critic's two jobs separate cleanly. Deciding *whether*
an answer is wrong measured AUC 0.447 - no signal at 1.5B. Saying *where* it
went wrong, given that it is wrong, beat chance by 2x. This script measures the
second job directly and compares every arm on identical inputs.
Protocol, identical for every arm:
* held-out HotpotQA trajectories (never seen in training) whose answer is
wrong and whose gold supporting documents are known
* the verdict is supplied - "INCORRECT" is
pre-filled - so only localisation is being tested, not judgement
* a hit = the critic names `information:DocK` where DocK is one of the two
documents that actually contain the answer
* greedy decoding for local models; temperature 0 for the API teacher
Reported: hit rate with a Wilson 95% CI, the random baseline, and an exact
McNemar test between every pair of arms. McNemar rather than comparing two
proportions because the arms answer the *same* 100 questions - a paired test
removes question difficulty and is far more sensitive at this sample size.
Each arm's raw outputs are cached, so a crash or a Groq rate limit costs only
the arm in progress.
"""
from __future__ import annotations
import argparse
import json
import math
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
import src.config # noqa: F401,E402 - loads .env for the Groq arm
from src.critic.prompts import CRITIC_INSTRUCTION, CRITIC_SYSTEM # noqa: E402
from src.critic.schema import ParseError, Stage, Verdict, parse_critique # noqa: E402
from src.critic.trajectory import Trajectory, read_trajectories # noqa: E402
PREFIX = "INCORRECT"
# --------------------------------------------------------------- scoring ----
def score(traj: Trajectory, text: str) -> dict:
"""Parse one critique and decide whether it localised the error."""
out = {"valid_format": False, "stage": None, "index": None, "hit": False}
try:
c = parse_critique(text, strict=True)
out["valid_format"] = True
except ParseError:
try:
c = parse_critique(text, strict=False)
except ParseError:
return out
if c.location is None:
return out
out["stage"] = c.location.stage.value
out["index"] = c.location.index
out["hit"] = (c.location.stage is Stage.INFORMATION
and c.location.index in traj.gold_indices)
return out
def wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float]:
if n == 0:
return 0.0, 0.0
p = k / n
den = 1 + z * z / n
centre = (p + z * z / (2 * n)) / den
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / den
return centre - half, centre + half
def mcnemar_exact(a_hits: list[bool], b_hits: list[bool]) -> tuple[int, int, float]:
"""Exact two-sided McNemar on paired outcomes.
Only discordant pairs carry information: b = A right & B wrong, c = the
reverse. Under H0 each discordant pair is a fair coin, so the p-value is
a binomial tail on min(b, c) out of b + c.
"""
b = sum(1 for x, y in zip(a_hits, b_hits) if x and not y)
c = sum(1 for x, y in zip(a_hits, b_hits) if y and not x)
n = b + c
if n == 0:
return b, c, 1.0
k = min(b, c)
tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
return b, c, min(1.0, 2 * tail)
# ------------------------------------------------------------ local arms ----
def run_local(arm: str, adapter: str | None, base: str, trajs: list[Trajectory],
max_passage_chars: int) -> list[str]:
import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
from src.critic.local_llm import release
tok = AutoTokenizer.from_pretrained(base)
tok.padding_side = "left"
tok.truncation_side = "left" # never cut the assistant cue off the end
if tok.pad_token is None:
tok.pad_token = tok.eos_token
model = AutoModelForCausalLM.from_pretrained(base, dtype=torch.bfloat16, device_map="cuda")
if adapter:
model = PeftModel.from_pretrained(model, adapter)
model.eval()
prompts = [
tok.apply_chat_template(
[{"role": "system", "content": CRITIC_SYSTEM},
{"role": "user", "content": f"{CRITIC_INSTRUCTION}\n\n{t.render(max_passage_chars)}"}],
tokenize=False, add_generation_prompt=True) + PREFIX
for t in trajs
]
outs: list[str] = []
t0 = time.time()
for i in range(0, len(prompts), 4):
enc = tok(prompts[i:i + 4], return_tensors="pt", padding=True, truncation=True,
max_length=1792).to(model.device)
with torch.inference_mode():
gen = model.generate(**enc, max_new_tokens=90, do_sample=False,
pad_token_id=tok.pad_token_id,
stop_strings=[""], tokenizer=tok)
outs.extend(PREFIX + s for s in tok.batch_decode(
gen[:, enc["input_ids"].shape[1]:], skip_special_tokens=True))
print(f" {arm}: {len(outs)}/{len(prompts)} "
f"({len(outs) / max(time.time() - t0, 1e-6) * 60:.0f}/min)", flush=True)
del model
release()
return outs
# -------------------------------------------------------------- API arm ----
GROQ_ARMS = {"groq": "openai/gpt-oss-120b", "groq20": "openai/gpt-oss-20b"}
def run_groq(trajs: list[Trajectory], max_passage_chars: int, cache: dict,
model: str = GROQ_ARMS["groq"]) -> list[str]:
"""An API model under identical conditions: same prompt, verdict pre-filled."""
from src.critic.supervise import Teacher
teacher = Teacher(model=model)
outs = []
for i, t in enumerate(trajs, 1):
if t.qid in cache:
outs.append(cache[t.qid])
continue
msgs = [
{"role": "system", "content": CRITIC_SYSTEM},
{"role": "user", "content":
f"{CRITIC_INSTRUCTION}\n\n{t.render(max_passage_chars)}\n\n"
"This answer is INCORRECT. Your output must begin with "
f"{PREFIX} and continue from there."},
]
try:
text = teacher.complete(msgs, temperature=0.0)
except Exception as exc: # noqa: BLE001
# One dropped TLS connection killed the first run at 42/100. Skip
# the item without caching it, so a re-run fills exactly the gaps.
print(f" groq: {t.qid} failed ({type(exc).__name__}); will retry next run")
time.sleep(5)
continue
# Normalise: the API model sometimes restates the prefix, sometimes not.
if not text.lstrip().startswith(""):
text = PREFIX + text
cache[t.qid] = text
outs.append(text)
if i % 10 == 0:
print(f" groq: {i}/{len(trajs)}", flush=True)
cache_path = getattr(run_groq, "cache_path", None)
if cache_path is not None: # checkpoint so a crash loses <= 10 items
cache_path.write_text(json.dumps(run_groq.raw, ensure_ascii=False),
encoding="utf-8")
return outs
# ------------------------------------------------------------------ main ----
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--arms", nargs="+", default=["base", "sft", "dqa", "groq20", "groq"])
ap.add_argument("--base", default="Qwen/Qwen2.5-1.5B-Instruct")
ap.add_argument("--sft", default="models/critic/sft")
ap.add_argument("--dqa", default="models/critic/dqa_step100")
ap.add_argument("--trajectories", default="data/critic/trajectories_eval.jsonl")
ap.add_argument("--n", type=int, default=100)
ap.add_argument("--max-passage-chars", type=int, default=450)
ap.add_argument("--out", default="results/critic_localization.json")
ap.add_argument("--exclude-from", default=None,
help="drop the first --exclude-n items of this benchmark file, so a "
"bigger set can be scored only on questions never analysed before")
ap.add_argument("--exclude-n", type=int, default=100)
ap.add_argument("--cache", default=None, help="raw-output cache (default: .raw.json)")
args = ap.parse_args()
def wrong_with_gold(path: str) -> list[Trajectory]:
return [t for t in read_trajectories(path)
if t.verdict is Verdict.INCORRECT and t.gold_indices]
trajs = wrong_with_gold(args.trajectories)[: args.n]
if args.exclude_from:
seen = {t.qid for t in wrong_with_gold(args.exclude_from)[: args.exclude_n]}
trajs = [t for t in trajs if t.qid not in seen]
baseline = sum(len(t.gold_indices) / len(t.passages) for t in trajs) / len(trajs)
print(f"[loc] {len(trajs)} held-out wrong answers with known gold documents")
print(f"[loc] random-guess baseline: {baseline:.1%}\n")
out_path = Path(args.out)
cache_path = Path(args.cache) if args.cache else out_path.with_suffix(".raw.json")
raw: dict = json.loads(cache_path.read_text(encoding="utf-8")) if cache_path.exists() else {}
adapters = {"base": None, "sft": args.sft, "dqa": args.dqa}
for arm in args.arms:
done = raw.get(arm, {})
if all(t.qid in done for t in trajs):
print(f"[loc] {arm}: cached")
continue
print(f"[loc] running {arm}")
if arm in GROQ_ARMS:
done = raw.setdefault(arm, {})
run_groq.cache_path, run_groq.raw = cache_path, raw
try:
run_groq(trajs, args.max_passage_chars, done, model=GROQ_ARMS[arm])
finally:
cache_path.write_text(json.dumps(raw, ensure_ascii=False), encoding="utf-8")
else:
if arm != "base" and not Path(adapters[arm]).exists():
print(f"[loc] skipping {arm}: {adapters[arm]} missing")
continue
outs = run_local(arm, adapters[arm], args.base, trajs, args.max_passage_chars)
raw[arm] = {t.qid: o for t, o in zip(trajs, outs)}
cache_path.write_text(json.dumps(raw, ensure_ascii=False), encoding="utf-8")
# ---- score ----
results: dict = {"n": len(trajs), "baseline": round(baseline, 4), "arms": {}}
hits_by_arm: dict[str, list[bool]] = {}
for arm in args.arms:
if arm not in raw:
continue
scored = [score(t, raw[arm].get(t.qid, "")) for t in trajs]
hits = [s["hit"] for s in scored]
hits_by_arm[arm] = hits
k = sum(hits)
lo, hi = wilson(k, len(trajs))
stages: dict[str, int] = {}
for s in scored:
if s["stage"]:
stages[s["stage"]] = stages.get(s["stage"], 0) + 1
# Overall hit rate rewards always naming a document. An API model that
# mostly blames the answer step instead (valid under the schema) scores
# low overall but can be precise when it does name one - so precision
# given a document was named is reported beside it.
named = sum(1 for s in scored if s["stage"] == Stage.INFORMATION.value)
nlo, nhi = wilson(k, named)
results["arms"][arm] = {
"hits": k, "hit_rate": round(k / len(trajs), 4),
"ci95": [round(lo, 4), round(hi, 4)],
"named_document": named,
"hit_given_named": round(k / max(named, 1), 4),
"hit_given_named_ci95": [round(nlo, 4), round(nhi, 4)],
"lift_over_chance": round((k / len(trajs)) / baseline, 2),
"valid_format": round(sum(s["valid_format"] for s in scored) / len(trajs), 4),
"stages": stages,
}
arms = list(hits_by_arm)
results["pairwise"] = {}
for i, a in enumerate(arms):
for b in arms[i + 1:]:
only_a, only_b, p = mcnemar_exact(hits_by_arm[a], hits_by_arm[b])
results["pairwise"][f"{a}_vs_{b}"] = {
"only_" + a: only_a, "only_" + b: only_b, "p_exact": round(p, 5)}
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text(json.dumps(results, indent=2), encoding="utf-8")
print(f"\n{'arm':<8}{'hit rate':>10}{'95% CI':>18}{'x chance':>10}{'format':>9}"
f"{'named doc':>11}{'hit|named':>11}")
print("-" * 77)
print(f"{'random':<8}{baseline:>10.1%}{'':>18}{1.0:>10.2f}")
for arm, r in results["arms"].items():
print(f"{arm:<8}{r['hit_rate']:>10.1%} [{r['ci95'][0]:.1%}, {r['ci95'][1]:.1%}]"
f"{r['lift_over_chance']:>10.2f}{r['valid_format']:>9.0%}"
f"{r['named_document']:>11}{r['hit_given_named']:>11.1%}")
print("\npaired comparisons (exact McNemar):")
for name, r in results["pairwise"].items():
vals = list(r.values())
print(f" {name:<16} discordant {vals[0]}/{vals[1]} p = {r['p_exact']:.4f}")
print(f"\n[loc] saved -> {out_path}")
if __name__ == "__main__":
main()