Spaces:
Running on Zero
Running on Zero
File size: 13,637 Bytes
e5c059f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 | """Localisation benchmark: can a critic find WHERE a wrong answer went wrong?
python scripts/critic_localization.py --arms base sft dqa groq
The ablation showed the critic's two jobs separate cleanly. Deciding *whether*
an answer is wrong measured AUC 0.447 - no signal at 1.5B. Saying *where* it
went wrong, given that it is wrong, beat chance by 2x. This script measures the
second job directly and compares every arm on identical inputs.
Protocol, identical for every arm:
* held-out HotpotQA trajectories (never seen in training) whose answer is
wrong and whose gold supporting documents are known
* the verdict is supplied - "<verdict>INCORRECT</verdict><location>" is
pre-filled - so only localisation is being tested, not judgement
* a hit = the critic names `information:DocK` where DocK is one of the two
documents that actually contain the answer
* greedy decoding for local models; temperature 0 for the API teacher
Reported: hit rate with a Wilson 95% CI, the random baseline, and an exact
McNemar test between every pair of arms. McNemar rather than comparing two
proportions because the arms answer the *same* 100 questions - a paired test
removes question difficulty and is far more sensitive at this sample size.
Each arm's raw outputs are cached, so a crash or a Groq rate limit costs only
the arm in progress.
"""
from __future__ import annotations
import argparse
import json
import math
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
import src.config # noqa: F401,E402 - loads .env for the Groq arm
from src.critic.prompts import CRITIC_INSTRUCTION, CRITIC_SYSTEM # noqa: E402
from src.critic.schema import ParseError, Stage, Verdict, parse_critique # noqa: E402
from src.critic.trajectory import Trajectory, read_trajectories # noqa: E402
PREFIX = "<verdict>INCORRECT</verdict><location>"
# --------------------------------------------------------------- scoring ----
def score(traj: Trajectory, text: str) -> dict:
"""Parse one critique and decide whether it localised the error."""
out = {"valid_format": False, "stage": None, "index": None, "hit": False}
try:
c = parse_critique(text, strict=True)
out["valid_format"] = True
except ParseError:
try:
c = parse_critique(text, strict=False)
except ParseError:
return out
if c.location is None:
return out
out["stage"] = c.location.stage.value
out["index"] = c.location.index
out["hit"] = (c.location.stage is Stage.INFORMATION
and c.location.index in traj.gold_indices)
return out
def wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float]:
if n == 0:
return 0.0, 0.0
p = k / n
den = 1 + z * z / n
centre = (p + z * z / (2 * n)) / den
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / den
return centre - half, centre + half
def mcnemar_exact(a_hits: list[bool], b_hits: list[bool]) -> tuple[int, int, float]:
"""Exact two-sided McNemar on paired outcomes.
Only discordant pairs carry information: b = A right & B wrong, c = the
reverse. Under H0 each discordant pair is a fair coin, so the p-value is
a binomial tail on min(b, c) out of b + c.
"""
b = sum(1 for x, y in zip(a_hits, b_hits) if x and not y)
c = sum(1 for x, y in zip(a_hits, b_hits) if y and not x)
n = b + c
if n == 0:
return b, c, 1.0
k = min(b, c)
tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
return b, c, min(1.0, 2 * tail)
# ------------------------------------------------------------ local arms ----
def run_local(arm: str, adapter: str | None, base: str, trajs: list[Trajectory],
max_passage_chars: int) -> list[str]:
import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
from src.critic.local_llm import release
tok = AutoTokenizer.from_pretrained(base)
tok.padding_side = "left"
tok.truncation_side = "left" # never cut the assistant cue off the end
if tok.pad_token is None:
tok.pad_token = tok.eos_token
model = AutoModelForCausalLM.from_pretrained(base, dtype=torch.bfloat16, device_map="cuda")
if adapter:
model = PeftModel.from_pretrained(model, adapter)
model.eval()
prompts = [
tok.apply_chat_template(
[{"role": "system", "content": CRITIC_SYSTEM},
{"role": "user", "content": f"{CRITIC_INSTRUCTION}\n\n{t.render(max_passage_chars)}"}],
tokenize=False, add_generation_prompt=True) + PREFIX
for t in trajs
]
outs: list[str] = []
t0 = time.time()
for i in range(0, len(prompts), 4):
enc = tok(prompts[i:i + 4], return_tensors="pt", padding=True, truncation=True,
max_length=1792).to(model.device)
with torch.inference_mode():
gen = model.generate(**enc, max_new_tokens=90, do_sample=False,
pad_token_id=tok.pad_token_id,
stop_strings=["</fix>"], tokenizer=tok)
outs.extend(PREFIX + s for s in tok.batch_decode(
gen[:, enc["input_ids"].shape[1]:], skip_special_tokens=True))
print(f" {arm}: {len(outs)}/{len(prompts)} "
f"({len(outs) / max(time.time() - t0, 1e-6) * 60:.0f}/min)", flush=True)
del model
release()
return outs
# -------------------------------------------------------------- API arm ----
GROQ_ARMS = {"groq": "openai/gpt-oss-120b", "groq20": "openai/gpt-oss-20b"}
def run_groq(trajs: list[Trajectory], max_passage_chars: int, cache: dict,
model: str = GROQ_ARMS["groq"]) -> list[str]:
"""An API model under identical conditions: same prompt, verdict pre-filled."""
from src.critic.supervise import Teacher
teacher = Teacher(model=model)
outs = []
for i, t in enumerate(trajs, 1):
if t.qid in cache:
outs.append(cache[t.qid])
continue
msgs = [
{"role": "system", "content": CRITIC_SYSTEM},
{"role": "user", "content":
f"{CRITIC_INSTRUCTION}\n\n{t.render(max_passage_chars)}\n\n"
"This answer is INCORRECT. Your output must begin with "
f"{PREFIX} and continue from there."},
]
try:
text = teacher.complete(msgs, temperature=0.0)
except Exception as exc: # noqa: BLE001
# One dropped TLS connection killed the first run at 42/100. Skip
# the item without caching it, so a re-run fills exactly the gaps.
print(f" groq: {t.qid} failed ({type(exc).__name__}); will retry next run")
time.sleep(5)
continue
# Normalise: the API model sometimes restates the prefix, sometimes not.
if not text.lstrip().startswith("<verdict>"):
text = PREFIX + text
cache[t.qid] = text
outs.append(text)
if i % 10 == 0:
print(f" groq: {i}/{len(trajs)}", flush=True)
cache_path = getattr(run_groq, "cache_path", None)
if cache_path is not None: # checkpoint so a crash loses <= 10 items
cache_path.write_text(json.dumps(run_groq.raw, ensure_ascii=False),
encoding="utf-8")
return outs
# ------------------------------------------------------------------ main ----
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--arms", nargs="+", default=["base", "sft", "dqa", "groq20", "groq"])
ap.add_argument("--base", default="Qwen/Qwen2.5-1.5B-Instruct")
ap.add_argument("--sft", default="models/critic/sft")
ap.add_argument("--dqa", default="models/critic/dqa_step100")
ap.add_argument("--trajectories", default="data/critic/trajectories_eval.jsonl")
ap.add_argument("--n", type=int, default=100)
ap.add_argument("--max-passage-chars", type=int, default=450)
ap.add_argument("--out", default="results/critic_localization.json")
ap.add_argument("--exclude-from", default=None,
help="drop the first --exclude-n items of this benchmark file, so a "
"bigger set can be scored only on questions never analysed before")
ap.add_argument("--exclude-n", type=int, default=100)
ap.add_argument("--cache", default=None, help="raw-output cache (default: <out>.raw.json)")
args = ap.parse_args()
def wrong_with_gold(path: str) -> list[Trajectory]:
return [t for t in read_trajectories(path)
if t.verdict is Verdict.INCORRECT and t.gold_indices]
trajs = wrong_with_gold(args.trajectories)[: args.n]
if args.exclude_from:
seen = {t.qid for t in wrong_with_gold(args.exclude_from)[: args.exclude_n]}
trajs = [t for t in trajs if t.qid not in seen]
baseline = sum(len(t.gold_indices) / len(t.passages) for t in trajs) / len(trajs)
print(f"[loc] {len(trajs)} held-out wrong answers with known gold documents")
print(f"[loc] random-guess baseline: {baseline:.1%}\n")
out_path = Path(args.out)
cache_path = Path(args.cache) if args.cache else out_path.with_suffix(".raw.json")
raw: dict = json.loads(cache_path.read_text(encoding="utf-8")) if cache_path.exists() else {}
adapters = {"base": None, "sft": args.sft, "dqa": args.dqa}
for arm in args.arms:
done = raw.get(arm, {})
if all(t.qid in done for t in trajs):
print(f"[loc] {arm}: cached")
continue
print(f"[loc] running {arm}")
if arm in GROQ_ARMS:
done = raw.setdefault(arm, {})
run_groq.cache_path, run_groq.raw = cache_path, raw
try:
run_groq(trajs, args.max_passage_chars, done, model=GROQ_ARMS[arm])
finally:
cache_path.write_text(json.dumps(raw, ensure_ascii=False), encoding="utf-8")
else:
if arm != "base" and not Path(adapters[arm]).exists():
print(f"[loc] skipping {arm}: {adapters[arm]} missing")
continue
outs = run_local(arm, adapters[arm], args.base, trajs, args.max_passage_chars)
raw[arm] = {t.qid: o for t, o in zip(trajs, outs)}
cache_path.write_text(json.dumps(raw, ensure_ascii=False), encoding="utf-8")
# ---- score ----
results: dict = {"n": len(trajs), "baseline": round(baseline, 4), "arms": {}}
hits_by_arm: dict[str, list[bool]] = {}
for arm in args.arms:
if arm not in raw:
continue
scored = [score(t, raw[arm].get(t.qid, "")) for t in trajs]
hits = [s["hit"] for s in scored]
hits_by_arm[arm] = hits
k = sum(hits)
lo, hi = wilson(k, len(trajs))
stages: dict[str, int] = {}
for s in scored:
if s["stage"]:
stages[s["stage"]] = stages.get(s["stage"], 0) + 1
# Overall hit rate rewards always naming a document. An API model that
# mostly blames the answer step instead (valid under the schema) scores
# low overall but can be precise when it does name one - so precision
# given a document was named is reported beside it.
named = sum(1 for s in scored if s["stage"] == Stage.INFORMATION.value)
nlo, nhi = wilson(k, named)
results["arms"][arm] = {
"hits": k, "hit_rate": round(k / len(trajs), 4),
"ci95": [round(lo, 4), round(hi, 4)],
"named_document": named,
"hit_given_named": round(k / max(named, 1), 4),
"hit_given_named_ci95": [round(nlo, 4), round(nhi, 4)],
"lift_over_chance": round((k / len(trajs)) / baseline, 2),
"valid_format": round(sum(s["valid_format"] for s in scored) / len(trajs), 4),
"stages": stages,
}
arms = list(hits_by_arm)
results["pairwise"] = {}
for i, a in enumerate(arms):
for b in arms[i + 1:]:
only_a, only_b, p = mcnemar_exact(hits_by_arm[a], hits_by_arm[b])
results["pairwise"][f"{a}_vs_{b}"] = {
"only_" + a: only_a, "only_" + b: only_b, "p_exact": round(p, 5)}
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text(json.dumps(results, indent=2), encoding="utf-8")
print(f"\n{'arm':<8}{'hit rate':>10}{'95% CI':>18}{'x chance':>10}{'format':>9}"
f"{'named doc':>11}{'hit|named':>11}")
print("-" * 77)
print(f"{'random':<8}{baseline:>10.1%}{'':>18}{1.0:>10.2f}")
for arm, r in results["arms"].items():
print(f"{arm:<8}{r['hit_rate']:>10.1%} [{r['ci95'][0]:.1%}, {r['ci95'][1]:.1%}]"
f"{r['lift_over_chance']:>10.2f}{r['valid_format']:>9.0%}"
f"{r['named_document']:>11}{r['hit_given_named']:>11.1%}")
print("\npaired comparisons (exact McNemar):")
for name, r in results["pairwise"].items():
vals = list(r.values())
print(f" {name:<16} discordant {vals[0]}/{vals[1]} p = {r['p_exact']:.4f}")
print(f"\n[loc] saved -> {out_path}")
if __name__ == "__main__":
main()
|