File size: 13,637 Bytes
e5c059f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
"""Localisation benchmark: can a critic find WHERE a wrong answer went wrong?



    python scripts/critic_localization.py --arms base sft dqa groq



The ablation showed the critic's two jobs separate cleanly. Deciding *whether*

an answer is wrong measured AUC 0.447 - no signal at 1.5B. Saying *where* it

went wrong, given that it is wrong, beat chance by 2x. This script measures the

second job directly and compares every arm on identical inputs.



Protocol, identical for every arm:

  * held-out HotpotQA trajectories (never seen in training) whose answer is

    wrong and whose gold supporting documents are known

  * the verdict is supplied - "<verdict>INCORRECT</verdict><location>" is

    pre-filled - so only localisation is being tested, not judgement

  * a hit = the critic names `information:DocK` where DocK is one of the two

    documents that actually contain the answer

  * greedy decoding for local models; temperature 0 for the API teacher



Reported: hit rate with a Wilson 95% CI, the random baseline, and an exact

McNemar test between every pair of arms. McNemar rather than comparing two

proportions because the arms answer the *same* 100 questions - a paired test

removes question difficulty and is far more sensitive at this sample size.



Each arm's raw outputs are cached, so a crash or a Groq rate limit costs only

the arm in progress.

"""

from __future__ import annotations

import argparse
import json
import math
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")

import src.config  # noqa: F401,E402  - loads .env for the Groq arm

from src.critic.prompts import CRITIC_INSTRUCTION, CRITIC_SYSTEM  # noqa: E402
from src.critic.schema import ParseError, Stage, Verdict, parse_critique  # noqa: E402
from src.critic.trajectory import Trajectory, read_trajectories  # noqa: E402

PREFIX = "<verdict>INCORRECT</verdict><location>"


# --------------------------------------------------------------- scoring ----
def score(traj: Trajectory, text: str) -> dict:
    """Parse one critique and decide whether it localised the error."""
    out = {"valid_format": False, "stage": None, "index": None, "hit": False}
    try:
        c = parse_critique(text, strict=True)
        out["valid_format"] = True
    except ParseError:
        try:
            c = parse_critique(text, strict=False)
        except ParseError:
            return out
    if c.location is None:
        return out
    out["stage"] = c.location.stage.value
    out["index"] = c.location.index
    out["hit"] = (c.location.stage is Stage.INFORMATION
                  and c.location.index in traj.gold_indices)
    return out


def wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float]:
    if n == 0:
        return 0.0, 0.0
    p = k / n
    den = 1 + z * z / n
    centre = (p + z * z / (2 * n)) / den
    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / den
    return centre - half, centre + half


def mcnemar_exact(a_hits: list[bool], b_hits: list[bool]) -> tuple[int, int, float]:
    """Exact two-sided McNemar on paired outcomes.



    Only discordant pairs carry information: b = A right & B wrong, c = the

    reverse. Under H0 each discordant pair is a fair coin, so the p-value is

    a binomial tail on min(b, c) out of b + c.

    """
    b = sum(1 for x, y in zip(a_hits, b_hits) if x and not y)
    c = sum(1 for x, y in zip(a_hits, b_hits) if y and not x)
    n = b + c
    if n == 0:
        return b, c, 1.0
    k = min(b, c)
    tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
    return b, c, min(1.0, 2 * tail)


# ------------------------------------------------------------ local arms ----
def run_local(arm: str, adapter: str | None, base: str, trajs: list[Trajectory],

              max_passage_chars: int) -> list[str]:
    import torch
    from peft import PeftModel
    from transformers import AutoModelForCausalLM, AutoTokenizer

    from src.critic.local_llm import release

    tok = AutoTokenizer.from_pretrained(base)
    tok.padding_side = "left"
    tok.truncation_side = "left"     # never cut the assistant cue off the end
    if tok.pad_token is None:
        tok.pad_token = tok.eos_token
    model = AutoModelForCausalLM.from_pretrained(base, dtype=torch.bfloat16, device_map="cuda")
    if adapter:
        model = PeftModel.from_pretrained(model, adapter)
    model.eval()

    prompts = [
        tok.apply_chat_template(
            [{"role": "system", "content": CRITIC_SYSTEM},
             {"role": "user", "content": f"{CRITIC_INSTRUCTION}\n\n{t.render(max_passage_chars)}"}],
            tokenize=False, add_generation_prompt=True) + PREFIX
        for t in trajs
    ]
    outs: list[str] = []
    t0 = time.time()
    for i in range(0, len(prompts), 4):
        enc = tok(prompts[i:i + 4], return_tensors="pt", padding=True, truncation=True,
                  max_length=1792).to(model.device)
        with torch.inference_mode():
            gen = model.generate(**enc, max_new_tokens=90, do_sample=False,
                                 pad_token_id=tok.pad_token_id,
                                 stop_strings=["</fix>"], tokenizer=tok)
        outs.extend(PREFIX + s for s in tok.batch_decode(
            gen[:, enc["input_ids"].shape[1]:], skip_special_tokens=True))
        print(f"    {arm}: {len(outs)}/{len(prompts)}  "
              f"({len(outs) / max(time.time() - t0, 1e-6) * 60:.0f}/min)", flush=True)
    del model
    release()
    return outs


# -------------------------------------------------------------- API arm ----
GROQ_ARMS = {"groq": "openai/gpt-oss-120b", "groq20": "openai/gpt-oss-20b"}


def run_groq(trajs: list[Trajectory], max_passage_chars: int, cache: dict,

             model: str = GROQ_ARMS["groq"]) -> list[str]:
    """An API model under identical conditions: same prompt, verdict pre-filled."""
    from src.critic.supervise import Teacher

    teacher = Teacher(model=model)
    outs = []
    for i, t in enumerate(trajs, 1):
        if t.qid in cache:
            outs.append(cache[t.qid])
            continue
        msgs = [
            {"role": "system", "content": CRITIC_SYSTEM},
            {"role": "user", "content":
                f"{CRITIC_INSTRUCTION}\n\n{t.render(max_passage_chars)}\n\n"
                "This answer is INCORRECT. Your output must begin with "
                f"{PREFIX} and continue from there."},
        ]
        try:
            text = teacher.complete(msgs, temperature=0.0)
        except Exception as exc:  # noqa: BLE001
            # One dropped TLS connection killed the first run at 42/100. Skip
            # the item without caching it, so a re-run fills exactly the gaps.
            print(f"    groq: {t.qid} failed ({type(exc).__name__}); will retry next run")
            time.sleep(5)
            continue
        # Normalise: the API model sometimes restates the prefix, sometimes not.
        if not text.lstrip().startswith("<verdict>"):
            text = PREFIX + text
        cache[t.qid] = text
        outs.append(text)
        if i % 10 == 0:
            print(f"    groq: {i}/{len(trajs)}", flush=True)
            cache_path = getattr(run_groq, "cache_path", None)
            if cache_path is not None:   # checkpoint so a crash loses <= 10 items
                cache_path.write_text(json.dumps(run_groq.raw, ensure_ascii=False),
                                      encoding="utf-8")
    return outs


# ------------------------------------------------------------------ main ----
def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--arms", nargs="+", default=["base", "sft", "dqa", "groq20", "groq"])
    ap.add_argument("--base", default="Qwen/Qwen2.5-1.5B-Instruct")
    ap.add_argument("--sft", default="models/critic/sft")
    ap.add_argument("--dqa", default="models/critic/dqa_step100")
    ap.add_argument("--trajectories", default="data/critic/trajectories_eval.jsonl")
    ap.add_argument("--n", type=int, default=100)
    ap.add_argument("--max-passage-chars", type=int, default=450)
    ap.add_argument("--out", default="results/critic_localization.json")
    ap.add_argument("--exclude-from", default=None,
                    help="drop the first --exclude-n items of this benchmark file, so a "
                         "bigger set can be scored only on questions never analysed before")
    ap.add_argument("--exclude-n", type=int, default=100)
    ap.add_argument("--cache", default=None, help="raw-output cache (default: <out>.raw.json)")
    args = ap.parse_args()

    def wrong_with_gold(path: str) -> list[Trajectory]:
        return [t for t in read_trajectories(path)
                if t.verdict is Verdict.INCORRECT and t.gold_indices]

    trajs = wrong_with_gold(args.trajectories)[: args.n]
    if args.exclude_from:
        seen = {t.qid for t in wrong_with_gold(args.exclude_from)[: args.exclude_n]}
        trajs = [t for t in trajs if t.qid not in seen]
    baseline = sum(len(t.gold_indices) / len(t.passages) for t in trajs) / len(trajs)
    print(f"[loc] {len(trajs)} held-out wrong answers with known gold documents")
    print(f"[loc] random-guess baseline: {baseline:.1%}\n")

    out_path = Path(args.out)
    cache_path = Path(args.cache) if args.cache else out_path.with_suffix(".raw.json")
    raw: dict = json.loads(cache_path.read_text(encoding="utf-8")) if cache_path.exists() else {}

    adapters = {"base": None, "sft": args.sft, "dqa": args.dqa}
    for arm in args.arms:
        done = raw.get(arm, {})
        if all(t.qid in done for t in trajs):
            print(f"[loc] {arm}: cached")
            continue
        print(f"[loc] running {arm}")
        if arm in GROQ_ARMS:
            done = raw.setdefault(arm, {})
            run_groq.cache_path, run_groq.raw = cache_path, raw
            try:
                run_groq(trajs, args.max_passage_chars, done, model=GROQ_ARMS[arm])
            finally:
                cache_path.write_text(json.dumps(raw, ensure_ascii=False), encoding="utf-8")
        else:
            if arm != "base" and not Path(adapters[arm]).exists():
                print(f"[loc] skipping {arm}: {adapters[arm]} missing")
                continue
            outs = run_local(arm, adapters[arm], args.base, trajs, args.max_passage_chars)
            raw[arm] = {t.qid: o for t, o in zip(trajs, outs)}
            cache_path.write_text(json.dumps(raw, ensure_ascii=False), encoding="utf-8")

    # ---- score ----
    results: dict = {"n": len(trajs), "baseline": round(baseline, 4), "arms": {}}
    hits_by_arm: dict[str, list[bool]] = {}
    for arm in args.arms:
        if arm not in raw:
            continue
        scored = [score(t, raw[arm].get(t.qid, "")) for t in trajs]
        hits = [s["hit"] for s in scored]
        hits_by_arm[arm] = hits
        k = sum(hits)
        lo, hi = wilson(k, len(trajs))
        stages: dict[str, int] = {}
        for s in scored:
            if s["stage"]:
                stages[s["stage"]] = stages.get(s["stage"], 0) + 1
        # Overall hit rate rewards always naming a document. An API model that
        # mostly blames the answer step instead (valid under the schema) scores
        # low overall but can be precise when it does name one - so precision
        # given a document was named is reported beside it.
        named = sum(1 for s in scored if s["stage"] == Stage.INFORMATION.value)
        nlo, nhi = wilson(k, named)
        results["arms"][arm] = {
            "hits": k, "hit_rate": round(k / len(trajs), 4),
            "ci95": [round(lo, 4), round(hi, 4)],
            "named_document": named,
            "hit_given_named": round(k / max(named, 1), 4),
            "hit_given_named_ci95": [round(nlo, 4), round(nhi, 4)],
            "lift_over_chance": round((k / len(trajs)) / baseline, 2),
            "valid_format": round(sum(s["valid_format"] for s in scored) / len(trajs), 4),
            "stages": stages,
        }

    arms = list(hits_by_arm)
    results["pairwise"] = {}
    for i, a in enumerate(arms):
        for b in arms[i + 1:]:
            only_a, only_b, p = mcnemar_exact(hits_by_arm[a], hits_by_arm[b])
            results["pairwise"][f"{a}_vs_{b}"] = {
                "only_" + a: only_a, "only_" + b: only_b, "p_exact": round(p, 5)}

    out_path.parent.mkdir(parents=True, exist_ok=True)
    out_path.write_text(json.dumps(results, indent=2), encoding="utf-8")

    print(f"\n{'arm':<8}{'hit rate':>10}{'95% CI':>18}{'x chance':>10}{'format':>9}"
          f"{'named doc':>11}{'hit|named':>11}")
    print("-" * 77)
    print(f"{'random':<8}{baseline:>10.1%}{'':>18}{1.0:>10.2f}")
    for arm, r in results["arms"].items():
        print(f"{arm:<8}{r['hit_rate']:>10.1%}   [{r['ci95'][0]:.1%}, {r['ci95'][1]:.1%}]"
              f"{r['lift_over_chance']:>10.2f}{r['valid_format']:>9.0%}"
              f"{r['named_document']:>11}{r['hit_given_named']:>11.1%}")
    print("\npaired comparisons (exact McNemar):")
    for name, r in results["pairwise"].items():
        vals = list(r.values())
        print(f"  {name:<16} discordant {vals[0]}/{vals[1]}   p = {r['p_exact']:.4f}")
    print(f"\n[loc] saved -> {out_path}")


if __name__ == "__main__":
    main()