File size: 9,386 Bytes
e5c059f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
"""Stage 6: evaluate the critic, reproducing the paper's ablation.

    python scripts/critic_evaluate.py --adapters base sft cja dqa --refine

Mirrors Table 3's ablation arms:

    base   no training        - the paper's "w/o Training" row, which scores
                               *worse* than using no critic at all (41.4 vs
                               44.5 F1). Reproducing that is the point: it is
                               the result that justifies the whole method.
    sft    cold-start only    - knows the schema, not the judgement
    cja    + stage 1          - conservative verdicts
    dqa    + stage 2 (gated)  - full CRITIC-R1

With `--refine`, the generator re-answers every flagged trajectory using the
critique as a hint (never an oracle - Appendix A.3), which gives the
end-to-end numbers: how many wrong answers got fixed, and how many right ones
got broken.

Memory note: critic and generator are the same base model, so the generator is
obtained by switching the LoRA adapters *off* rather than loading a second
copy. On an 8GB card that is the difference between fitting and not.
"""

from __future__ import annotations

import argparse
import json
import re
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")

import torch  # noqa: E402
from peft import PeftModel  # noqa: E402
from transformers import AutoModelForCausalLM, AutoTokenizer  # noqa: E402

from src.critic.evaluate import (  # noqa: E402
    evaluate_critic, evaluate_refinement, format_report, save_report,
)
from src.critic.local_llm import release, vram_report  # noqa: E402
from src.critic.prompts import (  # noqa: E402
    CRITIC_INSTRUCTION, CRITIC_SYSTEM, REFINE_SYSTEM,
)
from src.critic.schema import ParseError, Verdict, parse_critique  # noqa: E402
from src.critic.supervise import read_supervision  # noqa: E402
from src.critic.trajectory import read_trajectories  # noqa: E402

_ANSWER_RE = re.compile(r"<answer>(.*?)</answer>", re.S | re.I)


@torch.inference_mode()
def batched_generate(model, tok, prompts: list[str], max_new_tokens: int,
                     batch_size: int = 4, temperature: float = 0.0,
                     max_input: int = 1536, label: str = "") -> list[str]:
    """Greedy by default: evaluation should not be a lottery."""
    outputs: list[str] = []
    bs, i, t0 = batch_size, 0, time.time()
    while i < len(prompts):
        chunk = prompts[i:i + bs]
        try:
            enc = tok(chunk, return_tensors="pt", padding=True, truncation=True,
                      max_length=max_input).to(model.device)
            out = model.generate(
                **enc, max_new_tokens=max_new_tokens,
                do_sample=temperature > 0,
                temperature=temperature if temperature > 0 else None,
                pad_token_id=tok.pad_token_id)
            outputs.extend(tok.batch_decode(out[:, enc["input_ids"].shape[1]:],
                                            skip_special_tokens=True))
            i += len(chunk)
            if i % 40 < bs:
                rate = i / max(time.time() - t0, 1e-6)
                print(f"    {label} {i}/{len(prompts)}  {rate*60:.0f}/min", flush=True)
        except torch.cuda.OutOfMemoryError:
            release()
            if bs == 1:
                outputs.append("")
                i += 1
            else:
                bs = max(1, bs // 2)
                print(f"    OOM; batch -> {bs}")
    release()
    return outputs


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--base", default="Qwen/Qwen2.5-1.5B-Instruct")
    ap.add_argument("--adapters", nargs="+", default=["base", "sft", "cja", "dqa"])
    ap.add_argument("--models-dir", default="models/critic")
    ap.add_argument("--trajectories", default="data/critic/trajectories_eval.jsonl")
    ap.add_argument("--supervision", default="data/critic/supervision_eval.jsonl")
    ap.add_argument("--out", default="results/critic_eval.json")
    ap.add_argument("--limit", type=int, default=None)
    ap.add_argument("--batch-size", type=int, default=4)
    ap.add_argument("--refine", action="store_true", help="also measure end-to-end refinement")
    args = ap.parse_args()

    trajectories = read_trajectories(args.trajectories)
    if args.limit:
        trajectories = trajectories[: args.limit]
    ref_locs = {}
    if Path(args.supervision).exists():
        ref_locs = {s.qid: s.critique.get("location", "")
                    for s in read_supervision(args.supervision)}
    print(f"[eval] {len(trajectories)} eval trajectories | {vram_report()}")

    tok = AutoTokenizer.from_pretrained(args.base)
    tok.padding_side = "left"
    if tok.pad_token is None:
        tok.pad_token = tok.eos_token

    critic_prompts = [
        tok.apply_chat_template(
            [{"role": "system", "content": CRITIC_SYSTEM},
             {"role": "user", "content": f"{CRITIC_INSTRUCTION}\n\n{t.render(450)}"}],
            tokenize=False, add_generation_prompt=True)
        for t in trajectories
    ]

    report: dict = {"n": len(trajectories), "base_model": args.base, "arms": {}}

    for arm in args.adapters:
        adapter_path = Path(args.models_dir) / arm
        if arm != "base" and not adapter_path.exists():
            print(f"[eval] skipping '{arm}': {adapter_path} not found")
            continue

        print(f"\n[eval] === arm: {arm} ===")
        model = AutoModelForCausalLM.from_pretrained(
            args.base, dtype=torch.bfloat16, device_map="cuda")
        if arm != "base":
            model = PeftModel.from_pretrained(model, str(adapter_path))
        model.eval()

        critiques = batched_generate(model, tok, critic_prompts, 200,
                                     args.batch_size, label=f"{arm} critique")
        metrics = evaluate_critic(trajectories, critiques, ref_locs or None)
        arm_report: dict = {"critic": metrics.to_json()}

        refine_metrics = None
        if args.refine:
            # Refine only what the critic flagged - that is the behaviour in
            # production, and refining everything would measure a different
            # system.
            flagged: list[int] = []
            for i, text in enumerate(critiques):
                try:
                    v = parse_critique(text, strict=False).verdict
                except ParseError:
                    v = Verdict.CORRECT
                if v is Verdict.INCORRECT:
                    flagged.append(i)
            print(f"    critic flagged {len(flagged)}/{len(trajectories)}")

            refined: list[str | None] = [None] * len(trajectories)
            if flagged:
                prompts = []
                for i in flagged:
                    t = trajectories[i]
                    docs = "\n".join(f"Doc{p.index}({p.title}): {p.text[:450]}"
                                     for p in t.passages)
                    prompts.append(tok.apply_chat_template(
                        [{"role": "system", "content": REFINE_SYSTEM},
                         {"role": "user", "content":
                             f"<information>\n{docs}\n</information>\n\n"
                             f"<question>{t.question}</question>\n\n"
                             f"<previous_trajectory><think>{t.think}</think>"
                             f"<answer>{t.answer}</answer></previous_trajectory>\n\n"
                             f"<critique>{critiques[i]}</critique>\n\n"
                             "Re-solve. End with exactly one <answer>...</answer>."}],
                        tokenize=False, add_generation_prompt=True))

                # The generator is this same model with the adapters off - no
                # second set of weights.
                ctx = model.disable_adapter() if arm != "base" else torch.no_grad()
                with ctx:
                    outs = batched_generate(model, tok, prompts, 120,
                                            args.batch_size, label=f"{arm} refine")
                for i, text in zip(flagged, outs):
                    m = _ANSWER_RE.search(text)
                    refined[i] = (m.group(1).strip() if m
                                  else text.strip().split("\n")[-1].strip())
            refine_metrics = evaluate_refinement(trajectories, refined)
            arm_report["refinement"] = refine_metrics.to_json()

        report["arms"][arm] = arm_report
        print(format_report(metrics, refine_metrics, title=f"{arm}"))

        del model
        release()

    save_report(args.out, report)
    print(f"\n[eval] saved -> {args.out}")

    # Side-by-side, the shape the paper reports.
    if len(report["arms"]) > 1:
        print(f"\n{'arm':<8}{'falseAlarm':>12}{'precision':>11}{'recall':>9}"
              f"{'fmt':>7}{'F1 after':>10}{'harm':>8}")
        print("-" * 65)
        for arm, r in report["arms"].items():
            c, rf = r["critic"], r.get("refinement")
            print(f"{arm:<8}{c['false_alarm']:>11.1%}{c['precision']:>11.1%}"
                  f"{c['recall']:>9.1%}{c['valid_format_rate']:>7.0%}"
                  f"{(rf['f1_after'] if rf else 0):>10.3f}"
                  f"{(rf['harmed_of_right'] if rf else 0):>8.1%}")


if __name__ == "__main__":
    main()