"""Stage 6: evaluate the critic, reproducing the paper's ablation. python scripts/critic_evaluate.py --adapters base sft cja dqa --refine Mirrors Table 3's ablation arms: base no training - the paper's "w/o Training" row, which scores *worse* than using no critic at all (41.4 vs 44.5 F1). Reproducing that is the point: it is the result that justifies the whole method. sft cold-start only - knows the schema, not the judgement cja + stage 1 - conservative verdicts dqa + stage 2 (gated) - full CRITIC-R1 With `--refine`, the generator re-answers every flagged trajectory using the critique as a hint (never an oracle - Appendix A.3), which gives the end-to-end numbers: how many wrong answers got fixed, and how many right ones got broken. Memory note: critic and generator are the same base model, so the generator is obtained by switching the LoRA adapters *off* rather than loading a second copy. On an 8GB card that is the difference between fitting and not. """ from __future__ import annotations import argparse import json import re import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="replace") import torch # noqa: E402 from peft import PeftModel # noqa: E402 from transformers import AutoModelForCausalLM, AutoTokenizer # noqa: E402 from src.critic.evaluate import ( # noqa: E402 evaluate_critic, evaluate_refinement, format_report, save_report, ) from src.critic.local_llm import release, vram_report # noqa: E402 from src.critic.prompts import ( # noqa: E402 CRITIC_INSTRUCTION, CRITIC_SYSTEM, REFINE_SYSTEM, ) from src.critic.schema import ParseError, Verdict, parse_critique # noqa: E402 from src.critic.supervise import read_supervision # noqa: E402 from src.critic.trajectory import read_trajectories # noqa: E402 _ANSWER_RE = re.compile(r"(.*?)", re.S | re.I) @torch.inference_mode() def batched_generate(model, tok, prompts: list[str], max_new_tokens: int, batch_size: int = 4, temperature: float = 0.0, max_input: int = 1536, label: str = "") -> list[str]: """Greedy by default: evaluation should not be a lottery.""" outputs: list[str] = [] bs, i, t0 = batch_size, 0, time.time() while i < len(prompts): chunk = prompts[i:i + bs] try: enc = tok(chunk, return_tensors="pt", padding=True, truncation=True, max_length=max_input).to(model.device) out = model.generate( **enc, max_new_tokens=max_new_tokens, do_sample=temperature > 0, temperature=temperature if temperature > 0 else None, pad_token_id=tok.pad_token_id) outputs.extend(tok.batch_decode(out[:, enc["input_ids"].shape[1]:], skip_special_tokens=True)) i += len(chunk) if i % 40 < bs: rate = i / max(time.time() - t0, 1e-6) print(f" {label} {i}/{len(prompts)} {rate*60:.0f}/min", flush=True) except torch.cuda.OutOfMemoryError: release() if bs == 1: outputs.append("") i += 1 else: bs = max(1, bs // 2) print(f" OOM; batch -> {bs}") release() return outputs def main() -> None: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--base", default="Qwen/Qwen2.5-1.5B-Instruct") ap.add_argument("--adapters", nargs="+", default=["base", "sft", "cja", "dqa"]) ap.add_argument("--models-dir", default="models/critic") ap.add_argument("--trajectories", default="data/critic/trajectories_eval.jsonl") ap.add_argument("--supervision", default="data/critic/supervision_eval.jsonl") ap.add_argument("--out", default="results/critic_eval.json") ap.add_argument("--limit", type=int, default=None) ap.add_argument("--batch-size", type=int, default=4) ap.add_argument("--refine", action="store_true", help="also measure end-to-end refinement") args = ap.parse_args() trajectories = read_trajectories(args.trajectories) if args.limit: trajectories = trajectories[: args.limit] ref_locs = {} if Path(args.supervision).exists(): ref_locs = {s.qid: s.critique.get("location", "") for s in read_supervision(args.supervision)} print(f"[eval] {len(trajectories)} eval trajectories | {vram_report()}") tok = AutoTokenizer.from_pretrained(args.base) tok.padding_side = "left" if tok.pad_token is None: tok.pad_token = tok.eos_token critic_prompts = [ tok.apply_chat_template( [{"role": "system", "content": CRITIC_SYSTEM}, {"role": "user", "content": f"{CRITIC_INSTRUCTION}\n\n{t.render(450)}"}], tokenize=False, add_generation_prompt=True) for t in trajectories ] report: dict = {"n": len(trajectories), "base_model": args.base, "arms": {}} for arm in args.adapters: adapter_path = Path(args.models_dir) / arm if arm != "base" and not adapter_path.exists(): print(f"[eval] skipping '{arm}': {adapter_path} not found") continue print(f"\n[eval] === arm: {arm} ===") model = AutoModelForCausalLM.from_pretrained( args.base, dtype=torch.bfloat16, device_map="cuda") if arm != "base": model = PeftModel.from_pretrained(model, str(adapter_path)) model.eval() critiques = batched_generate(model, tok, critic_prompts, 200, args.batch_size, label=f"{arm} critique") metrics = evaluate_critic(trajectories, critiques, ref_locs or None) arm_report: dict = {"critic": metrics.to_json()} refine_metrics = None if args.refine: # Refine only what the critic flagged - that is the behaviour in # production, and refining everything would measure a different # system. flagged: list[int] = [] for i, text in enumerate(critiques): try: v = parse_critique(text, strict=False).verdict except ParseError: v = Verdict.CORRECT if v is Verdict.INCORRECT: flagged.append(i) print(f" critic flagged {len(flagged)}/{len(trajectories)}") refined: list[str | None] = [None] * len(trajectories) if flagged: prompts = [] for i in flagged: t = trajectories[i] docs = "\n".join(f"Doc{p.index}({p.title}): {p.text[:450]}" for p in t.passages) prompts.append(tok.apply_chat_template( [{"role": "system", "content": REFINE_SYSTEM}, {"role": "user", "content": f"\n{docs}\n\n\n" f"{t.question}\n\n" f"{t.think}" f"{t.answer}\n\n" f"{critiques[i]}\n\n" "Re-solve. End with exactly one ...."}], tokenize=False, add_generation_prompt=True)) # The generator is this same model with the adapters off - no # second set of weights. ctx = model.disable_adapter() if arm != "base" else torch.no_grad() with ctx: outs = batched_generate(model, tok, prompts, 120, args.batch_size, label=f"{arm} refine") for i, text in zip(flagged, outs): m = _ANSWER_RE.search(text) refined[i] = (m.group(1).strip() if m else text.strip().split("\n")[-1].strip()) refine_metrics = evaluate_refinement(trajectories, refined) arm_report["refinement"] = refine_metrics.to_json() report["arms"][arm] = arm_report print(format_report(metrics, refine_metrics, title=f"{arm}")) del model release() save_report(args.out, report) print(f"\n[eval] saved -> {args.out}") # Side-by-side, the shape the paper reports. if len(report["arms"]) > 1: print(f"\n{'arm':<8}{'falseAlarm':>12}{'precision':>11}{'recall':>9}" f"{'fmt':>7}{'F1 after':>10}{'harm':>8}") print("-" * 65) for arm, r in report["arms"].items(): c, rf = r["critic"], r.get("refinement") print(f"{arm:<8}{c['false_alarm']:>11.1%}{c['precision']:>11.1%}" f"{c['recall']:>9.1%}{c['valid_format_rate']:>7.0%}" f"{(rf['f1_after'] if rf else 0):>10.3f}" f"{(rf['harmed_of_right'] if rf else 0):>8.1%}") if __name__ == "__main__": main()