Spaces:
Running on Zero
Running on Zero
File size: 9,386 Bytes
e5c059f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 | """Stage 6: evaluate the critic, reproducing the paper's ablation.
python scripts/critic_evaluate.py --adapters base sft cja dqa --refine
Mirrors Table 3's ablation arms:
base no training - the paper's "w/o Training" row, which scores
*worse* than using no critic at all (41.4 vs
44.5 F1). Reproducing that is the point: it is
the result that justifies the whole method.
sft cold-start only - knows the schema, not the judgement
cja + stage 1 - conservative verdicts
dqa + stage 2 (gated) - full CRITIC-R1
With `--refine`, the generator re-answers every flagged trajectory using the
critique as a hint (never an oracle - Appendix A.3), which gives the
end-to-end numbers: how many wrong answers got fixed, and how many right ones
got broken.
Memory note: critic and generator are the same base model, so the generator is
obtained by switching the LoRA adapters *off* rather than loading a second
copy. On an 8GB card that is the difference between fitting and not.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
import torch # noqa: E402
from peft import PeftModel # noqa: E402
from transformers import AutoModelForCausalLM, AutoTokenizer # noqa: E402
from src.critic.evaluate import ( # noqa: E402
evaluate_critic, evaluate_refinement, format_report, save_report,
)
from src.critic.local_llm import release, vram_report # noqa: E402
from src.critic.prompts import ( # noqa: E402
CRITIC_INSTRUCTION, CRITIC_SYSTEM, REFINE_SYSTEM,
)
from src.critic.schema import ParseError, Verdict, parse_critique # noqa: E402
from src.critic.supervise import read_supervision # noqa: E402
from src.critic.trajectory import read_trajectories # noqa: E402
_ANSWER_RE = re.compile(r"<answer>(.*?)</answer>", re.S | re.I)
@torch.inference_mode()
def batched_generate(model, tok, prompts: list[str], max_new_tokens: int,
batch_size: int = 4, temperature: float = 0.0,
max_input: int = 1536, label: str = "") -> list[str]:
"""Greedy by default: evaluation should not be a lottery."""
outputs: list[str] = []
bs, i, t0 = batch_size, 0, time.time()
while i < len(prompts):
chunk = prompts[i:i + bs]
try:
enc = tok(chunk, return_tensors="pt", padding=True, truncation=True,
max_length=max_input).to(model.device)
out = model.generate(
**enc, max_new_tokens=max_new_tokens,
do_sample=temperature > 0,
temperature=temperature if temperature > 0 else None,
pad_token_id=tok.pad_token_id)
outputs.extend(tok.batch_decode(out[:, enc["input_ids"].shape[1]:],
skip_special_tokens=True))
i += len(chunk)
if i % 40 < bs:
rate = i / max(time.time() - t0, 1e-6)
print(f" {label} {i}/{len(prompts)} {rate*60:.0f}/min", flush=True)
except torch.cuda.OutOfMemoryError:
release()
if bs == 1:
outputs.append("")
i += 1
else:
bs = max(1, bs // 2)
print(f" OOM; batch -> {bs}")
release()
return outputs
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--base", default="Qwen/Qwen2.5-1.5B-Instruct")
ap.add_argument("--adapters", nargs="+", default=["base", "sft", "cja", "dqa"])
ap.add_argument("--models-dir", default="models/critic")
ap.add_argument("--trajectories", default="data/critic/trajectories_eval.jsonl")
ap.add_argument("--supervision", default="data/critic/supervision_eval.jsonl")
ap.add_argument("--out", default="results/critic_eval.json")
ap.add_argument("--limit", type=int, default=None)
ap.add_argument("--batch-size", type=int, default=4)
ap.add_argument("--refine", action="store_true", help="also measure end-to-end refinement")
args = ap.parse_args()
trajectories = read_trajectories(args.trajectories)
if args.limit:
trajectories = trajectories[: args.limit]
ref_locs = {}
if Path(args.supervision).exists():
ref_locs = {s.qid: s.critique.get("location", "")
for s in read_supervision(args.supervision)}
print(f"[eval] {len(trajectories)} eval trajectories | {vram_report()}")
tok = AutoTokenizer.from_pretrained(args.base)
tok.padding_side = "left"
if tok.pad_token is None:
tok.pad_token = tok.eos_token
critic_prompts = [
tok.apply_chat_template(
[{"role": "system", "content": CRITIC_SYSTEM},
{"role": "user", "content": f"{CRITIC_INSTRUCTION}\n\n{t.render(450)}"}],
tokenize=False, add_generation_prompt=True)
for t in trajectories
]
report: dict = {"n": len(trajectories), "base_model": args.base, "arms": {}}
for arm in args.adapters:
adapter_path = Path(args.models_dir) / arm
if arm != "base" and not adapter_path.exists():
print(f"[eval] skipping '{arm}': {adapter_path} not found")
continue
print(f"\n[eval] === arm: {arm} ===")
model = AutoModelForCausalLM.from_pretrained(
args.base, dtype=torch.bfloat16, device_map="cuda")
if arm != "base":
model = PeftModel.from_pretrained(model, str(adapter_path))
model.eval()
critiques = batched_generate(model, tok, critic_prompts, 200,
args.batch_size, label=f"{arm} critique")
metrics = evaluate_critic(trajectories, critiques, ref_locs or None)
arm_report: dict = {"critic": metrics.to_json()}
refine_metrics = None
if args.refine:
# Refine only what the critic flagged - that is the behaviour in
# production, and refining everything would measure a different
# system.
flagged: list[int] = []
for i, text in enumerate(critiques):
try:
v = parse_critique(text, strict=False).verdict
except ParseError:
v = Verdict.CORRECT
if v is Verdict.INCORRECT:
flagged.append(i)
print(f" critic flagged {len(flagged)}/{len(trajectories)}")
refined: list[str | None] = [None] * len(trajectories)
if flagged:
prompts = []
for i in flagged:
t = trajectories[i]
docs = "\n".join(f"Doc{p.index}({p.title}): {p.text[:450]}"
for p in t.passages)
prompts.append(tok.apply_chat_template(
[{"role": "system", "content": REFINE_SYSTEM},
{"role": "user", "content":
f"<information>\n{docs}\n</information>\n\n"
f"<question>{t.question}</question>\n\n"
f"<previous_trajectory><think>{t.think}</think>"
f"<answer>{t.answer}</answer></previous_trajectory>\n\n"
f"<critique>{critiques[i]}</critique>\n\n"
"Re-solve. End with exactly one <answer>...</answer>."}],
tokenize=False, add_generation_prompt=True))
# The generator is this same model with the adapters off - no
# second set of weights.
ctx = model.disable_adapter() if arm != "base" else torch.no_grad()
with ctx:
outs = batched_generate(model, tok, prompts, 120,
args.batch_size, label=f"{arm} refine")
for i, text in zip(flagged, outs):
m = _ANSWER_RE.search(text)
refined[i] = (m.group(1).strip() if m
else text.strip().split("\n")[-1].strip())
refine_metrics = evaluate_refinement(trajectories, refined)
arm_report["refinement"] = refine_metrics.to_json()
report["arms"][arm] = arm_report
print(format_report(metrics, refine_metrics, title=f"{arm}"))
del model
release()
save_report(args.out, report)
print(f"\n[eval] saved -> {args.out}")
# Side-by-side, the shape the paper reports.
if len(report["arms"]) > 1:
print(f"\n{'arm':<8}{'falseAlarm':>12}{'precision':>11}{'recall':>9}"
f"{'fmt':>7}{'F1 after':>10}{'harm':>8}")
print("-" * 65)
for arm, r in report["arms"].items():
c, rf = r["critic"], r.get("refinement")
print(f"{arm:<8}{c['false_alarm']:>11.1%}{c['precision']:>11.1%}"
f"{c['recall']:>9.1%}{c['valid_format_rate']:>7.0%}"
f"{(rf['f1_after'] if rf else 0):>10.3f}"
f"{(rf['harmed_of_right'] if rf else 0):>8.1%}")
if __name__ == "__main__":
main()
|