"""Stage 3-5: cold-start SFT, then two-stage GRPO (CRITIC-R1 Sec 3.3.2). python scripts/critic_train.py --phase all --steps 60 Phases, in the paper's order: sft teach the output schema, so RL starts above the format cliff cja GRPO stage 1 - conservative judgement: when NOT to intervene dqa GRPO stage 2 - gated diagnostic quality, initialised from stage 1 Each phase saves LoRA adapters (a few MB) and a JSONL log of per-step reward statistics, so a run can be inspected or resumed without re-training. """ from __future__ import annotations import argparse import json import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="replace") import torch # noqa: E402 from peft import LoraConfig, PeftModel, get_peft_model # noqa: E402 from transformers import AutoModelForCausalLM, AutoTokenizer # noqa: E402 from src.critic.grpo import GRPOConfig, GRPOTrainer # noqa: E402 from src.critic.local_llm import release, vram_report # noqa: E402 from src.critic.prompts import CRITIC_INSTRUCTION, CRITIC_SYSTEM # noqa: E402 from src.critic.rewards import critique_reward # noqa: E402 from src.critic.schema import ( # noqa: E402 Critique, Location, ParseError, Verdict, parse_critique, ) from src.critic.sft import SFTConfig, train_sft # noqa: E402 from src.critic.supervise import read_supervision # noqa: E402 from src.critic.trajectory import read_trajectories # noqa: E402 # Table 9: LoRA r=16, alpha=16, all linear layers. LORA_TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"] def build_dataset(traj_path: str, sup_path: str | None, tokenizer, max_passage_chars: int = 450) -> list[dict]: """Pair each trajectory with its reference critique and a rendered prompt. When supervision is missing (stage 1 needs only verdicts, which are free), a reference is derived from the gold answer so CJA can still train. """ trajectories = read_trajectories(traj_path) supervision = {s.qid: s for s in read_supervision(sup_path)} if sup_path and Path( sup_path).exists() else {} dataset = [] for t in trajectories: sup = supervision.get(t.qid) if sup is not None: ref = sup.to_critique() elif t.verdict is Verdict.CORRECT: ref = Critique(Verdict.CORRECT, None, "Answer is consistent with the retrieved evidence.", "keep") else: idx = t.gold_indices[0] if t.gold_indices else None ref = Critique( t.verdict, Location.parse(f"information:Doc{idx}") if idx else Location.parse("answer"), "The answer does not match the supporting evidence.", f"Re-read Doc{idx} and extract the exact entity requested." if idx else "Extract the exact answer span the question asks for.") prompt = tokenizer.apply_chat_template( [{"role": "system", "content": CRITIC_SYSTEM}, {"role": "user", "content": f"{CRITIC_INSTRUCTION}\n\n{t.render(max_passage_chars)}"}], tokenize=False, add_generation_prompt=True) # Stage 2 supplies the verdict instead of asking for it. # # The paper gates DQA on the critic's own verdict being right, which # assumes stage 1 produced a usable one. Ours did not: measured AUC # for verdict discrimination is 0.447 - no signal - so the critic # always says CORRECT, the gate never opens on an INCORRECT # trajectory, and r_DQA is identically zero. Observed directly: DQA # sat at reward -0.100 with 75% flat groups, learning nothing. # # So the verdict is pre-filled from ground truth and the critic # generates only location/reason/fix. That trains the capability # which demonstrably works (localisation 44% vs 21% chance on 100 # held-out wrong answers, binomial p=2e-7) instead of one that does # not - and it matches how this # would deploy in ResearchOS, where the existing mean-of-top-5 # confidence gate already supplies a reliable verdict. prefix = f"{ref.verdict.value}" dataset.append({"qid": t.qid, "prompt": prompt, "reference": ref, "target": ref.render(), "prefix": prefix, "has_teacher": sup is not None and sup.source == "teacher"}) return dataset def make_reward_fn(stage: int, prefilled: bool = False): """Wrap critique_reward, returning (reward, diagnostics) for logging. `prefilled` reattaches the verdict prefix that was appended to the prompt, so the reward still scores a complete, parseable critique. """ def fn(meta: dict, text: str): ref: Critique = meta["reference"] if prefilled: text = meta["prefix"] + text bd = critique_reward(text, ref, stage=stage, trust_diagnosis=meta.get("has_teacher", True)) # Compare verdicts directly. Inferring a match from `bd.verdict > 0` # scored a correct UNSURE->UNSURE abstention as a mismatch, because # that cell of Table 7 is exactly 0.0 - and abstention is the very # behaviour CJA exists to encourage, so the metric would have hidden # the effect being trained for. try: verdict_match = parse_critique(text, strict=True).verdict is ref.verdict except ParseError: verdict_match = False return bd.total, {"parse_error": bd.parse_error, "gate_open": bd.gate_open, "verdict_match": verdict_match} return fn def load_model(base: str, adapter: str | None, lr_target_dtype=torch.bfloat16): tok = AutoTokenizer.from_pretrained(base) tok.padding_side = "left" # Truncate the START of an over-long prompt, never the end. The chat # template's "<|im_start|>assistant" cue is the last thing in the prompt, # and right-truncation (the default) deletes it - after which the model # has no signal that it is its turn and simply continues the trajectory. # 37% of these prompts exceed the 1536-token cap, so this was live. tok.truncation_side = "left" if tok.pad_token is None: tok.pad_token = tok.eos_token model = AutoModelForCausalLM.from_pretrained(base, dtype=lr_target_dtype, device_map="cuda") if adapter and Path(adapter).exists(): model = PeftModel.from_pretrained(model, adapter, is_trainable=True) print(f"[train] resumed adapters from {adapter}") else: model = get_peft_model(model, LoraConfig( r=16, lora_alpha=16, lora_dropout=0.0, task_type="CAUSAL_LM", target_modules=LORA_TARGETS)) # Checkpointing only engages in train() mode - a silent 3x memory cost if # this is forgotten, and the first VRAM probe for this project hit exactly # that, reporting a false OOM at batch size 1. model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) model.enable_input_require_grads() model.train() trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) print(f"[train] {base} + LoRA: {trainable/1e6:.1f}M trainable | {vram_report()}") return model, tok def main() -> None: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--phase", default="all", choices=["all", "sft", "cja", "dqa"]) ap.add_argument("--base", default="Qwen/Qwen2.5-1.5B-Instruct") ap.add_argument("--trajectories", default="data/critic/trajectories_train.jsonl") ap.add_argument("--supervision", default="data/critic/supervision_train.jsonl") ap.add_argument("--out", default="models/critic") ap.add_argument("--steps", type=int, default=60, help="GRPO steps per stage") ap.add_argument("--sft-epochs", type=int, default=2) ap.add_argument("--group-size", type=int, default=4) ap.add_argument("--prompts-per-step", type=int, default=2) ap.add_argument("--max-new-tokens", type=int, default=200) # 120 truncated ap.add_argument("--max-prompt-tokens", type=int, default=1536) ap.add_argument("--limit", type=int, default=None) args = ap.parse_args() out = Path(args.out) out.mkdir(parents=True, exist_ok=True) phases = ["sft", "cja", "dqa"] if args.phase == "all" else [args.phase] report: dict = {"base": args.base, "phases": {}} started = time.time() # ---------------- SFT ---------------- if "sft" in phases: model, tok = load_model(args.base, None) data = build_dataset(args.trajectories, args.supervision, tok) if args.limit: data = data[: args.limit] print(f"[train] SFT on {len(data)} examples") report["phases"]["sft"] = train_sft( model, tok, data, SFTConfig(epochs=args.sft_epochs), out / "sft") del model release() grpo_cfg = GRPOConfig( group_size=args.group_size, prompts_per_step=args.prompts_per_step, max_new_tokens=args.max_new_tokens, max_prompt_tokens=args.max_prompt_tokens, gen_batch=min(8, args.group_size * args.prompts_per_step), ) # ---------------- GRPO stage 1: CJA ---------------- if "cja" in phases: model, tok = load_model(args.base, str(out / "sft")) data = build_dataset(args.trajectories, args.supervision, tok) if args.limit: data = data[: args.limit] trainer = GRPOTrainer(model, tok, make_reward_fn(stage=1), grpo_cfg, out) trainer.train(data, args.steps, tag="cja") trainer.save("cja") report["phases"]["cja"] = { "steps": args.steps, "reward_first": round(trainer.history[0].reward_mean, 4), "reward_last": round(trainer.history[-1].reward_mean, 4), "format_first": round(trainer.history[0].valid_format_rate, 3), "format_last": round(trainer.history[-1].valid_format_rate, 3), } del model, trainer release() # ---------------- GRPO stage 2: gated DQA ---------------- if "dqa" in phases: # Table 9: "Stage-2 initialization: Stage-1 LoRA adapter". init = out / "cja" if (out / "cja").exists() else out / "sft" model, tok = load_model(args.base, str(init)) data = build_dataset(args.trajectories, args.supervision, tok) # Stage 2 trains on every trajectory that has a real error location - # the gold supporting document the generator ignored. That label is # free and correct for all 343 non-CORRECT trajectories, where only 25 # have teacher reason/fix text. Filtering to the teacher-labelled 25 # discarded 93% of the usable localisation signal; the reason/fix # rewards are instead switched off per-example via trust_diagnosis. data = [d for d in data if d["reference"].verdict is not Verdict.CORRECT and d["reference"].location is not None] if args.limit: data = data[: args.limit] for d in data: d["prompt"] = d["prompt"] + d["prefix"] print(f"[train] DQA on {len(data)} located trajectories " f"(verdict pre-filled; critic writes location/reason/fix)") trainer = GRPOTrainer(model, tok, make_reward_fn(stage=2, prefilled=True), grpo_cfg, out) trainer.train(data, args.steps, tag="dqa") trainer.save("dqa") report["phases"]["dqa"] = { "steps": args.steps, "reward_first": round(trainer.history[0].reward_mean, 4), "reward_last": round(trainer.history[-1].reward_mean, 4), "gate_first": round(trainer.history[0].gate_open_rate, 3), "gate_last": round(trainer.history[-1].gate_open_rate, 3), } del model, trainer release() report["minutes"] = round((time.time() - started) / 60, 1) (out / "train_report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") print("\n" + json.dumps(report, indent=2)) if __name__ == "__main__": main()