"""Replay production R1-R5 arithmetic on frozen step-324 and SFT VALID outputs.""" from __future__ import annotations import hashlib import json from collections import defaultdict from pathlib import Path import numpy as np import pandas as pd from src.contracts.parser import parse_output from src.contracts.prompt import code_for_sample from src.rewards.score_sample import score_sample ROOT = Path(__file__).resolve().parents[1] RUN = ROOT / "results/phase7/matched_vllm_step324_fullvalid" CORPUS = ROOT / "data/processed/corpus.parquet" ANNOTATIONS = ROOT / "data/processed/annotations.jsonl" VALID = ROOT / "data/splits/chronological/valid.json" GRAPH_CACHE = ROOT / "data/cpg_cache/phase7_valid_reward_graphs" OUTPUT = ROOT / "results/phase7/step324_reward_alignment.json" MODEL_FILES = { "sft_v3_step500": RUN / "sft_v3_step500_predictions.jsonl", "grpo_step324": RUN / "grpo_step324_predictions.jsonl", } def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def mean_or_none(values: list[float]) -> float | None: return float(np.mean(values)) if values else None def replay(path: Path, corpus: pd.DataFrame, annotations: dict[str, dict]) -> dict: records = [] with path.open() as stream: for line in stream: if not line.strip(): continue row = json.loads(line) parsed = parse_output(row["completion"], contract_version=2) source = corpus.loc[str(row["sample_id"])] target = int(row["target"]) ann = annotations[str(row["sample_id"])] func_len = len(code_for_sample(source).splitlines()) reward = score_sample( parsed, sample_id=str(row["sample_id"]), target=target, annotation=ann, func_len=func_len, cache_dir=str(GRAPH_CACHE), r4=None, ) gold = "VULNERABLE" if target == 1 else "SAFE" predicted = parsed.label if parsed.format_ok else "INVALID" counterfactual_total = reward.total safe_empty_r2 = bool(target == 0 and predicted == "SAFE" and not parsed.statement_lines and reward.r2 == 1.0) if safe_empty_r2: # Fixed-output arithmetic only: neutralize the empty/empty IoU bonus. counterfactual_total -= reward.r1 * float(reward.r2) records.append({ "sample_id": str(row["sample_id"]), "gold": gold, "predicted": predicted, "correct": bool(parsed.format_ok and parsed.label == gold), "format_ok": bool(parsed.format_ok), "nonempty_statements": bool(parsed.statement_lines), "r1": reward.r1, "r2": reward.r2, "r3": reward.r3, "r5": reward.r5, "total_without_r4": reward.total, "total_neutral_safe_empty_r2": counterfactual_total, "safe_empty_r2_counterfactual_applied": safe_empty_r2, "fn_safe_statement_penalty": bool(target == 1 and predicted == "SAFE" and parsed.statement_lines and reward.r5 <= -0.5), }) def summarize(items: list[dict]) -> dict: return { "n": len(items), "correct_fraction": mean_or_none([float(x["correct"]) for x in items]), "r1_mean": mean_or_none([x["r1"] for x in items]), "r2_mean_including_gated_zero": mean_or_none([float(x["r2"] or 0.0) for x in items]), "r2_eligible_fraction": mean_or_none([float(x["r2"] is not None) for x in items]), "r3_mean_including_gated_zero": mean_or_none([float(x["r3"] or 0.0) for x in items]), "r3_eligible_fraction": mean_or_none([float(x["r3"] is not None) for x in items]), "r5_mean": mean_or_none([x["r5"] for x in items]), "total_without_r4_mean": mean_or_none([x["total_without_r4"] for x in items]), "total_neutral_safe_empty_r2_mean": mean_or_none( [x["total_neutral_safe_empty_r2"] for x in items]), } by_category: dict[str, list[dict]] = defaultdict(list) by_gold: dict[str, list[dict]] = defaultdict(list) for record in records: by_category[f"gold_{record['gold']}_pred_{record['predicted']}"].append(record) by_gold[record["gold"]].append(record) all_rows = summarize(records) return { "n_rows": len(records), "by_gold_label": {key: summarize(value) for key, value in sorted(by_gold.items())}, "by_gold_predicted_label": {key: summarize(value) for key, value in sorted(by_category.items())}, "specific_mechanisms": { "correct_safe_rows_with_empty_statements": sum(x["safe_empty_r2_counterfactual_applied"] for x in records), "safe_empty_r2_bonus_total_fixed_output": float(sum( x["total_without_r4"] - x["total_neutral_safe_empty_r2"] for x in records)), "vulnerable_false_negatives_with_safe_statement_penalty": sum(x["fn_safe_statement_penalty"] for x in records), "vulnerable_false_negative_rows": sum(x["gold"] == "VULNERABLE" and x["predicted"] == "SAFE" for x in records), }, "all_rows": all_rows, "scope": "production score_sample fixed-output replay with r4=None; no group difficulty weights, policy gradients, training, or reward semantics changed", } def main() -> dict: valid_pair_ids = set(map(str, json.loads(VALID.read_text()))) corpus = pd.read_parquet(CORPUS).set_index("sample_id", drop=False) valid_rows = corpus[corpus["pair_id"].astype(str).isin(valid_pair_ids)] if len(valid_rows) != 2208: raise ValueError(f"expected all 2,208 VALID rows, found {len(valid_rows)}") annotations = {} with ANNOTATIONS.open() as stream: for line in stream: if line.strip(): record = json.loads(line) annotations[record["sample_id"]] = record reports = {name: replay(path, corpus, annotations) for name, path in MODEL_FILES.items()} result = { "purpose": "fixed-output audit of R1/R2/R3/R5 mechanics on matched full chronological VALID predictions", "models": reports, "provenance": { "valid_split_sha256": sha256(VALID), "corpus_sha256": sha256(CORPUS), "annotations_sha256": sha256(ANNOTATIONS), "sft_predictions_sha256": sha256(MODEL_FILES["sft_v3_step500"]), "grpo_predictions_sha256": sha256(MODEL_FILES["grpo_step324"]), "test_read": False, }, "limitations": [ "R4 is omitted because the matched offline evaluation did not produce judge scores.", "Reported totals are unweighted fixed-output arithmetic; they do not reconstruct GRPO group-normalized advantages.", "The neutral SAFE-empty R2 number is a counterfactual over unchanged outputs and does not predict a retrained policy.", "R3 is skipped where frozen trace artifacts are unavailable or marked unverifiable by the reward implementation.", ], } OUTPUT.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") print(json.dumps(result, indent=2)) return result if __name__ == "__main__": main()