tracegrpo / scripts /audit_step324_valid_reward_alignment.py
Dhruv1000's picture
Archive complete TraceGRPO project results and documentation (2026-10-09)
646ccfc verified
Raw History Blame Contribute Delete
7.48 kB
"""Replay production R1-R5 arithmetic on frozen step-324 and SFT VALID outputs."""
from __future__ import annotations
import hashlib
import json
from collections import defaultdict
from pathlib import Path
import numpy as np
import pandas as pd
from src.contracts.parser import parse_output
from src.contracts.prompt import code_for_sample
from src.rewards.score_sample import score_sample
ROOT = Path(__file__).resolve().parents[1]
RUN = ROOT / "results/phase7/matched_vllm_step324_fullvalid"
CORPUS = ROOT / "data/processed/corpus.parquet"
ANNOTATIONS = ROOT / "data/processed/annotations.jsonl"
VALID = ROOT / "data/splits/chronological/valid.json"
GRAPH_CACHE = ROOT / "data/cpg_cache/phase7_valid_reward_graphs"
OUTPUT = ROOT / "results/phase7/step324_reward_alignment.json"
MODEL_FILES = {
"sft_v3_step500": RUN / "sft_v3_step500_predictions.jsonl",
"grpo_step324": RUN / "grpo_step324_predictions.jsonl",
}
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def mean_or_none(values: list[float]) -> float | None:
return float(np.mean(values)) if values else None
def replay(path: Path, corpus: pd.DataFrame, annotations: dict[str, dict]) -> dict:
records = []
with path.open() as stream:
for line in stream:
if not line.strip():
continue
row = json.loads(line)
parsed = parse_output(row["completion"], contract_version=2)
source = corpus.loc[str(row["sample_id"])]
target = int(row["target"])
ann = annotations[str(row["sample_id"])]
func_len = len(code_for_sample(source).splitlines())
reward = score_sample(
parsed,
sample_id=str(row["sample_id"]),
target=target,
annotation=ann,
func_len=func_len,
cache_dir=str(GRAPH_CACHE),
r4=None,
)
gold = "VULNERABLE" if target == 1 else "SAFE"
predicted = parsed.label if parsed.format_ok else "INVALID"
counterfactual_total = reward.total
safe_empty_r2 = bool(target == 0 and predicted == "SAFE"
and not parsed.statement_lines and reward.r2 == 1.0)
if safe_empty_r2:
# Fixed-output arithmetic only: neutralize the empty/empty IoU bonus.
counterfactual_total -= reward.r1 * float(reward.r2)
records.append({
"sample_id": str(row["sample_id"]),
"gold": gold,
"predicted": predicted,
"correct": bool(parsed.format_ok and parsed.label == gold),
"format_ok": bool(parsed.format_ok),
"nonempty_statements": bool(parsed.statement_lines),
"r1": reward.r1,
"r2": reward.r2,
"r3": reward.r3,
"r5": reward.r5,
"total_without_r4": reward.total,
"total_neutral_safe_empty_r2": counterfactual_total,
"safe_empty_r2_counterfactual_applied": safe_empty_r2,
"fn_safe_statement_penalty": bool(target == 1 and predicted == "SAFE"
and parsed.statement_lines and reward.r5 <= -0.5),
})
def summarize(items: list[dict]) -> dict:
return {
"n": len(items),
"correct_fraction": mean_or_none([float(x["correct"]) for x in items]),
"r1_mean": mean_or_none([x["r1"] for x in items]),
"r2_mean_including_gated_zero": mean_or_none([float(x["r2"] or 0.0) for x in items]),
"r2_eligible_fraction": mean_or_none([float(x["r2"] is not None) for x in items]),
"r3_mean_including_gated_zero": mean_or_none([float(x["r3"] or 0.0) for x in items]),
"r3_eligible_fraction": mean_or_none([float(x["r3"] is not None) for x in items]),
"r5_mean": mean_or_none([x["r5"] for x in items]),
"total_without_r4_mean": mean_or_none([x["total_without_r4"] for x in items]),
"total_neutral_safe_empty_r2_mean": mean_or_none(
[x["total_neutral_safe_empty_r2"] for x in items]),
}
by_category: dict[str, list[dict]] = defaultdict(list)
by_gold: dict[str, list[dict]] = defaultdict(list)
for record in records:
by_category[f"gold_{record['gold']}_pred_{record['predicted']}"].append(record)
by_gold[record["gold"]].append(record)
all_rows = summarize(records)
return {
"n_rows": len(records),
"by_gold_label": {key: summarize(value) for key, value in sorted(by_gold.items())},
"by_gold_predicted_label": {key: summarize(value) for key, value in sorted(by_category.items())},
"specific_mechanisms": {
"correct_safe_rows_with_empty_statements": sum(x["safe_empty_r2_counterfactual_applied"] for x in records),
"safe_empty_r2_bonus_total_fixed_output": float(sum(
x["total_without_r4"] - x["total_neutral_safe_empty_r2"] for x in records)),
"vulnerable_false_negatives_with_safe_statement_penalty": sum(x["fn_safe_statement_penalty"] for x in records),
"vulnerable_false_negative_rows": sum(x["gold"] == "VULNERABLE" and x["predicted"] == "SAFE" for x in records),
},
"all_rows": all_rows,
"scope": "production score_sample fixed-output replay with r4=None; no group difficulty weights, policy gradients, training, or reward semantics changed",
}
def main() -> dict:
valid_pair_ids = set(map(str, json.loads(VALID.read_text())))
corpus = pd.read_parquet(CORPUS).set_index("sample_id", drop=False)
valid_rows = corpus[corpus["pair_id"].astype(str).isin(valid_pair_ids)]
if len(valid_rows) != 2208:
raise ValueError(f"expected all 2,208 VALID rows, found {len(valid_rows)}")
annotations = {}
with ANNOTATIONS.open() as stream:
for line in stream:
if line.strip():
record = json.loads(line)
annotations[record["sample_id"]] = record
reports = {name: replay(path, corpus, annotations) for name, path in MODEL_FILES.items()}
result = {
"purpose": "fixed-output audit of R1/R2/R3/R5 mechanics on matched full chronological VALID predictions",
"models": reports,
"provenance": {
"valid_split_sha256": sha256(VALID),
"corpus_sha256": sha256(CORPUS),
"annotations_sha256": sha256(ANNOTATIONS),
"sft_predictions_sha256": sha256(MODEL_FILES["sft_v3_step500"]),
"grpo_predictions_sha256": sha256(MODEL_FILES["grpo_step324"]),
"test_read": False,
},
"limitations": [
"R4 is omitted because the matched offline evaluation did not produce judge scores.",
"Reported totals are unweighted fixed-output arithmetic; they do not reconstruct GRPO group-normalized advantages.",
"The neutral SAFE-empty R2 number is a counterfactual over unchanged outputs and does not predict a retrained policy.",
"R3 is skipped where frozen trace artifacts are unavailable or marked unverifiable by the reward implementation.",
],
}
OUTPUT.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
print(json.dumps(result, indent=2))
return result
if __name__ == "__main__":
main()