Download scripts/audit_step324_valid_reward_alignment.py from Dhruv1000/tracegrpo: direct link, hf CLI and curl.
- Browser
- Download file 7.48 kB
-
https://huggingface.co/Dhruv1000/tracegrpo/resolve/main/scripts/audit_step324_valid_reward_alignment.py
- Command line
-
hf download hf://Dhruv1000/tracegrpo/scripts/audit_step324_valid_reward_alignment.py
-
curl -L -o audit_step324_valid_reward_alignment.py https://huggingface.co/Dhruv1000/tracegrpo/resolve/main/scripts/audit_step324_valid_reward_alignment.py
7.48 kB
| """Replay production R1-R5 arithmetic on frozen step-324 and SFT VALID outputs.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| from collections import defaultdict | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| from src.contracts.parser import parse_output | |
| from src.contracts.prompt import code_for_sample | |
| from src.rewards.score_sample import score_sample | |
| ROOT = Path(__file__).resolve().parents[1] | |
| RUN = ROOT / "results/phase7/matched_vllm_step324_fullvalid" | |
| CORPUS = ROOT / "data/processed/corpus.parquet" | |
| ANNOTATIONS = ROOT / "data/processed/annotations.jsonl" | |
| VALID = ROOT / "data/splits/chronological/valid.json" | |
| GRAPH_CACHE = ROOT / "data/cpg_cache/phase7_valid_reward_graphs" | |
| OUTPUT = ROOT / "results/phase7/step324_reward_alignment.json" | |
| MODEL_FILES = { | |
| "sft_v3_step500": RUN / "sft_v3_step500_predictions.jsonl", | |
| "grpo_step324": RUN / "grpo_step324_predictions.jsonl", | |
| } | |
| def sha256(path: Path) -> str: | |
| return hashlib.sha256(path.read_bytes()).hexdigest() | |
| def mean_or_none(values: list[float]) -> float | None: | |
| return float(np.mean(values)) if values else None | |
| def replay(path: Path, corpus: pd.DataFrame, annotations: dict[str, dict]) -> dict: | |
| records = [] | |
| with path.open() as stream: | |
| for line in stream: | |
| if not line.strip(): | |
| continue | |
| row = json.loads(line) | |
| parsed = parse_output(row["completion"], contract_version=2) | |
| source = corpus.loc[str(row["sample_id"])] | |
| target = int(row["target"]) | |
| ann = annotations[str(row["sample_id"])] | |
| func_len = len(code_for_sample(source).splitlines()) | |
| reward = score_sample( | |
| parsed, | |
| sample_id=str(row["sample_id"]), | |
| target=target, | |
| annotation=ann, | |
| func_len=func_len, | |
| cache_dir=str(GRAPH_CACHE), | |
| r4=None, | |
| ) | |
| gold = "VULNERABLE" if target == 1 else "SAFE" | |
| predicted = parsed.label if parsed.format_ok else "INVALID" | |
| counterfactual_total = reward.total | |
| safe_empty_r2 = bool(target == 0 and predicted == "SAFE" | |
| and not parsed.statement_lines and reward.r2 == 1.0) | |
| if safe_empty_r2: | |
| # Fixed-output arithmetic only: neutralize the empty/empty IoU bonus. | |
| counterfactual_total -= reward.r1 * float(reward.r2) | |
| records.append({ | |
| "sample_id": str(row["sample_id"]), | |
| "gold": gold, | |
| "predicted": predicted, | |
| "correct": bool(parsed.format_ok and parsed.label == gold), | |
| "format_ok": bool(parsed.format_ok), | |
| "nonempty_statements": bool(parsed.statement_lines), | |
| "r1": reward.r1, | |
| "r2": reward.r2, | |
| "r3": reward.r3, | |
| "r5": reward.r5, | |
| "total_without_r4": reward.total, | |
| "total_neutral_safe_empty_r2": counterfactual_total, | |
| "safe_empty_r2_counterfactual_applied": safe_empty_r2, | |
| "fn_safe_statement_penalty": bool(target == 1 and predicted == "SAFE" | |
| and parsed.statement_lines and reward.r5 <= -0.5), | |
| }) | |
| def summarize(items: list[dict]) -> dict: | |
| return { | |
| "n": len(items), | |
| "correct_fraction": mean_or_none([float(x["correct"]) for x in items]), | |
| "r1_mean": mean_or_none([x["r1"] for x in items]), | |
| "r2_mean_including_gated_zero": mean_or_none([float(x["r2"] or 0.0) for x in items]), | |
| "r2_eligible_fraction": mean_or_none([float(x["r2"] is not None) for x in items]), | |
| "r3_mean_including_gated_zero": mean_or_none([float(x["r3"] or 0.0) for x in items]), | |
| "r3_eligible_fraction": mean_or_none([float(x["r3"] is not None) for x in items]), | |
| "r5_mean": mean_or_none([x["r5"] for x in items]), | |
| "total_without_r4_mean": mean_or_none([x["total_without_r4"] for x in items]), | |
| "total_neutral_safe_empty_r2_mean": mean_or_none( | |
| [x["total_neutral_safe_empty_r2"] for x in items]), | |
| } | |
| by_category: dict[str, list[dict]] = defaultdict(list) | |
| by_gold: dict[str, list[dict]] = defaultdict(list) | |
| for record in records: | |
| by_category[f"gold_{record['gold']}_pred_{record['predicted']}"].append(record) | |
| by_gold[record["gold"]].append(record) | |
| all_rows = summarize(records) | |
| return { | |
| "n_rows": len(records), | |
| "by_gold_label": {key: summarize(value) for key, value in sorted(by_gold.items())}, | |
| "by_gold_predicted_label": {key: summarize(value) for key, value in sorted(by_category.items())}, | |
| "specific_mechanisms": { | |
| "correct_safe_rows_with_empty_statements": sum(x["safe_empty_r2_counterfactual_applied"] for x in records), | |
| "safe_empty_r2_bonus_total_fixed_output": float(sum( | |
| x["total_without_r4"] - x["total_neutral_safe_empty_r2"] for x in records)), | |
| "vulnerable_false_negatives_with_safe_statement_penalty": sum(x["fn_safe_statement_penalty"] for x in records), | |
| "vulnerable_false_negative_rows": sum(x["gold"] == "VULNERABLE" and x["predicted"] == "SAFE" for x in records), | |
| }, | |
| "all_rows": all_rows, | |
| "scope": "production score_sample fixed-output replay with r4=None; no group difficulty weights, policy gradients, training, or reward semantics changed", | |
| } | |
| def main() -> dict: | |
| valid_pair_ids = set(map(str, json.loads(VALID.read_text()))) | |
| corpus = pd.read_parquet(CORPUS).set_index("sample_id", drop=False) | |
| valid_rows = corpus[corpus["pair_id"].astype(str).isin(valid_pair_ids)] | |
| if len(valid_rows) != 2208: | |
| raise ValueError(f"expected all 2,208 VALID rows, found {len(valid_rows)}") | |
| annotations = {} | |
| with ANNOTATIONS.open() as stream: | |
| for line in stream: | |
| if line.strip(): | |
| record = json.loads(line) | |
| annotations[record["sample_id"]] = record | |
| reports = {name: replay(path, corpus, annotations) for name, path in MODEL_FILES.items()} | |
| result = { | |
| "purpose": "fixed-output audit of R1/R2/R3/R5 mechanics on matched full chronological VALID predictions", | |
| "models": reports, | |
| "provenance": { | |
| "valid_split_sha256": sha256(VALID), | |
| "corpus_sha256": sha256(CORPUS), | |
| "annotations_sha256": sha256(ANNOTATIONS), | |
| "sft_predictions_sha256": sha256(MODEL_FILES["sft_v3_step500"]), | |
| "grpo_predictions_sha256": sha256(MODEL_FILES["grpo_step324"]), | |
| "test_read": False, | |
| }, | |
| "limitations": [ | |
| "R4 is omitted because the matched offline evaluation did not produce judge scores.", | |
| "Reported totals are unweighted fixed-output arithmetic; they do not reconstruct GRPO group-normalized advantages.", | |
| "The neutral SAFE-empty R2 number is a counterfactual over unchanged outputs and does not predict a retrained policy.", | |
| "R3 is skipped where frozen trace artifacts are unavailable or marked unverifiable by the reward implementation.", | |
| ], | |
| } | |
| OUTPUT.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") | |
| print(json.dumps(result, indent=2)) | |
| return result | |
| if __name__ == "__main__": | |
| main() | |