File size: 6,011 Bytes
646ccfc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | #!/usr/bin/env python3
"""Decompose paired VALID IoU changes by SFT-to-GRPO label transition.
Run from the repository root. This is descriptive analysis of the frozen
chronological VALID outputs; it does not access the TEST split.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import random
from collections import defaultdict
from pathlib import Path
from src.rewards.checker import statement_iou
def load_rows(path: Path) -> dict[str, dict]:
rows = {}
with path.open() as handle:
for line in handle:
if not line.strip():
continue
row = json.loads(line)
sid = row["sample_id"]
if sid in rows:
raise ValueError(f"duplicate sample_id: {sid}")
rows[sid] = row
return rows
def predicted_label(row: dict) -> str:
return row.get("pred_label") if row.get("format_ok") else "INVALID"
def row_iou(row: dict) -> float:
# Match validation evaluation: invalid/unparsed rows contribute zero;
# parsed rows are scored regardless of their predicted class.
if not row.get("format_ok"):
return 0.0
return statement_iou(set(row.get("pred_lines") or []),
set(row.get("gold_lines") or []))
def sha256(path: Path) -> str:
h = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
def bootstrap_mean(values: list[float], *, seed: int, reps: int = 20_000) -> list[float] | None:
if not values:
return None
rng = random.Random(seed)
n = len(values)
means = sorted(sum(rng.choice(values) for _ in range(n)) / n
for _ in range(reps))
return [means[int(0.025 * reps)], means[int(0.975 * reps) - 1]]
def analyze(sft_path: Path, grpo_path: Path, *, seed: int = 20261008) -> dict:
sft, grpo = load_rows(sft_path), load_rows(grpo_path)
if sft.keys() != grpo.keys():
raise ValueError("SFT and GRPO sample_id sets differ")
grouped: dict[tuple[str, str], list[tuple[float, float]]] = defaultdict(list)
overall_sft, overall_grpo = [], []
class_parity = True
for sid, a in sft.items():
b = grpo[sid]
if int(a["target"]) != int(b["target"]):
class_parity = False
raise ValueError(f"target mismatch for {sid}")
if set(a.get("gold_lines") or []) != set(b.get("gold_lines") or []):
raise ValueError(f"gold line mismatch for {sid}")
if int(a["target"]) != 1:
continue
sa, gb = row_iou(a), row_iou(b)
overall_sft.append(sa)
overall_grpo.append(gb)
grouped[(predicted_label(a), predicted_label(b))].append((sa, gb))
n = len(overall_sft)
transitions = {}
for (sft_label, grpo_label), values in sorted(grouped.items()):
ds = [b - a for a, b in values]
transitions[f"{sft_label}->{grpo_label}"] = {
"n": len(values),
"sft_mean_iou": sum(a for a, _ in values) / len(values),
"grpo_mean_iou": sum(b for _, b in values) / len(values),
"paired_mean_delta": sum(ds) / len(ds),
"contribution_to_overall_mean_delta": sum(ds) / n,
}
both_vulnerable = grouped.get(("VULNERABLE", "VULNERABLE"), [])
common_deltas = [b - a for a, b in both_vulnerable]
overall_deltas = [b - a for a, b in zip(overall_sft, overall_grpo)]
return {
"analysis": "gold-vulnerable VALID localization by paired label transition",
"n_valid_rows_paired": len(sft),
"n_gold_vulnerable": n,
"gold_target_and_lines_parity_verified": class_parity,
"test_touched": False,
"overall": {
"sft_mean_iou": sum(overall_sft) / n,
"grpo_step200_mean_iou": sum(overall_grpo) / n,
"paired_mean_delta": sum(overall_deltas) / n,
"paired_row_bootstrap_95_ci": bootstrap_mean(overall_deltas, seed=seed),
},
"transitions": transitions,
"both_predicted_vulnerable": {
"n": len(common_deltas),
"sft_mean_iou": transitions.get("VULNERABLE->VULNERABLE", {}).get("sft_mean_iou"),
"grpo_mean_iou": transitions.get("VULNERABLE->VULNERABLE", {}).get("grpo_mean_iou"),
"paired_mean_delta": (sum(common_deltas) / len(common_deltas)
if common_deltas else None),
"paired_row_bootstrap_95_ci": bootstrap_mean(common_deltas, seed=seed + 1),
},
"bootstrap": {"unit": "gold-vulnerable sample (one per pair)",
"replicates": 20_000, "seed": seed},
"input_sha256": {str(sft_path): sha256(sft_path),
str(grpo_path): sha256(grpo_path)},
"interpretation_caveat": (
"Transitions condition on model predictions and are descriptive, not causal. "
"An overall IoU increase can reflect more vulnerable rows receiving a "
"VULNERABLE prediction and nonempty localization; it does not alone show "
"better localization precision on rows both models detect."
),
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--sft", type=Path,
default=Path("results/phase4/contract_v2/sft_val_step_500.jsonl"))
parser.add_argument("--grpo", type=Path,
default=Path("results/phase6/scale_20261007_v5/valid_predictions.jsonl"))
parser.add_argument("--output", type=Path,
default=Path("results/phase6/scale_20261007_v5/localization_transition_decomposition_step200.json"))
args = parser.parse_args()
result = analyze(args.sft, args.grpo)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, indent=2) + "\n")
print(json.dumps(result, indent=2))
if __name__ == "__main__":
main()
|