File size: 12,963 Bytes
646ccfc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 | """Offline localization breakdown for the matched full-VALID step-324 evaluation."""
from __future__ import annotations
import hashlib
import json
from collections import defaultdict
from pathlib import Path
import numpy as np
from src.rewards.cache import cache_path
from src.rewards.checker import statement_iou
RUN_DIR = Path("results/phase7/matched_vllm_step324_fullvalid")
OUTPUT = Path("results/phase7/step324_localization_breakdown.json")
CORRECTED_METRICS = Path("results/phase7/matched_vllm_step324_fullvalid/paired_metrics_corrected.json")
def _load_jsonl(path):
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
def _bootstrap(values, seed, replicates=10000):
arr = np.asarray(values, dtype=float)
if not len(arr):
return None
rng = np.random.default_rng(seed)
draws = arr[rng.integers(0, len(arr), size=(replicates, len(arr)))].mean(axis=1)
return [float(x) for x in np.quantile(draws, [.025, .975])]
def _summarize(rows, annotations, model_name, graph_dir):
vulnerable = []
safe = []
cache_trace_eligible = 0
trace_gt_count = 0
for row in rows:
ann = annotations[row["sample_id"]]
pred_lines = set(int(x) for x in row["pred_lines"])
trace_lines = [int(step["line"]) for step in row["parsed_trace"]]
if int(row["target"]) == 1:
l1 = set(int(x) for x in ann.get("l1_lines", []))
l2 = set(int(x) for x in ann.get("l2_lines", []))
gold_union = l1 | l2
vulnerable.append({
"sample_id": row["sample_id"], "pair_id": row["pair_id"],
"label_correct": bool(row["format_ok"] and row["pred_label"] == "VULNERABLE"),
"format_ok": bool(row["format_ok"]),
"predicted_label": row["pred_label"],
"pred_line_count": len(pred_lines), "pred_empty": not pred_lines,
"iou_l2": statement_iou(pred_lines, l2),
"iou_l1_union_l2": statement_iou(pred_lines, gold_union),
"l2_recall": len(pred_lines & l2) / len(l2) if l2 else None,
"l2_precision": len(pred_lines & l2) / len(pred_lines) if pred_lines else 0.0,
"trace_step_count": len(trace_lines),
"trace_gt_available": bool(ann.get("trace_gt")),
"trace_graph_available": cache_path(graph_dir, row["sample_id"]).exists(),
})
trace_gt_count += int(bool(ann.get("trace_gt")))
cache_trace_eligible += int(bool(ann.get("trace_gt")) and cache_path(graph_dir, row["sample_id"]).exists())
else:
safe.append({"sample_id": row["sample_id"], "pair_id": row["pair_id"],
"label_correct": bool(row["format_ok"] and row["pred_label"] == "SAFE"),
"format_ok": bool(row["format_ok"]),
"predicted_label": row["pred_label"],
"pred_line_count": len(pred_lines), "pred_empty": not pred_lines,
"trace_step_count": len(trace_lines)})
def localization_summary(group):
return {
"n": len(group),
"mean_iou_l2": float(np.mean([x["iou_l2"] for x in group])) if group else None,
"mean_iou_l1_union_l2": float(np.mean([x["iou_l1_union_l2"] for x in group])) if group else None,
"mean_l2_recall": float(np.mean([x["l2_recall"] for x in group if x["l2_recall"] is not None])) if any(x["l2_recall"] is not None for x in group) else None,
"mean_l2_precision": float(np.mean([x["l2_precision"] for x in group])) if group else None,
"empty_prediction_fraction": float(np.mean([x["pred_empty"] for x in group])) if group else None,
"mean_predicted_statement_lines": float(np.mean([x["pred_line_count"] for x in group])) if group else None,
}
correct = [x for x in vulnerable if x["label_correct"]]
incorrect = [x for x in vulnerable if not x["label_correct"]]
parsed_vulnerable = [x for x in vulnerable if x["format_ok"]]
return {
"model": model_name,
"vulnerable_all": localization_summary(vulnerable),
"vulnerable_parsed_only": localization_summary(parsed_vulnerable),
"vulnerable_label_correct": localization_summary(correct),
"vulnerable_label_incorrect": localization_summary(incorrect),
"safe_rows": {"n": len(safe),
"label_correct": sum(x["label_correct"] for x in safe),
"nonempty_statement_fraction": float(np.mean([not x["pred_empty"] for x in safe])) if safe else None,
"mean_statement_lines": float(np.mean([x["pred_line_count"] for x in safe])) if safe else None},
"trace_ground_truth": {"vulnerable_rows_with_trace_gt": trace_gt_count,
"vulnerable_rows_with_cached_dfg": cache_trace_eligible,
"vulnerable_rows_with_any_cached_graph": sum(x["trace_graph_available"] for x in vulnerable),
"trace_steps_present_fraction": float(np.mean([x["trace_step_count"] > 0 for x in vulnerable])) if vulnerable else None,
"mean_trace_steps": float(np.mean([x["trace_step_count"] for x in vulnerable])) if vulnerable else None},
"per_vulnerable_row": vulnerable,
}
def run():
output = json.loads((RUN_DIR / "paired_metrics.json").read_text())
sft = _load_jsonl(RUN_DIR / "sft_v3_step500_predictions.jsonl")
grpo = _load_jsonl(RUN_DIR / "grpo_step324_predictions.jsonl")
if len(sft) != 2208 or len(grpo) != 2208:
raise ValueError("expected both models to cover all 2,208 VALID rows")
if [(r["sample_id"], r["prompt_sha256"]) for r in sft] != [
(r["sample_id"], r["prompt_sha256"]) for r in grpo]:
raise ValueError("paired evaluation rows/prompts differ")
valid_pairs = set(json.loads(Path("data/splits/chronological/valid.json").read_text()))
if {str(r["pair_id"]) for r in sft} != valid_pairs:
raise ValueError("evaluation predictions do not exactly match chronological VALID")
annotations = {}
valid_ids = {r["sample_id"] for r in sft}
for line in Path("data/processed/annotations.jsonl").read_text().splitlines():
if line.strip():
ann = json.loads(line)
if ann.get("sample_id") in valid_ids:
annotations[ann["sample_id"]] = ann
if set(annotations) != valid_ids:
raise ValueError("some VALID sample annotations are missing")
# Parsed trace lines are extracted from the raw v2 completions. The cached R3 DFG
# artifacts were built for TRAIN, so VALID DFG trace scores are reported unavailable.
from src.contracts.parser import parse_output
for row in (*sft, *grpo):
parsed = parse_output(row["completion"], contract_version=2)
row["parsed_trace"] = [{"line": step.line} for step in parsed.trace]
graph_dir = Path("data/cpg_cache/reward_graphs")
sft_result = _summarize(sft, annotations, "sft_v3_step500", graph_dir)
grpo_result = _summarize(grpo, annotations, "grpo_step324", graph_dir)
sft_by_id = {r["sample_id"]: r for r in sft_result["per_vulnerable_row"]}
grpo_by_id = {r["sample_id"]: r for r in grpo_result["per_vulnerable_row"]}
paired = [grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"] for sid in sorted(sft_by_id)]
jointly_parsed_ids = sorted(sid for sid in sft_by_id
if sft_by_id[sid]["format_ok"] and grpo_by_id[sid]["format_ok"])
jointly_parsed_delta = [grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"]
for sid in jointly_parsed_ids]
union_delta = [grpo_by_id[sid]["iou_l1_union_l2"] - sft_by_id[sid]["iou_l1_union_l2"] for sid in sorted(sft_by_id)]
by_pair = defaultdict(list)
for sid in sorted(sft_by_id):
pid = grpo_by_id[sid]["pair_id"]
by_pair[pid].append((grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"],
grpo_by_id[sid]["iou_l1_union_l2"] - sft_by_id[sid]["iou_l1_union_l2"]))
pair_deltas = [np.mean(v, axis=0) for v in by_pair.values()]
rng = np.random.default_rng(42)
pick = rng.integers(0, len(pair_deltas), size=(10000, len(pair_deltas)))
boot = np.asarray(pair_deltas)[pick].mean(axis=1)
parsed_pair_values = []
for pair_id in sorted({grpo_by_id[sid]["pair_id"] for sid in jointly_parsed_ids}):
ids = [sid for sid in jointly_parsed_ids if grpo_by_id[sid]["pair_id"] == pair_id]
parsed_pair_values.append(float(np.mean([grpo_by_id[sid]["iou_l2"] -
sft_by_id[sid]["iou_l2"] for sid in ids])))
parsed_rng = np.random.default_rng(43)
parsed_pick = parsed_rng.integers(0, len(parsed_pair_values),
size=(10000, len(parsed_pair_values)))
parsed_boot = np.asarray(parsed_pair_values)[parsed_pick].mean(axis=1)
enriched = {}
for name, model_rows in (("sft_v3_step500", sft), ("grpo_step324", grpo)):
enriched[name] = [{**row,
"gold_lines": (list(annotations[row["sample_id"]].get("l2_lines", []))
if int(row["target"]) == 1 else [])}
for row in model_rows]
from scripts.eval_paired_adapters import paired_summary
corrected_paired_metrics = paired_summary(enriched)
result = {
"purpose": "offline full-VALID localization breakdown of preserved matched greedy outputs",
"n_rows": 2208, "n_vulnerable_rows": 1104, "n_pairs": 1104,
"models": {"sft_v3_step500": {k: v for k, v in sft_result.items() if k != "per_vulnerable_row"},
"grpo_step324": {k: v for k, v in grpo_result.items() if k != "per_vulnerable_row"}},
"paired_grpo_minus_sft_iou_delta": {
"mean_iou_l2_delta": float(np.mean(paired)),
"pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(boot[:, 0], [.025, .975])],
"parsed_both_models": {"n_vulnerable_rows": len(jointly_parsed_ids),
"mean_iou_l2_delta": float(np.mean(jointly_parsed_delta)),
"pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(parsed_boot, [.025, .975])]},
"mean_iou_l1_union_l2_delta": float(np.mean(union_delta)),
"l1_union_l2_pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(boot[:, 1], [.025, .975])]},
"corrected_matched_validation_metrics": corrected_paired_metrics,
"corrected_metric_note": "Recomputed from preserved prediction JSONL plus frozen VALID annotations; the original paired_metrics.json omitted gold_lines during generation, so its statement-IoU fields were invalid.",
"r3_evaluation": "unavailable: 0/831 VALID vulnerable rows with trace_gt had a cached DFG graph; no new graph builds were run",
"provenance": {
"sft_predictions_sha256": hashlib.sha256((RUN_DIR / "sft_v3_step500_predictions.jsonl").read_bytes()).hexdigest(),
"grpo_predictions_sha256": hashlib.sha256((RUN_DIR / "grpo_step324_predictions.jsonl").read_bytes()).hexdigest(),
"annotations_sha256": hashlib.sha256(Path("data/processed/annotations.jsonl").read_bytes()).hexdigest(),
"valid_split_sha256": hashlib.sha256(Path("data/splits/chronological/valid.json").read_bytes()).hexdigest(),
"paired_metrics_sha256": hashlib.sha256((RUN_DIR / "paired_metrics.json").read_bytes()).hexdigest(),
"test_read": False},
"limitations": ["Statement localization uses the frozen L2 and L1∪L2 line sets; no annotations were changed.",
"Trace step counts are descriptive only; DFG validity cannot be scored on VALID because no cached VALID DFGs exist.",
"Predictions are from one greedy pass per model and one frozen checkpoint pair."]}
result["models"]["sft_v3_step500"]["per_vulnerable_row"] = sft_result["per_vulnerable_row"]
result["models"]["grpo_step324"]["per_vulnerable_row"] = grpo_result["per_vulnerable_row"]
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
OUTPUT.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
corrected = {**corrected_paired_metrics,
"created_by": "scripts/analyze_step324_localization.py",
"gold_lines_source": "data/processed/annotations.jsonl, L2 for vulnerable rows and empty for SAFE rows",
"original_paired_metrics_sha256": hashlib.sha256((RUN_DIR / "paired_metrics.json").read_bytes()).hexdigest(),
"test_touched": False}
CORRECTED_METRICS.write_text(json.dumps(corrected, indent=2, sort_keys=True) + "\n")
return result
if __name__ == "__main__":
print(json.dumps(run(), indent=2))
|