| """Offline localization breakdown for the matched full-VALID step-324 evaluation.""" |
| from __future__ import annotations |
|
|
| import hashlib |
| import json |
| from collections import defaultdict |
| from pathlib import Path |
|
|
| import numpy as np |
|
|
| from src.rewards.cache import cache_path |
| from src.rewards.checker import statement_iou |
|
|
|
|
| RUN_DIR = Path("results/phase7/matched_vllm_step324_fullvalid") |
| OUTPUT = Path("results/phase7/step324_localization_breakdown.json") |
| CORRECTED_METRICS = Path("results/phase7/matched_vllm_step324_fullvalid/paired_metrics_corrected.json") |
|
|
|
|
| def _load_jsonl(path): |
| return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] |
|
|
|
|
| def _bootstrap(values, seed, replicates=10000): |
| arr = np.asarray(values, dtype=float) |
| if not len(arr): |
| return None |
| rng = np.random.default_rng(seed) |
| draws = arr[rng.integers(0, len(arr), size=(replicates, len(arr)))].mean(axis=1) |
| return [float(x) for x in np.quantile(draws, [.025, .975])] |
|
|
|
|
| def _summarize(rows, annotations, model_name, graph_dir): |
| vulnerable = [] |
| safe = [] |
| cache_trace_eligible = 0 |
| trace_gt_count = 0 |
| for row in rows: |
| ann = annotations[row["sample_id"]] |
| pred_lines = set(int(x) for x in row["pred_lines"]) |
| trace_lines = [int(step["line"]) for step in row["parsed_trace"]] |
| if int(row["target"]) == 1: |
| l1 = set(int(x) for x in ann.get("l1_lines", [])) |
| l2 = set(int(x) for x in ann.get("l2_lines", [])) |
| gold_union = l1 | l2 |
| vulnerable.append({ |
| "sample_id": row["sample_id"], "pair_id": row["pair_id"], |
| "label_correct": bool(row["format_ok"] and row["pred_label"] == "VULNERABLE"), |
| "format_ok": bool(row["format_ok"]), |
| "predicted_label": row["pred_label"], |
| "pred_line_count": len(pred_lines), "pred_empty": not pred_lines, |
| "iou_l2": statement_iou(pred_lines, l2), |
| "iou_l1_union_l2": statement_iou(pred_lines, gold_union), |
| "l2_recall": len(pred_lines & l2) / len(l2) if l2 else None, |
| "l2_precision": len(pred_lines & l2) / len(pred_lines) if pred_lines else 0.0, |
| "trace_step_count": len(trace_lines), |
| "trace_gt_available": bool(ann.get("trace_gt")), |
| "trace_graph_available": cache_path(graph_dir, row["sample_id"]).exists(), |
| }) |
| trace_gt_count += int(bool(ann.get("trace_gt"))) |
| cache_trace_eligible += int(bool(ann.get("trace_gt")) and cache_path(graph_dir, row["sample_id"]).exists()) |
| else: |
| safe.append({"sample_id": row["sample_id"], "pair_id": row["pair_id"], |
| "label_correct": bool(row["format_ok"] and row["pred_label"] == "SAFE"), |
| "format_ok": bool(row["format_ok"]), |
| "predicted_label": row["pred_label"], |
| "pred_line_count": len(pred_lines), "pred_empty": not pred_lines, |
| "trace_step_count": len(trace_lines)}) |
|
|
| def localization_summary(group): |
| return { |
| "n": len(group), |
| "mean_iou_l2": float(np.mean([x["iou_l2"] for x in group])) if group else None, |
| "mean_iou_l1_union_l2": float(np.mean([x["iou_l1_union_l2"] for x in group])) if group else None, |
| "mean_l2_recall": float(np.mean([x["l2_recall"] for x in group if x["l2_recall"] is not None])) if any(x["l2_recall"] is not None for x in group) else None, |
| "mean_l2_precision": float(np.mean([x["l2_precision"] for x in group])) if group else None, |
| "empty_prediction_fraction": float(np.mean([x["pred_empty"] for x in group])) if group else None, |
| "mean_predicted_statement_lines": float(np.mean([x["pred_line_count"] for x in group])) if group else None, |
| } |
|
|
| correct = [x for x in vulnerable if x["label_correct"]] |
| incorrect = [x for x in vulnerable if not x["label_correct"]] |
| parsed_vulnerable = [x for x in vulnerable if x["format_ok"]] |
| return { |
| "model": model_name, |
| "vulnerable_all": localization_summary(vulnerable), |
| "vulnerable_parsed_only": localization_summary(parsed_vulnerable), |
| "vulnerable_label_correct": localization_summary(correct), |
| "vulnerable_label_incorrect": localization_summary(incorrect), |
| "safe_rows": {"n": len(safe), |
| "label_correct": sum(x["label_correct"] for x in safe), |
| "nonempty_statement_fraction": float(np.mean([not x["pred_empty"] for x in safe])) if safe else None, |
| "mean_statement_lines": float(np.mean([x["pred_line_count"] for x in safe])) if safe else None}, |
| "trace_ground_truth": {"vulnerable_rows_with_trace_gt": trace_gt_count, |
| "vulnerable_rows_with_cached_dfg": cache_trace_eligible, |
| "vulnerable_rows_with_any_cached_graph": sum(x["trace_graph_available"] for x in vulnerable), |
| "trace_steps_present_fraction": float(np.mean([x["trace_step_count"] > 0 for x in vulnerable])) if vulnerable else None, |
| "mean_trace_steps": float(np.mean([x["trace_step_count"] for x in vulnerable])) if vulnerable else None}, |
| "per_vulnerable_row": vulnerable, |
| } |
|
|
|
|
| def run(): |
| output = json.loads((RUN_DIR / "paired_metrics.json").read_text()) |
| sft = _load_jsonl(RUN_DIR / "sft_v3_step500_predictions.jsonl") |
| grpo = _load_jsonl(RUN_DIR / "grpo_step324_predictions.jsonl") |
| if len(sft) != 2208 or len(grpo) != 2208: |
| raise ValueError("expected both models to cover all 2,208 VALID rows") |
| if [(r["sample_id"], r["prompt_sha256"]) for r in sft] != [ |
| (r["sample_id"], r["prompt_sha256"]) for r in grpo]: |
| raise ValueError("paired evaluation rows/prompts differ") |
| valid_pairs = set(json.loads(Path("data/splits/chronological/valid.json").read_text())) |
| if {str(r["pair_id"]) for r in sft} != valid_pairs: |
| raise ValueError("evaluation predictions do not exactly match chronological VALID") |
| annotations = {} |
| valid_ids = {r["sample_id"] for r in sft} |
| for line in Path("data/processed/annotations.jsonl").read_text().splitlines(): |
| if line.strip(): |
| ann = json.loads(line) |
| if ann.get("sample_id") in valid_ids: |
| annotations[ann["sample_id"]] = ann |
| if set(annotations) != valid_ids: |
| raise ValueError("some VALID sample annotations are missing") |
|
|
| |
| |
| from src.contracts.parser import parse_output |
| for row in (*sft, *grpo): |
| parsed = parse_output(row["completion"], contract_version=2) |
| row["parsed_trace"] = [{"line": step.line} for step in parsed.trace] |
|
|
| graph_dir = Path("data/cpg_cache/reward_graphs") |
| sft_result = _summarize(sft, annotations, "sft_v3_step500", graph_dir) |
| grpo_result = _summarize(grpo, annotations, "grpo_step324", graph_dir) |
| sft_by_id = {r["sample_id"]: r for r in sft_result["per_vulnerable_row"]} |
| grpo_by_id = {r["sample_id"]: r for r in grpo_result["per_vulnerable_row"]} |
| paired = [grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"] for sid in sorted(sft_by_id)] |
| jointly_parsed_ids = sorted(sid for sid in sft_by_id |
| if sft_by_id[sid]["format_ok"] and grpo_by_id[sid]["format_ok"]) |
| jointly_parsed_delta = [grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"] |
| for sid in jointly_parsed_ids] |
| union_delta = [grpo_by_id[sid]["iou_l1_union_l2"] - sft_by_id[sid]["iou_l1_union_l2"] for sid in sorted(sft_by_id)] |
| by_pair = defaultdict(list) |
| for sid in sorted(sft_by_id): |
| pid = grpo_by_id[sid]["pair_id"] |
| by_pair[pid].append((grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"], |
| grpo_by_id[sid]["iou_l1_union_l2"] - sft_by_id[sid]["iou_l1_union_l2"])) |
| pair_deltas = [np.mean(v, axis=0) for v in by_pair.values()] |
| rng = np.random.default_rng(42) |
| pick = rng.integers(0, len(pair_deltas), size=(10000, len(pair_deltas))) |
| boot = np.asarray(pair_deltas)[pick].mean(axis=1) |
| parsed_pair_values = [] |
| for pair_id in sorted({grpo_by_id[sid]["pair_id"] for sid in jointly_parsed_ids}): |
| ids = [sid for sid in jointly_parsed_ids if grpo_by_id[sid]["pair_id"] == pair_id] |
| parsed_pair_values.append(float(np.mean([grpo_by_id[sid]["iou_l2"] - |
| sft_by_id[sid]["iou_l2"] for sid in ids]))) |
| parsed_rng = np.random.default_rng(43) |
| parsed_pick = parsed_rng.integers(0, len(parsed_pair_values), |
| size=(10000, len(parsed_pair_values))) |
| parsed_boot = np.asarray(parsed_pair_values)[parsed_pick].mean(axis=1) |
| enriched = {} |
| for name, model_rows in (("sft_v3_step500", sft), ("grpo_step324", grpo)): |
| enriched[name] = [{**row, |
| "gold_lines": (list(annotations[row["sample_id"]].get("l2_lines", [])) |
| if int(row["target"]) == 1 else [])} |
| for row in model_rows] |
| from scripts.eval_paired_adapters import paired_summary |
| corrected_paired_metrics = paired_summary(enriched) |
| result = { |
| "purpose": "offline full-VALID localization breakdown of preserved matched greedy outputs", |
| "n_rows": 2208, "n_vulnerable_rows": 1104, "n_pairs": 1104, |
| "models": {"sft_v3_step500": {k: v for k, v in sft_result.items() if k != "per_vulnerable_row"}, |
| "grpo_step324": {k: v for k, v in grpo_result.items() if k != "per_vulnerable_row"}}, |
| "paired_grpo_minus_sft_iou_delta": { |
| "mean_iou_l2_delta": float(np.mean(paired)), |
| "pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(boot[:, 0], [.025, .975])], |
| "parsed_both_models": {"n_vulnerable_rows": len(jointly_parsed_ids), |
| "mean_iou_l2_delta": float(np.mean(jointly_parsed_delta)), |
| "pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(parsed_boot, [.025, .975])]}, |
| "mean_iou_l1_union_l2_delta": float(np.mean(union_delta)), |
| "l1_union_l2_pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(boot[:, 1], [.025, .975])]}, |
| "corrected_matched_validation_metrics": corrected_paired_metrics, |
| "corrected_metric_note": "Recomputed from preserved prediction JSONL plus frozen VALID annotations; the original paired_metrics.json omitted gold_lines during generation, so its statement-IoU fields were invalid.", |
| "r3_evaluation": "unavailable: 0/831 VALID vulnerable rows with trace_gt had a cached DFG graph; no new graph builds were run", |
| "provenance": { |
| "sft_predictions_sha256": hashlib.sha256((RUN_DIR / "sft_v3_step500_predictions.jsonl").read_bytes()).hexdigest(), |
| "grpo_predictions_sha256": hashlib.sha256((RUN_DIR / "grpo_step324_predictions.jsonl").read_bytes()).hexdigest(), |
| "annotations_sha256": hashlib.sha256(Path("data/processed/annotations.jsonl").read_bytes()).hexdigest(), |
| "valid_split_sha256": hashlib.sha256(Path("data/splits/chronological/valid.json").read_bytes()).hexdigest(), |
| "paired_metrics_sha256": hashlib.sha256((RUN_DIR / "paired_metrics.json").read_bytes()).hexdigest(), |
| "test_read": False}, |
| "limitations": ["Statement localization uses the frozen L2 and L1∪L2 line sets; no annotations were changed.", |
| "Trace step counts are descriptive only; DFG validity cannot be scored on VALID because no cached VALID DFGs exist.", |
| "Predictions are from one greedy pass per model and one frozen checkpoint pair."]} |
| result["models"]["sft_v3_step500"]["per_vulnerable_row"] = sft_result["per_vulnerable_row"] |
| result["models"]["grpo_step324"]["per_vulnerable_row"] = grpo_result["per_vulnerable_row"] |
| OUTPUT.parent.mkdir(parents=True, exist_ok=True) |
| OUTPUT.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") |
| corrected = {**corrected_paired_metrics, |
| "created_by": "scripts/analyze_step324_localization.py", |
| "gold_lines_source": "data/processed/annotations.jsonl, L2 for vulnerable rows and empty for SAFE rows", |
| "original_paired_metrics_sha256": hashlib.sha256((RUN_DIR / "paired_metrics.json").read_bytes()).hexdigest(), |
| "test_touched": False} |
| CORRECTED_METRICS.write_text(json.dumps(corrected, indent=2, sort_keys=True) + "\n") |
| return result |
|
|
|
|
| if __name__ == "__main__": |
| print(json.dumps(run(), indent=2)) |
|
|
