File size: 12,963 Bytes
646ccfc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
"""Offline localization breakdown for the matched full-VALID step-324 evaluation."""
from __future__ import annotations

import hashlib
import json
from collections import defaultdict
from pathlib import Path

import numpy as np

from src.rewards.cache import cache_path
from src.rewards.checker import statement_iou


RUN_DIR = Path("results/phase7/matched_vllm_step324_fullvalid")
OUTPUT = Path("results/phase7/step324_localization_breakdown.json")
CORRECTED_METRICS = Path("results/phase7/matched_vllm_step324_fullvalid/paired_metrics_corrected.json")


def _load_jsonl(path):
    return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]


def _bootstrap(values, seed, replicates=10000):
    arr = np.asarray(values, dtype=float)
    if not len(arr):
        return None
    rng = np.random.default_rng(seed)
    draws = arr[rng.integers(0, len(arr), size=(replicates, len(arr)))].mean(axis=1)
    return [float(x) for x in np.quantile(draws, [.025, .975])]


def _summarize(rows, annotations, model_name, graph_dir):
    vulnerable = []
    safe = []
    cache_trace_eligible = 0
    trace_gt_count = 0
    for row in rows:
        ann = annotations[row["sample_id"]]
        pred_lines = set(int(x) for x in row["pred_lines"])
        trace_lines = [int(step["line"]) for step in row["parsed_trace"]]
        if int(row["target"]) == 1:
            l1 = set(int(x) for x in ann.get("l1_lines", []))
            l2 = set(int(x) for x in ann.get("l2_lines", []))
            gold_union = l1 | l2
            vulnerable.append({
                "sample_id": row["sample_id"], "pair_id": row["pair_id"],
                "label_correct": bool(row["format_ok"] and row["pred_label"] == "VULNERABLE"),
                "format_ok": bool(row["format_ok"]),
                "predicted_label": row["pred_label"],
                "pred_line_count": len(pred_lines), "pred_empty": not pred_lines,
                "iou_l2": statement_iou(pred_lines, l2),
                "iou_l1_union_l2": statement_iou(pred_lines, gold_union),
                "l2_recall": len(pred_lines & l2) / len(l2) if l2 else None,
                "l2_precision": len(pred_lines & l2) / len(pred_lines) if pred_lines else 0.0,
                "trace_step_count": len(trace_lines),
                "trace_gt_available": bool(ann.get("trace_gt")),
                "trace_graph_available": cache_path(graph_dir, row["sample_id"]).exists(),
            })
            trace_gt_count += int(bool(ann.get("trace_gt")))
            cache_trace_eligible += int(bool(ann.get("trace_gt")) and cache_path(graph_dir, row["sample_id"]).exists())
        else:
            safe.append({"sample_id": row["sample_id"], "pair_id": row["pair_id"],
                         "label_correct": bool(row["format_ok"] and row["pred_label"] == "SAFE"),
                         "format_ok": bool(row["format_ok"]),
                         "predicted_label": row["pred_label"],
                         "pred_line_count": len(pred_lines), "pred_empty": not pred_lines,
                         "trace_step_count": len(trace_lines)})

    def localization_summary(group):
        return {
            "n": len(group),
            "mean_iou_l2": float(np.mean([x["iou_l2"] for x in group])) if group else None,
            "mean_iou_l1_union_l2": float(np.mean([x["iou_l1_union_l2"] for x in group])) if group else None,
            "mean_l2_recall": float(np.mean([x["l2_recall"] for x in group if x["l2_recall"] is not None])) if any(x["l2_recall"] is not None for x in group) else None,
            "mean_l2_precision": float(np.mean([x["l2_precision"] for x in group])) if group else None,
            "empty_prediction_fraction": float(np.mean([x["pred_empty"] for x in group])) if group else None,
            "mean_predicted_statement_lines": float(np.mean([x["pred_line_count"] for x in group])) if group else None,
        }

    correct = [x for x in vulnerable if x["label_correct"]]
    incorrect = [x for x in vulnerable if not x["label_correct"]]
    parsed_vulnerable = [x for x in vulnerable if x["format_ok"]]
    return {
        "model": model_name,
        "vulnerable_all": localization_summary(vulnerable),
        "vulnerable_parsed_only": localization_summary(parsed_vulnerable),
        "vulnerable_label_correct": localization_summary(correct),
        "vulnerable_label_incorrect": localization_summary(incorrect),
        "safe_rows": {"n": len(safe),
                      "label_correct": sum(x["label_correct"] for x in safe),
                      "nonempty_statement_fraction": float(np.mean([not x["pred_empty"] for x in safe])) if safe else None,
                      "mean_statement_lines": float(np.mean([x["pred_line_count"] for x in safe])) if safe else None},
        "trace_ground_truth": {"vulnerable_rows_with_trace_gt": trace_gt_count,
                                "vulnerable_rows_with_cached_dfg": cache_trace_eligible,
                                "vulnerable_rows_with_any_cached_graph": sum(x["trace_graph_available"] for x in vulnerable),
                                "trace_steps_present_fraction": float(np.mean([x["trace_step_count"] > 0 for x in vulnerable])) if vulnerable else None,
                                "mean_trace_steps": float(np.mean([x["trace_step_count"] for x in vulnerable])) if vulnerable else None},
        "per_vulnerable_row": vulnerable,
    }


def run():
    output = json.loads((RUN_DIR / "paired_metrics.json").read_text())
    sft = _load_jsonl(RUN_DIR / "sft_v3_step500_predictions.jsonl")
    grpo = _load_jsonl(RUN_DIR / "grpo_step324_predictions.jsonl")
    if len(sft) != 2208 or len(grpo) != 2208:
        raise ValueError("expected both models to cover all 2,208 VALID rows")
    if [(r["sample_id"], r["prompt_sha256"]) for r in sft] != [
            (r["sample_id"], r["prompt_sha256"]) for r in grpo]:
        raise ValueError("paired evaluation rows/prompts differ")
    valid_pairs = set(json.loads(Path("data/splits/chronological/valid.json").read_text()))
    if {str(r["pair_id"]) for r in sft} != valid_pairs:
        raise ValueError("evaluation predictions do not exactly match chronological VALID")
    annotations = {}
    valid_ids = {r["sample_id"] for r in sft}
    for line in Path("data/processed/annotations.jsonl").read_text().splitlines():
        if line.strip():
            ann = json.loads(line)
            if ann.get("sample_id") in valid_ids:
                annotations[ann["sample_id"]] = ann
    if set(annotations) != valid_ids:
        raise ValueError("some VALID sample annotations are missing")

    # Parsed trace lines are extracted from the raw v2 completions. The cached R3 DFG
    # artifacts were built for TRAIN, so VALID DFG trace scores are reported unavailable.
    from src.contracts.parser import parse_output
    for row in (*sft, *grpo):
        parsed = parse_output(row["completion"], contract_version=2)
        row["parsed_trace"] = [{"line": step.line} for step in parsed.trace]

    graph_dir = Path("data/cpg_cache/reward_graphs")
    sft_result = _summarize(sft, annotations, "sft_v3_step500", graph_dir)
    grpo_result = _summarize(grpo, annotations, "grpo_step324", graph_dir)
    sft_by_id = {r["sample_id"]: r for r in sft_result["per_vulnerable_row"]}
    grpo_by_id = {r["sample_id"]: r for r in grpo_result["per_vulnerable_row"]}
    paired = [grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"] for sid in sorted(sft_by_id)]
    jointly_parsed_ids = sorted(sid for sid in sft_by_id
                                if sft_by_id[sid]["format_ok"] and grpo_by_id[sid]["format_ok"])
    jointly_parsed_delta = [grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"]
                            for sid in jointly_parsed_ids]
    union_delta = [grpo_by_id[sid]["iou_l1_union_l2"] - sft_by_id[sid]["iou_l1_union_l2"] for sid in sorted(sft_by_id)]
    by_pair = defaultdict(list)
    for sid in sorted(sft_by_id):
        pid = grpo_by_id[sid]["pair_id"]
        by_pair[pid].append((grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"],
                             grpo_by_id[sid]["iou_l1_union_l2"] - sft_by_id[sid]["iou_l1_union_l2"]))
    pair_deltas = [np.mean(v, axis=0) for v in by_pair.values()]
    rng = np.random.default_rng(42)
    pick = rng.integers(0, len(pair_deltas), size=(10000, len(pair_deltas)))
    boot = np.asarray(pair_deltas)[pick].mean(axis=1)
    parsed_pair_values = []
    for pair_id in sorted({grpo_by_id[sid]["pair_id"] for sid in jointly_parsed_ids}):
        ids = [sid for sid in jointly_parsed_ids if grpo_by_id[sid]["pair_id"] == pair_id]
        parsed_pair_values.append(float(np.mean([grpo_by_id[sid]["iou_l2"] -
                                                 sft_by_id[sid]["iou_l2"] for sid in ids])))
    parsed_rng = np.random.default_rng(43)
    parsed_pick = parsed_rng.integers(0, len(parsed_pair_values),
                                      size=(10000, len(parsed_pair_values)))
    parsed_boot = np.asarray(parsed_pair_values)[parsed_pick].mean(axis=1)
    enriched = {}
    for name, model_rows in (("sft_v3_step500", sft), ("grpo_step324", grpo)):
        enriched[name] = [{**row,
                           "gold_lines": (list(annotations[row["sample_id"]].get("l2_lines", []))
                                          if int(row["target"]) == 1 else [])}
                          for row in model_rows]
    from scripts.eval_paired_adapters import paired_summary
    corrected_paired_metrics = paired_summary(enriched)
    result = {
        "purpose": "offline full-VALID localization breakdown of preserved matched greedy outputs",
        "n_rows": 2208, "n_vulnerable_rows": 1104, "n_pairs": 1104,
        "models": {"sft_v3_step500": {k: v for k, v in sft_result.items() if k != "per_vulnerable_row"},
                   "grpo_step324": {k: v for k, v in grpo_result.items() if k != "per_vulnerable_row"}},
        "paired_grpo_minus_sft_iou_delta": {
            "mean_iou_l2_delta": float(np.mean(paired)),
            "pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(boot[:, 0], [.025, .975])],
            "parsed_both_models": {"n_vulnerable_rows": len(jointly_parsed_ids),
                                   "mean_iou_l2_delta": float(np.mean(jointly_parsed_delta)),
                                   "pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(parsed_boot, [.025, .975])]},
            "mean_iou_l1_union_l2_delta": float(np.mean(union_delta)),
            "l1_union_l2_pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(boot[:, 1], [.025, .975])]},
        "corrected_matched_validation_metrics": corrected_paired_metrics,
        "corrected_metric_note": "Recomputed from preserved prediction JSONL plus frozen VALID annotations; the original paired_metrics.json omitted gold_lines during generation, so its statement-IoU fields were invalid.",
        "r3_evaluation": "unavailable: 0/831 VALID vulnerable rows with trace_gt had a cached DFG graph; no new graph builds were run",
        "provenance": {
            "sft_predictions_sha256": hashlib.sha256((RUN_DIR / "sft_v3_step500_predictions.jsonl").read_bytes()).hexdigest(),
            "grpo_predictions_sha256": hashlib.sha256((RUN_DIR / "grpo_step324_predictions.jsonl").read_bytes()).hexdigest(),
            "annotations_sha256": hashlib.sha256(Path("data/processed/annotations.jsonl").read_bytes()).hexdigest(),
            "valid_split_sha256": hashlib.sha256(Path("data/splits/chronological/valid.json").read_bytes()).hexdigest(),
            "paired_metrics_sha256": hashlib.sha256((RUN_DIR / "paired_metrics.json").read_bytes()).hexdigest(),
            "test_read": False},
        "limitations": ["Statement localization uses the frozen L2 and L1∪L2 line sets; no annotations were changed.",
                        "Trace step counts are descriptive only; DFG validity cannot be scored on VALID because no cached VALID DFGs exist.",
                        "Predictions are from one greedy pass per model and one frozen checkpoint pair."]}
    result["models"]["sft_v3_step500"]["per_vulnerable_row"] = sft_result["per_vulnerable_row"]
    result["models"]["grpo_step324"]["per_vulnerable_row"] = grpo_result["per_vulnerable_row"]
    OUTPUT.parent.mkdir(parents=True, exist_ok=True)
    OUTPUT.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
    corrected = {**corrected_paired_metrics,
                 "created_by": "scripts/analyze_step324_localization.py",
                 "gold_lines_source": "data/processed/annotations.jsonl, L2 for vulnerable rows and empty for SAFE rows",
                 "original_paired_metrics_sha256": hashlib.sha256((RUN_DIR / "paired_metrics.json").read_bytes()).hexdigest(),
                 "test_touched": False}
    CORRECTED_METRICS.write_text(json.dumps(corrected, indent=2, sort_keys=True) + "\n")
    return result


if __name__ == "__main__":
    print(json.dumps(run(), indent=2))