File size: 3,627 Bytes
646ccfc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
"""Fixed equal-weight ensemble/error-overlap analysis for two pairwise models."""
from __future__ import annotations

import argparse
import json
from pathlib import Path

import numpy as np


def wilson(k: int, n: int) -> list[float]:
    z=1.96; p=k/n; den=1+z*z/n; mid=p+z*z/(2*n)
    h=z*np.sqrt(p*(1-p)/n+z*z/(4*n*n))
    return [float((mid-h)/den),float((mid+h)/den)]


def run(char_path: Path, edit_path: Path, seed: int=42, bootstrap: int=10_000) -> dict:
    c=json.loads(char_path.read_text()); e=json.loads(edit_path.read_text())
    cd={x["pair_id"]:x for x in c["valid"]["pair_diagnostics"]}
    ed={x["pair_id"]:x for x in e["valid"]["pair_diagnostics"]}
    if set(cd)!=set(ed): raise ValueError("model VALID pair IDs differ")
    ids=sorted(cd)
    y=np.asarray([cd[i]["target"] for i in ids],dtype=np.int8)
    if any(cd[i]["target"]!=ed[i]["target"] for i in ids): raise ValueError("target disagreement")
    cs=np.asarray([cd[i]["margin"] for i in ids],dtype=np.float64)/float(c["train"]["margin_std_for_train_scale"])
    es=np.asarray([ed[i]["margin"] for i in ids],dtype=np.float64)/float(e["train"]["margin_std_for_train_scale"])
    cp=cs>0; ep=es>0; ensemble=(cs+es)/2>0
    cc=cp==y.astype(bool); ec=ep==y.astype(bool); jc=ensemble==y.astype(bool)
    both=cc&ec; char_only=cc&~ec; edit_only=~cc&ec; neither=~cc&~ec
    rng=np.random.default_rng(seed); picks=rng.integers(0,len(ids),size=(bootstrap,len(ids)))
    draws=jc[picks].mean(axis=1)
    hits=int(jc.sum())
    return {
        "method":"equal-weight mean of pairwise decision margins, each divided by its TRAIN margin standard deviation",
        "weight_selection":"fixed equal weights; no VALID fit or calibration",
        "train_scale_std":{"char_difference":float(c["train"]["margin_std_for_train_scale"]),
                           "edit_feature":float(e["train"]["margin_std_for_train_scale"])},
        "valid":{"pairs":len(ids),"ensemble_correct":hits,"accuracy":hits/len(ids),"wilson_95":wilson(hits,len(ids)),
                 "pair_bootstrap_95_ci":[float(v) for v in np.quantile(draws,[.025,.975])],
                 "char_correct":int(cc.sum()),"edit_correct":int(ec.sum()),
                 "error_overlap":{"both_correct":int(both.sum()),"char_only_correct":int(char_only.sum()),
                                  "edit_only_correct":int(edit_only.sum()),"both_wrong":int(neither.sum()),
                                  "either_model_correct":int((cc|ec).sum())},
                 "labels_per_pair":"one vulnerable-side target; pair accuracy equals P-Pass@1"},
        "snapshot":{"char_results":str(char_path),"edit_results":str(edit_path),
                    "char_valid_pair_ids_match":True,"test_read":False},
        "limitations":["Paired-input diagnostic; does not provide single-function detection.",
                       "Only one fixed equal-weight ensemble; no VALID-selected alternatives."],
    }


def main():
    p=argparse.ArgumentParser(description=__doc__)
    p.add_argument("--char",type=Path,default=Path("results/phase7/pairwise_diff_chronological.json"))
    p.add_argument("--edit",type=Path,default=Path("results/phase7/pairwise_edit_feature_chronological.json"))
    p.add_argument("--output",type=Path,default=Path("results/phase7/pairwise_ensemble_chronological.json"))
    p.add_argument("--bootstrap",type=int,default=10000)
    a=p.parse_args(); x=run(a.char,a.edit,bootstrap=a.bootstrap)
    a.output.parent.mkdir(parents=True,exist_ok=True);a.output.write_text(json.dumps(x,indent=2,sort_keys=True)+"\n")
    print(json.dumps(x,indent=2,sort_keys=True))

if __name__=="__main__":main()