| """Fixed equal-weight ensemble/error-overlap analysis for two pairwise models.""" |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| from pathlib import Path |
|
|
| import numpy as np |
|
|
|
|
| def wilson(k: int, n: int) -> list[float]: |
| z=1.96; p=k/n; den=1+z*z/n; mid=p+z*z/(2*n) |
| h=z*np.sqrt(p*(1-p)/n+z*z/(4*n*n)) |
| return [float((mid-h)/den),float((mid+h)/den)] |
|
|
|
|
| def run(char_path: Path, edit_path: Path, seed: int=42, bootstrap: int=10_000) -> dict: |
| c=json.loads(char_path.read_text()); e=json.loads(edit_path.read_text()) |
| cd={x["pair_id"]:x for x in c["valid"]["pair_diagnostics"]} |
| ed={x["pair_id"]:x for x in e["valid"]["pair_diagnostics"]} |
| if set(cd)!=set(ed): raise ValueError("model VALID pair IDs differ") |
| ids=sorted(cd) |
| y=np.asarray([cd[i]["target"] for i in ids],dtype=np.int8) |
| if any(cd[i]["target"]!=ed[i]["target"] for i in ids): raise ValueError("target disagreement") |
| cs=np.asarray([cd[i]["margin"] for i in ids],dtype=np.float64)/float(c["train"]["margin_std_for_train_scale"]) |
| es=np.asarray([ed[i]["margin"] for i in ids],dtype=np.float64)/float(e["train"]["margin_std_for_train_scale"]) |
| cp=cs>0; ep=es>0; ensemble=(cs+es)/2>0 |
| cc=cp==y.astype(bool); ec=ep==y.astype(bool); jc=ensemble==y.astype(bool) |
| both=cc&ec; char_only=cc&~ec; edit_only=~cc&ec; neither=~cc&~ec |
| rng=np.random.default_rng(seed); picks=rng.integers(0,len(ids),size=(bootstrap,len(ids))) |
| draws=jc[picks].mean(axis=1) |
| hits=int(jc.sum()) |
| return { |
| "method":"equal-weight mean of pairwise decision margins, each divided by its TRAIN margin standard deviation", |
| "weight_selection":"fixed equal weights; no VALID fit or calibration", |
| "train_scale_std":{"char_difference":float(c["train"]["margin_std_for_train_scale"]), |
| "edit_feature":float(e["train"]["margin_std_for_train_scale"])}, |
| "valid":{"pairs":len(ids),"ensemble_correct":hits,"accuracy":hits/len(ids),"wilson_95":wilson(hits,len(ids)), |
| "pair_bootstrap_95_ci":[float(v) for v in np.quantile(draws,[.025,.975])], |
| "char_correct":int(cc.sum()),"edit_correct":int(ec.sum()), |
| "error_overlap":{"both_correct":int(both.sum()),"char_only_correct":int(char_only.sum()), |
| "edit_only_correct":int(edit_only.sum()),"both_wrong":int(neither.sum()), |
| "either_model_correct":int((cc|ec).sum())}, |
| "labels_per_pair":"one vulnerable-side target; pair accuracy equals P-Pass@1"}, |
| "snapshot":{"char_results":str(char_path),"edit_results":str(edit_path), |
| "char_valid_pair_ids_match":True,"test_read":False}, |
| "limitations":["Paired-input diagnostic; does not provide single-function detection.", |
| "Only one fixed equal-weight ensemble; no VALID-selected alternatives."], |
| } |
|
|
|
|
| def main(): |
| p=argparse.ArgumentParser(description=__doc__) |
| p.add_argument("--char",type=Path,default=Path("results/phase7/pairwise_diff_chronological.json")) |
| p.add_argument("--edit",type=Path,default=Path("results/phase7/pairwise_edit_feature_chronological.json")) |
| p.add_argument("--output",type=Path,default=Path("results/phase7/pairwise_ensemble_chronological.json")) |
| p.add_argument("--bootstrap",type=int,default=10000) |
| a=p.parse_args(); x=run(a.char,a.edit,bootstrap=a.bootstrap) |
| a.output.parent.mkdir(parents=True,exist_ok=True);a.output.write_text(json.dumps(x,indent=2,sort_keys=True)+"\n") |
| print(json.dumps(x,indent=2,sort_keys=True)) |
|
|
| if __name__=="__main__":main() |
|
|
