Buckets:
| #!/usr/bin/env python3 | |
| """cwm3/baselines_dms/report_dms_ours.py — render OUR read-outs' layer-3 operating points (1 / 3 / 5 / 10 alerts per h, DMS-matched, same_recall) and the PAIRED | |
| ours − public-read-out / ours − surrogate cells from the UNCHANGED evaluator's CSVs (eval/operating_points.py with the 2026-09-17 additive `--pair-runs` / | |
| `--pair-surrogate` option). Nothing is recomputed here; every printed number is a CSV cell (per seed — no seed means are printed, so --verify can check every | |
| 3-dp token against the CSVs exactly as report_dms.verify does). Multi-seed verdicts use eval/operating_points.multi_verdict (>= ceil(2n/3) seed CIs exclude 0, | |
| same sign, |mean Δ| > sd over seeds — the report_dms.verdict_multi rule). | |
| Inputs : results/operating_points/<item>_<stamp>/{operating_points,paired_lead,paired_public,paired_public_skipped,dms_reference}.csv (newest dir per item) | |
| Output : one dated section appended to results/tables/dms_operating_points_ours.md (append-only; the frozen dms_operating_points.md is never touched) | |
| python baselines_dms/report_dms_ours.py --items ours_l3_dwfc ours_l3_dwfcrmv ours_l3_cur --primary ours_l3_dwfc --verify | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import importlib.util | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| OP_ROOT = CWM3 / "results" / "operating_points" | |
| TABLE = CWM3 / "results" / "tables" / "dms_operating_points_ours.md" | |
| sys.path.insert(0, str(CWM3 / "baselines_dms")) | |
| from report_dms import f3, verify # noqa: E402 (unchanged helpers) | |
| _spec = importlib.util.spec_from_file_location("operating_points", CWM3 / "eval" / "operating_points.py") | |
| _OP = importlib.util.module_from_spec(_spec) | |
| _spec.loader.exec_module(_OP) | |
| multi_verdict = _OP.multi_verdict | |
| LOAD_ORDER = ["dms_matched", "low10", "low5", "low3", "low1", "same_recall"] | |
| REGIMES = ["pooled", "manual", "engaged"] | |
| def newest(item: str) -> Path: | |
| ds = sorted(d for d in OP_ROOT.glob(f"{item}_*") if d.is_dir() and (d / "operating_points.csv").exists()) | |
| assert ds, f"no operating-point run for item {item}" | |
| return ds[-1] | |
| def rel(p: Path) -> str: | |
| return str(p.relative_to(CWM3)) | |
| def f1(x) -> str: | |
| return "--" if pd.isna(x) else f"{float(x):.1f}" | |
| def f2s(x) -> str: | |
| return "--" if pd.isna(x) else f"{float(x):+.2f}" | |
| def s3(x) -> str: | |
| return "--" if pd.isna(x) else f"{float(x):+.3f}" | |
| def load_item(item: str): | |
| d = newest(item) | |
| R = {"dir": d, "op": pd.read_csv(d / "operating_points.csv"), "pl": pd.read_csv(d / "paired_lead.csv"), "ref": pd.read_csv(d / "dms_reference.csv")} | |
| R["pub"] = pd.read_csv(d / "paired_public.csv") if (d / "paired_public.csv").exists() else pd.DataFrame() | |
| R["skip"] = pd.read_csv(d / "paired_public_skipped.csv") if (d / "paired_public_skipped.csv").exists() else pd.DataFrame() | |
| js = sorted(d.glob("run_operating_points_*Z.json")) | |
| R["run"] = json.load(open(js[-1])) if js else {} | |
| return R | |
| def loads_present(df: pd.DataFrame): | |
| return [l for l in LOAD_ORDER if l in set(df.load)] + sorted(set(df.load) - set(LOAD_ORDER)) | |
| def section_ours_vs_surrogate(item: str, label: str, R) -> list[str]: | |
| op, pl, ref = R["op"], R["pl"], R["ref"] | |
| L = [f"### {label} — item `{item}` (`{rel(R['dir'])}/`) vs the surrogate at equal alert load (per seed; all rows are CSV cells)", ""] | |
| L += ["Surrogate reference (own clean-val-recall-0.5 threshold, defines `dms_matched`): " + "; ".join( | |
| f"seed {r.seed}: thr {r.thr_dms:.4f} -> {r.alerts_per_hour_manual:.1f} / {r.alerts_per_hour_engaged:.1f} alerts per h manual / engaged" for r in ref.itertuples()) + ".", ""] | |
| L += ["| load | regime | score | seed | thr m / e | alerts/h m / e (achieved) | n gold / pos | alerts on gold | recall [CI] | precision [CI] | lead median s [IQR] | detected |", "|---|---|---|---|---|---|---|---|---|---|---|---|"] | |
| for lname in loads_present(op): | |
| for reg in REGIMES: | |
| for sname in ("head", "dms"): | |
| g = op[(op.load == lname) & (op.regime == reg) & (op.score == sname)].sort_values("seed") | |
| for r in g.itertuples(): | |
| L.append(f"| {lname} | {reg} | {'**ours** ' + label if sname == 'head' else 'att@3s'} | {r.seed} | {r.thr_manual:.4f} / {r.thr_engaged:.4f} | {r.alerts_per_hour_manual:.3f} / {r.alerts_per_hour_engaged:.3f} | " | |
| f"{int(r.n_gold)} / {int(r.n_pos)} | {int(r.n_alerts_gold)} | {f3(r.recall)} [{f3(r.recall_ci_lo)}, {f3(r.recall_ci_hi)}] | {f3(r.precision)} [{f3(r.precision_ci_lo)}, {f3(r.precision_ci_hi)}] | " | |
| f"{f3(r.lead_median)} [{f3(r.lead_q25)}, {f3(r.lead_q75)}] | {f3(r.lead_detected_frac)} of {int(r.n_pos_lead)} |") | |
| L += ["", f"Paired lead-time Δ (ours − surrogate, undetected -> 0; pre-stated rule CI ⊃ 0 -> EQUAL) and the multi-seed verdict:", "", | |
| "| load | regime | seed | n pos | both detected | Δ lead mean s [CI] | per-seed verdict | multi-seed (>= 2/3 rule) |", "|---|---|---|---|---|---|---|---|"] | |
| for lname in loads_present(pl): | |
| for reg in REGIMES: | |
| g = pl[(pl.load == lname) & (pl.regime == reg)].sort_values("seed") | |
| if not len(g): | |
| continue | |
| mv = multi_verdict(g.delta_lead0_mean, g.delta_lead0_ci_lo, g.delta_lead0_ci_hi, "OURS EARLIER", "SURROGATE EARLIER") | |
| for i, r in enumerate(g.itertuples()): | |
| L.append(f"| {lname} | {reg} | {r.seed} | {int(r.n_pos)} | {int(r.n_both_detected)} | {s3(r.delta_lead0_mean)} [{s3(r.delta_lead0_ci_lo)}, {s3(r.delta_lead0_ci_hi)}] | {str(r.verdict).strip()} | {mv if i == 0 else ''} |") | |
| L.append("") | |
| return L | |
| def section_paired(item: str, label: str, R, regimes, title) -> list[str]: | |
| pub = R["pub"] | |
| if not len(pub): | |
| return [f"### {title}", "", "(no paired_public.csv — the run was made without `--pair-runs`)", ""] | |
| L = [f"### {title}", ""] | |
| if len(R["skip"]): | |
| L += ["Skipped comparator seeds (layouts incomplete): " + "; ".join(f"{r.comparator} seed {r.seed} ({r.reason})" for r in R["skip"].itertuples()) + ".", ""] | |
| comps = list(dict.fromkeys(pub.comparator)) | |
| L += ["| load | regime | comparator | seed | thr source | alerts/h comp m / e | n common / pos | recall ours / comp | Δ recall [CI] | precision ours / comp | Δ precision [CI] | FAR ours / comp | Δ FAR [CI] | lead med ours / comp | both det. | Δ lead s [CI] | per-seed verdict (recall; lead) |", | |
| "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"] | |
| for lname in loads_present(pub): | |
| for reg in regimes: | |
| for c in comps: | |
| g = pub[(pub.load == lname) & (pub.regime == reg) & (pub.comparator == c)].sort_values("seed") | |
| for r in g.itertuples(): | |
| L.append(f"| {lname} | {reg} | {c} | {r.seed} | {r.thr_source} | {r.aph_comp_manual:.3f} / {r.aph_comp_engaged:.3f} | {int(r.n_common)} / {int(r.n_pos)} | {f3(r.recall_head)} / {f3(r.recall_comp)} | " | |
| f"{s3(r.d_recall)} [{s3(r.d_recall_ci_lo)}, {s3(r.d_recall_ci_hi)}] | {f3(r.precision_head)} / {f3(r.precision_comp)} | {s3(r.d_precision)} [{s3(r.d_precision_ci_lo)}, {s3(r.d_precision_ci_hi)}] | " | |
| f"{f3(r.far_head)} / {f3(r.far_comp)} | {s3(r.d_far)} [{s3(r.d_far_ci_lo)}, {s3(r.d_far_ci_hi)}] | {f3(r.lead_median_head)} / {f3(r.lead_median_comp)} | {int(r.n_both_detected)} of {int(r.n_pos_lead)} | " | |
| f"{s3(r.d_lead0_mean)} [{s3(r.d_lead0_ci_lo)}, {s3(r.d_lead0_ci_hi)}] | {r.verdict_recall}; {r.verdict_lead} |") | |
| L.append("") | |
| return L | |
| def section_verdict_matrix(item: str, label: str, R, regimes) -> list[str]: | |
| pub = R["pub"] | |
| if not len(pub): | |
| return [] | |
| comps = list(dict.fromkeys(pub.comparator)) | |
| L = [f"### Verdict matrix — {label} (item `{item}`): multi-seed rule >= ceil(2n/3) seed CIs exclude 0, same sign, |mean Δ| > sd; k/n = seed CIs excluding 0; per-seed values listed s0 / s1 / s2 (CSV cells)", "", | |
| "| load | regime | comparator | n seeds | recall ours (per seed) | recall comp (per seed) | Δ recall per seed | recall verdict | FAR ours / comp (per seed) | FAR verdict | Δ lead per seed | lead verdict |", "|---|---|---|---|---|---|---|---|---|---|---|---|"] | |
| for lname in loads_present(pub): | |
| for reg in regimes: | |
| for c in comps: | |
| g = pub[(pub.load == lname) & (pub.regime == reg) & (pub.comparator == c)].sort_values("seed") | |
| if not len(g): | |
| continue | |
| L.append(f"| {lname} | {reg} | {c} | {len(g)} | {' / '.join(f3(v) for v in g.recall_head)} | {' / '.join(f3(v) for v in g.recall_comp)} | {' / '.join(s3(v) for v in g.d_recall)} | " | |
| f"{multi_verdict(g.d_recall, g.d_recall_ci_lo, g.d_recall_ci_hi, 'OURS HIGHER', 'COMP HIGHER')} | " | |
| f"{' / '.join(f3(v) for v in g.far_head)} vs {' / '.join(f3(v) for v in g.far_comp)} | {multi_verdict(g.d_far, g.d_far_ci_lo, g.d_far_ci_hi, 'OURS MORE FA', 'OURS FEWER FA')} | " | |
| f"{' / '.join(s3(v) for v in g.d_lead0_mean)} | {multi_verdict(g.d_lead0_mean, g.d_lead0_ci_lo, g.d_lead0_ci_hi, 'OURS EARLIER', 'COMP EARLIER')} |") | |
| L.append("") | |
| return L | |
| def main(argv=None) -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--items", nargs="+", required=True) | |
| ap.add_argument("--labels", nargs="+", default=None, help="one label per item (default: the run json's --label)") | |
| ap.add_argument("--primary", default=None, help="item rendered with all regimes in the paired tables (others: pooled only)") | |
| ap.add_argument("--table", default=str(TABLE)) | |
| ap.add_argument("--verify", action="store_true") | |
| a = ap.parse_args(argv) | |
| stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| items = {it: load_item(it) for it in a.items} | |
| labels = {} | |
| for i, it in enumerate(a.items): | |
| cmd = items[it]["run"].get("command", "") | |
| lab = a.labels[i] if a.labels else (cmd.split("--label ")[1].split(" --")[0].strip() if "--label " in cmd else it) | |
| labels[it] = lab | |
| primary = a.primary or a.items[0] | |
| L = [f"## Our read-outs at 1 / 3 / 5 / 10 alerts per h and the DMS-matched load, PAIRED vs the public read-outs and the surrogate — {stamp}", "", | |
| "Evaluator `eval/operating_points.py` — the same evaluator, same gold (`warning_gold.build_gold`, legacy_first anchors), same 60 s per-route cooldown, same lead window [t0 − 10 s, t_anchor], " | |
| "same device-cluster bootstrap (1,000 resamples) and same seeds (0 / 1 / 2) as the 12 public sections of `results/tables/dms_operating_points.md`; run here with the 2026-09-17 additive " | |
| "`--pair-runs` / `--pair-surrogate` option (paired ours − comparator cells; the historical cells and CSVs are unchanged). Thresholds: for EVERY arm (ours and each public read-out alike) the " | |
| "threshold is bisected per regime on the FULL TEST split to the target alert load (1 / 3 / 5 / 10 per h, or the surrogate's load); no threshold is set on validation — the only validation-set " | |
| "quantity is the surrogate's own clean-val-recall-0.5 threshold, which defines the `dms_matched` load per seed. The public comparators re-use their published per-seed / per-load thresholds " | |
| "from the frozen runs (`thr source = published (<dir>)`; load targets and recomputed alerts/h asserted equal), so they sit exactly on their frozen-table rows; comparators without a frozen " | |
| "row are bisected by the same rule. Cells: common finite gold clean-test rows; FAR = share of gold NEGATIVE rows alerted at the fixed threshold (an enriched-gold share, not a natural rate — " | |
| "the per-hour alert load is matched by construction); every Δ = ours − comparator on identical resamples (bootstrap seed 0 for recall / precision / FAR, seed 2 for lead, as the existing " | |
| "cells). Recall / precision are shares of the enriched gold set, not natural rates. Runs:", ""] | |
| for it, R in items.items(): | |
| L.append(f"* item `{it}` — {labels[it]}: `{rel(R['dir'])}/` — `{R['run'].get('command', '?')}` (wall {R['run'].get('wall_s', float('nan')):.0f} s)") | |
| L.append("") | |
| for it, R in items.items(): | |
| L += section_ours_vs_surrogate(it, labels[it], R) | |
| for it, R in items.items(): | |
| regs = REGIMES if it == primary else ["pooled"] | |
| L += section_verdict_matrix(it, labels[it], R, regs) | |
| for it, R in items.items(): | |
| regs = REGIMES if it == primary else ["pooled"] | |
| L += section_paired(it, labels[it], R, regs, f"Paired cells per seed — {labels[it]} (item `{it}`) − comparator" + ("" if it == primary else " (pooled only; per-regime rows in the CSV)")) | |
| md = "\n".join(L) + "\n" | |
| table = Path(a.table) | |
| if not table.exists(): | |
| table.write_text("# Our read-outs at layer 3 (1 / 3 / 5 / 10 alerts per h, DMS-matched) and the paired ours-vs-public comparison (baselines_dms/report_dms_ours.py; sections appended per run; the frozen dms_operating_points.md is never modified)\n\n") | |
| with open(table, "a", encoding="utf-8") as fh: | |
| fh.write("\n" + md) | |
| print(f"appended {len(L)} lines to {table}") | |
| if a.verify: | |
| csvs = [p for R in items.values() for p in R["dir"].glob("*.csv")] | |
| return int(verify(table, csvs) > 0) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 13.6 kB
- Xet hash:
- da40feb2a6ae84702fee731d6602543de093052917458ebd8f56a5b434957435
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.