Buckets:
| """cwm3/eval/newtest_analysis.py — pre-registered analysis of the human-evaluation test set (docs/newtest_prereg.md §3; 2026-09-12). | |
| Inputs (all script outputs): results/newtest/manifest_<v>.csv (design: stratum, cell, regime, pi, N_cell, N_covered_cell, anchors, dongle, score_*_anchor, | |
| t_alert_*), thresholds_frozen_<v>.json (thresholds, exposure hours), the Task-1 consensus (cwm2/results/W1d/h5_consensus_latest.csv from | |
| h5_to_consensus.py on labels_newtest.jsonl), the Task-2 consensus (event_consensus_latest.csv from h5_to_consensus.py --task event), the raw store for the | |
| prefix onset intervals, and the frozen layouts (alert times at every threshold are recomputed with newtest_sample.alert_rows; --from-manifest uses the | |
| stored t_alert_* within [t-10, t+10] instead). | |
| Labels per clip i: W_i = Task-1 consensus warranted (unsure excluded); E_i = Task-2 event != none; onset o_i (route time) = the Task-1 prefix onset start | |
| (mean over raters) when compatibility in {briefly, sustained} and concern ongoing, else the Task-2 event onset; ongoing_at_anchor without a prefix onset -> | |
| o_i = t (flagged censored). Weights: HT w_i = (N_cell / N_covered_cell) / pi_i on S1 u S2 (a partition of the TEST exposure; the post-stratification factor | |
| covers the device-cap-uncovered strata, coverage reported); S3 unweighted, never in a natural rate. | |
| Estimators at each (system X, regime, operating point L in {val10, val20 primary; test10, test20 secondary}): | |
| event recall R = sum w E 1[X alert at thr_X(L) in [o-10, o+3]] / sum w E (HT on S1 u S2; unweighted on S1 u S2 u S3 as secondary) | |
| lead time lead = o - t_first_alert in [o-10, o+3]; undetected -> 0 (pre-stated); HT mean; both-detected median secondary; paired deltas A-B, A-C, B-C on | |
| identical events with the SAME device resamples; CI containing 0 -> "matches" | |
| false alarms FA/h = sum_{S1} w 1[X alert in [t-10, t+10]] (1 - W) / H_regime; precision = HT ratio; silent-time warranted / h = sum_{S2} w W / H | |
| pooled manual and engaged always separate; pooled = HT event-count-weighted mixture (natural exposure mix); equal-weight pooling as sensitivity | |
| AUROC score at the anchor vs W (unweighted; HT-weighted Mann-Whitney) — secondary | |
| C at 10 / h the DMS chain cannot reach 10 / h: reported at its floor threshold and with a seeded random thinning of its alerts to 10 / h (200 replicates) | |
| uncertainty device-cluster bootstrap (dongles with replacement, 2,000 resamples, shared across systems for every paired delta) | |
| wording rule CI contains 0 -> "matches (Δ, CI)"; CI > 0 -> "above"; CI < 0 -> "below"; never "significantly better", never "equivalent"; per-regime comparisons | |
| are descriptive unless >= 60 labelled events in the regime; MDE from the pre-registered formula is printed with the achieved n | |
| Cell-level warranted shares are printed only under the header "enriched — not a natural rate". Output: a section appended to results/tables/newtest_fixed_load.md | |
| (every stratum, both regimes, HT natural rates, coverage, the pre-registration sha256 of docs/newtest_prereg.md vs the registered one) + CSVs in | |
| results/newtest/analysis_<stamp>/ + run json. --self-test: synthetic frame + synthetic labels through the sampler's own stratified draw -> HT recall / FA | |
| totals unbiased within Monte-Carlo tolerance; estimator identities; wording rule; writes nothing. | |
| python eval/newtest_analysis.py [--version v1] [--n-boot 2000] [--from-manifest] [--self-test] (labels from h5_to_consensus.py --panel G1 [--task event]) | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import sys | |
| import time | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| os.environ.setdefault("OMP_NUM_THREADS", "4") | |
| import numpy as np # noqa: E402 | |
| import pandas as pd # noqa: E402 | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| EXPLORE = CWM3.parent | |
| CWM2 = EXPLORE / "cwm2" | |
| for p in (CWM3 / "eval", CWM3 / "common", CWM2 / "annotation_human"): | |
| sys.path.insert(0, str(p)) | |
| import newtest_sample as NS # noqa: E402 | |
| from runmeta import write_run_json # noqa: E402 | |
| REGIMES = ("manual", "engaged") | |
| SYSTEMS = ("A", "B", "C") | |
| POINTS_PRIMARY = ("val10", "val20") | |
| POINTS_SECONDARY = ("test10", "test20") | |
| POINTS = POINTS_PRIMARY + POINTS_SECONDARY | |
| PAIRS = (("A", "B"), ("A", "C"), ("B", "C")) | |
| REC_LO, REC_HI = 10.0, 3.0 # detection window [o-10, o+3] | |
| FA_WIN = 10.0 # alert attributed to an S1 event when within [t-10, t+10] | |
| MIN_EVENTS_REGIME = 60 | |
| NEWTEST = CWM3 / "results" / "newtest" | |
| TABLE = CWM3 / "results" / "tables" / "newtest_fixed_load.md" | |
| CONSENSUS = CWM2 / "results" / "W1d" / "h5_consensus_latest.csv" | |
| EVENT_CONS = CWM2 / "results" / "W1d" / "event_consensus_latest.csv" | |
| STORE = CWM2 / "annotation_human" / "labels_newtest.jsonl" | |
| PREREG = CWM3 / "docs" / "newtest_prereg.md" | |
| PREREG_HASH = NEWTEST / "prereg_hash.json" | |
| DEVIATIONS = NEWTEST / "deviations_v1.json" | |
| def apply_deviations(M, dev_path, log, key_cols=("stratum", "cell", "dev_group")): | |
| """Pre-analysis exclusions (eval/newtest_render.py deviations file: units that cannot be rendered / rated, reason per entry). The unit is dropped and | |
| its stratum's remaining draws are re-weighted: pi' = pi * n_remaining / n_stratum (unit non-response treated as ignorable within the stratum — the | |
| stratum is (cell, device-group), so the excluded unit's device / cell are still represented by its stratum mates when any remain; a stratum losing | |
| all its draws becomes uncovered and is added to the coverage note). Returns (M', table of exclusions applied).""" | |
| if not Path(dev_path).exists(): | |
| return M, [] | |
| dev = json.load(open(dev_path))["exclusions"] | |
| ex = {int(e["idx"]): e for e in dev if int(e["idx"]) in set(M.idx)} | |
| if not ex: | |
| return M, [] | |
| M = M.copy() | |
| rows = [] | |
| for idx, e in ex.items(): | |
| r = M[M.idx == idx].iloc[0] | |
| key = tuple(r[k] for k in key_cols) | |
| mates = M[(M[list(key_cols)] == pd.Series(key, index=list(key_cols))).all(axis=1) & (M.idx != idx)] | |
| n0 = int(r.n_stratum) | |
| if len(mates): | |
| M.loc[mates.index, "pi"] = M.loc[mates.index, "pi"] * (n0 - 1) / n0 if n0 > 1 else M.loc[mates.index, "pi"] | |
| M.loc[mates.index, "n_stratum"] = n0 - 1 | |
| else: # the stratum is now uncovered: shrink N_covered_cell for the whole cell | |
| same_cell = M[(M.stratum == r.stratum) & (M.cell == r.cell)] | |
| M.loc[same_cell.index, "N_covered_cell"] = M.loc[same_cell.index, "N_covered_cell"] - int(r.N_stratum) | |
| rows.append(dict(idx=idx, stratum=r.stratum, cell=r.cell, reason=e.get("reason"), n_stratum_before=n0, n_mates=int(len(mates)), N_stratum=int(r.N_stratum), details=e.get("details"))) | |
| M = M[M.idx != idx] | |
| log(f"deviations applied: {len(rows)} units excluded ({dict(Counter(x['reason'] for x in rows))}); strata re-weighted; " + "; ".join(f"{x['idx']} {x['cell']} {x['reason']}" for x in rows)) | |
| return M.reset_index(drop=True), rows | |
| # ----------------------------------------------------------------------------------------------------------------------- helpers | |
| def sha256_file(p): | |
| h = hashlib.sha256() | |
| with open(p, "rb") as f: | |
| for b in iter(lambda: f.read(1 << 20), b""): | |
| h.update(b) | |
| return h.hexdigest() | |
| def f3(x, nd=3): | |
| try: | |
| x = float(x) | |
| except (TypeError, ValueError): | |
| return "—" | |
| return "—" if not np.isfinite(x) else f"{x:.{nd}f}" | |
| def pct(v): | |
| v = np.asarray(v, float) | |
| v = v[np.isfinite(v)] | |
| return (float(np.percentile(v, 2.5)), float(np.percentile(v, 97.5))) if len(v) else (np.nan, np.nan) | |
| def wording(delta, lo, hi, what="", unit=""): | |
| """Pre-registered wording rule. CI containing 0 -> matches; else above / below. Never 'significantly', never 'equivalent'.""" | |
| if not (np.isfinite(lo) and np.isfinite(hi)): | |
| return f"not estimable{(' (' + what + ')') if what else ''}" | |
| verb = "matches" if lo <= 0 <= hi else ("above" if lo > 0 else "below") | |
| return f"{verb} ({'Δ ' if what else ''}{delta:+.3f}{unit}, CI [{lo:+.3f}, {hi:+.3f}]{(' ' + what) if what else ''})" | |
| def mde_paired(n, disc=0.4, alpha_z=1.96, power_z=0.84): | |
| """MDE of a paired recall difference with discordance `disc`: SE = sqrt(disc / n).""" | |
| return (alpha_z + power_z) * np.sqrt(disc / max(n, 1)) | |
| def mde_lead(n, sd_diff=3.0, alpha_z=1.96, power_z=0.84): | |
| return (alpha_z + power_z) * sd_diff / np.sqrt(max(n, 1)) | |
| class DeviceBoot: | |
| """Shared device-cluster resamples: the SAME device draws are used for every statistic (paired deltas are exact).""" | |
| def __init__(self, groups, n_boot, seed=0): | |
| self.groups = np.asarray(groups) | |
| self.ug = np.unique(self.groups) | |
| self.gidx = {g: np.flatnonzero(self.groups == g) for g in self.ug} | |
| rng = np.random.default_rng(seed) | |
| self.draws = [rng.choice(self.ug, len(self.ug), replace=True) for _ in range(n_boot)] | |
| def picks(self): | |
| for d in self.draws: | |
| yield np.concatenate([self.gidx[g] for g in d]) | |
| def run(self, fn): | |
| out = np.full(len(self.draws), np.nan) | |
| for b, pk in enumerate(self.picks()): | |
| try: | |
| out[b] = fn(pk) | |
| except (ZeroDivisionError, ValueError): | |
| out[b] = np.nan | |
| return out | |
| def ht_ratio(w, num, den): | |
| d = float(np.sum(w * den)) | |
| return float(np.sum(w * num)) / d if d > 0 else np.nan | |
| def weighted_auroc(y, s, w): | |
| """HT-weighted Mann-Whitney AUROC (ties count 1/2).""" | |
| y, s, w = np.asarray(y, int), np.asarray(s, float), np.asarray(w, float) | |
| fin = np.isfinite(s) | |
| y, s, w = y[fin], s[fin], w[fin] | |
| pos, neg = y == 1, y == 0 | |
| if not pos.any() or not neg.any(): | |
| return np.nan | |
| sp, wp, sn, wn = s[pos], w[pos], s[neg], w[neg] | |
| order = np.argsort(sn, kind="mergesort") | |
| sn, wn = sn[order], wn[order] | |
| cw = np.concatenate([[0.0], np.cumsum(wn)]) | |
| lo = np.searchsorted(sn, sp, "left") | |
| hi = np.searchsorted(sn, sp, "right") | |
| below = cw[lo] | |
| ties = cw[hi] - cw[lo] | |
| return float(np.sum(wp * (below + 0.5 * ties)) / (wp.sum() * wn.sum())) | |
| # ----------------------------------------------------------------------------------------------------------------------- alerts | |
| def alert_lists(thr_json, from_manifest, M, log): | |
| """{(X, regime, point): {uid: sorted alert times}} at the frozen thresholds — recomputed from the layouts (primary) or approximated from the manifest.""" | |
| out = {} | |
| if from_manifest: | |
| for X in SYSTEMS: | |
| for r in REGIMES: | |
| for p in POINTS: | |
| d = defaultdict(list) | |
| for row in M[M.layout_regime == r].itertuples(): | |
| v = getattr(row, f"t_alert_{X}_{p}") | |
| if np.isfinite(v): | |
| d[row.uid].append(float(v)) | |
| out[(X, r, p)] = {u: np.sort(np.asarray(v)) for u, v in d.items()} | |
| log("alerts from the manifest's t_alert_* (first alert within [t-10, t+10] only — approximation)") | |
| return out | |
| S, packs, hours, files = NS.load_scores(lambda m: None) | |
| for X in SYSTEMS: | |
| for r in REGIMES: | |
| d = S[(X, r, "test")] | |
| for p in POINTS: | |
| rows = NS.alert_rows(d["uid"], d["t"], d["s"], thr_json["thresholds"][X][r][p]["thr"]) | |
| out[(X, r, p)] = NS.by_route(d["uid"][rows], d["t"][rows]) | |
| log(f"alerts recomputed {X} {r}: " + ", ".join(f"{p} {sum(len(v) for v in out[(X, r, p)].values())}" for p in POINTS)) | |
| return out | |
| def first_alert(lists, uid, o): | |
| arr = lists.get(uid) | |
| if arr is None or not len(arr): | |
| return np.nan | |
| i = int(np.searchsorted(arr, o - REC_LO - 1e-9)) | |
| if i < len(arr) and arr[i] <= o + REC_HI + 1e-9: | |
| return float(arr[i]) | |
| return np.nan | |
| def thin_alerts(lists, keep_prob, seed): | |
| rng = np.random.default_rng(seed) | |
| return {u: v[rng.random(len(v)) < keep_prob] for u, v in lists.items()} | |
| # ----------------------------------------------------------------------------------------------------------------------- labels | |
| def load_labels(M, consensus_csv, event_csv, store_path, log): | |
| """-> DataFrame per clip idx with W (0/1/NaN), E (0/1/NaN), onset_route_s, onset_source, censored.""" | |
| L = M[["idx", "t_anchor", "anchor_s", "stratum"]].copy().set_index("idx") | |
| L["W"], L["E"], L["onset_route_s"], L["onset_source"], L["censored"] = np.nan, np.nan, np.nan, "", 0 | |
| L["W_posthoc"], L["posthoc_flag"] = np.nan, 0 | |
| L.attrs["panel"] = False | |
| if Path(consensus_csv).exists(): | |
| C = pd.read_csv(consensus_csv) | |
| C = C[C.status.isin(["consensus", "adjudicated", "panel_consensus"]) & C.y.notna()] | |
| L.attrs["panel"] = bool((C.status == "panel_consensus").any()) | |
| # post-hoc committee revisions (stage-2 note marker; SENSITIVITY ONLY): W_posthoc = A/S -> 1, D -> 0 where flagged, else the locked primary W | |
| if "posthoc_q4" in C: | |
| Cp = C.set_index("idx") | |
| common_p = L.index.intersection(Cp.index) | |
| q = Cp.posthoc_q4.reindex(common_p).fillna("").astype(str) | |
| L.loc[common_p, "posthoc_flag"] = q.isin(["A", "S", "D"]).astype(int).to_numpy() | |
| L.loc[common_p, "W_posthoc"] = np.where(q.isin(["A", "S"]), 1.0, np.where(q == "D", 0.0, Cp.y.reindex(common_p).to_numpy(float))) | |
| L.loc[L.index.intersection(C.idx), "W"] = C.set_index("idx").y.reindex(L.index.intersection(C.idx)).to_numpy() | |
| if Path(event_csv).exists(): | |
| Ec = pd.read_csv(event_csv) | |
| Ec = Ec[Ec.status.isin(["consensus", "adjudicated", "single_rated", "panel_consensus"]) & Ec.event_any.notna()].set_index("idx") | |
| L.attrs["panel"] = L.attrs.get("panel", False) or bool((Ec.status == "panel_consensus").any()) | |
| common = L.index.intersection(Ec.index) | |
| L.loc[common, "E"] = Ec.event_any.reindex(common).to_numpy() | |
| if "event_type_prefill_accepted" in Ec: # 2026-09-13 prefill deviation: provenance of the 'none' labels | |
| L["E_none_from_prefill"] = 0 | |
| L.loc[common, "E_none_from_prefill"] = ((Ec.event_any.reindex(common) == 0) & (Ec.event_type_prefill_accepted.reindex(common) == 1)).astype(int).to_numpy() | |
| L.attrs["n_none_prefill"] = int(L.E_none_from_prefill.sum()) | |
| L.attrs["n_none"] = int((L.E == 0).sum()) | |
| ev_on = Ec.event_onset_route_s.reindex(common) if "event_onset_route_s" in Ec else pd.Series(np.nan, index=common) | |
| ev_cens = Ec.onset_censored_at_anchor.reindex(common).fillna(0) if "onset_censored_at_anchor" in Ec else pd.Series(0, index=common) | |
| else: | |
| common, ev_on, ev_cens = L.index[:0], pd.Series(dtype=float), pd.Series(dtype=float) | |
| # prefix onsets from the raw store (compatibility in {briefly, sustained} and concern ongoing) | |
| pre_on = defaultdict(list) | |
| if Path(store_path).exists(): | |
| for line in open(store_path): | |
| try: | |
| r = json.loads(line) | |
| except json.JSONDecodeError: | |
| continue | |
| if r.get("stage") != "prefix" or r.get("kind") == "skip" or str(r.get("annotator", "")).upper().startswith("ADJ"): | |
| continue | |
| f = r.get("fields", {}) | |
| iv = f.get("onset_interval_s") | |
| if f.get("compatibility") in ("briefly_incompatible", "sustained_incompatible") and f.get("concern_persists") == "ongoing" and iv: | |
| pre_on[int(r["idx"])].append(float(iv[0]) - float(r.get("anchor_s", 5.0))) # seconds relative to the anchor (<= 0) | |
| for idx in L.index: | |
| if idx in pre_on: | |
| L.loc[idx, "onset_route_s"] = L.loc[idx, "t_anchor"] + float(np.mean(pre_on[idx])) | |
| L.loc[idx, "onset_source"] = "task1_prefix" | |
| elif idx in common and np.isfinite(ev_on.get(idx, np.nan)): | |
| L.loc[idx, "onset_route_s"] = float(ev_on[idx]) | |
| L.loc[idx, "onset_source"] = "task2_event" | |
| L.loc[idx, "censored"] = int(ev_cens.get(idx, 0)) | |
| elif idx in common and L.loc[idx, "E"] == 1: | |
| L.loc[idx, "onset_route_s"] = L.loc[idx, "t_anchor"] | |
| L.loc[idx, "onset_source"] = "anchor_censored" | |
| L.loc[idx, "censored"] = 1 | |
| log(f"labels: W available {int(L.W.notna().sum())} (warranted {int((L.W == 1).sum())}); E available {int(L.E.notna().sum())} (events {int((L.E == 1).sum())}); " | |
| f"onset sources {dict(Counter(L.onset_source[L.onset_source != '']))}; censored {int(L.censored.sum())}; label protocol {'committee consensus (panel)' if L.attrs.get('panel') else 'blind raters + adjudication'}; " | |
| f"E = none from an accepted stage-2 prefill: {L.attrs.get('n_none_prefill', 'n/a')} of {L.attrs.get('n_none', 'n/a')}") | |
| return L | |
| # ----------------------------------------------------------------------------------------------------------------------- core estimators | |
| def analyse(M, L, alerts, hours, thr_json, n_boot, log, seed=0): | |
| D = M.set_index("idx").join(L[["W", "E", "onset_route_s", "onset_source", "censored"]]) | |
| D["w"] = np.where(D.stratum.isin(["S1", "S2"]), (D.N_cell / D.N_covered_cell) / D.pi, np.nan) | |
| D["in_ht"] = D.stratum.isin(["S1", "S2"]) | |
| # detection and lead per system x point | |
| for X in SYSTEMS: | |
| for p in POINTS: | |
| ta = np.array([first_alert(alerts[(X, r, p)], u, o) if np.isfinite(o) else np.nan for r, u, o in zip(D.layout_regime, D.uid, D.onset_route_s)]) | |
| D[f"det_{X}_{p}"] = np.isfinite(ta) | |
| D[f"lead_{X}_{p}"] = np.where(np.isfinite(ta), D.onset_route_s.to_numpy() - ta, np.nan) | |
| D[f"lead0_{X}_{p}"] = np.where(np.isfinite(ta), D.onset_route_s.to_numpy() - ta, 0.0) | |
| # C thinning to 10/h (200 replicates): floor rate per regime from the thresholds json | |
| thin = {} | |
| for r in REGIMES: | |
| floor = thr_json["thresholds"]["C"][r]["val10"]["achieved_test"] | |
| keep = min(1.0, 10.0 / floor) if floor > 0 else 1.0 | |
| thin[r] = keep | |
| rows_main, rows_pair, rows_cells, rows_auroc = [], [], [], [] | |
| boots = {} | |
| for r in REGIMES + ("pooled",): | |
| m_r = np.ones(len(D), bool) if r == "pooled" else (D.regime == r).to_numpy() | |
| H = sum(hours.values()) if r == "pooled" else hours[r] | |
| sub = D[m_r] | |
| B = DeviceBoot(sub.dongle.to_numpy(), n_boot, seed) | |
| boots[r] = B | |
| ev = (sub.E == 1).to_numpy() & sub.in_ht.to_numpy() | |
| n_ev_ht = int(ev.sum()) | |
| n_ev_all = int((sub.E == 1).sum()) | |
| for p in POINTS: | |
| for X in SYSTEMS: | |
| w = sub.w.to_numpy() | |
| det = sub[f"det_{X}_{p}"].to_numpy() | |
| lead0 = sub[f"lead0_{X}_{p}"].to_numpy() | |
| lead = sub[f"lead_{X}_{p}"].to_numpy() | |
| fired = sub[f"fired_{X}_{p}"].to_numpy().astype(bool) | |
| W = sub.W.to_numpy() | |
| s1 = (sub.stratum == "S1").to_numpy() | |
| s2 = (sub.stratum == "S2").to_numpy() | |
| lab = np.isfinite(W) | |
| ev_all = (sub.E == 1).to_numpy() | |
| def rec_ht(pk): | |
| return ht_ratio(w[pk][ev[pk]], det[pk][ev[pk]], np.ones(int(ev[pk].sum()))) | |
| def rec_all(pk): | |
| e = ev_all[pk] | |
| return float(det[pk][e].mean()) if e.any() else np.nan | |
| def lead_ht(pk): | |
| return ht_ratio(w[pk][ev[pk]], lead0[pk][ev[pk]], np.ones(int(ev[pk].sum()))) | |
| def fa_h(pk): | |
| m = s1[pk] & lab[pk] | |
| return float(np.sum(w[pk][m] * fired[pk][m] * (1 - W[pk][m]))) / H | |
| def prec_ht(pk): | |
| m = s1[pk] & lab[pk] & fired[pk] | |
| return ht_ratio(w[pk][m], W[pk][m], np.ones(int(m.sum()))) | |
| def silent_h(pk): | |
| m = s2[pk] & lab[pk] | |
| return float(np.sum(w[pk][m] * W[pk][m])) / H | |
| stats = dict(recall_ht=rec_ht, recall_all=rec_all, lead0_ht=lead_ht, fa_per_h=fa_h, precision_ht=prec_ht, silent_warranted_per_h=silent_h) | |
| rec = dict(regime=r, point=p, system=X, n_events_ht=n_ev_ht, n_events_all=n_ev_all, n_labelled_W=int(lab.sum()), n_s1_labelled=int((s1 & lab).sum()), n_s2_labelled=int((s2 & lab).sum()), | |
| hours=H, thr=(thr_json["thresholds"][X][r][p]["thr"] if r != "pooled" else np.nan), reachable=(thr_json["thresholds"][X][r][p]["reachable"] if r != "pooled" else np.nan), | |
| achieved_test_alerts_per_h=(thr_json["thresholds"][X][r][p]["achieved_test"] if r != "pooled" else np.nan), descriptive_only=bool(n_ev_all < MIN_EVENTS_REGIME and r != "pooled")) | |
| full = np.arange(len(sub)) | |
| for k, fn in stats.items(): | |
| v = fn(full) | |
| bs = B.run(fn) if len(sub) else np.array([np.nan]) | |
| lo, hi = pct(bs) | |
| rec.update({k: v, f"{k}_lo": lo, f"{k}_hi": hi}) | |
| both = ev & det | |
| rec["lead_median_detected"] = float(np.median(lead[both])) if both.any() else np.nan | |
| rec["detected_share_ht"] = rec["recall_ht"] | |
| if X == "C" and p in ("val10", "test10") and r != "pooled": | |
| reps = [] | |
| for k in range(200): | |
| tl = thin_alerts(alerts[("C", r, p)], thin[r], 1000 + k) | |
| ta = np.array([first_alert(tl, u, o) if np.isfinite(o) else np.nan for u, o in zip(sub.uid, sub.onset_route_s)]) | |
| reps.append(ht_ratio(w[ev], np.isfinite(ta)[ev], np.ones(int(ev.sum())))) | |
| rec["recall_ht_thinned_to_10h"] = float(np.nanmean(reps)) if reps else np.nan | |
| rec["thin_keep_prob"] = thin[r] | |
| rows_main.append(rec) | |
| # paired deltas on identical events, same resamples | |
| for Xa, Xb in PAIRS: | |
| da, db = sub[f"det_{Xa}_{p}"].to_numpy(), sub[f"det_{Xb}_{p}"].to_numpy() | |
| la, lb = sub[f"lead0_{Xa}_{p}"].to_numpy(), sub[f"lead0_{Xb}_{p}"].to_numpy() | |
| w = sub.w.to_numpy() | |
| def d_rec(pk): | |
| e = ev[pk] | |
| return ht_ratio(w[pk][e], da[pk][e].astype(float) - db[pk][e].astype(float), np.ones(int(e.sum()))) | |
| def d_lead(pk): | |
| e = ev[pk] | |
| return ht_ratio(w[pk][e], la[pk][e] - lb[pk][e], np.ones(int(e.sum()))) | |
| full = np.arange(len(sub)) | |
| dr, dl = d_rec(full), d_lead(full) | |
| br, bl = B.run(d_rec), B.run(d_lead) | |
| disc = float((da[ev] != db[ev]).mean()) if ev.any() else np.nan | |
| rows_pair.append(dict(regime=r, point=p, pair=f"{Xa}-{Xb}", n_events_ht=n_ev_ht, delta_recall=dr, delta_recall_lo=pct(br)[0], delta_recall_hi=pct(br)[1], discordance=disc, | |
| delta_lead0=dl, delta_lead0_lo=pct(bl)[0], delta_lead0_hi=pct(bl)[1], mde_recall=mde_paired(n_ev_ht, disc if np.isfinite(disc) and disc > 0 else 0.4), | |
| mde_lead_s=mde_lead(n_ev_ht), wording_recall=wording(dr, *pct(br), what="recall"), wording_lead=wording(dl, *pct(bl), what="lead", unit=" s"), | |
| descriptive_only=bool(n_ev_all < MIN_EVENTS_REGIME and r != "pooled"))) | |
| # AUROC (secondary): score at the anchor vs W | |
| for X in SYSTEMS: | |
| s = sub[f"score_{X}_anchor"].to_numpy(float) | |
| W = sub.W.to_numpy() | |
| lab = np.isfinite(W) & np.isfinite(s) | |
| w = np.where(sub.in_ht.to_numpy(), sub.w.to_numpy(), np.nan) | |
| if lab.sum() >= 20 and len(np.unique(W[lab])) == 2: | |
| au = weighted_auroc(W[lab], s[lab], np.ones(int(lab.sum()))) | |
| b = B.run(lambda pk: weighted_auroc(W[pk][lab[pk]], s[pk][lab[pk]], np.ones(int(lab[pk].sum())))) | |
| m2 = lab & np.isfinite(w) | |
| auw = weighted_auroc(W[m2], s[m2], w[m2]) if m2.sum() >= 20 and len(np.unique(W[m2])) == 2 else np.nan | |
| rows_auroc.append(dict(regime=r, system=X, n=int(lab.sum()), n_pos=int(W[lab].sum()), auroc=au, auroc_lo=pct(b)[0], auroc_hi=pct(b)[1], auroc_ht=auw, n_ht=int(m2.sum()))) | |
| # enriched cell shares (never converted to natural rates here) | |
| for cell, g in sub[sub.W.notna()].groupby("cell"): | |
| rows_cells.append(dict(regime=r, cell=cell, stratum=g.stratum.iloc[0], n_labelled=len(g), warranted_share_enriched=float(g.W.mean()), n_events=int((g.E == 1).sum()), | |
| coverage=float(g.coverage_cell.iloc[0]) if "coverage_cell" in g else np.nan)) | |
| # pooled equal-weight sensitivity | |
| MR = pd.DataFrame(rows_main) | |
| for p in POINTS: | |
| for X in SYSTEMS: | |
| a_, b_ = MR[(MR.regime == "manual") & (MR.point == p) & (MR.system == X)], MR[(MR.regime == "engaged") & (MR.point == p) & (MR.system == X)] | |
| if len(a_) and len(b_): | |
| MR.loc[(MR.regime == "pooled") & (MR.point == p) & (MR.system == X), "recall_ht_equal_weight"] = 0.5 * (float(a_.recall_ht.iloc[0]) + float(b_.recall_ht.iloc[0])) | |
| return D, MR, pd.DataFrame(rows_pair), pd.DataFrame(rows_cells), pd.DataFrame(rows_auroc) | |
| # ----------------------------------------------------------------------------------------------------------------------- table | |
| def render_table(stamp, a, M, L, D, MR, PR, CE, AU, fc, thr_json, prereg_now, prereg_reg, log_lines): | |
| L_ = [f"## Fixed-load human evaluation — manifest {a.version}, run {stamp} (eval/newtest_analysis.py)", "", | |
| f"Pre-registration `docs/newtest_prereg.md` sha256 now `{prereg_now}`; registered `{prereg_reg or 'NOT FOUND'}` -> **{'MATCH' if prereg_now == prereg_reg else 'MISMATCH — analysis deviates from the registered protocol'}**. " | |
| f"Clips {len(M)} ({dict(Counter(M.stratum))}); labelled W {int(L.W.notna().sum())} ({int((L.W == 1).sum())} warranted), E {int(L.E.notna().sum())} ({int((L.E == 1).sum())} events); " | |
| f"HT frame = S1 u S2 (test exposure manual {fc['hours_test']['manual']:.1f} h / engaged {fc['hours_test']['engaged']:.1f} h); weights (N_cell / N_covered) / pi; S3 unweighted. " | |
| f"Bootstrap: {a.n_boot} device-cluster resamples shared across systems. Alerts {'from the manifest window (approximation)' if a.from_manifest else 'recomputed from the frozen layouts at the frozen thresholds'}.", ""] | |
| if L.attrs.get("panel"): | |
| L_ += ["**Label protocol (deviation 2026-09-13, `docs/newtest_prereg_deviations.md`): committee consensus of five members entered by one operator.** " | |
| "Pre-registered quality gates — Q4 binary κ ≥ 0.6, event_type κ ≥ 0.6, onset median |Δt| ≤ 1 s — **not applicable (not passed): no independent raters, inter-rater reliability not estimable, " | |
| "effective n_raters = 1; event timing (Task 2) was not blinded to the prefix judgement.**", ""] | |
| else: | |
| L_ += ["Label protocol: two blind raters + adjudication (pre-registered); quality gates reported by `h5_to_consensus.py`.", ""] | |
| if "n_none_prefill" in L.attrs: | |
| L_ += [f"**Stage-2 prefill provenance (deviation 2026-09-13):** of the {L.attrs['n_none']} clips labelled E = none (no event), **{L.attrs['n_none_prefill']} came from an accepted prefill** " | |
| "(CAN quiet and stage 1 = compatible + not warranted + no notable event; untouched at submit); the event-recall denominator counts only E = 1 clips, so these affect the estimate only through the " | |
| "events the prefill may have hidden — a reviewer can re-derive recall excluding them from `clips.csv` (`E_none_from_prefill`). Per-field accepted / edited shares: `h5_to_consensus.py` output.", ""] | |
| L_ += ["### Frozen operating points (alerts / h achieved on TEST at the frozen thresholds; C cannot reach 10 / h — floor shown)", "", "| system | regime | val10 | val20 | test10 | test20 |", "|---|---|---|---|---|---|"] | |
| for X in SYSTEMS: | |
| for r in REGIMES: | |
| cells = [] | |
| for p in POINTS: | |
| t = thr_json["thresholds"][X][r][p] | |
| cells.append(f"{t['achieved_test']:.1f}/h" + ("" if t["reachable"] else f" (floor; target {t['target_alerts_per_hour']:.0f})")) | |
| L_.append(f"| {X} | {r} | " + " | ".join(cells) + " |") | |
| L_ += ["", "### S1 device-cap coverage per cell (share of union events with pi > 0; HT weights post-stratified to the full cell)", "", "| cell | N | N covered | coverage | n drawn | labelled |", "|---|---|---|---|---|---|"] | |
| for row in fc["S1"]["table"]: | |
| cell = f"{row['regime']}|{row['pattern']}" | |
| nl = int(D[(D.cell == cell) & D.W.notna()].shape[0]) | |
| L_.append(f"| {cell} | {row['N_cell']} | {row['N_covered']} | {row['coverage']:.2f} | {row['n']} | {nl} |") | |
| for r in fc["S2"]["table"]: | |
| L_.append(f"| {r['regime']}|silent (S2) | {r['N_rows_frame']} rows | {r['N_covered']} | {r['coverage']:.3f} | {r['n']} | {int(D[(D.stratum == 'S2') & (D.regime == r['regime']) & D.W.notna()].shape[0])} |") | |
| for p in POINTS: | |
| tag = "PRIMARY" if p in POINTS_PRIMARY else "secondary" | |
| L_ += ["", f"### Operating point `{p}` ({tag}) — HT event recall, lead time (undetected -> 0), natural false alarms, silent-time warranted moments", "", | |
| "| regime | system | n events (HT / all) | recall HT [CI] | recall all-labelled | lead0 mean s [CI] | lead median detected s | FA / h [CI] | precision HT [CI] | silent warranted / h [CI] | note |", "|---|---|---|---|---|---|---|---|---|---|---|"] | |
| for r in REGIMES + ("pooled",): | |
| for X in SYSTEMS: | |
| g = MR[(MR.regime == r) & (MR.point == p) & (MR.system == X)] | |
| if not len(g): | |
| continue | |
| g = g.iloc[0] | |
| note = [] | |
| if r != "pooled" and not bool(g.reachable): | |
| note.append(f"C at floor {g.achieved_test_alerts_per_h:.1f}/h" + (f"; thinned to 10/h recall {f3(g.get('recall_ht_thinned_to_10h'))}" if "recall_ht_thinned_to_10h" in g and np.isfinite(g.get("recall_ht_thinned_to_10h", np.nan)) else "")) | |
| if bool(g.descriptive_only): | |
| note.append(f"descriptive (< {MIN_EVENTS_REGIME} events)") | |
| if r == "pooled" and np.isfinite(g.get("recall_ht_equal_weight", np.nan)): | |
| note.append(f"equal-weight pooling {f3(g.recall_ht_equal_weight)}") | |
| L_.append(f"| {r} | {X} | {int(g.n_events_ht)} / {int(g.n_events_all)} | {f3(g.recall_ht)} [{f3(g.recall_ht_lo, 2)}, {f3(g.recall_ht_hi, 2)}] | {f3(g.recall_all)} | {f3(g.lead0_ht, 2)} [{f3(g.lead0_ht_lo, 2)}, {f3(g.lead0_ht_hi, 2)}] | " | |
| f"{f3(g.lead_median_detected, 2)} | {f3(g.fa_per_h, 2)} [{f3(g.fa_per_h_lo, 2)}, {f3(g.fa_per_h_hi, 2)}] | {f3(g.precision_ht)} [{f3(g.precision_ht_lo, 2)}, {f3(g.precision_ht_hi, 2)}] | " | |
| f"{f3(g.silent_warranted_per_h, 2)} [{f3(g.silent_warranted_per_h_lo, 2)}, {f3(g.silent_warranted_per_h_hi, 2)}] | {'; '.join(note)} |") | |
| L_ += ["", f"Paired Δ at `{p}` on identical events (same device resamples); wording rule applied verbatim:", "", "| regime | pair | n events | Δ recall [CI] | discordance | MDE recall | Δ lead0 s [CI] | MDE lead s | wording |", "|---|---|---|---|---|---|---|---|---|"] | |
| for row in PR[PR.point == p].itertuples(): | |
| L_.append(f"| {row.regime} | {row.pair} | {row.n_events_ht} | {row.delta_recall:+.3f} [{f3(row.delta_recall_lo, 2)}, {f3(row.delta_recall_hi, 2)}] | {f3(row.discordance, 2)} | {f3(row.mde_recall, 2)} | " | |
| f"{row.delta_lead0:+.2f} [{f3(row.delta_lead0_lo, 2)}, {f3(row.delta_lead0_hi, 2)}] | {f3(row.mde_lead_s, 2)} | recall {row.wording_recall}; lead {row.wording_lead}{' (descriptive)' if row.descriptive_only else ''} |") | |
| L_ += ["", "### AUROC at the anchor (secondary; unweighted over all labelled clips; HT-weighted on S1 u S2)", "", "| regime | system | n / pos | AUROC [CI] | AUROC HT (n) |", "|---|---|---|---|---|"] | |
| for row in AU.itertuples(): | |
| L_.append(f"| {row.regime} | {row.system} | {row.n} / {row.n_pos} | {f3(row.auroc)} [{f3(row.auroc_lo, 2)}, {f3(row.auroc_hi, 2)}] | {f3(row.auroc_ht)} ({row.n_ht}) |") | |
| L_ += ["", "### Enriched — not a natural rate: warranted share per sampling cell (labelled clips)", "", "| regime | cell | stratum | n labelled | warranted share (enriched) | events | coverage |", "|---|---|---|---|---|---|---|"] | |
| for row in CE.itertuples(): | |
| L_.append(f"| {row.regime} | {row.cell} | {row.stratum} | {row.n_labelled} | {row.warranted_share_enriched:.2f} | {row.n_events} | {f3(row.coverage, 2)} |") | |
| L_ += ["", "Wording rule (pre-registered): CI containing 0 -> \"matches\"; CI excluding 0 -> \"above\" / \"below\" with (Δ, CI); never \"significantly better\", never \"equivalent\"; per-regime comparisons " | |
| f"with < {MIN_EVENTS_REGIME} labelled events are descriptive. Manual is the headline regime; engaged is reported separately in every table above; pooled = HT event-count-weighted mixture " | |
| "(equal-weight pooling shown as a sensitivity note).", ""] | |
| return "\n".join(L_) + "\n" | |
| # ----------------------------------------------------------------------------------------------------------------------- self-test | |
| def self_test(): | |
| """Synthetic frame -> sampler draw -> synthetic labels -> HT recall / FA totals unbiased; estimator identities; wording rule.""" | |
| rng = np.random.default_rng(7) | |
| # population of 1,500 union events over 12 devices (one holds 40 %), true W, E, detection indicators per system | |
| n = 1500 | |
| devs = rng.choice([f"D{i}" for i in range(12)], n, p=np.r_[0.4, np.full(11, 0.6 / 11)]) | |
| pop = pd.DataFrame(dict(uid=[f"r_{d}_{i % 9}" for i, d in enumerate(devs)], t=rng.uniform(0, 5000, n).round(1), dongle=devs, regime="manual")) | |
| pop["cell"] = rng.choice(["manual|A", "manual|B", "manual|C", "manual|A+B"], n, p=[0.3, 0.3, 0.3, 0.1]) | |
| pop["W"] = (rng.random(n) < 0.3).astype(int) | |
| pop["E"] = (rng.random(n) < 0.35).astype(int) | |
| for X, pdet in zip(SYSTEMS, (0.6, 0.45, 0.3)): | |
| pop[f"det_{X}"] = rng.random(n) < pdet | |
| pop[f"fired_{X}"] = pop.cell.str.contains(X) | (rng.random(n) < 0.05) | |
| truth = {X: (pop[f"det_{X}"] & (pop.E == 1)).sum() / (pop.E == 1).sum() for X in SYSTEMS} | |
| truth_fa = {X: float((pop[f"fired_{X}"] & (pop.W == 0)).sum()) for X in SYSTEMS} | |
| est, est_fa, cov_all = defaultdict(list), defaultdict(list), [] | |
| for rep in range(60): | |
| dev_used = Counter() | |
| drawn = [] | |
| rr = np.random.default_rng(100 + rep) | |
| for cell in sorted(pop.cell.unique(), key=lambda c: (pop.cell == c).sum()): | |
| units = pop[pop.cell == cell].to_dict("records") | |
| d, tbl, cov = NS.stratified_draw(units, 14, 10, dev_used, rr) | |
| for u in d: | |
| u["N_cell"] = len(units) | |
| drawn += d | |
| cov_all.append(cov["coverage"]) | |
| S = pd.DataFrame(drawn) | |
| S["w"] = (S.N_cell / S.N_covered_cell) / S.pi | |
| for X in SYSTEMS: | |
| e = S.E == 1 | |
| est[X].append(np.sum(S.w[e] * S[f"det_{X}"][e]) / np.sum(S.w[e])) | |
| est_fa[X].append(float(np.sum(S.w * S[f"fired_{X}"] * (1 - S.W)))) | |
| ok = True | |
| for X in SYSTEMS: | |
| bias = np.mean(est[X]) - truth[X] | |
| bias_fa = np.mean(est_fa[X]) / truth_fa[X] - 1 | |
| print(f"self-test recall {X}: truth {truth[X]:.3f} HT mean over 60 draws {np.mean(est[X]):.3f} (bias {bias:+.3f}, sd {np.std(est[X]):.3f}); FA total truth {truth_fa[X]:.0f} HT {np.mean(est_fa[X]):.0f} (rel bias {bias_fa:+.3f})") | |
| ok &= abs(bias) < 0.05 and abs(bias_fa) < 0.08 | |
| print(f"self-test coverage per cell (device cap 10): mean {np.mean(cov_all):.3f} min {np.min(cov_all):.3f} (post-stratification factor N_cell / N_covered applied)") | |
| # identities | |
| w = np.array([2.0, 3.0, 5.0]) | |
| assert ht_ratio(w, np.ones(3), np.ones(3)) == 1.0 and np.isnan(ht_ratio(w, np.ones(3), np.zeros(3))) | |
| assert weighted_auroc([1, 1, 0, 0], [0.9, 0.8, 0.2, 0.1], [1, 1, 1, 1]) == 1.0 and weighted_auroc([1, 0], [0.5, 0.5], [1, 1]) == 0.5 | |
| assert wording(0.1, -0.05, 0.2).startswith("matches") and wording(0.1, 0.02, 0.2).startswith("above") and wording(-0.1, -0.2, -0.02).startswith("below") | |
| assert "significant" not in wording(0.1, 0.02, 0.2) and "equivalent" not in wording(0.0, -0.1, 0.1) | |
| assert abs(mde_paired(75) - 0.204) < 0.01 and abs(mde_lead(75) - 0.97) < 0.02 | |
| B = DeviceBoot(np.array(["a", "a", "b", "c"]), 50, 0) | |
| assert len(B.run(lambda pk: float(len(pk)))) == 50 | |
| assert np.isnan(first_alert({"u": np.array([100.0])}, "u", 50.0)) and first_alert({"u": np.array([95.0, 100.0])}, "u", 104.0) == 95.0 and np.isnan(first_alert({"u": np.array([108.0])}, "u", 104.0)) | |
| # post-hoc sensitivity mapping: A/S -> 1, D -> 0, unflagged -> primary | |
| q = pd.Series(["A", "S", "D", ""]); y = pd.Series([0.0, 0.0, 1.0, 1.0]) | |
| wp = np.where(q.isin(["A", "S"]), 1.0, np.where(q == "D", 0.0, y.to_numpy())) | |
| assert list(wp) == [1.0, 1.0, 0.0, 1.0] and int(q.isin(["A", "S", "D"]).sum()) == 3 | |
| print("self-test identities OK (HT ratio, weighted AUROC, wording rule, MDE formula, detection window [o-10, o+3])") | |
| print("SELF-TEST", "PASS" if ok else "FAIL") | |
| return ok | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--version", default="v1") | |
| ap.add_argument("--n-boot", type=int, default=2000) | |
| ap.add_argument("--from-manifest", action="store_true") | |
| ap.add_argument("--consensus", default=str(CONSENSUS)) | |
| ap.add_argument("--event-consensus", default=str(EVENT_CONS)) | |
| ap.add_argument("--store", default=str(STORE)) | |
| ap.add_argument("--deviations", default=str(DEVIATIONS)) | |
| ap.add_argument("--self-test", action="store_true") | |
| a = ap.parse_args() | |
| if a.self_test: | |
| sys.exit(0 if self_test() else 1) | |
| t0 = time.time() | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| out = NEWTEST / f"analysis_{stamp}" | |
| out.mkdir(parents=True, exist_ok=True) | |
| lines = [] | |
| def log(m): | |
| line = f"[{time.strftime('%H:%M:%S')}] {m}" | |
| print(line, flush=True) | |
| lines.append(line) | |
| M = pd.read_csv(NEWTEST / f"manifest_{a.version}.csv") | |
| M, dev_rows = apply_deviations(M, a.deviations, log) | |
| fc = json.load(open(NEWTEST / f"frame_counts_{a.version}.json")) | |
| thr_json = json.load(open(NEWTEST / f"thresholds_frozen_{a.version}.json")) | |
| hours = thr_json["hours_test"] | |
| prereg_now = sha256_file(PREREG) if PREREG.exists() else "missing" | |
| prereg_reg = json.load(open(PREREG_HASH)).get("sha256") if PREREG_HASH.exists() else None | |
| log(f"pre-registration sha256 now {prereg_now} registered {prereg_reg} -> {'MATCH' if prereg_now == prereg_reg else 'MISMATCH'}") | |
| L = load_labels(M, a.consensus, a.event_consensus, a.store, log) | |
| if L.W.notna().sum() == 0 and L.E.notna().sum() == 0: | |
| log("no labels yet — nothing to estimate (the store fills as raters submit; run again later)") | |
| return | |
| alerts = alert_lists(thr_json, a.from_manifest, M, log) | |
| D, MR, PR, CE, AU = analyse(M, L, alerts, hours, thr_json, a.n_boot, log) | |
| # ---- sensitivity: post-hoc committee revisions applied (W_i replaced where flagged); the primary objects above are untouched | |
| n_ph = int(L.posthoc_flag.sum()) if "posthoc_flag" in L else 0 | |
| sens_md = "" | |
| if n_ph: | |
| L2 = L.copy() | |
| L2["W"] = np.where(L2.posthoc_flag == 1, L2.W_posthoc, L2.W) | |
| D2, MR2, PR2, CE2, AU2 = analyse(M, L2, alerts, hours, thr_json, a.n_boot, log, seed=0) | |
| for name, df in (("main", MR2), ("cells_enriched", CE2), ("auroc", AU2)): | |
| df.to_csv(out / f"sensitivity_posthoc_{name}.csv", index=False) | |
| rows_s = [] | |
| for r in REGIMES + ("pooled",): | |
| for p_ in POINTS: | |
| for X in SYSTEMS: | |
| g1 = MR[(MR.regime == r) & (MR.point == p_) & (MR.system == X)] | |
| g2 = MR2[(MR2.regime == r) & (MR2.point == p_) & (MR2.system == X)] | |
| if len(g1) and len(g2): | |
| g1, g2 = g1.iloc[0], g2.iloc[0] | |
| rows_s.append(f"| {r} | {p_} | {X} | {f3(g1.fa_per_h, 2)} → {f3(g2.fa_per_h, 2)} [{f3(g2.fa_per_h_lo, 2)}, {f3(g2.fa_per_h_hi, 2)}] | {f3(g1.precision_ht)} → {f3(g2.precision_ht)} | " | |
| f"{f3(g1.silent_warranted_per_h, 2)} → {f3(g2.silent_warranted_per_h, 2)} |") | |
| au_rows = [] | |
| for r_ in AU2.itertuples(): | |
| a1 = AU[(AU.regime == r_.regime) & (AU.system == r_.system)] | |
| au_rows.append(f"| {r_.regime} | {r_.system} | {f3(a1.auroc.iloc[0]) if len(a1) else '—'} → {f3(r_.auroc)} [{f3(r_.auroc_lo, 2)}, {f3(r_.auroc_hi, 2)}] |") | |
| verd = dict(Counter(L[L.posthoc_flag == 1].W_posthoc.map({1.0: "warranted", 0.0: "not_warranted"}))) | |
| sens_md = "\n".join([f"### SENSITIVITY — with post-hoc committee revisions applied ({n_ph} flagged clip(s); stage-2 note marker `[事后:…]` / `[posthoc:…]`; NOT the primary analysis)", "", | |
| "The locked stage-1 Q4 remains the primary label everywhere above. Here W_i is replaced by the committee's post-hoc verdict on the flagged clips only " | |
| f"({verd}); event recall and lead time do not depend on W_i and are unchanged. Values shown as primary → sensitivity [CI of the sensitivity value].", "", | |
| "| regime | point | system | FA / h | precision HT | silent warranted / h |", "|---|---|---|---|---|---|"] + rows_s + | |
| ["", "| regime | system | AUROC at anchor (unweighted) |", "|---|---|---|"] + au_rows + [""]) + "\n" | |
| if "E_none_from_prefill" in L: | |
| D["E_none_from_prefill"] = L.E_none_from_prefill.reindex(D.index).fillna(0).astype(int) | |
| for name, df in (("clips", D.reset_index()), ("main", MR), ("paired", PR), ("cells_enriched", CE), ("auroc", AU)): | |
| df.to_csv(out / f"{name}.csv", index=False) | |
| md = render_table(stamp, a, M, L, D, MR, PR, CE, AU, fc, thr_json, prereg_now, prereg_reg, lines) | |
| md += (sens_md if sens_md else f"Post-hoc committee revisions (stage-2 note marker): none flagged ({n_ph}); no sensitivity block.\n\n") | |
| if dev_rows: | |
| md += "### Pre-analysis exclusions applied (deviations log; strata re-weighted pi' = pi (n-1)/n)\n\n| idx | stratum | cell | reason | n stratum before | mates | N stratum |\n|---|---|---|---|---|---|---|\n" | |
| md += "".join(f"| {x['idx']} | {x['stratum']} | {x['cell']} | {x['reason']} | {x['n_stratum_before']} | {x['n_mates']} | {x['N_stratum']} |\n" for x in dev_rows) + "\n" | |
| TABLE.parent.mkdir(parents=True, exist_ok=True) | |
| if not TABLE.exists(): | |
| TABLE.write_text("# Human-evaluation test set — fixed-load results (eval/newtest_analysis.py; sections appended per run; pre-registered in docs/newtest_prereg.md)\n\n") | |
| with open(TABLE, "a") as f: | |
| f.write(md) | |
| print(md) | |
| write_run_json(out, "newtest_analysis", [Path(__file__), Path(NS.__file__)], [NEWTEST / f"manifest_{a.version}.csv", NEWTEST / f"thresholds_frozen_{a.version}.json"] + [Path(p) for p in (a.consensus, a.event_consensus, a.store) if Path(p).exists()], | |
| seed=0, t_start=t0, input_listing=False, extra=dict(version=a.version, n_boot=a.n_boot, from_manifest=a.from_manifest, prereg_sha256_now=prereg_now, prereg_sha256_registered=prereg_reg, | |
| n_labelled_W=int(L.W.notna().sum()), n_events=int((L.E == 1).sum()), log=lines, dms_fields_used=[])) | |
| log(f"DONE in {time.time() - t0:.0f}s -> {out}; section appended to {TABLE}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 44.2 kB
- Xet hash:
- d64c72168c417b9cf8555b2054fe7a095b63dda63bbd2b78b36989c82be1ecb9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.