Buckets:
| """cwm3/eval/report_final_verified.py — results/tables/stage_final_verified.md (APPEND-ONLY, dated section): the masked-protocol sensitivity of the FINAL joint | |
| model re-run with the human-VERIFIED driver-box mask (3 routes with a passenger in the causal box, owner's review of the 64 suspects on 2026-09-16; | |
| results/targets_kin_causal_routes_v2/driver_box_verified.json -> --mask-routes form driver_box_verified_mask.json) instead of the 64-route SUSPECT mask. | |
| python eval/report_final_verified.py [--seeds 0,1,2] [--out results/tables/stage_final_verified.md] [--warning] [--dry-run] | |
| Composes eval/report_joint.py blocks (load / render / verdict / claim / provenance; NOTHING is recomputed here) over the tags written by | |
| results/verified_mask_logs/run_chain.sh into results/paired_compare.csv: | |
| finalcv_s<k>_allminus_vs_indep I1_all_minus_road2drv_s<k>_final vs I1_indep_s<k>_final, v2 causal kin targets, verified mask (protocol b, 3 routes) | |
| finalcv_s<k>_allminus_vs_k0 B1 on the joint model: main vs the k0 heuristic / persistence, verified mask | |
| and puts next to every cell the all-rows (`finalc_*`) and 64-route-masked (`finalcm_*`) values of stage_final.md (the same CSV rows). | |
| --warning: appends to results/tables/warning_head_final_ext_rmv.md the pooled / manual / engaged clean-test AUROC (device-cluster CIs) of the rmv heads | |
| (item cwm3_final_ext_rmv) next to the unmasked (cwm3_final_ext) and 64-masked (cwm3_final_ext_rm) heads and their paired Δ vs the DMS att@3s chain, | |
| read from results/warning_gold/warning_gold.csv (latest run_stamp per item). | |
| Label-precision paragraph: the precision of the automatic switch labels (results/tables/review_tool_summary.md, Task B; Wilson CI) is STATED next to the | |
| hand_return cells; no cell is re-scored with human labels as targets. CPU only; no openpilot DMS field is read. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| CWM2 = CWM3.parent / "cwm2" | |
| for p in (CWM3 / "eval", CWM2 / "eval"): | |
| sys.path.insert(0, str(p)) | |
| import report_joint as RJ # noqa: E402 | |
| import kill_checks as KC # noqa: E402 | |
| V2 = CWM3 / "results" / "targets_kin_causal_routes_v2" | |
| VERIFIED, VMASK, SUSPECT = V2 / "driver_box_verified.json", V2 / "driver_box_verified_mask.json", V2 / "driver_box_suspect_routes.json" | |
| REVIEW = CWM3 / "results" / "tables" / "review_tool_summary.md" | |
| WGOLD = CWM3 / "results" / "warning_gold" / "warning_gold.csv" | |
| PROTOCOLS = (("verified mask (3 routes)", "finalcv"), ("suspect mask (64 routes)", "finalcm"), ("all rows", "finalc")) | |
| DRV = ("hand_leave", "hand_return", "head_turn_onset") | |
| VEH = ("brake_on", "gas_on", "steer_phase_on", "veh_any_on") | |
| CELLS = [f"drv_onset|{e}|3s" for e in DRV] + [f"veh_onset|{e}|3s" for e in VEH] | |
| STRATA = ("clean_test", "clean_test|manual", "clean_test|engaged") | |
| def ms(v): | |
| v = [float(x) for x in v if str(x) not in ("", "nan")] | |
| return "—" if not v else (f"{np.mean(v):.3f} ± {np.std(v, ddof=1):.3f}" if len(v) > 1 else f"{v[0]:.3f}") | |
| def fmt_d(r): | |
| return f"{float(r.delta):+.4f} [{float(r.delta_ci_lo):+.4f},{float(r.delta_ci_hi):+.4f}]" | |
| def npos(g): | |
| return f"{int(g.n.iloc[0])} / {int(g.n_pos.iloc[0])}" if len(g) and str(g.n_pos.iloc[0]).strip() not in ("", "nan") else "—" | |
| def mask_info(): | |
| v = json.loads(VERIFIED.read_text()) | |
| m = json.loads(VMASK.read_text()) | |
| s = json.loads(SUSPECT.read_text()) | |
| return {"n_verified_mask": len(m["flagged"]), "mask_uids": [r["uid"] for r in m["flagged"]], "n_driver": len(v["driver_routes"]), "n_unsure": len(v["unsure_routes"]), | |
| "n_suspect": s["n_routes_flagged"], "anchors": v["anchors_by_split"], "verified_written": v["written_utc"], "mask_written": m["written_utc"], "review_definition": v["definition"]} | |
| def precision_rows(): | |
| """Task B of the LAST review-tool summary: {(regime, event): dict(n, correct, wrong, timing, unsure, p, lo, hi)} — parsed, not recomputed.""" | |
| txt = REVIEW.read_text().split("### Task B")[-1] | |
| rx = re.compile(r"^\| (manual|engaged) \| (\w+) \| (\d+) \| (\d+) \| (\d+) \| (\d+) \| (\d+) \| ([0-9.]+) \[([0-9.]+), ([0-9.]+)\] \|") | |
| out = {} | |
| for line in txt.splitlines(): | |
| mt = rx.match(line.strip()) | |
| if mt: | |
| reg, ev, n, c, w, tm, un, p, lo, hi = mt.groups() | |
| out[(reg, ev)] = dict(n=int(n), correct=int(c), wrong=int(w), timing=int(tm), unsure=int(un), p=float(p), lo=float(lo), hi=float(hi)) | |
| return out | |
| def joint_section(seeds): | |
| mi = mask_info() | |
| dfs = {st: RJ.load(st, "allminus", "indep", seeds) for _, st in PROTOCOLS} | |
| k0s = {st: RJ.load(st, "allminus", "k0", seeds) for _, st in PROTOCOLS} | |
| L = [] | |
| # ---- side-by-side own endpoints | |
| L += ["### 1. Driver + vehicle onset cells (PR-AUC@3s), A = all_minus_road2drv (main), B = indep — verified mask next to the suspect mask and all rows", "", | |
| "Rows per cell: `finalcv` (3 verified wrong-occupant routes censored from every population), `finalcm` (64 suspect routes censored; stage_final.md §4), `finalc` (all rows; stage_final.md §1). " | |
| "Paired device-cluster bootstrap on identical rows, n_boot 1000, seeds 0 / 1 / 2; verdict = kill_checks.beats (≥ 2/3 seeds CI excl. 0, same sign, |mean Δ| > seed sd).", "", | |
| "| regime | cell | protocol | A s0 / s1 / s2 | B s0 / s1 / s2 | Δ s0 [CI] | Δ s1 [CI] | Δ s2 [CI] | mean Δ ± sd | n / pos | verdict |", "|---|---|---|---|---|---|---|---|---|---|---|"] | |
| summary = {} | |
| for regime in ("manual", "engaged"): | |
| for cell in CELLS: | |
| for label, st in PROTOCOLS: | |
| df = dfs[st] | |
| g = df[(df.regime == regime) & (df.cell == cell) & (df.metric == "pr_auc")].sort_values("seed") if len(df) else df | |
| if not len(g): | |
| L.append(f"| {regime} | {cell.replace('|', '/')} | {label} | no rows yet (`{st}_s<k>_allminus_vs_indep`) | | | | | | | |") | |
| summary[(regime, cell, st)] = None | |
| continue | |
| res = RJ.verdict(g) | |
| by = {int(s): r for s, r in zip(g.seed, g.itertuples())} | |
| A = " / ".join(f"{by[s].value_a:.4f}" if s in by else "—" for s in seeds) | |
| B = " / ".join(f"{by[s].value_b:.4f}" if s in by else "—" for s in seeds) | |
| D = [fmt_d(by[s]) if s in by else "—" for s in seeds] | |
| d = g.delta.astype(float).to_numpy() | |
| L.append(f"| {regime} | {cell.replace('|', '/')} | {label} | {A} | {B} | " + " | ".join(D) + f" | {d.mean():+.4f} ± {d.std(ddof=1) if len(d) > 1 else 0.0:.4f} | {npos(g)} | {res['verdict']} |") | |
| summary[(regime, cell, st)] = (d.mean(), d.std(ddof=1) if len(d) > 1 else 0.0, res["verdict"], npos(g)) | |
| # ---- compact summary | |
| L += ["", "### 1b. Compact: mean Δ ± sd (verdict) per protocol", "", "| regime | cell | verified mask (3) | suspect mask (64) | all rows |", "|---|---|---|---|---|"] | |
| for regime in ("manual", "engaged"): | |
| for cell in CELLS: | |
| cells = [] | |
| for _, st in PROTOCOLS: | |
| v = summary.get((regime, cell, st)) | |
| cells.append("no rows yet" if v is None else f"{v[0]:+.4f} ± {v[1]:.4f} ({v[2]}; {v[3]})") | |
| L.append(f"| {regime} | {cell.replace('|', '/')} | " + " | ".join(cells) + " |") | |
| # ---- B1 | |
| L += ["", "### 2. Claim B1 on the joint model — driver onset PR-AUC@3s vs the k0 heuristic (causal K0) / persistence (base rate), three protocols", "", | |
| "| regime | event | protocol | model s0 / s1 / s2 (mean ± sd) | k0 | persistence (base rate) | Δ s0 / s1 / s2 [CI] | n / pos | verdict |", "|---|---|---|---|---|---|---|---|---|"] | |
| b1_rule = {} | |
| for label, st in PROTOCOLS: | |
| df = k0s[st] | |
| verd = {} | |
| for regime in ("manual", "engaged"): | |
| for ev in DRV: | |
| cell = f"drv_onset|{ev}|3s" | |
| if not len(df): | |
| continue | |
| g = df[(df.regime == regime) & (df.cell == cell) & (df.metric == "pr_auc")].sort_values("seed") | |
| base = df[(df.regime == regime) & (df.cell == cell) & (df.metric == "base_rate")] | |
| if not len(g): | |
| continue | |
| res = RJ.verdict(g) | |
| verd[(regime, cell, "pr_auc")] = res | |
| vals = " / ".join(f"{float(v):.3f}" for v in g.value_a) | |
| cells = " / ".join(fmt_d(r) for r in g.itertuples()) | |
| L.append(f"| {regime} | {ev} | {label} | {vals} ({ms(g.value_a)}) | {ms(g.value_b)} | {ms(base.value_a) if len(base) else '—'} | {cells} | {npos(g)} | {res['verdict']} |") | |
| if not verd: | |
| L.append(f"| — | — | {label} | no rows yet (`{st}_s<k>_allminus_vs_k0`) | | | | | |") | |
| b1_rule[st] = ("NO ROWS", 0, {}) | |
| continue | |
| stb, _, _, _ = RJ.claim(verd, "B1") | |
| n_beats = sum(v["verdict"] == "BEATS" for v in verd.values()) | |
| b1_rule[st] = (stb, n_beats, {f"{r}|{c.split('|')[1]}": v["verdict"] for (r, c, _), v in verd.items()}) | |
| L += ["", "B1 rule (hand_leave AND head_turn_onset BEATS k0 in both regimes): " + " · ".join(f"{label} **{b1_rule[st][0]}** ({b1_rule[st][1]}/6 cells BEATS)" for label, st in PROTOCOLS) + ".", ""] | |
| # ---- full verified block + B2 claim, with the WORSE / BEATS lists of the three protocols | |
| dfv = dfs["finalcv"] | |
| claims = {} | |
| for label, st in PROTOCOLS: | |
| df = dfs[st] | |
| if len(df): | |
| _, v = RJ.render(df, sorted(df.seed.unique()), "allminus", "indep", "x") | |
| claims[st] = RJ.claim(v, "B2") | |
| if len(dfv): | |
| body, vv = RJ.render(dfv, sorted(dfv.seed.unique()), "allminus", "indep", "Verified-mask protocol (3 human-verified wrong-occupant routes censored): all_minus_road2drv vs indep — every cell") | |
| stv, worsev, bcv, bov = claims["finalcv"] | |
| L += ["### 3. Every cell under the verified mask (own endpoints, cross-leg C1 / C2, hazard NLL, emissions, nr / cooccur Brier)", ""] + body + ["", | |
| f"B2 verdict (verified mask): **{stv}** — own endpoints WORSE {[f'{r}|{c}' for r, c, _ in worsev] or 'none'}; cross-leg BEATS {[f'{r}|{c}|{m}' for r, c, m in bcv] or 'none'}; own BEATS {[f'{r}|{c}' for r, c, _ in bov] or 'none'}", ""] | |
| else: | |
| L += ["### 3. Every cell under the verified mask — no `finalcv_*` rows yet", ""] | |
| L += ["B2 across protocols: " + " · ".join(f"{label} **{claims[st][0]}** (own WORSE {[f'{r}|{c}' for r, c, _ in claims[st][1]] or 'none'}; own BEATS {[f'{r}|{c}' for r, c, _ in claims[st][3]] or 'none'})" | |
| if st in claims else f"{label} NO ROWS" for label, st in PROTOCOLS) + ".", ""] | |
| # ---- label precision paragraph | |
| pr = precision_rows() | |
| L += ["### 4. Label-precision-aware reading of the hand_return cells (precision stated, nothing re-scored)", ""] | |
| if pr: | |
| L += ["| regime | event | n labelled | correct | wrong | precision [Wilson 95 % CI] |", "|---|---|---|---|---|---|"] | |
| for reg in ("manual", "engaged"): | |
| for ev in DRV: | |
| x = pr.get((reg, ev)) | |
| if x: | |
| L.append(f"| {reg} | {ev} | {x['n']} | {x['correct']} | {x['wrong']} | {x['p']:.3f} [{x['lo']:.3f}, {x['hi']:.3f}] |") | |
| hr = {reg: pr.get((reg, "hand_return")) for reg in ("manual", "engaged")} | |
| pos = {} | |
| for reg in ("manual", "engaged"): | |
| v = summary.get((reg, "drv_onset|hand_return|3s", "finalcv")) or summary.get((reg, "drv_onset|hand_return|3s", "finalc")) | |
| pos[reg] = int(v[3].split("/")[1]) if v else None | |
| para = ["The owner's validation of 200 automatic switch labels (`results/tables/review_tool_summary.md`, Task B) puts the precision of the automatic `hand_return` label at " | |
| + " and ".join(f"**{hr[r]['p']:.2f}** [{hr[r]['lo']:.2f}, {hr[r]['hi']:.2f}] ({hr[r]['correct']}/{hr[r]['n']}, {r})" for r in ("manual", "engaged") if hr[r]) | |
| + f"; hand_leave {pr[('manual', 'hand_leave')]['p']:.2f} / {pr[('engaged', 'hand_leave')]['p']:.2f} and head_turn_onset {pr[('manual', 'head_turn_onset')]['p']:.2f} / {pr[('engaged', 'head_turn_onset')]['p']:.2f} (manual / engaged)."] | |
| bound = [] | |
| for r in ("manual", "engaged"): | |
| if hr[r] and pos[r]: | |
| lo_f, hi_f, mid_f = 1 - hr[r]["hi"], 1 - hr[r]["lo"], 1 - hr[r]["p"] | |
| bound.append(f"{r}: n_pos = {pos[r]:,} automatic positives in the hand_return population, of which {mid_f:.0%} ≈ {int(round(mid_f * pos[r])):,} " | |
| f"(CI {lo_f:.0%}–{hi_f:.0%} ≈ {int(round(lo_f * pos[r])):,}–{int(round(hi_f * pos[r])):,}) would not be confirmed hand-return events if the validated precision carries over " | |
| f"(the validation sample is the labelled item set, not a population sample — order-of-magnitude figures)") | |
| if bound: | |
| para.append("Bound on the positives: " + "; ".join(bound) + ".") | |
| para.append("Reading: (i) every hand_return Δ above is a difference in agreement with the AUTOMATIC label — both arms are scored on the same rows, so a label error shifts A and B alike and the paired Δ " | |
| "remains a between-model comparison on that definition, but the absolute PR-AUC level is not the PR-AUC on true hand-return events; (ii) the engaged hand_return cells, where " | |
| f"{(1 - hr['engaged']['p']):.0%} of the validated automatic positives were wrong, are the least interpretable of the six driver cells — their verdicts " | |
| + " / ".join(f"{summary[('engaged', 'drv_onset|hand_return|3s', st)][2]} ({label})" for label, st in PROTOCOLS if summary.get(('engaged', 'drv_onset|hand_return|3s', st))) | |
| + " under B2 and vs k0 " + " / ".join(f"{b1_rule[st][2].get('engaged|hand_return', '—')} ({label})" for label, st in PROTOCOLS if b1_rule.get(st)) | |
| + " carry no directional claim; (iii) the manual hand_return cells (precision " | |
| f"{hr['manual']['p']:.2f}) keep their verdicts as statements about the automatic definition — B2 " + " / ".join(f"{summary[('manual', 'drv_onset|hand_return|3s', st)][2]}" for _, st in PROTOCOLS if summary.get(('manual', 'drv_onset|hand_return|3s', st))) | |
| + ", vs k0 " + " / ".join(f"{b1_rule[st][2].get('manual|hand_return', '—')}" for _, st in PROTOCOLS if b1_rule.get(st)) | |
| + " (verified / suspect / all) — and should be worded 'on the automatic hand_return definition (precision 0.77)'; (iv) hand_leave (0.85 / 0.97) and head_turn_onset (0.97 / 1.00) are not " | |
| "affected by this caveat beyond a 0–15 % label-noise floor. Nothing was re-scored with the human labels as targets (the 200 items are a precision check, not a test set).") | |
| L += [""] + para | |
| else: | |
| L += ["review_tool_summary.md Task B table not found — precision not stated.", ""] | |
| # ---- provenance | |
| allrows = pd.concat([d for d in list(dfs.values()) + list(k0s.values()) if len(d)], ignore_index=True) | |
| prov, stale = RJ.provenance(allrows) if len(allrows) else ([], []) | |
| head = [f"Mask: `{VMASK.relative_to(CWM3)}` ({mi['n_verified_mask']} routes: " + ", ".join(f"`{u}`" for u in mi["mask_uids"]) + f"; written {mi['mask_written']}) = the `mask_routes` of " | |
| f"`{VERIFIED.relative_to(CWM3)}` ({mi['review_definition']}; {mi['n_driver']} verified driver, {mi['n_unsure']} unsure; written {mi['verified_written']}). Verified wrong-occupant anchors per split: " | |
| + "; ".join(f"{k} {v['verified_wrong']:,} of {v['total_flagged']:,} flagged" for k, v in mi["anchors"].items()) + f". The previous masked protocol (`finalcm_*`) censored all {mi['n_suspect']} suspect routes.", "", | |
| "Runs `I1_all_minus_road2drv_s<k>_final` (main) vs `I1_indep_s<k>_final` (assembled bases), driver populations from the v2 causal kin targets (`--kin-targets causal_routes --kin-version v2`), " | |
| "test split, both regimes; commands in `results/verified_mask_logs/run_chain.sh`, logs `results/verified_mask_logs/pc_finalcv_*.log`. Every number comes from `results/paired_compare.csv`.", "", | |
| "#### Provenance of the A / B layouts (guard 2): rows older than the layout are STALE", "", "| run | regime | layout mtime (UTC) | source sha256 | model_id / code | status |", "|---|---|---|---|---|---|", *prov, ""] | |
| if stale: | |
| head += [f"**STALE ROWS DETECTED for {stale} — verdicts VOID.**", ""] | |
| return head + L, {"summary": {f"{r}|{c}|{st}": v for (r, c, st), v in summary.items()}, "b1": b1_rule, "b2": {st: c[0] for st, c in claims.items()}, "stale": stale} | |
| def warning_section(): | |
| if not WGOLD.exists(): | |
| return ["warning_gold.csv missing"], {} | |
| W = pd.read_csv(WGOLD, low_memory=False) | |
| items = (("verified mask (3 routes), tag rmv", "cwm3_final_ext_rmv", "rmv"), ("suspect mask (64 routes), tag rm", "cwm3_final_ext_rm", "rm"), ("unmasked", "cwm3_final_ext", "")) | |
| heads = (("current (HGB)", "W_learned_curhgb_t15_w10_n979{s}"), ("current + dwell + face (HGB)", "W_learned_curhgb_t15_w10_n979dwfc{s}")) | |
| L1 = ["| head | mask protocol (item, warning_gold run) | pooled AUROC [CI] | manual AUROC [CI] | engaged AUROC [CI] | n / pos (pooled, manual, engaged) |", "|---|---|---|---|---|---|"] | |
| L2 = ["| head | mask protocol | stratum | Δ vs DMS att@3s s0 / s1 / s2 [CI] | mean Δ | verdict (kill_checks.beats) |", "|---|---|---|---|---|---|"] | |
| out = {} | |
| def level_cells(w, arm, col, seeds): | |
| cells, ns = [], [] | |
| for st in STRATA: | |
| g = w[(w.arm == arm) & (w.event_or_family == col) & (w.metric == "auroc") & (w.calib == "raw") & w.comparator.isna() & (w.stratum == st)] | |
| g = g[g.seed.astype(str).str.isdigit()].drop_duplicates("seed").sort_values("seed", key=lambda s: s.astype(int)) | |
| if not len(g): | |
| cells.append("—"); ns.append("—"); continue | |
| vals = [f"{float(r.value):.3f} [{float(r.ci_lo):.2f},{float(r.ci_hi):.2f}]" for r in g.itertuples()] | |
| cells.append(vals[0] + (f" (×{len(vals)} seeds identical)" if len(set(vals)) == 1 and len(vals) > 1 else "") if len(set(vals)) == 1 else " / ".join(vals)) | |
| ns.append(f"{int(g.n.iloc[0])}/{int(g.n_pos.iloc[0])}") | |
| return cells, ns | |
| for label, item, sfx in items: | |
| w = W[W.item == item] | |
| if not len(w): | |
| for hl, _ in heads: | |
| L1.append(f"| {hl} | {label} (`{item}`) | no rows yet | | | |") | |
| continue | |
| stamp = w.run_stamp.max() | |
| w = w[w.run_stamp == stamp] | |
| seeds = sorted(int(s) for s in w.seed.unique() if str(s).isdigit()) | |
| for hl, tmpl in heads: | |
| col = tmpl.format(s=sfx) | |
| arms = sorted(a for a in w[w.event_or_family == col].arm.unique() if a == "cwm3_W" or str(a).startswith("abl_W")) | |
| if not arms: | |
| L1.append(f"| {hl} | {label} (`{item}`, {stamp}) | column `{col}` absent | | | |") | |
| continue | |
| for arm in arms: | |
| cells, ns = level_cells(w, arm, col, seeds) | |
| L1.append(f"| {hl} | {label} (`{item}`, {stamp}) | " + " | ".join(cells) + " | " + ", ".join(ns) + " |") | |
| out[f"{item}|{col}"] = dict(zip(STRATA, cells)) | |
| for st in STRATA: | |
| pr = KC.paired(w, arm, "legacy_att", "gold_W", col, metric="auroc", stratum=st) | |
| pr = pr[pr.seed.astype(str).str.isdigit()] | |
| if not len(pr): | |
| continue | |
| va = KC.values(w, arm, "gold_W", metric="auroc", stratum=st, score=col) | |
| vb = w[(w.arm == "legacy_att") & (w.stratum == st) & (w.metric == "auroc") & (w.calib == "raw") & w.comparator.isna()] | |
| res = KC.beats(pr, va[va.seed.astype(str).str.isdigit()], vb[vb.seed.astype(str).str.isdigit()], min_seeds=3) | |
| ds = [] | |
| for s in seeds: | |
| x = pr[pr.seed.astype(int) == s] | |
| ds.append(f"{float(x.delta.iloc[0]):+.3f} [{float(x.delta_ci_lo.iloc[0]):+.3f},{float(x.delta_ci_hi.iloc[0]):+.3f}]" if len(x) else "—") | |
| L2.append(f"| {hl} | {label} | {st} | " + " / ".join(ds) + f" | {res.get('mean_delta', float('nan')):+.3f} | {res['verdict']} ({res.get('ci_excl_zero', '?')}/{len(seeds)} CI excl. 0) |") | |
| out[f"{item}|{col}|dms|{st}"] = res["verdict"] | |
| cells, ns = level_cells(w, "legacy_att", "att@3s", seeds) | |
| L1.append(f"| DMS att@3s (legacy_att, comparator) | {label} (`{item}`, {stamp}) | " + " | ".join(cells) + " | " + ", ".join(ns) + " |") | |
| L = ["Clean-test AUROC (raw) per stratum of the row-masked heads under the three mask protocols; every number is read from `results/warning_gold/warning_gold.csv` (latest run_stamp per item; " | |
| "device-cluster bootstrap CIs of the unchanged `cwm2/eval/warning_gold.py`). The gold is the clean HUMAN gold (episode labels), so the precision of the AUTOMATIC switch labels " | |
| "(review_tool_summary.md Task B) does not enter these numbers; a mask only removes the gold rows on the masked routes, which is why n / pos differ between protocols. " | |
| "Current-state heads are seed-invariant by construction (the joint model's seed enters no current-state feature), so the three seeds coincide for the learned heads; the DMS comparator differs per seed.", "", | |
| "#### A. AUROC per stratum", ""] + L1 + ["", "#### B. Paired Δ vs the DMS attention chain (identical rows / resamples)", ""] + L2 | |
| return L, out | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0,1,2") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "stage_final_verified.md")) | |
| ap.add_argument("--warning", action="store_true", help="also append the rmv-vs-rm-vs-unmasked AUROC section to --warning-out") | |
| ap.add_argument("--warning-out", default=str(CWM3 / "results" / "tables" / "warning_head_final_ext_rmv.md")) | |
| ap.add_argument("--dry-run", action="store_true", help="print only; write nothing") | |
| ap.add_argument("--skip-joint", action="store_true", help="do not (re-)append the joint-model section (use with --warning once the rmv item exists)") | |
| a = ap.parse_args() | |
| seeds = [int(s) for s in a.seeds.split(",")] | |
| stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| if a.skip_joint: | |
| body, dec = None, None | |
| else: | |
| body, dec = joint_section(seeds) | |
| title = "# Stage FINAL — verified-mask sensitivity (owner's human review of the 64 driver-box suspect routes, 2026-09-16); append-only" | |
| if body is not None: | |
| sec = [f"## Appended {stamp} — masked protocol re-run with the VERIFIED mask (3 routes) next to the 64-route suspect mask and all rows (eval/report_final_verified.py)", ""] + body | |
| txt = "\n".join(sec) + "\n" | |
| if a.dry_run: | |
| print(txt) | |
| else: | |
| out = Path(a.out) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| with open(out, "a") as f: | |
| if not out.exists() or out.stat().st_size == 0: | |
| f.write(title + "\n\n") | |
| f.write(("\n---\n\n" if out.stat().st_size else "") + txt) | |
| (out.parent / "stage_final_verified_decision.json").write_text(json.dumps({"stamp": stamp, "seeds": seeds, **dec}, indent=1, default=str)) | |
| print(txt) | |
| print(f"appended to {out}; decision json {out.parent / 'stage_final_verified_decision.json'}") | |
| if a.warning: | |
| wl, wd = warning_section() | |
| wsec = [f"## Appended {stamp} — verified-mask (rmv) vs suspect-mask (rm) vs unmasked heads: clean-test AUROC per stratum (eval/report_final_verified.py --warning)", ""] + wl | |
| wtxt = "\n".join(wsec) + "\n" | |
| if a.dry_run: | |
| print(wtxt) | |
| else: | |
| wo = Path(a.warning_out) | |
| with open(wo, "a") as f: | |
| f.write("\n\n---\n\n" + wtxt) | |
| print(wtxt) | |
| print(f"appended to {wo}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 24.4 kB
- Xet hash:
- f02a665f5a93f9f9b621aa9d45162689d9211e612e41191e918460ad84cf2638
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.