Buckets:
| """cwm3/eval/report_final_ncroi.py — results/tables/stage_final_ncroi.md: roadmap E4, the joint-level NON-CAUSAL ROI sensitivity row (2026-09-13; pre-registered in | |
| docs/decisions.md BEFORE training) from results/paired_compare.csv. Composes eval/report_joint.py blocks (load / render / verdict / claim / provenance); nothing is | |
| recomputed here; every number comes from the CSV rows written by eval/run_joint_evals_final_ncroi.sh. | |
| python eval/report_final_ncroi.py [--seeds 0,1,2] [--out results/tables/stage_final_ncroi.md] | |
| Blocks (tags): finalc_s<k>_allminusncroi_vs_allminus (a) ncroi variant vs the FROZEN main on the v2 CAUSAL driver targets — the pre-registered comparison | |
| finalc_s<k>_allminusncroi_vs_indepncroi (b) interaction effect on the ncroi bases; finalc_s<k>_indepncroi_vs_indep the U-base swap alone | |
| finalcm_* masked protocol; finalc/finalcm_*_vs_k0 B1 on the variant; final_* DESCRIPTIVE cwm2 route-median targets (the leak picture) | |
| finalP_* P cells | |
| PRE-REGISTERED EXPECTED READING: the joint-level confirmation of K4 — on the causal targets the causal ROI is NOT worse than the non-causal ROI, i.e. A = all_minus_ncroi | |
| BEATS the frozen main on no driver own endpoint (drv_onset hand_leave / hand_return / head_turn_onset @3s); a non-causal "gain" shows only on the route-median targets. | |
| Verdicts per cell (kill_checks.beats); the frozen main stays the headline whatever the outcome. Also writes results/tables/stage_final_ncroi_decision.json. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import report_joint as RJ # noqa: E402 | |
| EV = ("hand_leave", "hand_return", "head_turn_onset") | |
| DRV = [f"drv_onset|{e}|3s" for e in EV] | |
| def block(stage, a, b, claim, title, seeds): | |
| df = RJ.load(stage, a, b, seeds) | |
| if not len(df): | |
| return df, [f"### {title}: A = {a}, B = {b} — no rows yet (tags `{stage}_s<k>_{a}_vs_{b}`)", ""], {}, ("NO ROWS", [], [], []) | |
| body, verdicts = RJ.render(df, sorted(df.seed.unique()), a, b, title) | |
| return df, body, verdicts, RJ.claim(verdicts, claim) | |
| def ms(v): | |
| v = [float(x) for x in v if str(x) not in ("", "nan")] | |
| return "—" if not v else (f"{np.mean(v):.3f} ± {np.std(v, ddof=1):.3f}" if len(v) > 1 else f"{v[0]:.3f}") | |
| def fmt(ks): | |
| return [f"{r}|{c}" + (f"|{m}" if m != "pr_auc" else "") for (r, c, m) in ks] or "none" | |
| def drv_table(verdicts, title): | |
| lines = [f"### {title}", "", "| regime | driver cell | mean Δ (A − B) | verdict |", "|---|---|---|---|"] | |
| for r in ("manual", "engaged"): | |
| for c in DRV: | |
| v = verdicts.get((r, c, "pr_auc")) | |
| lines.append(f"| {r} | {c.replace('|', '/')} | {v['mean_delta']:+.4f} | {v['verdict']} |" if v else f"| {r} | {c.replace('|', '/')} | — | NO ROWS |") | |
| return lines + [""] | |
| def b1_lines(seeds, dfs): | |
| out, js = [], {} | |
| for proto, stage in (("all rows", "finalc"), ("suspect routes masked", "finalcm")): | |
| out += [f"### B1 on the ncroi variant — driver onset PR-AUC@3s (v2 causal targets) vs the k0 heuristic / persistence, protocol: {proto}", "", | |
| "| variant | regime | event | model | k0 | persistence (base rate) | Δ per seed [CI] | verdict |", "|---|---|---|---|---|---|---|---|"] | |
| for var in ("indepncroi", "allminusncroi"): | |
| if proto != "all rows" and var == "indepncroi": | |
| continue | |
| df = RJ.load(stage, var, "k0", seeds) | |
| dfs.append(df) | |
| if not len(df): | |
| out.append(f"| {var} | — | — | no rows yet (`{stage}_s<k>_{var}_vs_k0`) | | | | |") | |
| continue | |
| verd = {} | |
| for reg in ("manual", "engaged"): | |
| for ev in EV: | |
| cell = f"drv_onset|{ev}|3s" | |
| g = df[(df.regime == reg) & (df.cell == cell) & (df.metric == "pr_auc")].sort_values("seed") | |
| base = df[(df.regime == reg) & (df.cell == cell) & (df.metric == "base_rate")] | |
| if not len(g): | |
| continue | |
| res = RJ.verdict(g) | |
| verd[(reg, cell, "pr_auc")] = res | |
| cells = " / ".join(f"{float(r.delta):+.3f} [{float(r.delta_ci_lo):+.3f},{float(r.delta_ci_hi):+.3f}]" for r in g.itertuples()) | |
| out.append(f"| {var} | {reg} | {ev} | {ms(g.value_a)} | {ms(g.value_b)} | {ms(base.value_a) if len(base) else '—'} | {cells} | {res['verdict']} |") | |
| stb, _, _, _ = RJ.claim(verd, "B1") | |
| n_beats = sum(v["verdict"] == "BEATS" for v in verd.values()) | |
| js[f"{var}|{proto}"] = {"verdict": stb, "beats": n_beats, "cells": {f"{r}|{c}": v["verdict"] for (r, c, _), v in verd.items()}} | |
| out.append(f"| {var} | **B1 ({proto})** | | | | | {n_beats}/6 cells BEATS k0; rule (hand_leave AND head_turn_onset BEATS in both regimes) | **{stb}** |") | |
| out.append("") | |
| return out, js | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0,1,2") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "stage_final_ncroi.md")) | |
| a = ap.parse_args() | |
| seeds = [int(s) for s in a.seeds.split(",")] | |
| dfs = [] | |
| dA, bodyA, vA, (stA, worseA, bcA, boA) = block("finalc", "allminusncroi", "allminus", "B2", "(a) ncroi variant vs the FROZEN main — v2 CAUSAL kin targets", seeds) | |
| dB, bodyB, vB, (stB, worseB, bcB, boB) = block("finalc", "allminusncroi", "indepncroi", "B2", "(b) interaction effect on the ncroi bases: all_minus_road2drv_ncroi vs indep_ncroi (B2-style)", seeds) | |
| dC, bodyC, vC, (stC, worseC, bcC, boC) = block("finalc", "indepncroi", "indep", "B2", "U-base swap alone: indep_ncroi vs indep (assembled bases, no training) — causal targets", seeds) | |
| dAm, bodyAm, vAm, (stAm, worseAm, bcAm, boAm) = block("finalcm", "allminusncroi", "allminus", "B2", "Masked protocol (64 v2 suspect routes censored): ncroi variant vs FROZEN main", seeds) | |
| dBm, bodyBm, vBm, _ = block("finalcm", "allminusncroi", "indepncroi", "B2", "Masked protocol: all_minus_road2drv_ncroi vs indep_ncroi", seeds) | |
| dL1, bodyL1, vL1, _ = block("final", "allminusncroi", "allminus", "B2", "DESCRIPTIVE — cwm2 ROUTE-MEDIAN (non-causal) kin targets: ncroi variant vs FROZEN main (the leak picture)", seeds) | |
| dL2, bodyL2, vL2, _ = block("final", "indepncroi", "indep", "B2", "DESCRIPTIVE — cwm2 route-median kin targets: indep_ncroi vs indep", seeds) | |
| dP1, bodyP1, vP1, _ = block("finalP", "allminusncroi", "allminus", "B3", "P cells (road-pressure onset PR-AUC@3s, eval/pressure_eval.py): ncroi variant vs FROZEN main", seeds) | |
| dP2, bodyP2, vP2, _ = block("finalP", "allminusncroi", "indepncroi", "B3", "P cells: all_minus_road2drv_ncroi vs indep_ncroi", seeds) | |
| dfs += [dA, dB, dC, dAm, dBm, dL1, dL2, dP1, dP2] | |
| b1, b1_json = b1_lines(seeds, dfs) | |
| # ---- the pre-registered expected reading | |
| drvA = {(r, c): vA[(r, c, "pr_auc")]["verdict"] for r in ("manual", "engaged") for c in DRV if (r, c, "pr_auc") in vA} | |
| ncroi_beats = [f"{r}|{c}" for (r, c), v in drvA.items() if v == "BEATS"] # the non-causal ROI beating the causal one on the causal targets | |
| ncroi_worse = [f"{r}|{c}" for (r, c), v in drvA.items() if v == "WORSE"] # the causal ROI better | |
| drvAm = {(r, c): vAm[(r, c, "pr_auc")]["verdict"] for r in ("manual", "engaged") for c in DRV if (r, c, "pr_auc") in vAm} | |
| drvL = {(r, c): vL1[(r, c, "pr_auc")]["verdict"] for r in ("manual", "engaged") for c in DRV if (r, c, "pr_auc") in vL1} | |
| n_seeds = int(dA.seed.nunique()) if len(dA) else 0 | |
| insuff = any(v["verdict"] == "INSUFFICIENT" for (r, c, m), v in vA.items() if c in DRV) | |
| if not len(dA): | |
| reading, rtxt = "NOT DECIDABLE", "no finalc_s<k>_allminusncroi_vs_allminus rows yet" | |
| elif n_seeds < 3 or insuff or len(drvA) < 6: | |
| reading, rtxt = "INSUFFICIENT", f"{n_seeds}/3 seeds paired, {len(drvA)}/6 driver cells" | |
| elif not ncroi_beats: | |
| reading, rtxt = "EXPECTED READING HOLDS", f"the non-causal ROI beats the causal ROI on no driver own endpoint of the causal targets (causal better on {ncroi_worse or 'none'}; the rest NOT_WORSE / INCONCLUSIVE)" | |
| else: | |
| reading, rtxt = "EXPECTED READING DOES NOT HOLD", f"the non-causal ROI BEATS the causal ROI on {ncroi_beats} (causal better on {ncroi_worse or 'none'})" | |
| allrows = pd.concat([d for d in dfs if len(d)], ignore_index=True) if any(len(d) for d in dfs) else pd.DataFrame(columns=["a", "b", "regime", "run_stamp"]) | |
| prov, stale = RJ.provenance(allrows) if len(allrows) else ([], []) | |
| void = f" — **VOID: stale rows for {stale}**" if stale else "" | |
| L = ["# Stage FINAL — E4 sensitivity row: the frozen joint recipe on a NON-CAUSAL ROI driver base (`U0_kin_can_noK_s<k>_ncroi_clip8_ln`: survivor recipe, cwm2 route-median kinematics), 3 seeds, test split, both regimes", "", | |
| "Runs `I1_{indep,all_minus_road2drv}_s<k>_final_ncroi` = the FINAL command line with only the U base swapped (V0 `can4s_stats2`, E0 `pooled_P_v2`, seeds, recipe, selection unchanged; " | |
| "driver inputs = the cwm2 route-median (non-causal) kinematics the survivor recipe used before K4). The frozen main `I1_all_minus_road2drv_s<k>_final` (causal ROI, " | |
| "`causal_routes/v2`) and `I1_indep_s<k>_final` are read as B layouts only. Paired device-cluster bootstrap on identical rows (eval/paired_compare_joint.py, n_boot 1000; " | |
| "verdict = kill_checks.beats: ≥ 2/3 seeds CI excl. 0, same sign, |mean Δ| > seed sd). Driver populations from the **v2 causal kin targets** (`finalc_*`; rows = the " | |
| "intersection of both layouts' valid rows = rows with a causal driver box), masked protocol `finalcm_*`; the cwm2 route-median targets (`final_*`) are DESCRIPTIVE only " | |
| "(the targets carry the same non-causal ROI as the ncroi inputs — the leak the K4 protocol removed). P cells by eval/pressure_eval.py (`finalP_*`). Identity of the " | |
| "assembled `indep_ncroi` with its bases: `results/tables/identity_final_ncroi.md`. Every number comes from `results/paired_compare.csv`.", "", | |
| "**Pre-registered expected reading (docs/decisions.md, 2026-09-13, before training):** the joint-level confirmation of K4 — on the causal targets the causal ROI is NOT worse than " | |
| "the non-causal ROI, i.e. A = all_minus_ncroi BEATS the frozen main on no driver own endpoint; any non-causal \"gain\" shows only on the route-median targets. Per-cell verdicts; " | |
| "the frozen main stays the headline whatever the outcome.", "", | |
| f"## READING: **{reading}**{void}", "", f"- {rtxt}", | |
| f"- driver cells, causal targets (A = ncroi − B = frozen main): " + "; ".join(f"{r}|{c.split('|')[1]} {v}" for (r, c), v in drvA.items()), | |
| f"- driver cells, masked protocol: " + ("; ".join(f"{r}|{c.split('|')[1]} {v}" for (r, c), v in drvAm.items()) or "no rows"), | |
| f"- driver cells, cwm2 ROUTE-MEDIAN targets (descriptive): " + ("; ".join(f"{r}|{c.split('|')[1]} {v}" for (r, c), v in drvL.items()) or "no rows"), | |
| f"- other own endpoints WORSE for ncroi vs frozen main: {fmt([k for k in worseA if k[1] not in DRV])}; own BEATS: {fmt(boA)}; cross-leg BEATS: {fmt(bcA)}", "", | |
| "## Provenance of the A / B layouts (guard 2): layout mtime, source hash, model_id / code hash, completeness; rows older than the layout are STALE", "", | |
| "| run | regime | layout mtime (UTC) | source sha256 | model_id / code | status |", "|---|---|---|---|---|---|", *prov, "", | |
| *([f"**STALE ROWS DETECTED for {stale} — every verdict is VOID; recompute (eval/run_joint_evals_final_ncroi.sh).**", ""] if stale else []), | |
| "## 1. (a) ncroi variant vs the FROZEN main — v2 causal kin targets (the pre-registered comparison)", ""] + drv_table(vA, "Driver own endpoints of (a)") + bodyA + \ | |
| ["", f"B2-style status of (a) for reference: **{stA}** — own WORSE {fmt(worseA)}; own BEATS {fmt(boA)}; cross BEATS {fmt(bcA)}", "", | |
| "## 2. DESCRIPTIVE — the same pairs on the cwm2 route-median (non-causal) targets: the leak picture", ""] + drv_table(vL1, "Driver cells of ncroi vs frozen main on the ROUTE-MEDIAN targets") + bodyL1 + [""] + bodyL2 + ["", | |
| "## 3. (b) Interaction effect on the ncroi bases", ""] + bodyB + ["", f"B2-style verdict (b): **{stB}** — own WORSE {fmt(worseB)}; cross BEATS {fmt(bcB)}; own BEATS {fmt(boB)}", "", | |
| "## 4. Attribution — the U-base swap alone (indep_ncroi vs indep, no training), causal targets", ""] + drv_table(vC, "Driver cells: indep_ncroi vs indep") + bodyC + ["", | |
| "## 5. Masked protocol (sensitivity to the 64 suspect routes)", ""] + drv_table(vAm, "Driver cells: ncroi vs frozen main, masked") + bodyAm + [""] + bodyBm + ["", | |
| "## 6. P cells (road-pressure chains, detached head)", ""] + bodyP1 + [""] + bodyP2 + ["", | |
| "## 7. B1 on the ncroi variant (driver cells vs k0 / persistence)", ""] + b1 | |
| out = Path(a.out); out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text("\n".join(L) + "\n") | |
| dec = {"reading": reading, "detail": rtxt, "driver_cells_causal_a": {f"{r}|{c}": v for (r, c), v in drvA.items()}, "driver_cells_masked_a": {f"{r}|{c}": v for (r, c), v in drvAm.items()}, | |
| "driver_cells_route_median_a": {f"{r}|{c}": v for (r, c), v in drvL.items()}, "ncroi_beats_causal": ncroi_beats, "ncroi_worse_causal": ncroi_worse, | |
| "own_worse_a": fmt(worseA), "own_beats_a": fmt(boA), "cross_beats_a": fmt(bcA), "B2_style_a": stA, "B2_style_b": stB, "B1": b1_json, "stale": stale, "seeds": seeds, "n_seeds_paired_a": n_seeds, | |
| "own_endpoints_a": {f"{r}|{c}": {"mean_delta": v["mean_delta"], "verdict": v["verdict"]} for (r, c, m), v in vA.items() if c in RJ.OWN and m == "pr_auc"}} | |
| (out.parent / "stage_final_ncroi_decision.json").write_text(json.dumps(dec, indent=1, default=str)) | |
| print("\n".join(L[:12])) | |
| print(f"\nREADING {reading}: {rtxt}\nwritten {out} and {out.parent / 'stage_final_ncroi_decision.json'}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 14.3 kB
- Xet hash:
- 9443d89462ab944d255bb3ff8f32084b3ae1e532024b7cfaf09b2c4af2d27b49
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.