Buckets:
| """cwm3/eval/report_final.py — results/tables/stage_final.md: the FINAL joint model (roadmap §四 E1 / E2, claims B2 and B1) from results/paired_compare.csv. | |
| python eval/report_final.py [--seeds 0,1,2] [--out results/tables/stage_final.md] | |
| Composes eval/report_joint.py blocks (load / render / claim / provenance; nothing is recomputed here) over the tags written by eval/run_joint_evals_final.sh: | |
| finalc_s<k>_{allminus,all}_vs_indep, finalc_s<k>_allminus_vs_all (v2 causal kin targets = the targets the U base was trained on: own endpoints incl. driver cells, | |
| cross-leg C1 / C2, hazard NLL, emissions) | |
| finalcm_s<k>_{allminus,all}_vs_indep (the same, 64 v2 driver-box-suspect routes censored) | |
| final_s<k>_{allminus,all}_vs_indep (cwm2 route-median kin targets; continuity with stage_I1_v2b; driver cells descriptive) | |
| finalc_s<k>_{indep,allminus,all}_vs_k0, finalcm_s<k>_..._vs_k0 (driver cells vs the k0 heuristic; persistence = base-rate rows; claim B1 on the joint model) | |
| Pre-stated rule (roadmap §三): main model = all_minus_road2drv iff no own endpoint is WORSE vs indep on the final bases (finalc), else all. | |
| Also writes results/tables/stage_final_decision.json {main_variant, main_dir_pattern, rule, worse_cells} for the downstream steps (ablations, warning head). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import report_joint as RJ # noqa: E402 | |
| VARIANT_DIR = {"indep": "I1_indep_s<k>_final", "allminus": "I1_all_minus_road2drv_s<k>_final", "all": "I1_all_s<k>_final"} | |
| EV = ("hand_leave", "hand_return", "head_turn_onset") | |
| def block(stage, a, b, claim, title, seeds): | |
| df = RJ.load(stage, a, b, seeds) | |
| if not len(df): | |
| return df, [f"### {title}: A = {a}, B = {b} — no rows yet (tags `{stage}_s<k>_{a}_vs_{b}`)", ""], {}, ("NO ROWS", [], [], []) | |
| body, verdicts = RJ.render(df, sorted(df.seed.unique()), a, b, title) | |
| return df, body, verdicts, RJ.claim(verdicts, claim) | |
| def fmt_list(ks): | |
| return [f"{r}|{c}" + (f"|{m}" if len(k) > 2 and m not in ("pr_auc",) else "") for k in ks for (r, c, m) in [k if len(k) == 3 else (*k, "")]] or "none" | |
| def ms(v): | |
| import numpy as np | |
| v = [float(x) for x in v if str(x) not in ("", "nan")] | |
| return "—" if not v else (f"{np.mean(v):.3f} ± {np.std(v, ddof=1):.3f}" if len(v) > 1 else f"{v[0]:.3f}") | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0,1,2") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "stage_final.md")) | |
| a = ap.parse_args() | |
| seeds = [int(s) for s in a.seeds.split(",")] | |
| dfs = [] | |
| # ---- B2 blocks (causal v2 targets) | |
| d1, body1, v1, (st1, worse1, bc1, bo1) = block("finalc", "allminus", "indep", "B2", "B2 main comparison (v2 causal kin targets)", seeds) | |
| d2, body2, v2, (st2, worse2, bc2, bo2) = block("finalc", "all", "indep", "B2", "B2 secondary comparison (v2 causal kin targets)", seeds) | |
| d3, body3, v3, (st3, worse3, bc3, bo3) = block("finalc", "allminus", "all", "B3", "all_minus_road2drv vs all (v2 causal kin targets; B3-style: no cell WORSE, ≥ 1 BEATS)", seeds) | |
| dfs += [d1, d2, d3] | |
| # ---- pre-stated rule | |
| if not len(d1): | |
| main_v, rule_txt = None, "NOT DECIDABLE YET (no finalc allminus vs indep rows)" | |
| elif not worse1: | |
| main_v, rule_txt = "allminus", "no own endpoint WORSE vs indep on the final bases → **main model = `all_minus_road2drv`**" | |
| else: | |
| main_v, rule_txt = "all", f"own endpoint(s) WORSE vs indep for all_minus_road2drv: {[f'{r}|{c}' for r, c, _ in worse1]} → **main model = `all`** (pre-stated fallback)" + \ | |
| ("" if not worse2 else f"; note `all` is itself WORSE on {[f'{r}|{c}' for r, c, _ in worse2]}") | |
| # ---- masked own endpoints (sensitivity) | |
| d4, body4, v4, (st4, worse4, bc4, bo4) = block("finalcm", "allminus", "indep", "B2", "Masked protocol (64 v2 suspect routes censored): all_minus_road2drv vs indep", seeds) | |
| d5, body5, v5, (st5, worse5, bc5, bo5) = block("finalcm", "all", "indep", "B2", "Masked protocol: all vs indep", seeds) | |
| dfs += [d4, d5] | |
| # ---- cwm2-target continuity | |
| d6, body6, v6, (st6, worse6, bc6, bo6) = block("final", "allminus", "indep", "B2", "Continuity with stage_I1_v2b (cwm2 route-median kin targets; driver cells DESCRIPTIVE: causal inputs scored on the old targets)", seeds) | |
| d7, body7, v7, (st7, worse7, bc7, bo7) = block("final", "all", "indep", "B2", "Continuity (cwm2 kin targets): all vs indep", seeds) | |
| dfs += [d6, d7] | |
| # ---- B1 blocks: driver cells vs k0 / persistence | |
| b1_lines, b1_json = [], {} | |
| for proto, stage in (("all rows", "finalc"), ("suspect routes masked", "finalcm")): | |
| b1_lines += [f"### B1 — driver onset PR-AUC@3s on the v2 causal targets vs the k0 heuristic / persistence, protocol: {proto}", "", | |
| "| variant | regime | event | model | k0 | persistence (base rate) | Δ s0 / s1 / s2 [CI] | verdict |", "|---|---|---|---|---|---|---|---|"] | |
| for var in ("indep", "allminus", "all"): | |
| df = RJ.load(stage, var, "k0", seeds) | |
| dfs.append(df) | |
| if not len(df): | |
| b1_lines.append(f"| {var} | — | — | no rows yet (`{stage}_s<k>_{var}_vs_k0`) | | | | |") | |
| continue | |
| verd = {} | |
| for reg in ("manual", "engaged"): | |
| for ev in EV: | |
| cell = f"drv_onset|{ev}|3s" | |
| g = df[(df.regime == reg) & (df.cell == cell) & (df.metric == "pr_auc")].sort_values("seed") | |
| base = df[(df.regime == reg) & (df.cell == cell) & (df.metric == "base_rate")] | |
| if not len(g): | |
| continue | |
| res = RJ.verdict(g) | |
| verd[(reg, cell, "pr_auc")] = res | |
| cells = " / ".join(f"{float(r.delta):+.3f} [{float(r.delta_ci_lo):+.3f},{float(r.delta_ci_hi):+.3f}]" for r in g.itertuples()) | |
| b1_lines.append(f"| {var} | {reg} | {ev} | {ms(g.value_a)} | {ms(g.value_b)} | {ms(base.value_a) if len(base) else '—'} | {cells} | {res['verdict']} |") | |
| stb, wb, bb, _ = RJ.claim(verd, "B1") | |
| n_beats = sum(v["verdict"] == "BEATS" for v in verd.values()) | |
| b1_json[f"{var}|{proto}"] = {"verdict": stb, "beats": n_beats, "cells": {f"{r}|{c}": v["verdict"] for (r, c, _), v in verd.items()}} | |
| b1_lines.append(f"| {var} | **B1 ({proto})** | | | | | {n_beats}/6 cells BEATS k0; rule (hand_leave AND head_turn_onset BEATS in both regimes) | **{stb}** |") | |
| b1_lines.append("") | |
| allrows = pd.concat([d for d in dfs if len(d)], ignore_index=True) if any(len(d) for d in dfs) else pd.DataFrame(columns=["a", "b", "regime", "run_stamp"]) | |
| prov, stale = RJ.provenance(allrows) if len(allrows) else ([], []) | |
| void = f" — **VOID: stale rows for {stale}**" if stale else "" | |
| lines = ["# Stage FINAL — the frozen CWM3 model on the final bases (V0 can4s stats2 · U0 kin_can_noK clip8+LN causal v2 · E0 pooled_P), 3 seeds, test split, both regimes", "", | |
| "Paired device-cluster bootstrap on identical rows (eval/paired_compare_joint.py, n_boot 1000; verdict = kill_checks.beats: ≥ 2/3 seeds CI excl. 0, same sign, |mean Δ| > seed sd). " | |
| "Driver populations from the **v2 causal kin targets** (`results/targets_kin_causal_routes_v2`, the targets the U base was trained on; tags `finalc_*`), the masked protocol censors the 64 v2 " | |
| "driver-box-suspect routes (`finalcm_*`); the cwm2 route-median targets are kept for continuity with `stage_I1_v2b.md` (`final_*`, driver cells descriptive). " | |
| "Bases: `V0_can4s_s<k>_stats2`, `U0_kin_can_noK_s<k>_kcv2_clip8_ln`, `E0_pooled_P_s<k>_v2`; runs `I1_{indep,all_minus_road2drv,all}_s<k>_final` " | |
| "(indep = the assembled bases, no training; identity verified by eval/verify_identity_final.py). Every number comes from `results/paired_compare.csv`.", "", | |
| "## Provenance of the A / B layouts (guard 2): layout mtime, source hash (sha256 of the prediction pack), model_id / code hash, completeness; rows older than the layout are STALE", "", | |
| "| run | regime | layout mtime (UTC) | source sha256 | model_id / code | status |", "|---|---|---|---|---|---|", *prov, "", | |
| *([f"**STALE ROWS DETECTED for {stale} — every verdict below is VOID; recompute the pairs (eval/run_joint_evals_final.sh).**", ""] if stale else []), | |
| "## 1. Claim B2 — interaction I vs the independent branches (own endpoints incl. the driver cells on the causal targets; cross-leg C1 / C2)", ""] | |
| lines += body1 + ["", f"### B2 verdict (all_minus_road2drv vs indep): **{st1}**{void}", "", | |
| f"- own endpoints WORSE: {[f'{r}|{c}' for r, c, _ in worse1] or 'none'}", f"- cross-leg endpoints BEATS: {[f'{r}|{c}|{m}' for r, c, m in bc1] or 'none'}", | |
| f"- own endpoints BEATS: {[f'{r}|{c}' for r, c, _ in bo1] or 'none'}", | |
| "- INCONCLUSIVE own endpoints (mean Δ): " + (", ".join(f"{r}|{c} {v['mean_delta']:+.4f}" for (r, c, m), v in v1.items() if c in RJ.OWN and m == "pr_auc" and v["verdict"] == "INCONCLUSIVE") or "none"), ""] | |
| lines += body2 + ["", f"### B2-style verdict (all vs indep): **{st2}**{void} — WORSE own {[f'{r}|{c}' for r, c, _ in worse2] or 'none'}; BEATS cross {[f'{r}|{c}|{m}' for r, c, m in bc2] or 'none'}; BEATS own {[f'{r}|{c}' for r, c, _ in bo2] or 'none'}", ""] | |
| lines += body3 + ["", f"### all_minus_road2drv − all: **{st3}** — WORSE {[f'{r}|{c}' for r, c, _ in worse3] or 'none'}; BEATS cross {[f'{r}|{c}|{m}' for r, c, m in bc3] or 'none'}; BEATS own {[f'{r}|{c}' for r, c, _ in bo3] or 'none'}", ""] | |
| lines += ["## 2. Pre-stated rule → main model", "", f"Rule (roadmap §三, registered before these runs): main = `all_minus_road2drv` iff no own endpoint (veh / drv / D onset PR-AUC@3s, both regimes) is WORSE vs `indep` on the final bases, else `all`.", "", | |
| f"**Decision: {rule_txt}**" + void, "", | |
| f"Sensitivity — masked protocol own-endpoint WORSE: all_minus_road2drv {[f'{r}|{c}' for r, c, _ in worse4] or 'none'} ({st4}); all {[f'{r}|{c}' for r, c, _ in worse5] or 'none'} ({st5}).", ""] | |
| lines += ["## 3. Claim B1 on the joint model — driver cells vs the label-free references (k0 heuristic from the causal K0; persistence = base rate)", ""] + b1_lines | |
| lines += ["## 4. Masked protocol (sensitivity to the 64 suspect routes)", ""] + body4 + [""] + body5 + [""] | |
| lines += ["## 5. Continuity with the previous pack / base versions (cwm2 route-median kin targets)", ""] + body6 + ["", f"B2-style verdict: **{st6}** (driver cells descriptive here)", ""] + body7 + ["", f"B2-style verdict: **{st7}**", ""] | |
| out = Path(a.out); out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text("\n".join(lines) + "\n") | |
| dec = {"main_variant": {"allminus": "all_minus_road2drv", "all": "all"}.get(main_v), "main_dir_pattern": VARIANT_DIR.get(main_v, None), "rule": rule_txt, "B2_allminus_vs_indep": st1, "B2_all_vs_indep": st2, | |
| "worse_allminus": [f"{r}|{c}" for r, c, _ in worse1], "worse_all": [f"{r}|{c}" for r, c, _ in worse2], "B1": b1_json, "stale": stale, "seeds": seeds} | |
| (out.parent / "stage_final_decision.json").write_text(json.dumps(dec, indent=1)) | |
| print("\n".join(lines)) | |
| print(f"\nmain model: {dec['main_variant']} ({rule_txt})") | |
| print(f"written {out} and {out.parent / 'stage_final_decision.json'}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 11.9 kB
- Xet hash:
- 248c45cd3650571ff7665f7fbf40ce95c55b14c6d334e3f8bab12800b071926c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.