Buckets:
| """cwm3/eval/report_identity_invariance.py — innovation 2 table: identity-invariant driver trunk arms (u_adv lambda 0.1 / 0.3, u_devnorm) vs the survivor baseline | |
| U0_kin_can_noK_s<k>_kcv2_clip8_ln -> results/tables/identity_invariance.md. Inputs (script outputs only): | |
| results/shift_twin.csv tags IDtw_s<k>_<arm>_vs_ln (A = arm, B = baseline): twin_pair_auroc@3s / random_pair_auroc@3s per event, identity_probe_acc (+ majority in note) | |
| results/paired_compare.csv tags IDk4_ / IDk4m_ (arm − baseline, all rows / masked) and IDk0_ / IDk0m_ (arm vs k0), driver cells drv_onset|<event>|3s | |
| results/runs/<run>/TRAIN_DONE best val chain NLL (the lambda-adv choice is made on it, never on test) | |
| Kill rule (pre-stated): adopt an arm only if the twin gap (random − twin pair-AUROC) is <= 0.10 AND identity top-1 within 0.05 of the majority rate AND no K4 cell WORSE | |
| (kill_checks.beats, 3 seeds). python eval/report_identity_invariance.py [--arms adv01 adv03 devnorm] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3.parent / "cwm2" / "eval")) | |
| import kill_checks as KC # noqa: E402 | |
| EV = ("hand_leave", "hand_return", "head_turn_onset") | |
| REG = ("manual", "engaged") | |
| BASE = "U0_kin_can_noK_s{k}_kcv2_clip8_ln" | |
| def ms(v): | |
| v = np.asarray([x for x in v if x is not None and np.isfinite(x)], float) | |
| return "—" if not len(v) else (f"{v.mean():.3f} ± {v.std(ddof=1):.3f}" if len(v) > 1 else f"{v[0]:.3f}") | |
| def beats(g): | |
| if len(g) < 3: | |
| return "INSUFFICIENT" | |
| pr = pd.DataFrame({"seed": g.seed, "delta": g.delta.astype(float), "delta_ci_lo": g.delta_ci_lo.astype(float), "delta_ci_hi": g.delta_ci_hi.astype(float), "status": "secondary"}) | |
| return KC.beats(pr, pd.DataFrame({"value": g.value_a.astype(float)}), pd.DataFrame({"value": g.value_b.astype(float)}), min_seeds=3)["verdict"] | |
| def dcell(g): | |
| return "—" if not len(g) else " / ".join(f"{float(r.delta):+.3f} [{float(r.delta_ci_lo):+.3f},{float(r.delta_ci_hi):+.3f}]" for r in g.sort_values("seed").itertuples()) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--arms", nargs="+", default=["adv01", "adv03", "devnorm"]) | |
| ap.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2]) | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "identity_invariance.md")) | |
| a = ap.parse_args() | |
| tw = pd.read_csv(CWM3 / "results" / "shift_twin.csv", low_memory=False) | |
| tw = tw[tw.tag.str.startswith("IDtw_")].copy() | |
| tw["seed"] = tw.tag.str.extract(r"_s(\d+)_")[0].astype(int) | |
| tw["arm"] = tw.tag.str.replace(r"^IDtw_s\d+_", "", regex=True).str.replace("_vs_ln", "", regex=False) | |
| tw = tw.sort_values("run_stamp").groupby(["tag", "regime", "event", "statistic"], as_index=False).tail(1) | |
| pc = pd.read_csv(CWM3 / "results" / "paired_compare.csv", low_memory=False) | |
| pc = pc[pc.tag.str.startswith(("IDk4_", "IDk4m_", "IDk0_", "IDk0m_", "K4v2c_"))].copy() | |
| pc["seed"] = pc.tag.str.extract(r"_s(\d+)_")[0].astype(int) | |
| pc = pc.sort_values("run_stamp").groupby(["tag", "regime", "cell", "metric"], as_index=False).tail(1) | |
| for c in ("value_a", "value_b", "delta", "delta_ci_lo", "delta_ci_hi"): | |
| pc[c] = pd.to_numeric(pc[c], errors="coerce") | |
| L = ["# Identity-invariant driver trunk (innovation 2) — u_adv (GRL device head, lambda 0.1 / 0.3) and u_devnorm (causal per-route normalisation) vs the survivor baseline " | |
| "`U0_kin_can_noK_s<k>_kcv2_clip8_ln` (causal v2 inputs, stats v2, kclip 8, u_in_ln; 3 seeds)", "", | |
| "Arms: `models/identity.py` (recipe keys `u_adv` / `u_devnorm`, stages.yaml variants `kin_can_noK_clip8_ln_{adv01,adv03,devnorm}`); same data, seeds, recipe otherwise. " | |
| "Twin statistics: `eval/shift_twin.py --kin-source causal_routes --kin-version v2` (same-route twins Δt 30–60 s, |ΔvEgo| < 2 m/s, label flips; random negatives = same positives vs " | |
| "cross-route negatives; identity probe = multinomial LR from the frozen 128-d GRU state to the dongle id, GroupKFold(5) by route on 120k train rows). K4 cells: " | |
| "`eval/paired_compare_joint.py` on the causal v2 targets, identical rows, device-cluster bootstrap n_boot 1000; masked = the 64 v2 suspect routes removed from every population " | |
| "(evaluation-side; every model trained on all rows). Verdicts: kill_checks.beats (>= 2/3 seeds CI excl. 0, same sign, |mean Δ| > seed sd). " | |
| "**Kill rule (pre-stated)**: adopt an arm only if twin gap (random − twin) <= 0.10 AND identity top-1 within 0.05 of the majority rate AND no K4 cell WORSE vs the baseline.", ""] | |
| # ---- lambda-adv selection on val chain NLL | |
| L += ["## 0. Validation chain NLL (selection objective; the lambda-adv choice is made here, never on test)", "", "| arm | " + " | ".join(f"s{s}" for s in a.seeds) + " | mean |", "|---|" + "---|" * len(a.seeds) + "---|"] | |
| best_of = {} | |
| for arm in ["ln"] + a.arms: | |
| vals = [] | |
| for s in a.seeds: | |
| rd = CWM3 / "results" / "runs" / (BASE.format(k=s) + ("" if arm == "ln" else f"_{arm}")) | |
| p = rd / "TRAIN_DONE" | |
| vals.append(json.loads(p.read_text())["best"] if p.exists() else np.nan) | |
| best_of[arm] = vals | |
| L.append(f"| {'baseline (ln)' if arm == 'ln' else arm} | " + " | ".join(f"{v:.4f}" if np.isfinite(v) else "—" for v in vals) + f" | {np.nanmean(vals):.4f} |") | |
| adv_arms = [x for x in a.arms if x.startswith("adv")] | |
| if len(adv_arms) >= 2 and all(np.isfinite(best_of[x]).all() for x in adv_arms): | |
| pick = min(adv_arms, key=lambda x: np.mean(best_of[x])) | |
| L.append(f"\nlambda-adv selected on val chain NLL: **{pick}** (mean {np.mean(best_of[pick]):.4f} vs {', '.join(f'{x} {np.mean(best_of[x]):.4f}' for x in adv_arms if x != pick)}).") | |
| # ---- twin gap | |
| L += ["", "## 1. Same-route twin pair-AUROC@3s vs random-negative pair-AUROC — gap = random − twin (identity shortcut size); arm (A) vs baseline (B), mean ± sd over seeds", "", | |
| "| arm | regime | event | twin A | twin B | random A | random B | **gap A** | **gap B** | Δ twin A − B (s0 / s1 / s2) | verdict |", "|---|---|---|---|---|---|---|---|---|---|---|"] | |
| gaps = {} | |
| for arm in a.arms: | |
| for reg in REG: | |
| for ev in EV: | |
| g = tw[(tw.arm == arm) & (tw.regime == reg) & (tw.event == ev)] | |
| gt, gr = g[g.statistic == "twin_pair_auroc@3s"].sort_values("seed"), g[g.statistic == "random_pair_auroc@3s"].sort_values("seed") | |
| if not len(gt): | |
| continue | |
| gapA, gapB = gr.value_a.to_numpy(float) - gt.value_a.to_numpy(float), gr.value_b.to_numpy(float) - gt.value_b.to_numpy(float) | |
| gaps[(arm, reg, ev)] = (gapA, gapB) | |
| L.append(f"| {arm} | {reg} | {ev} | {ms(gt.value_a)} | {ms(gt.value_b)} | {ms(gr.value_a)} | {ms(gr.value_b)} | **{ms(gapA)}** | **{ms(gapB)}** | {dcell(gt)} | {beats(gt)} |") | |
| # ---- identity probe | |
| L += ["", "## 2. Identity probe — device top-1 from the frozen GRU state (train rows, GroupKFold by route) vs the majority rate", "", | |
| "| arm | regime | top-1 A (arm) | top-1 B (baseline) | majority | A − majority | verdict (<= 0.05) |", "|---|---|---|---|---|---|---|"] | |
| idp = {} | |
| for arm in a.arms: | |
| for reg in REG: | |
| g = tw[(tw.arm == arm) & (tw.regime == reg) & (tw.statistic == "identity_probe_acc")].sort_values("seed") | |
| if not len(g): | |
| continue | |
| maj = np.array([float(re.search(r"majority-class rate ([0-9.]+)", n).group(1)) for n in g.note]) | |
| va, vb = g.value_a.to_numpy(float), g.value_b.to_numpy(float) | |
| idp[(arm, reg)] = (va, vb, maj) | |
| L.append(f"| {arm} | {reg} | {ms(va)} | {ms(vb)} | {ms(maj)} | {ms(va - maj)} | {'PASS' if np.mean(va - maj) <= 0.05 else 'FAIL'} |") | |
| # ---- K4 cells | |
| L += ["", "## 3. K4 driver cells on the causal v2 targets — arm − baseline on identical rows (PR-AUC@3s), all rows and suspect routes masked; arm vs k0", "", | |
| "| arm | protocol | regime | event | arm | baseline | Δ (s0 / s1 / s2) | verdict | k0 | arm vs k0 |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| worse = {arm: [] for arm in a.arms} | |
| for arm in a.arms: | |
| for proto, pfx, kpfx in (("all rows", "IDk4_", "IDk0_"), ("masked", "IDk4m_", "IDk0m_")): | |
| for reg in REG: | |
| for ev in EV: | |
| cell = f"drv_onset|{ev}|3s" | |
| g = pc[pc.tag.str.startswith(pfx) & pc.tag.str.contains(f"_{arm}_vs_ln$") & (pc.regime == reg) & (pc.cell == cell) & (pc.metric == "pr_auc")] | |
| gk = pc[pc.tag.str.startswith(kpfx) & pc.tag.str.contains(f"_{arm}_vs_k0$") & (pc.regime == reg) & (pc.cell == cell) & (pc.metric == "pr_auc")] | |
| if not len(g): | |
| continue | |
| v = beats(g) | |
| if v == "WORSE": | |
| worse[arm].append(f"{proto}|{reg}|{ev}") | |
| L.append(f"| {arm} | {proto} | {reg} | {ev} | {ms(g.value_a)} | {ms(g.value_b)} | {dcell(g)} | {v} | {ms(gk.value_b)} | {beats(gk) if len(gk) else '—'} |") | |
| # ---- verdicts | |
| L += ["", "## 4. Verdicts (kill rule)", ""] | |
| for arm in a.arms: | |
| ga = [np.mean(v[0]) for k, v in gaps.items() if k[0] == arm] | |
| gb = [np.mean(v[1]) for k, v in gaps.items() if k[0] == arm] | |
| ip = [np.mean(v[0] - v[2]) for k, v in idp.items() if k[0] == arm] | |
| ipb = [np.mean(v[1] - v[2]) for k, v in idp.items() if k[0] == arm] | |
| c1 = bool(ga) and max(ga) <= 0.10 | |
| c2 = bool(ip) and max(ip) <= 0.05 | |
| c3 = not worse[arm] and any(pc.tag.str.contains(f"_{arm}_vs_ln$")) | |
| adopt = c1 and c2 and c3 | |
| L.append(f"- **{arm}**: twin gap arm {', '.join(f'{x:.3f}' for x in ga)} (baseline {', '.join(f'{x:.3f}' for x in gb)}; max {max(ga) if ga else float('nan'):.3f} {'<=' if c1 else '>'} 0.10) · " | |
| f"identity top-1 − majority arm {', '.join(f'{x:+.3f}' for x in ip)} (baseline {', '.join(f'{x:+.3f}' for x in ipb)}; {'within' if c2 else 'beyond'} 0.05) · " | |
| f"K4 cells WORSE: {worse[arm] if worse[arm] else 'none'} -> **{'ADOPT' if adopt else 'KILL (not adopted)'}**") | |
| L.append("") | |
| Path(a.out).write_text("\n".join(L) + "\n") | |
| print("\n".join(L[-len(a.arms) - 2:])) | |
| print(f"written {a.out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 10.7 kB
- Xet hash:
- 48f73e9df6228207b22ca26615adb704456a65ee3548937f2a212ca3c5bd1d78
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.