Buckets:
| """cwm3/probes/consolidate_exp1.py — P3 consolidation: the model-level cabin-latent controls already run (Exp1: U0 kin_can vs +vc / +crop / noK) | |
| read from results/paired_compare.csv (eval/paired_compare.py rows on identical test rows, device-cluster bootstrap) -> results/tables/latent_exp1_consolidated.md. | |
| Also reads results/exp1_kin_repro.csv for the per-run onset PR-AUC@3s against the cwm2 gru_kin_can reference. Read-only; numbers copied verbatim. | |
| """ | |
| from __future__ import annotations | |
| import csv | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| import common as C # noqa: E402 | |
| def main(): | |
| with open(C.CWM3 / "results" / "paired_compare.csv") as f: | |
| rows = [r for r in csv.DictReader(f) if r["tag"].startswith("exp1")] | |
| dedup = {} | |
| for r in rows: # re-runs of a tag: keep the latest stamp per (tag, regime, cell) | |
| k = (r["tag"], r["regime"], r["cell"]) | |
| if k not in dedup or r["run_stamp"] > dedup[k]["run_stamp"]: | |
| dedup[k] = r | |
| rows = list(dedup.values()) | |
| L = ["# Exp1 consolidation (P3, model level) — does a cabin latent add value on top of kinematics + CAN inside the trained U branch?", "", | |
| f"Generated {C.stamp()} by `probes/consolidate_exp1.py` from `results/paired_compare.csv` (tags exp1_*; identical test rows, shared-resample device-cluster " | |
| "bootstrap, n_boot 1000). Δ = A − B in onset PR-AUC@3s; A = U0 variant with the cabin stream (vc256 = full-frame causal PCA-256; crop256 = route-median ROI " | |
| "PCA-256, NON-causal) or without the K residual head (noK); B = U0_kin_can of the same seed. Bold = CI excludes 0.", "", | |
| C.md_table(["tag", "regime", "A", "B", "cell", "A", "B", "Δ [CI]", "P(Δ>0)", "n / pos"], | |
| [[r["tag"], r["regime"], r["a"], r["b"], r["cell"], f"{float(r['value_a']):.3f}", f"{float(r['value_b']):.3f}", | |
| (lambda d, lo, hi: (f"**{d:+.3f} [{lo:+.3f}, {hi:+.3f}]**" if (lo > 0 or hi < 0) else f"{d:+.3f} [{lo:+.3f}, {hi:+.3f}]"))(float(r["delta"]), float(r["delta_ci_lo"]), float(r["delta_ci_hi"])), | |
| f"{float(r['frac_boot_d_gt0']):.2f}", f"{r['n']} / {r['n_pos']}"] for r in sorted(rows, key=lambda r: (r["tag"], r["regime"], r["cell"]))]), ""] | |
| # summary counts | |
| def cnt(pref): | |
| sub = [r for r in rows if r["tag"].startswith(pref)] | |
| neg = sum(1 for r in sub if float(r["delta_ci_hi"]) < 0) | |
| pos = sum(1 for r in sub if float(r["delta_ci_lo"]) > 0) | |
| return len(sub), neg, pos, sum(float(r["delta"]) for r in sub) / max(1, len(sub)) | |
| L += ["## Summary", ""] | |
| for pref, label in (("exp1_s0_vc", "+vc256 seed 0"), ("exp1_s1_vc", "+vc256 seed 1"), ("exp1_s0_crop", "+crop256 seed 0"), ("exp1_s0_noK", "noK (chains only) seed 0")): | |
| n, neg, pos, mean = cnt(pref) | |
| L.append(f"* {label}: {n} cells, {neg} with CI < 0 (hurt), {pos} with CI > 0 (helped), mean Δ {mean:+.4f}") | |
| with open(C.CWM3 / "results" / "exp1_kin_repro.csv") as f: | |
| kr = list(csv.DictReader(f)) | |
| cols = kr[0].keys() | |
| L += ["", "## Per-run onset PR-AUC@3s vs the cwm2 gru_kin_can reference (results/exp1_kin_repro.csv, tau = 3s, latest row per run)", ""] | |
| latest = {} | |
| for r in kr: | |
| if r.get("tau") in ("3s", "15", "3"): | |
| latest[(r["run_dir"] if "run_dir" in r else r.get("run", ""), r["regime"], r["event"])] = r | |
| keys = [k for k in ("run_dir", "run", "regime", "event", "value", "ci_lo", "ci_hi", "ref_mean", "ref_sd", "delta") if k in cols] | |
| L.append(C.md_table(keys, [[(f"{float(r[k]):.3f}" if k in ("value", "ci_lo", "ci_hi", "ref_mean", "ref_sd", "delta") and r[k] not in ("", None) else r[k]) for k in keys] | |
| for r in sorted(latest.values(), key=lambda r: (r.get("run_dir", r.get("run", "")), r["regime"], r["event"]))])) | |
| (C.TABLES / "latent_exp1_consolidated.md").write_text("\n".join(L) + "\n") | |
| print(f"-> {C.TABLES / 'latent_exp1_consolidated.md'} ({len(rows)} paired rows)") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 4.11 kB
- Xet hash:
- 67f1560ca7f41b4031eb6fa07db1fc94f259e88792c54ee5cd85b0dfc7ab30a9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.