Buckets:
| """cwm3/eval/kin_repro.py — Exp1 comparator: does the cwm3 U0 branch (both regimes in ONE model) reproduce cwm2's gru_kin_can? | |
| Populations and metric are EXACTLY those of cwm2/eval/kin_predictability.py::metric_rows -> s12_wm/eval_s13.transition_metrics: | |
| per event (hand_return = hands_on 0->1, hand_leave = hands_on 1->0, head_turn_onset = head_turned 0->1), rows inactive at the anchor | |
| with EM (AND s12 M) valid on the first tau steps; y = first onset < tau; score = 1 - prod_{k<tau}(1 - h_k). CI = driver-cluster | |
| bootstrap (s3_can_baseline/eval.py bootstrap_ci, n_boot 300, seed 0) exactly as the reference CSV rows were produced. | |
| Reference: cwm2/results/targets_kin/predictability.csv (latest run_stamp per regime; variant gru_kin_can, seeds 0-2, tau 3s and 1s/5s). | |
| Reads the cwm3 run's pred_{regime}_test.npz (hz_on / hz_off of the U chains) — no model code, CPU only. | |
| Output (append-only): results/exp1_kin_repro.csv one row per (run, regime, event, tau) with value, CI, reference mean/sd over seeds, | |
| delta vs the reference seed mean and the +/-0.02 verdict; run.json. | |
| python kin_repro.py --run-dirs results/runs/U0_kin_can_s0 [...] [--n-boot 300] [--taus 1s,3s,5s] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import importlib.util | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| EXPLORE = CWM3.parent | |
| CWM2 = EXPLORE / "cwm2" | |
| KIN = CWM2 / "results" / "targets_kin" | |
| S12R = EXPLORE / "s12_wm" / "results" | |
| for p in (EXPLORE / "s12_wm", CWM3 / "common"): | |
| sys.path.insert(0, str(p)) | |
| import eval_s13 as e13 # noqa: E402 | |
| import episodes as ep # noqa: E402 | |
| from runmeta import write_run_json, sha256_file # noqa: E402 | |
| _spec = importlib.util.spec_from_file_location("s3eval", EXPLORE / "s3_can_baseline" / "eval.py") | |
| s3eval = importlib.util.module_from_spec(_spec) | |
| _spec.loader.exec_module(s3eval) | |
| EVENTS = ["hand_return", "hand_leave", "head_turn_onset"] | |
| TAUS = e13.TAUS | |
| OUT = CWM3 / "results" / "exp1_kin_repro.csv" | |
| COLS = ["run_stamp", "run_dir", "stage", "variant", "seed", "regime", "event", "tau", "metric", "value", "ci_lo", "ci_hi", "base_rate", "n", "n_pos", | |
| "n_dongles", "n_boot", "ref_variant", "ref_mean", "ref_sd", "ref_seeds", "delta_vs_ref_mean", "within_0.02", "ref_run_stamp", "notes"] | |
| TOL = 0.02 | |
| def load_kin_chains(regime, split="test"): | |
| """kin_predictability.load_split -> chains(): (z0, z, EM & M) per event.""" | |
| d = np.load(KIN / f"kin_{regime}_{split}.npz", allow_pickle=True) | |
| s = np.load(S12R / f"targets_{regime}_{split}.npz", allow_pickle=True) | |
| assert np.array_equal(s["uid"], d["uid"]) and np.allclose(s["t"], d["t"]) | |
| Mc = s["M"].astype(bool) | |
| S, S0, EM = d["S"], d["S0"], d["EM"] & Mc[:, :, None] | |
| chains = [(S0[:, 0], S[:, :, 0], EM[:, :, 0]), (1 - S0[:, 0], 1 - S[:, :, 0], EM[:, :, 1]), (S0[:, 1], S[:, :, 1], EM[:, :, 2])] | |
| return chains, d["dongle"] | |
| def reference_rows(regime): | |
| """latest run_stamp rows of predictability.csv for this regime: {(variant, seed, event, tau): row}.""" | |
| rows = list(csv.DictReader(open(KIN / "predictability.csv"))) | |
| rows = [r for r in rows if r["regime"] == regime and r["metric"] == "onset_prauc"] | |
| stamp = max(r["run_stamp"] for r in rows) | |
| return {(r["variant"], int(r["seed"]), r["event"], r["tau"]): r for r in rows if r["run_stamp"] == stamp}, stamp | |
| def hazards_for_event(z, ev): | |
| """pred_{regime}_test.npz of a U run -> h_on[N,25] for the event (hazard of the onset of the event's chain state).""" | |
| chains = [str(c) for c in z["chain_names"]] | |
| hz_on, hz_off = z["hz_on"].astype(np.float64), z["hz_off"].astype(np.float64) | |
| if ev == "hand_return": | |
| return hz_on[..., chains.index("hands_on")] | |
| if ev == "hand_leave": | |
| return hz_off[..., chains.index("hands_on")] | |
| if ev == "head_turn_onset": | |
| return hz_on[..., chains.index("head_turned")] | |
| raise ValueError(ev) | |
| def evaluate_run(run_dir, regimes, taus, n_boot, stamp, log): | |
| rd = Path(run_dir) | |
| cfg = json.loads((rd / "config.json").read_text())["cfg"] | |
| seed = int(json.loads((rd / "config.json").read_text())["args"]["seed"]) | |
| rows = [] | |
| for regime in regimes: | |
| pf = rd / f"pred_{regime}_test.npz" | |
| if not pf.exists(): | |
| log(f" {rd.name}: {pf.name} missing — skipped") | |
| continue | |
| z = np.load(pf, allow_pickle=True) | |
| assert str(z["branch"]) == "U" | |
| chains, dongle = load_kin_chains(regime) | |
| ref, ref_stamp = reference_rows(regime) | |
| valid = z["valid"].astype(bool) | |
| for j, ev in enumerate(EVENTS): | |
| z0, zz, em = chains[j] | |
| h = hazards_for_event(z, ev) | |
| zp = np.concatenate([z0[:, None], zz[:, :-1]], 1) | |
| M = em | |
| onset = (np.nan_to_num(zz, nan=0.0) > 0.5) & (np.nan_to_num(zp, nan=1.0) < 0.5) & M | |
| first = np.where(onset.any(1), onset.argmax(1), -1) | |
| inact0 = np.nan_to_num(zp[:, 0], nan=1.0) < 0.5 | |
| for tau in taus: | |
| k = TAUS[tau] | |
| sel = inact0 & M[:, :k].all(1) & valid | |
| y = ((first[sel] >= 0) & (first[sel] < k)).astype(np.float32) | |
| p = ep.onset_prob_within(h[sel], np.zeros(sel.sum()), k).astype(np.float64) | |
| from sklearn.metrics import average_precision_score | |
| ap = float(average_precision_score(y, p)) | |
| lo, hi = s3eval.bootstrap_ci(y, p, dongle[sel], metric="pr_auc", n_boot=n_boot, seed=0) | |
| refs = [float(ref[("gru_kin_can", s, ev, tau)]["value"]) for s in (0, 1, 2) if ("gru_kin_can", s, ev, tau) in ref] | |
| rm, rs = (float(np.mean(refs)), float(np.std(refs))) if refs else (np.nan, np.nan) | |
| rows.append({"run_stamp": stamp, "run_dir": str(rd), "stage": cfg["stage"], "variant": cfg["variant"], "seed": seed, "regime": regime, | |
| "event": ev, "tau": tau, "metric": "onset_prauc", "value": ap, "ci_lo": lo, "ci_hi": hi, "base_rate": float(y.mean()), | |
| "n": int(sel.sum()), "n_pos": int(y.sum()), "n_dongles": int(len(np.unique(dongle[sel]))), "n_boot": n_boot, | |
| "ref_variant": "gru_kin_can", "ref_mean": rm, "ref_sd": rs, "ref_seeds": len(refs), "delta_vs_ref_mean": ap - rm, | |
| "within_0.02": bool(abs(ap - rm) <= TOL) if refs else "", "ref_run_stamp": ref_stamp, | |
| "notes": "populations = kin_predictability.metric_rows (inactive at anchor, EM&M valid on first tau steps); " | |
| "Kh from the non-causal route-median ROI -> diagnostic"}) | |
| log(f" {rd.name} {regime:8s} {ev:16s} @{tau}: AP {ap:.4f} [{lo:.3f},{hi:.3f}] ref {rm:.4f}±{rs:.4f} Δ {ap - rm:+.4f} " | |
| f"{'OK' if abs(ap - rm) <= TOL else 'OUT'} n={int(sel.sum())} pos={int(y.sum())}") | |
| return rows | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--run-dirs", nargs="+", required=True) | |
| ap.add_argument("--regimes", default="manual,engaged") | |
| ap.add_argument("--taus", default="1s,3s,5s") | |
| ap.add_argument("--n-boot", type=int, default=300) | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| logp = CWM3 / "results" / "exp1_kin_repro.log" | |
| def log(m): | |
| line = f"[{time.strftime('%H:%M:%S')}] {m}" | |
| print(line, flush=True) | |
| with open(logp, "a") as f: | |
| f.write(line + "\n") | |
| rows = [] | |
| for rd in a.run_dirs: | |
| rows += evaluate_run(rd, a.regimes.split(","), a.taus.split(","), a.n_boot, stamp, log) | |
| new = not OUT.exists() | |
| with open(OUT, "a", newline="") as f: | |
| w = csv.DictWriter(f, fieldnames=COLS) | |
| if new: | |
| w.writeheader() | |
| for r in rows: | |
| w.writerow(r) | |
| log(f"appended {len(rows)} rows to {OUT}") | |
| write_run_json(CWM3 / "results", "exp1_kin_repro", [Path(__file__), EXPLORE / "s12_wm" / "eval_s13.py", EXPLORE / "s3_can_baseline" / "eval.py"], | |
| [KIN / "predictability.csv"] + [Path(rd) / f"pred_{r}_test.npz" for rd in a.run_dirs for r in a.regimes.split(",") | |
| if (Path(rd) / f"pred_{r}_test.npz").exists()], | |
| seed=0, t_start=t0, input_listing=False, extra={"run_stamp": stamp, "n_rows": len(rows), "n_boot": a.n_boot, "tolerance": TOL, | |
| "dms_fields_used": []}) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 8.7 kB
- Xet hash:
- 360955d97607413b3fe7c1da6c2be9608af5b149ec81ad8f5689af1b76740f03
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.