Buckets:
| """cwm3/eval/kp_residual_eval.py — LANE F1 forecaster evaluation (R2 WP4 batch 2; pre-registration docs/decisions.md 2026-09-17T19:33:12Z). CPU, numpy. | |
| python eval/kp_residual_eval.py [--arms mb_full cwm_full mb_dense mb_1hz] [--seeds 0 1 2] [--split test] [--n-boot 500] [--out results/tables/kp_residual_forecaster] | |
| Truth (recomputed from results/kp_residual/store.npz, never from a run): the 11 upper-body joints at t + 0.5 / 1 / 2 / 3 s (frame-normalised x/W, y/H) with their | |
| validity; the current pose reference `ref` is the run's own (last valid position <= t under its observation window; pred_kp_*.npz) so that the model, the | |
| zero-velocity and the constant-velocity baselines share one reference per anchor. Scored rows = anchors whose forecaster output is valid. | |
| MPJPE_h(anchor) = mean over joints valid at t + h and with a reference of || pred − true || x 1000 (frame-normalised units x 1000) | |
| baselines zero-velocity ZV = ref; constant-velocity CV = ref + (ref − pose(t − 0.5 s)) x h / 0.5 on joints valid at t − 0.5 s (else ref) | |
| MOTION CLASSES label-free, from the mean joint displacement over [t − 1, t] (speed_past) and [t, t + 1] (speed_fut) on joints valid at both ends (>= 6 joints), | |
| theta = the TRAIN-split (both regimes) 70th percentile of speed_past (recorded): still (both < theta), motion onset (past < theta <= fut), | |
| sustained motion (both >= theta), recovery (past >= theta > fut); undefined rows are reported as 'undefined' | |
| states P(hands_on), P(head_turned) at +1 / +2 / +3 s (kin-v2 S_c) -> Brier / AUROC vs persistence (S0) | |
| Paired comparisons on identical anchors (device-cluster bootstrap, n_boot 500, seed 0; Δ = A − B in MPJPE, lower better): each arm vs ZV and vs CV; mb_full vs cwm_full | |
| (matched params), mb_full vs mb_dense (longer history) and vs mb_1hz (downsampled observation); verdict over the 3 seeds by kill_checks.beats on the negated | |
| deltas (BEATS = A lower). Outputs (new / append-only): <out>.json (every number) and <out>.md (a dated section). No CAN column, label or DMS field is read. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| CWM2 = CWM3.parent / "cwm2" | |
| for p in (CWM3, CWM3 / "common", CWM2 / "eval"): | |
| if str(p) not in sys.path: | |
| sys.path.insert(0, str(p)) | |
| import kill_checks as KC # noqa: E402 | |
| from runmeta import write_run_json # noqa: E402 | |
| STORE = CWM3 / "results" / "kp_residual" | |
| RUNS = CWM3 / "results" / "runs" | |
| REGIMES = ("manual", "engaged") | |
| HT = np.array([5, 10, 20, 30]); HS = ("0.5s", "1s", "2s", "3s") | |
| CLASSES = ("still", "onset", "sustained", "recovery") | |
| STATE_NAMES = ("hands_on", "head_turned"); STATE_H = ("1s", "2s", "3s") | |
| MIN_JOINTS_SPEED = 6 | |
| THETA_PCT = 70.0 | |
| class NpStore: | |
| def __init__(self): | |
| z = np.load(STORE / "store.npz", allow_pickle=True) | |
| self.kp = z["kp"]; self.valid = z["valid"]; self.lut = z["lut"].astype(np.int64); self.lut_off = z["route_lut_off"]; self.lut_len = z["route_lut_len"] | |
| self.L = len(self.lut) | |
| def gather(self, pos, route, offsets): | |
| """-> xy [N,T,11,2] f32 (0 where invalid), valid [N,T,11]""" | |
| idx = pos[:, None] + np.asarray(offsets)[None] | |
| lo = self.lut_off[route][:, None] | |
| inside = (idx >= lo) & (idx < lo + self.lut_len[route][:, None]) & (pos[:, None] >= 0) | |
| fi = self.lut[np.clip(idx, 0, self.L - 1)] | |
| fi = np.where(inside, fi, -1) | |
| ok = fi >= 0 | |
| k = self.kp[np.clip(fi, 0, None)].astype(np.float32) | |
| v = self.valid[np.clip(fi, 0, None)] & ok[..., None] | |
| return np.nan_to_num(k[..., :2]) * v[..., None], v | |
| def load_anchors(regime, split): | |
| z = np.load(STORE / f"anchors_{regime}_{split}.npz", allow_pickle=True) | |
| d = {k: z[k] for k in ("t", "dongle", "route", "lut_pos", "S", "S0")} | |
| d["uid"] = np.load(STORE / "store.npz", allow_pickle=True)["route_names"].astype(str)[d["route"]] | |
| return d | |
| def speeds(store, a): | |
| xy, v = store.gather(a["lut_pos"], a["route"], [-10, 0, 10]) | |
| def sp(i, j): | |
| both = v[:, i] & v[:, j] | |
| d = np.linalg.norm(xy[:, i] - xy[:, j], axis=-1) | |
| n = both.sum(1) | |
| s = np.where(n >= MIN_JOINTS_SPEED, (d * both).sum(1) / np.maximum(n, 1), np.nan) | |
| return s | |
| return sp(0, 1), sp(1, 2) | |
| def motion_class(sp_past, sp_fut, theta): | |
| c = np.full(len(sp_past), -1, np.int8) | |
| ok = np.isfinite(sp_past) & np.isfinite(sp_fut) | |
| c[ok & (sp_past < theta) & (sp_fut < theta)] = 0 | |
| c[ok & (sp_past < theta) & (sp_fut >= theta)] = 1 | |
| c[ok & (sp_past >= theta) & (sp_fut >= theta)] = 2 | |
| c[ok & (sp_past >= theta) & (sp_fut < theta)] = 3 | |
| return c | |
| def theta_from_train(store): | |
| vals = [] | |
| for r in REGIMES: | |
| a = load_anchors(r, "train") | |
| sel = np.random.default_rng(0).choice(len(a["t"]), min(200_000, len(a["t"])), replace=False) | |
| a = {k: v[sel] for k, v in a.items()} | |
| sp, _ = speeds(store, a) | |
| vals.append(sp[np.isfinite(sp)]) | |
| return float(np.percentile(np.concatenate(vals), THETA_PCT)) | |
| def per_anchor_mpjpe(pred_abs, true_xy, mask): | |
| """pred_abs / true_xy [N,H,J,2], mask [N,H,J] -> [N,H] (NaN when no joint)""" | |
| d = np.linalg.norm(pred_abs - true_xy, axis=-1) * mask | |
| n = mask.sum(-1) | |
| return np.where(n > 0, d.sum(-1) / np.maximum(n, 1), np.nan) * 1000.0 | |
| def boot_mean_diff(a, b, dev, n_boot, seed=0): | |
| """paired device-cluster bootstrap of mean(a) − mean(b) on rows where both finite -> dict""" | |
| ok = np.isfinite(a) & np.isfinite(b) | |
| a, b, dev = a[ok], b[ok], dev[ok] | |
| uniq, inv = np.unique(dev, return_inverse=True) | |
| D = len(uniq) | |
| sa = np.bincount(inv, weights=a, minlength=D); sb = np.bincount(inv, weights=b, minlength=D); n = np.bincount(inv, minlength=D).astype(np.float64) | |
| rng = np.random.default_rng(seed) | |
| W = rng.multinomial(D, np.full(D, 1.0 / D), size=n_boot).astype(np.float64) # [n_boot, D] device counts | |
| den = W @ n | |
| ma, mb = (W @ sa) / den, (W @ sb) / den | |
| d = ma - mb | |
| pct = lambda v: [float(np.percentile(v, 2.5)), float(np.percentile(v, 97.5))] | |
| return {"value_a": float(a.mean()), "value_b": float(b.mean()), "delta": float(a.mean() - b.mean()), "ci_d": pct(d), "ci_a": pct(ma), "ci_b": pct(mb), | |
| "n": int(ok.sum()), "n_dev": int(D)} | |
| def auroc(y, s): | |
| from sklearn.metrics import roc_auc_score | |
| return float(roc_auc_score(y, s)) if 0 < y.sum() < len(y) else float("nan") | |
| def verdict(rows): | |
| """rows: list of dicts per seed with delta / ci_d (A − B, lower better) and value_a / value_b -> kill_checks.beats on the NEGATED deltas""" | |
| pr = pd.DataFrame({"seed": range(len(rows)), "delta": [-r["delta"] for r in rows], "delta_ci_lo": [-r["ci_d"][1] for r in rows], "delta_ci_hi": [-r["ci_d"][0] for r in rows], | |
| "status": "secondary"}) | |
| va, vb = pd.DataFrame({"value": [r["value_a"] for r in rows]}), pd.DataFrame({"value": [r["value_b"] for r in rows]}) | |
| return KC.beats(pr, va, vb, min_seeds=3) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--arms", nargs="*", default=["mb_full", "cwm_full", "mb_dense", "mb_1hz"]) | |
| ap.add_argument("--seeds", nargs="*", type=int, default=[0, 1, 2]) | |
| ap.add_argument("--split", default="test") | |
| ap.add_argument("--n-boot", type=int, default=500) | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "kp_residual_forecaster")) | |
| a = ap.parse_args() | |
| t_start = time.time() | |
| store = NpStore() | |
| theta = theta_from_train(store) | |
| print(f"theta (train {THETA_PCT:g}th pct of the 1 s mean joint displacement) = {theta:.5f}") | |
| res = {"split": a.split, "theta": theta, "theta_rule": f"TRAIN {THETA_PCT:g}th percentile of speed_past (both regimes, 200k-anchor sample per regime)", "n_boot": a.n_boot, | |
| "arms": {}, "pairs": {}, "params": {}, "runs": {}} | |
| # truth per regime (independent of the arm) ------------------------------------------------------------------------------- | |
| truth = {} | |
| for r in REGIMES: | |
| an = load_anchors(r, a.split) | |
| fxy, fv = store.gather(an["lut_pos"], an["route"], HT) | |
| pxy, pv = store.gather(an["lut_pos"], an["route"], [-5]) | |
| sp_p, sp_f = speeds(store, an) | |
| truth[r] = {"an": an, "fxy": fxy, "fv": fv, "p05": pxy[:, 0], "p05v": pv[:, 0], "cls": motion_class(sp_p, sp_f, theta)} | |
| res.setdefault("class_share", {})[r] = {c: float((truth[r]["cls"] == i).mean()) for i, c in enumerate(CLASSES)} | {"undefined": float((truth[r]["cls"] < 0).mean()), "N": int(len(an["t"]))} | |
| per_anchor = {} # (arm, seed, regime) -> {"model": [N,H], "zv": [N,H], "cv": [N,H]} | |
| for arm in a.arms: | |
| for s in a.seeds: | |
| rd = RUNS / f"KPF_{arm}_s{s}" | |
| if not (rd / "PRED_DONE").exists(): | |
| print(f"skip {rd.name}: PRED_DONE missing"); continue | |
| cfg = json.loads((rd / "config.json").read_text()) | |
| res["params"][arm] = cfg["params"]; res["runs"][f"{arm}_s{s}"] = json.loads((rd / "TRAIN_DONE").read_text()) | {"gpu_wall_s": json.loads((rd / f"run_train_kp_residual_{arm}_s{s}_latest.json").read_text()).get("gpu_wall_s")} | |
| for r in REGIMES: | |
| T = truth[r] | |
| z = np.load(rd / f"pred_kp_{r}_{a.split}.npz", allow_pickle=True) | |
| assert np.array_equal(z["route_names"].astype(str)[z["route"]], T["an"]["uid"]) and np.allclose(z["t"], T["an"]["t"]), f"{rd.name} {r}: uid/t witness" | |
| ref = z["ref"].astype(np.float32); ref_ok = z["ref_ok"]; ok = z["valid"] | |
| mask = T["fv"] & ref_ok[:, None, :] & ok[:, None, None] | |
| pred = ref[:, None] + z["res_top1"].astype(np.float32) | |
| zv = np.repeat(ref[:, None], len(HT), 1) | |
| vel_ok = T["p05v"] & ref_ok | |
| vel = (ref - T["p05"]) * vel_ok[..., None] # displacement over the last 0.5 s | |
| cv = ref[:, None] + vel[:, None] * (HT / 5.0)[None, :, None, None] | |
| per_anchor[(arm, s, r)] = {"model": per_anchor_mpjpe(pred, T["fxy"], mask), "zv": per_anchor_mpjpe(zv, T["fxy"], mask), "cv": per_anchor_mpjpe(cv, T["fxy"], mask), | |
| "p_state": z["p_state"].astype(np.float32), "ok": ok} | |
| # per-arm tables ---------------------------------------------------------------------------------------------------------- | |
| for arm in a.arms: | |
| for s in a.seeds: | |
| for r in REGIMES: | |
| key = (arm, s, r) | |
| if key not in per_anchor: | |
| continue | |
| P, T = per_anchor[key], truth[r] | |
| rec = {"n_rows": int(np.isfinite(P["model"][:, 0]).sum())} | |
| for name in ("model", "zv", "cv"): | |
| rec[f"mpjpe_{name}"] = {h: float(np.nanmean(P[name][:, i])) for i, h in enumerate(HS)} | |
| rec[f"mpjpe_{name}_by_class"] = {c: {h: float(np.nanmean(P[name][T["cls"] == ci, i])) if (T["cls"] == ci).any() else float("nan") for i, h in enumerate(HS)} | |
| for ci, c in enumerate(CLASSES)} | |
| st = {} | |
| S, S0 = T["an"]["S"], T["an"]["S0"] | |
| for hi, h in enumerate(STATE_H): | |
| for si, sn in enumerate(STATE_NAMES): | |
| y = S[:, hi, si]; m = np.isfinite(y) & np.isfinite(S0[:, si]) & P["ok"] | |
| yb = y[m] > 0.5; p = P["p_state"][m, hi, si]; pers = S0[m, si] | |
| st[f"{sn}@{h}"] = {"brier_model": float(np.mean((p - yb) ** 2)), "brier_persistence": float(np.mean((pers - yb) ** 2)), "auroc_model": auroc(yb, p), | |
| "auroc_persistence": auroc(yb, pers), "n": int(m.sum()), "pos_rate": float(yb.mean())} | |
| rec["states"] = st | |
| res["arms"][f"{arm}_s{s}_{r}"] = rec | |
| # paired comparisons ------------------------------------------------------------------------------------------------------ | |
| pairs = [(arm, "model", arm, "zv", f"{arm}_vs_zv") for arm in a.arms] + [(arm, "model", arm, "cv", f"{arm}_vs_cv") for arm in a.arms] | |
| for b in ("cwm_full", "mb_dense", "mb_1hz"): | |
| if "mb_full" in a.arms and b in a.arms: | |
| pairs.append(("mb_full", "model", b, "model", f"mb_full_vs_{b}")) | |
| for arm_a, ka, arm_b, kb, tag in pairs: | |
| for r in REGIMES: | |
| for i, h in enumerate(HS): | |
| rows = [] | |
| for s in a.seeds: | |
| if (arm_a, s, r) not in per_anchor or (arm_b, s, r) not in per_anchor: | |
| continue | |
| rows.append(boot_mean_diff(per_anchor[(arm_a, s, r)][ka][:, i], per_anchor[(arm_b, s, r)][kb][:, i], truth[r]["an"]["dongle"], a.n_boot)) | |
| if rows: | |
| res["pairs"][f"{tag}|{r}|{h}"] = {"seeds": rows, "verdict": verdict(rows) if len(rows) >= 3 else {"verdict": "INSUFFICIENT", "n_seeds": len(rows)}} | |
| # class-split deltas at 1 s and 3 s (descriptive, mean over seeds) | |
| for ci, c in enumerate(CLASSES): | |
| for i in (1, 3): | |
| ds_ = [] | |
| for s in a.seeds: | |
| if (arm_a, s, r) in per_anchor and (arm_b, s, r) in per_anchor: | |
| sel = truth[r]["cls"] == ci | |
| x, y = per_anchor[(arm_a, s, r)][ka][sel, i], per_anchor[(arm_b, s, r)][kb][sel, i] | |
| m = np.isfinite(x) & np.isfinite(y) | |
| ds_.append(float(np.mean(x[m] - y[m])) if m.any() else float("nan")) | |
| if ds_: | |
| res["pairs"].setdefault("class_delta", {})[f"{tag}|{r}|{c}|{HS[i]}"] = {"deltas": ds_, "mean": float(np.nanmean(ds_))} | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| res["stamp"] = stamp | |
| out = Path(a.out) | |
| out.with_suffix(".json").write_text(json.dumps(res, indent=1, default=float)) | |
| # markdown section ----------------------------------------------------------------------------------------------------------- | |
| L = [f"## {stamp} — LANE F1 forecaster evaluation, split {a.split} (pre-registration 2026-09-17T19:33:12Z; n_boot {a.n_boot}; theta = {theta:.5f})", "", | |
| "MPJPE in frame-normalised units x 1000 (mean over scored anchors; 3-seed mean ± sd of the per-seed means). ZV = zero-velocity, CV = constant-velocity (last 0.5 s). Params per arm: " | |
| + ", ".join(f"{k} {v['total']:,}" for k, v in res["params"].items()), "", | |
| "| regime | arm | " + " | ".join(f"model {h}" for h in HS) + " | " + " | ".join(f"ZV {h}" for h in HS) + " | " + " | ".join(f"CV {h}" for h in HS) + " | n rows |", | |
| "|---|---|" + "---|" * (3 * len(HS)) + "---|"] | |
| def ms(vals): | |
| v = np.array(vals, float); return f"{v.mean():.2f} ± {v.std(ddof=1) if len(v) > 1 else 0:.2f}" | |
| for r in REGIMES: | |
| for arm in a.arms: | |
| recs = [res["arms"][k] for k in res["arms"] if k.startswith(f"{arm}_s") and k.endswith(f"_{r}")] | |
| if not recs: | |
| continue | |
| cells = [ms([x[f"mpjpe_{n}"][h] for x in recs]) for n in ("model", "zv", "cv") for h in HS] | |
| L.append(f"| {r} | {arm} | " + " | ".join(cells) + f" | {recs[0]['n_rows']:,} |") | |
| L += ["", "### Paired comparisons (Δ MPJPE = A − B, lower better; per-seed Δ [CI]; verdict = kill_checks.beats on the negated Δ: BEATS = A lower)", "", | |
| "| pair | regime | horizon | " + " | ".join(f"Δ s{s} [CI]" for s in a.seeds) + " | mean Δ | verdict |", "|---|---|---|" + "---|" * len(a.seeds) + "---|---|"] | |
| for k, v in res["pairs"].items(): | |
| if k == "class_delta": | |
| continue | |
| tag, r, h = k.split("|") | |
| d = [f"{x['delta']:+.2f} [{x['ci_d'][0]:+.2f},{x['ci_d'][1]:+.2f}]" for x in v["seeds"]] | |
| L.append(f"| {tag} | {r} | {h} | " + " | ".join(d + ["—"] * (len(a.seeds) - len(d))) + f" | {np.mean([x['delta'] for x in v['seeds']]):+.2f} | {v['verdict']['verdict']} |") | |
| L += ["", "### MPJPE by motion class (model; 3-seed mean; anchors per class share in brackets)", "", | |
| "| regime | arm | class | share | " + " | ".join(HS) + " | ZV " + " | ZV ".join(HS) + " |", "|---|---|---|---|" + "---|" * (2 * len(HS))] | |
| for r in REGIMES: | |
| for arm in a.arms: | |
| recs = [res["arms"][k] for k in res["arms"] if k.startswith(f"{arm}_s") and k.endswith(f"_{r}")] | |
| if not recs: | |
| continue | |
| for c in CLASSES: | |
| L.append(f"| {r} | {arm} | {c} | {res['class_share'][r][c]:.3f} | " + " | ".join(f"{np.nanmean([x['mpjpe_model_by_class'][c][h] for x in recs]):.2f}" for h in HS) | |
| + " | " + " | ".join(f"{np.nanmean([x['mpjpe_zv_by_class'][c][h] for x in recs]):.2f}" for h in HS) + " |") | |
| L += ["", "### Future-state probabilities (mb_full; 3-seed mean): Brier / AUROC vs persistence (S0)", "", | |
| "| regime | state@h | Brier model | Brier persistence | AUROC model | AUROC persistence | n | pos rate |", "|---|---|---|---|---|---|---|---|"] | |
| for r in REGIMES: | |
| recs = [res["arms"][k] for k in res["arms"] if k.startswith("mb_full_s") and k.endswith(f"_{r}")] | |
| if recs: | |
| for sh in recs[0]["states"]: | |
| g = lambda f: np.nanmean([x["states"][sh][f] for x in recs]) | |
| L.append(f"| {r} | {sh} | {g('brier_model'):.4f} | {g('brier_persistence'):.4f} | {g('auroc_model'):.4f} | {g('auroc_persistence'):.4f} | {recs[0]['states'][sh]['n']:,} | {g('pos_rate'):.3f} |") | |
| L += ["", f"Runs: " + "; ".join(f"{k}: best ep {v['best_epoch']}, val L1 {v['best_val_l1_top1']:.4f}, {v['epochs_run']} ep, {v['gpu_wall_s']:.0f} s" for k, v in res["runs"].items()), ""] | |
| with open(out.with_suffix(".md"), "a") as f: | |
| f.write("\n".join(L) + "\n") | |
| write_run_json(STORE, f"kp_residual_eval_{a.split}", [Path(__file__)], [STORE / "store.npz"], seed=0, t_start=t_start, extra={"stamp": stamp, "arms": a.arms, "seeds": a.seeds, "out": str(out)}, input_listing=False) | |
| print(f"written {out.with_suffix('.md')} and .json ({time.time() - t_start:.0f}s)") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 18.5 kB
- Xet hash:
- b6ff90732d96ea034e35344d9434365135cb996f146759b68d0316b857cc2b8d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.