Buckets:
| """cwm3/eval/predict_kpf_holdout.py — the FROZEN keypoint forecaster `KPF_<arm>_s<k>` predicts the BATON-holdout anchors (candidate (iii) hold-out | |
| confirmation; pre-registration docs/decisions.md 2026-09-18T03:41:14Z; owner decision row 499). Mechanism = eval/predict_holdout.py for the forecaster: | |
| * the frozen run dir results/runs/<run> is READ-ONLY; results/runs/<run>_holdout/ receives copies of config.json / BEST / TRAIN_DONE / history.jsonl and a | |
| symlink to the BEST checkpoint (never copied); | |
| * the model is rebuilt from the frozen config.json (arm, d / depth / heads / mlp_ratio, `scale_fallback` = the recorded TRAIN median shoulder width — NEVER | |
| recomputed on the hold-out; the per-anchor causal normalisation ref / scale is part of the frozen forward, models/kp_forecaster.causal_normalise) and the | |
| checkpoint's state dict is loaded; the parameter count is asserted equal to the frozen record; | |
| * the data layer (train/train_kp_residual.Store / load_anchors / route_names) is pointed at the hold-out store results/holdout/kp_residual/ | |
| (data/build_kp_residual_store.py --holdout: store.npz + anchors_{r}_holdout.npz on the hold-out s12 grid); the anchors' uid / t are asserted against | |
| s12_wm/results/targets_{r}_holdout.npz; | |
| * prediction re-uses train_kp_residual.predict UNCHANGED (the function that wrote the benchmark kpf_{r}_{train,val,test}.npz channel): kpf_{r}_holdout.npz | |
| (route / route_names / t / feat f16 [N,10] = 6 state probabilities + 3 mode probabilities + valid bit / valid / names) + pred_kp_{r}_holdout.npz + PRED_DONE | |
| + run json. The joint model reads the channel through eval/predict_holdout.py --u-kpf-dir results/runs/<run>_holdout. | |
| No CAN column, label or openpilot DMS field is read (inputs = our own causal RTMW upper-body pose <= t). Evaluated once per frozen run. | |
| python eval/predict_kpf_holdout.py --run KPF_mb_full_s0 [--store-dir results/holdout/kp_residual] [--split holdout] [--batch 4096] [--cpu] | |
| """ | |
| from __future__ import annotations | |
| import os | |
| os.environ.setdefault("OMP_NUM_THREADS", "4") | |
| os.environ.setdefault("MKL_NUM_THREADS", "4") | |
| import argparse # noqa: E402 | |
| import json # noqa: E402 | |
| import shutil # noqa: E402 | |
| import sys # noqa: E402 | |
| import time # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| import torch # noqa: E402 | |
| torch.set_num_threads(4) | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| for p in (CWM3, CWM3 / "train", CWM3 / "common"): | |
| if str(p) not in sys.path: | |
| sys.path.insert(0, str(p)) | |
| import train_kp_residual as TK # noqa: E402 | |
| from models import kp_forecaster as kf # noqa: E402 | |
| from runmeta import write_run_json # noqa: E402 | |
| S12R = CWM3.parent / "s12_wm" / "results" | |
| COPY = ("config.json", "BEST", "TRAIN_DONE", "history.jsonl") | |
| def log(msg, out): | |
| line = f"[{time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}] {msg}" | |
| print(line, flush=True) | |
| with open(Path(out) / "predict.log", "a") as f: | |
| f.write(line + "\n") | |
| def build_frozen(src, device): | |
| """the frozen forecaster from results/runs/<run>: model (eval mode) + cfg + checkpoint path; nothing is fitted here.""" | |
| cfg = json.loads((src / "config.json").read_text()) | |
| arm = cfg["arm"] | |
| assert arm in TK.ARMS, arm | |
| best = (src / "BEST").read_text().strip() | |
| ckpt = src / best | |
| ck = torch.load(ckpt, map_location=device, weights_only=False) | |
| assert abs(float(ck["cfg"]["scale_fallback"]) - float(cfg["scale_fallback"])) < 1e-9, "checkpoint / config scale_fallback differ" | |
| model = kf.KpForecaster(**{k: v for k, v in TK.ARMS[arm].items()}, scale_fallback=float(cfg["scale_fallback"])).to(device) | |
| model.load_state_dict(ck["model"]) | |
| n = kf.n_params(model) | |
| assert n == int(cfg["params"]["total"]), (n, cfg["params"]) | |
| model.eval() | |
| return model, cfg, ckpt, ck.get("epoch") | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--run", required=True, help="frozen forecaster run dir name under results/runs (KPF_<arm>_s<k>; read-only)") | |
| ap.add_argument("--store-dir", default=str(CWM3 / "results" / "holdout" / "kp_residual"), help="hold-out KPF store (data/build_kp_residual_store.py --holdout)") | |
| ap.add_argument("--split", default="holdout") | |
| ap.add_argument("--batch", type=int, default=4096) | |
| ap.add_argument("--out", default="", help="default results/runs/<run>_<split>") | |
| ap.add_argument("--cpu", action="store_true") | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| src = CWM3 / "results" / "runs" / a.run | |
| assert (src / "TRAIN_DONE").exists() and (src / "PRED_DONE").exists() and (src / "BEST").exists(), f"{src}: not a finished forecaster run" | |
| store_dir = Path(a.store_dir) | |
| assert (store_dir / "store.npz").exists(), f"{store_dir}/store.npz missing (run data/build_kp_residual_store.py --holdout first)" | |
| for r in TK.REGIMES: | |
| assert (store_dir / f"anchors_{r}_{a.split}.npz").exists(), f"{store_dir}/anchors_{r}_{a.split}.npz missing" | |
| out = Path(a.out) if a.out else CWM3 / "results" / "runs" / f"{a.run}_{a.split}" | |
| out.mkdir(parents=True, exist_ok=True) | |
| for f in COPY: | |
| if (src / f).exists() and not (out / f).exists(): | |
| shutil.copy2(src / f, out / f) | |
| device = torch.device("cpu" if a.cpu else "cuda") | |
| model, cfg, ckpt, best_epoch = build_frozen(src, device) | |
| if not (out / ckpt.name).exists(): | |
| os.symlink(ckpt, out / ckpt.name) | |
| amp = bool(cfg.get("amp", True)) and device.type == "cuda" | |
| # point the trainer's data layer at the hold-out store (module globals read at call time; the benchmark store is not opened) | |
| TK.STORE = store_dir | |
| TK.ROUTE_NAMES = None | |
| store = TK.Store(device) | |
| log(f"=== predict_kpf_holdout {a.run} split={a.split} arm={cfg['arm']} params={cfg['params']['total']} scale_fallback={cfg['scale_fallback']:.6f} (frozen TRAIN) " | |
| f"ckpt={ckpt} best_epoch={best_epoch} store={store_dir} frames={store.n_frames:,} device={device} amp={amp} ===", out) | |
| cover, n_rows, anchors_in = {}, {}, [] | |
| for r in TK.REGIMES: | |
| a_np = TK.load_anchors(r, a.split) | |
| with np.load(S12R / f"targets_{r}_{a.split}.npz", allow_pickle=True) as z: | |
| assert np.array_equal(a_np["uid"], z["uid"].astype(str)) and np.allclose(a_np["t"], z["t"]), f"{r}/{a.split}: anchors != s12 grid (uid / t witness)" | |
| anchors_in.append(store_dir / f"anchors_{r}_{a.split}.npz") | |
| with torch.no_grad(): | |
| cover[f"{r}_{a.split}"] = TK.predict(model, store, a_np, TK.to_dev(a_np, device), a.batch, amp, out, r, a.split, full=True) | |
| n_rows[r] = int(len(a_np["t"])) | |
| with np.load(out / f"kpf_{r}_{a.split}.npz", allow_pickle=True) as k: | |
| uid = k["route_names"].astype(str)[k["route"]] | |
| assert np.array_equal(uid, a_np["uid"]) and np.allclose(k["t"], a_np["t"]) and k["feat"].shape == (n_rows[r], kf.KPF_DIM), "kpf file witness" | |
| fv = k["feat"][k["valid"]].astype(np.float32) | |
| log(f"predicted {r}/{a.split}: rows {n_rows[r]:,} anchor-frame-present share {float((a_np['lut_pos'] >= 0).mean()):.4f} valid share {cover[f'{r}_{a.split}']:.4f} " | |
| f"mean p_state(valid rows) {np.round(fv[:, :6].mean(0), 4).tolist() if len(fv) else '—'} mean p_mode {np.round(fv[:, 6:9].mean(0), 4).tolist() if len(fv) else '—'}", out) | |
| (out / "PRED_DONE").write_text(json.dumps({**cover, "splits": [a.split], "regimes": list(TK.REGIMES), "source_run": a.run, "store_dir": str(store_dir), "rows": n_rows})) | |
| write_run_json(str(out), f"predict_kpf_{a.split}", [Path(__file__), CWM3 / "train" / "train_kp_residual.py", CWM3 / "models" / "kp_forecaster.py", CWM3 / "data" / "build_kp_residual_store.py"], | |
| [ckpt, store_dir / "store.npz"] + anchors_in, seed=int(cfg["seed"]), t_start=t0, input_listing=False, | |
| extra={"source_run": a.run, "split": a.split, "cfg": cfg, "best_epoch": best_epoch, "coverage": cover, "rows": n_rows, "store_dir": str(store_dir), | |
| "gpu_wall_s": round(time.time() - t0, 1), "device": str(device), "amp": amp, "dms_fields_used": [], | |
| "note": "frozen forecaster checkpoint + frozen TRAIN scale_fallback; hold-out KPF store of results/holdout/kp_residual; evaluated once (pre-registration 2026-09-18T03:41:14Z)"}) | |
| log(f"PREDICT_KPF_HOLDOUT_DONE {a.run} ({time.time() - t0:.0f}s)", out) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 8.87 kB
- Xet hash:
- 8632d9c6a47c11a1b99a739aea51486e100edff3d422fb1de2499809d4d9edb0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.