Buckets:
| """cwm3/eval/pc_holdout.py — paired comparisons on the BATON-HOLDOUT split (WP5 confirmation batch; pre-registration docs/decisions.md 2026-09-17T11:45:58Z). | |
| Thin wrapper over eval/paired_compare_joint.py (`joint`) and eval/pressure_eval.py (`pressure`): the STATISTICS, cells, bootstrap, tags and CSV columns are those | |
| scripts' — unchanged. What changes is only WHERE the row-aligned target files are read from: the benchmark evaluators hard-code results/targets_kin_causal_routes_v2, | |
| cwm2/results/targets_demand and results/targets_pressure; the holdout versions live in results/holdout/ (data/holdout_pipeline.py). This wrapper substitutes those | |
| three directories (the s12 anchor grid s12_wm/results/targets_{r}_holdout.npz is read from its usual place) and appends the rows race-free to the shared | |
| results/paired_compare.csv under a flock (eval/pc_joint_locked.py pattern; several pairings may run in parallel). Guards: --split must be `holdout`, both layout dirs | |
| must be `*_holdout` run dirs written by eval/predict_holdout.py (never a frozen benchmark run dir), `joint` requires --kin-targets causal_routes --kin-version v2 (the | |
| v2 causal driver populations the benchmark tables use). CPU only; no openpilot DMS field. | |
| python eval/pc_holdout.py joint --regime manual --a results/runs/I1_all_minus_road2drv_s0_final_holdout --b results/runs/I1_indep_s0_final_holdout \ | |
| --split holdout --tag holdc_s0_allminus_vs_indep --kin-targets causal_routes --kin-version v2 | |
| python eval/pc_holdout.py pressure --regime manual --a results/runs/..._holdout --b results/runs/..._holdout --split holdout --tag holdP_s0_allminus_vs_indep | |
| """ | |
| from __future__ import annotations | |
| import csv | |
| import fcntl | |
| import os | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| H = CWM3 / "results" / "holdout" | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import paired_compare_joint as PCJ # noqa: E402 (puts cwm2/eval, s12_wm, cwm3/common on sys.path) | |
| import populations as P # noqa: E402 | |
| import pressure_eval as PE # noqa: E402 | |
| SHARED = CWM3 / "results" / "paired_compare.csv" | |
| LOCK = CWM3 / "results" / "paired_compare.csv.lock" | |
| TMP = CWM3 / "results" / "tmp_pc" | |
| KINC_H = H / "targets_kin_causal_routes_v2" | |
| def _opt(args, key, default=None): | |
| return args[args.index(key) + 1] if key in args else default | |
| def install_holdout_paths(tool, args): | |
| """point the evaluators' module-level target dirs at results/holdout/ (the layouts' rows = the holdout anchor grid).""" | |
| P.DEM = H / "targets_demand" # populations.load_demand -> results/holdout/targets_demand/demand_{r}_holdout.npz | |
| PE.PRESS = H / "targets_pressure" # pressure_eval.populations -> results/holdout/targets_pressure/pressure_{r}_holdout.npz | |
| if tool == "joint": | |
| assert _opt(args, "--kin-targets") == "causal_routes" and _opt(args, "--kin-version") == "v2", "holdout joint pairing: --kin-targets causal_routes --kin-version v2 required" | |
| _orig_load_all = P.load_all | |
| def load_all_holdout(regime, split="test"): | |
| assert split == "holdout", split | |
| # paired_compare_joint.main() has just installed its `load_kin_causal` closure, which reads the BENCHMARK dir results/targets_kin_causal_routes_v2 | |
| # (a local of main(); not redirectable). Same body, holdout dir: | |
| assert getattr(P.load_kin, "__name__", "") == "load_kin_causal", "expected paired_compare_joint's causal kin loader" | |
| def load_kin_causal(regime, split, tg): | |
| d = np.load(KINC_H / f"kin_causal_{regime}_{split}.npz", allow_pickle=True) | |
| assert np.array_equal(d["uid"], tg["uid"]) and np.allclose(d["t"], tg["t"]), "causal kin rows != targets rows" | |
| ok = d["row_ok"].astype(bool) | |
| S = d["S_c"].astype(np.float32); S[~ok] = np.nan | |
| S0 = d["S0_c"].astype(np.float32); S0[~ok] = np.nan | |
| EM = d["EM_c"].astype(bool) & tg["M"][:, :, None] & ok[:, None, None] | |
| return {"S": S, "S0": S0, "E": d["E_c"] * EM, "EM": EM, "E0": d["E0_c"], "K0": d["K0_c"], "Kvalid": d["Kvalid"] & ok[:, None]} | |
| P.load_kin = load_kin_causal | |
| out = _orig_load_all(regime, split) | |
| out["files"] = [p for p in out["files"] if Path(p).exists()] + [KINC_H / f"kin_causal_{regime}_{split}.npz"] | |
| return out | |
| P.load_all = load_all_holdout | |
| def main(): | |
| assert len(sys.argv) >= 2 and sys.argv[1] in ("joint", "pressure"), __doc__ | |
| tool, args = sys.argv[1], sys.argv[2:] | |
| assert _opt(args, "--split") == "holdout", "pc_holdout: --split holdout is mandatory" | |
| for k in ("--a", "--b"): | |
| v = _opt(args, k) | |
| assert v and Path(v).name.endswith("_holdout") and (Path(v) / "PRED_DONE").exists(), f"{k} must be a finished *_holdout run dir written by eval/predict_holdout.py (got {v})" | |
| assert "--b-k0" not in args and "--b-margin" not in args and "--b-persistence" not in args, "holdout batch: layout-vs-layout pairings only" | |
| tag, reg = _opt(args, "--tag", "untagged"), _opt(args, "--regime", "x") | |
| install_holdout_paths(tool, args) | |
| TMP.mkdir(parents=True, exist_ok=True) | |
| private = TMP / f"{tag}_{reg}_{os.getpid()}.csv" | |
| if private.exists(): | |
| private.unlink() | |
| mod = PCJ if tool == "joint" else PE | |
| mod.OUT = private # the module-level path main() appends to | |
| sys.argv = [str(CWM3 / "eval" / ("paired_compare_joint.py" if tool == "joint" else "pressure_eval.py"))] + args | |
| mod.main() | |
| if not private.exists(): | |
| print(f"no rows written for {tag} {reg}") | |
| return | |
| with open(private, newline="") as f: | |
| rows = list(csv.DictReader(f)) | |
| assert all(r["split"] == "holdout" for r in rows) | |
| t0 = time.time() | |
| with open(LOCK, "a") as lk: | |
| fcntl.flock(lk, fcntl.LOCK_EX) | |
| new = not SHARED.exists() or SHARED.stat().st_size == 0 | |
| with open(SHARED, "a", newline="") as f: | |
| w = csv.DictWriter(f, fieldnames=PCJ.COLS) | |
| if new: | |
| w.writeheader() | |
| for r in rows: | |
| w.writerow(r) | |
| f.flush() | |
| os.fsync(f.fileno()) | |
| fcntl.flock(lk, fcntl.LOCK_UN) | |
| private.unlink() | |
| print(f"locked-append {len(rows)} rows ({tag} {reg}, split holdout) to {SHARED} ({time.time() - t0:.1f}s)") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 6.66 kB
- Xet hash:
- 85f61e4a5cdf87a9cd6cf1418bddff1088f9eb6106e7362ace4bb2bdbe62ef21
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.