Buckets:
| """cwm3/eval/eval_cwm3.py — score cwm3 pred_layout files with the UNCHANGED cwm2 evaluator (eval_grid.py) into cwm3/results/eval_grid/. | |
| python eval_cwm3.py --item cwm3_exp2 --regime manual --runs V0_can4s_s0 V0_can8s_s0 [--with-can-only] [--cells veh_onset,d_onset,y_mae,y_crps] | |
| [--n-boot 1000] [--split test] | |
| --with-can-only adds cwm2/results/grid/can_only_direct_{regime}_s{0,1,2}/pred_layout_test.npz so every cell is scored on the SAME row | |
| intersection (eval_grid drops rows any arm did not score) — the fair reference for Exp2. --baselines is always on (persistence / | |
| k0_heuristic / current_signal_S0 rows, Exp1 / Exp3 references). Rows land in cwm3/results/eval_grid/metrics.csv (results_schema.COLUMNS, | |
| append-only); the cwm2 tables are never written. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| EXPLORE = CWM3.parent | |
| CWM2 = EXPLORE / "cwm2" | |
| OUT = CWM3 / "results" / "eval_grid" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--item", required=True) | |
| ap.add_argument("--regime", required=True, choices=["manual", "engaged"]) | |
| ap.add_argument("--runs", nargs="+", required=True, help="run dir names under results/runs (or absolute paths)") | |
| ap.add_argument("--split", default="test") | |
| ap.add_argument("--with-can-only", action="store_true") | |
| ap.add_argument("--cells", default="") | |
| ap.add_argument("--n-boot", type=int, default=1000) | |
| ap.add_argument("--n-boot-secondary", type=int, default=300) | |
| ap.add_argument("--workers", type=int, default=8) | |
| a = ap.parse_args() | |
| preds = [] | |
| for r in a.runs: | |
| rd = Path(r) if Path(r).is_absolute() else CWM3 / "results" / "runs" / r | |
| p = rd / f"pred_layout_{a.regime}_{a.split}.npz" | |
| assert p.exists(), p | |
| preds.append(str(p)) | |
| if a.with_can_only: | |
| for s in range(3): | |
| p = CWM2 / "results" / "grid" / f"can_only_direct_{a.regime}_s{s}" / f"pred_layout_{a.split}.npz" | |
| if p.exists(): | |
| preds.append(str(p)) | |
| OUT.mkdir(parents=True, exist_ok=True) | |
| cmd = [sys.executable, str(CWM2 / "eval" / "eval_grid.py"), "--regime", a.regime, "--pred", *preds, "--baselines", "--item", a.item, | |
| "--out-dir", str(OUT), "--n-boot", str(a.n_boot), "--n-boot-secondary", str(a.n_boot_secondary), "--workers", str(a.workers)] | |
| if a.cells: | |
| cmd += ["--cells", a.cells] | |
| print(" ".join(cmd), flush=True) | |
| sys.exit(subprocess.call(cmd, cwd=str(CWM2))) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 2.62 kB
- Xet hash:
- 3c71afe95713b746998206a8d7d9e5e30d9eb33f3e887f32927527d88420e73a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.