Buckets:
| """cwm3/eval/run_exp_evals.py — the CPU evaluation pass for experiments 1-3 over every finished run (PRED_DONE) of the given seeds. | |
| python run_exp_evals.py --seeds 0 [--n-boot 1000] [--exps 1,2,3] [--regimes manual,engaged] | |
| Exp1: eval/kin_repro.py (kin_predictability populations, cluster bootstrap 300) + eval_grid drv_onset / hazard_nll cells (item cwm3_exp1) | |
| Exp2: eval_grid veh_onset / y_mae / y_crps / nr_brier cells with the can_only_direct reference files on the same rows (item cwm3_exp2) | |
| Exp3: eval_grid d_onset / future_latent cells with current_signal_S0 (baselines) and can_only_direct on the same rows (item cwm3_exp3) | |
| Everything lands in results/eval_grid/metrics.csv and results/exp1_kin_repro.csv (append-only); log results/eval_grid/run_exp_evals.log. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import subprocess | |
| import sys | |
| import time | |
| from pathlib import Path | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| RUNS = CWM3 / "results" / "runs" | |
| PY = sys.executable | |
| VARIANTS = {1: ("U0", ["kin_can", "kin_can_vc", "kin_can_crop"]), 2: ("V0", ["can4s", "can8s"]), 3: ("E0", ["pooled", "tok4s", "tok8s"])} | |
| CELLS = {1: "drv_onset,hazard_nll", 2: "veh_onset,y_mae,y_crps,nr_brier", 3: "d_onset,future_latent"} | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0") | |
| ap.add_argument("--exps", default="1,2,3") | |
| ap.add_argument("--regimes", default="manual,engaged") | |
| ap.add_argument("--n-boot", type=int, default=1000) | |
| ap.add_argument("--workers", type=int, default=8) | |
| a = ap.parse_args() | |
| logp = CWM3 / "results" / "eval_grid" / "run_exp_evals.log" | |
| logp.parent.mkdir(parents=True, exist_ok=True) | |
| def log(m): | |
| line = f"[{time.strftime('%H:%M:%S')}] {m}" | |
| print(line, flush=True) | |
| with open(logp, "a") as f: | |
| f.write(line + "\n") | |
| def run(cmd): | |
| log(" ".join(cmd)) | |
| code = subprocess.call(cmd, cwd=str(CWM3)) | |
| log(f"exit {code}") | |
| return code | |
| seeds = [int(s) for s in a.seeds.split(",")] | |
| for e in [int(x) for x in a.exps.split(",")]: | |
| stage, variants = VARIANTS[e] | |
| runs = [f"{stage}_{v}_s{s}" for v in variants for s in seeds if (RUNS / f"{stage}_{v}_s{s}" / "PRED_DONE").exists()] | |
| if not runs: | |
| log(f"exp{e}: no finished runs — skipped") | |
| continue | |
| log(f"exp{e}: runs {runs}") | |
| if e == 1: | |
| run([PY, str(CWM3 / "eval" / "kin_repro.py"), "--run-dirs", *[str(RUNS / r) for r in runs], "--n-boot", "300"]) | |
| for regime in a.regimes.split(","): | |
| cmd = [PY, str(CWM3 / "eval" / "eval_cwm3.py"), "--item", f"cwm3_exp{e}", "--regime", regime, "--runs", *runs, "--cells", CELLS[e], | |
| "--n-boot", str(a.n_boot), "--workers", str(a.workers)] | |
| if e in (2, 3): | |
| cmd.append("--with-can-only") | |
| run(cmd) | |
| run([PY, str(CWM3 / "eval" / "summarize.py")]) | |
| log("EVALS DONE") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.01 kB
- Xet hash:
- 200184d20fd97dbe4681c899e9f89859ab244f2eeb6ba6b80a808ddb61e43c65
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.