Buckets:
| """cwm3/eval/merge_w_layouts.py — merge several W (arm cwm3_W) layout families of one joint run into ONE layout dir, so the UNCHANGED cwm2/eval/warning_gold.py | |
| scores every column in one item (it keys each --pred file by (arm, regime, seed, split) — warning_gold.py:290 — so two cwm3_W layouts of the same run overwrite | |
| each other; found 2026-09-12 when the base / K0-masked columns vanished behind the last-listed family). | |
| python eval/merge_w_layouts.py --run I1_all_minus_road2drv_s0_final --suffixes "" _k0m _k0old [--out-suffix _merged] | |
| Per (regime, split): loads results/runs/W_<run><suffix>/pred_layout_<regime>_<split>.npz, asserts identical uid / t, disjoint w_names, concatenates w / w_names | |
| (w_mask = AND of the sources), keeps the first source's meta and records `merged_from` = [{dir, sha256, w_names, model_id, warning_head}] plus | |
| model_id "W_<run><out-suffix>:merged(<n> families)". Values are copied bit-for-bit (float16). Output: results/runs/W_<run><out-suffix>/ (overwritten only if | |
| --force; the dir is a derived, reproducible artefact). No openpilot DMS field is read. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| RUNS = CWM3 / "results" / "runs" | |
| def sha(p): | |
| h = hashlib.sha256() | |
| with open(p, "rb") as f: | |
| for b in iter(lambda: f.read(1 << 22), b""): | |
| h.update(b) | |
| return h.hexdigest() | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--run", required=True) | |
| ap.add_argument("--suffixes", nargs="+", required=True, help="W dir suffixes to merge, in order ('' = the base W_<run>)") | |
| ap.add_argument("--out-suffix", default="_merged") | |
| ap.add_argument("--force", action="store_true") | |
| a = ap.parse_args() | |
| out = RUNS / f"W_{a.run}{a.out_suffix}" | |
| out.mkdir(parents=True, exist_ok=True) | |
| for reg in ("manual", "engaged"): | |
| for sp in ("test", "val"): | |
| op = out / f"pred_layout_{reg}_{sp}.npz" | |
| if op.exists() and not a.force: | |
| print(f"have {op}") | |
| continue | |
| srcs = [RUNS / f"W_{a.run}{s}" / f"pred_layout_{reg}_{sp}.npz" for s in a.suffixes] | |
| zs = [np.load(p, allow_pickle=True) for p in srcs] | |
| uid, t = zs[0]["uid"], zs[0]["t"] | |
| names, cols, masks, prov = [], [], [], [] | |
| for p, z in zip(srcs, zs): | |
| assert np.array_equal(z["uid"], uid) and np.allclose(z["t"], t), f"{p}: uid/t differ" | |
| wn = [str(x) for x in z["w_names"]] | |
| assert not (set(wn) & set(names)), f"{p}: duplicate column names {set(wn) & set(names)}" | |
| m = json.loads(str(z["meta"])) | |
| names += wn | |
| cols.append(z["w"]) | |
| masks.append(z["w_mask"].astype(bool)) | |
| prov.append({"dir": p.parent.name, "file": p.name, "sha256": sha(p), "w_names": wn, "model_id": m.get("model_id", ""), "warning_head": m.get("warning_head", {})}) | |
| meta = json.loads(str(zs[0]["meta"])) | |
| meta.update({"model_id": f"W_{a.run}{a.out_suffix}:merged({len(srcs)} families)", "merged_from": prov, | |
| "notes": "merge of several cwm3_W layout families of the same joint run (eval/merge_w_layouts.py): values copied bit-for-bit; " | |
| "needed because cwm2/eval/warning_gold.py keys layouts by (arm, regime, seed, split)"}) | |
| W = np.concatenate(cols, 1) | |
| assert W.shape == (len(uid), len(names)) | |
| np.savez_compressed(op, meta=np.array(json.dumps(meta)), uid=uid, t=t, w_names=np.array(names), w=W, w_mask=np.logical_and.reduce(masks)) | |
| print(f"wrote {op}: {len(uid)} rows x {len(names)} columns from {[p.parent.name for p in srcs]}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.88 kB
- Xet hash:
- 36fe0a80b71e0bc80cc8c2a0ab669ad99f48f752163d343dd55b2c61e2b3bdcb
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.