Buckets:
| """Prediction ensemble of existing runs (mean of pred_{val,test}.npz coordinates and of logits). | |
| python aide/eval/ensemble.py --name cwm3d_vis_sem_ens3_v2 --runs cwm3d_vis_sem_s42_v2,cwm3d_vis_sem_s123_v2,cwm3d_vis_sem_s456_v2 \ | |
| [--refs zvshrink=zv_shrink_val,kp=cwm3d_kp_s42_v2,b2=cwm3d_vis_sem_s42_v2] | |
| Writes results/runs/<name>/pred_{val,test}.npz, evaluates both splits with the harness and writes the run json. | |
| Reported as a separate "ensemble" row, never as a single model. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| for _v in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS"): | |
| os.environ.setdefault(_v, "4") | |
| import argparse # noqa: E402 | |
| import sys # noqa: E402 | |
| import time # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval.harness import parse_refs, run_harness, summary_line # noqa: E402 | |
| from common.runmeta import write_run_json # noqa: E402 | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--name", required=True) | |
| ap.add_argument("--runs", required=True) | |
| ap.add_argument("--refs", default="zvshrink=zv_shrink_val,simlpe=simlpe_kp_s42_v2,kp=cwm3d_kp_s42_v2,b2=cwm3d_vis_sem_s42_v2") | |
| ap.add_argument("--n-boot", type=int, default=10000) | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| runs = a.runs.split(",") | |
| out = P.RESULTS / "runs" / a.name | |
| out.mkdir(parents=True, exist_ok=True) | |
| preds = {} | |
| for split in ("val", "test"): | |
| zs = [np.load(P.RESULTS / "runs" / r / f"pred_{split}.npz", allow_pickle=False) for r in runs] | |
| ids = zs[0]["clip_id"].astype(str) | |
| arrs, logits = [], {} | |
| for z in zs: | |
| order = {c: i for i, c in enumerate(z["clip_id"].astype(str))} | |
| idx = [order[c] for c in ids] | |
| arrs.append(np.asarray(z["pred"], np.float64)[idx]) | |
| for t in P.TASKS: | |
| if f"logits_{t}" in z.files: | |
| logits.setdefault(t, []).append(np.asarray(z[f"logits_{t}"], np.float64)[idx]) | |
| pred = np.mean(arrs, 0) | |
| np.savez(out / f"pred_{split}.npz", pred=pred.astype(np.float32), clip_id=ids, split=np.array(split), | |
| **{f"logits_{t}": np.mean(v, 0).astype(np.float32) for t, v in logits.items()}) | |
| preds[split] = out / f"pred_{split}.npz" | |
| refs = parse_refs([r for r in a.refs.split(",") if r]) | |
| recs = {} | |
| for split in ("val", "test"): | |
| recs[split] = run_harness(preds[split], a.name, "ensemble", None, split, f"ensemble of {len(runs)} runs: {a.runs}", | |
| a.n_boot, refs=refs) | |
| print(summary_line(recs[split]), flush=True) | |
| tm, vm = recs["test"]["metrics"], recs["val"]["metrics"] | |
| keys = ("mpjpe_all", "mpjpe_hm", "pck05", "hm_h5", "diff_vs_ref_all", "diff_vs_kp_all", "diff_vs_kp_hm", | |
| "diff_vs_b2_all", "diff_vs_b2_hm", "f1_dbr", "f1_der", "f1_tcr", "f1_vcr") | |
| write_run_json(out, a.name, [__file__, P.__file__], [P.CACHE_NPZ] + [P.RESULTS / "runs" / r / "pred_test.npz" for r in runs], | |
| seed=None, t_start=t0, input_listing=False, | |
| extra={"members": runs, "val": {k: vm[k]["value"] for k in keys if k in vm}, | |
| "test": {k: tm[k]["value"] for k in keys if k in tm}, | |
| "test_metrics_json": recs["test"]["metrics_path"], "val_metrics_json": recs["val"]["metrics_path"]}) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.6 kB
- Xet hash:
- 7ad6efc037edb1e2f1bdf0fbc624721c039ba681db75b22dbeb30ea751059a37
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.