Buckets:
| """Evaluation harness: pred npz -> metrics.json + rows appended to results/tables/results.csv. | |
| python aide/eval/harness.py --pred results/runs/<name>/pred_test.npz --run <name> --model <m> \ | |
| [--seed 0] [--split test] [--notes "..."] [--n-boot 10000] [--variant protocol|alt] \ | |
| [--ref zvshrink=zv_shrink_val --ref simlpe=simlpe_kp_lr3e-4_s42 ...] | |
| --ref adds paired per-clip differences diff_vs_<name>_{all,hm} against results/runs/<run>/pred_<split>.npz | |
| (the zero-velocity reference diff_vs_ref_* is always computed). | |
| pred npz contract: pred [N,5,136,2] (px), clip_id [N] (str) — rows are aligned to the cache by | |
| clip_id; optional logits_dbr/logits_der/logits_tcr/logits_vcr [N,C]. | |
| Ground truth = frames 5..9 of the protocol sample (cache S10, or S10_alt with --variant alt). | |
| The paired difference is always computed against zero-velocity (copy frame 4 of S10). | |
| results.csv columns: run, model, seed, split, metric, value, ci_lo, ci_hi, n, hm_threshold, notes | |
| (append-only; a file lock serialises concurrent writers). | |
| """ | |
| from __future__ import annotations | |
| import os | |
| # ops rule (docs/decisions.md 2026-09-10, host CPU saturation): cap BLAS/OMP threads before numpy import | |
| for _v in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS"): | |
| os.environ.setdefault(_v, "4") | |
| import argparse | |
| import csv | |
| import fcntl | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval import metrics as M # noqa: E402 | |
| RESULTS_CSV = P.RESULTS / "tables" / "results.csv" | |
| CSV_COLS = ["run", "model", "seed", "split", "metric", "value", "ci_lo", "ci_hi", "n", "hm_threshold", "notes"] | |
| # metrics that go to the csv (everything else stays in metrics.json) | |
| CSV_METRICS = ["mpjpe_all", "dn_mpjpe", "pck05", "pck10", "pck05_diag", "mpjpe_hm", "dn_mpjpe_hm", "pck05_hm", | |
| "mpjpe_h1", "mpjpe_h2", "mpjpe_h3", "mpjpe_h4", "mpjpe_h5", | |
| "hm_h1", "hm_h2", "hm_h3", "hm_h4", "hm_h5", | |
| "mpjpe_body", "mpjpe_face", "mpjpe_lhand", "mpjpe_rhand", "mpjpe_conf0.1", | |
| "diff_vs_ref_all", "diff_vs_ref_hm", "diff_vs_zvshrink_all", "diff_vs_zvshrink_hm", | |
| "diff_vs_simlpe_all", "diff_vs_simlpe_hm", | |
| "f1_dbr", "f1_der", "f1_tcr", "f1_vcr", "acc_dbr", "acc_der", "acc_tcr", "acc_vcr"] | |
| def load_split(split: str, variant: str = "protocol"): | |
| c = np.load(P.CACHE_NPZ, allow_pickle=False) | |
| m = c["split"] == split | |
| key = "S10" if variant == "protocol" else "S10_alt" | |
| d = {"S10": c[key][m], "clip_id": c["clip_id"][m], "n_frames": c["n_frames"][m]} | |
| for t in P.TASKS: | |
| d[f"y_{t}"] = c[f"y_{t}"][m] | |
| return d | |
| def zero_velocity(S10): | |
| return np.repeat(S10[:, P.T_OBS - 1:P.T_OBS, :, :2], P.T_PRED, axis=1) | |
| def align(pred_npz: dict, clip_ids: np.ndarray): | |
| """Reorder pred rows to the cache order; require an exact clip set match.""" | |
| pid = np.asarray(pred_npz["clip_id"]).astype(str) | |
| assert len(set(pid)) == len(pid), "duplicate clip_id in pred" | |
| assert set(pid) == set(clip_ids.astype(str)), ( | |
| f"pred clip set != split clip set ({len(pid)} vs {len(clip_ids)})") | |
| pos = {c: i for i, c in enumerate(pid)} | |
| order = np.array([pos[c] for c in clip_ids.astype(str)]) | |
| return order | |
| def load_ref_preds(refs: dict, split: str, clip_ids: np.ndarray) -> dict: | |
| """refs {name: run-name or pred path} -> {name: pred [N,5,J,2] aligned to clip_ids}; missing refs skipped.""" | |
| out = {} | |
| for name, r in (refs or {}).items(): | |
| path = Path(r) if str(r).endswith(".npz") else P.RESULTS / "runs" / r / f"pred_{split}.npz" | |
| if not path.exists(): | |
| print(f"[harness] ref {name}: {path} missing, skipped") | |
| continue | |
| z = np.load(path, allow_pickle=False) | |
| out[name] = np.asarray(z["pred"], dtype=np.float64)[align(z, clip_ids)] | |
| return out | |
| def run_harness(pred_path, run: str, model: str, seed=None, split="test", notes="", n_boot=10000, | |
| variant="protocol", write_csv=True, boot_seed=0, refs=None) -> dict: | |
| pred_path = Path(pred_path) | |
| z = np.load(pred_path, allow_pickle=False) | |
| d = load_split(split, variant) | |
| order = align(z, d["clip_id"]) | |
| pred = np.asarray(z["pred"], dtype=np.float64)[order] | |
| N = len(d["clip_id"]) | |
| assert pred.shape == (N, P.T_PRED, P.J, 2), pred.shape | |
| assert np.isfinite(pred).all(), "non-finite predictions" | |
| gt = d["S10"][:, P.T_OBS:] | |
| ref = zero_velocity(d["S10"]) | |
| ref_preds = load_ref_preds(refs, split, d["clip_id"]) | |
| out, meta = M.evaluate(pred, gt, n_boot=n_boot, seed=boot_seed, ref_pred=ref, ref_preds=ref_preds) | |
| logits = {t: np.asarray(z[f"logits_{t}"])[order] for t in P.TASKS if f"logits_{t}" in z.files} | |
| if logits: | |
| out.update(M.evaluate_semantic(logits, {t: d[f"y_{t}"] for t in logits}, n_boot=n_boot, seed=boot_seed)) | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| rec = { | |
| "run": run, "model": model, "seed": seed, "split": split, "variant": variant, | |
| "n": N, "pred": str(pred_path), "utc": stamp, "notes": notes, | |
| "refs": {k: str(v) for k, v in (refs or {}).items() if k in ref_preds}, | |
| "protocol": {"driver_rule": P.DRIVER_RULE, "sample_grid": P.SAMPLE_GRID, | |
| "hm_top": P.hm_top(N), "hm_n_transitions": P.HM_N_TRANSITIONS, | |
| "pck_norm_px": P.PCK_NORM, "n_boot": n_boot}, | |
| "hm_threshold": meta["hm_threshold"], "hm_clip_ids": d["clip_id"][meta["hm_clip_ids_idx"]].tolist(), | |
| "meta": {k: v for k, v in meta.items() if k != "hm_clip_ids_idx"}, | |
| "metrics": out, | |
| } | |
| mpath = pred_path.parent / f"metrics_{split}{'' if variant == 'protocol' else '_' + variant}.json" | |
| if mpath.exists(): # append-only: keep earlier evaluations | |
| mpath = pred_path.parent / f"metrics_{split}_{stamp}.json" | |
| mpath.write_text(json.dumps(rec, indent=1)) | |
| rec["metrics_path"] = str(mpath) | |
| if write_csv: | |
| RESULTS_CSV.parent.mkdir(parents=True, exist_ok=True) | |
| note = f"t={stamp}" + (f";variant={variant}" if variant != "protocol" else "") + (f";{notes}" if notes else "") | |
| with open(RESULTS_CSV, "a", newline="") as f: | |
| fcntl.flock(f, fcntl.LOCK_EX) | |
| w = csv.writer(f) | |
| if f.tell() == 0: | |
| w.writerow(CSV_COLS) | |
| for k in CSV_METRICS: | |
| if k in out: | |
| v = out[k] | |
| w.writerow([run, model, seed, split, k, f"{v['value']:.6g}", f"{v['ci_lo']:.6g}", | |
| f"{v['ci_hi']:.6g}", v["n"], f"{meta['hm_threshold']:.4f}", note]) | |
| fcntl.flock(f, fcntl.LOCK_UN) | |
| return rec | |
| def summary_line(rec: dict) -> str: | |
| m = rec["metrics"] | |
| g = lambda k: m[k]["value"] if k in m else float("nan") # noqa: E731 | |
| s = (f"{rec['run']:28s} {rec['split']:5s} All {g('mpjpe_all'):6.2f} [{m['mpjpe_all']['ci_lo']:.2f},{m['mpjpe_all']['ci_hi']:.2f}]" | |
| f" d-n {g('dn_mpjpe'):.3f} HM {g('mpjpe_hm'):7.2f} PCK05 {g('pck05'):6.2f} PCK10 {g('pck10'):6.2f}" | |
| f" h5 {g('mpjpe_h5'):6.2f} HMh5 {g('hm_h5'):7.2f} dZV {g('diff_vs_ref_all'):+.2f}" | |
| f" [{m['diff_vs_ref_all']['ci_lo']:+.2f},{m['diff_vs_ref_all']['ci_hi']:+.2f}]") | |
| for name in ("zvshrink", "simlpe"): | |
| if f"diff_vs_{name}_all" in m: | |
| s += f" d{name} {g(f'diff_vs_{name}_all'):+.2f} [{m[f'diff_vs_{name}_all']['ci_lo']:+.2f},{m[f'diff_vs_{name}_all']['ci_hi']:+.2f}]" | |
| if "f1_dbr" in m: | |
| s += " F1 " + "/".join(f"{g(f'f1_{t}'):.1f}" for t in P.TASKS) | |
| return s | |
| def parse_refs(items) -> dict: | |
| refs = {} | |
| for it in items or []: | |
| k, v = it.split("=", 1) | |
| refs[k] = v | |
| return refs | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--pred", required=True) | |
| ap.add_argument("--run", required=True) | |
| ap.add_argument("--model", required=True) | |
| ap.add_argument("--seed", type=int, default=None) | |
| ap.add_argument("--split", default="test", choices=("train", "val", "test")) | |
| ap.add_argument("--notes", default="") | |
| ap.add_argument("--n-boot", type=int, default=10000) | |
| ap.add_argument("--variant", default="protocol", choices=("protocol", "alt")) | |
| ap.add_argument("--no-csv", action="store_true") | |
| ap.add_argument("--ref", action="append", default=[], help="name=run (repeatable)") | |
| a = ap.parse_args() | |
| rec = run_harness(a.pred, a.run, a.model, a.seed, a.split, a.notes, a.n_boot, a.variant, not a.no_csv, | |
| refs=parse_refs(a.ref)) | |
| print(summary_line(rec)) | |
| print("metrics ->", rec["metrics_path"]) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 8.78 kB
- Xet hash:
- e7dcbacd08f5b5ec54eb876cda6daa1286612e75f272917e5c5f8874b52b1fcd
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.