Buckets:
| """cwm3/aide/eval/report_5seed.py — 5-seed extension of the two locked AIDE main models (protocol C22; pre-registered reporting rule, decisions 2026-09-13). | |
| Reads metrics_test.json / metrics_val.json / run json of cwm3d_kp_s{42,123,456,789,1011}_v2 (CWM3-D-kp) and cwm3d_vis_sem_s{...}_v2 (B2 main). | |
| Rule (fixed before the runs): the locked 3 seeds (42 / 123 / 456) remain the headline rows of stage_a.md / stage_bc.md; this script writes the APPENDIX tables | |
| aide/results/tables/stage_a_5seed.md (kp) and aide/results/tables/stage_bc_5seed.md (vis+sem), | |
| each with the per-seed values (locked + new; a worse seed is reported, never dropped), the locked 3-seed mean ± sd, the 5-seed mean ± sd (ddof 1), and the | |
| margin over the best published value per geometry column (aide/eval/paper_rows.best_published) recomputed with the 5-seed mean and sd; and appends a compact | |
| section to stage_bc.md (report_stage_bc.py regenerations drop appended sections — the standalone file is authoritative). Numbers only from run outputs. | |
| python aide/eval/report_5seed.py [--no-append] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import glob | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval import paper_rows as PR # noqa: E402 | |
| from common.runmeta import write_run_json # noqa: E402 | |
| RUNS = P.RESULTS / "runs" | |
| LOCKED, NEW = (42, 123, 456), (789, 1011) | |
| GEOM = [("mpjpe_all", "All", 2), ("dn_mpjpe", "d-n", 3), ("mpjpe_hm", "HM", 2), ("dn_mpjpe_hm", "HM d-n", 3), ("pck05", "PCK@0.05", 2)] | |
| EXTRA = [("pck10", "PCK@0.10", 2), ("hm_h5", "HM h=5", 2)] | |
| F1 = [(f"f1_{t}", f"F1 {t.upper()}", 1) for t in P.TASKS] | |
| MODELS = [("kp", "CWM3-D-kp (5.8 M, geometry only; Stage A headline)", "cwm3d_kp_s{k}_v2", GEOM + EXTRA, "stage_a_5seed.md"), | |
| ("vis_sem", "CWM3-D vis+sem (B2 main, 6.3 M)", "cwm3d_vis_sem_s{k}_v2", GEOM + EXTRA + F1, "stage_bc_5seed.md")] | |
| def load_run(name): | |
| rd = RUNS / name | |
| mt, mv = rd / "metrics_test.json", rd / "metrics_val.json" | |
| fs = sorted(glob.glob(str(rd / f"run_{name}_2*.json"))) | |
| if not (mt.exists() and fs): | |
| return None | |
| t = json.load(open(mt))["metrics"] | |
| v = json.load(open(mv))["metrics"] if mv.exists() else {} | |
| d = json.load(open(fs[-1])) | |
| return {"test": {k: t[k]["value"] for k in t}, "val_all": v.get("mpjpe_all", {}).get("value"), "best_epoch": d.get("best_epoch"), "epochs": d.get("epochs_run"), | |
| "stop": d.get("stop_reason"), "gpu_min": d.get("gpu_minutes"), "t_start": d.get("t_start_utc"), "n_params": d.get("n_params"), "notes": d.get("config", {}).get("notes")} | |
| def ms(vals, nd): | |
| v = np.asarray([x for x in vals if x is not None], float) | |
| if not len(v): | |
| return "—" | |
| return f"{v.mean():.{nd}f} ± {v.std(ddof=1):.{nd}f}" if len(v) > 1 else f"{v[0]:.{nd}f}" | |
| def section(tag, title, pat, cols, stamp): | |
| runs = {k: load_run(pat.format(k=k)) for k in LOCKED + NEW} | |
| have = [k for k in LOCKED + NEW if runs[k]] | |
| L = [f"# 5-seed extension — {title} (protocol C22; appendix table, generated {stamp} by `aide/eval/report_5seed.py`)", "", | |
| "Pre-registered reporting rule (decisions 2026-09-13, before the runs): the locked seeds 42 / 123 / 456 stay the headline rows of stage_a.md / stage_bc.md; seeds 789 / 1011 " | |
| "were trained with the locked recipe verbatim (val-only selection, test evaluated once) and are reported here whatever their value. Test split n = 609; sd = seed sd (ddof 1).", "", | |
| "## 1. Per seed (test)", "", | |
| "| seed | status | run | best epoch / epochs / stop | GPU min | val All | " + " | ".join(c[1] for c in cols) + " |", "|---|---|---|---|---|---|" + "---|" * len(cols)] | |
| for k in LOCKED + NEW: | |
| r = runs[k] | |
| if r is None: | |
| L.append(f"| {k} | {'locked' if k in LOCKED else 'NEW'} | {pat.format(k=k)} | (not finished) | | | " + " | ".join("—" for _ in cols) + " |") | |
| continue | |
| L.append(f"| {k} | {'locked' if k in LOCKED else 'NEW'} | {pat.format(k=k)} | {r['best_epoch']} / {r['epochs']} / {r['stop']} | {r['gpu_min']} | {r['val_all']:.3f} | " | |
| + " | ".join(f"{r['test'].get(c, float('nan')):.{nd}f}" for c, _, nd in cols) + " |") | |
| L += ["", "## 2. Seed means", "", "| rows | n | " + " | ".join(c[1] for c in cols) + " |", "|---|---|" + "---|" * len(cols)] | |
| groups = [("locked 3 seeds (HEADLINE, unchanged)", [k for k in LOCKED if k in have]), ("new seeds 789 / 1011", [k for k in NEW if k in have]), ("all 5 seeds (appendix)", have)] | |
| stats = {} | |
| for gname, ks in groups: | |
| L.append(f"| {gname} | {len(ks)} | " + " | ".join(ms([runs[k]['test'].get(c) for k in ks], nd) for c, _, nd in cols) + " |") | |
| stats[gname] = {c: [runs[k]["test"].get(c) for k in ks] for c, _, _ in cols} | |
| if have: | |
| L.append("| worst seed of 5 (per column) | 5 | " + " | ".join((f"{max(x):.{nd}f}" if c in PR.LOWER_IS_BETTER else f"{min(x):.{nd}f}") for (c, _, nd), x in | |
| ((cc, [runs[k]["test"].get(cc[0]) for k in have]) for cc in cols)) + " |") | |
| L += ["", "## 3. Margin over the best published value (5 geometry columns; margin = ours − best; error columns negative = ours better, PCK positive = ours better)", "", | |
| "| column | best published (row) | locked 3-seed mean ± sd / margin | 5-seed mean ± sd / margin | margin in 5-seed sd units | worst of 5 seeds / its margin | every seed better than best published? |", | |
| "|---|---|---|---|---|---|---|"] | |
| margins = {} | |
| for c, lab, nd in GEOM: | |
| best, row = PR.best_published(c) | |
| v3 = np.asarray([runs[k]["test"][c] for k in LOCKED if k in have], float) | |
| v5 = np.asarray([runs[k]["test"][c] for k in have], float) | |
| lower = c in PR.LOWER_IS_BETTER | |
| m3, m5 = (v3.mean() - best) if len(v3) else float("nan"), (v5.mean() - best) if len(v5) else float("nan") | |
| sd5 = v5.std(ddof=1) if len(v5) > 1 else float("nan") | |
| worst = v5.max() if lower else v5.min() | |
| every = bool(np.all(v5 < best) if lower else np.all(v5 > best)) if len(v5) else False | |
| L.append(f"| {lab} | {best} ({row}) | {ms(v3, nd)} / {m3:+.{nd}f} | {ms(v5, nd)} / {m5:+.{nd}f} | {(m5 / sd5) if sd5 else float('nan'):+.1f} σ | {worst:.{nd}f} / {worst - best:+.{nd}f} | " | |
| f"{'yes (' + str(len(v5)) + '/' + str(len(v5)) + ')' if every else 'NO'} |") | |
| margins[c] = {"best": best, "row": row, "mean3": float(v3.mean()) if len(v3) else None, "mean5": float(v5.mean()) if len(v5) else None, "sd5": float(sd5), "margin5": float(m5), | |
| "worst5": float(worst), "every_seed_better": every, "n": int(len(v5))} | |
| new_worse = [] | |
| for k in NEW: | |
| if runs.get(k) and runs[k]["test"].get("mpjpe_all") is not None: | |
| m3 = np.mean([runs[j]["test"]["mpjpe_all"] for j in LOCKED if runs[j]]) | |
| if runs[k]["test"]["mpjpe_all"] > m3: | |
| new_worse.append(f"seed {k}: All {runs[k]['test']['mpjpe_all']:.2f} vs locked mean {m3:.2f}") | |
| L += ["", "## 4. Reading", "", f"- New seeds worse than the locked 3-seed All mean: {', '.join(new_worse) if new_worse else 'none'} (reported either way; nothing dropped).", | |
| "- The headline rows of stage_a.md / stage_bc.md are unchanged by construction; the 5-seed statistics above are the appendix numbers.", ""] | |
| return L, {"runs": {str(k): runs[k] for k in have}, "groups": {g: {c: v for c, v in s.items()} for g, s in stats.items()}, "margins": margins, "new_worse": new_worse} | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--no-append", action="store_true", help="do not append the vis+sem section to stage_bc.md") | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| summary, outs = {}, [] | |
| for tag, title, pat, cols, fname in MODELS: | |
| L, s = section(tag, title, pat, cols, stamp) | |
| out = P.RESULTS / "tables" / fname | |
| out.write_text("\n".join(L)) | |
| out.with_suffix(".json").write_text(json.dumps(s, indent=1, default=str)) | |
| summary[tag] = {"file": str(out), "margins": s["margins"], "new_worse": s["new_worse"], "n_runs": len(s["runs"])} | |
| outs.append(out) | |
| print(f"written {out}: seeds {sorted(int(k) for k in s['runs'])}") | |
| for c, m in s["margins"].items(): | |
| print(f" {c}: 5-seed {m['mean5']} ± {m['sd5']:.3f} margin {m['margin5']:+.3f} (best {m['best']} {m['row']}), every seed better: {m['every_seed_better']}") | |
| if not a.no_append: | |
| bc = P.RESULTS / "tables" / "stage_bc.md" | |
| L_vs, _ = section(*[m for m in MODELS if m[0] == "vis_sem"][0][:4], stamp) | |
| block = ["", "", f"## 9. 5-seed extension of the B2 main model (protocol C22; appended {stamp} by aide/eval/report_5seed.py — authoritative copy: `aide/results/tables/stage_bc_5seed.md`; " | |
| "a regeneration of this file by report_stage_bc.py drops this section)", ""] + [l.replace("# 5-seed extension", "### 5-seed extension", 1) if l.startswith("# ") else l.replace("## ", "### ", 1) if l.startswith("## ") else l for l in L_vs[2:]] | |
| with open(bc, "a") as f: | |
| f.write("\n".join(block) + "\n") | |
| print(f"appended §9 to {bc}") | |
| write_run_json(P.RESULTS / "tables", "report_5seed", [Path(__file__), PR.__file__, P.__file__], [P.CACHE_NPZ], seed=0, t_start=t0, input_listing=False, | |
| extra={"outs": [str(o) for o in outs], "summary": summary, "dms_fields_used": []}) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 9.71 kB
- Xet hash:
- 9deaa99ff412a0503fc6bab7003f2c91bca15d63765ed6392d8564638392ebaf
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.