#!/usr/bin/env python """Aggregate the runs of a table into tables/tab.tex, tab_common.tex, tab_paired.tex, tab.json and tables/numbers.tex. python scripts/aggregate.py --table 3 [--runs runs] [--out tables] Refuses to merge runs with inconsistent provenance or missing runs unless ``--allow-incomplete`` (marked INCOMPLETE in every output). """ from __future__ import annotations import argparse import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from pulse.aggregate import aggregate, collect, load_registry, table_spec, write_outputs # noqa: E402 def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--table", required=True, action="append", help="table id (repeatable)") ap.add_argument("--registry", default=None, help="default configs/tables.yaml") ap.add_argument("--runs", default="runs") ap.add_argument("--out", default="tables") ap.add_argument("--allow-incomplete", action="store_true") ap.add_argument("--allow-mixed-sha", action="store_true", help="merge runs from different commits (data/split/taxonomy must still agree)") ap.add_argument("--allow-dirty", action="store_true") ap.add_argument("--allow-mixed-device", action="store_true", help="merge runs from different device types / torch versions (not for the paper)") args = ap.parse_args(argv) reg = load_registry(args.registry) for t in args.table: spec = table_spec(reg, t) runs, missing = collect(spec, args.runs) summary = aggregate(spec, runs, missing=missing, allow_incomplete=args.allow_incomplete, require_same_sha=not args.allow_mixed_sha, allow_dirty=args.allow_dirty, allow_mixed_device=args.allow_mixed_device) for p in write_outputs(summary, args.out): print(f"wrote {p}") for row, r in summary["rows"].items(): m0 = spec["metrics"][0]["key"] print(f" {row:24s} {m0} own {r['own'][m0]['mean']:.4f} +- {r['own'][m0]['sd']:.4f}" f" common {r['common'][m0]['mean']:.4f} n_test {sum(r['n_test_recordings'].values())}") return 0 if __name__ == "__main__": sys.exit(main())