File size: 2,368 Bytes
dffbc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | #!/usr/bin/env python
"""Aggregate the runs of a table into tables/tab<N>.tex, tab<N>_common.tex,
tab<N>_paired.tex, tab<N>.json and tables/numbers.tex.
python scripts/aggregate.py --table 3 [--runs runs] [--out tables]
Refuses to merge runs with inconsistent provenance or missing runs unless
``--allow-incomplete`` (marked INCOMPLETE in every output).
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pulse.aggregate import aggregate, collect, load_registry, table_spec, write_outputs # noqa: E402
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--table", required=True, action="append", help="table id (repeatable)")
ap.add_argument("--registry", default=None, help="default configs/tables.yaml")
ap.add_argument("--runs", default="runs")
ap.add_argument("--out", default="tables")
ap.add_argument("--allow-incomplete", action="store_true")
ap.add_argument("--allow-mixed-sha", action="store_true",
help="merge runs from different commits (data/split/taxonomy must still agree)")
ap.add_argument("--allow-dirty", action="store_true")
ap.add_argument("--allow-mixed-device", action="store_true",
help="merge runs from different device types / torch versions (not for the paper)")
args = ap.parse_args(argv)
reg = load_registry(args.registry)
for t in args.table:
spec = table_spec(reg, t)
runs, missing = collect(spec, args.runs)
summary = aggregate(spec, runs, missing=missing, allow_incomplete=args.allow_incomplete,
require_same_sha=not args.allow_mixed_sha, allow_dirty=args.allow_dirty,
allow_mixed_device=args.allow_mixed_device)
for p in write_outputs(summary, args.out):
print(f"wrote {p}")
for row, r in summary["rows"].items():
m0 = spec["metrics"][0]["key"]
print(f" {row:24s} {m0} own {r['own'][m0]['mean']:.4f} +- {r['own'][m0]['sd']:.4f}"
f" common {r['common'][m0]['mean']:.4f} n_test {sum(r['n_test_recordings'].values())}")
return 0
if __name__ == "__main__":
sys.exit(main())
|