File size: 2,368 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
#!/usr/bin/env python
"""Aggregate the runs of a table into tables/tab<N>.tex, tab<N>_common.tex,
tab<N>_paired.tex, tab<N>.json and tables/numbers.tex.

    python scripts/aggregate.py --table 3 [--runs runs] [--out tables]

Refuses to merge runs with inconsistent provenance or missing runs unless
``--allow-incomplete`` (marked INCOMPLETE in every output).
"""
from __future__ import annotations

import argparse
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from pulse.aggregate import aggregate, collect, load_registry, table_spec, write_outputs  # noqa: E402


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--table", required=True, action="append", help="table id (repeatable)")
    ap.add_argument("--registry", default=None, help="default configs/tables.yaml")
    ap.add_argument("--runs", default="runs")
    ap.add_argument("--out", default="tables")
    ap.add_argument("--allow-incomplete", action="store_true")
    ap.add_argument("--allow-mixed-sha", action="store_true",
                    help="merge runs from different commits (data/split/taxonomy must still agree)")
    ap.add_argument("--allow-dirty", action="store_true")
    ap.add_argument("--allow-mixed-device", action="store_true",
                    help="merge runs from different device types / torch versions (not for the paper)")
    args = ap.parse_args(argv)
    reg = load_registry(args.registry)
    for t in args.table:
        spec = table_spec(reg, t)
        runs, missing = collect(spec, args.runs)
        summary = aggregate(spec, runs, missing=missing, allow_incomplete=args.allow_incomplete,
                            require_same_sha=not args.allow_mixed_sha, allow_dirty=args.allow_dirty,
                            allow_mixed_device=args.allow_mixed_device)
        for p in write_outputs(summary, args.out):
            print(f"wrote {p}")
        for row, r in summary["rows"].items():
            m0 = spec["metrics"][0]["key"]
            print(f"  {row:24s} {m0} own {r['own'][m0]['mean']:.4f} +- {r['own'][m0]['sd']:.4f}"
                  f"  common {r['common'][m0]['mean']:.4f}  n_test {sum(r['n_test_recordings'].values())}")
    return 0


if __name__ == "__main__":
    sys.exit(main())