File size: 18,811 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
"""Turn the runs of one table into LaTeX tables and number macros.

Rules (see also ``pulse.stats``):

* every run's metrics are recomputed from its per-recording statistics and
  must equal the stored test metrics;
* runs are merged only if their provenance agrees (data tag, split md5,
  taxonomy md5, availability md5, code version and, by default, git sha),
  all runs of a row used the same configuration including ``--set``
  overrides (``config_md5``; a leftover smoke-test run is refused), and all
  runs of a table ran on the same kind of device with the same torch version
  (``cpu`` or ``cuda:<GPU model>``; CPU and GPU results are not spliced);
* a row's value = mean over seeds within fold, then mean +- SD (ddof=1) over
  the fold means, on the row's own test recordings (``own``) and on the common
  test set (``common``: per fold, the recordings every row of the table loaded);
* comparisons with the reference row (the table's ``reference_row``, or a row's
  own ``reference``) use the common test set: paired t over
  fold means (n=5) and over fold x seed pairs (n=15), and positive-fold counts
  (a fold counts as positive when the row improves on the reference).
"""
from __future__ import annotations

import json
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple

import yaml

from .errors import ProvenanceError
from .metrics import compute_metrics
from .provenance import check_consistent, load_results
from .splits import load_split
from .stats import DDOF, compare_rows, fold_means, format_p, format_pm, mean_sd

REPO_ROOT = Path(__file__).resolve().parents[1]
RunKey = Tuple[int, int]  # (fold, seed)


def load_registry(path=None) -> Dict[str, dict]:
    p = Path(path) if path else REPO_ROOT / "configs" / "tables.yaml"
    with open(p, encoding="utf-8") as f:
        return yaml.safe_load(f)["tables"]


def table_spec(registry: Dict[str, dict], table: str) -> dict:
    if str(table) not in registry:
        raise KeyError(f"table {table!r} not in registry; known: {sorted(registry)}")
    spec = dict(registry[str(table)])
    spec["id"] = str(table)
    if "folds" not in spec:
        spec["folds"] = list(range(load_split(REPO_ROOT / spec["split"]).n_folds))
    for m in spec["metrics"]:
        m.setdefault("digits", 3)
        m.setdefault("better", "higher")
        m.setdefault("label", m["key"])
    return spec


def run_dir(runs_root, table: str, row: str, fold: int, seed: int) -> Path:
    return Path(runs_root) / str(table) / row / f"fold{fold}_seed{seed}"


def row_source(spec: dict, row: dict) -> Tuple[str, str]:
    """(table, row) whose runs ``row`` reads: its own, or those named by
    ``reuse: "<table>/<row>"`` (the row is then not run again, see run_table.py)."""
    src = row.get("reuse")
    if not src:
        return spec["id"], row["key"]
    t, sep, r = str(src).partition("/")
    if not (sep and t and r):
        raise ValueError(f"row {row['key']}: reuse must be '<table>/<row>', got {src!r}")
    return t, r


def collect(spec: dict, runs_root) -> Tuple[Dict[str, Dict[RunKey, dict]], List[str]]:
    runs: Dict[str, Dict[RunKey, dict]] = {}
    missing: List[str] = []
    for row in spec["rows"]:
        runs[row["key"]] = {}
        src_table, src_row = row_source(spec, row)
        for k in spec["folds"]:
            for s in spec["seeds"]:
                p = run_dir(runs_root, src_table, src_row, k, s) / "results.json"
                if p.is_file():
                    runs[row["key"]][(k, s)] = load_results(p)
                else:
                    missing.append(f"{row['key']}/fold{k}_seed{s}")
    return runs, missing


def _check_run(name: str, r: dict, spec: dict, fold: int, seed: int, tol: float) -> Dict[str, float]:
    prov = r["provenance"]
    if r["task"] != spec["task"]:
        raise ProvenanceError(f"{name}: task {r['task']!r} != table task {spec['task']!r}")
    if prov["fold"] != fold or prov["seed"] != seed:
        raise ProvenanceError(f"{name}: provenance says fold {prov['fold']} seed {prov['seed']}")
    own = compute_metrics(r)
    for k, v in r["metrics"].get("test", {}).items():
        if k in own and abs(own[k] - v) > tol * max(1.0, abs(v)):
            raise ProvenanceError(f"{name}: stored test {k}={v} != recomputed {own[k]}")
    for m in spec["metrics"]:
        if m["key"] not in own:
            raise ProvenanceError(f"{name}: metric {m['key']!r} not defined in results.json")
    return own


def compute_kind(prov: dict) -> str:
    """Hardware / software a run used: ``cpu`` or ``cuda:<GPU model>``, plus the torch version."""
    d = prov.get("device")
    if not isinstance(d, dict):
        return json.dumps(d)
    dev = str(d.get("device") or "")
    kind = f"cuda:{d.get('gpu_name')}" if dev.startswith("cuda") else (dev or "unknown")
    return f"{kind} torch {d.get('torch')}"


def check_runs_comparable(runs: Dict[str, Dict[RunKey, dict]],
                          allow_mixed_device: bool = False) -> List[str]:
    """Every run of a row must use the same configuration (config file + all
    ``--set`` overrides; ``config_md5``), so e.g. a 1-epoch smoke run left in
    the runs directory cannot be averaged with full runs; and every run of a
    table must have used the same kind of device and torch version (CPU and GPU
    results are not spliced into one table).  Returns the device kinds seen."""
    for row, d in runs.items():
        md5s = {}
        for (k, s), r in d.items():
            md5s.setdefault(str(r["provenance"].get("config_md5")), []).append(f"fold{k}_seed{s}")
        if len(md5s) > 1:
            detail = "; ".join(f"config_md5 {m}: {', '.join(v[:3])}" for m, v in md5s.items())
            raise ProvenanceError(f"row {row}: runs used different configurations or overrides "
                                  f"({detail}); rerun them with one configuration")
    kinds: Dict[str, List[str]] = {}
    for row, d in runs.items():
        for (k, s), r in d.items():
            kinds.setdefault(compute_kind(r["provenance"]), []).append(f"{row}/fold{k}_seed{s}")
    if len(kinds) > 1 and not allow_mixed_device:
        detail = "; ".join(f"{kd}: {len(v)} run(s), e.g. {v[0]}" for kd, v in kinds.items())
        raise ProvenanceError(f"runs were produced on different devices / torch versions ({detail}); "
                              "rerun them on one device type (or pass allow_mixed_device)")
    return sorted(kinds)


def _row_stats(values: Dict[RunKey, float]) -> Dict[str, Any]:
    fm = fold_means(values)
    mean, sd = mean_sd(list(fm.values()), DDOF)
    return {"per_run": {f"{k}_{s}": v for (k, s), v in sorted(values.items())},
            "fold_means": {str(k): v for k, v in fm.items()}, "mean": mean, "sd": sd,
            "n_folds": len(fm), "n_runs": len(values)}


def aggregate(spec: dict, runs: Dict[str, Dict[RunKey, dict]], *, missing: Optional[List[str]] = None,
              allow_incomplete: bool = False, require_same_sha: bool = True,
              allow_dirty: bool = False, allow_mixed_device: bool = False,
              tol: float = 1e-9) -> Dict[str, Any]:
    missing = list(missing or [])
    if missing and not allow_incomplete:
        raise ProvenanceError(f"table {spec['id']}: {len(missing)} run(s) missing, e.g. "
                              f"{missing[:5]} (use --allow-incomplete for a partial table)")
    flat = {f"{row}/fold{k}_seed{s}": r for row, d in runs.items() for (k, s), r in d.items()}
    if not flat:
        raise ProvenanceError(f"table {spec['id']}: no runs found")
    prov = check_consistent(flat, require_same_sha=require_same_sha, allow_dirty=allow_dirty)
    prov["compute"] = " | ".join(check_runs_comparable(runs, allow_mixed_device=allow_mixed_device))

    own: Dict[str, Dict[RunKey, Dict[str, float]]] = {}
    test_sets: Dict[str, Dict[int, frozenset]] = {}
    for row, d in runs.items():
        own[row], test_sets[row] = {}, {}
        for (k, s), r in d.items():
            own[row][(k, s)] = _check_run(f"{row}/fold{k}_seed{s}", r, spec, k, s, tol)
            ts = frozenset(r["provenance"]["loaded_recordings"]["test"])
            if k in test_sets[row] and test_sets[row][k] != ts:
                raise ProvenanceError(f"{row} fold {k}: seeds loaded different test recordings")
            test_sets[row][k] = ts
    folds = sorted({k for d in runs.values() for (k, _s) in d})
    common: Dict[int, List[str]] = {}
    for k in folds:
        sets = [test_sets[row][k] for row in runs if k in test_sets[row]]
        inter = frozenset.intersection(*sets) if sets else frozenset()
        if not inter:
            raise ProvenanceError(f"fold {k}: the rows share no test recording")
        common[k] = sorted(inter)

    rows_out: Dict[str, Any] = {}
    common_vals: Dict[str, Dict[str, Dict[RunKey, float]]] = {}
    for row, d in runs.items():
        cv = {(k, s): compute_metrics(r, recordings=common[k]) for (k, s), r in d.items()}
        common_vals[row] = {m["key"]: {key: v[m["key"]] for key, v in cv.items()}
                            for m in spec["metrics"]}
        label = next(x["label"] for x in spec["rows"] if x["key"] == row)
        rows_out[row] = {
            "label": label,
            "own": {m["key"]: _row_stats({key: v[m["key"]] for key, v in own[row].items()})
                    for m in spec["metrics"]},
            "common": {m["key"]: _row_stats(common_vals[row][m["key"]]) for m in spec["metrics"]},
            "n_test_recordings": {str(k): len(v) for k, v in sorted(test_sets[row].items())},
            "n_params": next((r.get("n_params") for r in d.values() if r.get("n_params")), None),
        }
    ref = spec.get("reference_row")
    # a row may name its own reference (``reference:`` in the registry), e.g. F5
    # compares each pre-trained row with the scratch row of the same fraction
    row_ref = {x["key"]: x.get("reference", ref) for x in spec["rows"]}
    for row in runs:
        rr = row_ref.get(row)
        if not rr or rr == row or rr not in runs or not runs[rr] or not runs[row]:
            continue  # no reference, or a row without any run (only with allow_incomplete)
        a_vals, r_vals = common_vals[row], common_vals[rr]
        if allow_incomplete:
            # partial table: compare on the (fold, seed) runs both rows have
            keys = set(next(iter(a_vals.values()))) & set(next(iter(r_vals.values())))
            if not keys:
                continue
            a_vals = {mk: {key: v[key] for key in keys} for mk, v in a_vals.items()}
            r_vals = {mk: {key: v[key] for key in keys} for mk, v in r_vals.items()}
        cmp = {}
        for m in spec["metrics"]:
            c = compare_rows(a_vals[m["key"]], r_vals[m["key"]])
            if m["better"] == "lower":
                c["positive_folds"] = sum(1 for x in c["fold_diffs"] if x < 0)
            c["better"] = m["better"]
            cmp[m["key"]] = c
        rows_out[row]["vs_reference"] = cmp
        rows_out[row]["reference"] = rr
    return {
        "table": spec["id"], "title": spec.get("title", ""), "task": spec["task"],
        "split": spec["split"], "seeds": spec["seeds"], "folds": folds,
        "metrics": spec["metrics"], "reference_row": ref, "ddof": DDOF,
        "complete": not missing, "missing_runs": missing,
        "provenance": prov,
        "common_test_recordings": {str(k): v for k, v in common.items()},
        "rows": rows_out,
    }


# ---------------------------------------------------------------------------
# LaTeX
# ---------------------------------------------------------------------------

def tex_escape(s: str) -> str:
    if "\\" in s or "$" in s:
        return s  # already TeX
    for a, b in (("&", r"\&"), ("%", r"\%"), ("#", r"\#"), ("_", r"\_")):
        s = s.replace(a, b)
    return s


def _header(summary: dict, what: str) -> List[str]:
    lines = [f"% Generated by scripts/aggregate.py for table {summary['table']} -- do not edit.",
             f"% {summary['title']}",
             f"% {what}",
             f"% mean +- SD (ddof={summary['ddof']}) over {len(summary['folds'])} fold means; each "
             f"fold = mean over seeds {summary['seeds']}.",
             f"% git {summary['provenance'].get('git_sha')}  data {summary['provenance'].get('data_tag')}"
             f"  split md5 {summary['provenance'].get('split_md5')}"]
    if not summary["complete"]:
        lines.append(f"% INCOMPLETE: {len(summary['missing_runs'])} run(s) missing: "
                     + ", ".join(summary["missing_runs"][:20]))
    return lines


def render_table(summary: dict, which: str = "own") -> str:
    ms = summary["metrics"]
    what = ("own test set per row" if which == "own" else
            "common test set (recordings loaded by every row), per fold: "
            + ", ".join(f"{k}:{len(v)}" for k, v in summary["common_test_recordings"].items()))
    lines = _header(summary, what)
    lines += [r"\begin{tabular}{l" + "c" * len(ms) + "}", r"\toprule",
              "Method & " + " & ".join(tex_escape(m["label"]) for m in ms) + r" \\", r"\midrule"]
    for row, r in summary["rows"].items():
        cells = [format_pm(r[which][m["key"]]["mean"], r[which][m["key"]]["sd"], m["digits"])
                 for m in ms]
        lines.append(tex_escape(r["label"]) + " & " + " & ".join(cells) + r" \\")
    lines += [r"\bottomrule", r"\end{tabular}", ""]
    return "\n".join(lines)


def render_paired(summary: dict) -> str:
    ref = summary["reference_row"]
    ms = summary["metrics"]
    own_refs = any(r.get("reference", ref) != ref for r in summary["rows"].values()
                   if "vs_reference" in r)
    what = ("each row's own reference row (named after 'vs')" if own_refs
            else f"reference row {ref!r}")
    lines = _header(summary, f"paired comparison with {what} on the common test set; "
                             "delta = row - reference; positive folds = folds where the row is better")
    lines += [r"\begin{tabular}{l" + "cccc" * len(ms) + "}", r"\toprule",
              "Method & " + " & ".join(
                  rf"$\Delta$ {tex_escape(m['label'])} & pos. folds & $p_{{5}}$ & $p_{{15}}$"
                  for m in ms) + r" \\", r"\midrule"]
    for row, r in summary["rows"].items():
        if "vs_reference" not in r:
            continue
        cells = []
        for m in ms:
            c = r["vs_reference"][m["key"]]
            t15 = c.get("t15")
            cells += [format_pm(c["delta_mean"], c["delta_sd"], m["digits"]),
                      f"{c['positive_folds']}/{len(c['folds'])}",
                      format_p(c["t5"]["p"]), format_p(t15["p"]) if t15 else "--"]
        label = r["label"]
        if own_refs:
            label += " vs " + summary["rows"][r["reference"]]["label"]
        lines.append(tex_escape(label) + " & " + " & ".join(cells) + r" \\")
    lines += [r"\bottomrule", r"\end{tabular}", ""]
    return "\n".join(lines)


def _num(v: Optional[float], fmt: str) -> str:
    """``format(v, fmt)``, or ``--`` for a missing / NaN value (e.g. an SD over one fold)."""
    return "--" if v is None or v != v else format(v, fmt)


def number_entries(summary: dict) -> Dict[str, str]:
    """``{"<table>/<row>/<metric>/<stat>": "0.626"}`` for numbers.tex."""
    out: Dict[str, str] = {}
    t = summary["table"]
    for row, r in summary["rows"].items():
        for m in summary["metrics"]:
            key, d = m["key"], m["digits"]
            base = f"{t}/{row}/{key}"
            for which, pre in (("own", ""), ("common", "c")):
                st = r[which][key]
                out[f"{base}/{pre}mean"] = _num(st["mean"], f".{d}f")
                out[f"{base}/{pre}sd"] = _num(st["sd"], f".{d}f")
                out[f"{base}/{pre}pm"] = format_pm(st["mean"], st["sd"], d)
            if "vs_reference" in r:
                c = r["vs_reference"][key]
                out[f"{base}/delta"] = _num(c["delta_mean"], f"+.{d}f")
                out[f"{base}/deltasd"] = _num(c["delta_sd"], f".{d}f")
                out[f"{base}/pos"] = f"{c['positive_folds']}/{len(c['folds'])}"
                out[f"{base}/p5"] = format_p(c["t5"]["p"])
                out[f"{base}/t5"] = _num(c["t5"]["t"], ".2f")
                if c.get("t15"):
                    out[f"{base}/p15"] = format_p(c["t15"]["p"])
                    out[f"{base}/t15"] = _num(c["t15"]["t"], ".2f")
        out[f"{t}/{row}/ntest"] = str(sum(r["n_test_recordings"].values()))
        if r.get("n_params"):
            out[f"{t}/{row}/params"] = f"{r['n_params'] / 1e6:.1f}M"
    out[f"{t}/ncommon"] = str(sum(len(v) for v in summary["common_test_recordings"].values()))
    return out


NUMBERS_PREAMBLE = r"""% Generated by scripts/aggregate.py -- do not edit.
% Usage in the paper: \pulsenum{<table>/<row>/<metric>/<stat>}, e.g. \pulsenum{3/syncfuse/macro_f1/pm}
% stats: mean sd pm (own test set), cmean csd cpm (common test set), delta deltasd pos p5 t5 p15 t15
% (vs the reference row, common test set), ntest, params; per table: ncommon.
\providecommand{\pulsenum}[1]{\ifcsname pulse@#1\endcsname\csname pulse@#1\endcsname\else
  \textbf{??}\PackageWarning{pulsenum}{undefined number \detokenize{#1}}\fi}
"""


def render_numbers(summaries: List[dict]) -> str:
    lines = [NUMBERS_PREAMBLE]
    for s in sorted(summaries, key=lambda x: x["table"]):
        lines.append(f"% ---- table {s['table']}: {s['title']}"
                     + ("" if s["complete"] else "  [INCOMPLETE]"))
        for k, v in sorted(number_entries(s).items()):
            lines.append(r"\expandafter\def\csname pulse@" + k + r"\endcsname{" + v + "}")
    return "\n".join(lines) + "\n"


def write_outputs(summary: dict, out_dir) -> List[Path]:
    out = Path(out_dir)
    out.mkdir(parents=True, exist_ok=True)
    t = summary["table"]
    files = {f"tab{t}.tex": render_table(summary, "own"),
             f"tab{t}_common.tex": render_table(summary, "common"),
             f"tab{t}.json": json.dumps(summary, indent=1) + "\n"}
    if any("vs_reference" in r for r in summary["rows"].values()):
        files[f"tab{t}_paired.tex"] = render_paired(summary)
    written = []
    for name, text in files.items():
        (out / name).write_text(text, encoding="utf-8", newline="\n")
        written.append(out / name)
    # only table summaries (tab<id>.json); reports such as t3_report's tab6_folds.json share the prefix
    summaries = []
    for p in sorted(out.glob("tab*.json")):
        d = json.loads(p.read_text(encoding="utf-8"))
        if isinstance(d, dict) and p.name == f"tab{d.get('table')}.json" and "rows" in d \
                and "metrics" in d:
            summaries.append(d)
    (out / "numbers.tex").write_text(render_numbers(summaries), encoding="utf-8", newline="\n")
    written.append(out / "numbers.tex")
    return written