"""Turn the runs of one table into LaTeX tables and number macros. Rules (see also ``pulse.stats``): * every run's metrics are recomputed from its per-recording statistics and must equal the stored test metrics; * runs are merged only if their provenance agrees (data tag, split md5, taxonomy md5, availability md5, code version and, by default, git sha), all runs of a row used the same configuration including ``--set`` overrides (``config_md5``; a leftover smoke-test run is refused), and all runs of a table ran on the same kind of device with the same torch version (``cpu`` or ``cuda:``; CPU and GPU results are not spliced); * a row's value = mean over seeds within fold, then mean +- SD (ddof=1) over the fold means, on the row's own test recordings (``own``) and on the common test set (``common``: per fold, the recordings every row of the table loaded); * comparisons with the reference row (the table's ``reference_row``, or a row's own ``reference``) use the common test set: paired t over fold means (n=5) and over fold x seed pairs (n=15), and positive-fold counts (a fold counts as positive when the row improves on the reference). """ from __future__ import annotations import json from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import yaml from .errors import ProvenanceError from .metrics import compute_metrics from .provenance import check_consistent, load_results from .splits import load_split from .stats import DDOF, compare_rows, fold_means, format_p, format_pm, mean_sd REPO_ROOT = Path(__file__).resolve().parents[1] RunKey = Tuple[int, int] # (fold, seed) def load_registry(path=None) -> Dict[str, dict]: p = Path(path) if path else REPO_ROOT / "configs" / "tables.yaml" with open(p, encoding="utf-8") as f: return yaml.safe_load(f)["tables"] def table_spec(registry: Dict[str, dict], table: str) -> dict: if str(table) not in registry: raise KeyError(f"table {table!r} not in registry; known: {sorted(registry)}") spec = dict(registry[str(table)]) spec["id"] = str(table) if "folds" not in spec: spec["folds"] = list(range(load_split(REPO_ROOT / spec["split"]).n_folds)) for m in spec["metrics"]: m.setdefault("digits", 3) m.setdefault("better", "higher") m.setdefault("label", m["key"]) return spec def run_dir(runs_root, table: str, row: str, fold: int, seed: int) -> Path: return Path(runs_root) / str(table) / row / f"fold{fold}_seed{seed}" def row_source(spec: dict, row: dict) -> Tuple[str, str]: """(table, row) whose runs ``row`` reads: its own, or those named by ``reuse: "/"`` (the row is then not run again, see run_table.py).""" src = row.get("reuse") if not src: return spec["id"], row["key"] t, sep, r = str(src).partition("/") if not (sep and t and r): raise ValueError(f"row {row['key']}: reuse must be '
/', got {src!r}") return t, r def collect(spec: dict, runs_root) -> Tuple[Dict[str, Dict[RunKey, dict]], List[str]]: runs: Dict[str, Dict[RunKey, dict]] = {} missing: List[str] = [] for row in spec["rows"]: runs[row["key"]] = {} src_table, src_row = row_source(spec, row) for k in spec["folds"]: for s in spec["seeds"]: p = run_dir(runs_root, src_table, src_row, k, s) / "results.json" if p.is_file(): runs[row["key"]][(k, s)] = load_results(p) else: missing.append(f"{row['key']}/fold{k}_seed{s}") return runs, missing def _check_run(name: str, r: dict, spec: dict, fold: int, seed: int, tol: float) -> Dict[str, float]: prov = r["provenance"] if r["task"] != spec["task"]: raise ProvenanceError(f"{name}: task {r['task']!r} != table task {spec['task']!r}") if prov["fold"] != fold or prov["seed"] != seed: raise ProvenanceError(f"{name}: provenance says fold {prov['fold']} seed {prov['seed']}") own = compute_metrics(r) for k, v in r["metrics"].get("test", {}).items(): if k in own and abs(own[k] - v) > tol * max(1.0, abs(v)): raise ProvenanceError(f"{name}: stored test {k}={v} != recomputed {own[k]}") for m in spec["metrics"]: if m["key"] not in own: raise ProvenanceError(f"{name}: metric {m['key']!r} not defined in results.json") return own def compute_kind(prov: dict) -> str: """Hardware / software a run used: ``cpu`` or ``cuda:``, plus the torch version.""" d = prov.get("device") if not isinstance(d, dict): return json.dumps(d) dev = str(d.get("device") or "") kind = f"cuda:{d.get('gpu_name')}" if dev.startswith("cuda") else (dev or "unknown") return f"{kind} torch {d.get('torch')}" def check_runs_comparable(runs: Dict[str, Dict[RunKey, dict]], allow_mixed_device: bool = False) -> List[str]: """Every run of a row must use the same configuration (config file + all ``--set`` overrides; ``config_md5``), so e.g. a 1-epoch smoke run left in the runs directory cannot be averaged with full runs; and every run of a table must have used the same kind of device and torch version (CPU and GPU results are not spliced into one table). Returns the device kinds seen.""" for row, d in runs.items(): md5s = {} for (k, s), r in d.items(): md5s.setdefault(str(r["provenance"].get("config_md5")), []).append(f"fold{k}_seed{s}") if len(md5s) > 1: detail = "; ".join(f"config_md5 {m}: {', '.join(v[:3])}" for m, v in md5s.items()) raise ProvenanceError(f"row {row}: runs used different configurations or overrides " f"({detail}); rerun them with one configuration") kinds: Dict[str, List[str]] = {} for row, d in runs.items(): for (k, s), r in d.items(): kinds.setdefault(compute_kind(r["provenance"]), []).append(f"{row}/fold{k}_seed{s}") if len(kinds) > 1 and not allow_mixed_device: detail = "; ".join(f"{kd}: {len(v)} run(s), e.g. {v[0]}" for kd, v in kinds.items()) raise ProvenanceError(f"runs were produced on different devices / torch versions ({detail}); " "rerun them on one device type (or pass allow_mixed_device)") return sorted(kinds) def _row_stats(values: Dict[RunKey, float]) -> Dict[str, Any]: fm = fold_means(values) mean, sd = mean_sd(list(fm.values()), DDOF) return {"per_run": {f"{k}_{s}": v for (k, s), v in sorted(values.items())}, "fold_means": {str(k): v for k, v in fm.items()}, "mean": mean, "sd": sd, "n_folds": len(fm), "n_runs": len(values)} def aggregate(spec: dict, runs: Dict[str, Dict[RunKey, dict]], *, missing: Optional[List[str]] = None, allow_incomplete: bool = False, require_same_sha: bool = True, allow_dirty: bool = False, allow_mixed_device: bool = False, tol: float = 1e-9) -> Dict[str, Any]: missing = list(missing or []) if missing and not allow_incomplete: raise ProvenanceError(f"table {spec['id']}: {len(missing)} run(s) missing, e.g. " f"{missing[:5]} (use --allow-incomplete for a partial table)") flat = {f"{row}/fold{k}_seed{s}": r for row, d in runs.items() for (k, s), r in d.items()} if not flat: raise ProvenanceError(f"table {spec['id']}: no runs found") prov = check_consistent(flat, require_same_sha=require_same_sha, allow_dirty=allow_dirty) prov["compute"] = " | ".join(check_runs_comparable(runs, allow_mixed_device=allow_mixed_device)) own: Dict[str, Dict[RunKey, Dict[str, float]]] = {} test_sets: Dict[str, Dict[int, frozenset]] = {} for row, d in runs.items(): own[row], test_sets[row] = {}, {} for (k, s), r in d.items(): own[row][(k, s)] = _check_run(f"{row}/fold{k}_seed{s}", r, spec, k, s, tol) ts = frozenset(r["provenance"]["loaded_recordings"]["test"]) if k in test_sets[row] and test_sets[row][k] != ts: raise ProvenanceError(f"{row} fold {k}: seeds loaded different test recordings") test_sets[row][k] = ts folds = sorted({k for d in runs.values() for (k, _s) in d}) common: Dict[int, List[str]] = {} for k in folds: sets = [test_sets[row][k] for row in runs if k in test_sets[row]] inter = frozenset.intersection(*sets) if sets else frozenset() if not inter: raise ProvenanceError(f"fold {k}: the rows share no test recording") common[k] = sorted(inter) rows_out: Dict[str, Any] = {} common_vals: Dict[str, Dict[str, Dict[RunKey, float]]] = {} for row, d in runs.items(): cv = {(k, s): compute_metrics(r, recordings=common[k]) for (k, s), r in d.items()} common_vals[row] = {m["key"]: {key: v[m["key"]] for key, v in cv.items()} for m in spec["metrics"]} label = next(x["label"] for x in spec["rows"] if x["key"] == row) rows_out[row] = { "label": label, "own": {m["key"]: _row_stats({key: v[m["key"]] for key, v in own[row].items()}) for m in spec["metrics"]}, "common": {m["key"]: _row_stats(common_vals[row][m["key"]]) for m in spec["metrics"]}, "n_test_recordings": {str(k): len(v) for k, v in sorted(test_sets[row].items())}, "n_params": next((r.get("n_params") for r in d.values() if r.get("n_params")), None), } ref = spec.get("reference_row") # a row may name its own reference (``reference:`` in the registry), e.g. F5 # compares each pre-trained row with the scratch row of the same fraction row_ref = {x["key"]: x.get("reference", ref) for x in spec["rows"]} for row in runs: rr = row_ref.get(row) if not rr or rr == row or rr not in runs or not runs[rr] or not runs[row]: continue # no reference, or a row without any run (only with allow_incomplete) a_vals, r_vals = common_vals[row], common_vals[rr] if allow_incomplete: # partial table: compare on the (fold, seed) runs both rows have keys = set(next(iter(a_vals.values()))) & set(next(iter(r_vals.values()))) if not keys: continue a_vals = {mk: {key: v[key] for key in keys} for mk, v in a_vals.items()} r_vals = {mk: {key: v[key] for key in keys} for mk, v in r_vals.items()} cmp = {} for m in spec["metrics"]: c = compare_rows(a_vals[m["key"]], r_vals[m["key"]]) if m["better"] == "lower": c["positive_folds"] = sum(1 for x in c["fold_diffs"] if x < 0) c["better"] = m["better"] cmp[m["key"]] = c rows_out[row]["vs_reference"] = cmp rows_out[row]["reference"] = rr return { "table": spec["id"], "title": spec.get("title", ""), "task": spec["task"], "split": spec["split"], "seeds": spec["seeds"], "folds": folds, "metrics": spec["metrics"], "reference_row": ref, "ddof": DDOF, "complete": not missing, "missing_runs": missing, "provenance": prov, "common_test_recordings": {str(k): v for k, v in common.items()}, "rows": rows_out, } # --------------------------------------------------------------------------- # LaTeX # --------------------------------------------------------------------------- def tex_escape(s: str) -> str: if "\\" in s or "$" in s: return s # already TeX for a, b in (("&", r"\&"), ("%", r"\%"), ("#", r"\#"), ("_", r"\_")): s = s.replace(a, b) return s def _header(summary: dict, what: str) -> List[str]: lines = [f"% Generated by scripts/aggregate.py for table {summary['table']} -- do not edit.", f"% {summary['title']}", f"% {what}", f"% mean +- SD (ddof={summary['ddof']}) over {len(summary['folds'])} fold means; each " f"fold = mean over seeds {summary['seeds']}.", f"% git {summary['provenance'].get('git_sha')} data {summary['provenance'].get('data_tag')}" f" split md5 {summary['provenance'].get('split_md5')}"] if not summary["complete"]: lines.append(f"% INCOMPLETE: {len(summary['missing_runs'])} run(s) missing: " + ", ".join(summary["missing_runs"][:20])) return lines def render_table(summary: dict, which: str = "own") -> str: ms = summary["metrics"] what = ("own test set per row" if which == "own" else "common test set (recordings loaded by every row), per fold: " + ", ".join(f"{k}:{len(v)}" for k, v in summary["common_test_recordings"].items())) lines = _header(summary, what) lines += [r"\begin{tabular}{l" + "c" * len(ms) + "}", r"\toprule", "Method & " + " & ".join(tex_escape(m["label"]) for m in ms) + r" \\", r"\midrule"] for row, r in summary["rows"].items(): cells = [format_pm(r[which][m["key"]]["mean"], r[which][m["key"]]["sd"], m["digits"]) for m in ms] lines.append(tex_escape(r["label"]) + " & " + " & ".join(cells) + r" \\") lines += [r"\bottomrule", r"\end{tabular}", ""] return "\n".join(lines) def render_paired(summary: dict) -> str: ref = summary["reference_row"] ms = summary["metrics"] own_refs = any(r.get("reference", ref) != ref for r in summary["rows"].values() if "vs_reference" in r) what = ("each row's own reference row (named after 'vs')" if own_refs else f"reference row {ref!r}") lines = _header(summary, f"paired comparison with {what} on the common test set; " "delta = row - reference; positive folds = folds where the row is better") lines += [r"\begin{tabular}{l" + "cccc" * len(ms) + "}", r"\toprule", "Method & " + " & ".join( rf"$\Delta$ {tex_escape(m['label'])} & pos. folds & $p_{{5}}$ & $p_{{15}}$" for m in ms) + r" \\", r"\midrule"] for row, r in summary["rows"].items(): if "vs_reference" not in r: continue cells = [] for m in ms: c = r["vs_reference"][m["key"]] t15 = c.get("t15") cells += [format_pm(c["delta_mean"], c["delta_sd"], m["digits"]), f"{c['positive_folds']}/{len(c['folds'])}", format_p(c["t5"]["p"]), format_p(t15["p"]) if t15 else "--"] label = r["label"] if own_refs: label += " vs " + summary["rows"][r["reference"]]["label"] lines.append(tex_escape(label) + " & " + " & ".join(cells) + r" \\") lines += [r"\bottomrule", r"\end{tabular}", ""] return "\n".join(lines) def _num(v: Optional[float], fmt: str) -> str: """``format(v, fmt)``, or ``--`` for a missing / NaN value (e.g. an SD over one fold).""" return "--" if v is None or v != v else format(v, fmt) def number_entries(summary: dict) -> Dict[str, str]: """``{"
///": "0.626"}`` for numbers.tex.""" out: Dict[str, str] = {} t = summary["table"] for row, r in summary["rows"].items(): for m in summary["metrics"]: key, d = m["key"], m["digits"] base = f"{t}/{row}/{key}" for which, pre in (("own", ""), ("common", "c")): st = r[which][key] out[f"{base}/{pre}mean"] = _num(st["mean"], f".{d}f") out[f"{base}/{pre}sd"] = _num(st["sd"], f".{d}f") out[f"{base}/{pre}pm"] = format_pm(st["mean"], st["sd"], d) if "vs_reference" in r: c = r["vs_reference"][key] out[f"{base}/delta"] = _num(c["delta_mean"], f"+.{d}f") out[f"{base}/deltasd"] = _num(c["delta_sd"], f".{d}f") out[f"{base}/pos"] = f"{c['positive_folds']}/{len(c['folds'])}" out[f"{base}/p5"] = format_p(c["t5"]["p"]) out[f"{base}/t5"] = _num(c["t5"]["t"], ".2f") if c.get("t15"): out[f"{base}/p15"] = format_p(c["t15"]["p"]) out[f"{base}/t15"] = _num(c["t15"]["t"], ".2f") out[f"{t}/{row}/ntest"] = str(sum(r["n_test_recordings"].values())) if r.get("n_params"): out[f"{t}/{row}/params"] = f"{r['n_params'] / 1e6:.1f}M" out[f"{t}/ncommon"] = str(sum(len(v) for v in summary["common_test_recordings"].values())) return out NUMBERS_PREAMBLE = r"""% Generated by scripts/aggregate.py -- do not edit. % Usage in the paper: \pulsenum{
///}, e.g. \pulsenum{3/syncfuse/macro_f1/pm} % stats: mean sd pm (own test set), cmean csd cpm (common test set), delta deltasd pos p5 t5 p15 t15 % (vs the reference row, common test set), ntest, params; per table: ncommon. \providecommand{\pulsenum}[1]{\ifcsname pulse@#1\endcsname\csname pulse@#1\endcsname\else \textbf{??}\PackageWarning{pulsenum}{undefined number \detokenize{#1}}\fi} """ def render_numbers(summaries: List[dict]) -> str: lines = [NUMBERS_PREAMBLE] for s in sorted(summaries, key=lambda x: x["table"]): lines.append(f"% ---- table {s['table']}: {s['title']}" + ("" if s["complete"] else " [INCOMPLETE]")) for k, v in sorted(number_entries(s).items()): lines.append(r"\expandafter\def\csname pulse@" + k + r"\endcsname{" + v + "}") return "\n".join(lines) + "\n" def write_outputs(summary: dict, out_dir) -> List[Path]: out = Path(out_dir) out.mkdir(parents=True, exist_ok=True) t = summary["table"] files = {f"tab{t}.tex": render_table(summary, "own"), f"tab{t}_common.tex": render_table(summary, "common"), f"tab{t}.json": json.dumps(summary, indent=1) + "\n"} if any("vs_reference" in r for r in summary["rows"].values()): files[f"tab{t}_paired.tex"] = render_paired(summary) written = [] for name, text in files.items(): (out / name).write_text(text, encoding="utf-8", newline="\n") written.append(out / name) # only table summaries (tab.json); reports such as t3_report's tab6_folds.json share the prefix summaries = [] for p in sorted(out.glob("tab*.json")): d = json.loads(p.read_text(encoding="utf-8")) if isinstance(d, dict) and p.name == f"tab{d.get('table')}.json" and "rows" in d \ and "metrics" in d: summaries.append(d) (out / "numbers.tex").write_text(render_numbers(summaries), encoding="utf-8", newline="\n") written.append(out / "numbers.tex") return written