PULSE-code / pulse /aggregate.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
18.8 kB
"""Turn the runs of one table into LaTeX tables and number macros.
Rules (see also ``pulse.stats``):
* every run's metrics are recomputed from its per-recording statistics and
must equal the stored test metrics;
* runs are merged only if their provenance agrees (data tag, split md5,
taxonomy md5, availability md5, code version and, by default, git sha),
all runs of a row used the same configuration including ``--set``
overrides (``config_md5``; a leftover smoke-test run is refused), and all
runs of a table ran on the same kind of device with the same torch version
(``cpu`` or ``cuda:<GPU model>``; CPU and GPU results are not spliced);
* a row's value = mean over seeds within fold, then mean +- SD (ddof=1) over
the fold means, on the row's own test recordings (``own``) and on the common
test set (``common``: per fold, the recordings every row of the table loaded);
* comparisons with the reference row (the table's ``reference_row``, or a row's
own ``reference``) use the common test set: paired t over
fold means (n=5) and over fold x seed pairs (n=15), and positive-fold counts
(a fold counts as positive when the row improves on the reference).
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import yaml
from .errors import ProvenanceError
from .metrics import compute_metrics
from .provenance import check_consistent, load_results
from .splits import load_split
from .stats import DDOF, compare_rows, fold_means, format_p, format_pm, mean_sd
REPO_ROOT = Path(__file__).resolve().parents[1]
RunKey = Tuple[int, int] # (fold, seed)
def load_registry(path=None) -> Dict[str, dict]:
p = Path(path) if path else REPO_ROOT / "configs" / "tables.yaml"
with open(p, encoding="utf-8") as f:
return yaml.safe_load(f)["tables"]
def table_spec(registry: Dict[str, dict], table: str) -> dict:
if str(table) not in registry:
raise KeyError(f"table {table!r} not in registry; known: {sorted(registry)}")
spec = dict(registry[str(table)])
spec["id"] = str(table)
if "folds" not in spec:
spec["folds"] = list(range(load_split(REPO_ROOT / spec["split"]).n_folds))
for m in spec["metrics"]:
m.setdefault("digits", 3)
m.setdefault("better", "higher")
m.setdefault("label", m["key"])
return spec
def run_dir(runs_root, table: str, row: str, fold: int, seed: int) -> Path:
return Path(runs_root) / str(table) / row / f"fold{fold}_seed{seed}"
def row_source(spec: dict, row: dict) -> Tuple[str, str]:
"""(table, row) whose runs ``row`` reads: its own, or those named by
``reuse: "<table>/<row>"`` (the row is then not run again, see run_table.py)."""
src = row.get("reuse")
if not src:
return spec["id"], row["key"]
t, sep, r = str(src).partition("/")
if not (sep and t and r):
raise ValueError(f"row {row['key']}: reuse must be '<table>/<row>', got {src!r}")
return t, r
def collect(spec: dict, runs_root) -> Tuple[Dict[str, Dict[RunKey, dict]], List[str]]:
runs: Dict[str, Dict[RunKey, dict]] = {}
missing: List[str] = []
for row in spec["rows"]:
runs[row["key"]] = {}
src_table, src_row = row_source(spec, row)
for k in spec["folds"]:
for s in spec["seeds"]:
p = run_dir(runs_root, src_table, src_row, k, s) / "results.json"
if p.is_file():
runs[row["key"]][(k, s)] = load_results(p)
else:
missing.append(f"{row['key']}/fold{k}_seed{s}")
return runs, missing
def _check_run(name: str, r: dict, spec: dict, fold: int, seed: int, tol: float) -> Dict[str, float]:
prov = r["provenance"]
if r["task"] != spec["task"]:
raise ProvenanceError(f"{name}: task {r['task']!r} != table task {spec['task']!r}")
if prov["fold"] != fold or prov["seed"] != seed:
raise ProvenanceError(f"{name}: provenance says fold {prov['fold']} seed {prov['seed']}")
own = compute_metrics(r)
for k, v in r["metrics"].get("test", {}).items():
if k in own and abs(own[k] - v) > tol * max(1.0, abs(v)):
raise ProvenanceError(f"{name}: stored test {k}={v} != recomputed {own[k]}")
for m in spec["metrics"]:
if m["key"] not in own:
raise ProvenanceError(f"{name}: metric {m['key']!r} not defined in results.json")
return own
def compute_kind(prov: dict) -> str:
"""Hardware / software a run used: ``cpu`` or ``cuda:<GPU model>``, plus the torch version."""
d = prov.get("device")
if not isinstance(d, dict):
return json.dumps(d)
dev = str(d.get("device") or "")
kind = f"cuda:{d.get('gpu_name')}" if dev.startswith("cuda") else (dev or "unknown")
return f"{kind} torch {d.get('torch')}"
def check_runs_comparable(runs: Dict[str, Dict[RunKey, dict]],
allow_mixed_device: bool = False) -> List[str]:
"""Every run of a row must use the same configuration (config file + all
``--set`` overrides; ``config_md5``), so e.g. a 1-epoch smoke run left in
the runs directory cannot be averaged with full runs; and every run of a
table must have used the same kind of device and torch version (CPU and GPU
results are not spliced into one table). Returns the device kinds seen."""
for row, d in runs.items():
md5s = {}
for (k, s), r in d.items():
md5s.setdefault(str(r["provenance"].get("config_md5")), []).append(f"fold{k}_seed{s}")
if len(md5s) > 1:
detail = "; ".join(f"config_md5 {m}: {', '.join(v[:3])}" for m, v in md5s.items())
raise ProvenanceError(f"row {row}: runs used different configurations or overrides "
f"({detail}); rerun them with one configuration")
kinds: Dict[str, List[str]] = {}
for row, d in runs.items():
for (k, s), r in d.items():
kinds.setdefault(compute_kind(r["provenance"]), []).append(f"{row}/fold{k}_seed{s}")
if len(kinds) > 1 and not allow_mixed_device:
detail = "; ".join(f"{kd}: {len(v)} run(s), e.g. {v[0]}" for kd, v in kinds.items())
raise ProvenanceError(f"runs were produced on different devices / torch versions ({detail}); "
"rerun them on one device type (or pass allow_mixed_device)")
return sorted(kinds)
def _row_stats(values: Dict[RunKey, float]) -> Dict[str, Any]:
fm = fold_means(values)
mean, sd = mean_sd(list(fm.values()), DDOF)
return {"per_run": {f"{k}_{s}": v for (k, s), v in sorted(values.items())},
"fold_means": {str(k): v for k, v in fm.items()}, "mean": mean, "sd": sd,
"n_folds": len(fm), "n_runs": len(values)}
def aggregate(spec: dict, runs: Dict[str, Dict[RunKey, dict]], *, missing: Optional[List[str]] = None,
allow_incomplete: bool = False, require_same_sha: bool = True,
allow_dirty: bool = False, allow_mixed_device: bool = False,
tol: float = 1e-9) -> Dict[str, Any]:
missing = list(missing or [])
if missing and not allow_incomplete:
raise ProvenanceError(f"table {spec['id']}: {len(missing)} run(s) missing, e.g. "
f"{missing[:5]} (use --allow-incomplete for a partial table)")
flat = {f"{row}/fold{k}_seed{s}": r for row, d in runs.items() for (k, s), r in d.items()}
if not flat:
raise ProvenanceError(f"table {spec['id']}: no runs found")
prov = check_consistent(flat, require_same_sha=require_same_sha, allow_dirty=allow_dirty)
prov["compute"] = " | ".join(check_runs_comparable(runs, allow_mixed_device=allow_mixed_device))
own: Dict[str, Dict[RunKey, Dict[str, float]]] = {}
test_sets: Dict[str, Dict[int, frozenset]] = {}
for row, d in runs.items():
own[row], test_sets[row] = {}, {}
for (k, s), r in d.items():
own[row][(k, s)] = _check_run(f"{row}/fold{k}_seed{s}", r, spec, k, s, tol)
ts = frozenset(r["provenance"]["loaded_recordings"]["test"])
if k in test_sets[row] and test_sets[row][k] != ts:
raise ProvenanceError(f"{row} fold {k}: seeds loaded different test recordings")
test_sets[row][k] = ts
folds = sorted({k for d in runs.values() for (k, _s) in d})
common: Dict[int, List[str]] = {}
for k in folds:
sets = [test_sets[row][k] for row in runs if k in test_sets[row]]
inter = frozenset.intersection(*sets) if sets else frozenset()
if not inter:
raise ProvenanceError(f"fold {k}: the rows share no test recording")
common[k] = sorted(inter)
rows_out: Dict[str, Any] = {}
common_vals: Dict[str, Dict[str, Dict[RunKey, float]]] = {}
for row, d in runs.items():
cv = {(k, s): compute_metrics(r, recordings=common[k]) for (k, s), r in d.items()}
common_vals[row] = {m["key"]: {key: v[m["key"]] for key, v in cv.items()}
for m in spec["metrics"]}
label = next(x["label"] for x in spec["rows"] if x["key"] == row)
rows_out[row] = {
"label": label,
"own": {m["key"]: _row_stats({key: v[m["key"]] for key, v in own[row].items()})
for m in spec["metrics"]},
"common": {m["key"]: _row_stats(common_vals[row][m["key"]]) for m in spec["metrics"]},
"n_test_recordings": {str(k): len(v) for k, v in sorted(test_sets[row].items())},
"n_params": next((r.get("n_params") for r in d.values() if r.get("n_params")), None),
}
ref = spec.get("reference_row")
# a row may name its own reference (``reference:`` in the registry), e.g. F5
# compares each pre-trained row with the scratch row of the same fraction
row_ref = {x["key"]: x.get("reference", ref) for x in spec["rows"]}
for row in runs:
rr = row_ref.get(row)
if not rr or rr == row or rr not in runs or not runs[rr] or not runs[row]:
continue # no reference, or a row without any run (only with allow_incomplete)
a_vals, r_vals = common_vals[row], common_vals[rr]
if allow_incomplete:
# partial table: compare on the (fold, seed) runs both rows have
keys = set(next(iter(a_vals.values()))) & set(next(iter(r_vals.values())))
if not keys:
continue
a_vals = {mk: {key: v[key] for key in keys} for mk, v in a_vals.items()}
r_vals = {mk: {key: v[key] for key in keys} for mk, v in r_vals.items()}
cmp = {}
for m in spec["metrics"]:
c = compare_rows(a_vals[m["key"]], r_vals[m["key"]])
if m["better"] == "lower":
c["positive_folds"] = sum(1 for x in c["fold_diffs"] if x < 0)
c["better"] = m["better"]
cmp[m["key"]] = c
rows_out[row]["vs_reference"] = cmp
rows_out[row]["reference"] = rr
return {
"table": spec["id"], "title": spec.get("title", ""), "task": spec["task"],
"split": spec["split"], "seeds": spec["seeds"], "folds": folds,
"metrics": spec["metrics"], "reference_row": ref, "ddof": DDOF,
"complete": not missing, "missing_runs": missing,
"provenance": prov,
"common_test_recordings": {str(k): v for k, v in common.items()},
"rows": rows_out,
}
# ---------------------------------------------------------------------------
# LaTeX
# ---------------------------------------------------------------------------
def tex_escape(s: str) -> str:
if "\\" in s or "$" in s:
return s # already TeX
for a, b in (("&", r"\&"), ("%", r"\%"), ("#", r"\#"), ("_", r"\_")):
s = s.replace(a, b)
return s
def _header(summary: dict, what: str) -> List[str]:
lines = [f"% Generated by scripts/aggregate.py for table {summary['table']} -- do not edit.",
f"% {summary['title']}",
f"% {what}",
f"% mean +- SD (ddof={summary['ddof']}) over {len(summary['folds'])} fold means; each "
f"fold = mean over seeds {summary['seeds']}.",
f"% git {summary['provenance'].get('git_sha')} data {summary['provenance'].get('data_tag')}"
f" split md5 {summary['provenance'].get('split_md5')}"]
if not summary["complete"]:
lines.append(f"% INCOMPLETE: {len(summary['missing_runs'])} run(s) missing: "
+ ", ".join(summary["missing_runs"][:20]))
return lines
def render_table(summary: dict, which: str = "own") -> str:
ms = summary["metrics"]
what = ("own test set per row" if which == "own" else
"common test set (recordings loaded by every row), per fold: "
+ ", ".join(f"{k}:{len(v)}" for k, v in summary["common_test_recordings"].items()))
lines = _header(summary, what)
lines += [r"\begin{tabular}{l" + "c" * len(ms) + "}", r"\toprule",
"Method & " + " & ".join(tex_escape(m["label"]) for m in ms) + r" \\", r"\midrule"]
for row, r in summary["rows"].items():
cells = [format_pm(r[which][m["key"]]["mean"], r[which][m["key"]]["sd"], m["digits"])
for m in ms]
lines.append(tex_escape(r["label"]) + " & " + " & ".join(cells) + r" \\")
lines += [r"\bottomrule", r"\end{tabular}", ""]
return "\n".join(lines)
def render_paired(summary: dict) -> str:
ref = summary["reference_row"]
ms = summary["metrics"]
own_refs = any(r.get("reference", ref) != ref for r in summary["rows"].values()
if "vs_reference" in r)
what = ("each row's own reference row (named after 'vs')" if own_refs
else f"reference row {ref!r}")
lines = _header(summary, f"paired comparison with {what} on the common test set; "
"delta = row - reference; positive folds = folds where the row is better")
lines += [r"\begin{tabular}{l" + "cccc" * len(ms) + "}", r"\toprule",
"Method & " + " & ".join(
rf"$\Delta$ {tex_escape(m['label'])} & pos. folds & $p_{{5}}$ & $p_{{15}}$"
for m in ms) + r" \\", r"\midrule"]
for row, r in summary["rows"].items():
if "vs_reference" not in r:
continue
cells = []
for m in ms:
c = r["vs_reference"][m["key"]]
t15 = c.get("t15")
cells += [format_pm(c["delta_mean"], c["delta_sd"], m["digits"]),
f"{c['positive_folds']}/{len(c['folds'])}",
format_p(c["t5"]["p"]), format_p(t15["p"]) if t15 else "--"]
label = r["label"]
if own_refs:
label += " vs " + summary["rows"][r["reference"]]["label"]
lines.append(tex_escape(label) + " & " + " & ".join(cells) + r" \\")
lines += [r"\bottomrule", r"\end{tabular}", ""]
return "\n".join(lines)
def _num(v: Optional[float], fmt: str) -> str:
"""``format(v, fmt)``, or ``--`` for a missing / NaN value (e.g. an SD over one fold)."""
return "--" if v is None or v != v else format(v, fmt)
def number_entries(summary: dict) -> Dict[str, str]:
"""``{"<table>/<row>/<metric>/<stat>": "0.626"}`` for numbers.tex."""
out: Dict[str, str] = {}
t = summary["table"]
for row, r in summary["rows"].items():
for m in summary["metrics"]:
key, d = m["key"], m["digits"]
base = f"{t}/{row}/{key}"
for which, pre in (("own", ""), ("common", "c")):
st = r[which][key]
out[f"{base}/{pre}mean"] = _num(st["mean"], f".{d}f")
out[f"{base}/{pre}sd"] = _num(st["sd"], f".{d}f")
out[f"{base}/{pre}pm"] = format_pm(st["mean"], st["sd"], d)
if "vs_reference" in r:
c = r["vs_reference"][key]
out[f"{base}/delta"] = _num(c["delta_mean"], f"+.{d}f")
out[f"{base}/deltasd"] = _num(c["delta_sd"], f".{d}f")
out[f"{base}/pos"] = f"{c['positive_folds']}/{len(c['folds'])}"
out[f"{base}/p5"] = format_p(c["t5"]["p"])
out[f"{base}/t5"] = _num(c["t5"]["t"], ".2f")
if c.get("t15"):
out[f"{base}/p15"] = format_p(c["t15"]["p"])
out[f"{base}/t15"] = _num(c["t15"]["t"], ".2f")
out[f"{t}/{row}/ntest"] = str(sum(r["n_test_recordings"].values()))
if r.get("n_params"):
out[f"{t}/{row}/params"] = f"{r['n_params'] / 1e6:.1f}M"
out[f"{t}/ncommon"] = str(sum(len(v) for v in summary["common_test_recordings"].values()))
return out
NUMBERS_PREAMBLE = r"""% Generated by scripts/aggregate.py -- do not edit.
% Usage in the paper: \pulsenum{<table>/<row>/<metric>/<stat>}, e.g. \pulsenum{3/syncfuse/macro_f1/pm}
% stats: mean sd pm (own test set), cmean csd cpm (common test set), delta deltasd pos p5 t5 p15 t15
% (vs the reference row, common test set), ntest, params; per table: ncommon.
\providecommand{\pulsenum}[1]{\ifcsname pulse@#1\endcsname\csname pulse@#1\endcsname\else
\textbf{??}\PackageWarning{pulsenum}{undefined number \detokenize{#1}}\fi}
"""
def render_numbers(summaries: List[dict]) -> str:
lines = [NUMBERS_PREAMBLE]
for s in sorted(summaries, key=lambda x: x["table"]):
lines.append(f"% ---- table {s['table']}: {s['title']}"
+ ("" if s["complete"] else " [INCOMPLETE]"))
for k, v in sorted(number_entries(s).items()):
lines.append(r"\expandafter\def\csname pulse@" + k + r"\endcsname{" + v + "}")
return "\n".join(lines) + "\n"
def write_outputs(summary: dict, out_dir) -> List[Path]:
out = Path(out_dir)
out.mkdir(parents=True, exist_ok=True)
t = summary["table"]
files = {f"tab{t}.tex": render_table(summary, "own"),
f"tab{t}_common.tex": render_table(summary, "common"),
f"tab{t}.json": json.dumps(summary, indent=1) + "\n"}
if any("vs_reference" in r for r in summary["rows"].values()):
files[f"tab{t}_paired.tex"] = render_paired(summary)
written = []
for name, text in files.items():
(out / name).write_text(text, encoding="utf-8", newline="\n")
written.append(out / name)
# only table summaries (tab<id>.json); reports such as t3_report's tab6_folds.json share the prefix
summaries = []
for p in sorted(out.glob("tab*.json")):
d = json.loads(p.read_text(encoding="utf-8"))
if isinstance(d, dict) and p.name == f"tab{d.get('table')}.json" and "rows" in d \
and "metrics" in d:
summaries.append(d)
(out / "numbers.tex").write_text(render_numbers(summaries), encoding="utf-8", newline="\n")
written.append(out / "numbers.tex")
return written