File size: 18,811 Bytes
dffbc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 | """Turn the runs of one table into LaTeX tables and number macros.
Rules (see also ``pulse.stats``):
* every run's metrics are recomputed from its per-recording statistics and
must equal the stored test metrics;
* runs are merged only if their provenance agrees (data tag, split md5,
taxonomy md5, availability md5, code version and, by default, git sha),
all runs of a row used the same configuration including ``--set``
overrides (``config_md5``; a leftover smoke-test run is refused), and all
runs of a table ran on the same kind of device with the same torch version
(``cpu`` or ``cuda:<GPU model>``; CPU and GPU results are not spliced);
* a row's value = mean over seeds within fold, then mean +- SD (ddof=1) over
the fold means, on the row's own test recordings (``own``) and on the common
test set (``common``: per fold, the recordings every row of the table loaded);
* comparisons with the reference row (the table's ``reference_row``, or a row's
own ``reference``) use the common test set: paired t over
fold means (n=5) and over fold x seed pairs (n=15), and positive-fold counts
(a fold counts as positive when the row improves on the reference).
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import yaml
from .errors import ProvenanceError
from .metrics import compute_metrics
from .provenance import check_consistent, load_results
from .splits import load_split
from .stats import DDOF, compare_rows, fold_means, format_p, format_pm, mean_sd
REPO_ROOT = Path(__file__).resolve().parents[1]
RunKey = Tuple[int, int] # (fold, seed)
def load_registry(path=None) -> Dict[str, dict]:
p = Path(path) if path else REPO_ROOT / "configs" / "tables.yaml"
with open(p, encoding="utf-8") as f:
return yaml.safe_load(f)["tables"]
def table_spec(registry: Dict[str, dict], table: str) -> dict:
if str(table) not in registry:
raise KeyError(f"table {table!r} not in registry; known: {sorted(registry)}")
spec = dict(registry[str(table)])
spec["id"] = str(table)
if "folds" not in spec:
spec["folds"] = list(range(load_split(REPO_ROOT / spec["split"]).n_folds))
for m in spec["metrics"]:
m.setdefault("digits", 3)
m.setdefault("better", "higher")
m.setdefault("label", m["key"])
return spec
def run_dir(runs_root, table: str, row: str, fold: int, seed: int) -> Path:
return Path(runs_root) / str(table) / row / f"fold{fold}_seed{seed}"
def row_source(spec: dict, row: dict) -> Tuple[str, str]:
"""(table, row) whose runs ``row`` reads: its own, or those named by
``reuse: "<table>/<row>"`` (the row is then not run again, see run_table.py)."""
src = row.get("reuse")
if not src:
return spec["id"], row["key"]
t, sep, r = str(src).partition("/")
if not (sep and t and r):
raise ValueError(f"row {row['key']}: reuse must be '<table>/<row>', got {src!r}")
return t, r
def collect(spec: dict, runs_root) -> Tuple[Dict[str, Dict[RunKey, dict]], List[str]]:
runs: Dict[str, Dict[RunKey, dict]] = {}
missing: List[str] = []
for row in spec["rows"]:
runs[row["key"]] = {}
src_table, src_row = row_source(spec, row)
for k in spec["folds"]:
for s in spec["seeds"]:
p = run_dir(runs_root, src_table, src_row, k, s) / "results.json"
if p.is_file():
runs[row["key"]][(k, s)] = load_results(p)
else:
missing.append(f"{row['key']}/fold{k}_seed{s}")
return runs, missing
def _check_run(name: str, r: dict, spec: dict, fold: int, seed: int, tol: float) -> Dict[str, float]:
prov = r["provenance"]
if r["task"] != spec["task"]:
raise ProvenanceError(f"{name}: task {r['task']!r} != table task {spec['task']!r}")
if prov["fold"] != fold or prov["seed"] != seed:
raise ProvenanceError(f"{name}: provenance says fold {prov['fold']} seed {prov['seed']}")
own = compute_metrics(r)
for k, v in r["metrics"].get("test", {}).items():
if k in own and abs(own[k] - v) > tol * max(1.0, abs(v)):
raise ProvenanceError(f"{name}: stored test {k}={v} != recomputed {own[k]}")
for m in spec["metrics"]:
if m["key"] not in own:
raise ProvenanceError(f"{name}: metric {m['key']!r} not defined in results.json")
return own
def compute_kind(prov: dict) -> str:
"""Hardware / software a run used: ``cpu`` or ``cuda:<GPU model>``, plus the torch version."""
d = prov.get("device")
if not isinstance(d, dict):
return json.dumps(d)
dev = str(d.get("device") or "")
kind = f"cuda:{d.get('gpu_name')}" if dev.startswith("cuda") else (dev or "unknown")
return f"{kind} torch {d.get('torch')}"
def check_runs_comparable(runs: Dict[str, Dict[RunKey, dict]],
allow_mixed_device: bool = False) -> List[str]:
"""Every run of a row must use the same configuration (config file + all
``--set`` overrides; ``config_md5``), so e.g. a 1-epoch smoke run left in
the runs directory cannot be averaged with full runs; and every run of a
table must have used the same kind of device and torch version (CPU and GPU
results are not spliced into one table). Returns the device kinds seen."""
for row, d in runs.items():
md5s = {}
for (k, s), r in d.items():
md5s.setdefault(str(r["provenance"].get("config_md5")), []).append(f"fold{k}_seed{s}")
if len(md5s) > 1:
detail = "; ".join(f"config_md5 {m}: {', '.join(v[:3])}" for m, v in md5s.items())
raise ProvenanceError(f"row {row}: runs used different configurations or overrides "
f"({detail}); rerun them with one configuration")
kinds: Dict[str, List[str]] = {}
for row, d in runs.items():
for (k, s), r in d.items():
kinds.setdefault(compute_kind(r["provenance"]), []).append(f"{row}/fold{k}_seed{s}")
if len(kinds) > 1 and not allow_mixed_device:
detail = "; ".join(f"{kd}: {len(v)} run(s), e.g. {v[0]}" for kd, v in kinds.items())
raise ProvenanceError(f"runs were produced on different devices / torch versions ({detail}); "
"rerun them on one device type (or pass allow_mixed_device)")
return sorted(kinds)
def _row_stats(values: Dict[RunKey, float]) -> Dict[str, Any]:
fm = fold_means(values)
mean, sd = mean_sd(list(fm.values()), DDOF)
return {"per_run": {f"{k}_{s}": v for (k, s), v in sorted(values.items())},
"fold_means": {str(k): v for k, v in fm.items()}, "mean": mean, "sd": sd,
"n_folds": len(fm), "n_runs": len(values)}
def aggregate(spec: dict, runs: Dict[str, Dict[RunKey, dict]], *, missing: Optional[List[str]] = None,
allow_incomplete: bool = False, require_same_sha: bool = True,
allow_dirty: bool = False, allow_mixed_device: bool = False,
tol: float = 1e-9) -> Dict[str, Any]:
missing = list(missing or [])
if missing and not allow_incomplete:
raise ProvenanceError(f"table {spec['id']}: {len(missing)} run(s) missing, e.g. "
f"{missing[:5]} (use --allow-incomplete for a partial table)")
flat = {f"{row}/fold{k}_seed{s}": r for row, d in runs.items() for (k, s), r in d.items()}
if not flat:
raise ProvenanceError(f"table {spec['id']}: no runs found")
prov = check_consistent(flat, require_same_sha=require_same_sha, allow_dirty=allow_dirty)
prov["compute"] = " | ".join(check_runs_comparable(runs, allow_mixed_device=allow_mixed_device))
own: Dict[str, Dict[RunKey, Dict[str, float]]] = {}
test_sets: Dict[str, Dict[int, frozenset]] = {}
for row, d in runs.items():
own[row], test_sets[row] = {}, {}
for (k, s), r in d.items():
own[row][(k, s)] = _check_run(f"{row}/fold{k}_seed{s}", r, spec, k, s, tol)
ts = frozenset(r["provenance"]["loaded_recordings"]["test"])
if k in test_sets[row] and test_sets[row][k] != ts:
raise ProvenanceError(f"{row} fold {k}: seeds loaded different test recordings")
test_sets[row][k] = ts
folds = sorted({k for d in runs.values() for (k, _s) in d})
common: Dict[int, List[str]] = {}
for k in folds:
sets = [test_sets[row][k] for row in runs if k in test_sets[row]]
inter = frozenset.intersection(*sets) if sets else frozenset()
if not inter:
raise ProvenanceError(f"fold {k}: the rows share no test recording")
common[k] = sorted(inter)
rows_out: Dict[str, Any] = {}
common_vals: Dict[str, Dict[str, Dict[RunKey, float]]] = {}
for row, d in runs.items():
cv = {(k, s): compute_metrics(r, recordings=common[k]) for (k, s), r in d.items()}
common_vals[row] = {m["key"]: {key: v[m["key"]] for key, v in cv.items()}
for m in spec["metrics"]}
label = next(x["label"] for x in spec["rows"] if x["key"] == row)
rows_out[row] = {
"label": label,
"own": {m["key"]: _row_stats({key: v[m["key"]] for key, v in own[row].items()})
for m in spec["metrics"]},
"common": {m["key"]: _row_stats(common_vals[row][m["key"]]) for m in spec["metrics"]},
"n_test_recordings": {str(k): len(v) for k, v in sorted(test_sets[row].items())},
"n_params": next((r.get("n_params") for r in d.values() if r.get("n_params")), None),
}
ref = spec.get("reference_row")
# a row may name its own reference (``reference:`` in the registry), e.g. F5
# compares each pre-trained row with the scratch row of the same fraction
row_ref = {x["key"]: x.get("reference", ref) for x in spec["rows"]}
for row in runs:
rr = row_ref.get(row)
if not rr or rr == row or rr not in runs or not runs[rr] or not runs[row]:
continue # no reference, or a row without any run (only with allow_incomplete)
a_vals, r_vals = common_vals[row], common_vals[rr]
if allow_incomplete:
# partial table: compare on the (fold, seed) runs both rows have
keys = set(next(iter(a_vals.values()))) & set(next(iter(r_vals.values())))
if not keys:
continue
a_vals = {mk: {key: v[key] for key in keys} for mk, v in a_vals.items()}
r_vals = {mk: {key: v[key] for key in keys} for mk, v in r_vals.items()}
cmp = {}
for m in spec["metrics"]:
c = compare_rows(a_vals[m["key"]], r_vals[m["key"]])
if m["better"] == "lower":
c["positive_folds"] = sum(1 for x in c["fold_diffs"] if x < 0)
c["better"] = m["better"]
cmp[m["key"]] = c
rows_out[row]["vs_reference"] = cmp
rows_out[row]["reference"] = rr
return {
"table": spec["id"], "title": spec.get("title", ""), "task": spec["task"],
"split": spec["split"], "seeds": spec["seeds"], "folds": folds,
"metrics": spec["metrics"], "reference_row": ref, "ddof": DDOF,
"complete": not missing, "missing_runs": missing,
"provenance": prov,
"common_test_recordings": {str(k): v for k, v in common.items()},
"rows": rows_out,
}
# ---------------------------------------------------------------------------
# LaTeX
# ---------------------------------------------------------------------------
def tex_escape(s: str) -> str:
if "\\" in s or "$" in s:
return s # already TeX
for a, b in (("&", r"\&"), ("%", r"\%"), ("#", r"\#"), ("_", r"\_")):
s = s.replace(a, b)
return s
def _header(summary: dict, what: str) -> List[str]:
lines = [f"% Generated by scripts/aggregate.py for table {summary['table']} -- do not edit.",
f"% {summary['title']}",
f"% {what}",
f"% mean +- SD (ddof={summary['ddof']}) over {len(summary['folds'])} fold means; each "
f"fold = mean over seeds {summary['seeds']}.",
f"% git {summary['provenance'].get('git_sha')} data {summary['provenance'].get('data_tag')}"
f" split md5 {summary['provenance'].get('split_md5')}"]
if not summary["complete"]:
lines.append(f"% INCOMPLETE: {len(summary['missing_runs'])} run(s) missing: "
+ ", ".join(summary["missing_runs"][:20]))
return lines
def render_table(summary: dict, which: str = "own") -> str:
ms = summary["metrics"]
what = ("own test set per row" if which == "own" else
"common test set (recordings loaded by every row), per fold: "
+ ", ".join(f"{k}:{len(v)}" for k, v in summary["common_test_recordings"].items()))
lines = _header(summary, what)
lines += [r"\begin{tabular}{l" + "c" * len(ms) + "}", r"\toprule",
"Method & " + " & ".join(tex_escape(m["label"]) for m in ms) + r" \\", r"\midrule"]
for row, r in summary["rows"].items():
cells = [format_pm(r[which][m["key"]]["mean"], r[which][m["key"]]["sd"], m["digits"])
for m in ms]
lines.append(tex_escape(r["label"]) + " & " + " & ".join(cells) + r" \\")
lines += [r"\bottomrule", r"\end{tabular}", ""]
return "\n".join(lines)
def render_paired(summary: dict) -> str:
ref = summary["reference_row"]
ms = summary["metrics"]
own_refs = any(r.get("reference", ref) != ref for r in summary["rows"].values()
if "vs_reference" in r)
what = ("each row's own reference row (named after 'vs')" if own_refs
else f"reference row {ref!r}")
lines = _header(summary, f"paired comparison with {what} on the common test set; "
"delta = row - reference; positive folds = folds where the row is better")
lines += [r"\begin{tabular}{l" + "cccc" * len(ms) + "}", r"\toprule",
"Method & " + " & ".join(
rf"$\Delta$ {tex_escape(m['label'])} & pos. folds & $p_{{5}}$ & $p_{{15}}$"
for m in ms) + r" \\", r"\midrule"]
for row, r in summary["rows"].items():
if "vs_reference" not in r:
continue
cells = []
for m in ms:
c = r["vs_reference"][m["key"]]
t15 = c.get("t15")
cells += [format_pm(c["delta_mean"], c["delta_sd"], m["digits"]),
f"{c['positive_folds']}/{len(c['folds'])}",
format_p(c["t5"]["p"]), format_p(t15["p"]) if t15 else "--"]
label = r["label"]
if own_refs:
label += " vs " + summary["rows"][r["reference"]]["label"]
lines.append(tex_escape(label) + " & " + " & ".join(cells) + r" \\")
lines += [r"\bottomrule", r"\end{tabular}", ""]
return "\n".join(lines)
def _num(v: Optional[float], fmt: str) -> str:
"""``format(v, fmt)``, or ``--`` for a missing / NaN value (e.g. an SD over one fold)."""
return "--" if v is None or v != v else format(v, fmt)
def number_entries(summary: dict) -> Dict[str, str]:
"""``{"<table>/<row>/<metric>/<stat>": "0.626"}`` for numbers.tex."""
out: Dict[str, str] = {}
t = summary["table"]
for row, r in summary["rows"].items():
for m in summary["metrics"]:
key, d = m["key"], m["digits"]
base = f"{t}/{row}/{key}"
for which, pre in (("own", ""), ("common", "c")):
st = r[which][key]
out[f"{base}/{pre}mean"] = _num(st["mean"], f".{d}f")
out[f"{base}/{pre}sd"] = _num(st["sd"], f".{d}f")
out[f"{base}/{pre}pm"] = format_pm(st["mean"], st["sd"], d)
if "vs_reference" in r:
c = r["vs_reference"][key]
out[f"{base}/delta"] = _num(c["delta_mean"], f"+.{d}f")
out[f"{base}/deltasd"] = _num(c["delta_sd"], f".{d}f")
out[f"{base}/pos"] = f"{c['positive_folds']}/{len(c['folds'])}"
out[f"{base}/p5"] = format_p(c["t5"]["p"])
out[f"{base}/t5"] = _num(c["t5"]["t"], ".2f")
if c.get("t15"):
out[f"{base}/p15"] = format_p(c["t15"]["p"])
out[f"{base}/t15"] = _num(c["t15"]["t"], ".2f")
out[f"{t}/{row}/ntest"] = str(sum(r["n_test_recordings"].values()))
if r.get("n_params"):
out[f"{t}/{row}/params"] = f"{r['n_params'] / 1e6:.1f}M"
out[f"{t}/ncommon"] = str(sum(len(v) for v in summary["common_test_recordings"].values()))
return out
NUMBERS_PREAMBLE = r"""% Generated by scripts/aggregate.py -- do not edit.
% Usage in the paper: \pulsenum{<table>/<row>/<metric>/<stat>}, e.g. \pulsenum{3/syncfuse/macro_f1/pm}
% stats: mean sd pm (own test set), cmean csd cpm (common test set), delta deltasd pos p5 t5 p15 t15
% (vs the reference row, common test set), ntest, params; per table: ncommon.
\providecommand{\pulsenum}[1]{\ifcsname pulse@#1\endcsname\csname pulse@#1\endcsname\else
\textbf{??}\PackageWarning{pulsenum}{undefined number \detokenize{#1}}\fi}
"""
def render_numbers(summaries: List[dict]) -> str:
lines = [NUMBERS_PREAMBLE]
for s in sorted(summaries, key=lambda x: x["table"]):
lines.append(f"% ---- table {s['table']}: {s['title']}"
+ ("" if s["complete"] else " [INCOMPLETE]"))
for k, v in sorted(number_entries(s).items()):
lines.append(r"\expandafter\def\csname pulse@" + k + r"\endcsname{" + v + "}")
return "\n".join(lines) + "\n"
def write_outputs(summary: dict, out_dir) -> List[Path]:
out = Path(out_dir)
out.mkdir(parents=True, exist_ok=True)
t = summary["table"]
files = {f"tab{t}.tex": render_table(summary, "own"),
f"tab{t}_common.tex": render_table(summary, "common"),
f"tab{t}.json": json.dumps(summary, indent=1) + "\n"}
if any("vs_reference" in r for r in summary["rows"].values()):
files[f"tab{t}_paired.tex"] = render_paired(summary)
written = []
for name, text in files.items():
(out / name).write_text(text, encoding="utf-8", newline="\n")
written.append(out / name)
# only table summaries (tab<id>.json); reports such as t3_report's tab6_folds.json share the prefix
summaries = []
for p in sorted(out.glob("tab*.json")):
d = json.loads(p.read_text(encoding="utf-8"))
if isinstance(d, dict) and p.name == f"tab{d.get('table')}.json" and "rows" in d \
and "metrics" in d:
summaries.append(d)
(out / "numbers.tex").write_text(render_numbers(summaries), encoding="utf-8", newline="\n")
written.append(out / "numbers.tex")
return written
|