Download pulse/aggregate.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 18.8 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/aggregate.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/pulse/aggregate.py
-
curl -L -o aggregate.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/aggregate.py
18.8 kB
| """Turn the runs of one table into LaTeX tables and number macros. | |
| Rules (see also ``pulse.stats``): | |
| * every run's metrics are recomputed from its per-recording statistics and | |
| must equal the stored test metrics; | |
| * runs are merged only if their provenance agrees (data tag, split md5, | |
| taxonomy md5, availability md5, code version and, by default, git sha), | |
| all runs of a row used the same configuration including ``--set`` | |
| overrides (``config_md5``; a leftover smoke-test run is refused), and all | |
| runs of a table ran on the same kind of device with the same torch version | |
| (``cpu`` or ``cuda:<GPU model>``; CPU and GPU results are not spliced); | |
| * a row's value = mean over seeds within fold, then mean +- SD (ddof=1) over | |
| the fold means, on the row's own test recordings (``own``) and on the common | |
| test set (``common``: per fold, the recordings every row of the table loaded); | |
| * comparisons with the reference row (the table's ``reference_row``, or a row's | |
| own ``reference``) use the common test set: paired t over | |
| fold means (n=5) and over fold x seed pairs (n=15), and positive-fold counts | |
| (a fold counts as positive when the row improves on the reference). | |
| """ | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Optional, Tuple | |
| import yaml | |
| from .errors import ProvenanceError | |
| from .metrics import compute_metrics | |
| from .provenance import check_consistent, load_results | |
| from .splits import load_split | |
| from .stats import DDOF, compare_rows, fold_means, format_p, format_pm, mean_sd | |
| REPO_ROOT = Path(__file__).resolve().parents[1] | |
| RunKey = Tuple[int, int] # (fold, seed) | |
| def load_registry(path=None) -> Dict[str, dict]: | |
| p = Path(path) if path else REPO_ROOT / "configs" / "tables.yaml" | |
| with open(p, encoding="utf-8") as f: | |
| return yaml.safe_load(f)["tables"] | |
| def table_spec(registry: Dict[str, dict], table: str) -> dict: | |
| if str(table) not in registry: | |
| raise KeyError(f"table {table!r} not in registry; known: {sorted(registry)}") | |
| spec = dict(registry[str(table)]) | |
| spec["id"] = str(table) | |
| if "folds" not in spec: | |
| spec["folds"] = list(range(load_split(REPO_ROOT / spec["split"]).n_folds)) | |
| for m in spec["metrics"]: | |
| m.setdefault("digits", 3) | |
| m.setdefault("better", "higher") | |
| m.setdefault("label", m["key"]) | |
| return spec | |
| def run_dir(runs_root, table: str, row: str, fold: int, seed: int) -> Path: | |
| return Path(runs_root) / str(table) / row / f"fold{fold}_seed{seed}" | |
| def row_source(spec: dict, row: dict) -> Tuple[str, str]: | |
| """(table, row) whose runs ``row`` reads: its own, or those named by | |
| ``reuse: "<table>/<row>"`` (the row is then not run again, see run_table.py).""" | |
| src = row.get("reuse") | |
| if not src: | |
| return spec["id"], row["key"] | |
| t, sep, r = str(src).partition("/") | |
| if not (sep and t and r): | |
| raise ValueError(f"row {row['key']}: reuse must be '<table>/<row>', got {src!r}") | |
| return t, r | |
| def collect(spec: dict, runs_root) -> Tuple[Dict[str, Dict[RunKey, dict]], List[str]]: | |
| runs: Dict[str, Dict[RunKey, dict]] = {} | |
| missing: List[str] = [] | |
| for row in spec["rows"]: | |
| runs[row["key"]] = {} | |
| src_table, src_row = row_source(spec, row) | |
| for k in spec["folds"]: | |
| for s in spec["seeds"]: | |
| p = run_dir(runs_root, src_table, src_row, k, s) / "results.json" | |
| if p.is_file(): | |
| runs[row["key"]][(k, s)] = load_results(p) | |
| else: | |
| missing.append(f"{row['key']}/fold{k}_seed{s}") | |
| return runs, missing | |
| def _check_run(name: str, r: dict, spec: dict, fold: int, seed: int, tol: float) -> Dict[str, float]: | |
| prov = r["provenance"] | |
| if r["task"] != spec["task"]: | |
| raise ProvenanceError(f"{name}: task {r['task']!r} != table task {spec['task']!r}") | |
| if prov["fold"] != fold or prov["seed"] != seed: | |
| raise ProvenanceError(f"{name}: provenance says fold {prov['fold']} seed {prov['seed']}") | |
| own = compute_metrics(r) | |
| for k, v in r["metrics"].get("test", {}).items(): | |
| if k in own and abs(own[k] - v) > tol * max(1.0, abs(v)): | |
| raise ProvenanceError(f"{name}: stored test {k}={v} != recomputed {own[k]}") | |
| for m in spec["metrics"]: | |
| if m["key"] not in own: | |
| raise ProvenanceError(f"{name}: metric {m['key']!r} not defined in results.json") | |
| return own | |
| def compute_kind(prov: dict) -> str: | |
| """Hardware / software a run used: ``cpu`` or ``cuda:<GPU model>``, plus the torch version.""" | |
| d = prov.get("device") | |
| if not isinstance(d, dict): | |
| return json.dumps(d) | |
| dev = str(d.get("device") or "") | |
| kind = f"cuda:{d.get('gpu_name')}" if dev.startswith("cuda") else (dev or "unknown") | |
| return f"{kind} torch {d.get('torch')}" | |
| def check_runs_comparable(runs: Dict[str, Dict[RunKey, dict]], | |
| allow_mixed_device: bool = False) -> List[str]: | |
| """Every run of a row must use the same configuration (config file + all | |
| ``--set`` overrides; ``config_md5``), so e.g. a 1-epoch smoke run left in | |
| the runs directory cannot be averaged with full runs; and every run of a | |
| table must have used the same kind of device and torch version (CPU and GPU | |
| results are not spliced into one table). Returns the device kinds seen.""" | |
| for row, d in runs.items(): | |
| md5s = {} | |
| for (k, s), r in d.items(): | |
| md5s.setdefault(str(r["provenance"].get("config_md5")), []).append(f"fold{k}_seed{s}") | |
| if len(md5s) > 1: | |
| detail = "; ".join(f"config_md5 {m}: {', '.join(v[:3])}" for m, v in md5s.items()) | |
| raise ProvenanceError(f"row {row}: runs used different configurations or overrides " | |
| f"({detail}); rerun them with one configuration") | |
| kinds: Dict[str, List[str]] = {} | |
| for row, d in runs.items(): | |
| for (k, s), r in d.items(): | |
| kinds.setdefault(compute_kind(r["provenance"]), []).append(f"{row}/fold{k}_seed{s}") | |
| if len(kinds) > 1 and not allow_mixed_device: | |
| detail = "; ".join(f"{kd}: {len(v)} run(s), e.g. {v[0]}" for kd, v in kinds.items()) | |
| raise ProvenanceError(f"runs were produced on different devices / torch versions ({detail}); " | |
| "rerun them on one device type (or pass allow_mixed_device)") | |
| return sorted(kinds) | |
| def _row_stats(values: Dict[RunKey, float]) -> Dict[str, Any]: | |
| fm = fold_means(values) | |
| mean, sd = mean_sd(list(fm.values()), DDOF) | |
| return {"per_run": {f"{k}_{s}": v for (k, s), v in sorted(values.items())}, | |
| "fold_means": {str(k): v for k, v in fm.items()}, "mean": mean, "sd": sd, | |
| "n_folds": len(fm), "n_runs": len(values)} | |
| def aggregate(spec: dict, runs: Dict[str, Dict[RunKey, dict]], *, missing: Optional[List[str]] = None, | |
| allow_incomplete: bool = False, require_same_sha: bool = True, | |
| allow_dirty: bool = False, allow_mixed_device: bool = False, | |
| tol: float = 1e-9) -> Dict[str, Any]: | |
| missing = list(missing or []) | |
| if missing and not allow_incomplete: | |
| raise ProvenanceError(f"table {spec['id']}: {len(missing)} run(s) missing, e.g. " | |
| f"{missing[:5]} (use --allow-incomplete for a partial table)") | |
| flat = {f"{row}/fold{k}_seed{s}": r for row, d in runs.items() for (k, s), r in d.items()} | |
| if not flat: | |
| raise ProvenanceError(f"table {spec['id']}: no runs found") | |
| prov = check_consistent(flat, require_same_sha=require_same_sha, allow_dirty=allow_dirty) | |
| prov["compute"] = " | ".join(check_runs_comparable(runs, allow_mixed_device=allow_mixed_device)) | |
| own: Dict[str, Dict[RunKey, Dict[str, float]]] = {} | |
| test_sets: Dict[str, Dict[int, frozenset]] = {} | |
| for row, d in runs.items(): | |
| own[row], test_sets[row] = {}, {} | |
| for (k, s), r in d.items(): | |
| own[row][(k, s)] = _check_run(f"{row}/fold{k}_seed{s}", r, spec, k, s, tol) | |
| ts = frozenset(r["provenance"]["loaded_recordings"]["test"]) | |
| if k in test_sets[row] and test_sets[row][k] != ts: | |
| raise ProvenanceError(f"{row} fold {k}: seeds loaded different test recordings") | |
| test_sets[row][k] = ts | |
| folds = sorted({k for d in runs.values() for (k, _s) in d}) | |
| common: Dict[int, List[str]] = {} | |
| for k in folds: | |
| sets = [test_sets[row][k] for row in runs if k in test_sets[row]] | |
| inter = frozenset.intersection(*sets) if sets else frozenset() | |
| if not inter: | |
| raise ProvenanceError(f"fold {k}: the rows share no test recording") | |
| common[k] = sorted(inter) | |
| rows_out: Dict[str, Any] = {} | |
| common_vals: Dict[str, Dict[str, Dict[RunKey, float]]] = {} | |
| for row, d in runs.items(): | |
| cv = {(k, s): compute_metrics(r, recordings=common[k]) for (k, s), r in d.items()} | |
| common_vals[row] = {m["key"]: {key: v[m["key"]] for key, v in cv.items()} | |
| for m in spec["metrics"]} | |
| label = next(x["label"] for x in spec["rows"] if x["key"] == row) | |
| rows_out[row] = { | |
| "label": label, | |
| "own": {m["key"]: _row_stats({key: v[m["key"]] for key, v in own[row].items()}) | |
| for m in spec["metrics"]}, | |
| "common": {m["key"]: _row_stats(common_vals[row][m["key"]]) for m in spec["metrics"]}, | |
| "n_test_recordings": {str(k): len(v) for k, v in sorted(test_sets[row].items())}, | |
| "n_params": next((r.get("n_params") for r in d.values() if r.get("n_params")), None), | |
| } | |
| ref = spec.get("reference_row") | |
| # a row may name its own reference (``reference:`` in the registry), e.g. F5 | |
| # compares each pre-trained row with the scratch row of the same fraction | |
| row_ref = {x["key"]: x.get("reference", ref) for x in spec["rows"]} | |
| for row in runs: | |
| rr = row_ref.get(row) | |
| if not rr or rr == row or rr not in runs or not runs[rr] or not runs[row]: | |
| continue # no reference, or a row without any run (only with allow_incomplete) | |
| a_vals, r_vals = common_vals[row], common_vals[rr] | |
| if allow_incomplete: | |
| # partial table: compare on the (fold, seed) runs both rows have | |
| keys = set(next(iter(a_vals.values()))) & set(next(iter(r_vals.values()))) | |
| if not keys: | |
| continue | |
| a_vals = {mk: {key: v[key] for key in keys} for mk, v in a_vals.items()} | |
| r_vals = {mk: {key: v[key] for key in keys} for mk, v in r_vals.items()} | |
| cmp = {} | |
| for m in spec["metrics"]: | |
| c = compare_rows(a_vals[m["key"]], r_vals[m["key"]]) | |
| if m["better"] == "lower": | |
| c["positive_folds"] = sum(1 for x in c["fold_diffs"] if x < 0) | |
| c["better"] = m["better"] | |
| cmp[m["key"]] = c | |
| rows_out[row]["vs_reference"] = cmp | |
| rows_out[row]["reference"] = rr | |
| return { | |
| "table": spec["id"], "title": spec.get("title", ""), "task": spec["task"], | |
| "split": spec["split"], "seeds": spec["seeds"], "folds": folds, | |
| "metrics": spec["metrics"], "reference_row": ref, "ddof": DDOF, | |
| "complete": not missing, "missing_runs": missing, | |
| "provenance": prov, | |
| "common_test_recordings": {str(k): v for k, v in common.items()}, | |
| "rows": rows_out, | |
| } | |
| # --------------------------------------------------------------------------- | |
| # LaTeX | |
| # --------------------------------------------------------------------------- | |
| def tex_escape(s: str) -> str: | |
| if "\\" in s or "$" in s: | |
| return s # already TeX | |
| for a, b in (("&", r"\&"), ("%", r"\%"), ("#", r"\#"), ("_", r"\_")): | |
| s = s.replace(a, b) | |
| return s | |
| def _header(summary: dict, what: str) -> List[str]: | |
| lines = [f"% Generated by scripts/aggregate.py for table {summary['table']} -- do not edit.", | |
| f"% {summary['title']}", | |
| f"% {what}", | |
| f"% mean +- SD (ddof={summary['ddof']}) over {len(summary['folds'])} fold means; each " | |
| f"fold = mean over seeds {summary['seeds']}.", | |
| f"% git {summary['provenance'].get('git_sha')} data {summary['provenance'].get('data_tag')}" | |
| f" split md5 {summary['provenance'].get('split_md5')}"] | |
| if not summary["complete"]: | |
| lines.append(f"% INCOMPLETE: {len(summary['missing_runs'])} run(s) missing: " | |
| + ", ".join(summary["missing_runs"][:20])) | |
| return lines | |
| def render_table(summary: dict, which: str = "own") -> str: | |
| ms = summary["metrics"] | |
| what = ("own test set per row" if which == "own" else | |
| "common test set (recordings loaded by every row), per fold: " | |
| + ", ".join(f"{k}:{len(v)}" for k, v in summary["common_test_recordings"].items())) | |
| lines = _header(summary, what) | |
| lines += [r"\begin{tabular}{l" + "c" * len(ms) + "}", r"\toprule", | |
| "Method & " + " & ".join(tex_escape(m["label"]) for m in ms) + r" \\", r"\midrule"] | |
| for row, r in summary["rows"].items(): | |
| cells = [format_pm(r[which][m["key"]]["mean"], r[which][m["key"]]["sd"], m["digits"]) | |
| for m in ms] | |
| lines.append(tex_escape(r["label"]) + " & " + " & ".join(cells) + r" \\") | |
| lines += [r"\bottomrule", r"\end{tabular}", ""] | |
| return "\n".join(lines) | |
| def render_paired(summary: dict) -> str: | |
| ref = summary["reference_row"] | |
| ms = summary["metrics"] | |
| own_refs = any(r.get("reference", ref) != ref for r in summary["rows"].values() | |
| if "vs_reference" in r) | |
| what = ("each row's own reference row (named after 'vs')" if own_refs | |
| else f"reference row {ref!r}") | |
| lines = _header(summary, f"paired comparison with {what} on the common test set; " | |
| "delta = row - reference; positive folds = folds where the row is better") | |
| lines += [r"\begin{tabular}{l" + "cccc" * len(ms) + "}", r"\toprule", | |
| "Method & " + " & ".join( | |
| rf"$\Delta$ {tex_escape(m['label'])} & pos. folds & $p_{{5}}$ & $p_{{15}}$" | |
| for m in ms) + r" \\", r"\midrule"] | |
| for row, r in summary["rows"].items(): | |
| if "vs_reference" not in r: | |
| continue | |
| cells = [] | |
| for m in ms: | |
| c = r["vs_reference"][m["key"]] | |
| t15 = c.get("t15") | |
| cells += [format_pm(c["delta_mean"], c["delta_sd"], m["digits"]), | |
| f"{c['positive_folds']}/{len(c['folds'])}", | |
| format_p(c["t5"]["p"]), format_p(t15["p"]) if t15 else "--"] | |
| label = r["label"] | |
| if own_refs: | |
| label += " vs " + summary["rows"][r["reference"]]["label"] | |
| lines.append(tex_escape(label) + " & " + " & ".join(cells) + r" \\") | |
| lines += [r"\bottomrule", r"\end{tabular}", ""] | |
| return "\n".join(lines) | |
| def _num(v: Optional[float], fmt: str) -> str: | |
| """``format(v, fmt)``, or ``--`` for a missing / NaN value (e.g. an SD over one fold).""" | |
| return "--" if v is None or v != v else format(v, fmt) | |
| def number_entries(summary: dict) -> Dict[str, str]: | |
| """``{"<table>/<row>/<metric>/<stat>": "0.626"}`` for numbers.tex.""" | |
| out: Dict[str, str] = {} | |
| t = summary["table"] | |
| for row, r in summary["rows"].items(): | |
| for m in summary["metrics"]: | |
| key, d = m["key"], m["digits"] | |
| base = f"{t}/{row}/{key}" | |
| for which, pre in (("own", ""), ("common", "c")): | |
| st = r[which][key] | |
| out[f"{base}/{pre}mean"] = _num(st["mean"], f".{d}f") | |
| out[f"{base}/{pre}sd"] = _num(st["sd"], f".{d}f") | |
| out[f"{base}/{pre}pm"] = format_pm(st["mean"], st["sd"], d) | |
| if "vs_reference" in r: | |
| c = r["vs_reference"][key] | |
| out[f"{base}/delta"] = _num(c["delta_mean"], f"+.{d}f") | |
| out[f"{base}/deltasd"] = _num(c["delta_sd"], f".{d}f") | |
| out[f"{base}/pos"] = f"{c['positive_folds']}/{len(c['folds'])}" | |
| out[f"{base}/p5"] = format_p(c["t5"]["p"]) | |
| out[f"{base}/t5"] = _num(c["t5"]["t"], ".2f") | |
| if c.get("t15"): | |
| out[f"{base}/p15"] = format_p(c["t15"]["p"]) | |
| out[f"{base}/t15"] = _num(c["t15"]["t"], ".2f") | |
| out[f"{t}/{row}/ntest"] = str(sum(r["n_test_recordings"].values())) | |
| if r.get("n_params"): | |
| out[f"{t}/{row}/params"] = f"{r['n_params'] / 1e6:.1f}M" | |
| out[f"{t}/ncommon"] = str(sum(len(v) for v in summary["common_test_recordings"].values())) | |
| return out | |
| NUMBERS_PREAMBLE = r"""% Generated by scripts/aggregate.py -- do not edit. | |
| % Usage in the paper: \pulsenum{<table>/<row>/<metric>/<stat>}, e.g. \pulsenum{3/syncfuse/macro_f1/pm} | |
| % stats: mean sd pm (own test set), cmean csd cpm (common test set), delta deltasd pos p5 t5 p15 t15 | |
| % (vs the reference row, common test set), ntest, params; per table: ncommon. | |
| \providecommand{\pulsenum}[1]{\ifcsname pulse@#1\endcsname\csname pulse@#1\endcsname\else | |
| \textbf{??}\PackageWarning{pulsenum}{undefined number \detokenize{#1}}\fi} | |
| """ | |
| def render_numbers(summaries: List[dict]) -> str: | |
| lines = [NUMBERS_PREAMBLE] | |
| for s in sorted(summaries, key=lambda x: x["table"]): | |
| lines.append(f"% ---- table {s['table']}: {s['title']}" | |
| + ("" if s["complete"] else " [INCOMPLETE]")) | |
| for k, v in sorted(number_entries(s).items()): | |
| lines.append(r"\expandafter\def\csname pulse@" + k + r"\endcsname{" + v + "}") | |
| return "\n".join(lines) + "\n" | |
| def write_outputs(summary: dict, out_dir) -> List[Path]: | |
| out = Path(out_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| t = summary["table"] | |
| files = {f"tab{t}.tex": render_table(summary, "own"), | |
| f"tab{t}_common.tex": render_table(summary, "common"), | |
| f"tab{t}.json": json.dumps(summary, indent=1) + "\n"} | |
| if any("vs_reference" in r for r in summary["rows"].values()): | |
| files[f"tab{t}_paired.tex"] = render_paired(summary) | |
| written = [] | |
| for name, text in files.items(): | |
| (out / name).write_text(text, encoding="utf-8", newline="\n") | |
| written.append(out / name) | |
| # only table summaries (tab<id>.json); reports such as t3_report's tab6_folds.json share the prefix | |
| summaries = [] | |
| for p in sorted(out.glob("tab*.json")): | |
| d = json.loads(p.read_text(encoding="utf-8")) | |
| if isinstance(d, dict) and p.name == f"tab{d.get('table')}.json" and "rows" in d \ | |
| and "metrics" in d: | |
| summaries.append(d) | |
| (out / "numbers.tex").write_text(render_numbers(summaries), encoding="utf-8", newline="\n") | |
| written.append(out / "numbers.tex") | |
| return written | |