"""Summarise speed_engines.py outputs: identity against the paper code and timings. python benchmarks/summarize_speed.py --runs --out benchmarks/results/speed_engines.csv For every cell (dataset, fold, seed, family) and engine: decoded-label mismatches, per-window argmax mismatches and the maximum absolute score difference against the reference engine, plus fit/predict seconds and the reference/engine time ratio. """ from __future__ import annotations import argparse import csv import json from collections import defaultdict from pathlib import Path import numpy as np ENGINES = ("ref", "v020", "v020fast") # paper code, speed="exact", speed="fast" def main(): ap = argparse.ArgumentParser() ap.add_argument("--runs", required=True) ap.add_argument("--out", required=True) ap.add_argument("--markdown", help="also write per-family markdown tables here") a = ap.parse_args() runs = Path(a.runs) records = defaultdict(dict) for js in sorted(runs.glob("*.json")): rec = json.loads(js.read_text()) if rec["engine"] not in ENGINES: continue key = (rec["dataset"], rec["fold"], rec["seed"], rec["family"]) with np.load(js.with_suffix(".npz")) as z: rec["proba"], rec["labels"] = z["proba"], z["labels"] records[key][rec["engine"]] = rec rows = [] for key, engines in sorted(records.items()): ref = engines.get("ref") if ref is None: continue for name, rec in sorted(engines.items()): t_ref = ref["fit_seconds"] + ref["predict_seconds"] t = rec["fit_seconds"] + rec["predict_seconds"] rows.append({ "dataset": key[0], "fold": key[1], "seed": key[2], "family": key[3], "engine": name, "n_test": rec["n_test"], "label_mismatches": int((rec["labels"] != ref["labels"]).sum()), "argmax_mismatches": int((rec["proba"].argmax(1) != ref["proba"].argmax(1)).sum()), "proba_max_abs_diff": float(np.abs(rec["proba"] - ref["proba"]).max()), "fit_seconds": rec["fit_seconds"], "predict_seconds": rec["predict_seconds"], "total_seconds": t, "speedup_vs_ref": t_ref / t if t > 0 else float("nan"), **{f"test_{k}": v for k, v in rec["scores"].items()}, "cpu_model": rec["cpu_model"], }) out = Path(a.out) out.parent.mkdir(parents=True, exist_ok=True) with out.open("w", newline="") as f: w = csv.DictWriter(f, fieldnames=list(rows[0])) w.writeheader() w.writerows(rows) by = defaultdict(list) for r in rows: by[r["engine"]].append(r) for name, rs in sorted(by.items()): cells = len(rs) lab = sum(r["label_mismatches"] for r in rs) am = sum(r["argmax_mismatches"] for r in rs) mad = max(r["proba_max_abs_diff"] for r in rs) tot = sum(r["total_seconds"] for r in rs) cpus = sorted({r["cpu_model"] for r in rs}) print(f"{name:10s} cells={cells:3d} label_mismatches={lab} argmax_mismatches={am} " f"max_abs_score_diff={mad:.3e} total_s={tot:.1f} cpus={cpus}") if a.markdown: write_markdown(rows, Path(a.markdown)) def write_markdown(rows, path): """Per-family tables over the seed-42 fold-0 cells of all streams.""" import pandas as pd d = pd.DataFrame(rows) d = d[(d.seed == 42) & (d.fold == 0)] lines = [] ident = d.groupby(["engine", "family"]).agg(cells=("dataset", "size"), label=("label_mismatches", "sum"), amax=("argmax_mismatches", "sum"), maxdiff=("proba_max_abs_diff", "max"), seconds=("total_seconds", "sum")).reset_index() lines.append("| Family | Engine | Cells | Decoded-label mismatches | Argmax mismatches | Max abs score diff | Fit+predict s (sum) | Speed-up vs paper code |") lines.append("|---|---|---:|---:|---:|---:|---:|---:|") ref = ident[ident.engine == "ref"].set_index("family").seconds for fam in ["wisp_ss", "wisp_rc", "wisp_so", "wisp_gis", "wisp_cis", "wisp_cse", "wisp_ese"]: for eng in ENGINES: r = ident[(ident.family == fam) & (ident.engine == eng)] if r.empty: continue r = r.iloc[0] lines.append(f"| {fam} | {eng} | {r['cells']} | {r['label']} | {r['amax']} | {r['maxdiff']:.3g} | {r['seconds']:,.0f} | " f"{ref[fam] / r['seconds']:.2f}x |") tot = d.groupby("engine").agg(cells=("dataset", "size"), seconds=("total_seconds", "sum"), label=("label_mismatches", "sum")).reset_index() lines.append("") lines.append("| Engine | Cells | Fit+predict s (sum) | Decoded-label mismatches |") lines.append("|---|---:|---:|---:|") for _, r in tot.iterrows(): lines.append(f"| {r.engine} | {r.cells} | {r.seconds:,.0f} | {r.label} |") per = d[d.engine.isin(["v020fast", "ref"]) & d.family.isin(["wisp_rc", "wisp_cis"])].pivot_table( index=["dataset", "family"], columns="engine", values="test_macro_f1") if not per.empty: lines.append("") lines.append("| Dataset | Family | Paper code macro-F1 | screened Ridge (v020fast) change |") lines.append("|---|---|---:|---:|") for (ds, fam), r in per.iterrows(): f = lambda k: "" if k not in r or pd.isna(r[k]) else f"{100 * (r[k] - r['ref']):+.2f}" lines.append(f"| {ds} | {fam} | {100 * r['ref']:.2f} | {f('v020fast')} |") path.write_text("\n".join(lines) + "\n") if __name__ == "__main__": main()