Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / benchmarks /summarize_speed.py
Zipeng365's picture
wisp-har 0.3.0: accuracy-against-memory figures, explained advantage figures, speed docs cleaned up
19383e1 verified
Raw History Blame Contribute Delete
5.78 kB
"""Summarise speed_engines.py outputs: identity against the paper code and timings.
python benchmarks/summarize_speed.py --runs <dir of *.json/*.npz> --out benchmarks/results/speed_engines.csv
For every cell (dataset, fold, seed, family) and engine: decoded-label mismatches, per-window
argmax mismatches and the maximum absolute score difference against the reference engine,
plus fit/predict seconds and the reference/engine time ratio.
"""
from __future__ import annotations
import argparse
import csv
import json
from collections import defaultdict
from pathlib import Path
import numpy as np
ENGINES = ("ref", "v020", "v020fast") # paper code, speed="exact", speed="fast"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--runs", required=True)
ap.add_argument("--out", required=True)
ap.add_argument("--markdown", help="also write per-family markdown tables here")
a = ap.parse_args()
runs = Path(a.runs)
records = defaultdict(dict)
for js in sorted(runs.glob("*.json")):
rec = json.loads(js.read_text())
if rec["engine"] not in ENGINES:
continue
key = (rec["dataset"], rec["fold"], rec["seed"], rec["family"])
with np.load(js.with_suffix(".npz")) as z:
rec["proba"], rec["labels"] = z["proba"], z["labels"]
records[key][rec["engine"]] = rec
rows = []
for key, engines in sorted(records.items()):
ref = engines.get("ref")
if ref is None:
continue
for name, rec in sorted(engines.items()):
t_ref = ref["fit_seconds"] + ref["predict_seconds"]
t = rec["fit_seconds"] + rec["predict_seconds"]
rows.append({
"dataset": key[0], "fold": key[1], "seed": key[2], "family": key[3], "engine": name,
"n_test": rec["n_test"],
"label_mismatches": int((rec["labels"] != ref["labels"]).sum()),
"argmax_mismatches": int((rec["proba"].argmax(1) != ref["proba"].argmax(1)).sum()),
"proba_max_abs_diff": float(np.abs(rec["proba"] - ref["proba"]).max()),
"fit_seconds": rec["fit_seconds"], "predict_seconds": rec["predict_seconds"],
"total_seconds": t, "speedup_vs_ref": t_ref / t if t > 0 else float("nan"),
**{f"test_{k}": v for k, v in rec["scores"].items()},
"cpu_model": rec["cpu_model"],
})
out = Path(a.out)
out.parent.mkdir(parents=True, exist_ok=True)
with out.open("w", newline="") as f:
w = csv.DictWriter(f, fieldnames=list(rows[0]))
w.writeheader()
w.writerows(rows)
by = defaultdict(list)
for r in rows:
by[r["engine"]].append(r)
for name, rs in sorted(by.items()):
cells = len(rs)
lab = sum(r["label_mismatches"] for r in rs)
am = sum(r["argmax_mismatches"] for r in rs)
mad = max(r["proba_max_abs_diff"] for r in rs)
tot = sum(r["total_seconds"] for r in rs)
cpus = sorted({r["cpu_model"] for r in rs})
print(f"{name:10s} cells={cells:3d} label_mismatches={lab} argmax_mismatches={am} "
f"max_abs_score_diff={mad:.3e} total_s={tot:.1f} cpus={cpus}")
if a.markdown:
write_markdown(rows, Path(a.markdown))
def write_markdown(rows, path):
"""Per-family tables over the seed-42 fold-0 cells of all streams."""
import pandas as pd
d = pd.DataFrame(rows)
d = d[(d.seed == 42) & (d.fold == 0)]
lines = []
ident = d.groupby(["engine", "family"]).agg(cells=("dataset", "size"), label=("label_mismatches", "sum"),
amax=("argmax_mismatches", "sum"),
maxdiff=("proba_max_abs_diff", "max"),
seconds=("total_seconds", "sum")).reset_index()
lines.append("| Family | Engine | Cells | Decoded-label mismatches | Argmax mismatches | Max abs score diff | Fit+predict s (sum) | Speed-up vs paper code |")
lines.append("|---|---|---:|---:|---:|---:|---:|---:|")
ref = ident[ident.engine == "ref"].set_index("family").seconds
for fam in ["wisp_ss", "wisp_rc", "wisp_so", "wisp_gis", "wisp_cis", "wisp_cse", "wisp_ese"]:
for eng in ENGINES:
r = ident[(ident.family == fam) & (ident.engine == eng)]
if r.empty:
continue
r = r.iloc[0]
lines.append(f"| {fam} | {eng} | {r['cells']} | {r['label']} | {r['amax']} | {r['maxdiff']:.3g} | {r['seconds']:,.0f} | "
f"{ref[fam] / r['seconds']:.2f}x |")
tot = d.groupby("engine").agg(cells=("dataset", "size"), seconds=("total_seconds", "sum"),
label=("label_mismatches", "sum")).reset_index()
lines.append("")
lines.append("| Engine | Cells | Fit+predict s (sum) | Decoded-label mismatches |")
lines.append("|---|---:|---:|---:|")
for _, r in tot.iterrows():
lines.append(f"| {r.engine} | {r.cells} | {r.seconds:,.0f} | {r.label} |")
per = d[d.engine.isin(["v020fast", "ref"]) & d.family.isin(["wisp_rc", "wisp_cis"])].pivot_table(
index=["dataset", "family"], columns="engine", values="test_macro_f1")
if not per.empty:
lines.append("")
lines.append("| Dataset | Family | Paper code macro-F1 | screened Ridge (v020fast) change |")
lines.append("|---|---|---:|---:|")
for (ds, fam), r in per.iterrows():
f = lambda k: "" if k not in r or pd.isna(r[k]) else f"{100 * (r[k] - r['ref']):+.2f}"
lines.append(f"| {ds} | {fam} | {100 * r['ref']:.2f} | {f('v020fast')} |")
path.write_text("\n".join(lines) + "\n")
if __name__ == "__main__":
main()