"""Summarise optuna_vs_random.py outputs (this package's benchmark, not a paper result). python benchmarks/summarize_optuna.py --runs --out benchmarks/results/optuna_vs_random.csv \ --markdown benchmarks/results/optuna_vs_random.md """ from __future__ import annotations import argparse import glob import json from pathlib import Path import pandas as pd def main(): ap = argparse.ArgumentParser() ap.add_argument("--runs", required=True) ap.add_argument("--out", required=True) ap.add_argument("--markdown") ap.add_argument("--expected-pairs", type=int, default=36, help="stream x seed pairs planned (12 x 3)") a = ap.parse_args() rows = [] for f in sorted(glob.glob(str(Path(a.runs) / "*.json"))): r = json.load(open(f)) rows.append({"dataset": r["dataset"], "fold": r["fold"], "seed": r["seed"], "strategy": r["strategy"], "budget": r["budget"], "best_recipe": json.dumps(r["best_recipe"], sort_keys=True), "best_seed": r["best_seed"], "validation_utility": r["best_validation_utility"], "validation_macro_f1": r["best_validation_macro_f1"], **{f"test_{k}": v for k, v in r["test"].items()}, "selection_hours": r["selection_seconds"] / 3600, "final_fit_seconds": r["final_fit_seconds"], "candidates": r["n_candidates"], "unique_fitted": r["n_unique_fitted"], "ok": r["n_ok"], "n_jobs": r["n_jobs"]}) d = pd.DataFrame(rows) Path(a.out).parent.mkdir(parents=True, exist_ok=True) d.to_csv(a.out, index=False) if d.empty or not a.markdown: print(d) return piv = d.pivot_table(index=["dataset", "seed"], columns="strategy", values=["validation_utility", "test_macro_f1", "selection_hours", "unique_fitted"]) complete = piv.dropna() lines = ["**This package's benchmark, not a paper result.** WISPSearch('optuna') with TPE against " "WISPSearch('random') (the paper's WISP-Random implementation) at the same budget; selection on " "validation only, the refitted winner scored once on test for reporting. One run per cell.", "", "Correction in 0.4.0: the WISP-Random arms on CAPTURE-24 and PAAL-ADL were rerun with the export's " "sampling rate and study setting passed to `WISPSearch.fit` (0.3.0 did not pass them to the search " "fingerprint; the Optuna strategy has no fingerprint). The superseded rows are in " "`optuna_vs_random_superseded.csv`.", "", f"Pairs with both strategies finished: {len(complete)} of {a.expected_pairs} planned " f"(stream x seed, fold 0, budget {int(d.budget.max())}); runs with only one strategy finished are " "listed with a dash for the other.", ""] lines += ["| Stream | Seed | Val. utility, Random | Val. utility, Optuna | Test macro-F1, Random | Test macro-F1, Optuna " "| Selection h, Random | Selection h, Optuna | Distinct fits, Random | Distinct fits, Optuna |", "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|"] for (ds, seed), r in piv.iterrows(): def g(v, s, fmt): x = r.get((v, s)) return "—" if pd.isna(x) else format(x, fmt) lines.append(f"| {ds} | {seed} | {g('validation_utility', 'random', '.4f')} | {g('validation_utility', 'optuna', '.4f')} " f"| {g('test_macro_f1', 'random', '.4f')} | {g('test_macro_f1', 'optuna', '.4f')} " f"| {g('selection_hours', 'random', '.2f')} | {g('selection_hours', 'optuna', '.2f')} " f"| {g('unique_fitted', 'random', '.0f')} | {g('unique_fitted', 'optuna', '.0f')} |") if len(complete): dv = complete[("validation_utility", "optuna")] - complete[("validation_utility", "random")] dt = complete[("test_macro_f1", "optuna")] - complete[("test_macro_f1", "random")] hr = complete[("selection_hours", "optuna")].sum() / complete[("selection_hours", "random")].sum() lines += ["", f"Over the {len(complete)} finished pairs: Optuna minus Random validation utility mean " f"{dv.mean():+.4f} (Optuna higher in {(dv > 0).sum()}, lower in {(dv < 0).sum()}); test macro-F1 " f"mean {100 * dt.mean():+.2f} points (higher in {(dt > 0).sum()}, lower in {(dt < 0).sum()}); " f"Optuna selection time {hr:.2f} times Random's."] Path(a.markdown).write_text("\n".join(lines) + "\n") print("\n".join(lines[-3:])) if __name__ == "__main__": main()