"""Summarise optuna_vs_random.py outputs (this package's benchmark, not a paper result).
python benchmarks/summarize_optuna.py --runs
--out benchmarks/results/optuna_vs_random.csv \
--markdown benchmarks/results/optuna_vs_random.md
"""
from __future__ import annotations
import argparse
import glob
import json
from pathlib import Path
import pandas as pd
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--runs", required=True)
ap.add_argument("--out", required=True)
ap.add_argument("--markdown")
ap.add_argument("--expected-pairs", type=int, default=36, help="stream x seed pairs planned (12 x 3)")
a = ap.parse_args()
rows = []
for f in sorted(glob.glob(str(Path(a.runs) / "*.json"))):
r = json.load(open(f))
rows.append({"dataset": r["dataset"], "fold": r["fold"], "seed": r["seed"], "strategy": r["strategy"],
"budget": r["budget"], "best_recipe": json.dumps(r["best_recipe"], sort_keys=True),
"best_seed": r["best_seed"], "validation_utility": r["best_validation_utility"],
"validation_macro_f1": r["best_validation_macro_f1"],
**{f"test_{k}": v for k, v in r["test"].items()},
"selection_hours": r["selection_seconds"] / 3600, "final_fit_seconds": r["final_fit_seconds"],
"candidates": r["n_candidates"], "unique_fitted": r["n_unique_fitted"], "ok": r["n_ok"],
"n_jobs": r["n_jobs"]})
d = pd.DataFrame(rows)
Path(a.out).parent.mkdir(parents=True, exist_ok=True)
d.to_csv(a.out, index=False)
if d.empty or not a.markdown:
print(d)
return
piv = d.pivot_table(index=["dataset", "seed"], columns="strategy",
values=["validation_utility", "test_macro_f1", "selection_hours", "unique_fitted"])
complete = piv.dropna()
lines = ["**This package's benchmark, not a paper result.** WISPSearch('optuna') with TPE against "
"WISPSearch('random') (the paper's WISP-Random implementation) at the same budget; selection on "
"validation only, the refitted winner scored once on test for reporting. One run per cell.", "",
"Correction in 0.4.0: the WISP-Random arms on CAPTURE-24 and PAAL-ADL were rerun with the export's "
"sampling rate and study setting passed to `WISPSearch.fit` (0.3.0 did not pass them to the search "
"fingerprint; the Optuna strategy has no fingerprint). The superseded rows are in "
"`optuna_vs_random_superseded.csv`.", "",
f"Pairs with both strategies finished: {len(complete)} of {a.expected_pairs} planned "
f"(stream x seed, fold 0, budget {int(d.budget.max())}); runs with only one strategy finished are "
"listed with a dash for the other.", ""]
lines += ["| Stream | Seed | Val. utility, Random | Val. utility, Optuna | Test macro-F1, Random | Test macro-F1, Optuna "
"| Selection h, Random | Selection h, Optuna | Distinct fits, Random | Distinct fits, Optuna |",
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|"]
for (ds, seed), r in piv.iterrows():
def g(v, s, fmt):
x = r.get((v, s))
return "—" if pd.isna(x) else format(x, fmt)
lines.append(f"| {ds} | {seed} | {g('validation_utility', 'random', '.4f')} | {g('validation_utility', 'optuna', '.4f')} "
f"| {g('test_macro_f1', 'random', '.4f')} | {g('test_macro_f1', 'optuna', '.4f')} "
f"| {g('selection_hours', 'random', '.2f')} | {g('selection_hours', 'optuna', '.2f')} "
f"| {g('unique_fitted', 'random', '.0f')} | {g('unique_fitted', 'optuna', '.0f')} |")
if len(complete):
dv = complete[("validation_utility", "optuna")] - complete[("validation_utility", "random")]
dt = complete[("test_macro_f1", "optuna")] - complete[("test_macro_f1", "random")]
hr = complete[("selection_hours", "optuna")].sum() / complete[("selection_hours", "random")].sum()
lines += ["", f"Over the {len(complete)} finished pairs: Optuna minus Random validation utility mean "
f"{dv.mean():+.4f} (Optuna higher in {(dv > 0).sum()}, lower in {(dv < 0).sum()}); test macro-F1 "
f"mean {100 * dt.mean():+.2f} points (higher in {(dt > 0).sum()}, lower in {(dt < 0).sum()}); "
f"Optuna selection time {hr:.2f} times Random's."]
Path(a.markdown).write_text("\n".join(lines) + "\n")
print("\n".join(lines[-3:]))
if __name__ == "__main__":
main()