Instructions to use Zipeng365/WISP with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Scikit-learn
How to use Zipeng365/WISP with Scikit-learn:
from huggingface_hub import hf_hub_download import joblib model = joblib.load( hf_hub_download("Zipeng365/WISP", "sklearn_model.joblib") ) # only load pickle files from sources you trust # read more about it here https://skops.readthedocs.io/en/stable/persistence.html - Notebooks
- Google Colab
- Kaggle
Download benchmarks/summarize_optuna.py from Zipeng365/WISP: direct link, hf CLI and curl.
- Browser
- Download file 4.66 kB
-
https://huggingface.co/Zipeng365/WISP/resolve/main/benchmarks/summarize_optuna.py
- Command line
-
hf download hf://Zipeng365/WISP/benchmarks/summarize_optuna.py
-
curl -L -o summarize_optuna.py https://huggingface.co/Zipeng365/WISP/resolve/main/benchmarks/summarize_optuna.py
4.66 kB
| """Summarise optuna_vs_random.py outputs (this package's benchmark, not a paper result). | |
| python benchmarks/summarize_optuna.py --runs <dir> --out benchmarks/results/optuna_vs_random.csv \ | |
| --markdown benchmarks/results/optuna_vs_random.md | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import glob | |
| import json | |
| from pathlib import Path | |
| import pandas as pd | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--runs", required=True) | |
| ap.add_argument("--out", required=True) | |
| ap.add_argument("--markdown") | |
| ap.add_argument("--expected-pairs", type=int, default=36, help="stream x seed pairs planned (12 x 3)") | |
| a = ap.parse_args() | |
| rows = [] | |
| for f in sorted(glob.glob(str(Path(a.runs) / "*.json"))): | |
| r = json.load(open(f)) | |
| rows.append({"dataset": r["dataset"], "fold": r["fold"], "seed": r["seed"], "strategy": r["strategy"], | |
| "budget": r["budget"], "best_recipe": json.dumps(r["best_recipe"], sort_keys=True), | |
| "best_seed": r["best_seed"], "validation_utility": r["best_validation_utility"], | |
| "validation_macro_f1": r["best_validation_macro_f1"], | |
| **{f"test_{k}": v for k, v in r["test"].items()}, | |
| "selection_hours": r["selection_seconds"] / 3600, "final_fit_seconds": r["final_fit_seconds"], | |
| "candidates": r["n_candidates"], "unique_fitted": r["n_unique_fitted"], "ok": r["n_ok"], | |
| "n_jobs": r["n_jobs"]}) | |
| d = pd.DataFrame(rows) | |
| Path(a.out).parent.mkdir(parents=True, exist_ok=True) | |
| d.to_csv(a.out, index=False) | |
| if d.empty or not a.markdown: | |
| print(d) | |
| return | |
| piv = d.pivot_table(index=["dataset", "seed"], columns="strategy", | |
| values=["validation_utility", "test_macro_f1", "selection_hours", "unique_fitted"]) | |
| complete = piv.dropna() | |
| lines = ["**This package's benchmark, not a paper result.** WISPSearch('optuna') with TPE against " | |
| "WISPSearch('random') (the paper's WISP-Random implementation) at the same budget; selection on " | |
| "validation only, the refitted winner scored once on test for reporting. One run per cell.", "", | |
| "Correction in 0.4.0: the WISP-Random arms on CAPTURE-24 and PAAL-ADL were rerun with the export's " | |
| "sampling rate and study setting passed to `WISPSearch.fit` (0.3.0 did not pass them to the search " | |
| "fingerprint; the Optuna strategy has no fingerprint). The superseded rows are in " | |
| "`optuna_vs_random_superseded.csv`.", "", | |
| f"Pairs with both strategies finished: {len(complete)} of {a.expected_pairs} planned " | |
| f"(stream x seed, fold 0, budget {int(d.budget.max())}); runs with only one strategy finished are " | |
| "listed with a dash for the other.", ""] | |
| lines += ["| Stream | Seed | Val. utility, Random | Val. utility, Optuna | Test macro-F1, Random | Test macro-F1, Optuna " | |
| "| Selection h, Random | Selection h, Optuna | Distinct fits, Random | Distinct fits, Optuna |", | |
| "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|"] | |
| for (ds, seed), r in piv.iterrows(): | |
| def g(v, s, fmt): | |
| x = r.get((v, s)) | |
| return "—" if pd.isna(x) else format(x, fmt) | |
| lines.append(f"| {ds} | {seed} | {g('validation_utility', 'random', '.4f')} | {g('validation_utility', 'optuna', '.4f')} " | |
| f"| {g('test_macro_f1', 'random', '.4f')} | {g('test_macro_f1', 'optuna', '.4f')} " | |
| f"| {g('selection_hours', 'random', '.2f')} | {g('selection_hours', 'optuna', '.2f')} " | |
| f"| {g('unique_fitted', 'random', '.0f')} | {g('unique_fitted', 'optuna', '.0f')} |") | |
| if len(complete): | |
| dv = complete[("validation_utility", "optuna")] - complete[("validation_utility", "random")] | |
| dt = complete[("test_macro_f1", "optuna")] - complete[("test_macro_f1", "random")] | |
| hr = complete[("selection_hours", "optuna")].sum() / complete[("selection_hours", "random")].sum() | |
| lines += ["", f"Over the {len(complete)} finished pairs: Optuna minus Random validation utility mean " | |
| f"{dv.mean():+.4f} (Optuna higher in {(dv > 0).sum()}, lower in {(dv < 0).sum()}); test macro-F1 " | |
| f"mean {100 * dt.mean():+.2f} points (higher in {(dt > 0).sum()}, lower in {(dt < 0).sum()}); " | |
| f"Optuna selection time {hr:.2f} times Random's."] | |
| Path(a.markdown).write_text("\n".join(lines) + "\n") | |
| print("\n".join(lines[-3:])) | |
| if __name__ == "__main__": | |
| main() | |