"""This package's benchmark (not a paper result): Optuna (TPE) against WISP-Random at equal budget. One task = one data stream, fold, seed and strategy. The search sees only the training and validation splits; after it returns, its refitted winner is scored once on the test split for reporting. Output: one JSON per task with the winner's validation utility, test metrics, selection and final-fit seconds, and the candidate history. python benchmarks/optuna_vs_random.py --data-root --dataset adl_wrist_accel_v1 \ --fold 0 --seed 42 --strategy optuna --budget 329 --n-jobs 2 --out """ from __future__ import annotations import argparse import json import os import platform import sys import time from pathlib import Path import numpy as np sys.path.insert(0, str(Path(__file__).resolve().parent)) from _frozen import load_fold, paper_scores # noqa: E402 def main(): ap = argparse.ArgumentParser() ap.add_argument("--data-root", required=True) ap.add_argument("--dataset", required=True) ap.add_argument("--fold", type=int, default=0) ap.add_argument("--seed", type=int, default=42) ap.add_argument("--strategy", choices=("random", "optuna", "evolution"), required=True) ap.add_argument("--budget", type=int, default=329) ap.add_argument("--n-jobs", type=int, default=1) ap.add_argument("--out", required=True) a = ap.parse_args() out = Path(a.out) / f"{a.dataset}__f{a.fold}__s{a.seed}__{a.strategy}__b{a.budget}.json" if out.exists(): print("exists", out) return out.parent.mkdir(parents=True, exist_ok=True) import wisp train, valid, _, test = load_fold(a.data_root, a.dataset, a.fold) K = train["n_classes"] t0 = time.perf_counter() search = wisp.WISPSearch(a.strategy, budget=a.budget, decoder="recipe", classes=np.arange(K), n_jobs=a.n_jobs, random_state=a.seed) # recording descriptors of the export for the search controllers' dataset fingerprint (0.4.0; # the Optuna strategy does not use a fingerprint) info = wisp.datasets.load_exported_fold(a.data_root, a.dataset, a.fold) search.fit(train["X"], train["y"], groups=train["subject"], time_index=train["time_index"], validation=(valid["X"], valid["y"], valid["subject"], valid["time_index"]), sampling_rate_hz=info.sampling_rate_hz, study_setting=info.study_setting) wall = time.perf_counter() - t0 pred = search.predict(test["X"], groups=test["subject"], time_index=test["time_index"]) ok = [h for h in search.history_ if h["status"] == "ok"] record = { "dataset": a.dataset, "fold": a.fold, "seed": a.seed, "strategy": a.strategy, "budget": a.budget, "n_jobs": a.n_jobs, "sampling_rate_hz": info.sampling_rate_hz, "study_setting": info.study_setting, "best_recipe": search.best_recipe_.to_dict(), "best_seed": search.best_seed_, "best_validation_utility": search.best_validation_["predictive_utility"], "best_validation_macro_f1": search.best_validation_.get("macro_f1"), "test": paper_scores(test["y"], pred, K), "selection_seconds": search.selection_seconds_, "final_fit_seconds": search.final_fit_seconds_, "wall_seconds": wall, "n_candidates": len(search.history_), "n_ok": len(ok), "n_unique_fitted": sum(1 for h in search.history_ if not h.get("repeat", False)), "history": search.history_, "python": platform.python_version(), "slurm_job_id": os.environ.get("SLURM_ARRAY_JOB_ID", os.environ.get("SLURM_JOB_ID")), } out.write_text(json.dumps(record, indent=1, default=str)) print(json.dumps({k: v for k, v in record.items() if k != "history"}, default=str)) if __name__ == "__main__": main()