Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / benchmarks /optuna_vs_random.py
Zipeng365's picture
wisp-har 0.4.0: WISP-Funnel search and the adaptive HMM switch (default decoder)
93a9282 verified
Raw History Blame Contribute Delete
3.78 kB
"""This package's benchmark (not a paper result): Optuna (TPE) against WISP-Random at equal budget.
One task = one data stream, fold, seed and strategy. The search sees only the training and
validation splits; after it returns, its refitted winner is scored once on the test split for
reporting. Output: one JSON per task with the winner's validation utility, test metrics,
selection and final-fit seconds, and the candidate history.
python benchmarks/optuna_vs_random.py --data-root <frozen folds> --dataset adl_wrist_accel_v1 \
--fold 0 --seed 42 --strategy optuna --budget 329 --n-jobs 2 --out <dir>
"""
from __future__ import annotations
import argparse
import json
import os
import platform
import sys
import time
from pathlib import Path
import numpy as np
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _frozen import load_fold, paper_scores # noqa: E402
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--data-root", required=True)
ap.add_argument("--dataset", required=True)
ap.add_argument("--fold", type=int, default=0)
ap.add_argument("--seed", type=int, default=42)
ap.add_argument("--strategy", choices=("random", "optuna", "evolution"), required=True)
ap.add_argument("--budget", type=int, default=329)
ap.add_argument("--n-jobs", type=int, default=1)
ap.add_argument("--out", required=True)
a = ap.parse_args()
out = Path(a.out) / f"{a.dataset}__f{a.fold}__s{a.seed}__{a.strategy}__b{a.budget}.json"
if out.exists():
print("exists", out)
return
out.parent.mkdir(parents=True, exist_ok=True)
import wisp
train, valid, _, test = load_fold(a.data_root, a.dataset, a.fold)
K = train["n_classes"]
t0 = time.perf_counter()
search = wisp.WISPSearch(a.strategy, budget=a.budget, decoder="recipe", classes=np.arange(K), n_jobs=a.n_jobs,
random_state=a.seed)
# recording descriptors of the export for the search controllers' dataset fingerprint (0.4.0;
# the Optuna strategy does not use a fingerprint)
info = wisp.datasets.load_exported_fold(a.data_root, a.dataset, a.fold)
search.fit(train["X"], train["y"], groups=train["subject"], time_index=train["time_index"],
validation=(valid["X"], valid["y"], valid["subject"], valid["time_index"]),
sampling_rate_hz=info.sampling_rate_hz, study_setting=info.study_setting)
wall = time.perf_counter() - t0
pred = search.predict(test["X"], groups=test["subject"], time_index=test["time_index"])
ok = [h for h in search.history_ if h["status"] == "ok"]
record = {
"dataset": a.dataset, "fold": a.fold, "seed": a.seed, "strategy": a.strategy, "budget": a.budget,
"n_jobs": a.n_jobs, "sampling_rate_hz": info.sampling_rate_hz, "study_setting": info.study_setting,
"best_recipe": search.best_recipe_.to_dict(), "best_seed": search.best_seed_,
"best_validation_utility": search.best_validation_["predictive_utility"],
"best_validation_macro_f1": search.best_validation_.get("macro_f1"),
"test": paper_scores(test["y"], pred, K),
"selection_seconds": search.selection_seconds_, "final_fit_seconds": search.final_fit_seconds_,
"wall_seconds": wall, "n_candidates": len(search.history_), "n_ok": len(ok),
"n_unique_fitted": sum(1 for h in search.history_ if not h.get("repeat", False)),
"history": search.history_, "python": platform.python_version(),
"slurm_job_id": os.environ.get("SLURM_ARRAY_JOB_ID", os.environ.get("SLURM_JOB_ID")),
}
out.write_text(json.dumps(record, indent=1, default=str))
print(json.dumps({k: v for k, v in record.items() if k != "history"}, default=str))
if __name__ == "__main__":
main()