Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / src /wisp /integrations /optuna.py
Zipeng365's picture
wisp-har 0.4.0: WISP-Funnel search and the adaptive HMM switch (default decoder)
93a9282 verified
Raw History Blame Contribute Delete
11.1 kB
"""Optuna integration: WISP recipes as an Optuna search space (optional extra ``[optuna]``).
The space is the existing CPU search grammar (``wisp.search.grammar``):
* family: the seven kernel families;
* readout: ExtraTrees / random forest / logistic regression for SS and GIS, Ridge /
logistic regression for RC and CIS (the grammar's ``change_classifier``);
* segment sizes (3 of {3,4,5,8,10,12,16,20,25}) and symbolic bins ({4,5,6,8,10}) for SS
and GIS; convolution kernels ({128,...,768}) for RC and CIS; intervals ({8,...,48}) for
CIS; shapelets ({16,...,64}) for GIS; ensemble weights for ESE;
* CIS block switches R/I/S (the paper's CIS block subsets; at least one block on);
* decoder: HMM or per-window argmax.
Optuna only proposes. Each candidate is fitted and scored by the existing cascade
(``wisp.search.cascade``) with the existing validation utility, exactly as in
WISP-Random/WISP-Evolution; the test split is never seen.
Use :func:`suggest_recipe` inside your own objective to tune a WISP recipe jointly with
other hyperparameters.
"""
from __future__ import annotations
import itertools
import json
import math
from typing import Any
from ..recipes import Recipe
__all__ = ["suggest_recipe", "recipe_operators", "evaluate_recipe", "run_study", "SEARCH_SPACE"]
_SHORT = {"har_sculpt_forest_hmm": "ss", "rocket_sketch_ridge_hmm": "rc", "symbolic_interval_forest": "so",
"hybrid_cpu_ensemble": "cse", "invariant_interval_shape_forest": "gis",
"state_rocket_interval_hmm": "cis", "posterior_cpu_ensemble": "ese"}
_SEGMENT_POOL = (3, 4, 5, 8, 10, 12, 16, 20, 25)
SEARCH_SPACE: dict[str, Any] = {
"family": tuple(_SHORT),
"forest_classifier": ("extratrees", "rf", "logreg"),
"linear_classifier": ("ridge", "logreg"),
"segment_sizes": tuple(",".join(map(str, c)) for c in itertools.combinations(_SEGMENT_POOL, 3)),
"symbolic_bins": (4, 5, 6, 8, 10),
"n_kernels": (128, 256, 384, 512, 768),
"n_intervals": (8, 16, 24, 32, 48),
"n_shapelets": (16, 24, 32, 48, 64),
"use_hmm": (True, False),
}
def suggest_recipe(trial, *, families=None, prefix: str = "wisp") -> Recipe:
"""Sample one WISP recipe from the grammar's space with an ``optuna.trial.Trial``.
``families`` restricts the family choice (kernel names, e.g.
``["state_rocket_interval_hmm"]``). Parameter names are prefixed with ``prefix`` and
the family's short name, so the space is conditional on the family.
"""
import optuna
choices = tuple(families) if families is not None else SEARCH_SPACE["family"]
family = trial.suggest_categorical(f"{prefix}.family", choices)
short = _SHORT[family]
p: dict[str, Any] = {}
if family in ("har_sculpt_forest_hmm", "invariant_interval_shape_forest"):
p["classifier"] = trial.suggest_categorical(f"{prefix}.{short}.classifier", SEARCH_SPACE["forest_classifier"])
seg = trial.suggest_categorical(f"{prefix}.{short}.segment_sizes", SEARCH_SPACE["segment_sizes"])
p["segment_sizes"] = tuple(int(v) for v in seg.split(","))
p["symbolic_bins"] = trial.suggest_categorical(f"{prefix}.{short}.symbolic_bins", SEARCH_SPACE["symbolic_bins"])
if family == "invariant_interval_shape_forest":
p["n_shapelets"] = trial.suggest_categorical(f"{prefix}.gis.n_shapelets", SEARCH_SPACE["n_shapelets"])
elif family in ("rocket_sketch_ridge_hmm", "state_rocket_interval_hmm"):
p["classifier"] = trial.suggest_categorical(f"{prefix}.{short}.classifier", SEARCH_SPACE["linear_classifier"])
p["n_kernels"] = trial.suggest_categorical(f"{prefix}.{short}.n_kernels", SEARCH_SPACE["n_kernels"])
if family == "state_rocket_interval_hmm":
p["n_intervals"] = trial.suggest_categorical(f"{prefix}.cis.n_intervals", SEARCH_SPACE["n_intervals"])
for block, key in (("R", "use_rocket"), ("I", "use_intervals"), ("S", "use_state")):
p[key] = trial.suggest_categorical(f"{prefix}.cis.block_{block}", (True, False))
if not (p["use_rocket"] or p["use_intervals"] or p["use_state"]):
raise optuna.TrialPruned("CIS needs at least one representation block")
if p["use_rocket"] and p["use_intervals"] and p["use_state"]:
for key in ("use_rocket", "use_intervals", "use_state"):
p.pop(key) # the full CIS: identical to the grammar's template
elif family == "posterior_cpu_ensemble":
w = [trial.suggest_float(f"{prefix}.ese.weight{i}", 0.05, 1.0) for i in range(3)]
p["weights"] = tuple(float(v) / sum(w) for v in w)
p["use_hmm"] = trial.suggest_categorical(f"{prefix}.{short}.use_hmm", SEARCH_SPACE["use_hmm"])
return Recipe(family, p)
def recipe_operators(recipe: Recipe) -> list[str]:
"""Operator labels of a recipe (they enter the utility's complexity term).
Labels of the family's first grammar template, with the readout and decoder labels
replaced as the grammar's mutations do; CIS block subsets use the labels of the paper's
CIS subset recipes (``wisp.ablations``).
"""
from wisp.search.grammar import _FIRST, _TEMPLATES, _replace_classifier_ops
params = recipe.params
toggles = [params.get(k, True) for k in ("use_rocket", "use_intervals", "use_state")]
if recipe.family == "state_rocket_interval_hmm" and not all(toggles):
ops = ["zscore"]
if toggles[0]:
ops += ["random_conv", "ppv_pool"]
if toggles[1]:
ops += ["dyadic_intervals", "quantile_distribution"]
if toggles[2]:
ops += ["magnitude", "jerk", "spectral", "autocorr", "symbolic_transition"]
ops.append(params.get("classifier", "ridge"))
else:
ops = list(_TEMPLATES[_FIRST[recipe.family]].operators)
if "classifier" in params:
ops = _replace_classifier_ops(ops, params["classifier"])
ops = [o for o in ops if o not in ("hmm", "argmax")]
return ops + ["hmm" if recipe.use_hmm else "argmax"]
def evaluate_recipe(recipe: Recipe, train, valid, *, seed: int = 42, speed: str = "exact",
sequence: str = "subject", n_jobs: int = 1, cost_weight: float = 0.01,
robustness_weight: float = 0.1, force_hmm: bool | None = None) -> tuple[str, dict]:
"""Fit on ``train`` and score on ``valid`` with the search cascade; returns (status, result).
``train``/``valid`` are ``wisp.core.data.HARData`` with disjoint groups. This calls the
frozen ``wisp.search.cascade.evaluate_candidate_cascade`` (validation fit, four stress
views, anti-collapse check) with the speed-tier engine as its estimator factory; the
utility is ``result["predictive_utility"]`` (the cascade score), as in
WISP-Random/WISP-Evolution. ``force_hmm`` (True/False) fits the candidate with that HMM
setting instead of the recipe's own (``None``, the default).
"""
from wisp.search.cascade import evaluate_candidate_cascade
from wisp.search.grammar import CandidateSpec
from .._engine import build_kernel
spec = recipe.kernel_spec(seed=seed, n_jobs=n_jobs)
cand = CandidateSpec(name=spec["name"], params=spec["params"], operators=recipe_operators(recipe), route="cpu")
def factory(spec_dict):
params = dict(spec_dict["params"])
use_hmm = bool(params.get("use_hmm", True)) if force_hmm is None else bool(force_hmm)
model = build_kernel(Recipe(spec_dict["name"], params), use_hmm=use_hmm,
speed=speed, sequence=sequence, seed=int(params.get("seed", seed)),
n_jobs=int(params.get("n_jobs", n_jobs)))
if sequence == "recording":
model.calibration_data = (valid.X, valid.y) # confidence scaling chosen on validation, as in search
return model
try:
cascade = evaluate_candidate_cascade(cand, train, valid, cost_weight=cost_weight,
robustness_weight=robustness_weight, max_stress_transforms=4,
estimator_factory=factory)
except Exception as exc: # same policy as the search controller: failures consume budget
return "failed", {"predictive_utility": -1e9, "error": repr(exc), "candidate": cand.to_dict()}
result = cascade.to_dict()
result["predictive_utility"] = float(result.get("cascade_score", -1e9))
return ("ok" if cascade.accepted else "rejected"), result
def run_study(train, valid, *, n_trials: int, sampler=None, seed: int = 42, speed: str = "exact",
sequence: str = "subject", n_jobs: int = 1, cost_weight: float = 0.01,
robustness_weight: float = 0.1, study=None, families=None, force_hmm: bool | None = None):
"""Optuna study maximising the validation utility. Returns (study, best_spec, best_result, history).
Repeated proposals of an already evaluated recipe reuse its result (no refit) and still
count as trials. The winner is the accepted candidate with the highest utility.
"""
import optuna
if study is None:
study = optuna.create_study(direction="maximize",
sampler=sampler if sampler is not None else optuna.samplers.TPESampler(seed=seed))
cache: dict[str, tuple[str, dict]] = {}
history: list[dict] = []
def objective(trial):
recipe = suggest_recipe(trial, families=families)
key = recipe.to_json()
if key not in cache:
cache[key] = evaluate_recipe(recipe, train, valid, seed=seed, speed=speed, sequence=sequence,
n_jobs=n_jobs, cost_weight=cost_weight, robustness_weight=robustness_weight,
force_hmm=force_hmm)
repeat = False
else:
repeat = True
status, result = cache[key]
utility = float(result.get("predictive_utility", -1e9))
trial.set_user_attr("status", status)
trial.set_user_attr("recipe", key)
trial.set_user_attr("repeat", repeat)
history.append({"trial": trial.number, "recipe": json.loads(key), "status": status, "repeat": repeat,
"validation_utility": utility, "validation_macro_f1": result.get("macro_f1"),
"validation_worst_class_f1": result.get("worst_class_f1"),
"fit_seconds": None if repeat else result.get("fit_seconds"),
"predict_seconds": None if repeat else result.get("predict_seconds")})
if status == "failed" or not math.isfinite(utility):
return float("nan")
return utility
study.optimize(objective, n_trials=int(n_trials), catch=())
ok = [(json.loads(k), r) for k, (s, r) in cache.items() if s == "ok"]
if not ok:
raise RuntimeError("No accepted candidate in the Optuna study")
best_recipe, best_result = max(ok, key=lambda kv: float(kv[1]["predictive_utility"]))
best_spec = Recipe.from_dict(best_recipe).kernel_spec(seed=seed, n_jobs=n_jobs)
return study, best_spec, best_result, history