"""Optuna integration: WISP recipes as an Optuna search space (optional extra ``[optuna]``). The space is the existing CPU search grammar (``wisp.search.grammar``): * family: the seven kernel families; * readout: ExtraTrees / random forest / logistic regression for SS and GIS, Ridge / logistic regression for RC and CIS (the grammar's ``change_classifier``); * segment sizes (3 of {3,4,5,8,10,12,16,20,25}) and symbolic bins ({4,5,6,8,10}) for SS and GIS; convolution kernels ({128,...,768}) for RC and CIS; intervals ({8,...,48}) for CIS; shapelets ({16,...,64}) for GIS; ensemble weights for ESE; * CIS block switches R/I/S (the paper's CIS block subsets; at least one block on); * decoder: HMM or per-window argmax. Optuna only proposes. Each candidate is fitted and scored by the existing cascade (``wisp.search.cascade``) with the existing validation utility, exactly as in WISP-Random/WISP-Evolution; the test split is never seen. Use :func:`suggest_recipe` inside your own objective to tune a WISP recipe jointly with other hyperparameters. """ from __future__ import annotations import itertools import json import math from typing import Any from ..recipes import Recipe __all__ = ["suggest_recipe", "recipe_operators", "evaluate_recipe", "run_study", "SEARCH_SPACE"] _SHORT = {"har_sculpt_forest_hmm": "ss", "rocket_sketch_ridge_hmm": "rc", "symbolic_interval_forest": "so", "hybrid_cpu_ensemble": "cse", "invariant_interval_shape_forest": "gis", "state_rocket_interval_hmm": "cis", "posterior_cpu_ensemble": "ese"} _SEGMENT_POOL = (3, 4, 5, 8, 10, 12, 16, 20, 25) SEARCH_SPACE: dict[str, Any] = { "family": tuple(_SHORT), "forest_classifier": ("extratrees", "rf", "logreg"), "linear_classifier": ("ridge", "logreg"), "segment_sizes": tuple(",".join(map(str, c)) for c in itertools.combinations(_SEGMENT_POOL, 3)), "symbolic_bins": (4, 5, 6, 8, 10), "n_kernels": (128, 256, 384, 512, 768), "n_intervals": (8, 16, 24, 32, 48), "n_shapelets": (16, 24, 32, 48, 64), "use_hmm": (True, False), } def suggest_recipe(trial, *, families=None, prefix: str = "wisp") -> Recipe: """Sample one WISP recipe from the grammar's space with an ``optuna.trial.Trial``. ``families`` restricts the family choice (kernel names, e.g. ``["state_rocket_interval_hmm"]``). Parameter names are prefixed with ``prefix`` and the family's short name, so the space is conditional on the family. """ import optuna choices = tuple(families) if families is not None else SEARCH_SPACE["family"] family = trial.suggest_categorical(f"{prefix}.family", choices) short = _SHORT[family] p: dict[str, Any] = {} if family in ("har_sculpt_forest_hmm", "invariant_interval_shape_forest"): p["classifier"] = trial.suggest_categorical(f"{prefix}.{short}.classifier", SEARCH_SPACE["forest_classifier"]) seg = trial.suggest_categorical(f"{prefix}.{short}.segment_sizes", SEARCH_SPACE["segment_sizes"]) p["segment_sizes"] = tuple(int(v) for v in seg.split(",")) p["symbolic_bins"] = trial.suggest_categorical(f"{prefix}.{short}.symbolic_bins", SEARCH_SPACE["symbolic_bins"]) if family == "invariant_interval_shape_forest": p["n_shapelets"] = trial.suggest_categorical(f"{prefix}.gis.n_shapelets", SEARCH_SPACE["n_shapelets"]) elif family in ("rocket_sketch_ridge_hmm", "state_rocket_interval_hmm"): p["classifier"] = trial.suggest_categorical(f"{prefix}.{short}.classifier", SEARCH_SPACE["linear_classifier"]) p["n_kernels"] = trial.suggest_categorical(f"{prefix}.{short}.n_kernels", SEARCH_SPACE["n_kernels"]) if family == "state_rocket_interval_hmm": p["n_intervals"] = trial.suggest_categorical(f"{prefix}.cis.n_intervals", SEARCH_SPACE["n_intervals"]) for block, key in (("R", "use_rocket"), ("I", "use_intervals"), ("S", "use_state")): p[key] = trial.suggest_categorical(f"{prefix}.cis.block_{block}", (True, False)) if not (p["use_rocket"] or p["use_intervals"] or p["use_state"]): raise optuna.TrialPruned("CIS needs at least one representation block") if p["use_rocket"] and p["use_intervals"] and p["use_state"]: for key in ("use_rocket", "use_intervals", "use_state"): p.pop(key) # the full CIS: identical to the grammar's template elif family == "posterior_cpu_ensemble": w = [trial.suggest_float(f"{prefix}.ese.weight{i}", 0.05, 1.0) for i in range(3)] p["weights"] = tuple(float(v) / sum(w) for v in w) p["use_hmm"] = trial.suggest_categorical(f"{prefix}.{short}.use_hmm", SEARCH_SPACE["use_hmm"]) return Recipe(family, p) def recipe_operators(recipe: Recipe) -> list[str]: """Operator labels of a recipe (they enter the utility's complexity term). Labels of the family's first grammar template, with the readout and decoder labels replaced as the grammar's mutations do; CIS block subsets use the labels of the paper's CIS subset recipes (``wisp.ablations``). """ from wisp.search.grammar import _FIRST, _TEMPLATES, _replace_classifier_ops params = recipe.params toggles = [params.get(k, True) for k in ("use_rocket", "use_intervals", "use_state")] if recipe.family == "state_rocket_interval_hmm" and not all(toggles): ops = ["zscore"] if toggles[0]: ops += ["random_conv", "ppv_pool"] if toggles[1]: ops += ["dyadic_intervals", "quantile_distribution"] if toggles[2]: ops += ["magnitude", "jerk", "spectral", "autocorr", "symbolic_transition"] ops.append(params.get("classifier", "ridge")) else: ops = list(_TEMPLATES[_FIRST[recipe.family]].operators) if "classifier" in params: ops = _replace_classifier_ops(ops, params["classifier"]) ops = [o for o in ops if o not in ("hmm", "argmax")] return ops + ["hmm" if recipe.use_hmm else "argmax"] def evaluate_recipe(recipe: Recipe, train, valid, *, seed: int = 42, speed: str = "exact", sequence: str = "subject", n_jobs: int = 1, cost_weight: float = 0.01, robustness_weight: float = 0.1, force_hmm: bool | None = None) -> tuple[str, dict]: """Fit on ``train`` and score on ``valid`` with the search cascade; returns (status, result). ``train``/``valid`` are ``wisp.core.data.HARData`` with disjoint groups. This calls the frozen ``wisp.search.cascade.evaluate_candidate_cascade`` (validation fit, four stress views, anti-collapse check) with the speed-tier engine as its estimator factory; the utility is ``result["predictive_utility"]`` (the cascade score), as in WISP-Random/WISP-Evolution. ``force_hmm`` (True/False) fits the candidate with that HMM setting instead of the recipe's own (``None``, the default). """ from wisp.search.cascade import evaluate_candidate_cascade from wisp.search.grammar import CandidateSpec from .._engine import build_kernel spec = recipe.kernel_spec(seed=seed, n_jobs=n_jobs) cand = CandidateSpec(name=spec["name"], params=spec["params"], operators=recipe_operators(recipe), route="cpu") def factory(spec_dict): params = dict(spec_dict["params"]) use_hmm = bool(params.get("use_hmm", True)) if force_hmm is None else bool(force_hmm) model = build_kernel(Recipe(spec_dict["name"], params), use_hmm=use_hmm, speed=speed, sequence=sequence, seed=int(params.get("seed", seed)), n_jobs=int(params.get("n_jobs", n_jobs))) if sequence == "recording": model.calibration_data = (valid.X, valid.y) # confidence scaling chosen on validation, as in search return model try: cascade = evaluate_candidate_cascade(cand, train, valid, cost_weight=cost_weight, robustness_weight=robustness_weight, max_stress_transforms=4, estimator_factory=factory) except Exception as exc: # same policy as the search controller: failures consume budget return "failed", {"predictive_utility": -1e9, "error": repr(exc), "candidate": cand.to_dict()} result = cascade.to_dict() result["predictive_utility"] = float(result.get("cascade_score", -1e9)) return ("ok" if cascade.accepted else "rejected"), result def run_study(train, valid, *, n_trials: int, sampler=None, seed: int = 42, speed: str = "exact", sequence: str = "subject", n_jobs: int = 1, cost_weight: float = 0.01, robustness_weight: float = 0.1, study=None, families=None, force_hmm: bool | None = None): """Optuna study maximising the validation utility. Returns (study, best_spec, best_result, history). Repeated proposals of an already evaluated recipe reuse its result (no refit) and still count as trials. The winner is the accepted candidate with the highest utility. """ import optuna if study is None: study = optuna.create_study(direction="maximize", sampler=sampler if sampler is not None else optuna.samplers.TPESampler(seed=seed)) cache: dict[str, tuple[str, dict]] = {} history: list[dict] = [] def objective(trial): recipe = suggest_recipe(trial, families=families) key = recipe.to_json() if key not in cache: cache[key] = evaluate_recipe(recipe, train, valid, seed=seed, speed=speed, sequence=sequence, n_jobs=n_jobs, cost_weight=cost_weight, robustness_weight=robustness_weight, force_hmm=force_hmm) repeat = False else: repeat = True status, result = cache[key] utility = float(result.get("predictive_utility", -1e9)) trial.set_user_attr("status", status) trial.set_user_attr("recipe", key) trial.set_user_attr("repeat", repeat) history.append({"trial": trial.number, "recipe": json.loads(key), "status": status, "repeat": repeat, "validation_utility": utility, "validation_macro_f1": result.get("macro_f1"), "validation_worst_class_f1": result.get("worst_class_f1"), "fit_seconds": None if repeat else result.get("fit_seconds"), "predict_seconds": None if repeat else result.get("predict_seconds")}) if status == "failed" or not math.isfinite(utility): return float("nan") return utility study.optimize(objective, n_trials=int(n_trials), catch=()) ok = [(json.loads(k), r) for k, (s, r) in cache.items() if s == "ok"] if not ok: raise RuntimeError("No accepted candidate in the Optuna study") best_recipe, best_result = max(ok, key=lambda kv: float(kv[1]["predictive_utility"])) best_spec = Recipe.from_dict(best_recipe).kernel_spec(seed=seed, n_jobs=n_jobs) return study, best_spec, best_result, history