"""WISPSelect and WISPSearch: choose a recipe on validation data, then refit on train+validation. Both wrap the existing implementations without changing their rules: * ``WISPSelect`` calls ``wisp_release.selection.fit_selector`` (WISP-Select5/Select7). * ``WISPSearch(strategy="random"|"evolution")`` calls ``wisp_fast.search.run_search``, i.e. the search controller of ``wisp.search`` (proposal grammar, cascade, stress views and validation utility unchanged) with candidates fitted by the speed-tier engine. * ``WISPSearch(strategy="optuna")`` proposes recipes with Optuna from the same grammar and scores each one with the same validation utility (``wisp.integrations.optuna``). * ``WISPSearch(strategy="funnel")`` is WISP-Funnel (``wisp_fast.funnel``): the Random and Evolution controllers in one run, ranking proposals on a training-only proxy, verifying a shortlist on the full splits and refitting each controller's winner on train+validation. ``decoder`` (default ``"adaptive"``) sets the decoder of the final model; see :mod:`wisp._adaptive` for the rule. ``decoder="recipe"`` is the paper configuration. Neither class accepts test data. Selection cost and final-model cost are recorded separately (``selection_seconds_`` and ``final_fit_seconds_``). """ from __future__ import annotations import contextlib import math import tempfile import threading import time from pathlib import Path import numpy as np from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.model_selection import GroupShuffleSplit from sklearn.utils.validation import check_is_fitted from . import _adaptive as adaptive from ._validation import (SEQUENCES, SPEEDS, as_windows, check_aligned, check_choice, check_disjoint_groups, check_targets, encode_labels, kernel_context) from .classifier import WISPClassifier from .recipes import Recipe, get _PATCH_LOCK = threading.Lock() _SHORT = {"wisp_ss": "ss", "wisp_rc": "rc", "wisp_so": "so", "wisp_cse": "cse", "wisp_gis": "gis", "wisp_cis": "cis", "wisp_ese": "ese"} #: messages of the "no candidate passed validation" errors handled by ``on_no_candidate="default"`` NO_CANDIDATE_ERRORS = ("No accepted initial candidate", "No full-validation candidate passed", "No accepted candidate in the Optuna study") _CONTROLLERS = ("random", "evolution") def _har(X, codes, groups, time_index, names, source, sampling_rate_hz=None, study_setting=None): from wisp.core.data import HARData if time_index is None: time_index = np.arange(len(X), dtype=np.float64) metadata = None if study_setting is None else {"study_setting": str(study_setting)} rate = None if sampling_rate_hz is None else float(sampling_rate_hz) return HARData(X=X, y=codes, subject=np.asarray(groups), time_index=np.asarray(time_index), label_names=list(names), source=source, sampling_rate_hz=rate, metadata=metadata) class _Parts: """Training and validation splits plus all fitting data (``full``) for the default-recipe fallback. ``full`` is ``None`` for an explicit validation tuple (training then validation windows are used); otherwise it is ``X`` in its original row order and ``valid_rows`` marks its validation rows. """ def __init__(self, train, valid, full=None, valid_rows=None): self.train, self.valid, self.full, self.valid_rows = train, valid, full, valid_rows class _SplitMixin: """Shared train/validation handling for WISPSelect and WISPSearch.""" def _splits(self, X, y, groups, time_index, validation, validation_size, sampling_rate_hz=None, study_setting=None): check_choice("speed", self.speed, SPEEDS) check_choice("sequence", self.sequence, SEQUENCES) if not isinstance(self.n_jobs, (int, np.integer)) or self.n_jobs < 1: raise ValueError("n_jobs must be a positive integer") Xw = (as_windows(X, self.layout)) y = check_targets(self, y) if groups is None: raise ValueError("groups (sequence ID of every window) is required: validation must use " "different sequences than training") groups = check_aligned("groups", groups, len(Xw)) time_index = check_aligned("time_index", time_index, len(Xw)) if len(y) != len(Xw): raise ValueError(f"y has {len(y)} labels for {len(Xw)} windows") if validation is not None and validation_size is not None: raise ValueError("Pass either validation=(X_val, y_val, groups_val[, time_val]) or validation_size") rows = adaptive._rows_mask(validation, len(Xw)) if validation is not None else None full = (Xw, y, groups, time_index) if validation is not None and rows is None: if not isinstance(validation, (tuple, list)) or len(validation) not in (3, 4): raise ValueError("validation must be (X_val, y_val, groups_val), (X_val, y_val, groups_val, time_val) " "or validation rows of X (integer indices or a boolean mask)") Xv = (as_windows(validation[0], self.layout)) yv = np.asarray(validation[1]) gv = check_aligned("validation groups", validation[2], len(Xv)) tv = check_aligned("validation time_index", validation[3] if len(validation) == 4 else None, len(Xv)) if len(yv) != len(Xv): raise ValueError("validation labels and windows differ in length") if Xv.shape[1:] != Xw.shape[1:]: raise ValueError("training and validation windows must share (T, C)") full = None elif rows is not None: Xv, yv, gv = Xw[rows], y[rows], groups[rows] tv = None if time_index is None else time_index[rows] Xw, y, groups = Xw[~rows], y[~rows], groups[~rows] time_index = None if time_index is None else time_index[~rows] else: size = 0.125 if validation_size is None else float(validation_size) if not 0 < size < 1: raise ValueError("validation_size must be in (0, 1)") split = GroupShuffleSplit(n_splits=1, test_size=size, random_state=int(self.random_state)) tr, va = next(split.split(Xw, y, groups)) rows = np.zeros(len(Xw), dtype=bool) rows[va] = True Xv, yv, gv = Xw[va], y[va], groups[va] tv = None if time_index is None else time_index[va] Xw, y, groups = Xw[tr], y[tr], groups[tr] time_index = None if time_index is None else time_index[tr] check_disjoint_groups(groups, gv) classes = self.classes if self.classes is not None else np.unique(np.concatenate([y, yv])) self.classes_, codes = encode_labels(y, classes) _, codes_v = encode_labels(yv, self.classes_) names = [str(c) for c in self.classes_.tolist()] info = dict(sampling_rate_hz=sampling_rate_hz, study_setting=study_setting) train = _har(Xw, codes, groups, time_index, names, "train", **info) valid = _har(Xv, codes_v, gv, tv, names, "validation", **info) return _Parts(train, valid, full, rows) def _decoder(self): return adaptive.normalize_decoder(self.decoder) @staticmethod def _candidate_hmm(decoder, train): """HMM flag forced on every candidate: None keeps each recipe's own flag.""" if decoder == "hmm": return True if decoder == "none" or (decoder == "adaptive" and not adaptive.has_sequences(train.subject)): return False return None def _refit_classifier(self, recipe, seed, decoder="recipe"): return WISPClassifier(recipe, decoder=decoder, classes=self.classes_, layout="NTC", speed=self.speed, sequence=self.sequence, n_jobs=self.n_jobs, random_state=int(seed)) def _adaptive_classifier(self, recipe, seed, train, valid): """``decoder="adaptive"`` for a selected recipe: fit on train, decide on valid, refit on both.""" est = self._refit_classifier(recipe, seed, decoder="adaptive") est.fit(train.X, self.classes_[train.y], groups=train.subject, time_index=train.time_index, validation=(valid.X, self.classes_[valid.y], valid.subject, valid.time_index)) return est def _default_recipe(self, parts, decoder, seed, exc): """``on_no_candidate="default"``: the default CIS recipe on all fitting data.""" est = self._refit_classifier(get("cis"), seed, decoder=decoder) start = time.perf_counter() if parts.full is None: t, v = parts.train, parts.valid est.fit(t.X, self.classes_[t.y], groups=t.subject, time_index=t.time_index, validation=(v.X, self.classes_[v.y], v.subject, v.time_index)) else: X, y, g, ti = parts.full est.fit(X, y, groups=g, time_index=ti, validation=parts.valid_rows) self.fallback_ = {"reason": str(exc), "recipe": "cis", "fit_scope": "training and validation windows (all fitting data)"} self.best_recipe_ = est.recipe_ self.best_seed_ = int(seed) self.best_validation_ = None self._finish(est, final_fit=time.perf_counter() - start) return est def _finish(self, est, *, final_fit): self.best_estimator_ = est self.final_fit_seconds_ = float(final_fit) self.decoder_ = est.decoder_ self.decoder_scores_ = est.decoder_scores_ self.decoder_reason_ = est.decoder_reason_ def predict(self, X, groups=None, time_index=None): check_is_fitted(self, "best_estimator_") return self.best_estimator_.predict((as_windows(X, self.layout)), groups=groups, time_index=time_index) def predict_proba(self, X): check_is_fitted(self, "best_estimator_") return self.best_estimator_.predict_proba((as_windows(X, self.layout))) def __sklearn_tags__(self): tags = super().__sklearn_tags__() tags.input_tags.three_d_array = True return tags @contextlib.contextmanager def _record_builds(store, force_hmm=None): """Record every family fitted by fit_selector (for per-candidate fit times). Replaces the module-level default builder with a recording wrapper around the same function, so fit_selector's code path (including its size tie-break) is unchanged. ``force_hmm`` (True/False) builds every family with that HMM setting instead of its own (``decoder="hmm"``/``"none"``, or ``"adaptive"`` without sequences); ``None`` changes nothing. """ import wisp_release.selection as sel original, reference = sel.build_family, sel.build_reference_family def flagged(kwargs): if force_hmm is None: return kwargs return {**kwargs, "overrides": {**(kwargs.get("overrides") or {}), "use_hmm": bool(force_hmm)}} def recording(*args, **kwargs): model = original(*args, **flagged(kwargs)) store.append((args[0] if args else kwargs.get("method_id"), model)) return model def sized(*args, **kwargs): return reference(*args, **flagged(kwargs)) with _PATCH_LOCK: sel.build_family = recording if force_hmm is not None: sel.build_reference_family = sized try: yield finally: sel.build_family = original sel.build_reference_family = reference def _decide_on_model(model, valid, n_classes): """Decision from a model fitted on training windows with an HMM smoother.""" with kernel_context(): P = model.predict_proba(valid.X) hmm = adaptive.hmm_labels(model, P, valid.subject, valid.time_index) return adaptive.compare(valid.y, hmm, adaptive.argmax_labels(P), n_classes) class WISPSelect(_SplitMixin, ClassifierMixin, BaseEstimator): """WISP-Select5 / WISP-Select7: validation selection among fixed families, then refit. Candidates are fitted on training windows and scored on validation windows; the winner is chosen by validation macro-F1, then full-vocabulary worst-class F1, then serialized model size, then the order SS, RC, SO, CSE, GIS, CIS, ESE. A fresh winner is fitted on train+validation. ``menu=5``: SS, RC, SO, GIS, CIS. ``menu=7`` adds CSE and ESE. decoder : {"adaptive", "recipe", "hmm", "none"}, default="adaptive" ``"recipe"``: every family with its own decoder (the paper). ``"adaptive"``: the selection is the paper's; then the winner's training fit decodes the validation windows with and without the HMM, and the HMM is kept only if its macro-F1 is strictly higher (the refit uses the chosen decoder). Without sequences (no training group with two or more windows) candidates are scored and refitted with argmax. ``"hmm"``/``"none"`` force that decoder for scoring and refit. """ def __init__(self, menu=5, *, decoder="adaptive", classes=None, layout="NTC", speed="exact", sequence="subject", n_jobs=1, random_state=42): self.menu = menu self.decoder = decoder self.classes = classes self.layout = layout self.speed = speed self.sequence = sequence self.n_jobs = n_jobs self.random_state = random_state def fit(self, X, y, groups=None, time_index=None, validation=None, validation_size=None): """``validation=(X_val, y_val, groups_val[, time_val])`` (the paper protocol), validation rows of ``X``, or ``validation_size`` to hold out that fraction of the groups (default 0.125).""" if self.menu not in (5, 7): raise ValueError("menu must be 5 or 7") decoder = self._decoder() from wisp_release.selection import fit_selector parts = self._splits(X, y, groups, time_index, validation, validation_size) train, valid = parts.train, parts.valid force = self._candidate_hmm(decoder, train) built = [] start = time.perf_counter() with _record_builds(built, force), kernel_context(): model, report = fit_selector(f"wisp_select{self.menu}", train, valid, seed=int(self.random_state), n_jobs=int(self.n_jobs), speed=self.speed, sequence=self.sequence) total = time.perf_counter() - start final_fit = selector_refit = float(getattr(model, "fit_seconds_", 0.0)) winner = report["selected_method_id"] fit_times = {} for family, m in built[:-1]: fit_times[family] = float(getattr(m, "fit_seconds_", math.nan)) self.history_ = [{**{k: v for k, v in rec.items() if k != "confusion_matrix"}, "recipe": _SHORT[rec["method_id"]], "fit_seconds": fit_times.get(rec["method_id"])} for rec in report["validation_candidates"]] self.selection_report_ = report self.best_recipe_ = get(_SHORT[winner]) recipe_hmm = bool(model.use_hmm) use_hmm, reason, scores, decision_s = recipe_hmm, adaptive.FIXED, None, 0.0 if force is not None: reason = adaptive.NO_SEQUENCES if decoder == "adaptive" else adaptive.FIXED elif decoder == "adaptive": t0 = time.perf_counter() trained = dict(built[:-1])[winner] choice, scores = _decide_on_model(trained, valid, len(self.classes_)) decision_s = time.perf_counter() - t0 reason = adaptive.VALIDATION if (choice == "hmm") != use_hmm: model, final_fit = self._refit_family(winner, train, valid, choice == "hmm", report) use_hmm = choice == "hmm" est = self._refit_classifier(self.best_recipe_, self.random_state) est._adopt(model, classes=self.classes_, input_shape=train.X.shape[1:], use_hmm=use_hmm, fit_seconds=final_fit, reason=reason, scores=scores, decision_seconds=decision_s) self._finish(est, final_fit=final_fit) self.decision_seconds_ = decision_s self.selection_seconds_ = total - selector_refit self.has_paper_results_ = bool(self.speed == "exact" and self.sequence == "subject" and force is None and use_hmm == recipe_hmm) return self def _refit_family(self, winner, train, valid, use_hmm, report): """Refit the selected family on train+validation with the chosen decoder.""" from wisp.core.data import concatenate_har from wisp_release.methods import build_family combined = concatenate_har(train, valid, source="train+validation") model = build_family(winner, seed=int(self.random_state), n_jobs=int(self.n_jobs), speed=self.speed, sequence=self.sequence, overrides={"use_hmm": bool(use_hmm)}) with kernel_context(): model.fit(combined.X, combined.y, groups=combined.subject, time_index=combined.time_index, n_classes=combined.n_classes) if use_hmm and report.get("confidence_scale") is not None: model.set_confidence_scale(report["confidence_scale"]) return model, float(getattr(model, "fit_seconds_", 0.0)) class WISPSearch(_SplitMixin, ClassifierMixin, BaseEstimator): """Recipe search on validation data, then one refit of the winner on train+validation. strategy : {"random", "evolution", "optuna", "funnel"} ``"random"``/``"evolution"`` are WISP-Random/WISP-Evolution (paper implementations). ``"optuna"`` proposes recipes with Optuna (default sampler TPE) from the same grammar and scores them with the same validation utility; it has no paper results. ``"funnel"`` is WISP-Funnel, the fast joint Random + Evolution search: both controllers propose ``budget`` candidates each from the CIS family and rank them on a training-only proxy (at most 4,096 training and 1,024 validation windows, a shared Ridge feature dictionary, cost weight 0), a diversified shortlist of 4 per controller is verified on the full splits with the screened Ridge solver, and each controller's winner is refitted on train+validation. It has no paper results; its benchmark record is in docs/RECIPES.md. controller : {"random", "evolution"}, default="random" ``strategy="funnel"`` only: which controller's model predicts. Both are fitted in the same run and kept in ``funnel_estimators_``. budget : int, default=329 Number of candidates evaluated (paper: 329, i.e. 9 initial + 20 generations x 16); for the funnel, proposals per controller. population : int, default=24 Population of the random/evolution controller (paper: 24). sampler : optuna sampler or None Only for ``strategy="optuna"``; ``None`` uses ``TPESampler(seed=random_state)``. decoder : {"adaptive", "recipe", "hmm", "none"}, default="adaptive" Decoder of the final model. ``"recipe"``: the winner's own HMM flag (the paper). ``"adaptive"``: the search is unchanged; then the winner fitted on training windows decodes the validation windows with and without the HMM, and the HMM is kept only if its macro-F1 is strictly higher. Without sequences (no training group with two or more windows) candidates are scored and refitted with argmax. ``"hmm"``/``"none"`` force that decoder for scoring and refit. on_no_candidate : {"raise", "default"}, default="raise" What happens when no candidate passes validation (every candidate rejected or collapsed). ``"raise"`` re-raises the search's error; ``"default"`` fits the default CIS recipe (``wisp.recipes.get("cis")``) on all fitting data instead and records the reason in ``fallback_``. n_jobs : int, default=1 Threads for tree ensembles inside each candidate. For the funnel: controller threads (1 runs the two controllers in series, 2 or more runs them in parallel; candidates and BLAS use one thread each). The results do not depend on it. output_dir : path or None Where the controller writes its candidate ledger; a temporary directory when None. For the funnel, ``speed``, ``sequence``, ``cost_weight`` and ``robustness_weight`` are fixed by its protocol (``sequence`` must be ``"subject"``). The paper's single search (one dataset, fold and seed, 329 candidates) had median cumulative candidate fitting time of 47.5 h (Random) and 61.3 h (Evolution). Try the fixed recipes and WISPSelect first (docs/RECIPES.md). """ def __init__(self, strategy="random", *, controller="random", budget=329, population=24, sampler=None, decoder="adaptive", on_no_candidate="raise", classes=None, layout="NTC", speed="exact", sequence="subject", n_jobs=1, random_state=42, cost_weight=0.01, robustness_weight=0.1, output_dir=None): self.strategy = strategy self.controller = controller self.budget = budget self.population = population self.sampler = sampler self.decoder = decoder self.on_no_candidate = on_no_candidate self.classes = classes self.layout = layout self.speed = speed self.sequence = sequence self.n_jobs = n_jobs self.random_state = random_state self.cost_weight = cost_weight self.robustness_weight = robustness_weight self.output_dir = output_dir def fit(self, X, y, groups=None, time_index=None, validation=None, validation_size=None, sampling_rate_hz=None, study_setting=None): """Search on ``validation`` (or a ``validation_size`` hold-out of the groups), then refit. ``sampling_rate_hz`` and ``study_setting`` describe the recordings for the search controllers' dataset fingerprint, which shapes their proposals (window duration from the sampling rate; a study setting containing "free", "home", "natural" or "wild" marks free-living data). The paper searches and the funnel benchmark record used the values of the WristHARBench exports (``wisp.datasets`` folds expose ``sampling_rate_hz`` and ``study_setting``). When omitted, the fingerprint assumes 50 Hz and an unknown setting. """ check_choice("strategy", self.strategy, ("random", "evolution", "optuna", "funnel")) check_choice("controller", self.controller, _CONTROLLERS) check_choice("on_no_candidate", self.on_no_candidate, ("raise", "default")) if not isinstance(self.budget, (int, np.integer)) or self.budget < 1: raise ValueError("budget must be a positive integer") decoder = self._decoder() if self.strategy == "funnel" and self.sequence != "subject": raise ValueError("strategy='funnel' supports sequence='subject' only") parts = self._splits(X, y, groups, time_index, validation, validation_size, sampling_rate_hz, study_setting) train, valid = parts.train, parts.valid force = self._candidate_hmm(decoder, train) self.fallback_ = None self.decision_seconds_ = 0.0 if self.strategy == "funnel": return self._fit_funnel(parts, decoder, force) start = time.perf_counter() try: with kernel_context(): if self.strategy == "optuna": best_spec, best_result = self._fit_optuna(train, valid, force) else: best_spec, best_result = self._fit_controller(train, valid, force) except RuntimeError as exc: if self.on_no_candidate != "default" or not any(m in str(exc) for m in NO_CANDIDATE_ERRORS): raise self.selection_seconds_ = time.perf_counter() - start self._default_recipe(parts, decoder, self.random_state, exc) self.has_paper_results_ = False return self self.selection_seconds_ = time.perf_counter() - start params = dict(best_spec["params"]) seed = int(params.pop("seed", self.random_state)) params.pop("n_jobs", None) self.best_recipe_ = Recipe(best_spec["name"], params) self.best_seed_ = seed self.best_validation_ = best_result if decoder == "adaptive" and force is None: est = self._adaptive_classifier(self.best_recipe_, seed, train, valid) self.decision_seconds_ = est.decision_seconds_ else: combined = _concat(train, valid) est = self._refit_classifier(self.best_recipe_, seed, decoder="recipe" if force is None else ("hmm" if force else "none")) est.fit(combined.X, self.classes_[combined.y], groups=combined.subject, time_index=combined.time_index) if force is not None: est.decoder_reason_ = adaptive.NO_SEQUENCES if decoder == "adaptive" else adaptive.FIXED if self.sequence == "recording" and best_result.get("confidence_scale") is not None: # a train+validation refit reuses the factor chosen for the train-only candidate with kernel_context(): est.model_.set_confidence_scale(best_result["confidence_scale"]) self._finish(est, final_fit=est.fit_seconds_) recipe_decoder = "hmm" if self.best_recipe_.use_hmm else "none" self.has_paper_results_ = bool(self.strategy in ("random", "evolution") and self.speed == "exact" and self.sequence == "subject" and int(self.budget) == 329 and int(self.population) == 24 and force is None and est.decoder_ == recipe_decoder) return self # ------------------------------------------------------------------ strategies def _fit_controller(self, train, valid, force=None): from wisp_fast.search import run_search from ._engine import RIDGE_SOLVER budget, pop = int(self.budget), int(self.population) if pop < 3: raise ValueError("population must be at least 3") per_generation = max(pop - max(2, pop // 3), 1) generations = max(1, -(-(budget - min(9, budget)) // per_generation)) with _output_dir(self.output_dir) as out: options = dict(output=out, population=pop, generations=generations, max_evaluations=budget, seed=int(self.random_state), candidate_workers=1, n_jobs_per_candidate=int(self.n_jobs), ridge_solver=RIDGE_SOLVER[self.speed], sequence=self.sequence, cost_weight=float(self.cost_weight), robustness_weight=float(self.robustness_weight)) if force is None: summary = run_search(f"wisp_{self.strategy}", train, valid, None, **options) else: summary = _run_search_forced(f"wisp_{self.strategy}", train, valid, force, **options) rows = [_ledger_row(r) for r in _read_ledger(Path(out) / "program_database.jsonl")] self.history_ = rows self.search_summary_ = {k: summary[k] for k in ("n_rows", "n_ok", "n_evaluated", "search_config")} best = summary["best"] return best["spec"], best["result"] def _fit_optuna(self, train, valid, force=None): from .integrations.optuna import run_study study, best_spec, best_result, history = run_study( train, valid, n_trials=int(self.budget), sampler=self.sampler, seed=int(self.random_state), speed=self.speed, sequence=self.sequence, n_jobs=int(self.n_jobs), cost_weight=float(self.cost_weight), robustness_weight=float(self.robustness_weight), force_hmm=force) self.study_ = study self.history_ = history return best_spec, best_result def _fit_funnel(self, parts, decoder, force): """WISP-Funnel: both controllers in one run; ``controller`` picks the predicting model.""" from wisp.core.data import concatenate_har from wisp_fast.funnel.engine import FORCE_HMM from wisp_fast.funnel.joint import fit_joint_funnel train, valid = parts.train, parts.valid if int(self.population) < 3: raise ValueError("population must be at least 3") workers = 2 if int(self.n_jobs) >= 2 else 1 start = time.perf_counter() token = FORCE_HMM.set(force) try: with _output_dir(self.output_dir) as out, kernel_context(): try: models, summaries, receipt = fit_joint_funnel( train, valid, output=out, seed=int(self.random_state), evaluations=int(self.budget), population=int(self.population), workers=workers) except RuntimeError as exc: if self.on_no_candidate != "default" or not any(m in str(exc) for m in NO_CANDIDATE_ERRORS): raise models = None failure = exc else: rows = {c: [_ledger_row(r) for r in _read_ledger(Path(out) / f"wisp_{c}" / "proxy" / "program_database.jsonl")] for c in _CONTROLLERS} finally: FORCE_HMM.reset(token) funnel_s = time.perf_counter() - start self.funnel_seconds_ = funnel_s if models is None: self.selection_seconds_ = funnel_s est = self._default_recipe(parts, decoder, self.random_state, failure) self.funnel_estimators_ = {c: est for c in _CONTROLLERS} self.funnel_summary_, self.funnel_receipt_, self.history_ = None, None, [] self.has_paper_results_ = False return self combined = None estimators, info, decided = {}, {}, {} for c in _CONTROLLERS: summary = summaries[f"wisp_{c}"] spec = summary["winner_recipe"] params = dict(spec["params"]) seed = int(params.pop("seed", self.random_state)) params.pop("n_jobs", None) recipe = Recipe(spec["name"], params) recipe_hmm = bool(spec["params"].get("use_hmm", True)) if force is None else bool(force) model, final_fit = models[f"wisp_{c}"], float(summary["refit_seconds"]) use_hmm, reason, scores, decision_s = recipe_hmm, adaptive.FIXED, None, 0.0 if force is not None and decoder == "adaptive": reason = adaptive.NO_SEQUENCES elif decoder == "adaptive": key = _spec_key(spec) if key not in decided: decided[key] = _funnel_decision(spec, train, valid, len(self.classes_)) choice, scores, decision_s = decided[key] use_hmm, reason = choice == "hmm", adaptive.VALIDATION if use_hmm != recipe_hmm: if combined is None: combined = concatenate_har(train, valid, source="train+validation") model, final_fit = _funnel_refit(spec, combined, use_hmm) est = self._refit_classifier(recipe, seed) est._adopt(model, classes=self.classes_, input_shape=train.X.shape[1:], use_hmm=use_hmm, fit_seconds=final_fit, reason=reason, scores=scores, decision_seconds=decision_s) estimators[c] = est info[c] = {k: summary.get(k) for k in ("proxy_seconds", "verification_seconds", "refit_seconds", "n_proposals", "n_proxy_ok", "n_unique_proxy_ok", "n_full_verified", "n_full_ok", "proxy_winner")} info[c].update(winner_id=summary["best"]["id"], recipe=recipe, seed=seed, validation=summary["best"]["result"]) chosen = self.controller best = summaries[f"wisp_{chosen}"] self.funnel_estimators_ = estimators self.funnel_summary_ = info self.funnel_receipt_ = receipt self.history_ = rows[chosen] self.best_recipe_ = info[chosen]["recipe"] self.best_seed_ = info[chosen]["seed"] self.best_validation_ = best["best"]["result"] self.selection_seconds_ = float(best["proxy_seconds"]) + float(best["verification_seconds"]) self.decision_seconds_ = estimators[chosen].decision_seconds_ self._finish(estimators[chosen], final_fit=estimators[chosen].fit_seconds_) self.has_paper_results_ = False return self def _spec_key(spec): import json return json.dumps([spec["name"], spec["params"]], sort_keys=True) def _funnel_decision(spec, train, valid, n_classes): """Fit the funnel winner on training windows with the HMM (as in its verification), decide on valid.""" from wisp_fast.funnel.selection import fit_on from wisp_fast.funnel.shared_gram import verify_candidate start = time.perf_counter() hmm_spec = {**spec, "params": {**spec["params"], "use_hmm": True}} with kernel_context(): model = fit_on(verify_candidate(hmm_spec, mode="exact", budget=None, ridge_solver="screened"), train) choice, scores = _decide_on_model(model, valid, n_classes) return choice, scores, time.perf_counter() - start def _funnel_refit(spec, combined, use_hmm): """Refit a funnel winner on train+validation with the chosen decoder (the engine's refit factory).""" from wisp_fast.funnel.search import make_candidate from wisp_fast.funnel.selection import fit_on flipped = {**spec, "params": {**spec["params"], "use_hmm": bool(use_hmm)}} start = time.perf_counter() with kernel_context(): model = fit_on(make_candidate(flipped, mode="exact", budget=None, ridge_solver="screened"), combined) return model, time.perf_counter() - start def _run_search_forced(method, train, valid, force_hmm, *, output, population, generations, max_evaluations, seed, candidate_workers, n_jobs_per_candidate, ridge_solver, sequence, cost_weight, robustness_weight): """``wisp_fast.search.run_search`` with every candidate built with HMM flag ``force_hmm``. The controller, its proposals, ledger and utility are unchanged; only the candidate factory overrides ``use_hmm`` (``decoder="hmm"``/``"none"``, or ``"adaptive"`` without sequences). """ from dataclasses import replace from wisp.search.profiles import get_profile from wisp_fast.search import _AcceleratedSearch, _configure from wisp_fast.selection import check_splits check_splits(train, valid) profile = replace(get_profile(method), max_evaluations=int(max_evaluations)) search = _AcceleratedSearch(train=train, valid=valid, output=output, profile=profile, test=None, population=population, generations=generations, seed=seed, cost_weight=cost_weight, robustness_weight=robustness_weight, candidate_workers=candidate_workers, n_jobs_per_candidate=n_jobs_per_candidate) _configure(search, "exact", None, ridge_solver, 192, 128, 4 if profile.use_stress else 0, "complete", sequence=sequence) base = search.create_candidate def create(spec_dict): return base({**spec_dict, "params": {**spec_dict.get("params", {}), "use_hmm": bool(force_hmm)}}) search.create_candidate = create try: summary = search.run() finally: search.memory.clear() return summary def _concat(train, valid): from wisp.core.data import concatenate_har return concatenate_har(train, valid, source="train+validation") @contextlib.contextmanager def _output_dir(path): if path is not None: Path(path).mkdir(parents=True, exist_ok=True) if any(Path(path).iterdir()): raise ValueError("output_dir must be empty: the search ledger is never appended to") yield str(path) return with tempfile.TemporaryDirectory(prefix="wisp_search_") as tmp: yield tmp def _read_ledger(path): import json if not path.exists(): return [] with path.open(encoding="utf-8") as f: return [json.loads(line) for line in f if line.strip()] def _ledger_row(row): res = row.get("result", {}) return {"id": row.get("id"), "generation": row.get("generation"), "status": row.get("status"), "recipe": {"name": row["spec"]["name"], "params": {k: v for k, v in row["spec"]["params"].items() if k not in ("seed", "n_jobs")}}, "seed": row["spec"]["params"].get("seed"), "validation_utility": res.get("predictive_utility"), "validation_macro_f1": res.get("macro_f1"), "validation_worst_class_f1": res.get("worst_class_f1"), "fit_seconds": res.get("fit_seconds"), "predict_seconds": res.get("predict_seconds")}