Instructions to use Zipeng365/WISP with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Scikit-learn
How to use Zipeng365/WISP with Scikit-learn:
from huggingface_hub import hf_hub_download import joblib model = joblib.load( hf_hub_download("Zipeng365/WISP", "sklearn_model.joblib") ) # only load pickle files from sources you trust # read more about it here https://skops.readthedocs.io/en/stable/persistence.html - Notebooks
- Google Colab
- Kaggle
Download src/wisp/selection.py from Zipeng365/WISP: direct link, hf CLI and curl.
- Browser
- Download file 37.3 kB
-
https://huggingface.co/Zipeng365/WISP/resolve/main/src/wisp/selection.py
- Command line
-
hf download hf://Zipeng365/WISP/src/wisp/selection.py
-
curl -L -o selection.py https://huggingface.co/Zipeng365/WISP/resolve/main/src/wisp/selection.py
37.3 kB
| """WISPSelect and WISPSearch: choose a recipe on validation data, then refit on train+validation. | |
| Both wrap the existing implementations without changing their rules: | |
| * ``WISPSelect`` calls ``wisp_release.selection.fit_selector`` (WISP-Select5/Select7). | |
| * ``WISPSearch(strategy="random"|"evolution")`` calls ``wisp_fast.search.run_search``, i.e. | |
| the search controller of ``wisp.search`` (proposal grammar, cascade, stress views and | |
| validation utility unchanged) with candidates fitted by the speed-tier engine. | |
| * ``WISPSearch(strategy="optuna")`` proposes recipes with Optuna from the same grammar and | |
| scores each one with the same validation utility (``wisp.integrations.optuna``). | |
| * ``WISPSearch(strategy="funnel")`` is WISP-Funnel (``wisp_fast.funnel``): the Random and | |
| Evolution controllers in one run, ranking proposals on a training-only proxy, verifying a | |
| shortlist on the full splits and refitting each controller's winner on train+validation. | |
| ``decoder`` (default ``"adaptive"``) sets the decoder of the final model; see | |
| :mod:`wisp._adaptive` for the rule. ``decoder="recipe"`` is the paper configuration. | |
| Neither class accepts test data. Selection cost and final-model cost are recorded | |
| separately (``selection_seconds_`` and ``final_fit_seconds_``). | |
| """ | |
| from __future__ import annotations | |
| import contextlib | |
| import math | |
| import tempfile | |
| import threading | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| from sklearn.base import BaseEstimator, ClassifierMixin | |
| from sklearn.model_selection import GroupShuffleSplit | |
| from sklearn.utils.validation import check_is_fitted | |
| from . import _adaptive as adaptive | |
| from ._validation import (SEQUENCES, SPEEDS, as_windows, check_aligned, check_choice, check_disjoint_groups, | |
| check_targets, encode_labels, kernel_context) | |
| from .classifier import WISPClassifier | |
| from .recipes import Recipe, get | |
| _PATCH_LOCK = threading.Lock() | |
| _SHORT = {"wisp_ss": "ss", "wisp_rc": "rc", "wisp_so": "so", "wisp_cse": "cse", "wisp_gis": "gis", | |
| "wisp_cis": "cis", "wisp_ese": "ese"} | |
| #: messages of the "no candidate passed validation" errors handled by ``on_no_candidate="default"`` | |
| NO_CANDIDATE_ERRORS = ("No accepted initial candidate", "No full-validation candidate passed", | |
| "No accepted candidate in the Optuna study") | |
| _CONTROLLERS = ("random", "evolution") | |
| def _har(X, codes, groups, time_index, names, source, sampling_rate_hz=None, study_setting=None): | |
| from wisp.core.data import HARData | |
| if time_index is None: | |
| time_index = np.arange(len(X), dtype=np.float64) | |
| metadata = None if study_setting is None else {"study_setting": str(study_setting)} | |
| rate = None if sampling_rate_hz is None else float(sampling_rate_hz) | |
| return HARData(X=X, y=codes, subject=np.asarray(groups), time_index=np.asarray(time_index), | |
| label_names=list(names), source=source, sampling_rate_hz=rate, metadata=metadata) | |
| class _Parts: | |
| """Training and validation splits plus all fitting data (``full``) for the default-recipe fallback. | |
| ``full`` is ``None`` for an explicit validation tuple (training then validation windows are | |
| used); otherwise it is ``X`` in its original row order and ``valid_rows`` marks its | |
| validation rows. | |
| """ | |
| def __init__(self, train, valid, full=None, valid_rows=None): | |
| self.train, self.valid, self.full, self.valid_rows = train, valid, full, valid_rows | |
| class _SplitMixin: | |
| """Shared train/validation handling for WISPSelect and WISPSearch.""" | |
| def _splits(self, X, y, groups, time_index, validation, validation_size, sampling_rate_hz=None, | |
| study_setting=None): | |
| check_choice("speed", self.speed, SPEEDS) | |
| check_choice("sequence", self.sequence, SEQUENCES) | |
| if not isinstance(self.n_jobs, (int, np.integer)) or self.n_jobs < 1: | |
| raise ValueError("n_jobs must be a positive integer") | |
| Xw = (as_windows(X, self.layout)) | |
| y = check_targets(self, y) | |
| if groups is None: | |
| raise ValueError("groups (sequence ID of every window) is required: validation must use " | |
| "different sequences than training") | |
| groups = check_aligned("groups", groups, len(Xw)) | |
| time_index = check_aligned("time_index", time_index, len(Xw)) | |
| if len(y) != len(Xw): | |
| raise ValueError(f"y has {len(y)} labels for {len(Xw)} windows") | |
| if validation is not None and validation_size is not None: | |
| raise ValueError("Pass either validation=(X_val, y_val, groups_val[, time_val]) or validation_size") | |
| rows = adaptive._rows_mask(validation, len(Xw)) if validation is not None else None | |
| full = (Xw, y, groups, time_index) | |
| if validation is not None and rows is None: | |
| if not isinstance(validation, (tuple, list)) or len(validation) not in (3, 4): | |
| raise ValueError("validation must be (X_val, y_val, groups_val), (X_val, y_val, groups_val, time_val) " | |
| "or validation rows of X (integer indices or a boolean mask)") | |
| Xv = (as_windows(validation[0], self.layout)) | |
| yv = np.asarray(validation[1]) | |
| gv = check_aligned("validation groups", validation[2], len(Xv)) | |
| tv = check_aligned("validation time_index", validation[3] if len(validation) == 4 else None, len(Xv)) | |
| if len(yv) != len(Xv): | |
| raise ValueError("validation labels and windows differ in length") | |
| if Xv.shape[1:] != Xw.shape[1:]: | |
| raise ValueError("training and validation windows must share (T, C)") | |
| full = None | |
| elif rows is not None: | |
| Xv, yv, gv = Xw[rows], y[rows], groups[rows] | |
| tv = None if time_index is None else time_index[rows] | |
| Xw, y, groups = Xw[~rows], y[~rows], groups[~rows] | |
| time_index = None if time_index is None else time_index[~rows] | |
| else: | |
| size = 0.125 if validation_size is None else float(validation_size) | |
| if not 0 < size < 1: | |
| raise ValueError("validation_size must be in (0, 1)") | |
| split = GroupShuffleSplit(n_splits=1, test_size=size, random_state=int(self.random_state)) | |
| tr, va = next(split.split(Xw, y, groups)) | |
| rows = np.zeros(len(Xw), dtype=bool) | |
| rows[va] = True | |
| Xv, yv, gv = Xw[va], y[va], groups[va] | |
| tv = None if time_index is None else time_index[va] | |
| Xw, y, groups = Xw[tr], y[tr], groups[tr] | |
| time_index = None if time_index is None else time_index[tr] | |
| check_disjoint_groups(groups, gv) | |
| classes = self.classes if self.classes is not None else np.unique(np.concatenate([y, yv])) | |
| self.classes_, codes = encode_labels(y, classes) | |
| _, codes_v = encode_labels(yv, self.classes_) | |
| names = [str(c) for c in self.classes_.tolist()] | |
| info = dict(sampling_rate_hz=sampling_rate_hz, study_setting=study_setting) | |
| train = _har(Xw, codes, groups, time_index, names, "train", **info) | |
| valid = _har(Xv, codes_v, gv, tv, names, "validation", **info) | |
| return _Parts(train, valid, full, rows) | |
| def _decoder(self): | |
| return adaptive.normalize_decoder(self.decoder) | |
| def _candidate_hmm(decoder, train): | |
| """HMM flag forced on every candidate: None keeps each recipe's own flag.""" | |
| if decoder == "hmm": | |
| return True | |
| if decoder == "none" or (decoder == "adaptive" and not adaptive.has_sequences(train.subject)): | |
| return False | |
| return None | |
| def _refit_classifier(self, recipe, seed, decoder="recipe"): | |
| return WISPClassifier(recipe, decoder=decoder, classes=self.classes_, layout="NTC", speed=self.speed, | |
| sequence=self.sequence, n_jobs=self.n_jobs, random_state=int(seed)) | |
| def _adaptive_classifier(self, recipe, seed, train, valid): | |
| """``decoder="adaptive"`` for a selected recipe: fit on train, decide on valid, refit on both.""" | |
| est = self._refit_classifier(recipe, seed, decoder="adaptive") | |
| est.fit(train.X, self.classes_[train.y], groups=train.subject, time_index=train.time_index, | |
| validation=(valid.X, self.classes_[valid.y], valid.subject, valid.time_index)) | |
| return est | |
| def _default_recipe(self, parts, decoder, seed, exc): | |
| """``on_no_candidate="default"``: the default CIS recipe on all fitting data.""" | |
| est = self._refit_classifier(get("cis"), seed, decoder=decoder) | |
| start = time.perf_counter() | |
| if parts.full is None: | |
| t, v = parts.train, parts.valid | |
| est.fit(t.X, self.classes_[t.y], groups=t.subject, time_index=t.time_index, | |
| validation=(v.X, self.classes_[v.y], v.subject, v.time_index)) | |
| else: | |
| X, y, g, ti = parts.full | |
| est.fit(X, y, groups=g, time_index=ti, validation=parts.valid_rows) | |
| self.fallback_ = {"reason": str(exc), "recipe": "cis", | |
| "fit_scope": "training and validation windows (all fitting data)"} | |
| self.best_recipe_ = est.recipe_ | |
| self.best_seed_ = int(seed) | |
| self.best_validation_ = None | |
| self._finish(est, final_fit=time.perf_counter() - start) | |
| return est | |
| def _finish(self, est, *, final_fit): | |
| self.best_estimator_ = est | |
| self.final_fit_seconds_ = float(final_fit) | |
| self.decoder_ = est.decoder_ | |
| self.decoder_scores_ = est.decoder_scores_ | |
| self.decoder_reason_ = est.decoder_reason_ | |
| def predict(self, X, groups=None, time_index=None): | |
| check_is_fitted(self, "best_estimator_") | |
| return self.best_estimator_.predict((as_windows(X, self.layout)), | |
| groups=groups, time_index=time_index) | |
| def predict_proba(self, X): | |
| check_is_fitted(self, "best_estimator_") | |
| return self.best_estimator_.predict_proba((as_windows(X, self.layout))) | |
| def __sklearn_tags__(self): | |
| tags = super().__sklearn_tags__() | |
| tags.input_tags.three_d_array = True | |
| return tags | |
| def _record_builds(store, force_hmm=None): | |
| """Record every family fitted by fit_selector (for per-candidate fit times). | |
| Replaces the module-level default builder with a recording wrapper around the same | |
| function, so fit_selector's code path (including its size tie-break) is unchanged. | |
| ``force_hmm`` (True/False) builds every family with that HMM setting instead of its own | |
| (``decoder="hmm"``/``"none"``, or ``"adaptive"`` without sequences); ``None`` changes nothing. | |
| """ | |
| import wisp_release.selection as sel | |
| original, reference = sel.build_family, sel.build_reference_family | |
| def flagged(kwargs): | |
| if force_hmm is None: | |
| return kwargs | |
| return {**kwargs, "overrides": {**(kwargs.get("overrides") or {}), "use_hmm": bool(force_hmm)}} | |
| def recording(*args, **kwargs): | |
| model = original(*args, **flagged(kwargs)) | |
| store.append((args[0] if args else kwargs.get("method_id"), model)) | |
| return model | |
| def sized(*args, **kwargs): | |
| return reference(*args, **flagged(kwargs)) | |
| with _PATCH_LOCK: | |
| sel.build_family = recording | |
| if force_hmm is not None: | |
| sel.build_reference_family = sized | |
| try: | |
| yield | |
| finally: | |
| sel.build_family = original | |
| sel.build_reference_family = reference | |
| def _decide_on_model(model, valid, n_classes): | |
| """Decision from a model fitted on training windows with an HMM smoother.""" | |
| with kernel_context(): | |
| P = model.predict_proba(valid.X) | |
| hmm = adaptive.hmm_labels(model, P, valid.subject, valid.time_index) | |
| return adaptive.compare(valid.y, hmm, adaptive.argmax_labels(P), n_classes) | |
| class WISPSelect(_SplitMixin, ClassifierMixin, BaseEstimator): | |
| """WISP-Select5 / WISP-Select7: validation selection among fixed families, then refit. | |
| Candidates are fitted on training windows and scored on validation windows; the winner | |
| is chosen by validation macro-F1, then full-vocabulary worst-class F1, then serialized | |
| model size, then the order SS, RC, SO, CSE, GIS, CIS, ESE. A fresh winner is fitted on | |
| train+validation. ``menu=5``: SS, RC, SO, GIS, CIS. ``menu=7`` adds CSE and ESE. | |
| decoder : {"adaptive", "recipe", "hmm", "none"}, default="adaptive" | |
| ``"recipe"``: every family with its own decoder (the paper). ``"adaptive"``: the | |
| selection is the paper's; then the winner's training fit decodes the validation | |
| windows with and without the HMM, and the HMM is kept only if its macro-F1 is strictly | |
| higher (the refit uses the chosen decoder). Without sequences (no training group with | |
| two or more windows) candidates are scored and refitted with argmax. ``"hmm"``/``"none"`` | |
| force that decoder for scoring and refit. | |
| """ | |
| def __init__(self, menu=5, *, decoder="adaptive", classes=None, layout="NTC", speed="exact", | |
| sequence="subject", n_jobs=1, random_state=42): | |
| self.menu = menu | |
| self.decoder = decoder | |
| self.classes = classes | |
| self.layout = layout | |
| self.speed = speed | |
| self.sequence = sequence | |
| self.n_jobs = n_jobs | |
| self.random_state = random_state | |
| def fit(self, X, y, groups=None, time_index=None, validation=None, validation_size=None): | |
| """``validation=(X_val, y_val, groups_val[, time_val])`` (the paper protocol), validation | |
| rows of ``X``, or ``validation_size`` to hold out that fraction of the groups (default 0.125).""" | |
| if self.menu not in (5, 7): | |
| raise ValueError("menu must be 5 or 7") | |
| decoder = self._decoder() | |
| from wisp_release.selection import fit_selector | |
| parts = self._splits(X, y, groups, time_index, validation, validation_size) | |
| train, valid = parts.train, parts.valid | |
| force = self._candidate_hmm(decoder, train) | |
| built = [] | |
| start = time.perf_counter() | |
| with _record_builds(built, force), kernel_context(): | |
| model, report = fit_selector(f"wisp_select{self.menu}", train, valid, seed=int(self.random_state), | |
| n_jobs=int(self.n_jobs), speed=self.speed, sequence=self.sequence) | |
| total = time.perf_counter() - start | |
| final_fit = selector_refit = float(getattr(model, "fit_seconds_", 0.0)) | |
| winner = report["selected_method_id"] | |
| fit_times = {} | |
| for family, m in built[:-1]: | |
| fit_times[family] = float(getattr(m, "fit_seconds_", math.nan)) | |
| self.history_ = [{**{k: v for k, v in rec.items() if k != "confusion_matrix"}, | |
| "recipe": _SHORT[rec["method_id"]], "fit_seconds": fit_times.get(rec["method_id"])} | |
| for rec in report["validation_candidates"]] | |
| self.selection_report_ = report | |
| self.best_recipe_ = get(_SHORT[winner]) | |
| recipe_hmm = bool(model.use_hmm) | |
| use_hmm, reason, scores, decision_s = recipe_hmm, adaptive.FIXED, None, 0.0 | |
| if force is not None: | |
| reason = adaptive.NO_SEQUENCES if decoder == "adaptive" else adaptive.FIXED | |
| elif decoder == "adaptive": | |
| t0 = time.perf_counter() | |
| trained = dict(built[:-1])[winner] | |
| choice, scores = _decide_on_model(trained, valid, len(self.classes_)) | |
| decision_s = time.perf_counter() - t0 | |
| reason = adaptive.VALIDATION | |
| if (choice == "hmm") != use_hmm: | |
| model, final_fit = self._refit_family(winner, train, valid, choice == "hmm", report) | |
| use_hmm = choice == "hmm" | |
| est = self._refit_classifier(self.best_recipe_, self.random_state) | |
| est._adopt(model, classes=self.classes_, input_shape=train.X.shape[1:], use_hmm=use_hmm, | |
| fit_seconds=final_fit, reason=reason, scores=scores, decision_seconds=decision_s) | |
| self._finish(est, final_fit=final_fit) | |
| self.decision_seconds_ = decision_s | |
| self.selection_seconds_ = total - selector_refit | |
| self.has_paper_results_ = bool(self.speed == "exact" and self.sequence == "subject" and force is None | |
| and use_hmm == recipe_hmm) | |
| return self | |
| def _refit_family(self, winner, train, valid, use_hmm, report): | |
| """Refit the selected family on train+validation with the chosen decoder.""" | |
| from wisp.core.data import concatenate_har | |
| from wisp_release.methods import build_family | |
| combined = concatenate_har(train, valid, source="train+validation") | |
| model = build_family(winner, seed=int(self.random_state), n_jobs=int(self.n_jobs), speed=self.speed, | |
| sequence=self.sequence, overrides={"use_hmm": bool(use_hmm)}) | |
| with kernel_context(): | |
| model.fit(combined.X, combined.y, groups=combined.subject, time_index=combined.time_index, | |
| n_classes=combined.n_classes) | |
| if use_hmm and report.get("confidence_scale") is not None: | |
| model.set_confidence_scale(report["confidence_scale"]) | |
| return model, float(getattr(model, "fit_seconds_", 0.0)) | |
| class WISPSearch(_SplitMixin, ClassifierMixin, BaseEstimator): | |
| """Recipe search on validation data, then one refit of the winner on train+validation. | |
| strategy : {"random", "evolution", "optuna", "funnel"} | |
| ``"random"``/``"evolution"`` are WISP-Random/WISP-Evolution (paper implementations). | |
| ``"optuna"`` proposes recipes with Optuna (default sampler TPE) from the same grammar | |
| and scores them with the same validation utility; it has no paper results. | |
| ``"funnel"`` is WISP-Funnel, the fast joint Random + Evolution search: both | |
| controllers propose ``budget`` candidates each from the CIS family and rank them on a | |
| training-only proxy (at most 4,096 training and 1,024 validation windows, a shared | |
| Ridge feature dictionary, cost weight 0), a diversified shortlist of 4 per controller | |
| is verified on the full splits with the screened Ridge solver, and each controller's | |
| winner is refitted on train+validation. It has no paper results; its benchmark record | |
| is in docs/RECIPES.md. | |
| controller : {"random", "evolution"}, default="random" | |
| ``strategy="funnel"`` only: which controller's model predicts. Both are fitted in the | |
| same run and kept in ``funnel_estimators_``. | |
| budget : int, default=329 | |
| Number of candidates evaluated (paper: 329, i.e. 9 initial + 20 generations x 16); | |
| for the funnel, proposals per controller. | |
| population : int, default=24 | |
| Population of the random/evolution controller (paper: 24). | |
| sampler : optuna sampler or None | |
| Only for ``strategy="optuna"``; ``None`` uses ``TPESampler(seed=random_state)``. | |
| decoder : {"adaptive", "recipe", "hmm", "none"}, default="adaptive" | |
| Decoder of the final model. ``"recipe"``: the winner's own HMM flag (the paper). | |
| ``"adaptive"``: the search is unchanged; then the winner fitted on training windows | |
| decodes the validation windows with and without the HMM, and the HMM is kept only if | |
| its macro-F1 is strictly higher. Without sequences (no training group with two or more | |
| windows) candidates are scored and refitted with argmax. ``"hmm"``/``"none"`` force that | |
| decoder for scoring and refit. | |
| on_no_candidate : {"raise", "default"}, default="raise" | |
| What happens when no candidate passes validation (every candidate rejected or | |
| collapsed). ``"raise"`` re-raises the search's error; ``"default"`` fits the default CIS | |
| recipe (``wisp.recipes.get("cis")``) on all fitting data instead and records the reason | |
| in ``fallback_``. | |
| n_jobs : int, default=1 | |
| Threads for tree ensembles inside each candidate. For the funnel: controller threads | |
| (1 runs the two controllers in series, 2 or more runs them in parallel; candidates and | |
| BLAS use one thread each). The results do not depend on it. | |
| output_dir : path or None | |
| Where the controller writes its candidate ledger; a temporary directory when None. | |
| For the funnel, ``speed``, ``sequence``, ``cost_weight`` and ``robustness_weight`` are | |
| fixed by its protocol (``sequence`` must be ``"subject"``). | |
| The paper's single search (one dataset, fold and seed, 329 candidates) had median | |
| cumulative candidate fitting time of 47.5 h (Random) and 61.3 h (Evolution). Try the | |
| fixed recipes and WISPSelect first (docs/RECIPES.md). | |
| """ | |
| def __init__(self, strategy="random", *, controller="random", budget=329, population=24, sampler=None, | |
| decoder="adaptive", on_no_candidate="raise", classes=None, layout="NTC", speed="exact", | |
| sequence="subject", n_jobs=1, random_state=42, cost_weight=0.01, robustness_weight=0.1, | |
| output_dir=None): | |
| self.strategy = strategy | |
| self.controller = controller | |
| self.budget = budget | |
| self.population = population | |
| self.sampler = sampler | |
| self.decoder = decoder | |
| self.on_no_candidate = on_no_candidate | |
| self.classes = classes | |
| self.layout = layout | |
| self.speed = speed | |
| self.sequence = sequence | |
| self.n_jobs = n_jobs | |
| self.random_state = random_state | |
| self.cost_weight = cost_weight | |
| self.robustness_weight = robustness_weight | |
| self.output_dir = output_dir | |
| def fit(self, X, y, groups=None, time_index=None, validation=None, validation_size=None, sampling_rate_hz=None, | |
| study_setting=None): | |
| """Search on ``validation`` (or a ``validation_size`` hold-out of the groups), then refit. | |
| ``sampling_rate_hz`` and ``study_setting`` describe the recordings for the search | |
| controllers' dataset fingerprint, which shapes their proposals (window duration from the | |
| sampling rate; a study setting containing "free", "home", "natural" or "wild" marks | |
| free-living data). The paper searches and the funnel benchmark record used the values of | |
| the WristHARBench exports (``wisp.datasets`` folds expose ``sampling_rate_hz`` and | |
| ``study_setting``). When omitted, the fingerprint assumes 50 Hz and an unknown setting. | |
| """ | |
| check_choice("strategy", self.strategy, ("random", "evolution", "optuna", "funnel")) | |
| check_choice("controller", self.controller, _CONTROLLERS) | |
| check_choice("on_no_candidate", self.on_no_candidate, ("raise", "default")) | |
| if not isinstance(self.budget, (int, np.integer)) or self.budget < 1: | |
| raise ValueError("budget must be a positive integer") | |
| decoder = self._decoder() | |
| if self.strategy == "funnel" and self.sequence != "subject": | |
| raise ValueError("strategy='funnel' supports sequence='subject' only") | |
| parts = self._splits(X, y, groups, time_index, validation, validation_size, sampling_rate_hz, study_setting) | |
| train, valid = parts.train, parts.valid | |
| force = self._candidate_hmm(decoder, train) | |
| self.fallback_ = None | |
| self.decision_seconds_ = 0.0 | |
| if self.strategy == "funnel": | |
| return self._fit_funnel(parts, decoder, force) | |
| start = time.perf_counter() | |
| try: | |
| with kernel_context(): | |
| if self.strategy == "optuna": | |
| best_spec, best_result = self._fit_optuna(train, valid, force) | |
| else: | |
| best_spec, best_result = self._fit_controller(train, valid, force) | |
| except RuntimeError as exc: | |
| if self.on_no_candidate != "default" or not any(m in str(exc) for m in NO_CANDIDATE_ERRORS): | |
| raise | |
| self.selection_seconds_ = time.perf_counter() - start | |
| self._default_recipe(parts, decoder, self.random_state, exc) | |
| self.has_paper_results_ = False | |
| return self | |
| self.selection_seconds_ = time.perf_counter() - start | |
| params = dict(best_spec["params"]) | |
| seed = int(params.pop("seed", self.random_state)) | |
| params.pop("n_jobs", None) | |
| self.best_recipe_ = Recipe(best_spec["name"], params) | |
| self.best_seed_ = seed | |
| self.best_validation_ = best_result | |
| if decoder == "adaptive" and force is None: | |
| est = self._adaptive_classifier(self.best_recipe_, seed, train, valid) | |
| self.decision_seconds_ = est.decision_seconds_ | |
| else: | |
| combined = _concat(train, valid) | |
| est = self._refit_classifier(self.best_recipe_, seed, | |
| decoder="recipe" if force is None else ("hmm" if force else "none")) | |
| est.fit(combined.X, self.classes_[combined.y], groups=combined.subject, time_index=combined.time_index) | |
| if force is not None: | |
| est.decoder_reason_ = adaptive.NO_SEQUENCES if decoder == "adaptive" else adaptive.FIXED | |
| if self.sequence == "recording" and best_result.get("confidence_scale") is not None: | |
| # a train+validation refit reuses the factor chosen for the train-only candidate | |
| with kernel_context(): | |
| est.model_.set_confidence_scale(best_result["confidence_scale"]) | |
| self._finish(est, final_fit=est.fit_seconds_) | |
| recipe_decoder = "hmm" if self.best_recipe_.use_hmm else "none" | |
| self.has_paper_results_ = bool(self.strategy in ("random", "evolution") and self.speed == "exact" | |
| and self.sequence == "subject" and int(self.budget) == 329 | |
| and int(self.population) == 24 and force is None | |
| and est.decoder_ == recipe_decoder) | |
| return self | |
| # ------------------------------------------------------------------ strategies | |
| def _fit_controller(self, train, valid, force=None): | |
| from wisp_fast.search import run_search | |
| from ._engine import RIDGE_SOLVER | |
| budget, pop = int(self.budget), int(self.population) | |
| if pop < 3: | |
| raise ValueError("population must be at least 3") | |
| per_generation = max(pop - max(2, pop // 3), 1) | |
| generations = max(1, -(-(budget - min(9, budget)) // per_generation)) | |
| with _output_dir(self.output_dir) as out: | |
| options = dict(output=out, population=pop, generations=generations, max_evaluations=budget, | |
| seed=int(self.random_state), candidate_workers=1, n_jobs_per_candidate=int(self.n_jobs), | |
| ridge_solver=RIDGE_SOLVER[self.speed], sequence=self.sequence, | |
| cost_weight=float(self.cost_weight), robustness_weight=float(self.robustness_weight)) | |
| if force is None: | |
| summary = run_search(f"wisp_{self.strategy}", train, valid, None, **options) | |
| else: | |
| summary = _run_search_forced(f"wisp_{self.strategy}", train, valid, force, **options) | |
| rows = [_ledger_row(r) for r in _read_ledger(Path(out) / "program_database.jsonl")] | |
| self.history_ = rows | |
| self.search_summary_ = {k: summary[k] for k in ("n_rows", "n_ok", "n_evaluated", "search_config")} | |
| best = summary["best"] | |
| return best["spec"], best["result"] | |
| def _fit_optuna(self, train, valid, force=None): | |
| from .integrations.optuna import run_study | |
| study, best_spec, best_result, history = run_study( | |
| train, valid, n_trials=int(self.budget), sampler=self.sampler, seed=int(self.random_state), | |
| speed=self.speed, sequence=self.sequence, n_jobs=int(self.n_jobs), | |
| cost_weight=float(self.cost_weight), robustness_weight=float(self.robustness_weight), | |
| force_hmm=force) | |
| self.study_ = study | |
| self.history_ = history | |
| return best_spec, best_result | |
| def _fit_funnel(self, parts, decoder, force): | |
| """WISP-Funnel: both controllers in one run; ``controller`` picks the predicting model.""" | |
| from wisp.core.data import concatenate_har | |
| from wisp_fast.funnel.engine import FORCE_HMM | |
| from wisp_fast.funnel.joint import fit_joint_funnel | |
| train, valid = parts.train, parts.valid | |
| if int(self.population) < 3: | |
| raise ValueError("population must be at least 3") | |
| workers = 2 if int(self.n_jobs) >= 2 else 1 | |
| start = time.perf_counter() | |
| token = FORCE_HMM.set(force) | |
| try: | |
| with _output_dir(self.output_dir) as out, kernel_context(): | |
| try: | |
| models, summaries, receipt = fit_joint_funnel( | |
| train, valid, output=out, seed=int(self.random_state), evaluations=int(self.budget), | |
| population=int(self.population), workers=workers) | |
| except RuntimeError as exc: | |
| if self.on_no_candidate != "default" or not any(m in str(exc) for m in NO_CANDIDATE_ERRORS): | |
| raise | |
| models = None | |
| failure = exc | |
| else: | |
| rows = {c: [_ledger_row(r) for r in | |
| _read_ledger(Path(out) / f"wisp_{c}" / "proxy" / "program_database.jsonl")] | |
| for c in _CONTROLLERS} | |
| finally: | |
| FORCE_HMM.reset(token) | |
| funnel_s = time.perf_counter() - start | |
| self.funnel_seconds_ = funnel_s | |
| if models is None: | |
| self.selection_seconds_ = funnel_s | |
| est = self._default_recipe(parts, decoder, self.random_state, failure) | |
| self.funnel_estimators_ = {c: est for c in _CONTROLLERS} | |
| self.funnel_summary_, self.funnel_receipt_, self.history_ = None, None, [] | |
| self.has_paper_results_ = False | |
| return self | |
| combined = None | |
| estimators, info, decided = {}, {}, {} | |
| for c in _CONTROLLERS: | |
| summary = summaries[f"wisp_{c}"] | |
| spec = summary["winner_recipe"] | |
| params = dict(spec["params"]) | |
| seed = int(params.pop("seed", self.random_state)) | |
| params.pop("n_jobs", None) | |
| recipe = Recipe(spec["name"], params) | |
| recipe_hmm = bool(spec["params"].get("use_hmm", True)) if force is None else bool(force) | |
| model, final_fit = models[f"wisp_{c}"], float(summary["refit_seconds"]) | |
| use_hmm, reason, scores, decision_s = recipe_hmm, adaptive.FIXED, None, 0.0 | |
| if force is not None and decoder == "adaptive": | |
| reason = adaptive.NO_SEQUENCES | |
| elif decoder == "adaptive": | |
| key = _spec_key(spec) | |
| if key not in decided: | |
| decided[key] = _funnel_decision(spec, train, valid, len(self.classes_)) | |
| choice, scores, decision_s = decided[key] | |
| use_hmm, reason = choice == "hmm", adaptive.VALIDATION | |
| if use_hmm != recipe_hmm: | |
| if combined is None: | |
| combined = concatenate_har(train, valid, source="train+validation") | |
| model, final_fit = _funnel_refit(spec, combined, use_hmm) | |
| est = self._refit_classifier(recipe, seed) | |
| est._adopt(model, classes=self.classes_, input_shape=train.X.shape[1:], use_hmm=use_hmm, | |
| fit_seconds=final_fit, reason=reason, scores=scores, decision_seconds=decision_s) | |
| estimators[c] = est | |
| info[c] = {k: summary.get(k) for k in ("proxy_seconds", "verification_seconds", "refit_seconds", | |
| "n_proposals", "n_proxy_ok", "n_unique_proxy_ok", | |
| "n_full_verified", "n_full_ok", "proxy_winner")} | |
| info[c].update(winner_id=summary["best"]["id"], recipe=recipe, seed=seed, | |
| validation=summary["best"]["result"]) | |
| chosen = self.controller | |
| best = summaries[f"wisp_{chosen}"] | |
| self.funnel_estimators_ = estimators | |
| self.funnel_summary_ = info | |
| self.funnel_receipt_ = receipt | |
| self.history_ = rows[chosen] | |
| self.best_recipe_ = info[chosen]["recipe"] | |
| self.best_seed_ = info[chosen]["seed"] | |
| self.best_validation_ = best["best"]["result"] | |
| self.selection_seconds_ = float(best["proxy_seconds"]) + float(best["verification_seconds"]) | |
| self.decision_seconds_ = estimators[chosen].decision_seconds_ | |
| self._finish(estimators[chosen], final_fit=estimators[chosen].fit_seconds_) | |
| self.has_paper_results_ = False | |
| return self | |
| def _spec_key(spec): | |
| import json | |
| return json.dumps([spec["name"], spec["params"]], sort_keys=True) | |
| def _funnel_decision(spec, train, valid, n_classes): | |
| """Fit the funnel winner on training windows with the HMM (as in its verification), decide on valid.""" | |
| from wisp_fast.funnel.selection import fit_on | |
| from wisp_fast.funnel.shared_gram import verify_candidate | |
| start = time.perf_counter() | |
| hmm_spec = {**spec, "params": {**spec["params"], "use_hmm": True}} | |
| with kernel_context(): | |
| model = fit_on(verify_candidate(hmm_spec, mode="exact", budget=None, ridge_solver="screened"), train) | |
| choice, scores = _decide_on_model(model, valid, n_classes) | |
| return choice, scores, time.perf_counter() - start | |
| def _funnel_refit(spec, combined, use_hmm): | |
| """Refit a funnel winner on train+validation with the chosen decoder (the engine's refit factory).""" | |
| from wisp_fast.funnel.search import make_candidate | |
| from wisp_fast.funnel.selection import fit_on | |
| flipped = {**spec, "params": {**spec["params"], "use_hmm": bool(use_hmm)}} | |
| start = time.perf_counter() | |
| with kernel_context(): | |
| model = fit_on(make_candidate(flipped, mode="exact", budget=None, ridge_solver="screened"), combined) | |
| return model, time.perf_counter() - start | |
| def _run_search_forced(method, train, valid, force_hmm, *, output, population, generations, max_evaluations, | |
| seed, candidate_workers, n_jobs_per_candidate, ridge_solver, sequence, cost_weight, | |
| robustness_weight): | |
| """``wisp_fast.search.run_search`` with every candidate built with HMM flag ``force_hmm``. | |
| The controller, its proposals, ledger and utility are unchanged; only the candidate factory | |
| overrides ``use_hmm`` (``decoder="hmm"``/``"none"``, or ``"adaptive"`` without sequences). | |
| """ | |
| from dataclasses import replace | |
| from wisp.search.profiles import get_profile | |
| from wisp_fast.search import _AcceleratedSearch, _configure | |
| from wisp_fast.selection import check_splits | |
| check_splits(train, valid) | |
| profile = replace(get_profile(method), max_evaluations=int(max_evaluations)) | |
| search = _AcceleratedSearch(train=train, valid=valid, output=output, profile=profile, test=None, | |
| population=population, generations=generations, seed=seed, | |
| cost_weight=cost_weight, robustness_weight=robustness_weight, | |
| candidate_workers=candidate_workers, n_jobs_per_candidate=n_jobs_per_candidate) | |
| _configure(search, "exact", None, ridge_solver, 192, 128, 4 if profile.use_stress else 0, "complete", | |
| sequence=sequence) | |
| base = search.create_candidate | |
| def create(spec_dict): | |
| return base({**spec_dict, "params": {**spec_dict.get("params", {}), "use_hmm": bool(force_hmm)}}) | |
| search.create_candidate = create | |
| try: | |
| summary = search.run() | |
| finally: | |
| search.memory.clear() | |
| return summary | |
| def _concat(train, valid): | |
| from wisp.core.data import concatenate_har | |
| return concatenate_har(train, valid, source="train+validation") | |
| def _output_dir(path): | |
| if path is not None: | |
| Path(path).mkdir(parents=True, exist_ok=True) | |
| if any(Path(path).iterdir()): | |
| raise ValueError("output_dir must be empty: the search ledger is never appended to") | |
| yield str(path) | |
| return | |
| with tempfile.TemporaryDirectory(prefix="wisp_search_") as tmp: | |
| yield tmp | |
| def _read_ledger(path): | |
| import json | |
| if not path.exists(): | |
| return [] | |
| with path.open(encoding="utf-8") as f: | |
| return [json.loads(line) for line in f if line.strip()] | |
| def _ledger_row(row): | |
| res = row.get("result", {}) | |
| return {"id": row.get("id"), "generation": row.get("generation"), "status": row.get("status"), | |
| "recipe": {"name": row["spec"]["name"], | |
| "params": {k: v for k, v in row["spec"]["params"].items() if k not in ("seed", "n_jobs")}}, | |
| "seed": row["spec"]["params"].get("seed"), | |
| "validation_utility": res.get("predictive_utility"), "validation_macro_f1": res.get("macro_f1"), | |
| "validation_worst_class_f1": res.get("worst_class_f1"), | |
| "fit_seconds": res.get("fit_seconds"), "predict_seconds": res.get("predict_seconds")} | |