Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / src /wisp /selection.py
Zipeng365's picture
wisp-har 0.4.0: WISP-Funnel search and the adaptive HMM switch (default decoder)
93a9282 verified
Raw History Blame Contribute Delete
37.3 kB
"""WISPSelect and WISPSearch: choose a recipe on validation data, then refit on train+validation.
Both wrap the existing implementations without changing their rules:
* ``WISPSelect`` calls ``wisp_release.selection.fit_selector`` (WISP-Select5/Select7).
* ``WISPSearch(strategy="random"|"evolution")`` calls ``wisp_fast.search.run_search``, i.e.
the search controller of ``wisp.search`` (proposal grammar, cascade, stress views and
validation utility unchanged) with candidates fitted by the speed-tier engine.
* ``WISPSearch(strategy="optuna")`` proposes recipes with Optuna from the same grammar and
scores each one with the same validation utility (``wisp.integrations.optuna``).
* ``WISPSearch(strategy="funnel")`` is WISP-Funnel (``wisp_fast.funnel``): the Random and
Evolution controllers in one run, ranking proposals on a training-only proxy, verifying a
shortlist on the full splits and refitting each controller's winner on train+validation.
``decoder`` (default ``"adaptive"``) sets the decoder of the final model; see
:mod:`wisp._adaptive` for the rule. ``decoder="recipe"`` is the paper configuration.
Neither class accepts test data. Selection cost and final-model cost are recorded
separately (``selection_seconds_`` and ``final_fit_seconds_``).
"""
from __future__ import annotations
import contextlib
import math
import tempfile
import threading
import time
from pathlib import Path
import numpy as np
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.model_selection import GroupShuffleSplit
from sklearn.utils.validation import check_is_fitted
from . import _adaptive as adaptive
from ._validation import (SEQUENCES, SPEEDS, as_windows, check_aligned, check_choice, check_disjoint_groups,
check_targets, encode_labels, kernel_context)
from .classifier import WISPClassifier
from .recipes import Recipe, get
_PATCH_LOCK = threading.Lock()
_SHORT = {"wisp_ss": "ss", "wisp_rc": "rc", "wisp_so": "so", "wisp_cse": "cse", "wisp_gis": "gis",
"wisp_cis": "cis", "wisp_ese": "ese"}
#: messages of the "no candidate passed validation" errors handled by ``on_no_candidate="default"``
NO_CANDIDATE_ERRORS = ("No accepted initial candidate", "No full-validation candidate passed",
"No accepted candidate in the Optuna study")
_CONTROLLERS = ("random", "evolution")
def _har(X, codes, groups, time_index, names, source, sampling_rate_hz=None, study_setting=None):
from wisp.core.data import HARData
if time_index is None:
time_index = np.arange(len(X), dtype=np.float64)
metadata = None if study_setting is None else {"study_setting": str(study_setting)}
rate = None if sampling_rate_hz is None else float(sampling_rate_hz)
return HARData(X=X, y=codes, subject=np.asarray(groups), time_index=np.asarray(time_index),
label_names=list(names), source=source, sampling_rate_hz=rate, metadata=metadata)
class _Parts:
"""Training and validation splits plus all fitting data (``full``) for the default-recipe fallback.
``full`` is ``None`` for an explicit validation tuple (training then validation windows are
used); otherwise it is ``X`` in its original row order and ``valid_rows`` marks its
validation rows.
"""
def __init__(self, train, valid, full=None, valid_rows=None):
self.train, self.valid, self.full, self.valid_rows = train, valid, full, valid_rows
class _SplitMixin:
"""Shared train/validation handling for WISPSelect and WISPSearch."""
def _splits(self, X, y, groups, time_index, validation, validation_size, sampling_rate_hz=None,
study_setting=None):
check_choice("speed", self.speed, SPEEDS)
check_choice("sequence", self.sequence, SEQUENCES)
if not isinstance(self.n_jobs, (int, np.integer)) or self.n_jobs < 1:
raise ValueError("n_jobs must be a positive integer")
Xw = (as_windows(X, self.layout))
y = check_targets(self, y)
if groups is None:
raise ValueError("groups (sequence ID of every window) is required: validation must use "
"different sequences than training")
groups = check_aligned("groups", groups, len(Xw))
time_index = check_aligned("time_index", time_index, len(Xw))
if len(y) != len(Xw):
raise ValueError(f"y has {len(y)} labels for {len(Xw)} windows")
if validation is not None and validation_size is not None:
raise ValueError("Pass either validation=(X_val, y_val, groups_val[, time_val]) or validation_size")
rows = adaptive._rows_mask(validation, len(Xw)) if validation is not None else None
full = (Xw, y, groups, time_index)
if validation is not None and rows is None:
if not isinstance(validation, (tuple, list)) or len(validation) not in (3, 4):
raise ValueError("validation must be (X_val, y_val, groups_val), (X_val, y_val, groups_val, time_val) "
"or validation rows of X (integer indices or a boolean mask)")
Xv = (as_windows(validation[0], self.layout))
yv = np.asarray(validation[1])
gv = check_aligned("validation groups", validation[2], len(Xv))
tv = check_aligned("validation time_index", validation[3] if len(validation) == 4 else None, len(Xv))
if len(yv) != len(Xv):
raise ValueError("validation labels and windows differ in length")
if Xv.shape[1:] != Xw.shape[1:]:
raise ValueError("training and validation windows must share (T, C)")
full = None
elif rows is not None:
Xv, yv, gv = Xw[rows], y[rows], groups[rows]
tv = None if time_index is None else time_index[rows]
Xw, y, groups = Xw[~rows], y[~rows], groups[~rows]
time_index = None if time_index is None else time_index[~rows]
else:
size = 0.125 if validation_size is None else float(validation_size)
if not 0 < size < 1:
raise ValueError("validation_size must be in (0, 1)")
split = GroupShuffleSplit(n_splits=1, test_size=size, random_state=int(self.random_state))
tr, va = next(split.split(Xw, y, groups))
rows = np.zeros(len(Xw), dtype=bool)
rows[va] = True
Xv, yv, gv = Xw[va], y[va], groups[va]
tv = None if time_index is None else time_index[va]
Xw, y, groups = Xw[tr], y[tr], groups[tr]
time_index = None if time_index is None else time_index[tr]
check_disjoint_groups(groups, gv)
classes = self.classes if self.classes is not None else np.unique(np.concatenate([y, yv]))
self.classes_, codes = encode_labels(y, classes)
_, codes_v = encode_labels(yv, self.classes_)
names = [str(c) for c in self.classes_.tolist()]
info = dict(sampling_rate_hz=sampling_rate_hz, study_setting=study_setting)
train = _har(Xw, codes, groups, time_index, names, "train", **info)
valid = _har(Xv, codes_v, gv, tv, names, "validation", **info)
return _Parts(train, valid, full, rows)
def _decoder(self):
return adaptive.normalize_decoder(self.decoder)
@staticmethod
def _candidate_hmm(decoder, train):
"""HMM flag forced on every candidate: None keeps each recipe's own flag."""
if decoder == "hmm":
return True
if decoder == "none" or (decoder == "adaptive" and not adaptive.has_sequences(train.subject)):
return False
return None
def _refit_classifier(self, recipe, seed, decoder="recipe"):
return WISPClassifier(recipe, decoder=decoder, classes=self.classes_, layout="NTC", speed=self.speed,
sequence=self.sequence, n_jobs=self.n_jobs, random_state=int(seed))
def _adaptive_classifier(self, recipe, seed, train, valid):
"""``decoder="adaptive"`` for a selected recipe: fit on train, decide on valid, refit on both."""
est = self._refit_classifier(recipe, seed, decoder="adaptive")
est.fit(train.X, self.classes_[train.y], groups=train.subject, time_index=train.time_index,
validation=(valid.X, self.classes_[valid.y], valid.subject, valid.time_index))
return est
def _default_recipe(self, parts, decoder, seed, exc):
"""``on_no_candidate="default"``: the default CIS recipe on all fitting data."""
est = self._refit_classifier(get("cis"), seed, decoder=decoder)
start = time.perf_counter()
if parts.full is None:
t, v = parts.train, parts.valid
est.fit(t.X, self.classes_[t.y], groups=t.subject, time_index=t.time_index,
validation=(v.X, self.classes_[v.y], v.subject, v.time_index))
else:
X, y, g, ti = parts.full
est.fit(X, y, groups=g, time_index=ti, validation=parts.valid_rows)
self.fallback_ = {"reason": str(exc), "recipe": "cis",
"fit_scope": "training and validation windows (all fitting data)"}
self.best_recipe_ = est.recipe_
self.best_seed_ = int(seed)
self.best_validation_ = None
self._finish(est, final_fit=time.perf_counter() - start)
return est
def _finish(self, est, *, final_fit):
self.best_estimator_ = est
self.final_fit_seconds_ = float(final_fit)
self.decoder_ = est.decoder_
self.decoder_scores_ = est.decoder_scores_
self.decoder_reason_ = est.decoder_reason_
def predict(self, X, groups=None, time_index=None):
check_is_fitted(self, "best_estimator_")
return self.best_estimator_.predict((as_windows(X, self.layout)),
groups=groups, time_index=time_index)
def predict_proba(self, X):
check_is_fitted(self, "best_estimator_")
return self.best_estimator_.predict_proba((as_windows(X, self.layout)))
def __sklearn_tags__(self):
tags = super().__sklearn_tags__()
tags.input_tags.three_d_array = True
return tags
@contextlib.contextmanager
def _record_builds(store, force_hmm=None):
"""Record every family fitted by fit_selector (for per-candidate fit times).
Replaces the module-level default builder with a recording wrapper around the same
function, so fit_selector's code path (including its size tie-break) is unchanged.
``force_hmm`` (True/False) builds every family with that HMM setting instead of its own
(``decoder="hmm"``/``"none"``, or ``"adaptive"`` without sequences); ``None`` changes nothing.
"""
import wisp_release.selection as sel
original, reference = sel.build_family, sel.build_reference_family
def flagged(kwargs):
if force_hmm is None:
return kwargs
return {**kwargs, "overrides": {**(kwargs.get("overrides") or {}), "use_hmm": bool(force_hmm)}}
def recording(*args, **kwargs):
model = original(*args, **flagged(kwargs))
store.append((args[0] if args else kwargs.get("method_id"), model))
return model
def sized(*args, **kwargs):
return reference(*args, **flagged(kwargs))
with _PATCH_LOCK:
sel.build_family = recording
if force_hmm is not None:
sel.build_reference_family = sized
try:
yield
finally:
sel.build_family = original
sel.build_reference_family = reference
def _decide_on_model(model, valid, n_classes):
"""Decision from a model fitted on training windows with an HMM smoother."""
with kernel_context():
P = model.predict_proba(valid.X)
hmm = adaptive.hmm_labels(model, P, valid.subject, valid.time_index)
return adaptive.compare(valid.y, hmm, adaptive.argmax_labels(P), n_classes)
class WISPSelect(_SplitMixin, ClassifierMixin, BaseEstimator):
"""WISP-Select5 / WISP-Select7: validation selection among fixed families, then refit.
Candidates are fitted on training windows and scored on validation windows; the winner
is chosen by validation macro-F1, then full-vocabulary worst-class F1, then serialized
model size, then the order SS, RC, SO, CSE, GIS, CIS, ESE. A fresh winner is fitted on
train+validation. ``menu=5``: SS, RC, SO, GIS, CIS. ``menu=7`` adds CSE and ESE.
decoder : {"adaptive", "recipe", "hmm", "none"}, default="adaptive"
``"recipe"``: every family with its own decoder (the paper). ``"adaptive"``: the
selection is the paper's; then the winner's training fit decodes the validation
windows with and without the HMM, and the HMM is kept only if its macro-F1 is strictly
higher (the refit uses the chosen decoder). Without sequences (no training group with
two or more windows) candidates are scored and refitted with argmax. ``"hmm"``/``"none"``
force that decoder for scoring and refit.
"""
def __init__(self, menu=5, *, decoder="adaptive", classes=None, layout="NTC", speed="exact",
sequence="subject", n_jobs=1, random_state=42):
self.menu = menu
self.decoder = decoder
self.classes = classes
self.layout = layout
self.speed = speed
self.sequence = sequence
self.n_jobs = n_jobs
self.random_state = random_state
def fit(self, X, y, groups=None, time_index=None, validation=None, validation_size=None):
"""``validation=(X_val, y_val, groups_val[, time_val])`` (the paper protocol), validation
rows of ``X``, or ``validation_size`` to hold out that fraction of the groups (default 0.125)."""
if self.menu not in (5, 7):
raise ValueError("menu must be 5 or 7")
decoder = self._decoder()
from wisp_release.selection import fit_selector
parts = self._splits(X, y, groups, time_index, validation, validation_size)
train, valid = parts.train, parts.valid
force = self._candidate_hmm(decoder, train)
built = []
start = time.perf_counter()
with _record_builds(built, force), kernel_context():
model, report = fit_selector(f"wisp_select{self.menu}", train, valid, seed=int(self.random_state),
n_jobs=int(self.n_jobs), speed=self.speed, sequence=self.sequence)
total = time.perf_counter() - start
final_fit = selector_refit = float(getattr(model, "fit_seconds_", 0.0))
winner = report["selected_method_id"]
fit_times = {}
for family, m in built[:-1]:
fit_times[family] = float(getattr(m, "fit_seconds_", math.nan))
self.history_ = [{**{k: v for k, v in rec.items() if k != "confusion_matrix"},
"recipe": _SHORT[rec["method_id"]], "fit_seconds": fit_times.get(rec["method_id"])}
for rec in report["validation_candidates"]]
self.selection_report_ = report
self.best_recipe_ = get(_SHORT[winner])
recipe_hmm = bool(model.use_hmm)
use_hmm, reason, scores, decision_s = recipe_hmm, adaptive.FIXED, None, 0.0
if force is not None:
reason = adaptive.NO_SEQUENCES if decoder == "adaptive" else adaptive.FIXED
elif decoder == "adaptive":
t0 = time.perf_counter()
trained = dict(built[:-1])[winner]
choice, scores = _decide_on_model(trained, valid, len(self.classes_))
decision_s = time.perf_counter() - t0
reason = adaptive.VALIDATION
if (choice == "hmm") != use_hmm:
model, final_fit = self._refit_family(winner, train, valid, choice == "hmm", report)
use_hmm = choice == "hmm"
est = self._refit_classifier(self.best_recipe_, self.random_state)
est._adopt(model, classes=self.classes_, input_shape=train.X.shape[1:], use_hmm=use_hmm,
fit_seconds=final_fit, reason=reason, scores=scores, decision_seconds=decision_s)
self._finish(est, final_fit=final_fit)
self.decision_seconds_ = decision_s
self.selection_seconds_ = total - selector_refit
self.has_paper_results_ = bool(self.speed == "exact" and self.sequence == "subject" and force is None
and use_hmm == recipe_hmm)
return self
def _refit_family(self, winner, train, valid, use_hmm, report):
"""Refit the selected family on train+validation with the chosen decoder."""
from wisp.core.data import concatenate_har
from wisp_release.methods import build_family
combined = concatenate_har(train, valid, source="train+validation")
model = build_family(winner, seed=int(self.random_state), n_jobs=int(self.n_jobs), speed=self.speed,
sequence=self.sequence, overrides={"use_hmm": bool(use_hmm)})
with kernel_context():
model.fit(combined.X, combined.y, groups=combined.subject, time_index=combined.time_index,
n_classes=combined.n_classes)
if use_hmm and report.get("confidence_scale") is not None:
model.set_confidence_scale(report["confidence_scale"])
return model, float(getattr(model, "fit_seconds_", 0.0))
class WISPSearch(_SplitMixin, ClassifierMixin, BaseEstimator):
"""Recipe search on validation data, then one refit of the winner on train+validation.
strategy : {"random", "evolution", "optuna", "funnel"}
``"random"``/``"evolution"`` are WISP-Random/WISP-Evolution (paper implementations).
``"optuna"`` proposes recipes with Optuna (default sampler TPE) from the same grammar
and scores them with the same validation utility; it has no paper results.
``"funnel"`` is WISP-Funnel, the fast joint Random + Evolution search: both
controllers propose ``budget`` candidates each from the CIS family and rank them on a
training-only proxy (at most 4,096 training and 1,024 validation windows, a shared
Ridge feature dictionary, cost weight 0), a diversified shortlist of 4 per controller
is verified on the full splits with the screened Ridge solver, and each controller's
winner is refitted on train+validation. It has no paper results; its benchmark record
is in docs/RECIPES.md.
controller : {"random", "evolution"}, default="random"
``strategy="funnel"`` only: which controller's model predicts. Both are fitted in the
same run and kept in ``funnel_estimators_``.
budget : int, default=329
Number of candidates evaluated (paper: 329, i.e. 9 initial + 20 generations x 16);
for the funnel, proposals per controller.
population : int, default=24
Population of the random/evolution controller (paper: 24).
sampler : optuna sampler or None
Only for ``strategy="optuna"``; ``None`` uses ``TPESampler(seed=random_state)``.
decoder : {"adaptive", "recipe", "hmm", "none"}, default="adaptive"
Decoder of the final model. ``"recipe"``: the winner's own HMM flag (the paper).
``"adaptive"``: the search is unchanged; then the winner fitted on training windows
decodes the validation windows with and without the HMM, and the HMM is kept only if
its macro-F1 is strictly higher. Without sequences (no training group with two or more
windows) candidates are scored and refitted with argmax. ``"hmm"``/``"none"`` force that
decoder for scoring and refit.
on_no_candidate : {"raise", "default"}, default="raise"
What happens when no candidate passes validation (every candidate rejected or
collapsed). ``"raise"`` re-raises the search's error; ``"default"`` fits the default CIS
recipe (``wisp.recipes.get("cis")``) on all fitting data instead and records the reason
in ``fallback_``.
n_jobs : int, default=1
Threads for tree ensembles inside each candidate. For the funnel: controller threads
(1 runs the two controllers in series, 2 or more runs them in parallel; candidates and
BLAS use one thread each). The results do not depend on it.
output_dir : path or None
Where the controller writes its candidate ledger; a temporary directory when None.
For the funnel, ``speed``, ``sequence``, ``cost_weight`` and ``robustness_weight`` are
fixed by its protocol (``sequence`` must be ``"subject"``).
The paper's single search (one dataset, fold and seed, 329 candidates) had median
cumulative candidate fitting time of 47.5 h (Random) and 61.3 h (Evolution). Try the
fixed recipes and WISPSelect first (docs/RECIPES.md).
"""
def __init__(self, strategy="random", *, controller="random", budget=329, population=24, sampler=None,
decoder="adaptive", on_no_candidate="raise", classes=None, layout="NTC", speed="exact",
sequence="subject", n_jobs=1, random_state=42, cost_weight=0.01, robustness_weight=0.1,
output_dir=None):
self.strategy = strategy
self.controller = controller
self.budget = budget
self.population = population
self.sampler = sampler
self.decoder = decoder
self.on_no_candidate = on_no_candidate
self.classes = classes
self.layout = layout
self.speed = speed
self.sequence = sequence
self.n_jobs = n_jobs
self.random_state = random_state
self.cost_weight = cost_weight
self.robustness_weight = robustness_weight
self.output_dir = output_dir
def fit(self, X, y, groups=None, time_index=None, validation=None, validation_size=None, sampling_rate_hz=None,
study_setting=None):
"""Search on ``validation`` (or a ``validation_size`` hold-out of the groups), then refit.
``sampling_rate_hz`` and ``study_setting`` describe the recordings for the search
controllers' dataset fingerprint, which shapes their proposals (window duration from the
sampling rate; a study setting containing "free", "home", "natural" or "wild" marks
free-living data). The paper searches and the funnel benchmark record used the values of
the WristHARBench exports (``wisp.datasets`` folds expose ``sampling_rate_hz`` and
``study_setting``). When omitted, the fingerprint assumes 50 Hz and an unknown setting.
"""
check_choice("strategy", self.strategy, ("random", "evolution", "optuna", "funnel"))
check_choice("controller", self.controller, _CONTROLLERS)
check_choice("on_no_candidate", self.on_no_candidate, ("raise", "default"))
if not isinstance(self.budget, (int, np.integer)) or self.budget < 1:
raise ValueError("budget must be a positive integer")
decoder = self._decoder()
if self.strategy == "funnel" and self.sequence != "subject":
raise ValueError("strategy='funnel' supports sequence='subject' only")
parts = self._splits(X, y, groups, time_index, validation, validation_size, sampling_rate_hz, study_setting)
train, valid = parts.train, parts.valid
force = self._candidate_hmm(decoder, train)
self.fallback_ = None
self.decision_seconds_ = 0.0
if self.strategy == "funnel":
return self._fit_funnel(parts, decoder, force)
start = time.perf_counter()
try:
with kernel_context():
if self.strategy == "optuna":
best_spec, best_result = self._fit_optuna(train, valid, force)
else:
best_spec, best_result = self._fit_controller(train, valid, force)
except RuntimeError as exc:
if self.on_no_candidate != "default" or not any(m in str(exc) for m in NO_CANDIDATE_ERRORS):
raise
self.selection_seconds_ = time.perf_counter() - start
self._default_recipe(parts, decoder, self.random_state, exc)
self.has_paper_results_ = False
return self
self.selection_seconds_ = time.perf_counter() - start
params = dict(best_spec["params"])
seed = int(params.pop("seed", self.random_state))
params.pop("n_jobs", None)
self.best_recipe_ = Recipe(best_spec["name"], params)
self.best_seed_ = seed
self.best_validation_ = best_result
if decoder == "adaptive" and force is None:
est = self._adaptive_classifier(self.best_recipe_, seed, train, valid)
self.decision_seconds_ = est.decision_seconds_
else:
combined = _concat(train, valid)
est = self._refit_classifier(self.best_recipe_, seed,
decoder="recipe" if force is None else ("hmm" if force else "none"))
est.fit(combined.X, self.classes_[combined.y], groups=combined.subject, time_index=combined.time_index)
if force is not None:
est.decoder_reason_ = adaptive.NO_SEQUENCES if decoder == "adaptive" else adaptive.FIXED
if self.sequence == "recording" and best_result.get("confidence_scale") is not None:
# a train+validation refit reuses the factor chosen for the train-only candidate
with kernel_context():
est.model_.set_confidence_scale(best_result["confidence_scale"])
self._finish(est, final_fit=est.fit_seconds_)
recipe_decoder = "hmm" if self.best_recipe_.use_hmm else "none"
self.has_paper_results_ = bool(self.strategy in ("random", "evolution") and self.speed == "exact"
and self.sequence == "subject" and int(self.budget) == 329
and int(self.population) == 24 and force is None
and est.decoder_ == recipe_decoder)
return self
# ------------------------------------------------------------------ strategies
def _fit_controller(self, train, valid, force=None):
from wisp_fast.search import run_search
from ._engine import RIDGE_SOLVER
budget, pop = int(self.budget), int(self.population)
if pop < 3:
raise ValueError("population must be at least 3")
per_generation = max(pop - max(2, pop // 3), 1)
generations = max(1, -(-(budget - min(9, budget)) // per_generation))
with _output_dir(self.output_dir) as out:
options = dict(output=out, population=pop, generations=generations, max_evaluations=budget,
seed=int(self.random_state), candidate_workers=1, n_jobs_per_candidate=int(self.n_jobs),
ridge_solver=RIDGE_SOLVER[self.speed], sequence=self.sequence,
cost_weight=float(self.cost_weight), robustness_weight=float(self.robustness_weight))
if force is None:
summary = run_search(f"wisp_{self.strategy}", train, valid, None, **options)
else:
summary = _run_search_forced(f"wisp_{self.strategy}", train, valid, force, **options)
rows = [_ledger_row(r) for r in _read_ledger(Path(out) / "program_database.jsonl")]
self.history_ = rows
self.search_summary_ = {k: summary[k] for k in ("n_rows", "n_ok", "n_evaluated", "search_config")}
best = summary["best"]
return best["spec"], best["result"]
def _fit_optuna(self, train, valid, force=None):
from .integrations.optuna import run_study
study, best_spec, best_result, history = run_study(
train, valid, n_trials=int(self.budget), sampler=self.sampler, seed=int(self.random_state),
speed=self.speed, sequence=self.sequence, n_jobs=int(self.n_jobs),
cost_weight=float(self.cost_weight), robustness_weight=float(self.robustness_weight),
force_hmm=force)
self.study_ = study
self.history_ = history
return best_spec, best_result
def _fit_funnel(self, parts, decoder, force):
"""WISP-Funnel: both controllers in one run; ``controller`` picks the predicting model."""
from wisp.core.data import concatenate_har
from wisp_fast.funnel.engine import FORCE_HMM
from wisp_fast.funnel.joint import fit_joint_funnel
train, valid = parts.train, parts.valid
if int(self.population) < 3:
raise ValueError("population must be at least 3")
workers = 2 if int(self.n_jobs) >= 2 else 1
start = time.perf_counter()
token = FORCE_HMM.set(force)
try:
with _output_dir(self.output_dir) as out, kernel_context():
try:
models, summaries, receipt = fit_joint_funnel(
train, valid, output=out, seed=int(self.random_state), evaluations=int(self.budget),
population=int(self.population), workers=workers)
except RuntimeError as exc:
if self.on_no_candidate != "default" or not any(m in str(exc) for m in NO_CANDIDATE_ERRORS):
raise
models = None
failure = exc
else:
rows = {c: [_ledger_row(r) for r in
_read_ledger(Path(out) / f"wisp_{c}" / "proxy" / "program_database.jsonl")]
for c in _CONTROLLERS}
finally:
FORCE_HMM.reset(token)
funnel_s = time.perf_counter() - start
self.funnel_seconds_ = funnel_s
if models is None:
self.selection_seconds_ = funnel_s
est = self._default_recipe(parts, decoder, self.random_state, failure)
self.funnel_estimators_ = {c: est for c in _CONTROLLERS}
self.funnel_summary_, self.funnel_receipt_, self.history_ = None, None, []
self.has_paper_results_ = False
return self
combined = None
estimators, info, decided = {}, {}, {}
for c in _CONTROLLERS:
summary = summaries[f"wisp_{c}"]
spec = summary["winner_recipe"]
params = dict(spec["params"])
seed = int(params.pop("seed", self.random_state))
params.pop("n_jobs", None)
recipe = Recipe(spec["name"], params)
recipe_hmm = bool(spec["params"].get("use_hmm", True)) if force is None else bool(force)
model, final_fit = models[f"wisp_{c}"], float(summary["refit_seconds"])
use_hmm, reason, scores, decision_s = recipe_hmm, adaptive.FIXED, None, 0.0
if force is not None and decoder == "adaptive":
reason = adaptive.NO_SEQUENCES
elif decoder == "adaptive":
key = _spec_key(spec)
if key not in decided:
decided[key] = _funnel_decision(spec, train, valid, len(self.classes_))
choice, scores, decision_s = decided[key]
use_hmm, reason = choice == "hmm", adaptive.VALIDATION
if use_hmm != recipe_hmm:
if combined is None:
combined = concatenate_har(train, valid, source="train+validation")
model, final_fit = _funnel_refit(spec, combined, use_hmm)
est = self._refit_classifier(recipe, seed)
est._adopt(model, classes=self.classes_, input_shape=train.X.shape[1:], use_hmm=use_hmm,
fit_seconds=final_fit, reason=reason, scores=scores, decision_seconds=decision_s)
estimators[c] = est
info[c] = {k: summary.get(k) for k in ("proxy_seconds", "verification_seconds", "refit_seconds",
"n_proposals", "n_proxy_ok", "n_unique_proxy_ok",
"n_full_verified", "n_full_ok", "proxy_winner")}
info[c].update(winner_id=summary["best"]["id"], recipe=recipe, seed=seed,
validation=summary["best"]["result"])
chosen = self.controller
best = summaries[f"wisp_{chosen}"]
self.funnel_estimators_ = estimators
self.funnel_summary_ = info
self.funnel_receipt_ = receipt
self.history_ = rows[chosen]
self.best_recipe_ = info[chosen]["recipe"]
self.best_seed_ = info[chosen]["seed"]
self.best_validation_ = best["best"]["result"]
self.selection_seconds_ = float(best["proxy_seconds"]) + float(best["verification_seconds"])
self.decision_seconds_ = estimators[chosen].decision_seconds_
self._finish(estimators[chosen], final_fit=estimators[chosen].fit_seconds_)
self.has_paper_results_ = False
return self
def _spec_key(spec):
import json
return json.dumps([spec["name"], spec["params"]], sort_keys=True)
def _funnel_decision(spec, train, valid, n_classes):
"""Fit the funnel winner on training windows with the HMM (as in its verification), decide on valid."""
from wisp_fast.funnel.selection import fit_on
from wisp_fast.funnel.shared_gram import verify_candidate
start = time.perf_counter()
hmm_spec = {**spec, "params": {**spec["params"], "use_hmm": True}}
with kernel_context():
model = fit_on(verify_candidate(hmm_spec, mode="exact", budget=None, ridge_solver="screened"), train)
choice, scores = _decide_on_model(model, valid, n_classes)
return choice, scores, time.perf_counter() - start
def _funnel_refit(spec, combined, use_hmm):
"""Refit a funnel winner on train+validation with the chosen decoder (the engine's refit factory)."""
from wisp_fast.funnel.search import make_candidate
from wisp_fast.funnel.selection import fit_on
flipped = {**spec, "params": {**spec["params"], "use_hmm": bool(use_hmm)}}
start = time.perf_counter()
with kernel_context():
model = fit_on(make_candidate(flipped, mode="exact", budget=None, ridge_solver="screened"), combined)
return model, time.perf_counter() - start
def _run_search_forced(method, train, valid, force_hmm, *, output, population, generations, max_evaluations,
seed, candidate_workers, n_jobs_per_candidate, ridge_solver, sequence, cost_weight,
robustness_weight):
"""``wisp_fast.search.run_search`` with every candidate built with HMM flag ``force_hmm``.
The controller, its proposals, ledger and utility are unchanged; only the candidate factory
overrides ``use_hmm`` (``decoder="hmm"``/``"none"``, or ``"adaptive"`` without sequences).
"""
from dataclasses import replace
from wisp.search.profiles import get_profile
from wisp_fast.search import _AcceleratedSearch, _configure
from wisp_fast.selection import check_splits
check_splits(train, valid)
profile = replace(get_profile(method), max_evaluations=int(max_evaluations))
search = _AcceleratedSearch(train=train, valid=valid, output=output, profile=profile, test=None,
population=population, generations=generations, seed=seed,
cost_weight=cost_weight, robustness_weight=robustness_weight,
candidate_workers=candidate_workers, n_jobs_per_candidate=n_jobs_per_candidate)
_configure(search, "exact", None, ridge_solver, 192, 128, 4 if profile.use_stress else 0, "complete",
sequence=sequence)
base = search.create_candidate
def create(spec_dict):
return base({**spec_dict, "params": {**spec_dict.get("params", {}), "use_hmm": bool(force_hmm)}})
search.create_candidate = create
try:
summary = search.run()
finally:
search.memory.clear()
return summary
def _concat(train, valid):
from wisp.core.data import concatenate_har
return concatenate_har(train, valid, source="train+validation")
@contextlib.contextmanager
def _output_dir(path):
if path is not None:
Path(path).mkdir(parents=True, exist_ok=True)
if any(Path(path).iterdir()):
raise ValueError("output_dir must be empty: the search ledger is never appended to")
yield str(path)
return
with tempfile.TemporaryDirectory(prefix="wisp_search_") as tmp:
yield tmp
def _read_ledger(path):
import json
if not path.exists():
return []
with path.open(encoding="utf-8") as f:
return [json.loads(line) for line in f if line.strip()]
def _ledger_row(row):
res = row.get("result", {})
return {"id": row.get("id"), "generation": row.get("generation"), "status": row.get("status"),
"recipe": {"name": row["spec"]["name"],
"params": {k: v for k, v in row["spec"]["params"].items() if k not in ("seed", "n_jobs")}},
"seed": row["spec"]["params"].get("seed"),
"validation_utility": res.get("predictive_utility"), "validation_macro_f1": res.get("macro_f1"),
"validation_worst_class_f1": res.get("worst_class_f1"),
"fit_seconds": res.get("fit_seconds"), "predict_seconds": res.get("predict_seconds")}