Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / tests /test_api_selection.py
Zipeng365's picture
wisp-har 0.4.0: WISP-Funnel search and the adaptive HMM switch (default decoder)
93a9282 verified
Raw History Blame Contribute Delete
5.23 kB
"""WISPSelect / WISPSearch on fixture windows: small-budget smoke runs and input checks."""
import numpy as np
import pytest
from numpy.testing import assert_array_equal
from wisp import WISPClassifier, WISPSearch, WISPSelect
from wisp.recipes import Recipe
def test_select_matches_fit_selector(train_windows, valid_windows, test_windows):
from wisp.selection import _har
from wisp_release.selection import fit_selector
X, y, g, t = train_windows
Xv, yv, gv, tv = valid_windows
sel = WISPSelect(5, decoder="recipe").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
names = ["0", "1", "2"]
model, report = fit_selector("wisp_select5", _har(X, y, g, t, names, "t"), _har(Xv, yv, gv, tv, names, "v"),
seed=42, n_jobs=1)
assert sel.best_recipe_.provenance["method_id"] == report["selected_method_id"]
assert [h["recipe"] for h in sel.history_] == ["ss", "rc", "so", "gis", "cis"]
assert all(h["fit_seconds"] > 0 for h in sel.history_)
assert sel.selection_seconds_ > 0 and sel.final_fit_seconds_ > 0
Xq, _, gq, tq = test_windows
assert_array_equal(sel.predict(Xq, groups=gq, time_index=tq), model.predict(Xq, groups=gq, time_index=tq))
assert isinstance(sel.best_estimator_, WISPClassifier) and sel.best_estimator_.has_paper_results_
def test_select7_and_validation_size(train_windows):
X, y, g, t = train_windows
sel = WISPSelect(7).fit(X, y, groups=g, validation_size=0.34)
assert len(sel.history_) == 7
with pytest.raises(ValueError):
WISPSelect(6).fit(X, y, groups=g)
def test_group_overlap_and_missing_groups(train_windows, valid_windows):
X, y, g, t = train_windows
Xv, yv, gv, tv = valid_windows
with pytest.raises(ValueError, match="overlap"):
WISPSelect().fit(X, y, groups=g, validation=(Xv, yv, np.full(len(yv), g[0])))
with pytest.raises(ValueError, match="groups"):
WISPSearch(budget=3).fit(X, y)
with pytest.raises(ValueError, match="either"):
WISPSelect().fit(X, y, groups=g, validation=(Xv, yv, gv), validation_size=0.2)
@pytest.mark.parametrize("strategy,budget", [("random", 6), ("evolution", 12)])
def test_search_small_budget(strategy, budget, train_windows, valid_windows, test_windows, tmp_path):
X, y, g, t = train_windows
Xv, yv, gv, tv = valid_windows
search = WISPSearch(strategy, budget=budget, population=4, output_dir=tmp_path / strategy)
search.fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
assert len(search.history_) == budget
assert (tmp_path / strategy / "program_database.jsonl").exists()
assert isinstance(search.best_recipe_, Recipe)
assert search.has_paper_results_ is False # paper budget is 329
ok = [h for h in search.history_ if h["status"] == "ok"]
best = max(ok, key=lambda h: h["validation_utility"])
assert best["validation_utility"] == search.best_validation_["predictive_utility"]
assert best["recipe"]["name"] == search.best_recipe_.family
assert search.predict(test_windows[0]).shape == (len(test_windows[0]),)
assert search.final_fit_seconds_ > 0 and search.selection_seconds_ > 0
def test_search_optuna(train_windows, valid_windows):
pytest.importorskip("optuna")
X, y, g, t = train_windows
Xv, yv, gv, tv = valid_windows
search = WISPSearch("optuna", budget=8).fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
assert len(search.study_.trials) == 8 and len(search.history_) >= 1
assert search.has_paper_results_ is False
ok = [h for h in search.history_ if h["status"] == "ok"]
assert max(h["validation_utility"] for h in ok) == pytest.approx(search.best_validation_["predictive_utility"])
def test_suggest_recipe_in_user_study():
optuna = pytest.importorskip("optuna")
from wisp.integrations.optuna import recipe_operators, suggest_recipe
seen = set()
def objective(trial):
r = suggest_recipe(trial)
assert r.family in ("har_sculpt_forest_hmm", "rocket_sketch_ridge_hmm", "symbolic_interval_forest",
"hybrid_cpu_ensemble", "invariant_interval_shape_forest", "state_rocket_interval_hmm",
"posterior_cpu_ensemble")
ops = recipe_operators(r)
assert ops[-1] == ("hmm" if r.use_hmm else "argmax")
seen.add(r.family)
return trial.suggest_float("other_hyperparameter", 0, 1)
study = optuna.create_study(sampler=optuna.samplers.RandomSampler(seed=0))
study.optimize(objective, n_trials=40)
assert len(seen) >= 5
def test_operator_lists_follow_grammar():
from wisp.ablations import get_recipe_spec
from wisp.integrations.optuna import recipe_operators
from wisp.search.grammar import initial_cpu_candidates
templates = initial_cpu_candidates(0)
for idx in (0, 2, 4, 5, 6, 7, 8):
tpl = templates[idx]
params = {k: v for k, v in tpl.params.items() if k != "seed"}
assert recipe_operators(Recipe(tpl.name, params)) == tpl.operators
spec = get_recipe_spec("cis_ablation_RS_direct")
cand = spec.to_candidate(seed=0, n_jobs=1)
assert recipe_operators(Recipe(cand.name, cand.params)) == cand.operators