Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
File size: 5,230 Bytes
4731b11
 
 
 
 
 
 
 
 
 
 
 
 
 
93a9282
4731b11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
"""WISPSelect / WISPSearch on fixture windows: small-budget smoke runs and input checks."""
import numpy as np
import pytest
from numpy.testing import assert_array_equal

from wisp import WISPClassifier, WISPSearch, WISPSelect
from wisp.recipes import Recipe


def test_select_matches_fit_selector(train_windows, valid_windows, test_windows):
    from wisp.selection import _har
    from wisp_release.selection import fit_selector
    X, y, g, t = train_windows
    Xv, yv, gv, tv = valid_windows
    sel = WISPSelect(5, decoder="recipe").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
    names = ["0", "1", "2"]
    model, report = fit_selector("wisp_select5", _har(X, y, g, t, names, "t"), _har(Xv, yv, gv, tv, names, "v"),
                                 seed=42, n_jobs=1)
    assert sel.best_recipe_.provenance["method_id"] == report["selected_method_id"]
    assert [h["recipe"] for h in sel.history_] == ["ss", "rc", "so", "gis", "cis"]
    assert all(h["fit_seconds"] > 0 for h in sel.history_)
    assert sel.selection_seconds_ > 0 and sel.final_fit_seconds_ > 0
    Xq, _, gq, tq = test_windows
    assert_array_equal(sel.predict(Xq, groups=gq, time_index=tq), model.predict(Xq, groups=gq, time_index=tq))
    assert isinstance(sel.best_estimator_, WISPClassifier) and sel.best_estimator_.has_paper_results_


def test_select7_and_validation_size(train_windows):
    X, y, g, t = train_windows
    sel = WISPSelect(7).fit(X, y, groups=g, validation_size=0.34)
    assert len(sel.history_) == 7
    with pytest.raises(ValueError):
        WISPSelect(6).fit(X, y, groups=g)


def test_group_overlap_and_missing_groups(train_windows, valid_windows):
    X, y, g, t = train_windows
    Xv, yv, gv, tv = valid_windows
    with pytest.raises(ValueError, match="overlap"):
        WISPSelect().fit(X, y, groups=g, validation=(Xv, yv, np.full(len(yv), g[0])))
    with pytest.raises(ValueError, match="groups"):
        WISPSearch(budget=3).fit(X, y)
    with pytest.raises(ValueError, match="either"):
        WISPSelect().fit(X, y, groups=g, validation=(Xv, yv, gv), validation_size=0.2)


@pytest.mark.parametrize("strategy,budget", [("random", 6), ("evolution", 12)])
def test_search_small_budget(strategy, budget, train_windows, valid_windows, test_windows, tmp_path):
    X, y, g, t = train_windows
    Xv, yv, gv, tv = valid_windows
    search = WISPSearch(strategy, budget=budget, population=4, output_dir=tmp_path / strategy)
    search.fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
    assert len(search.history_) == budget
    assert (tmp_path / strategy / "program_database.jsonl").exists()
    assert isinstance(search.best_recipe_, Recipe)
    assert search.has_paper_results_ is False  # paper budget is 329
    ok = [h for h in search.history_ if h["status"] == "ok"]
    best = max(ok, key=lambda h: h["validation_utility"])
    assert best["validation_utility"] == search.best_validation_["predictive_utility"]
    assert best["recipe"]["name"] == search.best_recipe_.family
    assert search.predict(test_windows[0]).shape == (len(test_windows[0]),)
    assert search.final_fit_seconds_ > 0 and search.selection_seconds_ > 0


def test_search_optuna(train_windows, valid_windows):
    pytest.importorskip("optuna")
    X, y, g, t = train_windows
    Xv, yv, gv, tv = valid_windows
    search = WISPSearch("optuna", budget=8).fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
    assert len(search.study_.trials) == 8 and len(search.history_) >= 1
    assert search.has_paper_results_ is False
    ok = [h for h in search.history_ if h["status"] == "ok"]
    assert max(h["validation_utility"] for h in ok) == pytest.approx(search.best_validation_["predictive_utility"])


def test_suggest_recipe_in_user_study():
    optuna = pytest.importorskip("optuna")
    from wisp.integrations.optuna import recipe_operators, suggest_recipe
    seen = set()

    def objective(trial):
        r = suggest_recipe(trial)
        assert r.family in ("har_sculpt_forest_hmm", "rocket_sketch_ridge_hmm", "symbolic_interval_forest",
                            "hybrid_cpu_ensemble", "invariant_interval_shape_forest", "state_rocket_interval_hmm",
                            "posterior_cpu_ensemble")
        ops = recipe_operators(r)
        assert ops[-1] == ("hmm" if r.use_hmm else "argmax")
        seen.add(r.family)
        return trial.suggest_float("other_hyperparameter", 0, 1)

    study = optuna.create_study(sampler=optuna.samplers.RandomSampler(seed=0))
    study.optimize(objective, n_trials=40)
    assert len(seen) >= 5


def test_operator_lists_follow_grammar():
    from wisp.ablations import get_recipe_spec
    from wisp.integrations.optuna import recipe_operators
    from wisp.search.grammar import initial_cpu_candidates
    templates = initial_cpu_candidates(0)
    for idx in (0, 2, 4, 5, 6, 7, 8):
        tpl = templates[idx]
        params = {k: v for k, v in tpl.params.items() if k != "seed"}
        assert recipe_operators(Recipe(tpl.name, params)) == tpl.operators
    spec = get_recipe_spec("cis_ablation_RS_direct")
    cand = spec.to_candidate(seed=0, n_jobs=1)
    assert recipe_operators(Recipe(cand.name, cand.params)) == cand.operators