Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / tests /test_adaptive_decoder.py
Zipeng365's picture
wisp-har 0.4.0: WISP-Funnel search and the adaptive HMM switch (default decoder)
93a9282 verified
Raw History Blame Contribute Delete
11.1 kB
"""decoder="adaptive" (the default since 0.4.0): decision rule, edge cases and bit-identity of the final model."""
import warnings
import numpy as np
import pytest
from numpy.testing import assert_array_equal
from sklearn.linear_model import LogisticRegression
from threadpoolctl import threadpool_limits
from wisp import HMMSmoother, WISPClassifier, WISPSearch, WISPSelect
def sequence_fixture(n_subjects=8, per_subject=40, block=10, T=48, C=3, K=3, noise=1.4, seed=0, offset=0):
"""Windows whose labels persist in blocks within each group, with noisy per-window evidence,
so sequence decoding can help."""
rng = np.random.default_rng(seed)
X, y, g, t = [], [], [], []
tt = np.arange(T)
for s in range(n_subjects):
labels = np.repeat(rng.permutation(np.resize(np.arange(K), per_subject // block)), block)[:per_subject]
for i, k in enumerate(labels):
w = np.stack([np.sin(2 * np.pi * (k + 1 + c) * tt / T + rng.normal()) for c in range(C)], 1)
X.append(w + rng.normal(0, noise, (T, C)))
y.append(k), g.append(s + offset), t.append(i)
return np.asarray(X), np.asarray(y), np.asarray(g), np.asarray(t, dtype=float)
@pytest.fixture(scope="module")
def seq_data():
return sequence_fixture(seed=0), sequence_fixture(n_subjects=3, seed=1, offset=100), \
sequence_fixture(n_subjects=4, seed=2, offset=200)
def _same_outputs(a, b, Xq, gq, tq):
assert_array_equal(a.predict_proba(Xq), b.predict_proba(Xq))
assert_array_equal(a.predict(Xq, groups=gq, time_index=tq), b.predict(Xq, groups=gq, time_index=tq))
@pytest.mark.parametrize("recipe", ["rc", "cis", "ss"])
def test_adaptive_final_model_equals_fixed_decoder_model(recipe, seq_data):
"""Whatever the switch chooses, the final model is the fixed-decoder model with that setting."""
(X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
with threadpool_limits(1):
for kwargs in ({}, {"validation": (Xv, yv, gv, tv)}):
ada = WISPClassifier(recipe).fit(X, y, groups=g, time_index=t, **kwargs)
assert ada.decoder_reason_ == "validation"
assert set(ada.decoder_scores_) == {"hmm", "none"}
assert ada.decoder_ == ("hmm" if ada.decoder_scores_["hmm"] > ada.decoder_scores_["none"] else "none")
fixed = WISPClassifier(recipe, decoder=ada.decoder_).fit(X, y, groups=g, time_index=t, **kwargs)
_same_outputs(ada, fixed, Xq, gq, tq)
assert ada.has_paper_results_ == fixed.has_paper_results_
def test_adaptive_keeps_hmm_bit_identical_to_fixed_hmm(seq_data):
"""When the switch keeps the HMM, the final model equals decoder="hmm" (and "recipe") bit for bit."""
(X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
with threadpool_limits(1):
ada = WISPClassifier("cis").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
assert ada.decoder_ == "hmm", ada.decoder_scores_ # blocks of 10 windows: smoothing helps here
for decoder in ("hmm", "recipe"):
fixed = WISPClassifier("cis", decoder=decoder).fit(X, y, groups=g, time_index=t,
validation=(Xv, yv, gv, tv))
_same_outputs(ada, fixed, Xq, gq, tq)
assert ada.has_paper_results_ is True and ada.decision_seconds_ > 0
# validation windows are fitting data: the tuple form fits training then validation windows
both = WISPClassifier("cis", decoder="hmm").fit(np.concatenate([X, Xv]), np.concatenate([y, yv]),
groups=np.concatenate([g, gv]),
time_index=np.concatenate([t, tv]))
_same_outputs(ada, both, Xq, gq, tq)
def test_adaptive_with_readout(seq_data):
(X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
ada = WISPClassifier("cis:RS", readout=LogisticRegression(max_iter=300)).fit(
X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
assert ada.decoder_reason_ == "validation"
fixed = WISPClassifier("cis:RS", decoder=ada.decoder_, readout=LogisticRegression(max_iter=300)).fit(
X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
_same_outputs(ada, fixed, Xq, gq, tq)
def test_no_sequences_gives_none():
"""Every window its own group: HMM decoding would only reweight by the class prior."""
X, y, _, _ = sequence_fixture(n_subjects=4, per_subject=30)
g = np.arange(len(y))
clf = WISPClassifier("rc").fit(X, y, groups=g)
assert clf.decoder_ == "none" and clf.decoder_reason_ == "no sequences" and clf.decision_seconds_ == 0
assert clf.decoder_scores_ is None
with pytest.raises(ValueError, match="needs groups"):
WISPClassifier("rc", decoder="hmm").fit(X, y)
# explicit decoder="hmm" keeps today's behaviour (prior-weighted one-window decoding)
hmm = WISPClassifier("rc", decoder="hmm").fit(X, y, groups=g)
P = hmm.predict_proba(X[:20])
prior = hmm.model_.smoother_.prior_
assert_array_equal(hmm.predict(X[:20], groups=np.arange(20)), np.argmax(np.log(np.clip(P, 1e-12, 1)) +
np.log(np.clip(prior, 1e-12, 1)), 1))
def test_edge_cases_no_groups_and_one_group(seq_data):
(X, y, g, t), _, _ = seq_data
with warnings.catch_warnings():
warnings.simplefilter("error")
clf = WISPClassifier("rc").fit(X, y)
assert clf.decoder_ == "none" and clf.decoder_reason_ == "no groups"
with pytest.warns(UserWarning, match="at least 2 groups"):
one = WISPClassifier("rc").fit(X, y, groups=np.zeros(len(y), int), time_index=np.arange(len(y)))
assert one.decoder_reason_ == "fewer than 2 groups" and one.decoder_ == "hmm" # the recipe's own decoder
def test_auto_alias_is_deprecated(seq_data):
(X, y, g, t), _, (Xq, _, gq, tq) = seq_data
with pytest.warns(FutureWarning, match="decoder='recipe'"):
auto = WISPClassifier("rc", decoder="auto").fit(X, y, groups=g, time_index=t)
recipe = WISPClassifier("rc", decoder="recipe").fit(X, y, groups=g, time_index=t)
_same_outputs(auto, recipe, Xq, gq, tq)
with pytest.raises(ValueError, match="decoder"):
WISPClassifier("rc", decoder="viterbi").fit(X, y, groups=g)
def test_validation_rows_equal_validation_tuple(seq_data):
(X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
Xa, ya, ga, ta = (np.concatenate(p) for p in ((X, Xv), (y, yv), (g, gv), (t, tv)))
rows = np.arange(len(X), len(Xa))
a = WISPClassifier("rc").fit(Xa, ya, groups=ga, time_index=ta, validation=rows)
b = WISPClassifier("rc").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
assert a.decoder_ == b.decoder_ and a.decoder_scores_ == b.decoder_scores_
_same_outputs(a, b, Xq, gq, tq) # same rows in the same order
def test_select_adaptive(seq_data):
(X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
with threadpool_limits(1):
rec = WISPSelect(5, decoder="recipe").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
ada = WISPSelect(5).fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
assert ada.best_recipe_.id == rec.best_recipe_.id # the selection itself is the paper's
assert [h["macro_f1"] for h in ada.history_] == [h["macro_f1"] for h in rec.history_]
assert ada.decoder_reason_ == "validation"
if ada.decoder_ == "hmm":
_same_outputs(ada, rec, Xq, gq, tq)
assert ada.has_paper_results_
else:
fixed = WISPClassifier(ada.best_recipe_, decoder="none").fit(X, y, groups=g, time_index=t,
validation=(Xv, yv, gv, tv))
_same_outputs(ada, fixed, Xq, gq, tq)
assert not ada.has_paper_results_
def _singletons():
X, y, _, _ = sequence_fixture(n_subjects=4, per_subject=30, seed=3)
Xv, yv, _, _ = sequence_fixture(n_subjects=2, per_subject=30, seed=4)
return X, y, np.arange(len(y)), Xv, yv, np.arange(len(yv)) + 10_000
def test_select_scores_candidates_direct_without_sequences():
from wisp_release.scoring import score
X, y, g, Xv, yv, gv = _singletons()
with threadpool_limits(1):
ada = WISPSelect(5).fit(X, y, groups=g, validation=(Xv, yv, gv))
none = WISPSelect(5, decoder="none").fit(X, y, groups=g, validation=(Xv, yv, gv))
assert ada.decoder_ == "none" and ada.decoder_reason_ == "no sequences"
assert [h["macro_f1"] for h in ada.history_] == [h["macro_f1"] for h in none.history_]
for h in ada.history_:
clf = WISPClassifier(h["recipe"], decoder="none", classes=[0, 1, 2]).fit(X, y)
assert h["macro_f1"] == score(yv, clf.predict(Xv), 3)["macro_f1"]
assert_array_equal(ada.predict(Xv, groups=gv), none.predict(Xv, groups=gv))
@pytest.mark.parametrize("strategy", ["random", "funnel"])
def test_search_scores_candidates_direct_without_sequences(strategy):
X, y, g, Xv, yv, gv = _singletons()
kw = dict(budget=8, population=4)
if strategy == "random":
kw["cost_weight"] = 0.0 # the utility's time term would differ between two runs
ada = WISPSearch(strategy, **kw).fit(X, y, groups=g, validation=(Xv, yv, gv))
none = WISPSearch(strategy, decoder="none", **kw).fit(X, y, groups=g, validation=(Xv, yv, gv))
assert ada.decoder_ == "none" and ada.decoder_reason_ == "no sequences"
assert [h["validation_macro_f1"] for h in ada.history_] == [h["validation_macro_f1"] for h in none.history_]
assert [h["validation_utility"] for h in ada.history_] == [h["validation_utility"] for h in none.history_]
assert ada.best_validation_["predictive_utility"] == none.best_validation_["predictive_utility"]
assert_array_equal(ada.predict_proba(Xv), none.predict_proba(Xv))
def test_hmm_smoother_adaptive(seq_data):
from wisp.cpu.hmm import TransitionSmoother
(X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
clf = WISPClassifier("rc", decoder="none").fit(X, y)
Pv, Pq = clf.predict_proba(Xv), clf.predict_proba(Xq)
plain = HMMSmoother().fit(y, groups=g, time_index=t) # no validation scores: decodes as before
assert plain.decoder_ == "hmm" and plain.decoder_reason_ == "no validation scores"
kernel = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=3)
assert_array_equal(plain.decode(Pq, groups=gq, time_index=tq), kernel.predict_from_proba(Pq, groups=gq, time_index=tq))
ada = HMMSmoother().fit(y, groups=g, time_index=t, validation=(Pv, yv, gv, tv))
assert ada.decoder_reason_ == "validation"
if ada.decoder_ == "none":
assert_array_equal(ada.decode(Pq, groups=gq, time_index=tq), np.argmax(Pq, 1))
lone = HMMSmoother().fit(y, groups=np.arange(len(y)), validation=(Pv, yv, np.arange(len(yv)) + 10_000))
assert lone.decoder_ == "none" and lone.decoder_reason_ == "no sequences"
assert_array_equal(lone.decode(Pq, groups=gq), np.argmax(Pq, 1))