Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / tests /test_engine.py
Zipeng365's picture
wisp-har 0.2.0: accelerated implementation with speed (exact/fast) and HMM sequence (subject/recording) options
ac7f7cb verified
Raw History Blame Contribute Delete
7.11 kB
import pickle
import numpy as np
import unittest
from unittest.mock import patch
from contextlib import nullcontext
from numpy.testing import assert_allclose, assert_array_equal
from threadpoolctl import threadpool_limits
from wisp.cpu.algorithms import build_cpu_estimator
from wisp.cpu.hmm import TransitionSmoother
from wisp.registry import CANONICAL_METHOD_IDS
from wisp_release.methods import build_reference_family as original
from wisp_fast.decoder import FastSmoother
from wisp_fast.engine import accelerate_checkpoint, build_candidate, build_family, TurboBudget
def windows():
rng = np.random.default_rng(415)
X = rng.normal(size=(38, 42, 3))
y = np.resize([0, 1, 3], len(X))
return X[:27], y[:27], X[27:]
def check_full_recipe_and_converted_checkpoint(family):
X, y, Q = windows()
g, t = np.arange(len(X)) % 4, np.arange(len(X))[::-1]
gq, tq = np.arange(len(Q)) % 3, np.arange(len(Q))[::-1]
with threadpool_limits(1):
ref = original(family, seed=9).fit(X, y, groups=g, time_index=t, n_classes=5)
fast = build_family(family, seed=9).fit(X, y, groups=g, time_index=t, n_classes=5)
ref_p = ref.predict_proba(Q)
ref_y = ref.predict(Q, groups=gq, time_index=tq)
fast_from_ref = accelerate_checkpoint(ref)
for m in (fast, fast_from_ref):
p = m.predict_proba(Q)
assert_allclose(p, ref_p, atol=2e-10, rtol=1e-9)
assert_array_equal(p[:, [2, 4]], 0)
assert_array_equal(m.decode_proba(p, groups=gq, time_index=tq), ref_y)
assert_array_equal(m.predict(Q, batch_size=3, groups=gq, time_index=tq), ref_y)
assert_array_equal(pickle.loads(pickle.dumps(fast_from_ref)).predict(Q), ref.predict(Q))
def check_turbo_and_chronology(family):
X, y, Q = windows()
g, t = np.arange(len(X)) % 4, np.arange(len(X))[::-1]
b = TurboBudget(kernels=4, intervals=2, shapelets=2, trees=4, max_steps=28, max_train=12)
with threadpool_limits(1):
m = build_family(family, mode="turbo", budget=b).fit(X, y, groups=g, time_index=t, n_classes=5)
assert m.fit_windows_ == 12
ref_s = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=5)
assert_array_equal(m.smoother_.transition_, ref_s.transition_)
p = m.predict_proba(Q)
assert_allclose(p.sum(axis=1), 1)
assert_array_equal(p[:, [2, 4]], 0)
def check_custom_recipes():
X, y, Q = windows()
for name, params in [
("har_sculpt_forest_hmm", dict(classifier="rf", segment_sizes=(3, 6), symbolic_bins=5)),
("rocket_sketch_ridge_hmm", dict(classifier="logreg", n_kernels=11)),
("symbolic_interval_forest", {}),
("hybrid_cpu_ensemble", {}),
("invariant_interval_shape_forest", dict(n_shapelets=3)),
("state_rocket_interval_hmm", dict(n_kernels=9, n_intervals=4)),
("posterior_cpu_ensemble", dict(weights=(0.0, 0.4, 0.6))),
]:
spec = dict(name=name, route="cpu", params=dict(seed=33, n_jobs=1, use_hmm=False, **params))
with threadpool_limits(1):
ref = build_cpu_estimator(spec).fit(X, y)
fast = build_candidate(spec).fit(X, y)
assert_allclose(fast.predict_proba(Q), ref.predict_proba(Q), atol=2e-10, rtol=1e-9)
def check_transition_and_tie_order(mode):
rng = np.random.default_rng(911)
g = rng.integers(0, 5, 100) if mode != "none" else None
t = rng.integers(0, 10, 100) if mode == "time" else None
y = rng.integers(0, 4, 100)
a = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=6)
b = FastSmoother().fit(y, groups=g, time_index=t, n_classes=6)
assert_array_equal(a.prior_, b.prior_)
assert_array_equal(a.transition_, b.transition_)
for p in (rng.uniform(0, 1, (100, 6)), np.ones((100, 6)), np.zeros((100, 6))):
assert_array_equal(a.predict_from_proba(p, groups=g, time_index=t),
b.predict_from_proba(p, groups=g, time_index=t))
class EngineTests(unittest.TestCase):
def test_full_recipe_and_checkpoint(self):
for family in CANONICAL_METHOD_IDS:
with self.subTest(family=family):
check_full_recipe_and_converted_checkpoint(family)
def test_turbo_and_chronology(self):
for family in CANONICAL_METHOD_IDS:
with self.subTest(family=family):
check_turbo_and_chronology(family)
def test_custom_recipes(self):
check_custom_recipes()
def test_controlled_recipes(self):
from wisp.ablations import CIS_ABLATIONS, build_recipe as ref_builder
from wisp_fast.engine import build_recipe
X, y, Q = windows()
g, t = np.arange(len(X)) % 3, np.arange(len(X))[::-1]
qg, qt = np.arange(len(Q)) % 3, np.arange(len(Q))[::-1]
for recipe in CIS_ABLATIONS:
with self.subTest(recipe_id=recipe.recipe_id), threadpool_limits(1):
ref = ref_builder(recipe.recipe_id, seed=8, n_jobs=1)
fast = build_recipe(recipe.recipe_id, seed=8)
ref.fit(X, y, groups=g, time_index=t, n_classes=5)
fast.fit(X, y, groups=g, time_index=t, n_classes=5)
assert (fast.smoother_ is not None) == recipe.use_hmm
assert len(fast.blocks_) == sum((recipe.use_rocket, recipe.use_intervals, recipe.use_state))
assert type(ref.model_.named_steps["clf"]) is type(fast.model_.named_steps["clf"])
assert_allclose(ref.predict_proba(Q), fast.predict_proba(Q), atol=2e-10, rtol=1e-9)
assert_array_equal(ref.predict(Q, groups=qg, time_index=qt),
fast.predict(Q, groups=qg, time_index=qt))
def test_forest_kernel_and_fallback(self):
from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier
from wisp_fast.forest import local_proba, ForestPlan
rng = np.random.default_rng(611)
X = rng.normal(size=(110, 9))
y = rng.choice([0, 2, 5], size=len(X))
weight = rng.uniform(0.1, 2, size=len(X))
Q = rng.normal(size=(27, 9))
Q[0, 2] = np.nan
for cls in (ExtraTreesClassifier, RandomForestClassifier):
with self.subTest(cls=cls):
model = cls(n_estimators=11, n_jobs=1, random_state=9,
min_samples_leaf=4).fit(X, y, sample_weight=weight)
ref = model.predict_proba(Q)
plan = ForestPlan(model)
assert_array_equal(local_proba(model, Q, plan), ref)
plan.table = None
assert_array_equal(local_proba(model, Q, plan), ref)
def test_decoder(self):
for backend in ("default", "numpy"):
ctx = patch("wisp_fast.decoder._native", lambda: None) if backend == "numpy" else nullcontext()
with ctx:
for mode in ("none", "group", "time"):
with self.subTest(backend=backend, mode=mode):
check_transition_and_tie_order(mode)
if __name__ == "__main__":
unittest.main()