import pickle import numpy as np import unittest from unittest.mock import patch from contextlib import nullcontext from numpy.testing import assert_allclose, assert_array_equal from threadpoolctl import threadpool_limits from wisp.cpu.algorithms import build_cpu_estimator from wisp.cpu.hmm import TransitionSmoother from wisp.registry import CANONICAL_METHOD_IDS from wisp_release.methods import build_reference_family as original from wisp_fast.decoder import FastSmoother from wisp_fast.engine import accelerate_checkpoint, build_candidate, build_family, TurboBudget def windows(): rng = np.random.default_rng(415) X = rng.normal(size=(38, 42, 3)) y = np.resize([0, 1, 3], len(X)) return X[:27], y[:27], X[27:] def check_full_recipe_and_converted_checkpoint(family): X, y, Q = windows() g, t = np.arange(len(X)) % 4, np.arange(len(X))[::-1] gq, tq = np.arange(len(Q)) % 3, np.arange(len(Q))[::-1] with threadpool_limits(1): ref = original(family, seed=9).fit(X, y, groups=g, time_index=t, n_classes=5) fast = build_family(family, seed=9).fit(X, y, groups=g, time_index=t, n_classes=5) ref_p = ref.predict_proba(Q) ref_y = ref.predict(Q, groups=gq, time_index=tq) fast_from_ref = accelerate_checkpoint(ref) for m in (fast, fast_from_ref): p = m.predict_proba(Q) assert_allclose(p, ref_p, atol=2e-10, rtol=1e-9) assert_array_equal(p[:, [2, 4]], 0) assert_array_equal(m.decode_proba(p, groups=gq, time_index=tq), ref_y) assert_array_equal(m.predict(Q, batch_size=3, groups=gq, time_index=tq), ref_y) assert_array_equal(pickle.loads(pickle.dumps(fast_from_ref)).predict(Q), ref.predict(Q)) def check_turbo_and_chronology(family): X, y, Q = windows() g, t = np.arange(len(X)) % 4, np.arange(len(X))[::-1] b = TurboBudget(kernels=4, intervals=2, shapelets=2, trees=4, max_steps=28, max_train=12) with threadpool_limits(1): m = build_family(family, mode="turbo", budget=b).fit(X, y, groups=g, time_index=t, n_classes=5) assert m.fit_windows_ == 12 ref_s = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=5) assert_array_equal(m.smoother_.transition_, ref_s.transition_) p = m.predict_proba(Q) assert_allclose(p.sum(axis=1), 1) assert_array_equal(p[:, [2, 4]], 0) def check_custom_recipes(): X, y, Q = windows() for name, params in [ ("har_sculpt_forest_hmm", dict(classifier="rf", segment_sizes=(3, 6), symbolic_bins=5)), ("rocket_sketch_ridge_hmm", dict(classifier="logreg", n_kernels=11)), ("symbolic_interval_forest", {}), ("hybrid_cpu_ensemble", {}), ("invariant_interval_shape_forest", dict(n_shapelets=3)), ("state_rocket_interval_hmm", dict(n_kernels=9, n_intervals=4)), ("posterior_cpu_ensemble", dict(weights=(0.0, 0.4, 0.6))), ]: spec = dict(name=name, route="cpu", params=dict(seed=33, n_jobs=1, use_hmm=False, **params)) with threadpool_limits(1): ref = build_cpu_estimator(spec).fit(X, y) fast = build_candidate(spec).fit(X, y) assert_allclose(fast.predict_proba(Q), ref.predict_proba(Q), atol=2e-10, rtol=1e-9) def check_transition_and_tie_order(mode): rng = np.random.default_rng(911) g = rng.integers(0, 5, 100) if mode != "none" else None t = rng.integers(0, 10, 100) if mode == "time" else None y = rng.integers(0, 4, 100) a = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=6) b = FastSmoother().fit(y, groups=g, time_index=t, n_classes=6) assert_array_equal(a.prior_, b.prior_) assert_array_equal(a.transition_, b.transition_) for p in (rng.uniform(0, 1, (100, 6)), np.ones((100, 6)), np.zeros((100, 6))): assert_array_equal(a.predict_from_proba(p, groups=g, time_index=t), b.predict_from_proba(p, groups=g, time_index=t)) class EngineTests(unittest.TestCase): def test_full_recipe_and_checkpoint(self): for family in CANONICAL_METHOD_IDS: with self.subTest(family=family): check_full_recipe_and_converted_checkpoint(family) def test_turbo_and_chronology(self): for family in CANONICAL_METHOD_IDS: with self.subTest(family=family): check_turbo_and_chronology(family) def test_custom_recipes(self): check_custom_recipes() def test_controlled_recipes(self): from wisp.ablations import CIS_ABLATIONS, build_recipe as ref_builder from wisp_fast.engine import build_recipe X, y, Q = windows() g, t = np.arange(len(X)) % 3, np.arange(len(X))[::-1] qg, qt = np.arange(len(Q)) % 3, np.arange(len(Q))[::-1] for recipe in CIS_ABLATIONS: with self.subTest(recipe_id=recipe.recipe_id), threadpool_limits(1): ref = ref_builder(recipe.recipe_id, seed=8, n_jobs=1) fast = build_recipe(recipe.recipe_id, seed=8) ref.fit(X, y, groups=g, time_index=t, n_classes=5) fast.fit(X, y, groups=g, time_index=t, n_classes=5) assert (fast.smoother_ is not None) == recipe.use_hmm assert len(fast.blocks_) == sum((recipe.use_rocket, recipe.use_intervals, recipe.use_state)) assert type(ref.model_.named_steps["clf"]) is type(fast.model_.named_steps["clf"]) assert_allclose(ref.predict_proba(Q), fast.predict_proba(Q), atol=2e-10, rtol=1e-9) assert_array_equal(ref.predict(Q, groups=qg, time_index=qt), fast.predict(Q, groups=qg, time_index=qt)) def test_forest_kernel_and_fallback(self): from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier from wisp_fast.forest import local_proba, ForestPlan rng = np.random.default_rng(611) X = rng.normal(size=(110, 9)) y = rng.choice([0, 2, 5], size=len(X)) weight = rng.uniform(0.1, 2, size=len(X)) Q = rng.normal(size=(27, 9)) Q[0, 2] = np.nan for cls in (ExtraTreesClassifier, RandomForestClassifier): with self.subTest(cls=cls): model = cls(n_estimators=11, n_jobs=1, random_state=9, min_samples_leaf=4).fit(X, y, sample_weight=weight) ref = model.predict_proba(Q) plan = ForestPlan(model) assert_array_equal(local_proba(model, Q, plan), ref) plan.table = None assert_array_equal(local_proba(model, Q, plan), ref) def test_decoder(self): for backend in ("default", "numpy"): ctx = patch("wisp_fast.decoder._native", lambda: None) if backend == "numpy" else nullcontext() with ctx: for mode in ("none", "group", "time"): with self.subTest(backend=backend, mode=mode): check_transition_and_tie_order(mode) if __name__ == "__main__": unittest.main()