"""decoder="adaptive" (the default since 0.4.0): decision rule, edge cases and bit-identity of the final model.""" import warnings import numpy as np import pytest from numpy.testing import assert_array_equal from sklearn.linear_model import LogisticRegression from threadpoolctl import threadpool_limits from wisp import HMMSmoother, WISPClassifier, WISPSearch, WISPSelect def sequence_fixture(n_subjects=8, per_subject=40, block=10, T=48, C=3, K=3, noise=1.4, seed=0, offset=0): """Windows whose labels persist in blocks within each group, with noisy per-window evidence, so sequence decoding can help.""" rng = np.random.default_rng(seed) X, y, g, t = [], [], [], [] tt = np.arange(T) for s in range(n_subjects): labels = np.repeat(rng.permutation(np.resize(np.arange(K), per_subject // block)), block)[:per_subject] for i, k in enumerate(labels): w = np.stack([np.sin(2 * np.pi * (k + 1 + c) * tt / T + rng.normal()) for c in range(C)], 1) X.append(w + rng.normal(0, noise, (T, C))) y.append(k), g.append(s + offset), t.append(i) return np.asarray(X), np.asarray(y), np.asarray(g), np.asarray(t, dtype=float) @pytest.fixture(scope="module") def seq_data(): return sequence_fixture(seed=0), sequence_fixture(n_subjects=3, seed=1, offset=100), \ sequence_fixture(n_subjects=4, seed=2, offset=200) def _same_outputs(a, b, Xq, gq, tq): assert_array_equal(a.predict_proba(Xq), b.predict_proba(Xq)) assert_array_equal(a.predict(Xq, groups=gq, time_index=tq), b.predict(Xq, groups=gq, time_index=tq)) @pytest.mark.parametrize("recipe", ["rc", "cis", "ss"]) def test_adaptive_final_model_equals_fixed_decoder_model(recipe, seq_data): """Whatever the switch chooses, the final model is the fixed-decoder model with that setting.""" (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data with threadpool_limits(1): for kwargs in ({}, {"validation": (Xv, yv, gv, tv)}): ada = WISPClassifier(recipe).fit(X, y, groups=g, time_index=t, **kwargs) assert ada.decoder_reason_ == "validation" assert set(ada.decoder_scores_) == {"hmm", "none"} assert ada.decoder_ == ("hmm" if ada.decoder_scores_["hmm"] > ada.decoder_scores_["none"] else "none") fixed = WISPClassifier(recipe, decoder=ada.decoder_).fit(X, y, groups=g, time_index=t, **kwargs) _same_outputs(ada, fixed, Xq, gq, tq) assert ada.has_paper_results_ == fixed.has_paper_results_ def test_adaptive_keeps_hmm_bit_identical_to_fixed_hmm(seq_data): """When the switch keeps the HMM, the final model equals decoder="hmm" (and "recipe") bit for bit.""" (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data with threadpool_limits(1): ada = WISPClassifier("cis").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv)) assert ada.decoder_ == "hmm", ada.decoder_scores_ # blocks of 10 windows: smoothing helps here for decoder in ("hmm", "recipe"): fixed = WISPClassifier("cis", decoder=decoder).fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv)) _same_outputs(ada, fixed, Xq, gq, tq) assert ada.has_paper_results_ is True and ada.decision_seconds_ > 0 # validation windows are fitting data: the tuple form fits training then validation windows both = WISPClassifier("cis", decoder="hmm").fit(np.concatenate([X, Xv]), np.concatenate([y, yv]), groups=np.concatenate([g, gv]), time_index=np.concatenate([t, tv])) _same_outputs(ada, both, Xq, gq, tq) def test_adaptive_with_readout(seq_data): (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data ada = WISPClassifier("cis:RS", readout=LogisticRegression(max_iter=300)).fit( X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv)) assert ada.decoder_reason_ == "validation" fixed = WISPClassifier("cis:RS", decoder=ada.decoder_, readout=LogisticRegression(max_iter=300)).fit( X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv)) _same_outputs(ada, fixed, Xq, gq, tq) def test_no_sequences_gives_none(): """Every window its own group: HMM decoding would only reweight by the class prior.""" X, y, _, _ = sequence_fixture(n_subjects=4, per_subject=30) g = np.arange(len(y)) clf = WISPClassifier("rc").fit(X, y, groups=g) assert clf.decoder_ == "none" and clf.decoder_reason_ == "no sequences" and clf.decision_seconds_ == 0 assert clf.decoder_scores_ is None with pytest.raises(ValueError, match="needs groups"): WISPClassifier("rc", decoder="hmm").fit(X, y) # explicit decoder="hmm" keeps today's behaviour (prior-weighted one-window decoding) hmm = WISPClassifier("rc", decoder="hmm").fit(X, y, groups=g) P = hmm.predict_proba(X[:20]) prior = hmm.model_.smoother_.prior_ assert_array_equal(hmm.predict(X[:20], groups=np.arange(20)), np.argmax(np.log(np.clip(P, 1e-12, 1)) + np.log(np.clip(prior, 1e-12, 1)), 1)) def test_edge_cases_no_groups_and_one_group(seq_data): (X, y, g, t), _, _ = seq_data with warnings.catch_warnings(): warnings.simplefilter("error") clf = WISPClassifier("rc").fit(X, y) assert clf.decoder_ == "none" and clf.decoder_reason_ == "no groups" with pytest.warns(UserWarning, match="at least 2 groups"): one = WISPClassifier("rc").fit(X, y, groups=np.zeros(len(y), int), time_index=np.arange(len(y))) assert one.decoder_reason_ == "fewer than 2 groups" and one.decoder_ == "hmm" # the recipe's own decoder def test_auto_alias_is_deprecated(seq_data): (X, y, g, t), _, (Xq, _, gq, tq) = seq_data with pytest.warns(FutureWarning, match="decoder='recipe'"): auto = WISPClassifier("rc", decoder="auto").fit(X, y, groups=g, time_index=t) recipe = WISPClassifier("rc", decoder="recipe").fit(X, y, groups=g, time_index=t) _same_outputs(auto, recipe, Xq, gq, tq) with pytest.raises(ValueError, match="decoder"): WISPClassifier("rc", decoder="viterbi").fit(X, y, groups=g) def test_validation_rows_equal_validation_tuple(seq_data): (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data Xa, ya, ga, ta = (np.concatenate(p) for p in ((X, Xv), (y, yv), (g, gv), (t, tv))) rows = np.arange(len(X), len(Xa)) a = WISPClassifier("rc").fit(Xa, ya, groups=ga, time_index=ta, validation=rows) b = WISPClassifier("rc").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv)) assert a.decoder_ == b.decoder_ and a.decoder_scores_ == b.decoder_scores_ _same_outputs(a, b, Xq, gq, tq) # same rows in the same order def test_select_adaptive(seq_data): (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data with threadpool_limits(1): rec = WISPSelect(5, decoder="recipe").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv)) ada = WISPSelect(5).fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv)) assert ada.best_recipe_.id == rec.best_recipe_.id # the selection itself is the paper's assert [h["macro_f1"] for h in ada.history_] == [h["macro_f1"] for h in rec.history_] assert ada.decoder_reason_ == "validation" if ada.decoder_ == "hmm": _same_outputs(ada, rec, Xq, gq, tq) assert ada.has_paper_results_ else: fixed = WISPClassifier(ada.best_recipe_, decoder="none").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv)) _same_outputs(ada, fixed, Xq, gq, tq) assert not ada.has_paper_results_ def _singletons(): X, y, _, _ = sequence_fixture(n_subjects=4, per_subject=30, seed=3) Xv, yv, _, _ = sequence_fixture(n_subjects=2, per_subject=30, seed=4) return X, y, np.arange(len(y)), Xv, yv, np.arange(len(yv)) + 10_000 def test_select_scores_candidates_direct_without_sequences(): from wisp_release.scoring import score X, y, g, Xv, yv, gv = _singletons() with threadpool_limits(1): ada = WISPSelect(5).fit(X, y, groups=g, validation=(Xv, yv, gv)) none = WISPSelect(5, decoder="none").fit(X, y, groups=g, validation=(Xv, yv, gv)) assert ada.decoder_ == "none" and ada.decoder_reason_ == "no sequences" assert [h["macro_f1"] for h in ada.history_] == [h["macro_f1"] for h in none.history_] for h in ada.history_: clf = WISPClassifier(h["recipe"], decoder="none", classes=[0, 1, 2]).fit(X, y) assert h["macro_f1"] == score(yv, clf.predict(Xv), 3)["macro_f1"] assert_array_equal(ada.predict(Xv, groups=gv), none.predict(Xv, groups=gv)) @pytest.mark.parametrize("strategy", ["random", "funnel"]) def test_search_scores_candidates_direct_without_sequences(strategy): X, y, g, Xv, yv, gv = _singletons() kw = dict(budget=8, population=4) if strategy == "random": kw["cost_weight"] = 0.0 # the utility's time term would differ between two runs ada = WISPSearch(strategy, **kw).fit(X, y, groups=g, validation=(Xv, yv, gv)) none = WISPSearch(strategy, decoder="none", **kw).fit(X, y, groups=g, validation=(Xv, yv, gv)) assert ada.decoder_ == "none" and ada.decoder_reason_ == "no sequences" assert [h["validation_macro_f1"] for h in ada.history_] == [h["validation_macro_f1"] for h in none.history_] assert [h["validation_utility"] for h in ada.history_] == [h["validation_utility"] for h in none.history_] assert ada.best_validation_["predictive_utility"] == none.best_validation_["predictive_utility"] assert_array_equal(ada.predict_proba(Xv), none.predict_proba(Xv)) def test_hmm_smoother_adaptive(seq_data): from wisp.cpu.hmm import TransitionSmoother (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data clf = WISPClassifier("rc", decoder="none").fit(X, y) Pv, Pq = clf.predict_proba(Xv), clf.predict_proba(Xq) plain = HMMSmoother().fit(y, groups=g, time_index=t) # no validation scores: decodes as before assert plain.decoder_ == "hmm" and plain.decoder_reason_ == "no validation scores" kernel = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=3) assert_array_equal(plain.decode(Pq, groups=gq, time_index=tq), kernel.predict_from_proba(Pq, groups=gq, time_index=tq)) ada = HMMSmoother().fit(y, groups=g, time_index=t, validation=(Pv, yv, gv, tv)) assert ada.decoder_reason_ == "validation" if ada.decoder_ == "none": assert_array_equal(ada.decode(Pq, groups=gq, time_index=tq), np.argmax(Pq, 1)) lone = HMMSmoother().fit(y, groups=np.arange(len(y)), validation=(Pv, yv, np.arange(len(yv)) + 10_000)) assert lone.decoder_ == "none" and lone.decoder_reason_ == "no sequences" assert_array_equal(lone.decode(Pq, groups=gq), np.argmax(Pq, 1))