Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
File size: 11,056 Bytes
93a9282
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
"""decoder="adaptive" (the default since 0.4.0): decision rule, edge cases and bit-identity of the final model."""
import warnings

import numpy as np
import pytest
from numpy.testing import assert_array_equal
from sklearn.linear_model import LogisticRegression
from threadpoolctl import threadpool_limits

from wisp import HMMSmoother, WISPClassifier, WISPSearch, WISPSelect


def sequence_fixture(n_subjects=8, per_subject=40, block=10, T=48, C=3, K=3, noise=1.4, seed=0, offset=0):
    """Windows whose labels persist in blocks within each group, with noisy per-window evidence,
    so sequence decoding can help."""
    rng = np.random.default_rng(seed)
    X, y, g, t = [], [], [], []
    tt = np.arange(T)
    for s in range(n_subjects):
        labels = np.repeat(rng.permutation(np.resize(np.arange(K), per_subject // block)), block)[:per_subject]
        for i, k in enumerate(labels):
            w = np.stack([np.sin(2 * np.pi * (k + 1 + c) * tt / T + rng.normal()) for c in range(C)], 1)
            X.append(w + rng.normal(0, noise, (T, C)))
            y.append(k), g.append(s + offset), t.append(i)
    return np.asarray(X), np.asarray(y), np.asarray(g), np.asarray(t, dtype=float)


@pytest.fixture(scope="module")
def seq_data():
    return sequence_fixture(seed=0), sequence_fixture(n_subjects=3, seed=1, offset=100), \
        sequence_fixture(n_subjects=4, seed=2, offset=200)


def _same_outputs(a, b, Xq, gq, tq):
    assert_array_equal(a.predict_proba(Xq), b.predict_proba(Xq))
    assert_array_equal(a.predict(Xq, groups=gq, time_index=tq), b.predict(Xq, groups=gq, time_index=tq))


@pytest.mark.parametrize("recipe", ["rc", "cis", "ss"])
def test_adaptive_final_model_equals_fixed_decoder_model(recipe, seq_data):
    """Whatever the switch chooses, the final model is the fixed-decoder model with that setting."""
    (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
    with threadpool_limits(1):
        for kwargs in ({}, {"validation": (Xv, yv, gv, tv)}):
            ada = WISPClassifier(recipe).fit(X, y, groups=g, time_index=t, **kwargs)
            assert ada.decoder_reason_ == "validation"
            assert set(ada.decoder_scores_) == {"hmm", "none"}
            assert ada.decoder_ == ("hmm" if ada.decoder_scores_["hmm"] > ada.decoder_scores_["none"] else "none")
            fixed = WISPClassifier(recipe, decoder=ada.decoder_).fit(X, y, groups=g, time_index=t, **kwargs)
            _same_outputs(ada, fixed, Xq, gq, tq)
            assert ada.has_paper_results_ == fixed.has_paper_results_


def test_adaptive_keeps_hmm_bit_identical_to_fixed_hmm(seq_data):
    """When the switch keeps the HMM, the final model equals decoder="hmm" (and "recipe") bit for bit."""
    (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
    with threadpool_limits(1):
        ada = WISPClassifier("cis").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
        assert ada.decoder_ == "hmm", ada.decoder_scores_  # blocks of 10 windows: smoothing helps here
        for decoder in ("hmm", "recipe"):
            fixed = WISPClassifier("cis", decoder=decoder).fit(X, y, groups=g, time_index=t,
                                                               validation=(Xv, yv, gv, tv))
            _same_outputs(ada, fixed, Xq, gq, tq)
        assert ada.has_paper_results_ is True and ada.decision_seconds_ > 0
        # validation windows are fitting data: the tuple form fits training then validation windows
        both = WISPClassifier("cis", decoder="hmm").fit(np.concatenate([X, Xv]), np.concatenate([y, yv]),
                                                        groups=np.concatenate([g, gv]),
                                                        time_index=np.concatenate([t, tv]))
        _same_outputs(ada, both, Xq, gq, tq)


def test_adaptive_with_readout(seq_data):
    (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
    ada = WISPClassifier("cis:RS", readout=LogisticRegression(max_iter=300)).fit(
        X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
    assert ada.decoder_reason_ == "validation"
    fixed = WISPClassifier("cis:RS", decoder=ada.decoder_, readout=LogisticRegression(max_iter=300)).fit(
        X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
    _same_outputs(ada, fixed, Xq, gq, tq)


def test_no_sequences_gives_none():
    """Every window its own group: HMM decoding would only reweight by the class prior."""
    X, y, _, _ = sequence_fixture(n_subjects=4, per_subject=30)
    g = np.arange(len(y))
    clf = WISPClassifier("rc").fit(X, y, groups=g)
    assert clf.decoder_ == "none" and clf.decoder_reason_ == "no sequences" and clf.decision_seconds_ == 0
    assert clf.decoder_scores_ is None
    with pytest.raises(ValueError, match="needs groups"):
        WISPClassifier("rc", decoder="hmm").fit(X, y)
    # explicit decoder="hmm" keeps today's behaviour (prior-weighted one-window decoding)
    hmm = WISPClassifier("rc", decoder="hmm").fit(X, y, groups=g)
    P = hmm.predict_proba(X[:20])
    prior = hmm.model_.smoother_.prior_
    assert_array_equal(hmm.predict(X[:20], groups=np.arange(20)), np.argmax(np.log(np.clip(P, 1e-12, 1)) +
                                                                            np.log(np.clip(prior, 1e-12, 1)), 1))


def test_edge_cases_no_groups_and_one_group(seq_data):
    (X, y, g, t), _, _ = seq_data
    with warnings.catch_warnings():
        warnings.simplefilter("error")
        clf = WISPClassifier("rc").fit(X, y)
    assert clf.decoder_ == "none" and clf.decoder_reason_ == "no groups"
    with pytest.warns(UserWarning, match="at least 2 groups"):
        one = WISPClassifier("rc").fit(X, y, groups=np.zeros(len(y), int), time_index=np.arange(len(y)))
    assert one.decoder_reason_ == "fewer than 2 groups" and one.decoder_ == "hmm"  # the recipe's own decoder


def test_auto_alias_is_deprecated(seq_data):
    (X, y, g, t), _, (Xq, _, gq, tq) = seq_data
    with pytest.warns(FutureWarning, match="decoder='recipe'"):
        auto = WISPClassifier("rc", decoder="auto").fit(X, y, groups=g, time_index=t)
    recipe = WISPClassifier("rc", decoder="recipe").fit(X, y, groups=g, time_index=t)
    _same_outputs(auto, recipe, Xq, gq, tq)
    with pytest.raises(ValueError, match="decoder"):
        WISPClassifier("rc", decoder="viterbi").fit(X, y, groups=g)


def test_validation_rows_equal_validation_tuple(seq_data):
    (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
    Xa, ya, ga, ta = (np.concatenate(p) for p in ((X, Xv), (y, yv), (g, gv), (t, tv)))
    rows = np.arange(len(X), len(Xa))
    a = WISPClassifier("rc").fit(Xa, ya, groups=ga, time_index=ta, validation=rows)
    b = WISPClassifier("rc").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
    assert a.decoder_ == b.decoder_ and a.decoder_scores_ == b.decoder_scores_
    _same_outputs(a, b, Xq, gq, tq)  # same rows in the same order


def test_select_adaptive(seq_data):
    (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
    with threadpool_limits(1):
        rec = WISPSelect(5, decoder="recipe").fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
        ada = WISPSelect(5).fit(X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
    assert ada.best_recipe_.id == rec.best_recipe_.id  # the selection itself is the paper's
    assert [h["macro_f1"] for h in ada.history_] == [h["macro_f1"] for h in rec.history_]
    assert ada.decoder_reason_ == "validation"
    if ada.decoder_ == "hmm":
        _same_outputs(ada, rec, Xq, gq, tq)
        assert ada.has_paper_results_
    else:
        fixed = WISPClassifier(ada.best_recipe_, decoder="none").fit(X, y, groups=g, time_index=t,
                                                                     validation=(Xv, yv, gv, tv))
        _same_outputs(ada, fixed, Xq, gq, tq)
        assert not ada.has_paper_results_


def _singletons():
    X, y, _, _ = sequence_fixture(n_subjects=4, per_subject=30, seed=3)
    Xv, yv, _, _ = sequence_fixture(n_subjects=2, per_subject=30, seed=4)
    return X, y, np.arange(len(y)), Xv, yv, np.arange(len(yv)) + 10_000


def test_select_scores_candidates_direct_without_sequences():
    from wisp_release.scoring import score
    X, y, g, Xv, yv, gv = _singletons()
    with threadpool_limits(1):
        ada = WISPSelect(5).fit(X, y, groups=g, validation=(Xv, yv, gv))
        none = WISPSelect(5, decoder="none").fit(X, y, groups=g, validation=(Xv, yv, gv))
    assert ada.decoder_ == "none" and ada.decoder_reason_ == "no sequences"
    assert [h["macro_f1"] for h in ada.history_] == [h["macro_f1"] for h in none.history_]
    for h in ada.history_:
        clf = WISPClassifier(h["recipe"], decoder="none", classes=[0, 1, 2]).fit(X, y)
        assert h["macro_f1"] == score(yv, clf.predict(Xv), 3)["macro_f1"]
    assert_array_equal(ada.predict(Xv, groups=gv), none.predict(Xv, groups=gv))


@pytest.mark.parametrize("strategy", ["random", "funnel"])
def test_search_scores_candidates_direct_without_sequences(strategy):
    X, y, g, Xv, yv, gv = _singletons()
    kw = dict(budget=8, population=4)
    if strategy == "random":
        kw["cost_weight"] = 0.0  # the utility's time term would differ between two runs
    ada = WISPSearch(strategy, **kw).fit(X, y, groups=g, validation=(Xv, yv, gv))
    none = WISPSearch(strategy, decoder="none", **kw).fit(X, y, groups=g, validation=(Xv, yv, gv))
    assert ada.decoder_ == "none" and ada.decoder_reason_ == "no sequences"
    assert [h["validation_macro_f1"] for h in ada.history_] == [h["validation_macro_f1"] for h in none.history_]
    assert [h["validation_utility"] for h in ada.history_] == [h["validation_utility"] for h in none.history_]
    assert ada.best_validation_["predictive_utility"] == none.best_validation_["predictive_utility"]
    assert_array_equal(ada.predict_proba(Xv), none.predict_proba(Xv))


def test_hmm_smoother_adaptive(seq_data):
    from wisp.cpu.hmm import TransitionSmoother
    (X, y, g, t), (Xv, yv, gv, tv), (Xq, _, gq, tq) = seq_data
    clf = WISPClassifier("rc", decoder="none").fit(X, y)
    Pv, Pq = clf.predict_proba(Xv), clf.predict_proba(Xq)
    plain = HMMSmoother().fit(y, groups=g, time_index=t)       # no validation scores: decodes as before
    assert plain.decoder_ == "hmm" and plain.decoder_reason_ == "no validation scores"
    kernel = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=3)
    assert_array_equal(plain.decode(Pq, groups=gq, time_index=tq), kernel.predict_from_proba(Pq, groups=gq, time_index=tq))
    ada = HMMSmoother().fit(y, groups=g, time_index=t, validation=(Pv, yv, gv, tv))
    assert ada.decoder_reason_ == "validation"
    if ada.decoder_ == "none":
        assert_array_equal(ada.decode(Pq, groups=gq, time_index=tq), np.argmax(Pq, 1))
    lone = HMMSmoother().fit(y, groups=np.arange(len(y)), validation=(Pv, yv, np.arange(len(yv)) + 10_000))
    assert lone.decoder_ == "none" and lone.decoder_reason_ == "no sequences"
    assert_array_equal(lone.decode(Pq, groups=gq), np.argmax(Pq, 1))