Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
File size: 11,119 Bytes
4731b11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93a9282
4731b11
 
 
 
 
 
93a9282
 
4731b11
 
 
 
 
 
 
 
 
 
93a9282
 
4731b11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93a9282
4731b11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93a9282
 
4731b11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
"""Optuna integration: WISP recipes as an Optuna search space (optional extra ``[optuna]``).

The space is the existing CPU search grammar (``wisp.search.grammar``):

* family: the seven kernel families;
* readout: ExtraTrees / random forest / logistic regression for SS and GIS, Ridge /
  logistic regression for RC and CIS (the grammar's ``change_classifier``);
* segment sizes (3 of {3,4,5,8,10,12,16,20,25}) and symbolic bins ({4,5,6,8,10}) for SS
  and GIS; convolution kernels ({128,...,768}) for RC and CIS; intervals ({8,...,48}) for
  CIS; shapelets ({16,...,64}) for GIS; ensemble weights for ESE;
* CIS block switches R/I/S (the paper's CIS block subsets; at least one block on);
* decoder: HMM or per-window argmax.

Optuna only proposes. Each candidate is fitted and scored by the existing cascade
(``wisp.search.cascade``) with the existing validation utility, exactly as in
WISP-Random/WISP-Evolution; the test split is never seen.

Use :func:`suggest_recipe` inside your own objective to tune a WISP recipe jointly with
other hyperparameters.
"""
from __future__ import annotations

import itertools
import json
import math
from typing import Any

from ..recipes import Recipe

__all__ = ["suggest_recipe", "recipe_operators", "evaluate_recipe", "run_study", "SEARCH_SPACE"]

_SHORT = {"har_sculpt_forest_hmm": "ss", "rocket_sketch_ridge_hmm": "rc", "symbolic_interval_forest": "so",
          "hybrid_cpu_ensemble": "cse", "invariant_interval_shape_forest": "gis",
          "state_rocket_interval_hmm": "cis", "posterior_cpu_ensemble": "ese"}
_SEGMENT_POOL = (3, 4, 5, 8, 10, 12, 16, 20, 25)
SEARCH_SPACE: dict[str, Any] = {
    "family": tuple(_SHORT),
    "forest_classifier": ("extratrees", "rf", "logreg"),
    "linear_classifier": ("ridge", "logreg"),
    "segment_sizes": tuple(",".join(map(str, c)) for c in itertools.combinations(_SEGMENT_POOL, 3)),
    "symbolic_bins": (4, 5, 6, 8, 10),
    "n_kernels": (128, 256, 384, 512, 768),
    "n_intervals": (8, 16, 24, 32, 48),
    "n_shapelets": (16, 24, 32, 48, 64),
    "use_hmm": (True, False),
}


def suggest_recipe(trial, *, families=None, prefix: str = "wisp") -> Recipe:
    """Sample one WISP recipe from the grammar's space with an ``optuna.trial.Trial``.

    ``families`` restricts the family choice (kernel names, e.g.
    ``["state_rocket_interval_hmm"]``). Parameter names are prefixed with ``prefix`` and
    the family's short name, so the space is conditional on the family.
    """
    import optuna

    choices = tuple(families) if families is not None else SEARCH_SPACE["family"]
    family = trial.suggest_categorical(f"{prefix}.family", choices)
    short = _SHORT[family]
    p: dict[str, Any] = {}
    if family in ("har_sculpt_forest_hmm", "invariant_interval_shape_forest"):
        p["classifier"] = trial.suggest_categorical(f"{prefix}.{short}.classifier", SEARCH_SPACE["forest_classifier"])
        seg = trial.suggest_categorical(f"{prefix}.{short}.segment_sizes", SEARCH_SPACE["segment_sizes"])
        p["segment_sizes"] = tuple(int(v) for v in seg.split(","))
        p["symbolic_bins"] = trial.suggest_categorical(f"{prefix}.{short}.symbolic_bins", SEARCH_SPACE["symbolic_bins"])
        if family == "invariant_interval_shape_forest":
            p["n_shapelets"] = trial.suggest_categorical(f"{prefix}.gis.n_shapelets", SEARCH_SPACE["n_shapelets"])
    elif family in ("rocket_sketch_ridge_hmm", "state_rocket_interval_hmm"):
        p["classifier"] = trial.suggest_categorical(f"{prefix}.{short}.classifier", SEARCH_SPACE["linear_classifier"])
        p["n_kernels"] = trial.suggest_categorical(f"{prefix}.{short}.n_kernels", SEARCH_SPACE["n_kernels"])
        if family == "state_rocket_interval_hmm":
            p["n_intervals"] = trial.suggest_categorical(f"{prefix}.cis.n_intervals", SEARCH_SPACE["n_intervals"])
            for block, key in (("R", "use_rocket"), ("I", "use_intervals"), ("S", "use_state")):
                p[key] = trial.suggest_categorical(f"{prefix}.cis.block_{block}", (True, False))
            if not (p["use_rocket"] or p["use_intervals"] or p["use_state"]):
                raise optuna.TrialPruned("CIS needs at least one representation block")
            if p["use_rocket"] and p["use_intervals"] and p["use_state"]:
                for key in ("use_rocket", "use_intervals", "use_state"):
                    p.pop(key)  # the full CIS: identical to the grammar's template
    elif family == "posterior_cpu_ensemble":
        w = [trial.suggest_float(f"{prefix}.ese.weight{i}", 0.05, 1.0) for i in range(3)]
        p["weights"] = tuple(float(v) / sum(w) for v in w)
    p["use_hmm"] = trial.suggest_categorical(f"{prefix}.{short}.use_hmm", SEARCH_SPACE["use_hmm"])
    return Recipe(family, p)


def recipe_operators(recipe: Recipe) -> list[str]:
    """Operator labels of a recipe (they enter the utility's complexity term).

    Labels of the family's first grammar template, with the readout and decoder labels
    replaced as the grammar's mutations do; CIS block subsets use the labels of the paper's
    CIS subset recipes (``wisp.ablations``).
    """
    from wisp.search.grammar import _FIRST, _TEMPLATES, _replace_classifier_ops

    params = recipe.params
    toggles = [params.get(k, True) for k in ("use_rocket", "use_intervals", "use_state")]
    if recipe.family == "state_rocket_interval_hmm" and not all(toggles):
        ops = ["zscore"]
        if toggles[0]:
            ops += ["random_conv", "ppv_pool"]
        if toggles[1]:
            ops += ["dyadic_intervals", "quantile_distribution"]
        if toggles[2]:
            ops += ["magnitude", "jerk", "spectral", "autocorr", "symbolic_transition"]
        ops.append(params.get("classifier", "ridge"))
    else:
        ops = list(_TEMPLATES[_FIRST[recipe.family]].operators)
        if "classifier" in params:
            ops = _replace_classifier_ops(ops, params["classifier"])
    ops = [o for o in ops if o not in ("hmm", "argmax")]
    return ops + ["hmm" if recipe.use_hmm else "argmax"]


def evaluate_recipe(recipe: Recipe, train, valid, *, seed: int = 42, speed: str = "exact",
                    sequence: str = "subject", n_jobs: int = 1, cost_weight: float = 0.01,
                    robustness_weight: float = 0.1, force_hmm: bool | None = None) -> tuple[str, dict]:
    """Fit on ``train`` and score on ``valid`` with the search cascade; returns (status, result).

    ``train``/``valid`` are ``wisp.core.data.HARData`` with disjoint groups. This calls the
    frozen ``wisp.search.cascade.evaluate_candidate_cascade`` (validation fit, four stress
    views, anti-collapse check) with the speed-tier engine as its estimator factory; the
    utility is ``result["predictive_utility"]`` (the cascade score), as in
    WISP-Random/WISP-Evolution. ``force_hmm`` (True/False) fits the candidate with that HMM
    setting instead of the recipe's own (``None``, the default).
    """
    from wisp.search.cascade import evaluate_candidate_cascade
    from wisp.search.grammar import CandidateSpec
    from .._engine import build_kernel

    spec = recipe.kernel_spec(seed=seed, n_jobs=n_jobs)
    cand = CandidateSpec(name=spec["name"], params=spec["params"], operators=recipe_operators(recipe), route="cpu")

    def factory(spec_dict):
        params = dict(spec_dict["params"])
        use_hmm = bool(params.get("use_hmm", True)) if force_hmm is None else bool(force_hmm)
        model = build_kernel(Recipe(spec_dict["name"], params), use_hmm=use_hmm,
                             speed=speed, sequence=sequence, seed=int(params.get("seed", seed)),
                             n_jobs=int(params.get("n_jobs", n_jobs)))
        if sequence == "recording":
            model.calibration_data = (valid.X, valid.y)  # confidence scaling chosen on validation, as in search
        return model

    try:
        cascade = evaluate_candidate_cascade(cand, train, valid, cost_weight=cost_weight,
                                             robustness_weight=robustness_weight, max_stress_transforms=4,
                                             estimator_factory=factory)
    except Exception as exc:  # same policy as the search controller: failures consume budget
        return "failed", {"predictive_utility": -1e9, "error": repr(exc), "candidate": cand.to_dict()}
    result = cascade.to_dict()
    result["predictive_utility"] = float(result.get("cascade_score", -1e9))
    return ("ok" if cascade.accepted else "rejected"), result


def run_study(train, valid, *, n_trials: int, sampler=None, seed: int = 42, speed: str = "exact",
              sequence: str = "subject", n_jobs: int = 1, cost_weight: float = 0.01,
              robustness_weight: float = 0.1, study=None, families=None, force_hmm: bool | None = None):
    """Optuna study maximising the validation utility. Returns (study, best_spec, best_result, history).

    Repeated proposals of an already evaluated recipe reuse its result (no refit) and still
    count as trials. The winner is the accepted candidate with the highest utility.
    """
    import optuna

    if study is None:
        study = optuna.create_study(direction="maximize",
                                    sampler=sampler if sampler is not None else optuna.samplers.TPESampler(seed=seed))
    cache: dict[str, tuple[str, dict]] = {}
    history: list[dict] = []

    def objective(trial):
        recipe = suggest_recipe(trial, families=families)
        key = recipe.to_json()
        if key not in cache:
            cache[key] = evaluate_recipe(recipe, train, valid, seed=seed, speed=speed, sequence=sequence,
                                         n_jobs=n_jobs, cost_weight=cost_weight, robustness_weight=robustness_weight,
                                         force_hmm=force_hmm)
            repeat = False
        else:
            repeat = True
        status, result = cache[key]
        utility = float(result.get("predictive_utility", -1e9))
        trial.set_user_attr("status", status)
        trial.set_user_attr("recipe", key)
        trial.set_user_attr("repeat", repeat)
        history.append({"trial": trial.number, "recipe": json.loads(key), "status": status, "repeat": repeat,
                        "validation_utility": utility, "validation_macro_f1": result.get("macro_f1"),
                        "validation_worst_class_f1": result.get("worst_class_f1"),
                        "fit_seconds": None if repeat else result.get("fit_seconds"),
                        "predict_seconds": None if repeat else result.get("predict_seconds")})
        if status == "failed" or not math.isfinite(utility):
            return float("nan")
        return utility

    study.optimize(objective, n_trials=int(n_trials), catch=())
    ok = [(json.loads(k), r) for k, (s, r) in cache.items() if s == "ok"]
    if not ok:
        raise RuntimeError("No accepted candidate in the Optuna study")
    best_recipe, best_result = max(ok, key=lambda kv: float(kv[1]["predictive_utility"]))
    best_spec = Recipe.from_dict(best_recipe).kernel_spec(seed=seed, n_jobs=n_jobs)
    return study, best_spec, best_result, history