Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
File size: 5,157 Bytes
93a9282
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
"""decoder="adaptive" (the 0.4.0 default) on one WristHARBench cell, against the saved paper results.

Decision data are the paper's validation split; the test split is scored once.

* ``--group fixed``: ``WISPClassifier(<family>, decoder="adaptive").fit(train, validation=valid)``
  for the seven fixed families.
* ``--group select``: ``WISPSelect(5 | 7, decoder="adaptive")``.
* ``--group search``: the adaptive step of ``WISPSearch`` applied to the recipe each paper search
  selected at budget 329 (``results/search_selections.csv``), refitted with the candidate seed of
  the retained checkpoint (``--checkpoints``); the searches are not rerun.

The paper (fixed decoder) metrics come from ``results/run_results.csv``; when the switch keeps
the recipe's decoder, the refit equals the saved model.

  python benchmarks/adaptive_decoder.py --data-root <fold exports> --results results \\
      --checkpoints <models dir> --dataset adl_wrist_accel_v1 --fold 0 --seed 42 --group fixed --out <dir>
"""
from __future__ import annotations

import argparse
import csv
import json
import sys
import time
from pathlib import Path

import numpy as np

sys.path.insert(0, str(Path(__file__).resolve().parent))
from _frozen import load_fold, paper_scores  # noqa: E402

METRICS = ("macro_f1", "accuracy", "balanced_accuracy", "worst_class_f1")
FIXED = ("ss", "rc", "so", "gis", "cis", "cse", "ese")


def saved(results, method, d, f, s):
    with open(Path(results) / "run_results.csv", newline="") as fh:
        for r in csv.DictReader(fh):
            if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"])) == (method, d, f, s):
                return {m: float(r[m]) for m in METRICS}
    return None


def selected(results, method, d, f, s):
    with open(Path(results) / "search_selections.csv", newline="") as fh:
        for r in csv.DictReader(fh):
            if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"]), float(r["budget"])) == \
                    (method, d, f, s, 329.0):
                return json.loads(r["canonical_recipe"])
    raise LookupError((method, d, f, s))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--data-root", required=True)
    ap.add_argument("--results", default=str(Path(__file__).resolve().parents[1] / "results"))
    ap.add_argument("--checkpoints")
    ap.add_argument("--dataset", required=True)
    ap.add_argument("--fold", type=int, required=True)
    ap.add_argument("--seed", type=int, required=True)
    ap.add_argument("--group", required=True, choices=["fixed", "select", "search"])
    ap.add_argument("--out", required=True)
    a = ap.parse_args()
    import wisp
    from wisp.recipes import Recipe

    tr, va, _, te = load_fold(a.data_root, a.dataset, a.fold)
    K = tr["n_classes"]
    fit_args = (tr["X"], tr["y"])
    fit_kw = dict(groups=tr["subject"], time_index=tr["time_index"],
                  validation=(va["X"], va["y"], va["subject"], va["time_index"]))
    jobs = []
    if a.group == "fixed":
        jobs = [(f"wisp_{s}", lambda s=s: wisp.WISPClassifier(s, classes=np.arange(K), random_state=a.seed)) for s in FIXED]
    elif a.group == "select":
        jobs = [(f"wisp_select{m}", lambda m=m: wisp.WISPSelect(m, classes=np.arange(K), random_state=a.seed)) for m in (5, 7)]
    else:
        from wisp_release.checkpoints import load_checkpoint
        for method in ("wisp_random", "wisp_evolution"):
            spec = selected(a.results, method, a.dataset, a.fold, a.seed)
            folder = Path(a.checkpoints) / method / a.dataset / f"fold{a.fold}" / f"seed{a.seed}"
            meta = json.loads((folder / "checkpoint.json").read_text())
            rstate = int(load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"]).seed)
            recipe = Recipe(spec["name"], {k: v for k, v in spec["params"].items() if k not in ("seed", "n_jobs")})
            jobs.append((method, lambda recipe=recipe, rstate=rstate: wisp.WISPClassifier(
                recipe, classes=np.arange(K), random_state=rstate)))
    rows = []
    for method, make in jobs:
        est = make()                                   # decoder="adaptive" is the default
        t0 = time.perf_counter()
        est.fit(*fit_args, **fit_kw)
        wall = time.perf_counter() - t0
        final = getattr(est, "best_estimator_", est)
        pred = est.predict(te["X"], groups=te["subject"], time_index=te["time_index"])
        rows.append({"method": method, "decoder": final.decoder_, "reason": final.decoder_reason_,
                     "scores": final.decoder_scores_, "recipe_decoder": "hmm" if final.recipe_.use_hmm else "none",
                     "decision_seconds": est.decision_seconds_, "wall_seconds": wall,
                     "metrics": paper_scores(te["y"], pred, K), "saved": saved(a.results, method, a.dataset, a.fold, a.seed)})
    out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
    (out / f"{a.group}__{a.dataset}__f{a.fold}__s{a.seed}.json").write_text(json.dumps(rows, indent=1, default=str))
    print(json.dumps(rows, default=str))


if __name__ == "__main__":
    main()