Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / benchmarks /adaptive_decoder.py
Zipeng365's picture
wisp-har 0.4.0: WISP-Funnel search and the adaptive HMM switch (default decoder)
93a9282 verified
Raw History Blame Contribute Delete
5.16 kB
"""decoder="adaptive" (the 0.4.0 default) on one WristHARBench cell, against the saved paper results.
Decision data are the paper's validation split; the test split is scored once.
* ``--group fixed``: ``WISPClassifier(<family>, decoder="adaptive").fit(train, validation=valid)``
for the seven fixed families.
* ``--group select``: ``WISPSelect(5 | 7, decoder="adaptive")``.
* ``--group search``: the adaptive step of ``WISPSearch`` applied to the recipe each paper search
selected at budget 329 (``results/search_selections.csv``), refitted with the candidate seed of
the retained checkpoint (``--checkpoints``); the searches are not rerun.
The paper (fixed decoder) metrics come from ``results/run_results.csv``; when the switch keeps
the recipe's decoder, the refit equals the saved model.
python benchmarks/adaptive_decoder.py --data-root <fold exports> --results results \\
--checkpoints <models dir> --dataset adl_wrist_accel_v1 --fold 0 --seed 42 --group fixed --out <dir>
"""
from __future__ import annotations
import argparse
import csv
import json
import sys
import time
from pathlib import Path
import numpy as np
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _frozen import load_fold, paper_scores # noqa: E402
METRICS = ("macro_f1", "accuracy", "balanced_accuracy", "worst_class_f1")
FIXED = ("ss", "rc", "so", "gis", "cis", "cse", "ese")
def saved(results, method, d, f, s):
with open(Path(results) / "run_results.csv", newline="") as fh:
for r in csv.DictReader(fh):
if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"])) == (method, d, f, s):
return {m: float(r[m]) for m in METRICS}
return None
def selected(results, method, d, f, s):
with open(Path(results) / "search_selections.csv", newline="") as fh:
for r in csv.DictReader(fh):
if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"]), float(r["budget"])) == \
(method, d, f, s, 329.0):
return json.loads(r["canonical_recipe"])
raise LookupError((method, d, f, s))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--data-root", required=True)
ap.add_argument("--results", default=str(Path(__file__).resolve().parents[1] / "results"))
ap.add_argument("--checkpoints")
ap.add_argument("--dataset", required=True)
ap.add_argument("--fold", type=int, required=True)
ap.add_argument("--seed", type=int, required=True)
ap.add_argument("--group", required=True, choices=["fixed", "select", "search"])
ap.add_argument("--out", required=True)
a = ap.parse_args()
import wisp
from wisp.recipes import Recipe
tr, va, _, te = load_fold(a.data_root, a.dataset, a.fold)
K = tr["n_classes"]
fit_args = (tr["X"], tr["y"])
fit_kw = dict(groups=tr["subject"], time_index=tr["time_index"],
validation=(va["X"], va["y"], va["subject"], va["time_index"]))
jobs = []
if a.group == "fixed":
jobs = [(f"wisp_{s}", lambda s=s: wisp.WISPClassifier(s, classes=np.arange(K), random_state=a.seed)) for s in FIXED]
elif a.group == "select":
jobs = [(f"wisp_select{m}", lambda m=m: wisp.WISPSelect(m, classes=np.arange(K), random_state=a.seed)) for m in (5, 7)]
else:
from wisp_release.checkpoints import load_checkpoint
for method in ("wisp_random", "wisp_evolution"):
spec = selected(a.results, method, a.dataset, a.fold, a.seed)
folder = Path(a.checkpoints) / method / a.dataset / f"fold{a.fold}" / f"seed{a.seed}"
meta = json.loads((folder / "checkpoint.json").read_text())
rstate = int(load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"]).seed)
recipe = Recipe(spec["name"], {k: v for k, v in spec["params"].items() if k not in ("seed", "n_jobs")})
jobs.append((method, lambda recipe=recipe, rstate=rstate: wisp.WISPClassifier(
recipe, classes=np.arange(K), random_state=rstate)))
rows = []
for method, make in jobs:
est = make() # decoder="adaptive" is the default
t0 = time.perf_counter()
est.fit(*fit_args, **fit_kw)
wall = time.perf_counter() - t0
final = getattr(est, "best_estimator_", est)
pred = est.predict(te["X"], groups=te["subject"], time_index=te["time_index"])
rows.append({"method": method, "decoder": final.decoder_, "reason": final.decoder_reason_,
"scores": final.decoder_scores_, "recipe_decoder": "hmm" if final.recipe_.use_hmm else "none",
"decision_seconds": est.decision_seconds_, "wall_seconds": wall,
"metrics": paper_scores(te["y"], pred, K), "saved": saved(a.results, method, a.dataset, a.fold, a.seed)})
out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
(out / f"{a.group}__{a.dataset}__f{a.fold}__s{a.seed}.json").write_text(json.dumps(rows, indent=1, default=str))
print(json.dumps(rows, default=str))
if __name__ == "__main__":
main()