"""decoder="adaptive" (the 0.4.0 default) on one WristHARBench cell, against the saved paper results. Decision data are the paper's validation split; the test split is scored once. * ``--group fixed``: ``WISPClassifier(, decoder="adaptive").fit(train, validation=valid)`` for the seven fixed families. * ``--group select``: ``WISPSelect(5 | 7, decoder="adaptive")``. * ``--group search``: the adaptive step of ``WISPSearch`` applied to the recipe each paper search selected at budget 329 (``results/search_selections.csv``), refitted with the candidate seed of the retained checkpoint (``--checkpoints``); the searches are not rerun. The paper (fixed decoder) metrics come from ``results/run_results.csv``; when the switch keeps the recipe's decoder, the refit equals the saved model. python benchmarks/adaptive_decoder.py --data-root --results results \\ --checkpoints --dataset adl_wrist_accel_v1 --fold 0 --seed 42 --group fixed --out """ from __future__ import annotations import argparse import csv import json import sys import time from pathlib import Path import numpy as np sys.path.insert(0, str(Path(__file__).resolve().parent)) from _frozen import load_fold, paper_scores # noqa: E402 METRICS = ("macro_f1", "accuracy", "balanced_accuracy", "worst_class_f1") FIXED = ("ss", "rc", "so", "gis", "cis", "cse", "ese") def saved(results, method, d, f, s): with open(Path(results) / "run_results.csv", newline="") as fh: for r in csv.DictReader(fh): if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"])) == (method, d, f, s): return {m: float(r[m]) for m in METRICS} return None def selected(results, method, d, f, s): with open(Path(results) / "search_selections.csv", newline="") as fh: for r in csv.DictReader(fh): if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"]), float(r["budget"])) == \ (method, d, f, s, 329.0): return json.loads(r["canonical_recipe"]) raise LookupError((method, d, f, s)) def main(): ap = argparse.ArgumentParser() ap.add_argument("--data-root", required=True) ap.add_argument("--results", default=str(Path(__file__).resolve().parents[1] / "results")) ap.add_argument("--checkpoints") ap.add_argument("--dataset", required=True) ap.add_argument("--fold", type=int, required=True) ap.add_argument("--seed", type=int, required=True) ap.add_argument("--group", required=True, choices=["fixed", "select", "search"]) ap.add_argument("--out", required=True) a = ap.parse_args() import wisp from wisp.recipes import Recipe tr, va, _, te = load_fold(a.data_root, a.dataset, a.fold) K = tr["n_classes"] fit_args = (tr["X"], tr["y"]) fit_kw = dict(groups=tr["subject"], time_index=tr["time_index"], validation=(va["X"], va["y"], va["subject"], va["time_index"])) jobs = [] if a.group == "fixed": jobs = [(f"wisp_{s}", lambda s=s: wisp.WISPClassifier(s, classes=np.arange(K), random_state=a.seed)) for s in FIXED] elif a.group == "select": jobs = [(f"wisp_select{m}", lambda m=m: wisp.WISPSelect(m, classes=np.arange(K), random_state=a.seed)) for m in (5, 7)] else: from wisp_release.checkpoints import load_checkpoint for method in ("wisp_random", "wisp_evolution"): spec = selected(a.results, method, a.dataset, a.fold, a.seed) folder = Path(a.checkpoints) / method / a.dataset / f"fold{a.fold}" / f"seed{a.seed}" meta = json.loads((folder / "checkpoint.json").read_text()) rstate = int(load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"]).seed) recipe = Recipe(spec["name"], {k: v for k, v in spec["params"].items() if k not in ("seed", "n_jobs")}) jobs.append((method, lambda recipe=recipe, rstate=rstate: wisp.WISPClassifier( recipe, classes=np.arange(K), random_state=rstate))) rows = [] for method, make in jobs: est = make() # decoder="adaptive" is the default t0 = time.perf_counter() est.fit(*fit_args, **fit_kw) wall = time.perf_counter() - t0 final = getattr(est, "best_estimator_", est) pred = est.predict(te["X"], groups=te["subject"], time_index=te["time_index"]) rows.append({"method": method, "decoder": final.decoder_, "reason": final.decoder_reason_, "scores": final.decoder_scores_, "recipe_decoder": "hmm" if final.recipe_.use_hmm else "none", "decision_seconds": est.decision_seconds_, "wall_seconds": wall, "metrics": paper_scores(te["y"], pred, K), "saved": saved(a.results, method, a.dataset, a.fold, a.seed)}) out = Path(a.out); out.mkdir(parents=True, exist_ok=True) (out / f"{a.group}__{a.dataset}__f{a.fold}__s{a.seed}.json").write_text(json.dumps(rows, indent=1, default=str)) print(json.dumps(rows, default=str)) if __name__ == "__main__": main()