Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / scripts /build_search_best.py
Zipeng365's picture
wisp-har 0.3.0: scikit-learn interface, recipe catalog, Optuna search, cost-accuracy figures
4731b11 verified
Raw History Blame Contribute Delete
5.38 kB
"""Generate src/wisp/_data/search_best.json from results/search_selections.csv.
Rule (validation only; no test metric is read): for each data stream take the 30 final
recipes of the budget-329 searches (5 folds x 3 seeds x WISP-Random/WISP-Evolution).
Each final recipe is the candidate with the highest validation utility in its search.
The stream's best recipe is the recipe selected most often; ties are broken by the
number of folds it covers, then by the canonical JSON text (deterministic, never needed
for the released files). Only ``canonical_recipe``, ``dataset_id``, ``fold``,
``model_seed``, ``method_id`` and ``budget`` are read.
Usage: python scripts/build_search_best.py [--check]
"""
from __future__ import annotations
import argparse
import csv
import json
from collections import defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SOURCE = ROOT / "results" / "search_selections.csv"
TARGET = ROOT / "src" / "wisp" / "_data" / "search_best.json"
ALIASES = ROOT / "src" / "wisp" / "_data" / "dataset_aliases.json"
FINAL_BUDGET = 329
# Canonical forms of catalog recipes, for the "same as" column.
FIXED_CANONICAL = {
"ss": {"name": "har_sculpt_forest_hmm", "params": {"classifier": "extratrees", "segment_sizes": [4, 8, 16], "symbolic_bins": 6, "use_hmm": True}},
"rc": {"name": "rocket_sketch_ridge_hmm", "params": {"classifier": "ridge", "n_kernels": 384, "use_hmm": True}},
"so": {"name": "symbolic_interval_forest", "params": {"use_hmm": True}},
"cse": {"name": "hybrid_cpu_ensemble", "params": {"use_hmm": True}},
"gis": {"name": "invariant_interval_shape_forest", "params": {"classifier": "extratrees", "n_shapelets": 32, "segment_sizes": [4, 8, 16], "symbolic_bins": 6, "use_hmm": True}},
"cis": {"name": "state_rocket_interval_hmm", "params": {"classifier": "ridge", "n_intervals": 24, "n_kernels": 384, "use_hmm": True}},
"ese": {"name": "posterior_cpu_ensemble", "params": {"use_hmm": True, "weights": [0.34, 0.33, 0.33]}},
}
DISPLAY = {"adl": "ADL", "capture24": "CAPTURE-24", "domino": "DOMINO", "gotov": "GOTOV", "handy": "HANDY",
"harmes_left": "HARMES left wrist", "harmes_right": "HARMES right wrist", "iuwds": "IUWDS",
"paal_adl": "PAAL-ADL", "wisdm": "WISDM", "mhealth": "mHealth", "pamap2": "PAMAP2"}
def _same_as(recipe):
for rid, canon in FIXED_CANONICAL.items():
if recipe["name"] == canon["name"] and recipe["params"] == canon["params"]:
return rid
flipped = dict(canon["params"], use_hmm=False)
if recipe["name"] == canon["name"] and recipe["params"] == flipped:
return f"{rid} with decoder='none'"
return None
def build():
aliases = json.loads(ALIASES.read_text())
short = {}
for name, full in aliases.items():
short.setdefault(full, name) # first alias listed is the primary short name
runs = defaultdict(list)
with SOURCE.open(newline="") as f:
for row in csv.DictReader(f):
if int(float(row["budget"])) != FINAL_BUDGET:
continue
runs[row["dataset_id"]].append(row)
out = []
for dataset_id in sorted(runs, key=lambda d: list(short).index(d)):
rows = runs[dataset_id]
if len(rows) != 30:
raise ValueError(f"{dataset_id}: expected 30 final searches, found {len(rows)}")
groups = defaultdict(list)
for row in rows:
groups[row["canonical_recipe"]].append(row)
ranked = sorted(groups.items(),
key=lambda kv: (-len(kv[1]), -len({r["fold"] for r in kv[1]}), kv[0]))
text, chosen = ranked[0]
runner_up = ranked[1][1] if len(ranked) > 1 else []
recipe = json.loads(text)
name = short[dataset_id]
out.append({
"id": f"best:{name}",
"family": recipe["name"],
"params": recipe["params"],
"paper_name": f"WISP search best recipe ({DISPLAY[name]})",
"source": "search_best",
"provenance": {
"dataset_id": dataset_id,
"selected": len(chosen),
"searches": len(rows),
"folds_covered": len({r["fold"] for r in chosen}),
"folds_total": len({r["fold"] for r in rows}),
"cells": sorted([f"{r['method_id']}/fold{r['fold']}/seed{r['model_seed']}" for r in chosen]),
"runner_up_selected": len(runner_up),
"same_as": _same_as(recipe),
"selection_rule": ("most frequently selected final recipe among the 30 budget-329 searches "
"(validation utility only); ties by folds covered"),
"source_file": "results/search_selections.csv",
},
})
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--check", action="store_true", help="fail if the committed file differs")
a = ap.parse_args()
text = json.dumps(build(), indent=1, sort_keys=False) + "\n"
if a.check:
if TARGET.read_text() != text:
raise SystemExit("search_best.json is out of date; rerun scripts/build_search_best.py")
print("search_best.json is up to date")
return
TARGET.write_text(text)
print(f"wrote {TARGET.relative_to(ROOT)}")
if __name__ == "__main__":
main()