Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / benchmarks /memory_final_models.py
Zipeng365's picture
wisp-har 0.4.0: WISP-Funnel search and the adaptive HMM switch (default decoder)
93a9282 verified
Raw History Blame Contribute Delete
5.21 kB
"""Peak host memory of one WISP final model: train+validation fit plus test prediction, one thread.
For WISP-Random and WISP-Evolution the final model of a search is the recipe it selected at
budget 329 (``results/search_selections.csv``), refitted with the candidate seed read from the
retained checkpoint of that cell, so the refit is the saved model. WISP-CIS is the fixed recipe.
The four metrics are compared with ``results/run_results.csv`` to confirm the refit is the
saved model. Peak memory is the process's ``ru_maxrss`` (whole process: imports, data, fit,
prediction), the same measure as the paper's efficiency benchmark. Run one process per cell.
python benchmarks/memory_final_models.py --data-root <fold exports> --results <results dir> \
--checkpoints <dir with wisp_random/ and wisp_evolution/> --method wisp_random \
--dataset adl_wrist_accel_v1 --out <dir>
"""
from __future__ import annotations
import argparse
import csv
import json
import platform
import resource
import sys
import time
from pathlib import Path
import numpy as np
from threadpoolctl import threadpool_limits
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _frozen import load_fold # noqa: E402
METRICS = ("macro_f1", "accuracy", "balanced_accuracy", "worst_class_f1")
def selected_recipe(results, method, dataset, fold, seed):
with open(Path(results) / "search_selections.csv", newline="") as f:
for r in csv.DictReader(f):
if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"]), float(r["budget"])) == \
(method, dataset, fold, seed, 329.0):
return json.loads(r["canonical_recipe"]), r["candidate_id"]
raise LookupError((method, dataset, fold, seed))
def saved_metrics(results, method, dataset, fold, seed):
with open(Path(results) / "run_results.csv", newline="") as f:
for r in csv.DictReader(f):
if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"])) == (method, dataset, fold, seed):
return {m: float(r[m]) for m in METRICS}
return None
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--data-root", required=True)
ap.add_argument("--results", required=True)
ap.add_argument("--checkpoints")
ap.add_argument("--method", required=True, choices=["wisp_random", "wisp_evolution", "wisp_cis"])
ap.add_argument("--dataset", required=True)
ap.add_argument("--fold", type=int, default=0)
ap.add_argument("--seed", type=int, default=42)
ap.add_argument("--out", required=True)
ap.add_argument("--blas-threads", type=int, default=1,
help="BLAS threads (default 1; a retry with more threads can change LAPACK's operation order)")
a = ap.parse_args()
from wisp import WISPClassifier
from wisp.metrics import score
if a.method == "wisp_cis":
recipe, candidate, rstate = "cis", None, a.seed
else:
from wisp_release.checkpoints import load_checkpoint
recipe, candidate = selected_recipe(a.results, a.method, a.dataset, a.fold, a.seed)
folder = Path(a.checkpoints) / a.method / a.dataset / f"fold{a.fold}" / f"seed{a.seed}"
meta = json.loads((folder / "checkpoint.json").read_text())
rstate = int(load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"]).seed)
del meta
_, _, trval, test = load_fold(a.data_root, a.dataset, a.fold)
K = trval["n_classes"]
with threadpool_limits(a.blas_threads):
clf = WISPClassifier(recipe, decoder="recipe", classes=np.arange(K), random_state=rstate, n_jobs=1)
t0 = time.perf_counter()
clf.fit(trval["X"], trval["y"], groups=trval["subject"], time_index=trval["time_index"])
fit_s = time.perf_counter() - t0
t1 = time.perf_counter()
pred = clf.predict(test["X"], groups=test["subject"], time_index=test["time_index"])
predict_s = time.perf_counter() - t1
peak_gib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 ** 2 # Linux: KiB
got = {m: float(v) for m, v in score(test["y"], np.asarray(pred, dtype=np.int64), K).items() if m in METRICS}
saved = saved_metrics(a.results, a.method, a.dataset, a.fold, a.seed)
rec = {"method": a.method, "dataset": a.dataset, "fold": a.fold, "seed": a.seed,
"recipe": recipe, "candidate_id": candidate, "random_state": rstate,
"peak_rss_gib": peak_gib, "fit_seconds": fit_s, "predict_seconds": predict_s,
"metrics": got, "saved_metrics": saved,
"max_abs_metric_diff": None if saved is None else max(abs(got[m] - saved[m]) for m in METRICS),
"blas_threads": a.blas_threads, "n_trval": int(len(trval["y"])), "n_test": int(len(test["y"])),
"cpu_model": next((l.split(":", 1)[1].strip() for l in open("/proc/cpuinfo") if l.startswith("model name")), ""),
"python": platform.python_version()}
out = Path(a.out)
out.mkdir(parents=True, exist_ok=True)
(out / f"{a.method}__{a.dataset}__f{a.fold}__s{a.seed}.json").write_text(json.dumps(rec, indent=1))
print(json.dumps(rec))
if __name__ == "__main__":
main()