Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / benchmarks /check_checkpoints.py
Zipeng365's picture
wisp-har 0.3.0: scikit-learn interface, recipe catalog, Optuna search, cost-accuracy figures
4731b11 verified
Raw History Blame Contribute Delete
2.68 kB
"""Load retained WISP-Random/WISP-Evolution checkpoints with this package and rescore them.
Checks that the frozen kernel module paths still unpickle the retained models and that
their locked test predictions reproduce the saved four metrics in run_results.csv.
Uses the paper decoder of each checkpoint (no refit). Single thread.
python benchmarks/check_checkpoints.py --models <dir with wisp_random/...> \
--data-root <frozen fold root> --results results/run_results.csv --out out.json
"""
from __future__ import annotations
import argparse
import csv
import json
import sys
from pathlib import Path
from threadpoolctl import threadpool_limits
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _frozen import load_fold, paper_scores # noqa: E402
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--models", required=True)
ap.add_argument("--data-root", required=True)
ap.add_argument("--results", required=True)
ap.add_argument("--dataset", default="adl_wrist_accel_v1")
ap.add_argument("--fold", type=int, default=0)
ap.add_argument("--out", required=True)
a = ap.parse_args()
from wisp_release.checkpoints import load_checkpoint, predict_checkpoint
saved = {(r["method_id"], int(r["model_seed"])): r for r in csv.DictReader(open(a.results))
if r["dataset_id"] == a.dataset and int(r["fold"]) == a.fold}
_, _, _, test = load_fold(a.data_root, a.dataset, a.fold)
rows = []
for method in ("wisp_random", "wisp_evolution"):
for seed in (42, 43, 44):
folder = Path(a.models) / method / a.dataset / f"fold{a.fold}" / f"seed{seed}"
meta = json.loads((folder / "checkpoint.json").read_text())
with threadpool_limits(1):
model = load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"])
pred = predict_checkpoint(model, test["X"], subject=test["subject"],
time_index=test["time_index"], n_jobs=1)
got = paper_scores(test["y"], pred, test["n_classes"])
ref = saved[(method, seed)]
diff = {k: abs(got[k] - float(ref[k])) for k in got}
rows.append({"method": method, "seed": seed, "model_class": type(model).__name__,
"model_module": type(model).__module__, "observed": got,
"expected": {k: float(ref[k]) for k in got}, "abs_diff": diff,
"max_abs_diff": max(diff.values())})
print(json.dumps(rows[-1]))
Path(a.out).write_text(json.dumps(rows, indent=1))
if __name__ == "__main__":
main()