Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
File size: 2,678 Bytes
4731b11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
"""Load retained WISP-Random/WISP-Evolution checkpoints with this package and rescore them.

Checks that the frozen kernel module paths still unpickle the retained models and that
their locked test predictions reproduce the saved four metrics in run_results.csv.
Uses the paper decoder of each checkpoint (no refit). Single thread.

  python benchmarks/check_checkpoints.py --models <dir with wisp_random/...> \
      --data-root <frozen fold root> --results results/run_results.csv --out out.json
"""
from __future__ import annotations

import argparse
import csv
import json
import sys
from pathlib import Path

from threadpoolctl import threadpool_limits

sys.path.insert(0, str(Path(__file__).resolve().parent))
from _frozen import load_fold, paper_scores  # noqa: E402


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--models", required=True)
    ap.add_argument("--data-root", required=True)
    ap.add_argument("--results", required=True)
    ap.add_argument("--dataset", default="adl_wrist_accel_v1")
    ap.add_argument("--fold", type=int, default=0)
    ap.add_argument("--out", required=True)
    a = ap.parse_args()
    from wisp_release.checkpoints import load_checkpoint, predict_checkpoint

    saved = {(r["method_id"], int(r["model_seed"])): r for r in csv.DictReader(open(a.results))
             if r["dataset_id"] == a.dataset and int(r["fold"]) == a.fold}
    _, _, _, test = load_fold(a.data_root, a.dataset, a.fold)
    rows = []
    for method in ("wisp_random", "wisp_evolution"):
        for seed in (42, 43, 44):
            folder = Path(a.models) / method / a.dataset / f"fold{a.fold}" / f"seed{seed}"
            meta = json.loads((folder / "checkpoint.json").read_text())
            with threadpool_limits(1):
                model = load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"])
                pred = predict_checkpoint(model, test["X"], subject=test["subject"],
                                          time_index=test["time_index"], n_jobs=1)
            got = paper_scores(test["y"], pred, test["n_classes"])
            ref = saved[(method, seed)]
            diff = {k: abs(got[k] - float(ref[k])) for k in got}
            rows.append({"method": method, "seed": seed, "model_class": type(model).__name__,
                         "model_module": type(model).__module__, "observed": got,
                         "expected": {k: float(ref[k]) for k in got}, "abs_diff": diff,
                         "max_abs_diff": max(diff.values())})
            print(json.dumps(rows[-1]))
    Path(a.out).write_text(json.dumps(rows, indent=1))


if __name__ == "__main__":
    main()