Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
File size: 5,214 Bytes
19383e1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93a9282
19383e1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
"""Peak host memory of one WISP final model: train+validation fit plus test prediction, one thread.

For WISP-Random and WISP-Evolution the final model of a search is the recipe it selected at
budget 329 (``results/search_selections.csv``), refitted with the candidate seed read from the
retained checkpoint of that cell, so the refit is the saved model. WISP-CIS is the fixed recipe.
The four metrics are compared with ``results/run_results.csv`` to confirm the refit is the
saved model. Peak memory is the process's ``ru_maxrss`` (whole process: imports, data, fit,
prediction), the same measure as the paper's efficiency benchmark. Run one process per cell.

  python benchmarks/memory_final_models.py --data-root <fold exports> --results <results dir> \
      --checkpoints <dir with wisp_random/ and wisp_evolution/> --method wisp_random \
      --dataset adl_wrist_accel_v1 --out <dir>
"""
from __future__ import annotations

import argparse
import csv
import json
import platform
import resource
import sys
import time
from pathlib import Path

import numpy as np
from threadpoolctl import threadpool_limits

sys.path.insert(0, str(Path(__file__).resolve().parent))
from _frozen import load_fold  # noqa: E402

METRICS = ("macro_f1", "accuracy", "balanced_accuracy", "worst_class_f1")


def selected_recipe(results, method, dataset, fold, seed):
    with open(Path(results) / "search_selections.csv", newline="") as f:
        for r in csv.DictReader(f):
            if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"]), float(r["budget"])) == \
                    (method, dataset, fold, seed, 329.0):
                return json.loads(r["canonical_recipe"]), r["candidate_id"]
    raise LookupError((method, dataset, fold, seed))


def saved_metrics(results, method, dataset, fold, seed):
    with open(Path(results) / "run_results.csv", newline="") as f:
        for r in csv.DictReader(f):
            if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"])) == (method, dataset, fold, seed):
                return {m: float(r[m]) for m in METRICS}
    return None


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--data-root", required=True)
    ap.add_argument("--results", required=True)
    ap.add_argument("--checkpoints")
    ap.add_argument("--method", required=True, choices=["wisp_random", "wisp_evolution", "wisp_cis"])
    ap.add_argument("--dataset", required=True)
    ap.add_argument("--fold", type=int, default=0)
    ap.add_argument("--seed", type=int, default=42)
    ap.add_argument("--out", required=True)
    ap.add_argument("--blas-threads", type=int, default=1,
                    help="BLAS threads (default 1; a retry with more threads can change LAPACK's operation order)")
    a = ap.parse_args()

    from wisp import WISPClassifier
    from wisp.metrics import score

    if a.method == "wisp_cis":
        recipe, candidate, rstate = "cis", None, a.seed
    else:
        from wisp_release.checkpoints import load_checkpoint
        recipe, candidate = selected_recipe(a.results, a.method, a.dataset, a.fold, a.seed)
        folder = Path(a.checkpoints) / a.method / a.dataset / f"fold{a.fold}" / f"seed{a.seed}"
        meta = json.loads((folder / "checkpoint.json").read_text())
        rstate = int(load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"]).seed)
        del meta

    _, _, trval, test = load_fold(a.data_root, a.dataset, a.fold)
    K = trval["n_classes"]
    with threadpool_limits(a.blas_threads):
        clf = WISPClassifier(recipe, decoder="recipe", classes=np.arange(K), random_state=rstate, n_jobs=1)
        t0 = time.perf_counter()
        clf.fit(trval["X"], trval["y"], groups=trval["subject"], time_index=trval["time_index"])
        fit_s = time.perf_counter() - t0
        t1 = time.perf_counter()
        pred = clf.predict(test["X"], groups=test["subject"], time_index=test["time_index"])
        predict_s = time.perf_counter() - t1
    peak_gib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 ** 2   # Linux: KiB
    got = {m: float(v) for m, v in score(test["y"], np.asarray(pred, dtype=np.int64), K).items() if m in METRICS}
    saved = saved_metrics(a.results, a.method, a.dataset, a.fold, a.seed)
    rec = {"method": a.method, "dataset": a.dataset, "fold": a.fold, "seed": a.seed,
           "recipe": recipe, "candidate_id": candidate, "random_state": rstate,
           "peak_rss_gib": peak_gib, "fit_seconds": fit_s, "predict_seconds": predict_s,
           "metrics": got, "saved_metrics": saved,
           "max_abs_metric_diff": None if saved is None else max(abs(got[m] - saved[m]) for m in METRICS),
           "blas_threads": a.blas_threads, "n_trval": int(len(trval["y"])), "n_test": int(len(test["y"])),
           "cpu_model": next((l.split(":", 1)[1].strip() for l in open("/proc/cpuinfo") if l.startswith("model name")), ""),
           "python": platform.python_version()}
    out = Path(a.out)
    out.mkdir(parents=True, exist_ok=True)
    (out / f"{a.method}__{a.dataset}__f{a.fold}__s{a.seed}.json").write_text(json.dumps(rec, indent=1))
    print(json.dumps(rec))


if __name__ == "__main__":
    main()