"""Load retained WISP-Random/WISP-Evolution checkpoints with this package and rescore them.
Checks that the frozen kernel module paths still unpickle the retained models and that
their locked test predictions reproduce the saved four metrics in run_results.csv.
Uses the paper decoder of each checkpoint (no refit). Single thread.
python benchmarks/check_checkpoints.py --models
\
--data-root --results results/run_results.csv --out out.json
"""
from __future__ import annotations
import argparse
import csv
import json
import sys
from pathlib import Path
from threadpoolctl import threadpool_limits
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _frozen import load_fold, paper_scores # noqa: E402
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--models", required=True)
ap.add_argument("--data-root", required=True)
ap.add_argument("--results", required=True)
ap.add_argument("--dataset", default="adl_wrist_accel_v1")
ap.add_argument("--fold", type=int, default=0)
ap.add_argument("--out", required=True)
a = ap.parse_args()
from wisp_release.checkpoints import load_checkpoint, predict_checkpoint
saved = {(r["method_id"], int(r["model_seed"])): r for r in csv.DictReader(open(a.results))
if r["dataset_id"] == a.dataset and int(r["fold"]) == a.fold}
_, _, _, test = load_fold(a.data_root, a.dataset, a.fold)
rows = []
for method in ("wisp_random", "wisp_evolution"):
for seed in (42, 43, 44):
folder = Path(a.models) / method / a.dataset / f"fold{a.fold}" / f"seed{seed}"
meta = json.loads((folder / "checkpoint.json").read_text())
with threadpool_limits(1):
model = load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"])
pred = predict_checkpoint(model, test["X"], subject=test["subject"],
time_index=test["time_index"], n_jobs=1)
got = paper_scores(test["y"], pred, test["n_classes"])
ref = saved[(method, seed)]
diff = {k: abs(got[k] - float(ref[k])) for k in got}
rows.append({"method": method, "seed": seed, "model_class": type(model).__name__,
"model_module": type(model).__module__, "observed": got,
"expected": {k: float(ref[k]) for k in got}, "abs_diff": diff,
"max_abs_diff": max(diff.values())})
print(json.dumps(rows[-1]))
Path(a.out).write_text(json.dumps(rows, indent=1))
if __name__ == "__main__":
main()