"""Load retained WISP-Random/WISP-Evolution checkpoints with this package and rescore them. Checks that the frozen kernel module paths still unpickle the retained models and that their locked test predictions reproduce the saved four metrics in run_results.csv. Uses the paper decoder of each checkpoint (no refit). Single thread. python benchmarks/check_checkpoints.py --models \ --data-root --results results/run_results.csv --out out.json """ from __future__ import annotations import argparse import csv import json import sys from pathlib import Path from threadpoolctl import threadpool_limits sys.path.insert(0, str(Path(__file__).resolve().parent)) from _frozen import load_fold, paper_scores # noqa: E402 def main(): ap = argparse.ArgumentParser() ap.add_argument("--models", required=True) ap.add_argument("--data-root", required=True) ap.add_argument("--results", required=True) ap.add_argument("--dataset", default="adl_wrist_accel_v1") ap.add_argument("--fold", type=int, default=0) ap.add_argument("--out", required=True) a = ap.parse_args() from wisp_release.checkpoints import load_checkpoint, predict_checkpoint saved = {(r["method_id"], int(r["model_seed"])): r for r in csv.DictReader(open(a.results)) if r["dataset_id"] == a.dataset and int(r["fold"]) == a.fold} _, _, _, test = load_fold(a.data_root, a.dataset, a.fold) rows = [] for method in ("wisp_random", "wisp_evolution"): for seed in (42, 43, 44): folder = Path(a.models) / method / a.dataset / f"fold{a.fold}" / f"seed{seed}" meta = json.loads((folder / "checkpoint.json").read_text()) with threadpool_limits(1): model = load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"]) pred = predict_checkpoint(model, test["X"], subject=test["subject"], time_index=test["time_index"], n_jobs=1) got = paper_scores(test["y"], pred, test["n_classes"]) ref = saved[(method, seed)] diff = {k: abs(got[k] - float(ref[k])) for k in got} rows.append({"method": method, "seed": seed, "model_class": type(model).__name__, "model_module": type(model).__module__, "observed": got, "expected": {k: float(ref[k]) for k in got}, "abs_diff": diff, "max_abs_diff": max(diff.values())}) print(json.dumps(rows[-1])) Path(a.out).write_text(json.dumps(rows, indent=1)) if __name__ == "__main__": main()