"""Peak host memory of one WISP final model: train+validation fit plus test prediction, one thread. For WISP-Random and WISP-Evolution the final model of a search is the recipe it selected at budget 329 (``results/search_selections.csv``), refitted with the candidate seed read from the retained checkpoint of that cell, so the refit is the saved model. WISP-CIS is the fixed recipe. The four metrics are compared with ``results/run_results.csv`` to confirm the refit is the saved model. Peak memory is the process's ``ru_maxrss`` (whole process: imports, data, fit, prediction), the same measure as the paper's efficiency benchmark. Run one process per cell. python benchmarks/memory_final_models.py --data-root --results \ --checkpoints --method wisp_random \ --dataset adl_wrist_accel_v1 --out """ from __future__ import annotations import argparse import csv import json import platform import resource import sys import time from pathlib import Path import numpy as np from threadpoolctl import threadpool_limits sys.path.insert(0, str(Path(__file__).resolve().parent)) from _frozen import load_fold # noqa: E402 METRICS = ("macro_f1", "accuracy", "balanced_accuracy", "worst_class_f1") def selected_recipe(results, method, dataset, fold, seed): with open(Path(results) / "search_selections.csv", newline="") as f: for r in csv.DictReader(f): if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"]), float(r["budget"])) == \ (method, dataset, fold, seed, 329.0): return json.loads(r["canonical_recipe"]), r["candidate_id"] raise LookupError((method, dataset, fold, seed)) def saved_metrics(results, method, dataset, fold, seed): with open(Path(results) / "run_results.csv", newline="") as f: for r in csv.DictReader(f): if (r["method_id"], r["dataset_id"], int(r["fold"]), int(r["model_seed"])) == (method, dataset, fold, seed): return {m: float(r[m]) for m in METRICS} return None def main(): ap = argparse.ArgumentParser() ap.add_argument("--data-root", required=True) ap.add_argument("--results", required=True) ap.add_argument("--checkpoints") ap.add_argument("--method", required=True, choices=["wisp_random", "wisp_evolution", "wisp_cis"]) ap.add_argument("--dataset", required=True) ap.add_argument("--fold", type=int, default=0) ap.add_argument("--seed", type=int, default=42) ap.add_argument("--out", required=True) ap.add_argument("--blas-threads", type=int, default=1, help="BLAS threads (default 1; a retry with more threads can change LAPACK's operation order)") a = ap.parse_args() from wisp import WISPClassifier from wisp.metrics import score if a.method == "wisp_cis": recipe, candidate, rstate = "cis", None, a.seed else: from wisp_release.checkpoints import load_checkpoint recipe, candidate = selected_recipe(a.results, a.method, a.dataset, a.fold, a.seed) folder = Path(a.checkpoints) / a.method / a.dataset / f"fold{a.fold}" / f"seed{a.seed}" meta = json.loads((folder / "checkpoint.json").read_text()) rstate = int(load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"]).seed) del meta _, _, trval, test = load_fold(a.data_root, a.dataset, a.fold) K = trval["n_classes"] with threadpool_limits(a.blas_threads): clf = WISPClassifier(recipe, decoder="recipe", classes=np.arange(K), random_state=rstate, n_jobs=1) t0 = time.perf_counter() clf.fit(trval["X"], trval["y"], groups=trval["subject"], time_index=trval["time_index"]) fit_s = time.perf_counter() - t0 t1 = time.perf_counter() pred = clf.predict(test["X"], groups=test["subject"], time_index=test["time_index"]) predict_s = time.perf_counter() - t1 peak_gib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 ** 2 # Linux: KiB got = {m: float(v) for m, v in score(test["y"], np.asarray(pred, dtype=np.int64), K).items() if m in METRICS} saved = saved_metrics(a.results, a.method, a.dataset, a.fold, a.seed) rec = {"method": a.method, "dataset": a.dataset, "fold": a.fold, "seed": a.seed, "recipe": recipe, "candidate_id": candidate, "random_state": rstate, "peak_rss_gib": peak_gib, "fit_seconds": fit_s, "predict_seconds": predict_s, "metrics": got, "saved_metrics": saved, "max_abs_metric_diff": None if saved is None else max(abs(got[m] - saved[m]) for m in METRICS), "blas_threads": a.blas_threads, "n_trval": int(len(trval["y"])), "n_test": int(len(test["y"])), "cpu_model": next((l.split(":", 1)[1].strip() for l in open("/proc/cpuinfo") if l.startswith("model name")), ""), "python": platform.python_version()} out = Path(a.out) out.mkdir(parents=True, exist_ok=True) (out / f"{a.method}__{a.dataset}__f{a.fold}__s{a.seed}.json").write_text(json.dumps(rec, indent=1)) print(json.dumps(rec)) if __name__ == "__main__": main()