Instructions to use Zipeng365/WISP with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Scikit-learn
How to use Zipeng365/WISP with Scikit-learn:
from huggingface_hub import hf_hub_download import joblib model = joblib.load( hf_hub_download("Zipeng365/WISP", "sklearn_model.joblib") ) # only load pickle files from sources you trust # read more about it here https://skops.readthedocs.io/en/stable/persistence.html - Notebooks
- Google Colab
- Kaggle
Download tests/test_equivalence.py from Zipeng365/WISP: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/Zipeng365/WISP/resolve/main/tests/test_equivalence.py
- Command line
-
hf download hf://Zipeng365/WISP/tests/test_equivalence.py
-
curl -L -o test_equivalence.py https://huggingface.co/Zipeng365/WISP/resolve/main/tests/test_equivalence.py
11.3 kB
| """Equivalence tests E1-E7 on the frozen ADL-Wrist fold 0 against the paper code and saved results. | |
| Run with ``WISP_FROZEN_DATA=<fold export root>`` (and optionally ``WISP_SAVED_RESULTS``, | |
| ``WISP_MODEL_INVENTORY``, ``WISP_CHECKPOINTS`` and ``WISP_EQUIV_REPORT=<jsonl>`` to write | |
| observed values). Single thread for bit-level comparisons. Comparisons with saved CSV | |
| values use the CSV's decimal text; differences of a few 1e-17 come from that rounding and | |
| are reported, never hidden. | |
| """ | |
| from __future__ import annotations | |
| import csv | |
| import json | |
| import os | |
| from functools import lru_cache | |
| from pathlib import Path | |
| import numpy as np | |
| import pytest | |
| from numpy.testing import assert_array_equal | |
| from threadpoolctl import threadpool_limits | |
| from conftest import frozen_root, saved_results_dir | |
| pytestmark = pytest.mark.skipif(frozen_root() is None or os.environ.get("WISP_RUN_EQUIVALENCE") != "1", | |
| reason="set WISP_FROZEN_DATA and WISP_RUN_EQUIVALENCE=1 for the equivalence checks") | |
| DATASET, FOLD = "adl_wrist_accel_v1", 0 | |
| FIXED = {"ss": "wisp_ss", "rc": "wisp_rc", "so": "wisp_so", "cse": "wisp_cse", "gis": "wisp_gis", | |
| "cis": "wisp_cis", "ese": "wisp_ese"} | |
| METRICS = ("macro_f1", "accuracy", "balanced_accuracy", "worst_class_f1") | |
| CSV_TOL = 1e-15 # decimal text of the saved CSVs; observed differences are reported | |
| def report(**record): | |
| path = os.environ.get("WISP_EQUIV_REPORT") | |
| if path: | |
| with open(path, "a") as f: | |
| f.write(json.dumps(record, default=float) + "\n") | |
| def fold(dataset=DATASET, k=FOLD): | |
| from wisp.datasets import load_exported_fold | |
| d = load_exported_fold(frozen_root(), dataset, k) | |
| def f64(s): | |
| return np.asarray(s.X, dtype=np.float64), s.y, s.groups, s.time_index | |
| tr, va, te = f64(d.train), f64(d.val), f64(d.test) | |
| trval = tuple(np.concatenate([a, b]) for a, b in zip(tr, va)) | |
| return tr, va, trval, te, len(d.label_names) | |
| def scores(y, pred, K): | |
| from wisp.metrics import score | |
| s = score(np.asarray(y, dtype=np.int64), np.asarray(pred, dtype=np.int64), K) | |
| return {k: s[k] for k in METRICS} | |
| def saved_rows(name): | |
| with open(saved_results_dir() / name, newline="") as f: | |
| return list(csv.DictReader(f)) | |
| def fitted_pair(short, seed): | |
| """(WISPClassifier, reference kernel) fitted on train+validation, paper protocol.""" | |
| from wisp import WISPClassifier | |
| from wisp.registry import build_method | |
| _, _, (X, y, g, t), _, K = fold() | |
| with threadpool_limits(1): | |
| clf = WISPClassifier(short, decoder="recipe", classes=np.arange(K), random_state=seed, n_jobs=1).fit( | |
| X, y, groups=g, time_index=t) | |
| ref = build_method(FIXED[short], seed=seed, n_jobs=1).fit(X, y, groups=g, time_index=t, n_classes=K) | |
| return clf, ref | |
| def test_e1_e2_fixed_families(short, seed): | |
| _, _, _, (Xq, yq, gq, tq), K = fold() | |
| clf, ref = fitted_pair(short, seed) | |
| with threadpool_limits(1): | |
| pred = clf.predict(Xq, groups=gq, time_index=tq) | |
| pred_ref = ref.predict(Xq, groups=gq, time_index=tq) | |
| P, P_ref = clf.predict_proba(Xq), ref.predict_proba(Xq) | |
| got = scores(yq, pred, K) | |
| row = next(r for r in saved_rows("run_results.csv") if r["dataset_id"] == DATASET and int(r["fold"]) == FOLD | |
| and r["method_id"] == FIXED[short] and int(r["model_seed"]) == seed) | |
| expected = {k: float(row[k]) for k in METRICS} | |
| diff = {k: abs(got[k] - expected[k]) for k in METRICS} | |
| report(test="E1/E2", recipe=short, seed=seed, label_mismatches=int((pred != pred_ref).sum()), | |
| argmax_mismatches=int((P.argmax(1) != P_ref.argmax(1)).sum()), proba_max_abs_diff=float(np.abs(P - P_ref).max()), | |
| observed=got, expected=expected, max_metric_abs_diff=max(diff.values())) | |
| assert_array_equal(pred, pred_ref) # E1: identical decoded labels | |
| assert max(diff.values()) <= CSV_TOL, (got, expected) # E2: saved four metrics | |
| def test_e3_cis_subsets(tag, decoder): | |
| from wisp import WISPClassifier | |
| _, _, (X, y, g, t), (Xq, yq, gq, tq), K = fold() | |
| with threadpool_limits(1): | |
| clf = WISPClassifier(f"cis:{tag}", decoder=decoder, classes=np.arange(K), random_state=42).fit( | |
| X, y, groups=g, time_index=t) | |
| pred = clf.predict(Xq, groups=gq, time_index=tq) | |
| got = scores(yq, pred, K) | |
| mid = f"cis_ablation_{tag}_{'viterbi' if decoder == 'hmm' else 'direct'}" | |
| row = next(r for r in saved_rows("cis_and_budget_runs.csv") if r["dataset_id"] == DATASET | |
| and int(r["fold"]) == FOLD and int(r["model_seed"]) == 42 and r["method_id"] == mid) | |
| expected = {k: float(row[k]) for k in METRICS} | |
| diff = {k: abs(got[k] - expected[k]) for k in METRICS} | |
| report(test="E3", recipe=f"cis:{tag}", decoder=decoder, paper_id=mid, observed=got, expected=expected, | |
| max_metric_abs_diff=max(diff.values()), has_paper_results=clf.has_paper_results_) | |
| assert clf.has_paper_results_ | |
| assert max(diff.values()) <= CSV_TOL, (got, expected) | |
| def _family_features(model, recipe, X): | |
| if recipe.family == "rocket_sketch_ridge_hmm": | |
| return model.sketch_.transform(X) | |
| if recipe.family in ("har_sculpt_forest_hmm", "symbolic_interval_forest"): | |
| return model.feature_extractor_.transform(X) | |
| return model._features(X) | |
| def test_e4_representation_bit_identical(name): | |
| from wisp import WISPRepresentation, recipes | |
| from wisp._engine import build_kernel | |
| (X, y, g, t), _, _, (Xq, _, _, _), K = fold() | |
| recipe = recipes.get(name) | |
| with threadpool_limits(1): | |
| rep = WISPRepresentation(name, random_state=42).fit(X) | |
| model = build_kernel(recipe, use_hmm=True, engine="reference", seed=42).fit(X, y, groups=g, time_index=t, | |
| n_classes=K) | |
| Z, F = rep.transform(Xq), _family_features(model, recipe, Xq) | |
| report(test="E4", recipe=name, shape=list(Z.shape), bit_identical=bool(np.array_equal(Z, F)), | |
| max_abs_diff=float(np.abs(Z - F).max())) | |
| assert_array_equal(Z, F) | |
| E5_STREAMS = ("adl", "capture24", "paal_adl", "pamap2") # best recipes not identical to a fixed recipe | |
| def test_e5_search_best_refit(stream): | |
| """Refit each non-overlapping best recipe on the first cell that selected it. | |
| The catalog recipe carries no candidate seed, so the catalog refit (random_state = model | |
| seed) is not expected to be bit-identical; the observed difference is reported. When the | |
| retained checkpoint of that cell is available (WISP_CHECKPOINTS), the refit is repeated | |
| with the candidate seed read from it, which must reproduce the saved metrics. | |
| """ | |
| from wisp import WISPClassifier, recipes | |
| recipe = recipes.best(stream) | |
| prov = recipe.provenance | |
| cell = sorted(prov["cells"])[0] | |
| method, fold_txt, seed_txt = cell.split("/") | |
| k, seed = int(fold_txt[4:]), int(seed_txt[4:]) | |
| _, _, (X, y, g, t), (Xq, yq, gq, tq), K = fold(prov["dataset_id"], k) | |
| row = next(r for r in saved_rows("cis_and_budget_runs.csv") if r["dataset_id"] == prov["dataset_id"] | |
| and int(r["fold"]) == k and int(r["model_seed"]) == seed and r["method_id"] == method | |
| and r["kind"] == "anytime_replay" and float(r["budget"]) == 329) | |
| expected = {m: float(row[m]) for m in METRICS} | |
| seeds = {"catalog_random_state": seed} | |
| ckpt_root = os.environ.get("WISP_CHECKPOINTS") | |
| if ckpt_root: | |
| from wisp_release.checkpoints import load_checkpoint | |
| folder = Path(ckpt_root) / method / prov["dataset_id"] / f"fold{k}" / f"seed{seed}" | |
| meta = json.loads((folder / "checkpoint.json").read_text()) | |
| model = load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"]) | |
| seeds["checkpoint_candidate_seed"] = int(model.seed) | |
| out = {} | |
| for label, s in seeds.items(): | |
| with threadpool_limits(1): | |
| clf = WISPClassifier(recipe, decoder="recipe", classes=np.arange(K), random_state=s).fit( | |
| X, y, groups=g, time_index=t) | |
| got = scores(yq, clf.predict(Xq, groups=gq, time_index=tq), K) | |
| out[label] = {"seed": s, "observed": got, "diff": {m: got[m] - expected[m] for m in METRICS}} | |
| report(test="E5", stream=stream, cell=cell, expected=expected, runs=out) | |
| if "checkpoint_candidate_seed" in out: | |
| assert max(abs(v) for v in out["checkpoint_candidate_seed"]["diff"].values()) <= CSV_TOL | |
| def test_e6_select(menu, seed): | |
| from wisp import WISPSelect | |
| inv = os.environ.get("WISP_MODEL_INVENTORY") | |
| if not inv: | |
| pytest.skip("set WISP_MODEL_INVENTORY to models/model_inventory.csv") | |
| with open(inv, newline="") as f: | |
| row = next(r for r in csv.DictReader(f) if r["method"] == f"wisp_select{menu}" and r["dataset"] == DATASET | |
| and int(r["fold"]) == FOLD and int(r["seed"]) == seed) | |
| (X, y, g, t), (Xv, yv, gv, tv), _, (Xq, yq, gq, tq), K = fold() | |
| with threadpool_limits(1): | |
| sel = WISPSelect(menu, decoder="recipe", classes=np.arange(K), random_state=seed).fit( | |
| X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv)) | |
| got = scores(yq, sel.predict(Xq, groups=gq, time_index=tq), K) | |
| chosen = sel.best_recipe_.provenance["method_id"] | |
| saved = next(r for r in saved_rows("run_results.csv") if r["dataset_id"] == DATASET and int(r["fold"]) == FOLD | |
| and r["method_id"] == f"wisp_select{menu}" and int(r["model_seed"]) == seed) | |
| expected = {m: float(saved[m]) for m in METRICS} | |
| report(test="E6", menu=menu, seed=seed, selected=chosen, expected_selected=row["selected_method"], | |
| observed=got, expected=expected, max_metric_abs_diff=max(abs(got[m] - expected[m]) for m in METRICS), | |
| selection_seconds=sel.selection_seconds_, final_fit_seconds=sel.final_fit_seconds_) | |
| assert chosen == row["selected_method"] | |
| def test_e7_hmm_smoother_on_real_scores(short): | |
| from wisp import HMMSmoother | |
| from wisp.cpu.hmm import TransitionSmoother | |
| _, _, (X, y, g, t), (Xq, yq, gq, tq), K = fold() | |
| clf, ref = fitted_pair(short, 42) | |
| P = clf.predict_proba(Xq) | |
| sm = HMMSmoother(decoder="recipe").fit(y, groups=g, time_index=t, classes=np.arange(K)) | |
| kernel = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=K) | |
| got = sm.decode(P, groups=gq, time_index=tq) | |
| exp = kernel.predict_from_proba(P, groups=gq, time_index=tq) | |
| report(test="E7", recipe=short, mismatches=int((got != exp).sum()), n=len(got), | |
| family_decoder_mismatches=int((got != ref.predict(Xq, groups=gq, time_index=tq)).sum())) | |
| assert_array_equal(got, exp) | |