Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / tests /test_equivalence.py
Zipeng365's picture
wisp-har 0.4.0: WISP-Funnel search and the adaptive HMM switch (default decoder)
93a9282 verified
Raw History Blame Contribute Delete
11.3 kB
"""Equivalence tests E1-E7 on the frozen ADL-Wrist fold 0 against the paper code and saved results.
Run with ``WISP_FROZEN_DATA=<fold export root>`` (and optionally ``WISP_SAVED_RESULTS``,
``WISP_MODEL_INVENTORY``, ``WISP_CHECKPOINTS`` and ``WISP_EQUIV_REPORT=<jsonl>`` to write
observed values). Single thread for bit-level comparisons. Comparisons with saved CSV
values use the CSV's decimal text; differences of a few 1e-17 come from that rounding and
are reported, never hidden.
"""
from __future__ import annotations
import csv
import json
import os
from functools import lru_cache
from pathlib import Path
import numpy as np
import pytest
from numpy.testing import assert_array_equal
from threadpoolctl import threadpool_limits
from conftest import frozen_root, saved_results_dir
pytestmark = pytest.mark.skipif(frozen_root() is None or os.environ.get("WISP_RUN_EQUIVALENCE") != "1",
reason="set WISP_FROZEN_DATA and WISP_RUN_EQUIVALENCE=1 for the equivalence checks")
DATASET, FOLD = "adl_wrist_accel_v1", 0
FIXED = {"ss": "wisp_ss", "rc": "wisp_rc", "so": "wisp_so", "cse": "wisp_cse", "gis": "wisp_gis",
"cis": "wisp_cis", "ese": "wisp_ese"}
METRICS = ("macro_f1", "accuracy", "balanced_accuracy", "worst_class_f1")
CSV_TOL = 1e-15 # decimal text of the saved CSVs; observed differences are reported
def report(**record):
path = os.environ.get("WISP_EQUIV_REPORT")
if path:
with open(path, "a") as f:
f.write(json.dumps(record, default=float) + "\n")
@lru_cache(maxsize=4)
def fold(dataset=DATASET, k=FOLD):
from wisp.datasets import load_exported_fold
d = load_exported_fold(frozen_root(), dataset, k)
def f64(s):
return np.asarray(s.X, dtype=np.float64), s.y, s.groups, s.time_index
tr, va, te = f64(d.train), f64(d.val), f64(d.test)
trval = tuple(np.concatenate([a, b]) for a, b in zip(tr, va))
return tr, va, trval, te, len(d.label_names)
def scores(y, pred, K):
from wisp.metrics import score
s = score(np.asarray(y, dtype=np.int64), np.asarray(pred, dtype=np.int64), K)
return {k: s[k] for k in METRICS}
def saved_rows(name):
with open(saved_results_dir() / name, newline="") as f:
return list(csv.DictReader(f))
@lru_cache(maxsize=32)
def fitted_pair(short, seed):
"""(WISPClassifier, reference kernel) fitted on train+validation, paper protocol."""
from wisp import WISPClassifier
from wisp.registry import build_method
_, _, (X, y, g, t), _, K = fold()
with threadpool_limits(1):
clf = WISPClassifier(short, decoder="recipe", classes=np.arange(K), random_state=seed, n_jobs=1).fit(
X, y, groups=g, time_index=t)
ref = build_method(FIXED[short], seed=seed, n_jobs=1).fit(X, y, groups=g, time_index=t, n_classes=K)
return clf, ref
@pytest.mark.parametrize("seed", [42, 43, 44])
@pytest.mark.parametrize("short", list(FIXED))
def test_e1_e2_fixed_families(short, seed):
_, _, _, (Xq, yq, gq, tq), K = fold()
clf, ref = fitted_pair(short, seed)
with threadpool_limits(1):
pred = clf.predict(Xq, groups=gq, time_index=tq)
pred_ref = ref.predict(Xq, groups=gq, time_index=tq)
P, P_ref = clf.predict_proba(Xq), ref.predict_proba(Xq)
got = scores(yq, pred, K)
row = next(r for r in saved_rows("run_results.csv") if r["dataset_id"] == DATASET and int(r["fold"]) == FOLD
and r["method_id"] == FIXED[short] and int(r["model_seed"]) == seed)
expected = {k: float(row[k]) for k in METRICS}
diff = {k: abs(got[k] - expected[k]) for k in METRICS}
report(test="E1/E2", recipe=short, seed=seed, label_mismatches=int((pred != pred_ref).sum()),
argmax_mismatches=int((P.argmax(1) != P_ref.argmax(1)).sum()), proba_max_abs_diff=float(np.abs(P - P_ref).max()),
observed=got, expected=expected, max_metric_abs_diff=max(diff.values()))
assert_array_equal(pred, pred_ref) # E1: identical decoded labels
assert max(diff.values()) <= CSV_TOL, (got, expected) # E2: saved four metrics
@pytest.mark.parametrize("decoder", ["hmm", "none"])
@pytest.mark.parametrize("tag", ["R", "I", "S", "RI", "RS", "IS", "RIS"])
def test_e3_cis_subsets(tag, decoder):
from wisp import WISPClassifier
_, _, (X, y, g, t), (Xq, yq, gq, tq), K = fold()
with threadpool_limits(1):
clf = WISPClassifier(f"cis:{tag}", decoder=decoder, classes=np.arange(K), random_state=42).fit(
X, y, groups=g, time_index=t)
pred = clf.predict(Xq, groups=gq, time_index=tq)
got = scores(yq, pred, K)
mid = f"cis_ablation_{tag}_{'viterbi' if decoder == 'hmm' else 'direct'}"
row = next(r for r in saved_rows("cis_and_budget_runs.csv") if r["dataset_id"] == DATASET
and int(r["fold"]) == FOLD and int(r["model_seed"]) == 42 and r["method_id"] == mid)
expected = {k: float(row[k]) for k in METRICS}
diff = {k: abs(got[k] - expected[k]) for k in METRICS}
report(test="E3", recipe=f"cis:{tag}", decoder=decoder, paper_id=mid, observed=got, expected=expected,
max_metric_abs_diff=max(diff.values()), has_paper_results=clf.has_paper_results_)
assert clf.has_paper_results_
assert max(diff.values()) <= CSV_TOL, (got, expected)
def _family_features(model, recipe, X):
if recipe.family == "rocket_sketch_ridge_hmm":
return model.sketch_.transform(X)
if recipe.family in ("har_sculpt_forest_hmm", "symbolic_interval_forest"):
return model.feature_extractor_.transform(X)
return model._features(X)
@pytest.mark.parametrize("name", ["ss", "rc", "so", "gis", "cis", "cis:R", "cis:I", "cis:S", "cis:RI", "cis:RS", "cis:IS"])
def test_e4_representation_bit_identical(name):
from wisp import WISPRepresentation, recipes
from wisp._engine import build_kernel
(X, y, g, t), _, _, (Xq, _, _, _), K = fold()
recipe = recipes.get(name)
with threadpool_limits(1):
rep = WISPRepresentation(name, random_state=42).fit(X)
model = build_kernel(recipe, use_hmm=True, engine="reference", seed=42).fit(X, y, groups=g, time_index=t,
n_classes=K)
Z, F = rep.transform(Xq), _family_features(model, recipe, Xq)
report(test="E4", recipe=name, shape=list(Z.shape), bit_identical=bool(np.array_equal(Z, F)),
max_abs_diff=float(np.abs(Z - F).max()))
assert_array_equal(Z, F)
E5_STREAMS = ("adl", "capture24", "paal_adl", "pamap2") # best recipes not identical to a fixed recipe
@pytest.mark.parametrize("stream", E5_STREAMS)
def test_e5_search_best_refit(stream):
"""Refit each non-overlapping best recipe on the first cell that selected it.
The catalog recipe carries no candidate seed, so the catalog refit (random_state = model
seed) is not expected to be bit-identical; the observed difference is reported. When the
retained checkpoint of that cell is available (WISP_CHECKPOINTS), the refit is repeated
with the candidate seed read from it, which must reproduce the saved metrics.
"""
from wisp import WISPClassifier, recipes
recipe = recipes.best(stream)
prov = recipe.provenance
cell = sorted(prov["cells"])[0]
method, fold_txt, seed_txt = cell.split("/")
k, seed = int(fold_txt[4:]), int(seed_txt[4:])
_, _, (X, y, g, t), (Xq, yq, gq, tq), K = fold(prov["dataset_id"], k)
row = next(r for r in saved_rows("cis_and_budget_runs.csv") if r["dataset_id"] == prov["dataset_id"]
and int(r["fold"]) == k and int(r["model_seed"]) == seed and r["method_id"] == method
and r["kind"] == "anytime_replay" and float(r["budget"]) == 329)
expected = {m: float(row[m]) for m in METRICS}
seeds = {"catalog_random_state": seed}
ckpt_root = os.environ.get("WISP_CHECKPOINTS")
if ckpt_root:
from wisp_release.checkpoints import load_checkpoint
folder = Path(ckpt_root) / method / prov["dataset_id"] / f"fold{k}" / f"seed{seed}"
meta = json.loads((folder / "checkpoint.json").read_text())
model = load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"])
seeds["checkpoint_candidate_seed"] = int(model.seed)
out = {}
for label, s in seeds.items():
with threadpool_limits(1):
clf = WISPClassifier(recipe, decoder="recipe", classes=np.arange(K), random_state=s).fit(
X, y, groups=g, time_index=t)
got = scores(yq, clf.predict(Xq, groups=gq, time_index=tq), K)
out[label] = {"seed": s, "observed": got, "diff": {m: got[m] - expected[m] for m in METRICS}}
report(test="E5", stream=stream, cell=cell, expected=expected, runs=out)
if "checkpoint_candidate_seed" in out:
assert max(abs(v) for v in out["checkpoint_candidate_seed"]["diff"].values()) <= CSV_TOL
@pytest.mark.parametrize("seed", [42, 43, 44])
@pytest.mark.parametrize("menu", [5, 7])
def test_e6_select(menu, seed):
from wisp import WISPSelect
inv = os.environ.get("WISP_MODEL_INVENTORY")
if not inv:
pytest.skip("set WISP_MODEL_INVENTORY to models/model_inventory.csv")
with open(inv, newline="") as f:
row = next(r for r in csv.DictReader(f) if r["method"] == f"wisp_select{menu}" and r["dataset"] == DATASET
and int(r["fold"]) == FOLD and int(r["seed"]) == seed)
(X, y, g, t), (Xv, yv, gv, tv), _, (Xq, yq, gq, tq), K = fold()
with threadpool_limits(1):
sel = WISPSelect(menu, decoder="recipe", classes=np.arange(K), random_state=seed).fit(
X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv))
got = scores(yq, sel.predict(Xq, groups=gq, time_index=tq), K)
chosen = sel.best_recipe_.provenance["method_id"]
saved = next(r for r in saved_rows("run_results.csv") if r["dataset_id"] == DATASET and int(r["fold"]) == FOLD
and r["method_id"] == f"wisp_select{menu}" and int(r["model_seed"]) == seed)
expected = {m: float(saved[m]) for m in METRICS}
report(test="E6", menu=menu, seed=seed, selected=chosen, expected_selected=row["selected_method"],
observed=got, expected=expected, max_metric_abs_diff=max(abs(got[m] - expected[m]) for m in METRICS),
selection_seconds=sel.selection_seconds_, final_fit_seconds=sel.final_fit_seconds_)
assert chosen == row["selected_method"]
@pytest.mark.parametrize("short", ["cis", "rc", "ss"])
def test_e7_hmm_smoother_on_real_scores(short):
from wisp import HMMSmoother
from wisp.cpu.hmm import TransitionSmoother
_, _, (X, y, g, t), (Xq, yq, gq, tq), K = fold()
clf, ref = fitted_pair(short, 42)
P = clf.predict_proba(Xq)
sm = HMMSmoother(decoder="recipe").fit(y, groups=g, time_index=t, classes=np.arange(K))
kernel = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=K)
got = sm.decode(P, groups=gq, time_index=tq)
exp = kernel.predict_from_proba(P, groups=gq, time_index=tq)
report(test="E7", recipe=short, mismatches=int((got != exp).sum()), n=len(got),
family_decoder_mismatches=int((got != ref.predict(Xq, groups=gq, time_index=tq)).sum()))
assert_array_equal(got, exp)