"""Equivalence tests E1-E7 on the frozen ADL-Wrist fold 0 against the paper code and saved results. Run with ``WISP_FROZEN_DATA=`` (and optionally ``WISP_SAVED_RESULTS``, ``WISP_MODEL_INVENTORY``, ``WISP_CHECKPOINTS`` and ``WISP_EQUIV_REPORT=`` to write observed values). Single thread for bit-level comparisons. Comparisons with saved CSV values use the CSV's decimal text; differences of a few 1e-17 come from that rounding and are reported, never hidden. """ from __future__ import annotations import csv import json import os from functools import lru_cache from pathlib import Path import numpy as np import pytest from numpy.testing import assert_array_equal from threadpoolctl import threadpool_limits from conftest import frozen_root, saved_results_dir pytestmark = pytest.mark.skipif(frozen_root() is None or os.environ.get("WISP_RUN_EQUIVALENCE") != "1", reason="set WISP_FROZEN_DATA and WISP_RUN_EQUIVALENCE=1 for the equivalence checks") DATASET, FOLD = "adl_wrist_accel_v1", 0 FIXED = {"ss": "wisp_ss", "rc": "wisp_rc", "so": "wisp_so", "cse": "wisp_cse", "gis": "wisp_gis", "cis": "wisp_cis", "ese": "wisp_ese"} METRICS = ("macro_f1", "accuracy", "balanced_accuracy", "worst_class_f1") CSV_TOL = 1e-15 # decimal text of the saved CSVs; observed differences are reported def report(**record): path = os.environ.get("WISP_EQUIV_REPORT") if path: with open(path, "a") as f: f.write(json.dumps(record, default=float) + "\n") @lru_cache(maxsize=4) def fold(dataset=DATASET, k=FOLD): from wisp.datasets import load_exported_fold d = load_exported_fold(frozen_root(), dataset, k) def f64(s): return np.asarray(s.X, dtype=np.float64), s.y, s.groups, s.time_index tr, va, te = f64(d.train), f64(d.val), f64(d.test) trval = tuple(np.concatenate([a, b]) for a, b in zip(tr, va)) return tr, va, trval, te, len(d.label_names) def scores(y, pred, K): from wisp.metrics import score s = score(np.asarray(y, dtype=np.int64), np.asarray(pred, dtype=np.int64), K) return {k: s[k] for k in METRICS} def saved_rows(name): with open(saved_results_dir() / name, newline="") as f: return list(csv.DictReader(f)) @lru_cache(maxsize=32) def fitted_pair(short, seed): """(WISPClassifier, reference kernel) fitted on train+validation, paper protocol.""" from wisp import WISPClassifier from wisp.registry import build_method _, _, (X, y, g, t), _, K = fold() with threadpool_limits(1): clf = WISPClassifier(short, decoder="recipe", classes=np.arange(K), random_state=seed, n_jobs=1).fit( X, y, groups=g, time_index=t) ref = build_method(FIXED[short], seed=seed, n_jobs=1).fit(X, y, groups=g, time_index=t, n_classes=K) return clf, ref @pytest.mark.parametrize("seed", [42, 43, 44]) @pytest.mark.parametrize("short", list(FIXED)) def test_e1_e2_fixed_families(short, seed): _, _, _, (Xq, yq, gq, tq), K = fold() clf, ref = fitted_pair(short, seed) with threadpool_limits(1): pred = clf.predict(Xq, groups=gq, time_index=tq) pred_ref = ref.predict(Xq, groups=gq, time_index=tq) P, P_ref = clf.predict_proba(Xq), ref.predict_proba(Xq) got = scores(yq, pred, K) row = next(r for r in saved_rows("run_results.csv") if r["dataset_id"] == DATASET and int(r["fold"]) == FOLD and r["method_id"] == FIXED[short] and int(r["model_seed"]) == seed) expected = {k: float(row[k]) for k in METRICS} diff = {k: abs(got[k] - expected[k]) for k in METRICS} report(test="E1/E2", recipe=short, seed=seed, label_mismatches=int((pred != pred_ref).sum()), argmax_mismatches=int((P.argmax(1) != P_ref.argmax(1)).sum()), proba_max_abs_diff=float(np.abs(P - P_ref).max()), observed=got, expected=expected, max_metric_abs_diff=max(diff.values())) assert_array_equal(pred, pred_ref) # E1: identical decoded labels assert max(diff.values()) <= CSV_TOL, (got, expected) # E2: saved four metrics @pytest.mark.parametrize("decoder", ["hmm", "none"]) @pytest.mark.parametrize("tag", ["R", "I", "S", "RI", "RS", "IS", "RIS"]) def test_e3_cis_subsets(tag, decoder): from wisp import WISPClassifier _, _, (X, y, g, t), (Xq, yq, gq, tq), K = fold() with threadpool_limits(1): clf = WISPClassifier(f"cis:{tag}", decoder=decoder, classes=np.arange(K), random_state=42).fit( X, y, groups=g, time_index=t) pred = clf.predict(Xq, groups=gq, time_index=tq) got = scores(yq, pred, K) mid = f"cis_ablation_{tag}_{'viterbi' if decoder == 'hmm' else 'direct'}" row = next(r for r in saved_rows("cis_and_budget_runs.csv") if r["dataset_id"] == DATASET and int(r["fold"]) == FOLD and int(r["model_seed"]) == 42 and r["method_id"] == mid) expected = {k: float(row[k]) for k in METRICS} diff = {k: abs(got[k] - expected[k]) for k in METRICS} report(test="E3", recipe=f"cis:{tag}", decoder=decoder, paper_id=mid, observed=got, expected=expected, max_metric_abs_diff=max(diff.values()), has_paper_results=clf.has_paper_results_) assert clf.has_paper_results_ assert max(diff.values()) <= CSV_TOL, (got, expected) def _family_features(model, recipe, X): if recipe.family == "rocket_sketch_ridge_hmm": return model.sketch_.transform(X) if recipe.family in ("har_sculpt_forest_hmm", "symbolic_interval_forest"): return model.feature_extractor_.transform(X) return model._features(X) @pytest.mark.parametrize("name", ["ss", "rc", "so", "gis", "cis", "cis:R", "cis:I", "cis:S", "cis:RI", "cis:RS", "cis:IS"]) def test_e4_representation_bit_identical(name): from wisp import WISPRepresentation, recipes from wisp._engine import build_kernel (X, y, g, t), _, _, (Xq, _, _, _), K = fold() recipe = recipes.get(name) with threadpool_limits(1): rep = WISPRepresentation(name, random_state=42).fit(X) model = build_kernel(recipe, use_hmm=True, engine="reference", seed=42).fit(X, y, groups=g, time_index=t, n_classes=K) Z, F = rep.transform(Xq), _family_features(model, recipe, Xq) report(test="E4", recipe=name, shape=list(Z.shape), bit_identical=bool(np.array_equal(Z, F)), max_abs_diff=float(np.abs(Z - F).max())) assert_array_equal(Z, F) E5_STREAMS = ("adl", "capture24", "paal_adl", "pamap2") # best recipes not identical to a fixed recipe @pytest.mark.parametrize("stream", E5_STREAMS) def test_e5_search_best_refit(stream): """Refit each non-overlapping best recipe on the first cell that selected it. The catalog recipe carries no candidate seed, so the catalog refit (random_state = model seed) is not expected to be bit-identical; the observed difference is reported. When the retained checkpoint of that cell is available (WISP_CHECKPOINTS), the refit is repeated with the candidate seed read from it, which must reproduce the saved metrics. """ from wisp import WISPClassifier, recipes recipe = recipes.best(stream) prov = recipe.provenance cell = sorted(prov["cells"])[0] method, fold_txt, seed_txt = cell.split("/") k, seed = int(fold_txt[4:]), int(seed_txt[4:]) _, _, (X, y, g, t), (Xq, yq, gq, tq), K = fold(prov["dataset_id"], k) row = next(r for r in saved_rows("cis_and_budget_runs.csv") if r["dataset_id"] == prov["dataset_id"] and int(r["fold"]) == k and int(r["model_seed"]) == seed and r["method_id"] == method and r["kind"] == "anytime_replay" and float(r["budget"]) == 329) expected = {m: float(row[m]) for m in METRICS} seeds = {"catalog_random_state": seed} ckpt_root = os.environ.get("WISP_CHECKPOINTS") if ckpt_root: from wisp_release.checkpoints import load_checkpoint folder = Path(ckpt_root) / method / prov["dataset_id"] / f"fold{k}" / f"seed{seed}" meta = json.loads((folder / "checkpoint.json").read_text()) model = load_checkpoint(folder / "model.pkl", trust_pickle=True, expected_sha256=meta["sha256"]) seeds["checkpoint_candidate_seed"] = int(model.seed) out = {} for label, s in seeds.items(): with threadpool_limits(1): clf = WISPClassifier(recipe, decoder="recipe", classes=np.arange(K), random_state=s).fit( X, y, groups=g, time_index=t) got = scores(yq, clf.predict(Xq, groups=gq, time_index=tq), K) out[label] = {"seed": s, "observed": got, "diff": {m: got[m] - expected[m] for m in METRICS}} report(test="E5", stream=stream, cell=cell, expected=expected, runs=out) if "checkpoint_candidate_seed" in out: assert max(abs(v) for v in out["checkpoint_candidate_seed"]["diff"].values()) <= CSV_TOL @pytest.mark.parametrize("seed", [42, 43, 44]) @pytest.mark.parametrize("menu", [5, 7]) def test_e6_select(menu, seed): from wisp import WISPSelect inv = os.environ.get("WISP_MODEL_INVENTORY") if not inv: pytest.skip("set WISP_MODEL_INVENTORY to models/model_inventory.csv") with open(inv, newline="") as f: row = next(r for r in csv.DictReader(f) if r["method"] == f"wisp_select{menu}" and r["dataset"] == DATASET and int(r["fold"]) == FOLD and int(r["seed"]) == seed) (X, y, g, t), (Xv, yv, gv, tv), _, (Xq, yq, gq, tq), K = fold() with threadpool_limits(1): sel = WISPSelect(menu, decoder="recipe", classes=np.arange(K), random_state=seed).fit( X, y, groups=g, time_index=t, validation=(Xv, yv, gv, tv)) got = scores(yq, sel.predict(Xq, groups=gq, time_index=tq), K) chosen = sel.best_recipe_.provenance["method_id"] saved = next(r for r in saved_rows("run_results.csv") if r["dataset_id"] == DATASET and int(r["fold"]) == FOLD and r["method_id"] == f"wisp_select{menu}" and int(r["model_seed"]) == seed) expected = {m: float(saved[m]) for m in METRICS} report(test="E6", menu=menu, seed=seed, selected=chosen, expected_selected=row["selected_method"], observed=got, expected=expected, max_metric_abs_diff=max(abs(got[m] - expected[m]) for m in METRICS), selection_seconds=sel.selection_seconds_, final_fit_seconds=sel.final_fit_seconds_) assert chosen == row["selected_method"] @pytest.mark.parametrize("short", ["cis", "rc", "ss"]) def test_e7_hmm_smoother_on_real_scores(short): from wisp import HMMSmoother from wisp.cpu.hmm import TransitionSmoother _, _, (X, y, g, t), (Xq, yq, gq, tq), K = fold() clf, ref = fitted_pair(short, 42) P = clf.predict_proba(Xq) sm = HMMSmoother(decoder="recipe").fit(y, groups=g, time_index=t, classes=np.arange(K)) kernel = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=K) got = sm.decode(P, groups=gq, time_index=tq) exp = kernel.predict_from_proba(P, groups=gq, time_index=tq) report(test="E7", recipe=short, mismatches=int((got != exp).sum()), n=len(got), family_decoder_mismatches=int((got != ref.predict(Xq, groups=gq, time_index=tq)).sum())) assert_array_equal(got, exp)