Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / tests /test_search_shared.py
Zipeng365's picture
wisp-har 0.2.0: accelerated implementation with speed (exact/fast) and HMM sequence (subject/recording) options
ac7f7cb verified
Raw History Blame Contribute Delete
4.71 kB
import json
import tempfile
import unittest
from pathlib import Path
import numpy as np
from threadpoolctl import threadpool_limits
from wisp.core.data import HARData
from wisp.core.metrics import classification_report_dict
from wisp.search.cascade import evaluate_candidate_cascade
from wisp.search.grammar import initial_candidates
from wisp_fast.search import evaluate_recipes, fit_search, make_candidate
from wisp_fast.search_metrics import search_report
from wisp_fast.search_cache import SearchFeatures
from wisp_fast import features as fx
def splits():
rng=np.random.default_rng(55)
def build(subject):
y=np.resize([0,0,1,1,3,3],24)
x=rng.normal(size=(len(y),32,3))+np.sin(np.arange(32)[None,:,None]*(1+y[:,None,None])*.4)
return HARData(x,y,subject=np.repeat([subject,subject+1],12),
time_index=np.tile(np.arange(12),2),label_names=list("abcd"))
return build(1),build(3)
class SharedSearchTests(unittest.TestCase):
def test_scorer_search_semantics(self):
for a,b in [([0,0,2,3],[0,1,1,2]),([2,2,2],[1,1,1]),([3],[3]),([0,3],[0,3])]:
r=classification_report_dict(a,b)
s=search_report(a,b)
for k in ("weighted_f1","worst_class_f1","accuracy","balanced_accuracy","macro_f1"):
self.assertAlmostEqual(r[k],s[k],places=14)
self.assertEqual(r["confusion_matrix"],s["confusion_matrix"])
self.assertEqual(r["per_class_f1"],s["per_class_f1"])
def test_memoed_physics_is_seed_safe(self):
data,_=splits()
cache=SearchFeatures(16,[data.X])
for seed in (1,2,1):
model=fx.InvariantPhysicsFeatureExtractor(random_state=seed).fit(data.X)
expect=model.transform(data.X)
got=cache.transform([model],data.X)[0]
np.testing.assert_array_equal(expect,got)
self.assertGreater(cache.hits,1)
def test_stress_panel_and_full_metric_pairing(self):
a,b=splits()
specs=initial_candidates(seed=42)
with threadpool_limits(1):
scored,stats=evaluate_recipes(specs,a,b,mode="turbo",cost_weight=0,cache_mb=20)
for spec,(status,r) in zip(specs,scored):
# Match controller runtime injection, including one tree job.
spec.params["n_jobs"]=1
ref=evaluate_candidate_cascade(spec,a,b,cost_weight=0,
estimator_factory=lambda d:make_candidate(d,mode="turbo")).to_dict()
self.assertNotEqual(status,"failed",msg=r)
self.assertEqual(ref["confusion_matrix"],r["confusion_matrix"])
self.assertAlmostEqual(ref["cascade_score"],r["cascade_score"],places=14)
self.assertEqual(ref["cascade_stages"][2]["metrics"]["stress_reports"],
r["cascade_stages"][2]["metrics"]["stress_reports"])
self.assertGreater(stats["hits"],0)
def test_no_refit_and_zero_stress(self):
a,b=splits()
with tempfile.TemporaryDirectory() as tmp,threadpool_limits(1):
m,s=fit_search("wisp_evolution",a,b,mode="turbo",output=Path(tmp)/"run",
evaluations=3,stress_count=0,robustness_weight=0,refit=False)
self.assertIsNone(m)
self.assertEqual(s["stress_count"],0)
self.assertEqual(s["cached_stress"],[])
self.assertEqual(s["n_evaluated"],3)
class ScopeTests(unittest.TestCase):
def test_cis_scope(self):
a,b=splits()
with tempfile.TemporaryDirectory() as tmp,threadpool_limits(1):
for method in ("wisp_random","wisp_evolution"):
_,s=fit_search(method,a,b,mode="turbo",output=Path(tmp)/method,evaluations=11,
population=6,family_scope="cis",refit=False,cost_weight=0)
self.assertEqual(s["family_scope"],"cis")
rows=Path(tmp,method,"program_database.jsonl").read_text().splitlines()
self.assertEqual({json.loads(l)["spec"]["name"] for l in rows},{"state_rocket_interval_hmm"})
def test_related_scope(self):
a,b=splits()
allowed={"state_rocket_interval_hmm","rocket_sketch_ridge_hmm","posterior_cpu_ensemble"}
with tempfile.TemporaryDirectory() as tmp,threadpool_limits(1):
for method in ("wisp_random","wisp_evolution"):
fit_search(method,a,b,output=Path(tmp)/method,mode="turbo",evaluations=10,population=6,
family_scope="cis-related",refit=False)
rows=Path(tmp,method,"program_database.jsonl").read_text().splitlines()
self.assertEqual({json.loads(l)["spec"]["name"] for l in rows},allowed)