import json import tempfile import unittest from pathlib import Path import numpy as np from threadpoolctl import threadpool_limits from wisp.core.data import HARData from wisp.core.metrics import classification_report_dict from wisp.search.cascade import evaluate_candidate_cascade from wisp.search.grammar import initial_candidates from wisp_fast.search import evaluate_recipes, fit_search, make_candidate from wisp_fast.search_metrics import search_report from wisp_fast.search_cache import SearchFeatures from wisp_fast import features as fx def splits(): rng=np.random.default_rng(55) def build(subject): y=np.resize([0,0,1,1,3,3],24) x=rng.normal(size=(len(y),32,3))+np.sin(np.arange(32)[None,:,None]*(1+y[:,None,None])*.4) return HARData(x,y,subject=np.repeat([subject,subject+1],12), time_index=np.tile(np.arange(12),2),label_names=list("abcd")) return build(1),build(3) class SharedSearchTests(unittest.TestCase): def test_scorer_search_semantics(self): for a,b in [([0,0,2,3],[0,1,1,2]),([2,2,2],[1,1,1]),([3],[3]),([0,3],[0,3])]: r=classification_report_dict(a,b) s=search_report(a,b) for k in ("weighted_f1","worst_class_f1","accuracy","balanced_accuracy","macro_f1"): self.assertAlmostEqual(r[k],s[k],places=14) self.assertEqual(r["confusion_matrix"],s["confusion_matrix"]) self.assertEqual(r["per_class_f1"],s["per_class_f1"]) def test_memoed_physics_is_seed_safe(self): data,_=splits() cache=SearchFeatures(16,[data.X]) for seed in (1,2,1): model=fx.InvariantPhysicsFeatureExtractor(random_state=seed).fit(data.X) expect=model.transform(data.X) got=cache.transform([model],data.X)[0] np.testing.assert_array_equal(expect,got) self.assertGreater(cache.hits,1) def test_stress_panel_and_full_metric_pairing(self): a,b=splits() specs=initial_candidates(seed=42) with threadpool_limits(1): scored,stats=evaluate_recipes(specs,a,b,mode="turbo",cost_weight=0,cache_mb=20) for spec,(status,r) in zip(specs,scored): # Match controller runtime injection, including one tree job. spec.params["n_jobs"]=1 ref=evaluate_candidate_cascade(spec,a,b,cost_weight=0, estimator_factory=lambda d:make_candidate(d,mode="turbo")).to_dict() self.assertNotEqual(status,"failed",msg=r) self.assertEqual(ref["confusion_matrix"],r["confusion_matrix"]) self.assertAlmostEqual(ref["cascade_score"],r["cascade_score"],places=14) self.assertEqual(ref["cascade_stages"][2]["metrics"]["stress_reports"], r["cascade_stages"][2]["metrics"]["stress_reports"]) self.assertGreater(stats["hits"],0) def test_no_refit_and_zero_stress(self): a,b=splits() with tempfile.TemporaryDirectory() as tmp,threadpool_limits(1): m,s=fit_search("wisp_evolution",a,b,mode="turbo",output=Path(tmp)/"run", evaluations=3,stress_count=0,robustness_weight=0,refit=False) self.assertIsNone(m) self.assertEqual(s["stress_count"],0) self.assertEqual(s["cached_stress"],[]) self.assertEqual(s["n_evaluated"],3) class ScopeTests(unittest.TestCase): def test_cis_scope(self): a,b=splits() with tempfile.TemporaryDirectory() as tmp,threadpool_limits(1): for method in ("wisp_random","wisp_evolution"): _,s=fit_search(method,a,b,mode="turbo",output=Path(tmp)/method,evaluations=11, population=6,family_scope="cis",refit=False,cost_weight=0) self.assertEqual(s["family_scope"],"cis") rows=Path(tmp,method,"program_database.jsonl").read_text().splitlines() self.assertEqual({json.loads(l)["spec"]["name"] for l in rows},{"state_rocket_interval_hmm"}) def test_related_scope(self): a,b=splits() allowed={"state_rocket_interval_hmm","rocket_sketch_ridge_hmm","posterior_cpu_ensemble"} with tempfile.TemporaryDirectory() as tmp,threadpool_limits(1): for method in ("wisp_random","wisp_evolution"): fit_search(method,a,b,output=Path(tmp)/method,mode="turbo",evaluations=10,population=6, family_scope="cis-related",refit=False) rows=Path(tmp,method,"program_database.jsonl").read_text().splitlines() self.assertEqual({json.loads(l)["spec"]["name"] for l in rows},allowed)