import json from pathlib import Path from tempfile import TemporaryDirectory import unittest import numpy as np from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier from threadpoolctl import threadpool_limits from wisp_fast import features as fx from wisp_fast.search import fit_search from wisp_fast.search_cache import SearchFeatures from wisp_fast.search_forest import forest_fit_plan from test_search_shared import splits class SearchPlanTests(unittest.TestCase): def test_forest_seed_and_tree_parity(self): rng=np.random.default_rng(76) x=rng.normal(size=(71,24)) y=np.resize([0,2,3,3],len(x)) for kind,weight in ((ExtraTreesClassifier,"balanced"),(RandomForestClassifier,"balanced_subsample")): for jobs in (1,2): kw=dict(random_state=7,n_estimators=11,n_jobs=jobs,class_weight=weight) ref=kind(**kw).fit(x,y) got=kind(**kw) with forest_fit_plan(got): got.fit(x,y) self.assertNotIn("_make_estimator",got.__dict__) for a,b in zip(ref.estimators_,got.estimators_): self.assertEqual(a.get_params(),b.get_params()) for name in ("children_left","children_right","feature","threshold","value","weighted_n_node_samples"): np.testing.assert_array_equal(getattr(a.tree_,name),getattr(b.tree_,name)) np.testing.assert_allclose(ref.predict_proba(x),got.predict_proba(x),atol=1e-15,rtol=0) def test_interval_cache_normalization_and_seed(self): x=np.random.default_rng(22).normal(size=(28,43,3)) mem=SearchFeatures(4,[x]) for fast in (True,False): for seed in (3,11): for factory in (fx.IntervalDistributionSketch,fx.InvariantPhysicsFeatureExtractor): b=factory(fast_profile=fast,random_state=seed).fit(x) np.testing.assert_array_equal(mem.transform([b],x)[0],b.transform(x)) if fx._cis_intervals is not None: self.assertGreater(mem.rank_hits,1) self.assertGreater(mem.dyad_hits,1) self.assertLessEqual(mem.peak,mem.limit) def test_certificates_against_complete_ledgers(self): train,valid=splits() with TemporaryDirectory() as tmp,threadpool_limits(1): for method in ("wisp_random","wisp_evolution"): arms={} for policy in ("complete","certified"): path=Path(tmp)/method/policy _,s=fit_search(method,train,valid,output=path,mode="turbo",evaluations=25, population=6,cost_weight=0,refit=False,stress_cache_mb=0, stress_policy=policy,workers=2) rows=[json.loads(l) for l in (path/"program_database.jsonl").read_text().splitlines()] arms[policy]=s,rows (s,full),(t,short)=arms.values() self.assertEqual([r["id"] for r in full],[r["id"] for r in short]) self.assertEqual(s["best"]["id"],t["best"]["id"]) skipped=0 for row,original in zip(short,full): if row["status"]=="pruned": cert=row["result"]["prune_certificate"] refscore=original["result"]["predictive_utility"] idx=cert["incumbent_ledger_index"] self.assertEqual(short[idx]["result"]["predictive_utility"],cert["incumbent_utility"]) self.assertLess(cert["upper_bound"],cert["incumbent_utility"]) self.assertLessEqual(refscore,cert["upper_bound"]) self.assertNotIn("predictive_utility",row["result"]) if method=="wisp_evolution":self.assertEqual(row["generation"],short[-1]["generation"]) skipped+=cert["skipped_views"] else: self.assertEqual(row["result"]["predictive_utility"],original["result"]["predictive_utility"]) self.assertEqual(skipped,t["skipped_stress_views"]) if method=="wisp_random":self.assertGreater(skipped,0)