"""WISP-Random / WISP-Evolution search with split-local feature sharing. The proposal grammar, cascade, stress views and utility are those of ``wisp.search.controller``; candidates are fitted with the accelerated engine. """ from concurrent.futures import ThreadPoolExecutor from contextvars import copy_context from dataclasses import replace from functools import partial import json import pickle from pathlib import Path import time import numpy as np from wisp.core.data import concatenate_har from wisp.core.metrics import classification_report_dict from wisp.cpu.algorithms import estimate_pickle_size_mb from wisp.search.cascade import STRESS_TRANSFORMS, evaluate_candidate_cascade from wisp.search.controller import WISPSearchController from wisp.search.grammar import CandidateSpec from wisp.search.profiles import get_profile from .cache import _SEARCH from .decoder import check_sequence from .engine import RIDGE_SOLVERS, build_candidate from .selection import check_splits, fit_on from .search_cache import SearchFeatures from .search_metrics import search_report,SearchConfusion def make_candidate(spec_dict, *, mode="exact", budget=None, ridge_solver="original", sequence="subject", calibration=None): """Accelerated estimator for one search candidate. ``calibration`` (a validation HARData) is used only with ``sequence="recording"``: right after fitting, the confidence scaling is chosen on those windows. """ if ridge_solver not in RIDGE_SOLVERS or (mode!="exact" and ridge_solver!="original"): raise ValueError("The screened Ridge solver requires exact full-size features") model=build_candidate(spec_dict,mode=mode,budget=budget,ridge_solver=ridge_solver,sequence=sequence) if sequence=="recording" and calibration is not None: model.calibration_data=(calibration.X,calibration.y) return model class _AcceleratedSearch(WISPSearchController): def _evaluate_many(self, specs): # Random proposals are score-independent. Evolution may prune only after # generating its terminal batch; every breeding decision is then complete. terminal = self.rows_seen+len(specs) == self.profile.max_evaluations eligible = self.profile.search_strategy == "random" or terminal self.batch_bound = self.incumbent if self.stress_policy=="certified" and eligible else None # Update only between completed waves. Certificates always reference # earlier canonical ledger rows, independent of worker completion order. if self.stress_policy=="frontier" and eligible: rows=[] width=max(self._worker_count(),1) for start in range(0,len(specs),width): self.batch_bound=self.incumbent chunk=self._evaluate_batch(specs[start:start+width]) self._observe(chunk,self.rows_seen+start) rows.extend(chunk) else: rows = self._evaluate_batch(specs) self._observe(rows,self.rows_seen) self.rows_seen += len(rows) return rows def _observe(self,rows,start): for index,(status,result) in enumerate(rows,start): if status == "ok": score = result["predictive_utility"] if self.incumbent is None or score > self.incumbent[0]: self.incumbent = (score,index) elif status == "pruned": self.pruned_count += 1 self.skipped_views += result["prune_certificate"]["skipped_views"] def _stress_gate(self, metrics, observed): # Macro-F1 <= 1. The stress contribution w*s - .05*max(clean-s,0) # is monotone on [0,1] for w>=0. No equality pruning, so ties survive. score,index = self.batch_bound remaining = self.stress_count-len(observed) mean_upper = float(np.mean(observed+[1.0]*remaining)) if observed else 1.0 gap_lower = max(float(metrics["macro_f1"])-mean_upper,0.0) upper = float(metrics["score"] + self.robustness_weight*mean_upper - .05*gap_lower) if np.isfinite(upper) and upper < score: return dict(upper_bound=upper,incumbent_utility=score, incumbent_ledger_index=index,stress_f1_upper=mean_upper, observed_stress_f1=list(observed),skipped_views=remaining) return None def _evaluate_batch(self,specs): if self._worker_count() == 1 or len(specs)<2: return super()._evaluate_many(specs) with ThreadPoolExecutor(max_workers=min(self._worker_count(),len(specs))) as pool: jobs=[pool.submit(copy_context().run,self._evaluate_one,s) for s in specs] return [job.result() for job in jobs] def _batched_stress(self, model): # Each stress view is a distinct sequence panel. Never Viterbi-decode # a concatenated panel, even if the same subject IDs appear in each. P=model.predict_proba(self.stress_bundle) chunks=np.split(P,self.stress_count) return {name:model.decode_proba(p,groups=self.valid.subject,time_index=self.valid.time_index) for name,p in zip(self.stress_names,chunks)} def _evaluate_one(self, spec): token=_SEARCH.set(self.memory if self.memory.limit else None) built=[] def factory(spec_dict): built.append(self.create_candidate(spec_dict)) return built[-1] try: cascade=evaluate_candidate_cascade( spec,self.train,self.valid,estimator_factory=factory, cost_weight=self.cost_weight,robustness_weight=self.robustness_weight, max_stress_transforms=self.stress_count,stress_inputs=self.stress_inputs, reporter=self.reporter,stress_gate=self._stress_gate if self.batch_bound is not None else None, stress_predictor=self._batched_stress if self.stress_bundle is not None else None) result=cascade.to_dict() if self.sequence=="recording" and built: smoother=getattr(built[-1],"smoother_",None) result["sequence"]="recording" result["sequence_smoothing"]=bool(smoother is not None and smoother.active) result["confidence_scale"]=built[-1].confidence_scale if "prune_certificate" in result: del result["cascade_score"] del result["accepted"] # stress and final eligibility were not evaluated return "pruned",result result["predictive_utility"]=float(result["cascade_score"]) return ("ok" if cascade.accepted else "rejected",result) except Exception as exc: return ("failed",{"predictive_utility":-1e9,"breeding_score":-1e9, "error":repr(exc),"candidate":spec.to_dict()}) finally: _SEARCH.reset(token) def _final_refit(self, best, out_dir): """Refit the validation winner on train+validation with the accelerated engine.""" if self.test is None: return None model = self.create_candidate(best["spec"]) model.__dict__.pop("calibration_data", None) refit = concatenate_har(self.train, self.valid, source="train+validation") t0 = time.perf_counter() fit_on(model, refit) fit_s = time.perf_counter() - t0 # A train+validation refit reuses the scaling chosen for the train-only candidate. model.set_confidence_scale(best["result"].get("confidence_scale")) t1 = time.perf_counter() pred = model.predict(self.test.X, groups=self.test.subject, time_index=self.test.time_index) pred_s = time.perf_counter() - t1 metrics = classification_report_dict(self.test.y, pred, fit_seconds=fit_s, predict_seconds=pred_s, model_size_mb=estimate_pickle_size_mb(model), label_names=self.test.label_names) np.savez_compressed(out_dir / "locked_test_predictions.npz", y_true=self.test.y, y_pred=pred, subject=self.test.subject, time_index=self.test.time_index) with (out_dir / "selected_model.pkl").open("wb") as f: pickle.dump(model, f) (out_dir / "locked_test_metrics.json").write_text( json.dumps(metrics, indent=2, ensure_ascii=False, default=str), encoding="utf-8") return metrics def _configure(search, mode, budget, ridge_solver, cache_mb, stress_cache_mb, stress_count, stress_policy="complete", sequence="subject"): if stress_policy not in {"complete","certified","frontier"}: raise ValueError("stress_policy must be complete, certified or frontier") if stress_policy in ("certified","frontier") and (stress_count==0 or not np.isfinite(search.robustness_weight) or search.robustness_weight<0): raise ValueError("Certified bounds require stress_count>0 and finite nonnegative robustness_weight") search.stress_policy = stress_policy search.reporter=SearchConfusion(search.valid.y,search.train.n_classes) if stress_policy=="frontier" else search_report search.incumbent = search.batch_bound = None search.pruned_count = search.skipped_views = search.rows_seen = 0 if mode not in ("exact","turbo") or ridge_solver not in RIDGE_SOLVERS: raise ValueError("Unknown search execution mode or Ridge solver") if mode=="turbo" and ridge_solver!="original": raise ValueError("Turbo uses its fixed Ridge alpha") if cache_mb<0 or stress_cache_mb<0 or not isinstance(stress_count,int) or not 0<=stress_count<=4: raise ValueError("Invalid cache size or stress count") search.sequence=check_sequence(sequence) search.create_candidate=partial(make_candidate,mode=mode,budget=budget,ridge_solver=ridge_solver, sequence=sequence,calibration=search.valid) X=search.valid.X search.stress_count=stress_count search.stress_names=list(STRESS_TRANSFORMS)[:stress_count] search.stress_bundle=None search.stress_inputs={} remaining=stress_cache_mb*1024**2 if stress_count>1 and stress_count*len(X)<=1024 and stress_count*X.nbytes<=min(remaining,16*1024**2): arr=np.empty((stress_count*len(X),*X.shape[1:]),dtype=X.dtype) for i,name in enumerate(search.stress_names): arr[i*len(X):(i+1)*len(X)]=STRESS_TRANSFORMS[name](X) arr.flags.writeable=False search.stress_bundle=arr else: for name in search.stress_names: if X.nbytes<=remaining: transformed=STRESS_TRANSFORMS[name](X) transformed.flags.writeable=False search.stress_inputs[name]=transformed remaining-=transformed.nbytes arrays=[search.train.X,X,*search.stress_inputs.values()] if search.stress_bundle is not None:arrays.append(search.stress_bundle) search.memory=SearchFeatures(cache_mb,arrays) return search def evaluate_recipes(specs,train,valid,*,mode="exact",budget=None,workers=1,n_jobs=1,seed=42, cost_weight=.01,robustness_weight=.1,cache_mb=192,stress_cache_mb=128, ridge_solver="original",stress_count=4,sequence="subject"): """Evaluate an explicit ordered sequence with fresh split-local caches.""" check_splits(train,valid) if min(workers,n_jobs)<1:raise ValueError("Worker counts must be positive") controller=_AcceleratedSearch(train,valid,".",seed=seed,candidate_workers=workers, n_jobs_per_candidate=n_jobs,cost_weight=cost_weight,robustness_weight=robustness_weight) _configure(controller,mode,budget,ridge_solver,cache_mb,stress_cache_mb,stress_count,sequence=sequence) specs=[s if isinstance(s,CandidateSpec) else CandidateSpec.from_dict(s) for s in specs] # Preserve recorded effective candidate parameters; supply missing job settings. specs=[controller._runtime_spec(s) if "n_jobs" not in s.params else s for s in specs] rows=controller._evaluate_many(specs) stats=controller.memory.stats() controller.memory.clear() return rows, stats def run_search(method, train, valid, test=None, *, output, population, generations, max_evaluations, seed=42, candidate_workers=1, n_jobs_per_candidate=1, ridge_solver="original", sequence="subject", cost_weight=0.01, robustness_weight=0.1, cache_mb=192, stress_cache_mb=128): """Run WISP-Random or WISP-Evolution with explicit population/generations/budget. Selection uses validation only. With ``test``, the winner is refitted once on train+validation and scored on test (``selected_model.pkl``, ``locked_test_*``). """ check_splits(*[part for part in (train, valid, test) if part is not None]) profile = replace(get_profile(method), max_evaluations=int(max_evaluations)) search = _AcceleratedSearch(train=train, valid=valid, output=output, profile=profile, test=test, population=population, generations=generations, seed=seed, cost_weight=cost_weight, robustness_weight=robustness_weight, candidate_workers=candidate_workers, n_jobs_per_candidate=n_jobs_per_candidate) _configure(search, "exact", None, ridge_solver, cache_mb, stress_cache_mb, 4 if profile.use_stress else 0, "complete", sequence=sequence) try: summary = search.run() finally: search.memory.clear() return summary def fit_search(method,train,valid,*,output,mode="exact",seed=42,evaluations=None,workers=1, n_jobs=1,population=24,cost_weight=0.01,robustness_weight=0.1, stress_cache_mb=128,budget=None,cache_mb=192,ridge_solver="original", refit=True,stress_count=4,family_scope="all",stress_policy="complete",sequence="subject"): """Search with the paper's default budget (329 proposals) and population, then refit. Same proposal grammar and default stress/utility as the search controller. Training and clean prediction times include actual shared-cache costs, so rankings may change.""" check_splits(train,valid) if min(workers,n_jobs)<1 or population<3:raise ValueError("Invalid parallelism or population") scopes={"all":None, "cis":("state_rocket_interval_hmm",), "cis-related":("state_rocket_interval_hmm","rocket_sketch_ridge_hmm","posterior_cpu_ensemble")} if family_scope not in scopes:raise ValueError("family_scope must be all, cis or cis-related") profile=replace(get_profile(method),allowed_cpu_families=scopes[family_scope]) limit=(profile.max_evaluations if mode=="exact" else 25) if evaluations is None else evaluations if not isinstance(limit,int) or limit<1:raise ValueError("evaluations must be a positive integer") out=Path(output) out.mkdir(parents=True,exist_ok=True) if any(out.iterdir()):raise ValueError("Search requires a fresh empty output directory per run") per_generation=max(population-max(2,population//3),1) generations=max(1,(limit-min(9,limit)+per_generation-1)//per_generation) search=_AcceleratedSearch(train,valid,out,profile=replace(profile,max_evaluations=limit), population=population,generations=generations,seed=seed,candidate_workers=workers, n_jobs_per_candidate=n_jobs,cost_weight=cost_weight,robustness_weight=robustness_weight) _configure(search,mode,budget,ridge_solver,cache_mb,stress_cache_mb,stress_count,stress_policy,sequence) summary=search.run() if summary["n_evaluated"]!=limit:raise RuntimeError("Controller did not complete evaluation budget") stats=search.memory.stats() search.memory.clear() model=None if refit: model=search.create_candidate(summary["best"]["spec"]) model.__dict__.pop("calibration_data",None) fit_on(model,concatenate_har(train,valid,source="train+validation")) model.set_confidence_scale(summary["best"]["result"].get("confidence_scale")) summary.update(backend="wisp-fast-shared",mode=mode,ridge_solver=ridge_solver,sequence=sequence, family_scope=family_scope, fit_scope="train+validation" if refit else "candidate selection only",selection_uses_test=False, cached_stress=search.stress_names if search.stress_bundle is not None else list(search.stress_inputs), stress_count=stress_count,stress_policy=stress_policy,pruned_candidates=search.pruned_count, skipped_stress_views=search.skipped_views,batched_stress=search.stress_bundle is not None,feature_cache=stats, evaluations=limit,winner_recipe=summary["best"]["spec"], effective_budget=(vars(budget) if budget is not None else "mode defaults")) (out/"summary.json").write_text(json.dumps(summary,indent=2,default=str)) return model, summary