Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / src /wisp_fast /search.py
Zipeng365's picture
wisp-har 0.2.0: accelerated implementation with speed (exact/fast) and HMM sequence (subject/recording) options
ac7f7cb verified
Raw History Blame Contribute Delete
17 kB
"""WISP-Random / WISP-Evolution search with split-local feature sharing.
The proposal grammar, cascade, stress views and utility are those of
``wisp.search.controller``; candidates are fitted with the accelerated engine.
"""
from concurrent.futures import ThreadPoolExecutor
from contextvars import copy_context
from dataclasses import replace
from functools import partial
import json
import pickle
from pathlib import Path
import time
import numpy as np
from wisp.core.data import concatenate_har
from wisp.core.metrics import classification_report_dict
from wisp.cpu.algorithms import estimate_pickle_size_mb
from wisp.search.cascade import STRESS_TRANSFORMS, evaluate_candidate_cascade
from wisp.search.controller import WISPSearchController
from wisp.search.grammar import CandidateSpec
from wisp.search.profiles import get_profile
from .cache import _SEARCH
from .decoder import check_sequence
from .engine import RIDGE_SOLVERS, build_candidate
from .selection import check_splits, fit_on
from .search_cache import SearchFeatures
from .search_metrics import search_report,SearchConfusion
def make_candidate(spec_dict, *, mode="exact", budget=None, ridge_solver="original", sequence="subject",
calibration=None):
"""Accelerated estimator for one search candidate.
``calibration`` (a validation HARData) is used only with ``sequence="recording"``:
right after fitting, the confidence scaling is chosen on those windows.
"""
if ridge_solver not in RIDGE_SOLVERS or (mode!="exact" and ridge_solver!="original"):
raise ValueError("The screened Ridge solver requires exact full-size features")
model=build_candidate(spec_dict,mode=mode,budget=budget,ridge_solver=ridge_solver,sequence=sequence)
if sequence=="recording" and calibration is not None:
model.calibration_data=(calibration.X,calibration.y)
return model
class _AcceleratedSearch(WISPSearchController):
def _evaluate_many(self, specs):
# Random proposals are score-independent. Evolution may prune only after
# generating its terminal batch; every breeding decision is then complete.
terminal = self.rows_seen+len(specs) == self.profile.max_evaluations
eligible = self.profile.search_strategy == "random" or terminal
self.batch_bound = self.incumbent if self.stress_policy=="certified" and eligible else None
# Update only between completed waves. Certificates always reference
# earlier canonical ledger rows, independent of worker completion order.
if self.stress_policy=="frontier" and eligible:
rows=[]
width=max(self._worker_count(),1)
for start in range(0,len(specs),width):
self.batch_bound=self.incumbent
chunk=self._evaluate_batch(specs[start:start+width])
self._observe(chunk,self.rows_seen+start)
rows.extend(chunk)
else:
rows = self._evaluate_batch(specs)
self._observe(rows,self.rows_seen)
self.rows_seen += len(rows)
return rows
def _observe(self,rows,start):
for index,(status,result) in enumerate(rows,start):
if status == "ok":
score = result["predictive_utility"]
if self.incumbent is None or score > self.incumbent[0]:
self.incumbent = (score,index)
elif status == "pruned":
self.pruned_count += 1
self.skipped_views += result["prune_certificate"]["skipped_views"]
def _stress_gate(self, metrics, observed):
# Macro-F1 <= 1. The stress contribution w*s - .05*max(clean-s,0)
# is monotone on [0,1] for w>=0. No equality pruning, so ties survive.
score,index = self.batch_bound
remaining = self.stress_count-len(observed)
mean_upper = float(np.mean(observed+[1.0]*remaining)) if observed else 1.0
gap_lower = max(float(metrics["macro_f1"])-mean_upper,0.0)
upper = float(metrics["score"] + self.robustness_weight*mean_upper - .05*gap_lower)
if np.isfinite(upper) and upper < score:
return dict(upper_bound=upper,incumbent_utility=score,
incumbent_ledger_index=index,stress_f1_upper=mean_upper,
observed_stress_f1=list(observed),skipped_views=remaining)
return None
def _evaluate_batch(self,specs):
if self._worker_count() == 1 or len(specs)<2:
return super()._evaluate_many(specs)
with ThreadPoolExecutor(max_workers=min(self._worker_count(),len(specs))) as pool:
jobs=[pool.submit(copy_context().run,self._evaluate_one,s) for s in specs]
return [job.result() for job in jobs]
def _batched_stress(self, model):
# Each stress view is a distinct sequence panel. Never Viterbi-decode
# a concatenated panel, even if the same subject IDs appear in each.
P=model.predict_proba(self.stress_bundle)
chunks=np.split(P,self.stress_count)
return {name:model.decode_proba(p,groups=self.valid.subject,time_index=self.valid.time_index)
for name,p in zip(self.stress_names,chunks)}
def _evaluate_one(self, spec):
token=_SEARCH.set(self.memory if self.memory.limit else None)
built=[]
def factory(spec_dict):
built.append(self.create_candidate(spec_dict))
return built[-1]
try:
cascade=evaluate_candidate_cascade(
spec,self.train,self.valid,estimator_factory=factory,
cost_weight=self.cost_weight,robustness_weight=self.robustness_weight,
max_stress_transforms=self.stress_count,stress_inputs=self.stress_inputs,
reporter=self.reporter,stress_gate=self._stress_gate if self.batch_bound is not None else None,
stress_predictor=self._batched_stress if self.stress_bundle is not None else None)
result=cascade.to_dict()
if self.sequence=="recording" and built:
smoother=getattr(built[-1],"smoother_",None)
result["sequence"]="recording"
result["sequence_smoothing"]=bool(smoother is not None and smoother.active)
result["confidence_scale"]=built[-1].confidence_scale
if "prune_certificate" in result:
del result["cascade_score"]
del result["accepted"] # stress and final eligibility were not evaluated
return "pruned",result
result["predictive_utility"]=float(result["cascade_score"])
return ("ok" if cascade.accepted else "rejected",result)
except Exception as exc:
return ("failed",{"predictive_utility":-1e9,"breeding_score":-1e9,
"error":repr(exc),"candidate":spec.to_dict()})
finally:
_SEARCH.reset(token)
def _final_refit(self, best, out_dir):
"""Refit the validation winner on train+validation with the accelerated engine."""
if self.test is None:
return None
model = self.create_candidate(best["spec"])
model.__dict__.pop("calibration_data", None)
refit = concatenate_har(self.train, self.valid, source="train+validation")
t0 = time.perf_counter()
fit_on(model, refit)
fit_s = time.perf_counter() - t0
# A train+validation refit reuses the scaling chosen for the train-only candidate.
model.set_confidence_scale(best["result"].get("confidence_scale"))
t1 = time.perf_counter()
pred = model.predict(self.test.X, groups=self.test.subject, time_index=self.test.time_index)
pred_s = time.perf_counter() - t1
metrics = classification_report_dict(self.test.y, pred, fit_seconds=fit_s, predict_seconds=pred_s,
model_size_mb=estimate_pickle_size_mb(model),
label_names=self.test.label_names)
np.savez_compressed(out_dir / "locked_test_predictions.npz", y_true=self.test.y, y_pred=pred,
subject=self.test.subject, time_index=self.test.time_index)
with (out_dir / "selected_model.pkl").open("wb") as f:
pickle.dump(model, f)
(out_dir / "locked_test_metrics.json").write_text(
json.dumps(metrics, indent=2, ensure_ascii=False, default=str), encoding="utf-8")
return metrics
def _configure(search, mode, budget, ridge_solver, cache_mb, stress_cache_mb, stress_count,
stress_policy="complete", sequence="subject"):
if stress_policy not in {"complete","certified","frontier"}:
raise ValueError("stress_policy must be complete, certified or frontier")
if stress_policy in ("certified","frontier") and (stress_count==0 or not np.isfinite(search.robustness_weight)
or search.robustness_weight<0):
raise ValueError("Certified bounds require stress_count>0 and finite nonnegative robustness_weight")
search.stress_policy = stress_policy
search.reporter=SearchConfusion(search.valid.y,search.train.n_classes) if stress_policy=="frontier" else search_report
search.incumbent = search.batch_bound = None
search.pruned_count = search.skipped_views = search.rows_seen = 0
if mode not in ("exact","turbo") or ridge_solver not in RIDGE_SOLVERS:
raise ValueError("Unknown search execution mode or Ridge solver")
if mode=="turbo" and ridge_solver!="original":
raise ValueError("Turbo uses its fixed Ridge alpha")
if cache_mb<0 or stress_cache_mb<0 or not isinstance(stress_count,int) or not 0<=stress_count<=4:
raise ValueError("Invalid cache size or stress count")
search.sequence=check_sequence(sequence)
search.create_candidate=partial(make_candidate,mode=mode,budget=budget,ridge_solver=ridge_solver,
sequence=sequence,calibration=search.valid)
X=search.valid.X
search.stress_count=stress_count
search.stress_names=list(STRESS_TRANSFORMS)[:stress_count]
search.stress_bundle=None
search.stress_inputs={}
remaining=stress_cache_mb*1024**2
if stress_count>1 and stress_count*len(X)<=1024 and stress_count*X.nbytes<=min(remaining,16*1024**2):
arr=np.empty((stress_count*len(X),*X.shape[1:]),dtype=X.dtype)
for i,name in enumerate(search.stress_names):
arr[i*len(X):(i+1)*len(X)]=STRESS_TRANSFORMS[name](X)
arr.flags.writeable=False
search.stress_bundle=arr
else:
for name in search.stress_names:
if X.nbytes<=remaining:
transformed=STRESS_TRANSFORMS[name](X)
transformed.flags.writeable=False
search.stress_inputs[name]=transformed
remaining-=transformed.nbytes
arrays=[search.train.X,X,*search.stress_inputs.values()]
if search.stress_bundle is not None:arrays.append(search.stress_bundle)
search.memory=SearchFeatures(cache_mb,arrays)
return search
def evaluate_recipes(specs,train,valid,*,mode="exact",budget=None,workers=1,n_jobs=1,seed=42,
cost_weight=.01,robustness_weight=.1,cache_mb=192,stress_cache_mb=128,
ridge_solver="original",stress_count=4,sequence="subject"):
"""Evaluate an explicit ordered sequence with fresh split-local caches."""
check_splits(train,valid)
if min(workers,n_jobs)<1:raise ValueError("Worker counts must be positive")
controller=_AcceleratedSearch(train,valid,".",seed=seed,candidate_workers=workers,
n_jobs_per_candidate=n_jobs,cost_weight=cost_weight,robustness_weight=robustness_weight)
_configure(controller,mode,budget,ridge_solver,cache_mb,stress_cache_mb,stress_count,sequence=sequence)
specs=[s if isinstance(s,CandidateSpec) else CandidateSpec.from_dict(s) for s in specs]
# Preserve recorded effective candidate parameters; supply missing job settings.
specs=[controller._runtime_spec(s) if "n_jobs" not in s.params else s for s in specs]
rows=controller._evaluate_many(specs)
stats=controller.memory.stats()
controller.memory.clear()
return rows, stats
def run_search(method, train, valid, test=None, *, output, population, generations, max_evaluations,
seed=42, candidate_workers=1, n_jobs_per_candidate=1, ridge_solver="original",
sequence="subject", cost_weight=0.01, robustness_weight=0.1, cache_mb=192,
stress_cache_mb=128):
"""Run WISP-Random or WISP-Evolution with explicit population/generations/budget.
Selection uses validation only. With ``test``, the winner is refitted once on
train+validation and scored on test (``selected_model.pkl``, ``locked_test_*``).
"""
check_splits(*[part for part in (train, valid, test) if part is not None])
profile = replace(get_profile(method), max_evaluations=int(max_evaluations))
search = _AcceleratedSearch(train=train, valid=valid, output=output, profile=profile, test=test,
population=population, generations=generations, seed=seed,
cost_weight=cost_weight, robustness_weight=robustness_weight,
candidate_workers=candidate_workers, n_jobs_per_candidate=n_jobs_per_candidate)
_configure(search, "exact", None, ridge_solver, cache_mb, stress_cache_mb,
4 if profile.use_stress else 0, "complete", sequence=sequence)
try:
summary = search.run()
finally:
search.memory.clear()
return summary
def fit_search(method,train,valid,*,output,mode="exact",seed=42,evaluations=None,workers=1,
n_jobs=1,population=24,cost_weight=0.01,robustness_weight=0.1,
stress_cache_mb=128,budget=None,cache_mb=192,ridge_solver="original",
refit=True,stress_count=4,family_scope="all",stress_policy="complete",sequence="subject"):
"""Search with the paper's default budget (329 proposals) and population, then refit.
Same proposal grammar and default stress/utility as the search controller. Training and
clean prediction times include actual shared-cache costs, so rankings may change."""
check_splits(train,valid)
if min(workers,n_jobs)<1 or population<3:raise ValueError("Invalid parallelism or population")
scopes={"all":None, "cis":("state_rocket_interval_hmm",),
"cis-related":("state_rocket_interval_hmm","rocket_sketch_ridge_hmm","posterior_cpu_ensemble")}
if family_scope not in scopes:raise ValueError("family_scope must be all, cis or cis-related")
profile=replace(get_profile(method),allowed_cpu_families=scopes[family_scope])
limit=(profile.max_evaluations if mode=="exact" else 25) if evaluations is None else evaluations
if not isinstance(limit,int) or limit<1:raise ValueError("evaluations must be a positive integer")
out=Path(output)
out.mkdir(parents=True,exist_ok=True)
if any(out.iterdir()):raise ValueError("Search requires a fresh empty output directory per run")
per_generation=max(population-max(2,population//3),1)
generations=max(1,(limit-min(9,limit)+per_generation-1)//per_generation)
search=_AcceleratedSearch(train,valid,out,profile=replace(profile,max_evaluations=limit),
population=population,generations=generations,seed=seed,candidate_workers=workers,
n_jobs_per_candidate=n_jobs,cost_weight=cost_weight,robustness_weight=robustness_weight)
_configure(search,mode,budget,ridge_solver,cache_mb,stress_cache_mb,stress_count,stress_policy,sequence)
summary=search.run()
if summary["n_evaluated"]!=limit:raise RuntimeError("Controller did not complete evaluation budget")
stats=search.memory.stats()
search.memory.clear()
model=None
if refit:
model=search.create_candidate(summary["best"]["spec"])
model.__dict__.pop("calibration_data",None)
fit_on(model,concatenate_har(train,valid,source="train+validation"))
model.set_confidence_scale(summary["best"]["result"].get("confidence_scale"))
summary.update(backend="wisp-fast-shared",mode=mode,ridge_solver=ridge_solver,sequence=sequence,
family_scope=family_scope,
fit_scope="train+validation" if refit else "candidate selection only",selection_uses_test=False,
cached_stress=search.stress_names if search.stress_bundle is not None else list(search.stress_inputs),
stress_count=stress_count,stress_policy=stress_policy,pruned_candidates=search.pruned_count,
skipped_stress_views=search.skipped_views,batched_stress=search.stress_bundle is not None,feature_cache=stats,
evaluations=limit,winner_recipe=summary["best"]["spec"],
effective_budget=(vars(budget) if budget is not None else "mode defaults"))
(out/"summary.json").write_text(json.dumps(summary,indent=2,default=str))
return model, summary