Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
WISP / src /wisp_fast /cache.py
Zipeng365's picture
wisp-har 0.2.0: accelerated implementation with speed (exact/fast) and HMM sequence (subject/recording) options
ac7f7cb verified
Raw History Blame Contribute Delete
3.83 kB
"""Opt-in numeric feature cache. Keys include actual signals and fitted block state."""
from contextlib import contextmanager
from contextvars import ContextVar
from functools import lru_cache
import hashlib
import os
from pathlib import Path
import platform
import tempfile
import joblib
import numpy as np
import scipy
_SEARCH = ContextVar("wisp_search_memory", default=None)
_ACTIVE = ContextVar("wisp_feature_store", default=None)
@lru_cache(maxsize=1)
def _backend():
from wisp.cpu import features as original
from . import features as fast
digest = hashlib.sha256()
from importlib.util import find_spec
native = [find_spec("wisp_fast."+key) for key in ("_kernels","_cis_conv","_cis_intervals","_family_kernels")]
for path in [original.__file__, fast.__file__, __file__]+[s.origin for s in native if s is not None]:
if path:
digest.update(Path(path).read_bytes())
digest.update(repr((np.__version__,scipy.__version__,joblib.__version__, platform.machine(),
platform.processor())).encode())
return digest.hexdigest()
@contextmanager
def feature_cache(directory, *, max_entry_mb=512):
"""Reuse exact NPY feature matrices. Use a private, same-host cache directory.
Caller controls the directory lifetime. Bound any single entry with max_entry_mb.
Atomic replacement supports racing workers. There is no cross-filesystem lock.
"""
if directory is None:
yield
return
if max_entry_mb <= 0:
raise ValueError("max_entry_mb must be positive")
path = Path(directory) / _backend()
path.mkdir(parents=True, exist_ok=True)
token = _ACTIVE.set((path, max_entry_mb*1024**2))
try:
yield
finally:
_ACTIVE.reset(token)
def transform_blocks(blocks, X):
config = _ACTIVE.get()
shared_search = _SEARCH.get()
if shared_search is not None and config is None:
return shared_search.transform(blocks,X)
if config is None:
shared = (len(blocks) >= 2 and X.flags.c_contiguous and
all(getattr(b,"zscore",False) and hasattr(b,"transform_normalized") for b in blocks))
if shared:
from wisp.cpu.features import _as_3d
from .features import _normalize
z = _normalize(_as_3d(X))
return [b.transform_normalized(z) for b in blocks]
return [block.transform(X) for block in blocks]
directory, limit = config
signal = hashlib.sha256(np.ascontiguousarray(X).view(np.uint8)).digest()
features = []
for block in blocks:
# feature_names_ is display metadata produced during transform. Every
# fit-dependent quantity, including shapelet values, remains in the key.
state = {k:v for k,v in vars(block).items() if k not in ("feature_names_","_conv_plan","_quant_plan")}
key = joblib.hash((type(block).__module__, type(block).__qualname__, state,
X.dtype.str,X.shape,X.strides,signal),hash_name="sha1")
path = directory / (key+".npy")
try:
arr = np.load(path, allow_pickle=False, mmap_mode="r")
except FileNotFoundError:
arr = block.transform(X)
if arr.nbytes <= limit:
tmp = None
try:
with tempfile.NamedTemporaryFile(dir=directory, delete=False) as stream:
tmp = stream.name
np.save(stream, arr, allow_pickle=False)
os.replace(tmp,path)
finally:
if tmp and os.path.exists(tmp):
os.unlink(tmp)
else:
if hasattr(block, "feature_names_") and not block.feature_names_:
block.transform(X[:1])
features.append(arr)
return features