"""Search-scoped reuse of deterministic representation components on immutable splits.""" from collections import OrderedDict from copy import copy import hashlib from threading import RLock import weakref from functools import lru_cache import numpy as np from wisp.cpu import features as ref from . import features as fx @lru_cache(maxsize=128) def _dyadic(time): # Mirrors the release order, including deduplication for short windows. bounds = [] for parts in (2, 4, 8): width = max(2, (time+parts-1)//parts) bounds.extend((lo,min(lo+width,time)) for lo in range(0,time,width) if min(lo+width,time)-lo >= 2) return tuple(dict.fromkeys(bounds)) class SearchFeatures: """A private per-run LRU; share only deterministic blocks, never train labels or heads.""" def __init__(self, limit_mb, arrays): self.limit = int(limit_mb * 1024**2) self.known = {id(arr): (arr, i) for i, arr in enumerate(arrays)} self.derived = {} self.store = OrderedDict() # Tile data have much greater cardinality than the stable feature # blocks. Keep a separate bounded LRU to avoid flushing the hot set. self.tiles = OrderedDict() self.tile_limit = self.limit // 8 self.tile_bytes = self.tile_hits = self.rank_hits = self.dyad_hits = 0 self.bytes = self.peak = self.hits = self.misses = self.evictions = 0 self.lock = RLock() def clear(self): self.store.clear() self.known.clear() self.derived.clear() self.tiles.clear() self.tile_bytes = 0 self.bytes = 0 def stats(self): return dict(limit_bytes=self.limit, peak_bytes=self.peak, hits=self.hits, misses=self.misses, evictions=self.evictions, shapelet_tile_hits=self.tile_hits, interval_rank_hits=self.rank_hits, dyadic_block_hits=self.dyad_hits) def token(self, x): val = self.known.get(id(x)) if val is not None and x is val[0]: return val[1] with self.lock: entry = self.derived.get(id(x)) if entry is not None and entry[0]() is x: return entry[1] # Derived arrays (e.g. train-row cap and temporal downsampling) must # never be keyed by shape alone. Their allocation may change each fit. return (x.shape, x.dtype.str, x.strides, hashlib.blake2b(np.ascontiguousarray(x).view(np.uint8), digest_size=16).digest()) def sampled(self, x, steps, sample): key = (self.token(x),"time_sample",steps) arr = self.get(key,sample) # Weak entries tie identity to live allocations, even after eviction. idx = id(arr) def expired(ref): with self.lock: item = self.derived.get(idx) if item is not None and item[0] is ref: self.derived.pop(idx) with self.lock: self.derived[idx] = (weakref.ref(arr,expired),key) return arr def get(self, key, fn, *, tile=False): store = self.tiles if tile else self.store limit = self.tile_limit if tile else self.limit - self.tile_limit with self.lock: if key in store: self.hits += 1 if tile:self.tile_hits += 1 if key[1]=="interval_rank":self.rank_hits += 1 if key[1]=="dyad":self.dyad_hits += 1 store.move_to_end(key) return store[key] self.misses += 1 arr = fn() if arr.nbytes <= limit: with self.lock: if key not in store: used = self.tile_bytes if tile else self.bytes while used + arr.nbytes > limit: _, old = store.popitem(last=False) used -= old.nbytes self.evictions += 1 store[key] = arr used += arr.nbytes if tile:self.tile_bytes = used else:self.bytes = used self.peak = max(self.peak, self.bytes+self.tile_bytes) return arr def transform(self, blocks, x): token = self.token(x) def zscore(): return self.get((token, "z"), lambda: fx._normalize(ref._as_3d(x))) def interval(block, data, norm): def native_plan(b, rank, evaluate): order = self.get((token,"interval_rank",norm),rank) fixed = _dyadic(data.shape[1]) if b.include_dyadic else () if not fixed or tuple(b.intervals_[:len(fixed)]) != fixed: return evaluate(b.intervals_,order) params = (norm,b.fast_profile,tuple(b.quantiles),fixed) common = self.get((token,"dyad",params),lambda:evaluate(fixed,order)) suffix = b.intervals_[len(fixed):] return np.concatenate((common,evaluate(suffix,order)),axis=1) if suffix else common return block.transform_normalized(data,native_plan=native_plan) features=[] for b in blocks: if type(b) is fx.WearableFeatureExtractor: # Stateless window-local features. Constructor parameters fully # specify columns, including turbo choices. Fitted names are # cheap to copy and restored on warm hits for pickle-size parity. params=tuple(b.get_params(deep=False).items()) key=(token, "wear", params) def calculate(): return b.transform_normalized(zscore()) if x.flags.c_contiguous and b.zscore else b.transform(x) value=self.get(key,calculate) if not b.feature_names_: b.transform(x[:1]) elif type(b) is fx.InvariantPhysicsFeatureExtractor and x.flags.c_contiguous and b.interval_ is not None: # Upstream concatenates deterministic physics with a separate # seeded random-interval branch. Share the former across seeds. key=(token,"physics",b.segment_sizes,b.symbolic_bins,b.zscore,b.fast_profile,b.include_axis_features) def fixed(): base=copy(b) base.interval_=None return base.transform(x) common=self.get(key,fixed) xn=zscore() if b.zscore else np.nan_to_num(ref._as_3d(x)) if b.interval_.zscore: xn=self.get((token,"physics_second_z",b.zscore),lambda:fx._normalize(xn)) intervals=interval(b.interval_,xn,("physics",b.zscore,b.interval_.zscore)) value=np.concatenate((common,intervals),axis=1) elif type(b) is fx.IntervalDistributionSketch and x.flags.c_contiguous and b.zscore: value=interval(b,zscore(),"z") elif type(b) is fx.RandomShapeletSketch and x.flags.c_contiguous and b.zscore: def tile(key, fn): return self.get((token,"shapelet_windows",key),fn,tile=True) value=b.transform_normalized(zscore(),window_tile=tile) elif type(b) is fx.RandomConvSketch and x.flags.c_contiguous and b.zscore: value=b.transform_normalized(zscore()) else: # Fitted shapelets depend on actual training examples; they # are deliberately evaluated with the candidate's own object. value=b.transform(x) features.append(value) capture=getattr(self,"capture",None) if capture is not None:capture(x,blocks,features) return features