"""Grouped transition counting and Viterbi decoding on the release class axis. Two ways to form the sequences that the HMM smoother decodes: ``subject`` All windows of one participant (the ``subject``/``groups`` value) form one chain, ordered by ``time_index``. This is the original WISP behaviour. ``recording`` Each recording forms its own chain. Within one participant's rows, in the order they are supplied, a new recording starts whenever ``time_index`` goes backwards. Classifier scores are rescaled by one confidence factor chosen on validation data (``fit_confidence_scale``), and smoothing is skipped when most training recordings contain a single activity, because then there are no within-recording transitions to learn. """ from functools import lru_cache import ctypes from .native import lookup import numpy as np SEQUENCES = ("subject", "recording") # Smoothing is skipped when more than this fraction of training recordings has one label. SINGLE_LABEL_RECORDING_LIMIT = 0.5 # Candidate confidence-scaling factors, searched on validation negative log-likelihood. CONFIDENCE_SCALE_GRID = np.exp(np.linspace(np.log(0.01), np.log(10.0), 121)) def check_sequence(sequence): if sequence not in SEQUENCES: raise ValueError(f"sequence must be one of {', '.join(SEQUENCES)}; got {sequence!r}") return sequence def recording_ids(groups, time_index): """Split each participant into recordings at every backward step of time_index. Rows keep the order in which they are supplied; the k-th backward step within a participant starts recording k. Returns string IDs ``"|"``. """ g = np.asarray(groups).astype(str) if time_index is None: return g t = np.asarray(time_index, dtype=float) if t.shape != g.shape: raise ValueError("time_index must have one entry per window") part = np.zeros(len(t), dtype=np.int64) for subject in np.unique(g): rows = np.flatnonzero(g == subject) part[rows] = np.concatenate([[0], np.cumsum(np.diff(t[rows]) < 0)]) return np.char.add(np.char.add(g, "|"), part.astype(str)) def single_label_fraction(y, recordings): """Fraction of recordings with at least two windows whose windows all share one label.""" _, inverse, counts = np.unique(recordings, return_inverse=True, return_counts=True) pairs = np.unique(np.stack([inverse.ravel(), np.asarray(y, dtype=np.int64)]), axis=1) labels_per_recording = np.bincount(pairs[0], minlength=len(counts)) eligible = counts > 1 if not eligible.any(): return 1.0 return float(1 - np.mean(labels_per_recording[eligible] > 1)) def scale_confidence(proba, scale): """Divide log-probabilities by ``scale`` and renormalise (scale 1 leaves them unchanged).""" P = np.asarray(proba, dtype=np.float64) if scale == 1.0: return P Z = np.log(np.clip(P, 1e-12, 1.0)) / scale Z -= Z.max(axis=1, keepdims=True) Q = np.exp(Z) return Q / Q.sum(axis=1, keepdims=True) def fit_confidence_scale(proba, y): """Choose the factor on CONFIDENCE_SCALE_GRID with the lowest validation NLL.""" L = np.log(np.clip(np.asarray(proba, dtype=np.float64), 1e-12, 1.0)) y = np.asarray(y, dtype=int) best = (np.inf, 1.0) for scale in CONFIDENCE_SCALE_GRID: Z = L / scale Z -= Z.max(1, keepdims=True) Q = np.exp(Z) Q /= Q.sum(1, keepdims=True) nll = -float(np.mean(np.log(np.clip(Q[np.arange(len(y)), y], 1e-12, 1.0)))) if nll < best[0]: best = (nll, float(scale)) return best[1] @lru_cache(maxsize=1) def _native(): fn = lookup("wisp_viterbi") if fn is None: return None arr = np.ctypeslib.ndpointer f64 = arr(dtype=np.float64, flags="C_CONTIGUOUS") i32 = arr(dtype=np.int32, flags="C_CONTIGUOUS") i64 = arr(dtype=np.int64, flags="C_CONTIGUOUS") fn.argtypes = [f64, f64, f64, ctypes.c_int64, ctypes.c_int64, i32, f64, f64, i64] fn.restype = None return fn @lru_cache(maxsize=1) def _native_groups(): fn = lookup("wisp_viterbi_groups") if fn is not None: arr = np.ctypeslib.ndpointer f64 = arr(dtype=np.float64, flags="C_CONTIGUOUS") i32 = arr(dtype=np.int32, flags="C_CONTIGUOUS") i64 = arr(dtype=np.int64, flags="C_CONTIGUOUS") fn.argtypes = [f64, f64, f64, i64, ctypes.c_int64, ctypes.c_int64, i32, f64, f64, i64] fn.restype = None return fn def _order(n, groups, time_index): """Preserve original order within a group when timestamps are absent or tied.""" if groups is None: return np.arange(n), np.array([0, n], dtype=np.intp) g = np.asarray(groups) if g.shape != (n,): raise ValueError("groups must have one entry per window") if np.any(g != g): raise ValueError("Missing group ID") if time_index is None: order = np.argsort(g, kind="stable") else: time = np.asarray(time_index) if time.shape != (n,): raise ValueError("time_index must have one entry per window") order = np.lexsort((time, g)) sorted_groups = g[order] bounds = np.r_[0, np.flatnonzero(sorted_groups[1:] != sorted_groups[:-1]) + 1, n] return order, bounds class FastSmoother: """HMM smoother. ``sequence`` selects how windows are grouped into chains (see module docs).""" def __init__(self, sequence="subject"): self.sequence = check_sequence(sequence) @property def active(self): """False when recording-level smoothing was skipped at fit time.""" return getattr(self, "active_", True) @property def confidence_scale(self): return getattr(self, "confidence_scale_", 1.0) def set_confidence_scale(self, scale): scale = float(scale) if not np.isfinite(scale) or scale <= 0: raise ValueError("confidence scale must be finite and positive") self.confidence_scale_ = scale return self def fit(self, y, *, groups=None, time_index=None, n_classes=None): y = np.asarray(y, dtype=np.int64) k = int(n_classes) if n_classes is not None else int(y.max()) + 1 if y.ndim != 1 or y.size == 0 or np.any((y < 0) | (y >= k)): raise ValueError("Invalid y or n_classes") if getattr(self, "sequence", "subject") == "recording": self.confidence_scale_ = 1.0 self.active_ = True self.single_label_fraction_ = None if groups is not None: rec = recording_ids(groups, time_index) self.single_label_fraction_ = single_label_fraction(y, rec) self.active_ = self.single_label_fraction_ <= SINGLE_LABEL_RECORDING_LIMIT groups = rec A = np.ones((k, k), dtype=np.float64) if groups is not None: order, bounds = _order(len(y), groups, time_index) labels = y[order] edges = labels[:-1] * k + labels[1:] valid = np.ones(edges.size, dtype=bool) valid[bounds[1:-1] - 1] = False A += np.bincount(edges[valid], minlength=k*k).reshape(k, k) # Upstream adds persistence twice for ungrouped training and once otherwise. A.flat[::k + 1] += 4.0 if groups is None else 2.0 p = np.bincount(y, minlength=k).astype(np.float64) + 1.0 self.transition_ = A / A.sum(axis=1, keepdims=True) self.prior_ = p / p.sum() self.log_A_ = np.log(np.clip(self.transition_, 1e-12, 1.0)) self.log_pi_ = np.log(np.clip(self.prior_, 1e-12, 1.0)) return self def predict_from_proba(self, proba, *, groups=None, time_index=None): P = np.asarray(proba, dtype=np.float64) if P.ndim != 2 or P.shape[1] != len(self.prior_) or not np.isfinite(P).all(): raise ValueError("Invalid emissions") n, k = P.shape if n == 0: return np.empty(0, dtype=np.int64) if getattr(self, "sequence", "subject") == "recording": if not self.active: return P.argmax(1).astype(np.int64) P = scale_confidence(P, self.confidence_scale) if groups is not None: groups = recording_ids(groups, time_index) order, bounds = _order(n, groups, time_index) return self._decode_ordered(P,order,bounds) def _decode_ordered(self,P,order,bounds): n,k=P.shape P = np.clip(P[order], 1e-12, 1.0) P /= P.sum(axis=1, keepdims=True) E = np.ascontiguousarray(np.log(np.clip(P, 1e-12, 1.0))) back = np.empty((np.diff(bounds).max(), k), dtype=np.int32) prev, nxt = np.empty(k), np.empty(k) labels = np.empty(n, dtype=np.int64) grouped = _native_groups() if _native() is not None else None if grouped is not None: grouped(E, self.log_A_, self.log_pi_, np.ascontiguousarray(bounds, dtype=np.int64), len(bounds)-1, k, back, prev, nxt, labels) out = np.empty_like(labels) out[order] = labels return out fn = _native() for lo, hi in zip(bounds[:-1], bounds[1:]): if fn is not None: fn(E[lo:hi], self.log_A_, self.log_pi_, hi-lo, k, back, prev, nxt, labels[lo:hi]) else: prev[:] = self.log_pi_ + E[lo] cols = np.arange(k) for i in range(lo + 1, hi): scores = prev[:, None] + self.log_A_ pred = np.argmax(scores, axis=0) back[i-lo] = pred prev[:] = scores[pred, cols] + E[i] labels[hi-1] = np.argmax(prev) for i in range(hi-2, lo-1, -1): labels[i] = back[i-lo+1, labels[i+1]] out = np.empty_like(labels) out[order] = labels return out