"""Data entry points: WristHARBench folds and windowing of continuous signals. ``load_wristharbench`` needs the optional extra ``[hub]`` (``datasets``, ``huggingface_hub``, ``pandas``) the first time a fold is exported; afterwards the cached NPZ files are read with NumPy only. """ from __future__ import annotations import json import os from dataclasses import dataclass, field from importlib import resources from pathlib import Path from typing import Any import numpy as np from .recipes import dataset_id as _dataset_id __all__ = ["load_wristharbench", "load_exported_fold", "make_windows", "Split", "BenchmarkFold", "REPO_ID", "REVISION"] REPO_ID = "Zipeng365/WristHARBench" REVISION = "5e6af3c5c6dca050e73c06f49272f9656c0f259f" @dataclass class Split: """One partition: windows ``X`` (n, T, C), integer labels ``y`` on the global vocabulary, ``groups`` (participant IDs) and ``time_index`` (position within each participant).""" X: np.ndarray y: np.ndarray groups: np.ndarray time_index: np.ndarray def __len__(self) -> int: return len(self.y) @dataclass class BenchmarkFold: dataset_id: str fold: int train: Split val: Split test: Split label_names: list[str] sampling_rate_hz: float | None = None source: dict[str, Any] = field(default_factory=dict) @property def valid(self) -> Split: return self.val @property def study_setting(self) -> str | None: """The stream's study setting from the package's WristHARBench metadata (e.g. ``"free_living"``).""" settings = json.loads(_bench_text("datasets.json")).get(self.dataset_id, {}) return settings.get("metadata", {}).get("study_setting") @property def train_val(self) -> Split: """Training and validation windows concatenated (the paper's final-fit data).""" a, b = self.train, self.val return Split(np.concatenate([a.X, b.X]), np.concatenate([a.y, b.y]), np.concatenate([a.groups, b.groups]), np.concatenate([a.time_index, b.time_index])) @property def classes(self) -> np.ndarray: """The global class table (integer IDs ``0..K-1``) to pass as ``classes=``.""" return np.arange(len(self.label_names)) def _bench_text(name: str) -> str: return resources.files("wisp").joinpath("_data", "wristharbench", name).read_text(encoding="utf-8") def _default_cache() -> Path: root = os.environ.get("WISP_DATA_DIR") if root: return Path(root) return Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "wisp-har" / "wristharbench" def _read_split(path: Path) -> tuple[Split, list[str], float | None]: with np.load(path, allow_pickle=False) as z: X = np.asarray(z["X"]) y = np.asarray(z["y"], dtype=np.int64) groups = np.asarray(z["subject"]).astype(str) time_index = np.asarray(z["time_index"], dtype=np.float64) names = [str(v) for v in z["label_names"].tolist()] if "label_names" in z.files else None rate = float(np.asarray(z["sampling_rate_hz"]).reshape(-1)[0]) if "sampling_rate_hz" in z.files else None return Split(X, y, groups, time_index), names, rate def load_exported_fold(root, dataset: str, fold: int = 0) -> BenchmarkFold: """Read an already exported fold: ``//fold/{train,valid,test}.npz``. Only numeric arrays and Unicode strings are read (no pickled objects). When the NPZ files carry no ``label_names`` (older exports), the fold's ``export_manifest.json`` is used. """ did = _dataset_id(dataset) folder = Path(root) / did / f"fold{int(fold)}" parts, names, rate = {}, None, None for part in ("train", "valid", "test"): try: parts[part], part_names, part_rate = _read_split(folder / f"{part}.npz") except ValueError: # object arrays present: read the numeric keys only with np.load(folder / f"{part}.npz", allow_pickle=False) as z: parts[part] = Split(np.asarray(z["X"]), np.asarray(z["y"], dtype=np.int64), np.asarray(z["subject"]).astype(str), np.asarray(z["time_index"], dtype=np.float64)) part_names, part_rate = None, None names = names or part_names rate = rate or part_rate if names is None: manifest = folder / "export_manifest.json" if manifest.exists(): names = list(json.loads(manifest.read_text())["label_names"]) if names is None: raise ValueError(f"No label vocabulary found for {folder}") if rate is None: settings = json.loads(_bench_text("datasets.json")).get(did, {}) rate = settings.get("sampling_rate_hz") return BenchmarkFold(did, int(fold), parts["train"], parts["valid"], parts["test"], names, rate, {"root": str(Path(root)), "format": "exported NPZ"}) def _preprocess_signal(signal): import pandas as pd a = np.stack(signal).astype(np.float64) a = pd.DataFrame(a).ffill().bfill().to_numpy(dtype=np.float64) if a.ndim != 2 or not np.isfinite(a).all(): raise ValueError("Non-finite signal after registered imputation") mu, sd = a.mean(0, keepdims=True), a.std(0, keepdims=True) sd[sd == 0] = 1 return (a - mu) / sd def _config_name(did: str) -> str: return did + "_paper_snapshot_20260828" if did == "capture24_wearable_activity_v1" else did def _export_fold(did: str, fold: int, output: Path, *, local_data=None, revision: str = REVISION) -> None: """Paper export contract (same procedure as ``scripts/export_benchmark.py``) for one fold.""" import pandas as pd scope = json.loads(_bench_text("scope.json")) if did not in scope["datasets"]: raise ValueError(f"Unknown paper dataset: {did}") settings = json.loads(_bench_text("datasets.json"))[did] if local_data is None: from datasets import load_dataset frame = load_dataset(REPO_ID, _config_name(did), split="full", revision=revision).to_pandas() else: files = sorted((Path(local_data) / "data" / _config_name(did)).glob("*.parquet")) if not files: raise FileNotFoundError(f"No Parquet files for {_config_name(did)} under {local_data}") frame = pd.concat([pd.read_parquet(f) for f in files], ignore_index=True) if len(frame) != settings["sequences"] or frame.series_id.duplicated().any(): raise ValueError("Input scope does not match the frozen paper contract") frame["series_id"] = frame.series_id.astype(str) frame["subject_id"] = frame.subject_id.astype(str) vocabulary = sorted(frame.label.astype(str).unique()) encoded = {label: i for i, label in enumerate(vocabulary)} with resources.files("wisp").joinpath("_data", "wristharbench", "ordering", f"{did}.csv.gz").open("rb") as fh: ordering = pd.read_csv(fh, compression="gzip", dtype={"series_id": str, "subject_id": str}) if set(ordering.series_id) != set(frame.series_id): raise ValueError("Ordering sidecar does not cover the downloaded rows") folder = output / did / f"fold{fold}" folder.mkdir(parents=True, exist_ok=True) groups = [] for split in ("train", "validation", "test"): side = ordering[(ordering.fold == fold) & (ordering.split == split)].sort_values("split_row") part = side.merge(frame, on=["series_id", "subject_id"], how="left", validate="one_to_one") if not part[f"fold_{fold}"].eq(split).all(): raise ValueError("Published split assignments disagree with the frozen export") groups.append(set(part.subject_id)) X = np.stack([_preprocess_signal(x) for x in part.signal]).astype(np.float32) if X.shape[1:] != (settings["window_steps"], len(settings["channels"])): raise ValueError(f"Unexpected shape for {did}: {X.shape}") name = "valid" if split == "validation" else split tmp = folder / f".{name}.tmp.npz" np.savez_compressed(tmp, X=X, y=np.array([encoded[str(x)] for x in part.label], dtype=np.int64), subject=part.subject_id.to_numpy(dtype=str), series_id=part.series_id.to_numpy(dtype=str), time_index=part.time_index.to_numpy(dtype=float), label_names=np.asarray(vocabulary, dtype=str), sampling_rate_hz=np.asarray([settings["sampling_rate_hz"]])) os.replace(tmp, folder / f"{name}.npz") if any(groups[i] & groups[j] for i, j in ((0, 1), (0, 2), (1, 2))): raise ValueError("Participant leakage across split partitions") (folder / "export_source.json").write_text(json.dumps( {"repo_id": REPO_ID, "revision": revision, "configuration": _config_name(did), "fold": fold, "local_data": local_data is not None, "label_names": vocabulary}, indent=1)) def load_wristharbench(dataset: str, fold: int = 0, *, cache_dir=None, local_data=None, revision: str = REVISION) -> BenchmarkFold: """One official WristHARBench fold (paper view, participant-disjoint train/val/test). The fold is exported once from the Hugging Face dataset ``Zipeng365/WristHARBench`` at the pinned revision (or from a local snapshot via ``local_data``) and cached under ``cache_dir`` (default ``$WISP_DATA_DIR`` or ``~/.cache/wisp-har/wristharbench``). Access follows that repository's terms; authenticate with your own account if needed. ``dataset`` accepts short names (``"adl"``) and full IDs. """ did = _dataset_id(dataset) if int(fold) not in range(5): raise ValueError("fold must be 0..4 (the five paper splits)") cache = Path(cache_dir) if cache_dir is not None else _default_cache() folder = cache / did / f"fold{int(fold)}" if not all((folder / f"{p}.npz").exists() for p in ("train", "valid", "test")): _export_fold(did, int(fold), cache, local_data=local_data, revision=revision) data = load_exported_fold(cache, did, fold) data.source = {"repo_id": REPO_ID, "revision": revision, "cache": str(folder)} return data # --------------------------------------------------------------------------- windowing def _runs(values: np.ndarray) -> list[tuple[int, int]]: """Contiguous runs [start, stop) of equal values.""" if len(values) == 0: return [] change = np.flatnonzero(values[1:] != values[:-1]) + 1 bounds = np.r_[0, change, len(values)] return [(int(a), int(b)) for a, b in zip(bounds[:-1], bounds[1:])] def _is_missing(labels: np.ndarray) -> np.ndarray: if labels.dtype.kind in "fc": return ~np.isfinite(labels) if labels.dtype == object: return np.array([v is None or (isinstance(v, float) and not np.isfinite(v)) for v in labels.tolist()]) return np.zeros(len(labels), dtype=bool) def make_windows(signal, *, fs: float, window_s: float, step_s: float, labels=None, group=None, label_rule: str = "majority", min_label_fraction: float = 0.8): """Cut a continuous signal into fixed-length windows. Parameters ---------- signal : array of shape (n_samples, C) or (n_samples,) fs : float Sampling rate in Hz. window_s, step_s : float Window length and step in seconds (required: datasets differ). They are converted to ``round(window_s * fs)`` and ``round(step_s * fs)`` samples. labels : array of shape (n_samples,), optional Per-sample annotations; ``None``/NaN entries are missing. group : scalar or array of shape (n_samples,), optional Sequence ID (participant or recording). Windows never cross a change of ``group``; each contiguous run of one value is windowed separately. label_rule : {"majority", "constant"} Window labelling following the WristHARBench export rules (docs/specifications/execution.tex of the benchmark): ``"majority"`` assigns the most frequent valid annotation and keeps the window only if that label covers at least ``min_label_fraction`` of the valid annotations (CAPTURE-24 and IUWDS use 0.8); ``"constant"`` keeps only windows whose samples all carry one label (the datasets whose windows are built within each activity segment). Ties go to the smallest label. Returns ------- X : (n_windows, T, C) float64, y : (n_windows,) or None, groups : (n_windows,), time_index : (n_windows,) window start time in seconds from the start of the signal. """ x = np.asarray(signal, dtype=np.float64) if x.ndim == 1: x = x[:, None] if x.ndim != 2: raise ValueError("signal must have shape (n_samples, C) or (n_samples,)") n = len(x) if fs is None or window_s is None or step_s is None: raise ValueError("fs, window_s and step_s are required") win, step = int(round(float(window_s) * float(fs))), int(round(float(step_s) * float(fs))) if win < 2 or step < 1: raise ValueError("window_s * fs must give at least 2 samples and step_s * fs at least 1") if label_rule not in ("majority", "constant"): raise ValueError("label_rule must be 'majority' or 'constant'") if not 0 < float(min_label_fraction) <= 1: raise ValueError("min_label_fraction must be in (0, 1]") if group is None or np.ndim(group) == 0: g = np.full(n, 0 if group is None else group) else: g = np.asarray(group) if g.shape != (n,): raise ValueError("group must be a scalar or have one entry per sample") lab = None if labels is not None: lab = np.asarray(labels) if lab.shape != (n,): raise ValueError("labels must have one entry per sample") missing = _is_missing(lab) starts, ys, gs = [], [], [] for a, b in _runs(g): for s in range(a, b - win + 1, step): if lab is not None: seg, ok = lab[s:s + win], ~missing[s:s + win] if not ok.any(): continue values, counts = np.unique(seg[ok], return_counts=True) k = int(np.argmax(counts)) if label_rule == "constant": if len(values) != 1 or not ok.all(): continue elif counts[k] / ok.sum() < float(min_label_fraction): continue ys.append(values[k]) starts.append(s) gs.append(g[s]) if not starts: X = np.empty((0, win, x.shape[1])) else: X = np.stack([x[s:s + win] for s in starts]) y = None if lab is None else np.asarray(ys) groups = np.asarray(gs, dtype=g.dtype) time_index = np.asarray(starts, dtype=np.float64) / float(fs) return X, y, groups, time_index