Scikit-learn
human-activity-recognition
wearable
wrist
time-series
cpu
scikit-learn
File size: 14,811 Bytes
4731b11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93a9282
 
 
 
 
 
4731b11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
"""Data entry points: WristHARBench folds and windowing of continuous signals.

``load_wristharbench`` needs the optional extra ``[hub]`` (``datasets``,
``huggingface_hub``, ``pandas``) the first time a fold is exported; afterwards the cached
NPZ files are read with NumPy only.
"""
from __future__ import annotations

import json
import os
from dataclasses import dataclass, field
from importlib import resources
from pathlib import Path
from typing import Any

import numpy as np

from .recipes import dataset_id as _dataset_id

__all__ = ["load_wristharbench", "load_exported_fold", "make_windows", "Split", "BenchmarkFold",
           "REPO_ID", "REVISION"]

REPO_ID = "Zipeng365/WristHARBench"
REVISION = "5e6af3c5c6dca050e73c06f49272f9656c0f259f"


@dataclass
class Split:
    """One partition: windows ``X`` (n, T, C), integer labels ``y`` on the global vocabulary,
    ``groups`` (participant IDs) and ``time_index`` (position within each participant)."""
    X: np.ndarray
    y: np.ndarray
    groups: np.ndarray
    time_index: np.ndarray

    def __len__(self) -> int:
        return len(self.y)


@dataclass
class BenchmarkFold:
    dataset_id: str
    fold: int
    train: Split
    val: Split
    test: Split
    label_names: list[str]
    sampling_rate_hz: float | None = None
    source: dict[str, Any] = field(default_factory=dict)

    @property
    def valid(self) -> Split:
        return self.val

    @property
    def study_setting(self) -> str | None:
        """The stream's study setting from the package's WristHARBench metadata (e.g. ``"free_living"``)."""
        settings = json.loads(_bench_text("datasets.json")).get(self.dataset_id, {})
        return settings.get("metadata", {}).get("study_setting")

    @property
    def train_val(self) -> Split:
        """Training and validation windows concatenated (the paper's final-fit data)."""
        a, b = self.train, self.val
        return Split(np.concatenate([a.X, b.X]), np.concatenate([a.y, b.y]),
                     np.concatenate([a.groups, b.groups]), np.concatenate([a.time_index, b.time_index]))

    @property
    def classes(self) -> np.ndarray:
        """The global class table (integer IDs ``0..K-1``) to pass as ``classes=``."""
        return np.arange(len(self.label_names))


def _bench_text(name: str) -> str:
    return resources.files("wisp").joinpath("_data", "wristharbench", name).read_text(encoding="utf-8")


def _default_cache() -> Path:
    root = os.environ.get("WISP_DATA_DIR")
    if root:
        return Path(root)
    return Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "wisp-har" / "wristharbench"


def _read_split(path: Path) -> tuple[Split, list[str], float | None]:
    with np.load(path, allow_pickle=False) as z:
        X = np.asarray(z["X"])
        y = np.asarray(z["y"], dtype=np.int64)
        groups = np.asarray(z["subject"]).astype(str)
        time_index = np.asarray(z["time_index"], dtype=np.float64)
        names = [str(v) for v in z["label_names"].tolist()] if "label_names" in z.files else None
        rate = float(np.asarray(z["sampling_rate_hz"]).reshape(-1)[0]) if "sampling_rate_hz" in z.files else None
    return Split(X, y, groups, time_index), names, rate


def load_exported_fold(root, dataset: str, fold: int = 0) -> BenchmarkFold:
    """Read an already exported fold: ``<root>/<dataset_id>/fold<k>/{train,valid,test}.npz``.

    Only numeric arrays and Unicode strings are read (no pickled objects). When the NPZ files
    carry no ``label_names`` (older exports), the fold's ``export_manifest.json`` is used.
    """
    did = _dataset_id(dataset)
    folder = Path(root) / did / f"fold{int(fold)}"
    parts, names, rate = {}, None, None
    for part in ("train", "valid", "test"):
        try:
            parts[part], part_names, part_rate = _read_split(folder / f"{part}.npz")
        except ValueError:  # object arrays present: read the numeric keys only
            with np.load(folder / f"{part}.npz", allow_pickle=False) as z:
                parts[part] = Split(np.asarray(z["X"]), np.asarray(z["y"], dtype=np.int64),
                                    np.asarray(z["subject"]).astype(str), np.asarray(z["time_index"], dtype=np.float64))
            part_names, part_rate = None, None
        names = names or part_names
        rate = rate or part_rate
    if names is None:
        manifest = folder / "export_manifest.json"
        if manifest.exists():
            names = list(json.loads(manifest.read_text())["label_names"])
    if names is None:
        raise ValueError(f"No label vocabulary found for {folder}")
    if rate is None:
        settings = json.loads(_bench_text("datasets.json")).get(did, {})
        rate = settings.get("sampling_rate_hz")
    return BenchmarkFold(did, int(fold), parts["train"], parts["valid"], parts["test"], names, rate,
                         {"root": str(Path(root)), "format": "exported NPZ"})


def _preprocess_signal(signal):
    import pandas as pd
    a = np.stack(signal).astype(np.float64)
    a = pd.DataFrame(a).ffill().bfill().to_numpy(dtype=np.float64)
    if a.ndim != 2 or not np.isfinite(a).all():
        raise ValueError("Non-finite signal after registered imputation")
    mu, sd = a.mean(0, keepdims=True), a.std(0, keepdims=True)
    sd[sd == 0] = 1
    return (a - mu) / sd


def _config_name(did: str) -> str:
    return did + "_paper_snapshot_20260828" if did == "capture24_wearable_activity_v1" else did


def _export_fold(did: str, fold: int, output: Path, *, local_data=None, revision: str = REVISION) -> None:
    """Paper export contract (same procedure as ``scripts/export_benchmark.py``) for one fold."""
    import pandas as pd

    scope = json.loads(_bench_text("scope.json"))
    if did not in scope["datasets"]:
        raise ValueError(f"Unknown paper dataset: {did}")
    settings = json.loads(_bench_text("datasets.json"))[did]
    if local_data is None:
        from datasets import load_dataset
        frame = load_dataset(REPO_ID, _config_name(did), split="full", revision=revision).to_pandas()
    else:
        files = sorted((Path(local_data) / "data" / _config_name(did)).glob("*.parquet"))
        if not files:
            raise FileNotFoundError(f"No Parquet files for {_config_name(did)} under {local_data}")
        frame = pd.concat([pd.read_parquet(f) for f in files], ignore_index=True)
    if len(frame) != settings["sequences"] or frame.series_id.duplicated().any():
        raise ValueError("Input scope does not match the frozen paper contract")
    frame["series_id"] = frame.series_id.astype(str)
    frame["subject_id"] = frame.subject_id.astype(str)
    vocabulary = sorted(frame.label.astype(str).unique())
    encoded = {label: i for i, label in enumerate(vocabulary)}
    with resources.files("wisp").joinpath("_data", "wristharbench", "ordering", f"{did}.csv.gz").open("rb") as fh:
        ordering = pd.read_csv(fh, compression="gzip", dtype={"series_id": str, "subject_id": str})
    if set(ordering.series_id) != set(frame.series_id):
        raise ValueError("Ordering sidecar does not cover the downloaded rows")
    folder = output / did / f"fold{fold}"
    folder.mkdir(parents=True, exist_ok=True)
    groups = []
    for split in ("train", "validation", "test"):
        side = ordering[(ordering.fold == fold) & (ordering.split == split)].sort_values("split_row")
        part = side.merge(frame, on=["series_id", "subject_id"], how="left", validate="one_to_one")
        if not part[f"fold_{fold}"].eq(split).all():
            raise ValueError("Published split assignments disagree with the frozen export")
        groups.append(set(part.subject_id))
        X = np.stack([_preprocess_signal(x) for x in part.signal]).astype(np.float32)
        if X.shape[1:] != (settings["window_steps"], len(settings["channels"])):
            raise ValueError(f"Unexpected shape for {did}: {X.shape}")
        name = "valid" if split == "validation" else split
        tmp = folder / f".{name}.tmp.npz"
        np.savez_compressed(tmp, X=X, y=np.array([encoded[str(x)] for x in part.label], dtype=np.int64),
                            subject=part.subject_id.to_numpy(dtype=str), series_id=part.series_id.to_numpy(dtype=str),
                            time_index=part.time_index.to_numpy(dtype=float),
                            label_names=np.asarray(vocabulary, dtype=str),
                            sampling_rate_hz=np.asarray([settings["sampling_rate_hz"]]))
        os.replace(tmp, folder / f"{name}.npz")
    if any(groups[i] & groups[j] for i, j in ((0, 1), (0, 2), (1, 2))):
        raise ValueError("Participant leakage across split partitions")
    (folder / "export_source.json").write_text(json.dumps(
        {"repo_id": REPO_ID, "revision": revision, "configuration": _config_name(did), "fold": fold,
         "local_data": local_data is not None, "label_names": vocabulary}, indent=1))


def load_wristharbench(dataset: str, fold: int = 0, *, cache_dir=None, local_data=None,
                       revision: str = REVISION) -> BenchmarkFold:
    """One official WristHARBench fold (paper view, participant-disjoint train/val/test).

    The fold is exported once from the Hugging Face dataset ``Zipeng365/WristHARBench`` at
    the pinned revision (or from a local snapshot via ``local_data``) and cached under
    ``cache_dir`` (default ``$WISP_DATA_DIR`` or ``~/.cache/wisp-har/wristharbench``).
    Access follows that repository's terms; authenticate with your own account if needed.
    ``dataset`` accepts short names (``"adl"``) and full IDs.
    """
    did = _dataset_id(dataset)
    if int(fold) not in range(5):
        raise ValueError("fold must be 0..4 (the five paper splits)")
    cache = Path(cache_dir) if cache_dir is not None else _default_cache()
    folder = cache / did / f"fold{int(fold)}"
    if not all((folder / f"{p}.npz").exists() for p in ("train", "valid", "test")):
        _export_fold(did, int(fold), cache, local_data=local_data, revision=revision)
    data = load_exported_fold(cache, did, fold)
    data.source = {"repo_id": REPO_ID, "revision": revision, "cache": str(folder)}
    return data


# --------------------------------------------------------------------------- windowing
def _runs(values: np.ndarray) -> list[tuple[int, int]]:
    """Contiguous runs [start, stop) of equal values."""
    if len(values) == 0:
        return []
    change = np.flatnonzero(values[1:] != values[:-1]) + 1
    bounds = np.r_[0, change, len(values)]
    return [(int(a), int(b)) for a, b in zip(bounds[:-1], bounds[1:])]


def _is_missing(labels: np.ndarray) -> np.ndarray:
    if labels.dtype.kind in "fc":
        return ~np.isfinite(labels)
    if labels.dtype == object:
        return np.array([v is None or (isinstance(v, float) and not np.isfinite(v)) for v in labels.tolist()])
    return np.zeros(len(labels), dtype=bool)


def make_windows(signal, *, fs: float, window_s: float, step_s: float, labels=None, group=None,
                 label_rule: str = "majority", min_label_fraction: float = 0.8):
    """Cut a continuous signal into fixed-length windows.

    Parameters
    ----------
    signal : array of shape (n_samples, C) or (n_samples,)
    fs : float
        Sampling rate in Hz.
    window_s, step_s : float
        Window length and step in seconds (required: datasets differ). They are converted to
        ``round(window_s * fs)`` and ``round(step_s * fs)`` samples.
    labels : array of shape (n_samples,), optional
        Per-sample annotations; ``None``/NaN entries are missing.
    group : scalar or array of shape (n_samples,), optional
        Sequence ID (participant or recording). Windows never cross a change of ``group``;
        each contiguous run of one value is windowed separately.
    label_rule : {"majority", "constant"}
        Window labelling following the WristHARBench export rules
        (docs/specifications/execution.tex of the benchmark): ``"majority"`` assigns the most
        frequent valid annotation and keeps the window only if that label covers at least
        ``min_label_fraction`` of the valid annotations (CAPTURE-24 and IUWDS use 0.8);
        ``"constant"`` keeps only windows whose samples all carry one label (the datasets
        whose windows are built within each activity segment). Ties go to the smallest label.

    Returns
    -------
    X : (n_windows, T, C) float64, y : (n_windows,) or None, groups : (n_windows,),
    time_index : (n_windows,) window start time in seconds from the start of the signal.
    """
    x = np.asarray(signal, dtype=np.float64)
    if x.ndim == 1:
        x = x[:, None]
    if x.ndim != 2:
        raise ValueError("signal must have shape (n_samples, C) or (n_samples,)")
    n = len(x)
    if fs is None or window_s is None or step_s is None:
        raise ValueError("fs, window_s and step_s are required")
    win, step = int(round(float(window_s) * float(fs))), int(round(float(step_s) * float(fs)))
    if win < 2 or step < 1:
        raise ValueError("window_s * fs must give at least 2 samples and step_s * fs at least 1")
    if label_rule not in ("majority", "constant"):
        raise ValueError("label_rule must be 'majority' or 'constant'")
    if not 0 < float(min_label_fraction) <= 1:
        raise ValueError("min_label_fraction must be in (0, 1]")
    if group is None or np.ndim(group) == 0:
        g = np.full(n, 0 if group is None else group)
    else:
        g = np.asarray(group)
        if g.shape != (n,):
            raise ValueError("group must be a scalar or have one entry per sample")
    lab = None
    if labels is not None:
        lab = np.asarray(labels)
        if lab.shape != (n,):
            raise ValueError("labels must have one entry per sample")
        missing = _is_missing(lab)
    starts, ys, gs = [], [], []
    for a, b in _runs(g):
        for s in range(a, b - win + 1, step):
            if lab is not None:
                seg, ok = lab[s:s + win], ~missing[s:s + win]
                if not ok.any():
                    continue
                values, counts = np.unique(seg[ok], return_counts=True)
                k = int(np.argmax(counts))
                if label_rule == "constant":
                    if len(values) != 1 or not ok.all():
                        continue
                elif counts[k] / ok.sum() < float(min_label_fraction):
                    continue
                ys.append(values[k])
            starts.append(s)
            gs.append(g[s])
    if not starts:
        X = np.empty((0, win, x.shape[1]))
    else:
        X = np.stack([x[s:s + win] for s in starts])
    y = None if lab is None else np.asarray(ys)
    groups = np.asarray(gs, dtype=g.dtype)
    time_index = np.asarray(starts, dtype=np.float64) / float(fs)
    return X, y, groups, time_index