File size: 14,771 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
"""Strict loaders for the public PULSE files.

``load_stream``    -> the raw table of one file (column names + numeric values)
``load_modality``  -> the model input of one modality, ``float32 (T, F)``
``load_recording`` -> several modalities of one recording, cut to a common length

None of these returns ``None`` or skips anything: a missing file raises
:class:`MissingFileError`, a stream that does not provide the modality raises
:class:`ModalityUnavailableError`, and a stream that fails a quality rule raises
:class:`DataQualityError`.  Which recordings a task *expects* is decided before
loading, from the frozen availability table (see ``pulse.data.manifest``).
"""
from __future__ import annotations

import json
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Sequence, Tuple

import numpy as np

from ..errors import DataQualityError, MissingFileError, ModalityUnavailableError
from ..paths import DataRoot
from . import modalities as M

MAX_ABS_VALUE = 1e6
MAX_ZERO_FRACTION = 0.9


@dataclass
class Stream:
    """One file: feature column names, values (float64, NaN kept) and time."""

    rid: str
    stream: str
    path: str
    columns: List[str]
    values: np.ndarray            # (T, C) float64; non-numeric -> NaN
    time: Optional[np.ndarray]    # (T,) seconds, if the file has a time column
    raw_columns: List[str]        # all columns as in the file

    @property
    def n_rows(self) -> int:
        return int(self.values.shape[0])


def _read_table(path: str):
    import pandas as pd  # imported lazily: pandas must load before torch on some clusters

    sep = "\t" if path.endswith(".tsv") else ","
    df = pd.read_csv(path, sep=sep, low_memory=False)
    df.columns = [str(c).strip() for c in df.columns]
    return df


def stream_path(dr: DataRoot, rid: str, stream: str) -> str:
    if stream not in M.STREAM_FILES:
        raise KeyError(f"unknown stream {stream!r}; known: {sorted(M.STREAM_FILES)}")
    if stream == "mocap_skeleton":
        return str(dr.mocap_tsv_path(rid, must_exist=True))
    return str(dr.stream_path(rid, M.STREAM_FILES[stream], must_exist=True))


def load_stream(dr: DataRoot, rid: str, stream: str) -> Stream:
    """Read one raw file.  Feature columns exclude Frame/Time/time/UTC/'* Type'."""
    import pandas as pd

    path = stream_path(dr, rid, stream)
    df = _read_table(path)
    raw_cols = list(df.columns)
    tcol = "time" if "time" in df.columns else None
    time = df[tcol].to_numpy(dtype=np.float64) if tcol else None
    feat = [c for c in raw_cols
            if c not in M.SKIP_COLS and not any(c.endswith(s) for s in M.SKIP_COL_SUFFIXES)]
    if stream == "myo_pose":
        # categorical event stream: keep the labels as strings
        vals = df[feat].astype(str).to_numpy()
        return Stream(rid, stream, path, feat, vals, time, raw_cols)
    sub = df[feat]
    obj = sub.select_dtypes(include=["object"]).columns
    if len(obj):
        sub = sub.copy()
        sub[obj] = sub[obj].apply(pd.to_numeric, errors="coerce")
    vals = sub.to_numpy(dtype=np.float64)
    return Stream(rid, stream, path, feat, vals, time, raw_cols)


# ---------------------------------------------------------------------------
# feature selection per modality
# ---------------------------------------------------------------------------

def _require_columns(st: Stream, wanted: Sequence[str], modality: str, reason: str) -> List[int]:
    idx = {c: i for i, c in enumerate(st.columns)}
    missing = [c for c in wanted if c not in idx]
    if missing:
        raise ModalityUnavailableError(
            f"{st.rid}: {modality} needs {len(wanted)} columns, {len(missing)} missing "
            f"(e.g. {missing[0]!r}) in {st.path}", reason)
    return [idx[c] for c in wanted]


def select_feature_columns(st: Stream, modality: str) -> Tuple[List[str], List[int]]:
    """Return (column names, column indices) that form ``modality``."""
    spec = M.get_spec(modality)
    if spec.stream != st.stream:
        raise ValueError(f"{modality} is computed from {spec.stream}, not {st.stream}")
    if modality == "emg":
        cols = list(M.EMG_COLS)
        if len(st.columns) != len(cols):
            raise ModalityUnavailableError(
                f"{st.rid}: EMG file has {len(st.columns)} channels, expected 8", "emg_columns")
        return cols, _require_columns(st, cols, modality, "emg_columns")
    if modality == "pressure":
        cols = list(M.PRESSURE_COLS)
        if len(st.columns) != len(cols):
            raise ModalityUnavailableError(
                f"{st.rid}: pressure file has {len(st.columns)} cells, expected 50",
                "pressure_columns")
        return cols, _require_columns(st, cols, modality, "pressure_columns")
    if modality == "myo_quat":
        cols = list(M.MYO_QUAT_COLS)
        return cols, _require_columns(st, cols, modality, "myo_quat_columns")
    if modality in ("imu", "imu_wrist", "imu_finger"):
        n = len(st.columns)
        if n != M.IMU_N_COLS:
            raise ModalityUnavailableError(
                f"{st.rid}: IMU file has {n} columns, expected {M.IMU_N_COLS} "
                f"(10 sensors x 18); treated as a missing modality", f"imu_columns_{n}")
        if modality == "imu":
            return list(st.columns), list(range(n))
        pre = M.IMU_SENSOR_SUBSETS[modality]
        sel = [(c, i) for i, c in enumerate(st.columns) if c.startswith(pre)]
        if len(sel) != M.get_spec(modality).expected_dim:
            raise ModalityUnavailableError(
                f"{st.rid}: {modality} selects {len(sel)} columns", f"{modality}_columns")
        return [c for c, _ in sel], [i for _, i in sel]
    if modality in ("eyetrack", "eyetrack_gaze", "eyetrack_ocular"):
        core = list(M.EYETRACK_CORE_COLS)
        idx = _require_columns(st, core, modality, "eyetrack_core_columns")
        if modality == "eyetrack":
            return core, idx
        pats = M.EYETRACK_SUBSETS[modality]
        sel = [(c, i) for c, i in zip(core, idx) if any(p in c for p in pats)]
        return [c for c, _ in sel], [i for _, i in sel]
    if modality in ("mocap", "mocap_markers"):
        if not st.columns:
            raise ModalityUnavailableError(f"{st.rid}: no MoCap columns", "mocap_columns")
        if modality == "mocap" and "Hips_X" not in st.columns:
            raise ModalityUnavailableError(f"{st.rid}: skeleton TSV has no Hips_X", "mocap_no_hips")
        return list(st.columns), list(range(len(st.columns)))
    raise KeyError(modality)


def skeleton_hip_relative_velocity(arr: np.ndarray, cols: Sequence[str]) -> np.ndarray:
    """Original benchmark MoCap features: hip-relative positions + velocity.

    Input ``(T, F)`` with absolute ``*_X/_Y/_Z`` positions and quaternions.
    Output ``(T, F + N_pos)``: all features with positions minus the hip
    position, followed by the frame-to-frame difference of the position
    columns (first frame 0).
    """
    ci = {c: i for i, c in enumerate(cols)}
    hx, hy, hz = ci["Hips_X"], ci["Hips_Y"], ci["Hips_Z"]
    xs = [i for i, c in enumerate(cols) if c.endswith("_X")]
    ys = [i for i, c in enumerate(cols) if c.endswith("_Y")]
    zs = [i for i, c in enumerate(cols) if c.endswith("_Z")]
    pos = sorted(xs + ys + zs)
    rel = arr.copy()
    hip = arr[:, [hx, hy, hz]].copy()
    rel[:, xs] -= hip[:, [0]]
    rel[:, ys] -= hip[:, [1]]
    rel[:, zs] -= hip[:, [2]]
    p = rel[:, pos]
    vel = np.zeros_like(p)
    vel[1:] = p[1:] - p[:-1]
    return np.concatenate([rel, vel], axis=1)


def quality_check(arr: np.ndarray, rid: str, modality: str) -> None:
    spec = M.get_spec(modality)
    if arr.size == 0:
        raise DataQualityError(f"{rid}: {modality} is empty", "empty")
    max_abs = float(np.max(np.abs(arr)))
    if max_abs > MAX_ABS_VALUE:
        raise DataQualityError(
            f"{rid}: {modality} has |x| up to {max_abs:.3g} > {MAX_ABS_VALUE:g} (corrupted)",
            "extreme_values")
    if spec.zero_check:
        zf = float(np.mean(arr == 0.0))
        if zf > MAX_ZERO_FRACTION:
            raise DataQualityError(
                f"{rid}: {modality} is {100 * zf:.1f}% zeros (> {100 * MAX_ZERO_FRACTION:.0f}%, "
                "sensor dropout)", "mostly_zero")


@dataclass
class ModalityArray:
    rid: str
    modality: str
    data: np.ndarray          # (T, F) float32, NaN/inf mapped to 0
    columns: List[str]        # names of the F input columns (before any velocity)
    path: str


def modality_from_stream(st: Stream, modality: str) -> ModalityArray:
    cols, idx = select_feature_columns(st, modality)
    arr = np.nan_to_num(st.values[:, idx].astype(np.float64), nan=0.0, posinf=0.0, neginf=0.0)
    quality_check(arr, st.rid, modality)
    if modality == "mocap":
        arr = skeleton_hip_relative_velocity(arr, cols)
    spec = M.get_spec(modality)
    if spec.expected_dim is not None and arr.shape[1] != spec.expected_dim:
        raise ModalityUnavailableError(
            f"{st.rid}: {modality} has {arr.shape[1]} features, expected {spec.expected_dim}",
            f"{modality}_dim")
    return ModalityArray(st.rid, modality, arr.astype(np.float32), cols, st.path)


def load_modality(dr: DataRoot, rid: str, modality: str) -> ModalityArray:
    st = load_stream(dr, rid, M.base_stream(modality))
    return modality_from_stream(st, modality)


@dataclass
class RecordingArrays:
    rid: str
    arrays: Dict[str, np.ndarray]       # modality -> (T, F) float32, common T
    columns: Dict[str, List[str]]
    raw_lengths: Dict[str, int]         # rows before cutting to the common length
    #: frames dropped at the start of a modality by ``align_trimmed`` (see
    #: :func:`release_trim_window`); empty when no stream was shifted
    start_offsets: Dict[str, int] = field(default_factory=dict)

    @property
    def n_frames(self) -> int:
        return next(iter(self.arrays.values())).shape[0] if self.arrays else 0

    def concat(self, order: Optional[Sequence[str]] = None) -> np.ndarray:
        order = list(order or self.arrays.keys())
        return np.concatenate([self.arrays[m] for m in order], axis=1)


def release_trim_window(meta: dict) -> Optional[Tuple[int, int]]:
    """``(front, n)`` if some streams of a recording cover only frames
    ``[front, front + n)`` of its alignment timeline, else ``None``.

    In v1.1 the EMG, eye-tracking and pressure files of v2s6, v13s2 and v13s3
    were re-trimmed for the release to the MoCap capture window, while IMU,
    MoCap (TSV and marker CSV) and the Myo orientation streams still cover the
    whole timeline (``n_samples`` rows).  The window is the MoCap padding
    recorded in ``alignment_metadata.json`` (``mocap_padding.mocap_sq``:
    ``front_samples``, ``n_frames``); for v2s6 and v13s3 the re-trimmed files
    equal the earlier full-length streams sliced at exactly this window.
    """
    n_total = meta.get("n_samples")
    pads = meta.get("mocap_padding") or {}
    pad = pads.get("mocap_sq") or pads.get("mocap") or {}
    front, n = pad.get("front_samples"), pad.get("n_frames")
    if not n_total or front is None or n is None:
        return None
    front, n, n_total = int(front), int(n), int(n_total)
    if n <= 0 or n >= n_total or front + n > n_total:
        return None
    return front, n


def load_recording(dr: DataRoot, rid: str, modalities: Sequence[str],
                   gate_modalities: Sequence[str] = (),
                   align_trimmed: bool = False) -> RecordingArrays:
    """Load ``modalities`` of one recording, cut to the shortest stream.

    ``gate_modalities`` are loaded and validated (so a recording is used only if
    those streams are present and pass the same rules) but not returned.  This
    is how a 4-modality run is restricted to the recordings of a 5-modality run.

    ``align_trimmed`` (opt-in; default off keeps the original behaviour): when
    the requested modalities mix streams re-trimmed by the release (exactly
    ``n`` rows of :func:`release_trim_window`) with full-timeline streams, the
    full-timeline streams are first sliced to ``[front, front + n)`` so that
    frame ``i`` of every returned array is the same instant.  Without it, the
    three affected recordings are cut from the start and the IMU / MoCap
    streams lead the EMG / eye / pressure streams by 3.4-10.5 s.  A requested
    stream whose length is neither ``n`` nor at least ``front + n`` raises
    :class:`DataQualityError` (it cannot be placed on either timeline).
    """
    streams: Dict[str, Stream] = {}
    out: Dict[str, np.ndarray] = {}
    cols: Dict[str, List[str]] = {}
    lengths: Dict[str, int] = {}
    for mod in list(gate_modalities) + list(modalities):
        sname = M.base_stream(mod)
        if sname not in streams:
            streams[sname] = load_stream(dr, rid, sname)
        ma = modality_from_stream(streams[sname], mod)
        if mod in modalities:
            out[mod] = ma.data
            cols[mod] = ma.columns
            lengths[mod] = ma.data.shape[0]
    offsets: Dict[str, int] = {}
    if align_trimmed and out and len({a.shape[0] for a in out.values()}) > 1:
        win = release_trim_window(load_alignment_metadata(dr, rid))
        if win is not None:
            front, n = win
            short = [m for m, a in out.items() if a.shape[0] == n]
            full = [m for m, a in out.items() if a.shape[0] >= front + n and a.shape[0] != n]
            odd = {m: int(a.shape[0]) for m, a in out.items() if m not in short and m not in full}
            if short and odd:
                # neither the re-trimmed window nor the full timeline: shifting the
                # others but not these (or cutting all from the start) would misalign
                raise DataQualityError(
                    f"{rid}: streams {odd} are neither the re-trimmed window ({n} rows) nor "
                    f"cover the full timeline (>= {front + n} rows); cannot align", "trim_window")
            if short and full:
                for m in full:
                    out[m] = out[m][front:front + n]
                    offsets[m] = front
    if out:
        t = min(a.shape[0] for a in out.values())
        out = {m: a[:t] for m, a in out.items()}
    return RecordingArrays(rid, out, cols, lengths, offsets)


def load_alignment_metadata(dr: DataRoot, rid: str) -> dict:
    p = dr.alignment_metadata_path(rid)
    with open(p, encoding="utf-8") as f:
        return json.load(f)


__all__ = [
    "Stream", "ModalityArray", "RecordingArrays", "load_stream", "load_modality",
    "load_recording", "release_trim_window", "modality_from_stream", "select_feature_columns",
    "quality_check",
    "skeleton_hip_relative_velocity", "load_alignment_metadata", "MissingFileError",
]