PULSE-code / pulse /tasks /t5_data.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
17.9 kB
"""T5 samples: anchors, 3-class grasp-state labels and contact-event strata.
Definitions (camera-ready, v1.1 labels)
---------------------------------------
All streams are 100 Hz on disk and are decimated by ``downsample`` (5 -> 20 Hz)
by taking every 5th frame from frame 0.
* **Grasp-verb mask** (100 Hz): a frame is "grasp" if it lies inside any v1.1
segment whose ``action_name`` is in ``grasp_verbs`` (grasp, hold, pick_up,
move, place, put_down, pull, rotate, insert, remove). Segment ``[start,
end]`` covers frames ``round(100 start) .. round(100 end) - 1``, clipped to
the loaded frames. Every verb must belong to the frozen v1.1 label space
(``taxonomy_v1.1.json``, including the dropped ``support``); anything else
raises. Nouns are not used.
* **Contact** (20 Hz): at least ``contact_min_cells`` (3) of the 50 pressure
cells (both hands) read more than ``contact_cell_threshold_g`` (10 g).
* **Anchors** every ``anchor_stride_sec`` (0.25 s = 5 frames) at 20 Hz frame
``a`` with ``T_obs <= a <= T - T_fut``. The input is the past window
``[a - T_obs, a)`` (1 s = 20 frames); labels use only the future window
``[a, a + T_fut)`` (0.5 s = 10 frames).
* **Label**: NoGrasp (0) unless at least ``majority_threshold`` (50 %) of the
future window is grasp-verb; then SustainedGrasp (2) if the longest
contiguous contact run in the future window is at least
``sustained_min_sec`` (0.3 s = 6 frames), else AttemptedGrasp (1).
* **Event stratum** (Table 10): past / future window "in contact" if more than
50 % of its frames are contact frames: non-contact (no/no), pre-contact
(no/yes), steady-grip (yes/yes), release (yes/no). SustainedGrasp anchors
can only occur in pre-contact and steady-grip.
Sample set
----------
A configuration names its model ``inputs`` and its ``sample_streams`` (a
superset that always contains ``pressure``, which defines the labels). All
sample streams are loaded and cut to a common length, so rows that differ only
in their inputs (EMG vs EMG + pressure vs pressure only) see exactly the same
anchors and labels.
Release re-trim (v2s6, v13s2, v13s3)
------------------------------------
In v1.1 the EMG, eye-tracking and pressure files of these three recordings
were re-trimmed to the MoCap capture window ``[front, front + n)`` recorded in
``alignment_metadata.json`` (``mocap_padding.mocap_sq``), while IMU and MoCap
keep the full alignment timeline, and the annotations are on the full
timeline. When a loaded stream has exactly ``n`` rows, the full-timeline
streams are sliced to ``[front, front + n)`` and the annotation times are
shifted by ``front`` frames, so frame ``i`` of every array and of the label
timeline is the same instant (recorded per recording as ``timeline_offset``).
Normalisation
-------------
Per-channel z-score with the mean / SD (population) of the training frames,
each frame weighted by the number of kept training anchors whose observation
window contains it (the statistics of the training windows, as in the original
pipeline). Inner-validation and test data use the training statistics.
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Sequence, Tuple
import numpy as np
from ..data.annotations import load_segments
from ..data.io import load_alignment_metadata, load_modality
from ..errors import DataQualityError
from ..paths import DataRoot
from ..taxonomy import Taxonomy, TaxonomyError
SR_HZ = 100
CLASS_KEYS: Tuple[str, ...] = ("nograsp", "attempted", "sustained")
CLASS_NAMES: Tuple[str, ...] = ("NoGrasp", "AttemptedGrasp", "SustainedGrasp")
EVENT_NAMES: Tuple[str, ...] = ("non-contact", "pre-contact", "steady-grip", "release")
#: Events in which SustainedGrasp anchors can exist (future window in contact).
SUSTAINED_EVENTS: Tuple[str, ...] = ("pre-contact", "steady-grip")
DEFAULT_GRASP_VERBS: Tuple[str, ...] = ("grasp", "hold", "pick_up", "move", "place", "put_down",
"pull", "rotate", "insert", "remove")
@dataclass(frozen=True)
class LabelSpec:
downsample: int = 5
t_obs_sec: float = 1.0
t_fut_sec: float = 0.5
anchor_stride_sec: float = 0.25
majority_threshold: float = 0.5
grasp_verbs: Tuple[str, ...] = DEFAULT_GRASP_VERBS
contact_cell_threshold_g: float = 10.0
contact_min_cells: int = 3
sustained_min_sec: float = 0.3
@classmethod
def from_config(cls, cfg: Dict[str, Any]) -> "LabelSpec":
d = dict(cfg or {})
if "grasp_verbs" in d:
d["grasp_verbs"] = tuple(d["grasp_verbs"])
unknown = set(d) - set(cls.__dataclass_fields__)
if unknown:
raise ValueError(f"unknown T5 label settings: {sorted(unknown)}")
return cls(**d)
@property
def sr(self) -> int:
if SR_HZ % self.downsample:
raise ValueError("downsample must divide 100")
return SR_HZ // self.downsample
@property
def t_obs(self) -> int:
return int(round(self.t_obs_sec * self.sr))
@property
def t_fut(self) -> int:
return int(round(self.t_fut_sec * self.sr))
@property
def stride(self) -> int:
return max(1, int(round(self.anchor_stride_sec * self.sr)))
@property
def sustained_frames(self) -> int:
return int(round(self.sustained_min_sec * self.sr))
def check_against(self, tax: Taxonomy) -> None:
bad = [v for v in self.grasp_verbs if v not in tax.verbs]
if bad:
raise TaxonomyError(f"grasp verbs {bad} are not v1.1 verbs")
# ---------------------------------------------------------------------------
# label derivation (pure functions, unit-tested)
# ---------------------------------------------------------------------------
def grasp_mask(segments, n_frames: int, grasp_verbs: Sequence[str], offset_frames: int = 0,
tax: Optional[Taxonomy] = None) -> Tuple[np.ndarray, Dict[str, int]]:
"""100 Hz bool mask of grasp-verb frames for frames ``0 .. n_frames - 1``.
``offset_frames``: timeline frame of loaded frame 0 (release re-trim).
"""
gv = set(grasp_verbs)
mask = np.zeros(int(n_frames), dtype=bool)
st = {"segments": 0, "grasp_segments": 0, "grasp_clipped": 0, "grasp_outside": 0}
for s in segments:
st["segments"] += 1
if tax is not None and s.verb not in tax.verbs and s.verb not in tax.dropped_verbs:
raise TaxonomyError(f"{s.rid} segment {s.index}: verb {s.verb!r} is not a v1.1 verb")
if s.verb not in gv:
continue
st["grasp_segments"] += 1
i0 = int(round(s.start_sec * SR_HZ)) - offset_frames
i1 = int(round(s.end_sec * SR_HZ)) - offset_frames
j0, j1 = max(0, i0), min(int(n_frames), i1)
if j1 <= j0:
st["grasp_outside"] += 1
continue
if (j0, j1) != (i0, i1):
st["grasp_clipped"] += 1
mask[j0:j1] = True
return mask, st
def contact_frames(pressure_ds: np.ndarray, cell_threshold_g: float, min_cells: int) -> np.ndarray:
return (np.asarray(pressure_ds) > cell_threshold_g).sum(axis=1) >= min_cells
def longest_true_run(windows: np.ndarray) -> np.ndarray:
"""Longest run of True per row of a ``(N, T)`` bool array."""
w = np.asarray(windows, dtype=bool)
cur = np.zeros(w.shape[0], dtype=np.int64)
best = np.zeros(w.shape[0], dtype=np.int64)
for t in range(w.shape[1]):
cur = np.where(w[:, t], cur + 1, 0)
best = np.maximum(best, cur)
return best
def derive_anchors(mask_ds: np.ndarray, contact_ds: np.ndarray, spec: LabelSpec
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Return ``(anchors, labels, events)`` for one recording at the decimated rate."""
mask_ds = np.asarray(mask_ds, dtype=bool)
contact_ds = np.asarray(contact_ds, dtype=bool)
if mask_ds.shape != contact_ds.shape:
raise ValueError("mask and contact must have the same length")
T = mask_ds.shape[0]
to, tf = spec.t_obs, spec.t_fut
if T - tf < to:
e = np.zeros(0, dtype=np.int64)
return e, e.copy(), e.copy()
anchors = np.arange(to, T - tf + 1, spec.stride, dtype=np.int64)
fut = anchors[:, None] + np.arange(tf)[None, :]
past = anchors[:, None] - to + np.arange(to)[None, :]
is_grasp = mask_ds[fut].mean(axis=1) >= spec.majority_threshold
sustained = longest_true_run(contact_ds[fut]) >= spec.sustained_frames
labels = np.where(is_grasp, np.where(sustained, 2, 1), 0).astype(np.int64)
past_high = contact_ds[past].mean(axis=1) > 0.5
fut_high = contact_ds[fut].mean(axis=1) > 0.5
events = np.select([~past_high & ~fut_high, ~past_high & fut_high, past_high & fut_high],
[0, 1, 2], default=3).astype(np.int64)
return anchors, labels, events
# ---------------------------------------------------------------------------
# release re-trim (v2s6, v13s2, v13s3)
# ---------------------------------------------------------------------------
def release_trim_window(meta: dict) -> Optional[Tuple[int, int]]:
"""``(front, n)`` from ``mocap_padding`` if it covers part of the timeline, else None."""
n_total = meta.get("n_samples")
pads = meta.get("mocap_padding") or {}
pad = pads.get("mocap_sq") or pads.get("mocap") or {}
front, n = pad.get("front_samples"), pad.get("n_frames")
if not n_total or front is None or n is None:
return None
front, n, n_total = int(front), int(n), int(n_total)
if n <= 0 or n >= n_total or front + n > n_total:
return None
return front, n
def align_release_trim(rid: str, arrays: Dict[str, np.ndarray], meta: dict
) -> Tuple[Dict[str, np.ndarray], int]:
"""Put every stream on the same instants; return (arrays, timeline offset)."""
win = release_trim_window(meta)
if win is None:
return arrays, 0
front, n = win
trimmed = [m for m, a in arrays.items() if a.shape[0] == n]
if not trimmed:
return arrays, 0
out = dict(arrays)
for m, a in arrays.items():
if a.shape[0] == n:
continue
if a.shape[0] < front + n:
raise DataQualityError(
f"{rid}: {m} has {a.shape[0]} rows; cannot align it with the re-trimmed "
f"streams {trimmed} (window {front}+{n})", "release_trim_mismatch")
out[m] = a[front:front + n]
return out, front
# ---------------------------------------------------------------------------
# recordings and partitions
# ---------------------------------------------------------------------------
@dataclass
class RecordingSamples:
rid: str
arrays: Dict[str, np.ndarray] # model inputs, decimated, (T_ds, F) float32, raw units
anchors: np.ndarray # decimated frame index of each anchor
labels: np.ndarray
events: np.ndarray
n_frames: int # common length at 100 Hz
timeline_offset: int # 100 Hz frames (release re-trim), 0 otherwise
label_stats: Dict[str, int]
annotation_md5: str
def load_recording_samples(dr: DataRoot, rid: str, inputs: Sequence[str],
sample_streams: Sequence[str], spec: LabelSpec,
tax: Optional[Taxonomy] = None) -> RecordingSamples:
if "pressure" not in sample_streams:
raise ValueError("T5 sample_streams must contain 'pressure' (it defines the labels)")
missing = [m for m in inputs if m not in sample_streams]
if missing:
raise ValueError(f"inputs {missing} are not in sample_streams")
raw = {m: load_modality(dr, rid, m).data for m in sample_streams} # raises if absent
raw, offset = align_release_trim(rid, raw, load_alignment_metadata(dr, rid))
n = min(a.shape[0] for a in raw.values())
ann_path = dr.annotation_path(rid, must_exist=True)
segs = load_segments(dr, rid)
mask, st = grasp_mask(segs, n, spec.grasp_verbs, offset_frames=offset, tax=tax)
ds = spec.downsample
mask_ds = mask[:n:ds]
contact = contact_frames(raw["pressure"][:n:ds], spec.contact_cell_threshold_g,
spec.contact_min_cells)
anchors, labels, events = derive_anchors(mask_ds, contact, spec)
arrays = {m: np.ascontiguousarray(raw[m][:n:ds], dtype=np.float32) for m in inputs}
st = dict(st, n_anchors=int(anchors.size))
return RecordingSamples(rid, arrays, anchors, labels, events, int(n), int(offset), st,
hashlib.md5(ann_path.read_bytes()).hexdigest())
@dataclass
class Partition:
"""All anchors of one partition, windows gathered from concatenated arrays."""
name: str
rids: List[str]
inputs: List[str]
data: Dict[str, np.ndarray] # modality -> (sum T_ds, F) float32
rec_index: np.ndarray # (N,) recording index of each anchor
start: np.ndarray # (N,) global row of the first frame of the window
labels: np.ndarray
events: np.ndarray
t_obs: int
recordings: Dict[str, Dict[str, Any]] = field(default_factory=dict)
@property
def n(self) -> int:
return int(self.labels.size)
def subset(self, idx: np.ndarray) -> "Partition":
return Partition(self.name, self.rids, self.inputs, self.data, self.rec_index[idx],
self.start[idx], self.labels[idx], self.events[idx], self.t_obs,
self.recordings)
def class_counts(self, n_classes: int = 3) -> List[int]:
return np.bincount(self.labels, minlength=n_classes).astype(int).tolist()
def event_counts(self) -> Dict[str, int]:
c = np.bincount(self.events, minlength=len(EVENT_NAMES))
return {e: int(c[i]) for i, e in enumerate(EVENT_NAMES)}
def window_index(self, idx: np.ndarray) -> np.ndarray:
return self.start[idx][:, None] + np.arange(self.t_obs)[None, :]
def build_partition(name: str, recs: Sequence[RecordingSamples], inputs: Sequence[str],
spec: LabelSpec) -> Partition:
data: Dict[str, List[np.ndarray]] = {m: [] for m in inputs}
rec_index, start, labels, events = [], [], [], []
info: Dict[str, Dict[str, Any]] = {}
offset = 0
dims: Dict[str, int] = {}
for i, r in enumerate(recs):
T = next(iter(r.arrays.values())).shape[0]
for m in inputs:
a = r.arrays[m]
if a.shape[0] != T:
raise ValueError(f"{r.rid}: {m} has {a.shape[0]} frames, expected {T}")
if dims.setdefault(m, a.shape[1]) != a.shape[1]:
raise ValueError(f"{r.rid}: {m} has {a.shape[1]} features, others {dims[m]}")
data[m].append(a)
rec_index.append(np.full(r.anchors.size, i, dtype=np.int64))
start.append(offset + r.anchors - spec.t_obs)
labels.append(r.labels)
events.append(r.events)
info[r.rid] = {"n_frames_100hz": r.n_frames, "n_frames_ds": int(T),
"timeline_offset": r.timeline_offset,
"class_counts": np.bincount(r.labels, minlength=3).astype(int).tolist(),
**r.label_stats}
offset += T
cat = lambda xs, dt=np.int64: np.concatenate(xs).astype(dt) if xs else np.zeros(0, dt) # noqa: E731
return Partition(name, [r.rid for r in recs], list(inputs),
{m: np.concatenate(v, axis=0) if v else np.zeros((0, 0), np.float32)
for m, v in data.items()},
cat(rec_index), cat(start), cat(labels), cat(events), spec.t_obs, info)
def cap_per_class(p: Partition, per_class_max: Optional[int], seed: int,
n_classes: int = 3) -> Tuple[Partition, Dict[str, Any]]:
"""Keep at most ``per_class_max`` anchors per class (uniform without replacement)."""
before = p.class_counts(n_classes)
if not per_class_max:
return p, {"per_class_max": None, "before": before, "after": before}
rng = np.random.default_rng([int(seed), 5])
keep = []
for c in range(n_classes):
idx = np.flatnonzero(p.labels == c)
if idx.size > per_class_max:
idx = np.sort(rng.choice(idx, size=per_class_max, replace=False))
keep.append(idx)
sel = np.sort(np.concatenate(keep))
q = p.subset(sel)
return q, {"per_class_max": int(per_class_max), "before": before, "after": q.class_counts(n_classes)}
def window_stats(p: Partition) -> Dict[str, Tuple[np.ndarray, np.ndarray]]:
"""Mean / SD of the frames of ``p``'s observation windows (with multiplicity)."""
total = next(iter(p.data.values())).shape[0]
cover = np.zeros(total + 1, dtype=np.int64)
np.add.at(cover, p.start, 1)
np.add.at(cover, p.start + p.t_obs, -1)
w = np.cumsum(cover)[:total].astype(np.float64)
W = w.sum()
if W <= 0:
raise ValueError(f"{p.name}: no windows to compute statistics from")
chunk = 1 << 16
out = {}
for m, x in p.data.items():
s1 = np.zeros(x.shape[1], np.float64)
for i in range(0, total, chunk):
s1 += w[i:i + chunk] @ x[i:i + chunk].astype(np.float64)
mu = s1 / W
s2 = np.zeros(x.shape[1], np.float64)
for i in range(0, total, chunk):
s2 += w[i:i + chunk] @ (x[i:i + chunk].astype(np.float64) - mu) ** 2
sd = np.sqrt(s2 / W)
sd = np.where(sd < 1e-6, 1.0, sd)
out[m] = (mu.astype(np.float32), sd.astype(np.float32))
return out
def normalise(p: Partition, stats: Dict[str, Tuple[np.ndarray, np.ndarray]]) -> None:
for m in p.data:
mu, sd = stats[m]
p.data[m] = np.nan_to_num((p.data[m] - mu) / sd, nan=0.0, posinf=0.0,
neginf=0.0).astype(np.float32)
def class_weights(counts: Sequence[int]) -> np.ndarray:
ny = np.asarray(counts, dtype=np.float64)
return ny.sum() / (len(ny) * np.maximum(ny, 1))