Download pulse/tasks/t5_data.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 17.9 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/tasks/t5_data.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/pulse/tasks/t5_data.py
-
curl -L -o t5_data.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/tasks/t5_data.py
17.9 kB
| """T5 samples: anchors, 3-class grasp-state labels and contact-event strata. | |
| Definitions (camera-ready, v1.1 labels) | |
| --------------------------------------- | |
| All streams are 100 Hz on disk and are decimated by ``downsample`` (5 -> 20 Hz) | |
| by taking every 5th frame from frame 0. | |
| * **Grasp-verb mask** (100 Hz): a frame is "grasp" if it lies inside any v1.1 | |
| segment whose ``action_name`` is in ``grasp_verbs`` (grasp, hold, pick_up, | |
| move, place, put_down, pull, rotate, insert, remove). Segment ``[start, | |
| end]`` covers frames ``round(100 start) .. round(100 end) - 1``, clipped to | |
| the loaded frames. Every verb must belong to the frozen v1.1 label space | |
| (``taxonomy_v1.1.json``, including the dropped ``support``); anything else | |
| raises. Nouns are not used. | |
| * **Contact** (20 Hz): at least ``contact_min_cells`` (3) of the 50 pressure | |
| cells (both hands) read more than ``contact_cell_threshold_g`` (10 g). | |
| * **Anchors** every ``anchor_stride_sec`` (0.25 s = 5 frames) at 20 Hz frame | |
| ``a`` with ``T_obs <= a <= T - T_fut``. The input is the past window | |
| ``[a - T_obs, a)`` (1 s = 20 frames); labels use only the future window | |
| ``[a, a + T_fut)`` (0.5 s = 10 frames). | |
| * **Label**: NoGrasp (0) unless at least ``majority_threshold`` (50 %) of the | |
| future window is grasp-verb; then SustainedGrasp (2) if the longest | |
| contiguous contact run in the future window is at least | |
| ``sustained_min_sec`` (0.3 s = 6 frames), else AttemptedGrasp (1). | |
| * **Event stratum** (Table 10): past / future window "in contact" if more than | |
| 50 % of its frames are contact frames: non-contact (no/no), pre-contact | |
| (no/yes), steady-grip (yes/yes), release (yes/no). SustainedGrasp anchors | |
| can only occur in pre-contact and steady-grip. | |
| Sample set | |
| ---------- | |
| A configuration names its model ``inputs`` and its ``sample_streams`` (a | |
| superset that always contains ``pressure``, which defines the labels). All | |
| sample streams are loaded and cut to a common length, so rows that differ only | |
| in their inputs (EMG vs EMG + pressure vs pressure only) see exactly the same | |
| anchors and labels. | |
| Release re-trim (v2s6, v13s2, v13s3) | |
| ------------------------------------ | |
| In v1.1 the EMG, eye-tracking and pressure files of these three recordings | |
| were re-trimmed to the MoCap capture window ``[front, front + n)`` recorded in | |
| ``alignment_metadata.json`` (``mocap_padding.mocap_sq``), while IMU and MoCap | |
| keep the full alignment timeline, and the annotations are on the full | |
| timeline. When a loaded stream has exactly ``n`` rows, the full-timeline | |
| streams are sliced to ``[front, front + n)`` and the annotation times are | |
| shifted by ``front`` frames, so frame ``i`` of every array and of the label | |
| timeline is the same instant (recorded per recording as ``timeline_offset``). | |
| Normalisation | |
| ------------- | |
| Per-channel z-score with the mean / SD (population) of the training frames, | |
| each frame weighted by the number of kept training anchors whose observation | |
| window contains it (the statistics of the training windows, as in the original | |
| pipeline). Inner-validation and test data use the training statistics. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| from dataclasses import dataclass, field | |
| from typing import Any, Dict, List, Optional, Sequence, Tuple | |
| import numpy as np | |
| from ..data.annotations import load_segments | |
| from ..data.io import load_alignment_metadata, load_modality | |
| from ..errors import DataQualityError | |
| from ..paths import DataRoot | |
| from ..taxonomy import Taxonomy, TaxonomyError | |
| SR_HZ = 100 | |
| CLASS_KEYS: Tuple[str, ...] = ("nograsp", "attempted", "sustained") | |
| CLASS_NAMES: Tuple[str, ...] = ("NoGrasp", "AttemptedGrasp", "SustainedGrasp") | |
| EVENT_NAMES: Tuple[str, ...] = ("non-contact", "pre-contact", "steady-grip", "release") | |
| #: Events in which SustainedGrasp anchors can exist (future window in contact). | |
| SUSTAINED_EVENTS: Tuple[str, ...] = ("pre-contact", "steady-grip") | |
| DEFAULT_GRASP_VERBS: Tuple[str, ...] = ("grasp", "hold", "pick_up", "move", "place", "put_down", | |
| "pull", "rotate", "insert", "remove") | |
| class LabelSpec: | |
| downsample: int = 5 | |
| t_obs_sec: float = 1.0 | |
| t_fut_sec: float = 0.5 | |
| anchor_stride_sec: float = 0.25 | |
| majority_threshold: float = 0.5 | |
| grasp_verbs: Tuple[str, ...] = DEFAULT_GRASP_VERBS | |
| contact_cell_threshold_g: float = 10.0 | |
| contact_min_cells: int = 3 | |
| sustained_min_sec: float = 0.3 | |
| def from_config(cls, cfg: Dict[str, Any]) -> "LabelSpec": | |
| d = dict(cfg or {}) | |
| if "grasp_verbs" in d: | |
| d["grasp_verbs"] = tuple(d["grasp_verbs"]) | |
| unknown = set(d) - set(cls.__dataclass_fields__) | |
| if unknown: | |
| raise ValueError(f"unknown T5 label settings: {sorted(unknown)}") | |
| return cls(**d) | |
| def sr(self) -> int: | |
| if SR_HZ % self.downsample: | |
| raise ValueError("downsample must divide 100") | |
| return SR_HZ // self.downsample | |
| def t_obs(self) -> int: | |
| return int(round(self.t_obs_sec * self.sr)) | |
| def t_fut(self) -> int: | |
| return int(round(self.t_fut_sec * self.sr)) | |
| def stride(self) -> int: | |
| return max(1, int(round(self.anchor_stride_sec * self.sr))) | |
| def sustained_frames(self) -> int: | |
| return int(round(self.sustained_min_sec * self.sr)) | |
| def check_against(self, tax: Taxonomy) -> None: | |
| bad = [v for v in self.grasp_verbs if v not in tax.verbs] | |
| if bad: | |
| raise TaxonomyError(f"grasp verbs {bad} are not v1.1 verbs") | |
| # --------------------------------------------------------------------------- | |
| # label derivation (pure functions, unit-tested) | |
| # --------------------------------------------------------------------------- | |
| def grasp_mask(segments, n_frames: int, grasp_verbs: Sequence[str], offset_frames: int = 0, | |
| tax: Optional[Taxonomy] = None) -> Tuple[np.ndarray, Dict[str, int]]: | |
| """100 Hz bool mask of grasp-verb frames for frames ``0 .. n_frames - 1``. | |
| ``offset_frames``: timeline frame of loaded frame 0 (release re-trim). | |
| """ | |
| gv = set(grasp_verbs) | |
| mask = np.zeros(int(n_frames), dtype=bool) | |
| st = {"segments": 0, "grasp_segments": 0, "grasp_clipped": 0, "grasp_outside": 0} | |
| for s in segments: | |
| st["segments"] += 1 | |
| if tax is not None and s.verb not in tax.verbs and s.verb not in tax.dropped_verbs: | |
| raise TaxonomyError(f"{s.rid} segment {s.index}: verb {s.verb!r} is not a v1.1 verb") | |
| if s.verb not in gv: | |
| continue | |
| st["grasp_segments"] += 1 | |
| i0 = int(round(s.start_sec * SR_HZ)) - offset_frames | |
| i1 = int(round(s.end_sec * SR_HZ)) - offset_frames | |
| j0, j1 = max(0, i0), min(int(n_frames), i1) | |
| if j1 <= j0: | |
| st["grasp_outside"] += 1 | |
| continue | |
| if (j0, j1) != (i0, i1): | |
| st["grasp_clipped"] += 1 | |
| mask[j0:j1] = True | |
| return mask, st | |
| def contact_frames(pressure_ds: np.ndarray, cell_threshold_g: float, min_cells: int) -> np.ndarray: | |
| return (np.asarray(pressure_ds) > cell_threshold_g).sum(axis=1) >= min_cells | |
| def longest_true_run(windows: np.ndarray) -> np.ndarray: | |
| """Longest run of True per row of a ``(N, T)`` bool array.""" | |
| w = np.asarray(windows, dtype=bool) | |
| cur = np.zeros(w.shape[0], dtype=np.int64) | |
| best = np.zeros(w.shape[0], dtype=np.int64) | |
| for t in range(w.shape[1]): | |
| cur = np.where(w[:, t], cur + 1, 0) | |
| best = np.maximum(best, cur) | |
| return best | |
| def derive_anchors(mask_ds: np.ndarray, contact_ds: np.ndarray, spec: LabelSpec | |
| ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: | |
| """Return ``(anchors, labels, events)`` for one recording at the decimated rate.""" | |
| mask_ds = np.asarray(mask_ds, dtype=bool) | |
| contact_ds = np.asarray(contact_ds, dtype=bool) | |
| if mask_ds.shape != contact_ds.shape: | |
| raise ValueError("mask and contact must have the same length") | |
| T = mask_ds.shape[0] | |
| to, tf = spec.t_obs, spec.t_fut | |
| if T - tf < to: | |
| e = np.zeros(0, dtype=np.int64) | |
| return e, e.copy(), e.copy() | |
| anchors = np.arange(to, T - tf + 1, spec.stride, dtype=np.int64) | |
| fut = anchors[:, None] + np.arange(tf)[None, :] | |
| past = anchors[:, None] - to + np.arange(to)[None, :] | |
| is_grasp = mask_ds[fut].mean(axis=1) >= spec.majority_threshold | |
| sustained = longest_true_run(contact_ds[fut]) >= spec.sustained_frames | |
| labels = np.where(is_grasp, np.where(sustained, 2, 1), 0).astype(np.int64) | |
| past_high = contact_ds[past].mean(axis=1) > 0.5 | |
| fut_high = contact_ds[fut].mean(axis=1) > 0.5 | |
| events = np.select([~past_high & ~fut_high, ~past_high & fut_high, past_high & fut_high], | |
| [0, 1, 2], default=3).astype(np.int64) | |
| return anchors, labels, events | |
| # --------------------------------------------------------------------------- | |
| # release re-trim (v2s6, v13s2, v13s3) | |
| # --------------------------------------------------------------------------- | |
| def release_trim_window(meta: dict) -> Optional[Tuple[int, int]]: | |
| """``(front, n)`` from ``mocap_padding`` if it covers part of the timeline, else None.""" | |
| n_total = meta.get("n_samples") | |
| pads = meta.get("mocap_padding") or {} | |
| pad = pads.get("mocap_sq") or pads.get("mocap") or {} | |
| front, n = pad.get("front_samples"), pad.get("n_frames") | |
| if not n_total or front is None or n is None: | |
| return None | |
| front, n, n_total = int(front), int(n), int(n_total) | |
| if n <= 0 or n >= n_total or front + n > n_total: | |
| return None | |
| return front, n | |
| def align_release_trim(rid: str, arrays: Dict[str, np.ndarray], meta: dict | |
| ) -> Tuple[Dict[str, np.ndarray], int]: | |
| """Put every stream on the same instants; return (arrays, timeline offset).""" | |
| win = release_trim_window(meta) | |
| if win is None: | |
| return arrays, 0 | |
| front, n = win | |
| trimmed = [m for m, a in arrays.items() if a.shape[0] == n] | |
| if not trimmed: | |
| return arrays, 0 | |
| out = dict(arrays) | |
| for m, a in arrays.items(): | |
| if a.shape[0] == n: | |
| continue | |
| if a.shape[0] < front + n: | |
| raise DataQualityError( | |
| f"{rid}: {m} has {a.shape[0]} rows; cannot align it with the re-trimmed " | |
| f"streams {trimmed} (window {front}+{n})", "release_trim_mismatch") | |
| out[m] = a[front:front + n] | |
| return out, front | |
| # --------------------------------------------------------------------------- | |
| # recordings and partitions | |
| # --------------------------------------------------------------------------- | |
| class RecordingSamples: | |
| rid: str | |
| arrays: Dict[str, np.ndarray] # model inputs, decimated, (T_ds, F) float32, raw units | |
| anchors: np.ndarray # decimated frame index of each anchor | |
| labels: np.ndarray | |
| events: np.ndarray | |
| n_frames: int # common length at 100 Hz | |
| timeline_offset: int # 100 Hz frames (release re-trim), 0 otherwise | |
| label_stats: Dict[str, int] | |
| annotation_md5: str | |
| def load_recording_samples(dr: DataRoot, rid: str, inputs: Sequence[str], | |
| sample_streams: Sequence[str], spec: LabelSpec, | |
| tax: Optional[Taxonomy] = None) -> RecordingSamples: | |
| if "pressure" not in sample_streams: | |
| raise ValueError("T5 sample_streams must contain 'pressure' (it defines the labels)") | |
| missing = [m for m in inputs if m not in sample_streams] | |
| if missing: | |
| raise ValueError(f"inputs {missing} are not in sample_streams") | |
| raw = {m: load_modality(dr, rid, m).data for m in sample_streams} # raises if absent | |
| raw, offset = align_release_trim(rid, raw, load_alignment_metadata(dr, rid)) | |
| n = min(a.shape[0] for a in raw.values()) | |
| ann_path = dr.annotation_path(rid, must_exist=True) | |
| segs = load_segments(dr, rid) | |
| mask, st = grasp_mask(segs, n, spec.grasp_verbs, offset_frames=offset, tax=tax) | |
| ds = spec.downsample | |
| mask_ds = mask[:n:ds] | |
| contact = contact_frames(raw["pressure"][:n:ds], spec.contact_cell_threshold_g, | |
| spec.contact_min_cells) | |
| anchors, labels, events = derive_anchors(mask_ds, contact, spec) | |
| arrays = {m: np.ascontiguousarray(raw[m][:n:ds], dtype=np.float32) for m in inputs} | |
| st = dict(st, n_anchors=int(anchors.size)) | |
| return RecordingSamples(rid, arrays, anchors, labels, events, int(n), int(offset), st, | |
| hashlib.md5(ann_path.read_bytes()).hexdigest()) | |
| class Partition: | |
| """All anchors of one partition, windows gathered from concatenated arrays.""" | |
| name: str | |
| rids: List[str] | |
| inputs: List[str] | |
| data: Dict[str, np.ndarray] # modality -> (sum T_ds, F) float32 | |
| rec_index: np.ndarray # (N,) recording index of each anchor | |
| start: np.ndarray # (N,) global row of the first frame of the window | |
| labels: np.ndarray | |
| events: np.ndarray | |
| t_obs: int | |
| recordings: Dict[str, Dict[str, Any]] = field(default_factory=dict) | |
| def n(self) -> int: | |
| return int(self.labels.size) | |
| def subset(self, idx: np.ndarray) -> "Partition": | |
| return Partition(self.name, self.rids, self.inputs, self.data, self.rec_index[idx], | |
| self.start[idx], self.labels[idx], self.events[idx], self.t_obs, | |
| self.recordings) | |
| def class_counts(self, n_classes: int = 3) -> List[int]: | |
| return np.bincount(self.labels, minlength=n_classes).astype(int).tolist() | |
| def event_counts(self) -> Dict[str, int]: | |
| c = np.bincount(self.events, minlength=len(EVENT_NAMES)) | |
| return {e: int(c[i]) for i, e in enumerate(EVENT_NAMES)} | |
| def window_index(self, idx: np.ndarray) -> np.ndarray: | |
| return self.start[idx][:, None] + np.arange(self.t_obs)[None, :] | |
| def build_partition(name: str, recs: Sequence[RecordingSamples], inputs: Sequence[str], | |
| spec: LabelSpec) -> Partition: | |
| data: Dict[str, List[np.ndarray]] = {m: [] for m in inputs} | |
| rec_index, start, labels, events = [], [], [], [] | |
| info: Dict[str, Dict[str, Any]] = {} | |
| offset = 0 | |
| dims: Dict[str, int] = {} | |
| for i, r in enumerate(recs): | |
| T = next(iter(r.arrays.values())).shape[0] | |
| for m in inputs: | |
| a = r.arrays[m] | |
| if a.shape[0] != T: | |
| raise ValueError(f"{r.rid}: {m} has {a.shape[0]} frames, expected {T}") | |
| if dims.setdefault(m, a.shape[1]) != a.shape[1]: | |
| raise ValueError(f"{r.rid}: {m} has {a.shape[1]} features, others {dims[m]}") | |
| data[m].append(a) | |
| rec_index.append(np.full(r.anchors.size, i, dtype=np.int64)) | |
| start.append(offset + r.anchors - spec.t_obs) | |
| labels.append(r.labels) | |
| events.append(r.events) | |
| info[r.rid] = {"n_frames_100hz": r.n_frames, "n_frames_ds": int(T), | |
| "timeline_offset": r.timeline_offset, | |
| "class_counts": np.bincount(r.labels, minlength=3).astype(int).tolist(), | |
| **r.label_stats} | |
| offset += T | |
| cat = lambda xs, dt=np.int64: np.concatenate(xs).astype(dt) if xs else np.zeros(0, dt) # noqa: E731 | |
| return Partition(name, [r.rid for r in recs], list(inputs), | |
| {m: np.concatenate(v, axis=0) if v else np.zeros((0, 0), np.float32) | |
| for m, v in data.items()}, | |
| cat(rec_index), cat(start), cat(labels), cat(events), spec.t_obs, info) | |
| def cap_per_class(p: Partition, per_class_max: Optional[int], seed: int, | |
| n_classes: int = 3) -> Tuple[Partition, Dict[str, Any]]: | |
| """Keep at most ``per_class_max`` anchors per class (uniform without replacement).""" | |
| before = p.class_counts(n_classes) | |
| if not per_class_max: | |
| return p, {"per_class_max": None, "before": before, "after": before} | |
| rng = np.random.default_rng([int(seed), 5]) | |
| keep = [] | |
| for c in range(n_classes): | |
| idx = np.flatnonzero(p.labels == c) | |
| if idx.size > per_class_max: | |
| idx = np.sort(rng.choice(idx, size=per_class_max, replace=False)) | |
| keep.append(idx) | |
| sel = np.sort(np.concatenate(keep)) | |
| q = p.subset(sel) | |
| return q, {"per_class_max": int(per_class_max), "before": before, "after": q.class_counts(n_classes)} | |
| def window_stats(p: Partition) -> Dict[str, Tuple[np.ndarray, np.ndarray]]: | |
| """Mean / SD of the frames of ``p``'s observation windows (with multiplicity).""" | |
| total = next(iter(p.data.values())).shape[0] | |
| cover = np.zeros(total + 1, dtype=np.int64) | |
| np.add.at(cover, p.start, 1) | |
| np.add.at(cover, p.start + p.t_obs, -1) | |
| w = np.cumsum(cover)[:total].astype(np.float64) | |
| W = w.sum() | |
| if W <= 0: | |
| raise ValueError(f"{p.name}: no windows to compute statistics from") | |
| chunk = 1 << 16 | |
| out = {} | |
| for m, x in p.data.items(): | |
| s1 = np.zeros(x.shape[1], np.float64) | |
| for i in range(0, total, chunk): | |
| s1 += w[i:i + chunk] @ x[i:i + chunk].astype(np.float64) | |
| mu = s1 / W | |
| s2 = np.zeros(x.shape[1], np.float64) | |
| for i in range(0, total, chunk): | |
| s2 += w[i:i + chunk] @ (x[i:i + chunk].astype(np.float64) - mu) ** 2 | |
| sd = np.sqrt(s2 / W) | |
| sd = np.where(sd < 1e-6, 1.0, sd) | |
| out[m] = (mu.astype(np.float32), sd.astype(np.float32)) | |
| return out | |
| def normalise(p: Partition, stats: Dict[str, Tuple[np.ndarray, np.ndarray]]) -> None: | |
| for m in p.data: | |
| mu, sd = stats[m] | |
| p.data[m] = np.nan_to_num((p.data[m] - mu) / sd, nan=0.0, posinf=0.0, | |
| neginf=0.0).astype(np.float32) | |
| def class_weights(counts: Sequence[int]) -> np.ndarray: | |
| ny = np.asarray(counts, dtype=np.float64) | |
| return ny.sum() / (len(ny) * np.maximum(ny, 1)) | |