Download pulse/tasks/t2_data.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 16.4 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/tasks/t2_data.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/pulse/tasks/t2_data.py
-
curl -L -o t2_data.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/tasks/t2_data.py
16.4 kB
| """T2 samples: sliding windows (primary) and annotated segments (secondary). | |
| Timeline | |
| Annotations are in seconds on the recording's alignment timeline | |
| (``n_samples`` frames at 100 Hz). Every benchmark stream of a recording is | |
| placed on that timeline: a stream with ``n_samples`` (or ``n_samples + 1``) | |
| rows starts at frame 0; in v1.1 the EMG, eye-tracking and pressure files of | |
| v2s6, v13s2 and v13s3 were re-trimmed for the release to the MoCap capture | |
| window ``[front, front + n)`` (``alignment_metadata.json``, | |
| ``mocap_padding``) and start at frame ``front``. Any other row count raises. | |
| Evaluation span | |
| Per recording, the frames covered by *every* benchmark modality | |
| (MoCap, EMG, eye tracking, IMU, pressure) that the availability table marks | |
| usable, with the start rounded up to the 20 Hz grid. It is computed from | |
| the frozen availability table and the alignment metadata only, so it is the | |
| same for every model row whatever its input modalities: two rows that load | |
| a recording are scored on identical windows / segments, and the common | |
| test set of the aggregator compares like with like. | |
| Signals | |
| Each modality is loaded with the strict loaders (``pulse.data.io``), | |
| checked against the availability row count, cut to the span and decimated | |
| to 20 Hz (every 5th sample; the grid is aligned to the alignment timeline). | |
| Z-scoring uses per-modality mean / SD over the frames of the training | |
| samples (each frame counted once per sample that contains it), applied to | |
| train, inner-validation and test alike. | |
| Labels (frozen taxonomy v1.1) | |
| A segment is kept if ``Taxonomy.classify`` keeps it and it lasts at least | |
| ``segment.min_sec``. Sliding protocol: windows of ``window_sec`` at | |
| ``stride_sec`` over the span; a window takes the labels of the kept segment | |
| that contains its centre frame and is skipped if there is none. Segment | |
| protocol: one sample per kept segment inside the span, its own | |
| ``[start, end)``; segments longer than ``segment.max_frames`` are | |
| centre-cropped to that length. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import math | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| from dataclasses import dataclass, field | |
| from typing import Dict, List, Optional, Sequence, Tuple | |
| import numpy as np | |
| from ..data.annotations import AnnotationError, load_segments | |
| from ..data.availability import Availability | |
| from ..data.io import load_alignment_metadata, load_modality | |
| from ..data.modalities import BENCHMARK_MODALITIES | |
| from ..errors import DataQualityError | |
| from ..paths import DataRoot | |
| from ..taxonomy import Taxonomy | |
| FS = 100 # Hz, every released stream | |
| HEADS = ("verb_fine", "verb_composite", "noun", "hand") | |
| PROTOCOLS = ("sliding", "segment") | |
| # --------------------------------------------------------------------------- | |
| # timeline | |
| # --------------------------------------------------------------------------- | |
| class Timeline: | |
| rid: str | |
| n_total: int # alignment timeline length (frames, 100 Hz) | |
| offsets: Dict[str, int] # usable benchmark modality -> first timeline frame | |
| lengths: Dict[str, int] # usable benchmark modality -> rows (availability) | |
| start: int # evaluation span start (multiple of ds) | |
| n_frames_ds: int # span length at the decimated rate | |
| ds: int | |
| def end(self) -> int: | |
| return self.start + self.ds * self.n_frames_ds | |
| def _release_window(meta: dict) -> Optional[Tuple[int, int]]: | |
| pads = meta.get("mocap_padding") or {} | |
| pad = pads.get("mocap_sq") or pads.get("mocap") or {} | |
| front, n = pad.get("front_samples"), pad.get("n_frames") | |
| if front is None or n is None: | |
| return None | |
| return int(front), int(n) | |
| def recording_timeline(av: Availability, dr: DataRoot, rid: str, ds: int, | |
| modalities: Sequence[str] = BENCHMARK_MODALITIES) -> Timeline: | |
| rec = av.table["recordings"][rid] | |
| n_total = rec.get("alignment_n_samples") or rec.get("n_samples_100hz") | |
| if not n_total: | |
| raise DataQualityError(f"{rid}: no alignment length in the availability table", "no_length") | |
| n_total = int(n_total) | |
| lengths = {m: int(rec["modalities"][m]["n_frames"]) for m in modalities | |
| if rec["modalities"].get(m, {}).get("status") == "ok"} | |
| if not lengths: | |
| raise DataQualityError(f"{rid}: no usable benchmark modality", "no_modality") | |
| offsets: Dict[str, int] = {} | |
| window = None | |
| for m, L in lengths.items(): | |
| if L in (n_total, n_total + 1): | |
| offsets[m] = 0 | |
| continue | |
| if window is None: | |
| window = _release_window(load_alignment_metadata(dr, rid)) or (-1, -1) | |
| front, n = window | |
| if L == n and 0 <= front and front + n <= n_total: | |
| offsets[m] = front | |
| else: | |
| raise DataQualityError( | |
| f"{rid}: {m} has {L} rows; the alignment timeline has {n_total} frames and the " | |
| f"release window is {window} -- cannot place the stream", "timeline") | |
| a = max(offsets.values()) | |
| b = min(min(offsets[m] + L for m, L in lengths.items()), n_total) | |
| a = int(math.ceil(a / ds) * ds) | |
| n_ds = max(0, (b - a) // ds) | |
| return Timeline(rid, n_total, offsets, lengths, a, n_ds, ds) | |
| # --------------------------------------------------------------------------- | |
| # labels and samples | |
| # --------------------------------------------------------------------------- | |
| class LabelledSpan: | |
| start: int # timeline frame (100 Hz), inclusive | |
| end: int # exclusive | |
| labels: Tuple[int, int, int, int] | |
| def kept_segments(dr: DataRoot, rid: str, tax: Taxonomy, min_sec: float, | |
| counts: Dict[str, int]) -> List[LabelledSpan]: | |
| out: List[LabelledSpan] = [] | |
| for s in load_segments(dr, rid): | |
| counts["segments_total"] = counts.get("segments_total", 0) + 1 | |
| lab, why = tax.classify(s.verb, s.noun, s.hand) | |
| if lab is None: | |
| key = "dropped_" + why.split(":")[0].replace("dropped_", "") | |
| counts[key] = counts.get(key, 0) + 1 | |
| continue | |
| if s.duration_sec < min_sec: | |
| counts["dropped_short"] = counts.get("dropped_short", 0) + 1 | |
| continue | |
| out.append(LabelledSpan(int(round(s.start_sec * FS)), int(round(s.end_sec * FS)), | |
| tuple(int(lab[h]) for h in HEADS))) | |
| out.sort(key=lambda x: x.start) | |
| for a, b in zip(out, out[1:]): | |
| if b.start < a.end: | |
| raise AnnotationError(f"{rid}: kept segments overlap ({a.start / FS:.2f}-" | |
| f"{a.end / FS:.2f} s and {b.start / FS:.2f}-{b.end / FS:.2f} s)") | |
| return out | |
| def sliding_samples(tl: Timeline, spans: List[LabelledSpan], window_sec: float, | |
| stride_sec: float, counts: Dict[str, int]): | |
| ds = tl.ds | |
| win = int(round(window_sec * FS / ds)) | |
| stride = int(round(stride_sec * FS / ds)) | |
| if win < 1 or stride < 1: | |
| raise ValueError("window and stride must be at least one frame at the decimated rate") | |
| starts = np.asarray([s.start for s in spans], dtype=np.int64) | |
| ends = np.asarray([s.end for s in spans], dtype=np.int64) | |
| s0, s1, labs = [], [], [] | |
| for w0 in range(0, tl.n_frames_ds - win + 1, stride): | |
| counts["windows_total"] = counts.get("windows_total", 0) + 1 | |
| centre = tl.start + ds * (w0 + win // 2) | |
| i = int(np.searchsorted(starts, centre, side="right")) - 1 | |
| if i < 0 or centre >= ends[i]: | |
| counts["windows_unlabelled"] = counts.get("windows_unlabelled", 0) + 1 | |
| continue | |
| s0.append(w0) | |
| s1.append(w0 + win) | |
| labs.append(spans[i].labels) | |
| return s0, s1, labs | |
| def segment_samples(tl: Timeline, spans: List[LabelledSpan], max_frames: int, min_frames: int, | |
| counts: Dict[str, int]): | |
| ds = tl.ds | |
| s0, s1, labs = [], [], [] | |
| for sp in spans: | |
| a = int(math.floor((sp.start - tl.start) / ds + 0.5)) | |
| b = int(math.floor((sp.end - tl.start) / ds + 0.5)) | |
| if a < 0 or b > tl.n_frames_ds: | |
| counts["dropped_outside_span"] = counts.get("dropped_outside_span", 0) + 1 | |
| continue | |
| if b - a < min_frames: | |
| counts["dropped_too_few_frames"] = counts.get("dropped_too_few_frames", 0) + 1 | |
| continue | |
| if b - a > max_frames: | |
| c = (a + b) // 2 | |
| a = c - max_frames // 2 | |
| b = a + max_frames | |
| counts["cropped_long"] = counts.get("cropped_long", 0) + 1 | |
| s0.append(a) | |
| s1.append(b) | |
| labs.append(sp.labels) | |
| return s0, s1, labs | |
| # --------------------------------------------------------------------------- | |
| # recordings | |
| # --------------------------------------------------------------------------- | |
| class RecordingSamples: | |
| rid: str | |
| arrays: Dict[str, np.ndarray] # modality -> (n_frames_ds, F) float32 | |
| starts: np.ndarray # (N,) decimated frame index into arrays | |
| ends: np.ndarray # (N,) exclusive | |
| labels: np.ndarray # (N, 4) int64 in HEADS order | |
| timeline: Timeline | |
| counts: Dict[str, int] = field(default_factory=dict) | |
| def n(self) -> int: | |
| return int(self.starts.shape[0]) | |
| def load_recording_samples(dr: DataRoot, av: Availability, tax: Taxonomy, rid: str, | |
| modalities: Sequence[str], cfg: dict) -> RecordingSamples: | |
| ds = int(cfg["downsample"]) | |
| tl = recording_timeline(av, dr, rid, ds) | |
| arrays: Dict[str, np.ndarray] = {} | |
| for m in modalities: | |
| if m not in tl.offsets: | |
| raise DataQualityError(f"{rid}: {m} is not usable according to the availability table " | |
| "but the manifest expects it", "manifest") | |
| ma = load_modality(dr, rid, m) | |
| L = ma.data.shape[0] | |
| if L != tl.lengths[m]: | |
| raise DataQualityError(f"{rid}: {m} has {L} rows, the availability table says " | |
| f"{tl.lengths[m]} (data copy differs from v1.1?)", "rows") | |
| a = tl.start - tl.offsets[m] | |
| arr = ma.data[a:a + ds * tl.n_frames_ds:ds] | |
| if arr.shape[0] != tl.n_frames_ds: | |
| raise DataQualityError(f"{rid}: {m} does not cover the evaluation span", "span") | |
| arrays[m] = np.ascontiguousarray(arr, dtype=np.float32) | |
| counts: Dict[str, int] = {} | |
| spans = kept_segments(dr, rid, tax, float(cfg["segment"]["min_sec"]), counts) | |
| if cfg["protocol"] == "sliding": | |
| sc = cfg["sliding"] | |
| if sc.get("label", "centre") != "centre": | |
| raise ValueError("sliding.label must be 'centre'") | |
| s0, s1, labs = sliding_samples(tl, spans, float(sc["window_sec"]), float(sc["stride_sec"]), | |
| counts) | |
| elif cfg["protocol"] == "segment": | |
| s0, s1, labs = segment_samples(tl, spans, int(cfg["segment"]["max_frames"]), | |
| int(cfg.get("min_frames", 4)), counts) | |
| else: | |
| raise ValueError(f"protocol must be one of {PROTOCOLS}, got {cfg['protocol']!r}") | |
| counts["samples"] = len(s0) | |
| return RecordingSamples(rid, arrays, np.asarray(s0, np.int64), np.asarray(s1, np.int64), | |
| np.asarray(labs, np.int64).reshape(-1, len(HEADS)), tl, counts) | |
| def load_partition(dr: DataRoot, av: Availability, tax: Taxonomy, rids: Sequence[str], | |
| modalities: Sequence[str], cfg: dict, workers: int = 1, | |
| log=print, name: str = "") -> List[RecordingSamples]: | |
| t0 = time.time() | |
| def one(rid): | |
| return load_recording_samples(dr, av, tax, rid, modalities, cfg) | |
| if workers > 1 and len(rids) > 1: | |
| with ThreadPoolExecutor(max_workers=workers) as ex: | |
| recs = list(ex.map(one, rids)) | |
| else: | |
| recs = [one(r) for r in rids] | |
| log(f" loaded {name}: {len(recs)} recordings, {sum(r.n for r in recs)} samples " | |
| f"({time.time() - t0:.0f}s)") | |
| return recs | |
| # --------------------------------------------------------------------------- | |
| # normalisation | |
| # --------------------------------------------------------------------------- | |
| def normalisation_stats(recs: Sequence[RecordingSamples], modalities: Sequence[str] | |
| ) -> Dict[str, Tuple[np.ndarray, np.ndarray]]: | |
| """Per-modality mean / SD over the frames of the training samples.""" | |
| out = {} | |
| for m in modalities: | |
| s = s2 = None | |
| cnt = 0.0 | |
| for r in recs: | |
| if r.n == 0: | |
| continue | |
| T = r.arrays[m].shape[0] | |
| cov = np.zeros(T + 1, dtype=np.float64) | |
| np.add.at(cov, r.starts, 1.0) | |
| np.add.at(cov, r.ends, -1.0) | |
| w = np.cumsum(cov)[:T] | |
| x = r.arrays[m].astype(np.float64) | |
| ws, ws2 = w @ x, w @ (x * x) | |
| s = ws if s is None else s + ws | |
| s2 = ws2 if s2 is None else s2 + ws2 | |
| cnt += float(w.sum()) | |
| if not cnt: | |
| raise ValueError(f"no training frames for modality {m}") | |
| mu = s / cnt | |
| sd = np.sqrt(np.maximum(s2 / cnt - mu * mu, 0.0)) | |
| sd[sd < 1e-8] = 1.0 | |
| out[m] = (mu.astype(np.float32), sd.astype(np.float32)) | |
| return out | |
| def apply_normalisation(recs: Sequence[RecordingSamples], | |
| stats: Dict[str, Tuple[np.ndarray, np.ndarray]]) -> None: | |
| for r in recs: | |
| for m, (mu, sd) in stats.items(): | |
| z = (r.arrays[m] - mu) / sd | |
| r.arrays[m] = np.nan_to_num(z, nan=0.0, posinf=0.0, neginf=0.0).astype(np.float32) | |
| def stats_md5(stats: Dict[str, Tuple[np.ndarray, np.ndarray]]) -> str: | |
| h = hashlib.md5() | |
| for m in sorted(stats): | |
| h.update(m.encode()) | |
| h.update(np.ascontiguousarray(stats[m][0]).tobytes()) | |
| h.update(np.ascontiguousarray(stats[m][1]).tobytes()) | |
| return h.hexdigest() | |
| # --------------------------------------------------------------------------- | |
| # torch dataset | |
| # --------------------------------------------------------------------------- | |
| class SampleSet: | |
| """Flat view over the samples of several recordings (map-style dataset).""" | |
| def __init__(self, recs: Sequence[RecordingSamples], modalities: Sequence[str], | |
| index: Optional[np.ndarray] = None): | |
| self.recs = list(recs) | |
| self.modalities = list(modalities) | |
| full = np.asarray([(i, j) for i, r in enumerate(self.recs) for j in range(r.n)], | |
| dtype=np.int64).reshape(-1, 2) | |
| self.index = full if index is None else full[np.asarray(index, dtype=np.int64)] | |
| def __len__(self) -> int: | |
| return int(self.index.shape[0]) | |
| def __getitem__(self, k: int): | |
| i, j = self.index[k] | |
| r = self.recs[i] | |
| a, b = int(r.starts[j]), int(r.ends[j]) | |
| return {m: r.arrays[m][a:b] for m in self.modalities}, r.labels[j], int(i) | |
| def labels(self) -> np.ndarray: | |
| if not len(self): | |
| return np.zeros((0, len(HEADS)), np.int64) | |
| return np.stack([self.recs[i].labels[j] for i, j in self.index]) | |
| def rec_index(self) -> np.ndarray: | |
| return self.index[:, 0].copy() | |
| def modality_dims(self) -> Dict[str, int]: | |
| r = self.recs[0] | |
| return {m: int(r.arrays[m].shape[1]) for m in self.modalities} | |
| def collate(batch): | |
| import torch | |
| xs, ys, ri = zip(*batch) | |
| mods = list(xs[0]) | |
| lens = np.asarray([x[mods[0]].shape[0] for x in xs], dtype=np.int64) | |
| T = int(lens.max()) | |
| out = {} | |
| for m in mods: | |
| F = xs[0][m].shape[1] | |
| buf = np.zeros((len(xs), T, F), dtype=np.float32) | |
| for i, x in enumerate(xs): | |
| buf[i, :x[m].shape[0]] = x[m] | |
| out[m] = torch.from_numpy(buf) | |
| mask = torch.from_numpy(np.arange(T)[None, :] < lens[:, None]) | |
| y = torch.from_numpy(np.stack(ys).astype(np.int64)) | |
| return out, mask, y, torch.as_tensor(ri, dtype=torch.int64) | |
| def label_counts(labels: np.ndarray, sizes: Dict[str, int]) -> Dict[str, List[int]]: | |
| return {h: np.bincount(labels[:, i], minlength=sizes[h]).astype(int).tolist() | |
| for i, h in enumerate(HEADS)} | |