PULSE-code / pulse /data /modalities.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
8.01 kB
"""Modality registry: which file each stream lives in and which columns form
the model input.
The feature rules are the ones of the original benchmark pipeline, made
explicit and strict:
* ``emg`` 8 Myo channels ``emg_1..emg_8``. The Myo armband is worn on
the RIGHT forearm by every volunteer.
* ``imu`` 10 WitMotion sensors x 18 columns = 180 columns (WT0-WT7 on
the fingers, WT8 right wrist, WT9 left wrist). A file with any
other column count (volunteer v5: 72 columns, 4 sensors) does
not provide the modality.
* ``imu_wrist`` / ``imu_finger`` sensor subsets of ``imu`` (36 / 144 cols).
* ``eyetrack`` the 24 Dikablis core gaze columns below. Recordings missing
any of them (one-eye exports, e.g. v1s1 and v14s8) do not
provide the modality.
* ``eyetrack_gaze`` / ``eyetrack_ocular`` 2 scene-camera gaze columns / the
22 frame-independent ocular columns.
* ``pressure`` 50 fingertip cells ``R1(g)..R25(g), L1(g)..L25(g)``.
* ``mocap`` Qualisys skeleton TSV (``aligned_vXsY_s_Q.tsv``): every
position/quaternion column, positions made hip-relative, plus
the frame-to-frame velocity of the position columns.
* ``mocap_markers`` the 100 Hz marker CSV ``aligned_mocap_100hz.csv``
(marker XYZ, ``Type`` columns dropped).
* ``myo_quat`` Myo orientation quaternion (4 columns).
Quality rules (applied to the selected feature matrix after non-numeric
values and NaN/inf are mapped to 0): any ``|x| > 1e6`` is rejected as
corrupted; for modalities other than ``emg`` and ``pressure`` (which are
legitimately zero at rest) a zero fraction above 0.9 is rejected as dropout.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Dict, Optional, Tuple
#: Columns that are never features.
SKIP_COLS = frozenset({"Frame", "Time", "time", "UTC"})
SKIP_COL_SUFFIXES = (" Type",)
EMG_COLS: Tuple[str, ...] = tuple(f"emg_{i}" for i in range(1, 9))
PRESSURE_COLS: Tuple[str, ...] = tuple(
[f"R{i}(g)" for i in range(1, 26)] + [f"L{i}(g)" for i in range(1, 26)])
MYO_QUAT_COLS: Tuple[str, ...] = tuple(f"quat_{i}" for i in range(1, 5))
IMU_N_SENSORS = 10
IMU_COLS_PER_SENSOR = 18
IMU_N_COLS = IMU_N_SENSORS * IMU_COLS_PER_SENSOR # 180
IMU_SENSOR_SUBSETS: Dict[str, Tuple[str, ...]] = {
"imu_wrist": ("WT8_", "WT9_"),
"imu_finger": tuple(f"WT{i}_" for i in range(8)),
}
EYETRACK_CORE_COLS: Tuple[str, ...] = (
"Dikablis Glasses 3_Eye Data_Original_Pupil X",
"Dikablis Glasses 3_Eye Data_Original_Pupil Y",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Pupil X",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Pupil Y",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Pupil Area",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Pupil Height",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Pupil Width",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Fixations_Fixations",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Fixations_Fixations Duration",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Saccades_Saccades",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Saccades_Saccades Duration",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Saccades_Saccades Angle",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Pupil X",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Pupil Y",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Pupil Area",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Pupil Height",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Pupil Width",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Fixations_Fixations",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Fixations_Fixations Duration",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Saccades_Saccades",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Saccades_Saccades Duration",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Saccades_Saccades Angle",
"Dikablis Glasses 3_Field Data_Scene Cam_Original_Gaze_Gaze X",
"Dikablis Glasses 3_Field Data_Scene Cam_Original_Gaze_Gaze Y",
)
EYETRACK_SUBSETS: Dict[str, Tuple[str, ...]] = {
"eyetrack_gaze": ("Scene Cam_Original_Gaze",), # 2 columns
"eyetrack_ocular": ("Eye Data_Original",), # 22 columns
}
#: Raw streams (one file each). ``mocap_skeleton`` is the TSV.
STREAM_FILES: Dict[str, Optional[str]] = {
"emg": "aligned_emg_100hz.csv",
"imu": "aligned_imu_100hz.csv",
"eyetrack": "aligned_eyetrack_100hz.csv",
"pressure": "aligned_pressure_100hz.csv",
"mocap_markers": "aligned_mocap_100hz.csv",
"mocap_skeleton": None, # aligned_vXsY_s_Q.tsv in raw/ or at the recording root
"myo_quat": "aligned_myo_quat_100hz.csv",
"myo_pose": "aligned_myo_pose_100hz.csv",
}
#: Name of each stream in ``alignment_metadata.json`` (``aligned_modalities``).
#: This is the authority for whether a file must exist: the skeleton TSV is
#: ``mocap_sq``; ``mocap`` is the marker CSV.
STREAM_ALIGNMENT_NAME: Dict[str, str] = {
"emg": "emg",
"imu": "imu",
"eyetrack": "eyetrack",
"pressure": "pressure",
"mocap_markers": "mocap",
"mocap_skeleton": "mocap_sq",
"myo_quat": "myo_quat",
"myo_pose": "myo_pose",
}
#: Name of a stream in metadata/recordings.csv ``aligned_modalities`` (that
#: file has no entry for the skeleton TSV or the Myo orientation streams;
#: its ``mocap`` is the marker CSV).
STREAM_RECORDINGS_CSV_NAME: Dict[str, Optional[str]] = {
"emg": "emg",
"imu": "imu",
"eyetrack": "eyetrack",
"pressure": "pressure",
"mocap_markers": "mocap",
"mocap_skeleton": None,
"myo_quat": None,
"myo_pose": None,
}
@dataclass(frozen=True)
class ModalitySpec:
name: str
stream: str # raw stream it is computed from
expected_dim: Optional[int] # None: data dependent (mocap skeleton)
zero_check: bool = True # apply the >90 % zeros dropout rule
description: str = ""
extra: Dict[str, object] = field(default_factory=dict)
MODALITIES: Dict[str, ModalitySpec] = {
"emg": ModalitySpec("emg", "emg", 8, zero_check=False,
description="Myo sEMG, right forearm, 8 channels"),
"imu": ModalitySpec("imu", "imu", IMU_N_COLS,
description="10 IMUs x 18 columns (fingers WT0-7, wrists WT8/9)"),
"imu_wrist": ModalitySpec("imu_wrist", "imu", 36, description="IMU WT8+WT9 only"),
"imu_finger": ModalitySpec("imu_finger", "imu", 144, description="IMU WT0-WT7 only"),
"eyetrack": ModalitySpec("eyetrack", "eyetrack", 24, description="24 Dikablis core columns"),
"eyetrack_gaze": ModalitySpec("eyetrack_gaze", "eyetrack", 2,
description="scene-camera gaze X/Y"),
"eyetrack_ocular": ModalitySpec("eyetrack_ocular", "eyetrack", 22,
description="pupil + fixation/saccade columns"),
"pressure": ModalitySpec("pressure", "pressure", 50, zero_check=False,
description="fingertip pressure, 25 cells per hand"),
"mocap": ModalitySpec("mocap", "mocap_skeleton", None,
description="skeleton TSV, hip-relative + position velocity"),
"mocap_markers": ModalitySpec("mocap_markers", "mocap_markers", None,
description="100 Hz marker CSV (XYZ)"),
"myo_quat": ModalitySpec("myo_quat", "myo_quat", 4, description="Myo orientation"),
}
#: The five benchmark modalities in the paper's order.
BENCHMARK_MODALITIES: Tuple[str, ...] = ("mocap", "emg", "eyetrack", "imu", "pressure")
def get_spec(modality: str) -> ModalitySpec:
try:
return MODALITIES[modality]
except KeyError:
raise KeyError(f"unknown modality {modality!r}; known: {sorted(MODALITIES)}") from None
def base_stream(modality: str) -> str:
return get_spec(modality).stream