File size: 8,010 Bytes
dffbc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 | """Modality registry: which file each stream lives in and which columns form
the model input.
The feature rules are the ones of the original benchmark pipeline, made
explicit and strict:
* ``emg`` 8 Myo channels ``emg_1..emg_8``. The Myo armband is worn on
the RIGHT forearm by every volunteer.
* ``imu`` 10 WitMotion sensors x 18 columns = 180 columns (WT0-WT7 on
the fingers, WT8 right wrist, WT9 left wrist). A file with any
other column count (volunteer v5: 72 columns, 4 sensors) does
not provide the modality.
* ``imu_wrist`` / ``imu_finger`` sensor subsets of ``imu`` (36 / 144 cols).
* ``eyetrack`` the 24 Dikablis core gaze columns below. Recordings missing
any of them (one-eye exports, e.g. v1s1 and v14s8) do not
provide the modality.
* ``eyetrack_gaze`` / ``eyetrack_ocular`` 2 scene-camera gaze columns / the
22 frame-independent ocular columns.
* ``pressure`` 50 fingertip cells ``R1(g)..R25(g), L1(g)..L25(g)``.
* ``mocap`` Qualisys skeleton TSV (``aligned_vXsY_s_Q.tsv``): every
position/quaternion column, positions made hip-relative, plus
the frame-to-frame velocity of the position columns.
* ``mocap_markers`` the 100 Hz marker CSV ``aligned_mocap_100hz.csv``
(marker XYZ, ``Type`` columns dropped).
* ``myo_quat`` Myo orientation quaternion (4 columns).
Quality rules (applied to the selected feature matrix after non-numeric
values and NaN/inf are mapped to 0): any ``|x| > 1e6`` is rejected as
corrupted; for modalities other than ``emg`` and ``pressure`` (which are
legitimately zero at rest) a zero fraction above 0.9 is rejected as dropout.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Dict, Optional, Tuple
#: Columns that are never features.
SKIP_COLS = frozenset({"Frame", "Time", "time", "UTC"})
SKIP_COL_SUFFIXES = (" Type",)
EMG_COLS: Tuple[str, ...] = tuple(f"emg_{i}" for i in range(1, 9))
PRESSURE_COLS: Tuple[str, ...] = tuple(
[f"R{i}(g)" for i in range(1, 26)] + [f"L{i}(g)" for i in range(1, 26)])
MYO_QUAT_COLS: Tuple[str, ...] = tuple(f"quat_{i}" for i in range(1, 5))
IMU_N_SENSORS = 10
IMU_COLS_PER_SENSOR = 18
IMU_N_COLS = IMU_N_SENSORS * IMU_COLS_PER_SENSOR # 180
IMU_SENSOR_SUBSETS: Dict[str, Tuple[str, ...]] = {
"imu_wrist": ("WT8_", "WT9_"),
"imu_finger": tuple(f"WT{i}_" for i in range(8)),
}
EYETRACK_CORE_COLS: Tuple[str, ...] = (
"Dikablis Glasses 3_Eye Data_Original_Pupil X",
"Dikablis Glasses 3_Eye Data_Original_Pupil Y",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Pupil X",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Pupil Y",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Pupil Area",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Pupil Height",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Pupil Width",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Fixations_Fixations",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Fixations_Fixations Duration",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Saccades_Saccades",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Saccades_Saccades Duration",
"Dikablis Glasses 3_Eye Data_Original_Left Eye_Saccades_Saccades Angle",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Pupil X",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Pupil Y",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Pupil Area",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Pupil Height",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Pupil Width",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Fixations_Fixations",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Fixations_Fixations Duration",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Saccades_Saccades",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Saccades_Saccades Duration",
"Dikablis Glasses 3_Eye Data_Original_Right Eye_Saccades_Saccades Angle",
"Dikablis Glasses 3_Field Data_Scene Cam_Original_Gaze_Gaze X",
"Dikablis Glasses 3_Field Data_Scene Cam_Original_Gaze_Gaze Y",
)
EYETRACK_SUBSETS: Dict[str, Tuple[str, ...]] = {
"eyetrack_gaze": ("Scene Cam_Original_Gaze",), # 2 columns
"eyetrack_ocular": ("Eye Data_Original",), # 22 columns
}
#: Raw streams (one file each). ``mocap_skeleton`` is the TSV.
STREAM_FILES: Dict[str, Optional[str]] = {
"emg": "aligned_emg_100hz.csv",
"imu": "aligned_imu_100hz.csv",
"eyetrack": "aligned_eyetrack_100hz.csv",
"pressure": "aligned_pressure_100hz.csv",
"mocap_markers": "aligned_mocap_100hz.csv",
"mocap_skeleton": None, # aligned_vXsY_s_Q.tsv in raw/ or at the recording root
"myo_quat": "aligned_myo_quat_100hz.csv",
"myo_pose": "aligned_myo_pose_100hz.csv",
}
#: Name of each stream in ``alignment_metadata.json`` (``aligned_modalities``).
#: This is the authority for whether a file must exist: the skeleton TSV is
#: ``mocap_sq``; ``mocap`` is the marker CSV.
STREAM_ALIGNMENT_NAME: Dict[str, str] = {
"emg": "emg",
"imu": "imu",
"eyetrack": "eyetrack",
"pressure": "pressure",
"mocap_markers": "mocap",
"mocap_skeleton": "mocap_sq",
"myo_quat": "myo_quat",
"myo_pose": "myo_pose",
}
#: Name of a stream in metadata/recordings.csv ``aligned_modalities`` (that
#: file has no entry for the skeleton TSV or the Myo orientation streams;
#: its ``mocap`` is the marker CSV).
STREAM_RECORDINGS_CSV_NAME: Dict[str, Optional[str]] = {
"emg": "emg",
"imu": "imu",
"eyetrack": "eyetrack",
"pressure": "pressure",
"mocap_markers": "mocap",
"mocap_skeleton": None,
"myo_quat": None,
"myo_pose": None,
}
@dataclass(frozen=True)
class ModalitySpec:
name: str
stream: str # raw stream it is computed from
expected_dim: Optional[int] # None: data dependent (mocap skeleton)
zero_check: bool = True # apply the >90 % zeros dropout rule
description: str = ""
extra: Dict[str, object] = field(default_factory=dict)
MODALITIES: Dict[str, ModalitySpec] = {
"emg": ModalitySpec("emg", "emg", 8, zero_check=False,
description="Myo sEMG, right forearm, 8 channels"),
"imu": ModalitySpec("imu", "imu", IMU_N_COLS,
description="10 IMUs x 18 columns (fingers WT0-7, wrists WT8/9)"),
"imu_wrist": ModalitySpec("imu_wrist", "imu", 36, description="IMU WT8+WT9 only"),
"imu_finger": ModalitySpec("imu_finger", "imu", 144, description="IMU WT0-WT7 only"),
"eyetrack": ModalitySpec("eyetrack", "eyetrack", 24, description="24 Dikablis core columns"),
"eyetrack_gaze": ModalitySpec("eyetrack_gaze", "eyetrack", 2,
description="scene-camera gaze X/Y"),
"eyetrack_ocular": ModalitySpec("eyetrack_ocular", "eyetrack", 22,
description="pupil + fixation/saccade columns"),
"pressure": ModalitySpec("pressure", "pressure", 50, zero_check=False,
description="fingertip pressure, 25 cells per hand"),
"mocap": ModalitySpec("mocap", "mocap_skeleton", None,
description="skeleton TSV, hip-relative + position velocity"),
"mocap_markers": ModalitySpec("mocap_markers", "mocap_markers", None,
description="100 Hz marker CSV (XYZ)"),
"myo_quat": ModalitySpec("myo_quat", "myo_quat", 4, description="Myo orientation"),
}
#: The five benchmark modalities in the paper's order.
BENCHMARK_MODALITIES: Tuple[str, ...] = ("mocap", "emg", "eyetrack", "imu", "pressure")
def get_spec(modality: str) -> ModalitySpec:
try:
return MODALITIES[modality]
except KeyError:
raise KeyError(f"unknown modality {modality!r}; known: {sorted(MODALITIES)}") from None
def base_stream(modality: str) -> str:
return get_spec(modality).stream
|