File size: 7,288 Bytes
dffbc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | import json
import os
import shutil
import numpy as np
import pytest
from pulse.data import modalities as M
from pulse.data.availability import Availability, build_availability
from pulse.data.io import load_modality, load_recording, quality_check
from pulse.data.manifest import expected_manifest, verify_loaded
from pulse.errors import (AmbiguousFileError, DataQualityError, DataRootError,
ManifestMismatchError, MissingFileError, ModalityUnavailableError)
from pulse.paths import DataRoot, resolve_root
from pulse.splits import load_split
from conftest import mini_split
def test_root_resolution(tmp_path, mini_root, monkeypatch):
monkeypatch.delenv("PULSE_ROOT", raising=False)
with pytest.raises(DataRootError):
resolve_root(None)
with pytest.raises(DataRootError):
resolve_root(tmp_path / "nope")
monkeypatch.setenv("PULSE_ROOT", str(mini_root))
assert resolve_root(None) == mini_root.resolve()
monkeypatch.setenv("PULSE_ROOT", "$UNDEFINED_PULSE_VAR/x")
with pytest.raises(DataRootError):
resolve_root(None)
def test_tsv_locations(mini_root):
dr = DataRoot.from_arg(mini_root)
assert dr.mocap_tsv_path("v1s1").parent.name == "raw"
assert dr.mocap_tsv_path("v1s2").parent.name == "s2" # recording root
with pytest.raises(MissingFileError):
dr.mocap_tsv_path("v5s1")
shutil.copy(dr.mocap_tsv_path("v1s1"), dr.recording_dir("v1s1") / "aligned_v1s1_s_Q.tsv")
with pytest.raises(AmbiguousFileError):
dr.mocap_tsv_path("v1s1")
def test_modality_dims(mini_root):
dr = DataRoot.from_arg(mini_root)
dims = {m: load_modality(dr, "v1s1", m).data.shape[1]
for m in ("emg", "imu", "imu_wrist", "imu_finger", "eyetrack", "eyetrack_gaze",
"eyetrack_ocular", "pressure", "myo_quat")}
assert dims == {"emg": 8, "imu": 180, "imu_wrist": 36, "imu_finger": 144, "eyetrack": 24,
"eyetrack_gaze": 2, "eyetrack_ocular": 22, "pressure": 50, "myo_quat": 4}
mo = load_modality(dr, "v1s2", "mocap") # TSV at the recording root
n_feat = 7 * 5
assert mo.data.shape == (240, n_feat + 3 * 5)
assert np.allclose(mo.data[:, mo.columns.index("Hips_X")], 0)
assert mo.data.dtype == np.float32
def test_v5_imu_is_missing_modality(mini_root):
dr = DataRoot.from_arg(mini_root)
with pytest.raises(ModalityUnavailableError) as e:
load_modality(dr, "v5s1", "imu")
assert e.value.reason == "imu_columns_72"
with pytest.raises(ModalityUnavailableError):
load_modality(dr, "v5s1", "imu_wrist")
def test_eyetrack_core_columns(mini_root):
dr = DataRoot.from_arg(mini_root)
with pytest.raises(ModalityUnavailableError) as e:
load_modality(dr, "v2s2", "eyetrack")
assert e.value.reason == "eyetrack_core_columns"
def test_missing_file_raises(mini_root):
dr = DataRoot.from_arg(mini_root)
with pytest.raises(MissingFileError):
load_modality(dr, "v3s1", "pressure")
with pytest.raises(MissingFileError):
load_recording(dr, "v5s1", ["emg", "mocap"])
def test_quality_rules():
with pytest.raises(DataQualityError) as e:
quality_check(np.array([[0.0, 2e6]]), "x", "imu")
assert e.value.reason == "extreme_values"
z = np.zeros((100, 4))
with pytest.raises(DataQualityError):
quality_check(z, "x", "imu")
quality_check(np.zeros((100, 8)), "x", "emg") # EMG and pressure may rest at 0
quality_check(np.zeros((100, 50)), "x", "pressure")
def test_load_recording_gate(mini_root):
dr = DataRoot.from_arg(mini_root)
rec = load_recording(dr, "v1s1", ["emg", "imu"], gate_modalities=["pressure"])
assert set(rec.arrays) == {"emg", "imu"} and rec.concat().shape == (240, 188)
with pytest.raises(MissingFileError):
load_recording(dr, "v3s1", ["emg"], gate_modalities=["pressure"])
def _availability(mini_root, tmp_path):
dr = DataRoot.from_arg(mini_root)
table = build_availability(dr)
p = tmp_path / "availability.json"
p.write_text(json.dumps(table), encoding="utf-8")
return Availability.load(p), table
def test_availability_statuses(mini_root, tmp_path):
av, table = _availability(mini_root, tmp_path)
assert av.status("v5s1", "imu") == "unavailable:imu_columns_72"
assert av.status("v5s1", "mocap") == "absent"
assert av.status("v3s1", "pressure") == "absent"
assert av.status("v2s2", "eyetrack") == "unavailable:eyetrack_core_columns"
assert av.status("v1s2", "mocap") == "ok"
assert table["summary"]["mocap_tsv_locations"] == {"raw": 8, "recording_root": 1}
assert table["summary"]["errors"] == []
assert not av.has_annotation("v1s9") and av.has_annotation("v1s1")
def test_recording_without_any_data(mini_root, tmp_path):
import csv
with open(mini_root / "metadata" / "recordings.csv", "a", newline="", encoding="utf-8") as f:
csv.writer(f).writerow(["v6s3", "v6", "s3", "", "", "", "emg|imu", 0, "train"])
av, table = _availability(mini_root, tmp_path)
assert table["recordings"]["v6s3"]["no_data"] is True
assert table["summary"]["recordings_without_data"] == ["v6s3"]
assert av.status("v6s3", "emg") == "absent" and table["summary"]["errors"] == []
def test_availability_flags_missing_listed_file(mini_root, tmp_path):
os.remove(mini_root / "data" / "v6" / "s1" / "aligned_pressure_100hz.csv")
av, table = _availability(mini_root, tmp_path)
assert av.status("v6s1", "pressure") == "missing_file"
assert any("v6s1" in e for e in table["summary"]["errors"])
sp = load_split(mini_split(tmp_path / "split.json"))
with pytest.raises(ManifestMismatchError):
expected_manifest(av, sp.fold(2), ["emg", "pressure"])
def test_manifest_and_verify(mini_root, tmp_path):
av, _ = _availability(mini_root, tmp_path)
sp = load_split(mini_split(tmp_path / "split.json"))
man = expected_manifest(av, sp.fold(1), ["emg", "mocap"])
assert man.partitions["test"] == ["v2s1", "v2s2"] # v5 has no MoCap
assert man.excluded["test"] == {"v5s1": "mocap:absent", "v5s2": "mocap:absent"}
assert man.partitions["train"] == ["v1s1", "v1s2", "v3s1", "v3s2"]
m9 = expected_manifest(av, sp.fold(0), ["emg"], scenarios=["s9"])
assert m9.partitions["test"] == ["v1s9"]
lab = expected_manifest(av, sp.fold(0), ["emg"], scenarios=["s1", "s2", "s9"],
require_annotation=True)
assert "v1s9" in lab.excluded["test"]
gate = expected_manifest(av, sp.fold(2), ["emg"], gate_modalities=["pressure"])
assert gate.partitions["test"] == ["v6s1", "v6s2"]
verify_loaded(man, {p: list(v) for p, v in man.partitions.items()})
bad = {p: list(v) for p, v in man.partitions.items()}
bad["test"] = bad["test"][:1]
with pytest.raises(ManifestMismatchError, match="expected but not loaded"):
verify_loaded(man, bad)
bad["test"] = man.partitions["test"] + ["v5s1"]
with pytest.raises(ManifestMismatchError, match="loaded but not expected"):
verify_loaded(man, bad)
assert man.md5 == expected_manifest(av, sp.fold(1), ["emg", "mocap"]).md5
def test_registry_covers_benchmark_modalities():
for m in M.BENCHMARK_MODALITIES:
assert m in M.MODALITIES
|