File size: 7,288 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
import json
import os
import shutil

import numpy as np
import pytest

from pulse.data import modalities as M
from pulse.data.availability import Availability, build_availability
from pulse.data.io import load_modality, load_recording, quality_check
from pulse.data.manifest import expected_manifest, verify_loaded
from pulse.errors import (AmbiguousFileError, DataQualityError, DataRootError,
                          ManifestMismatchError, MissingFileError, ModalityUnavailableError)
from pulse.paths import DataRoot, resolve_root
from pulse.splits import load_split

from conftest import mini_split


def test_root_resolution(tmp_path, mini_root, monkeypatch):
    monkeypatch.delenv("PULSE_ROOT", raising=False)
    with pytest.raises(DataRootError):
        resolve_root(None)
    with pytest.raises(DataRootError):
        resolve_root(tmp_path / "nope")
    monkeypatch.setenv("PULSE_ROOT", str(mini_root))
    assert resolve_root(None) == mini_root.resolve()
    monkeypatch.setenv("PULSE_ROOT", "$UNDEFINED_PULSE_VAR/x")
    with pytest.raises(DataRootError):
        resolve_root(None)


def test_tsv_locations(mini_root):
    dr = DataRoot.from_arg(mini_root)
    assert dr.mocap_tsv_path("v1s1").parent.name == "raw"
    assert dr.mocap_tsv_path("v1s2").parent.name == "s2"   # recording root
    with pytest.raises(MissingFileError):
        dr.mocap_tsv_path("v5s1")
    shutil.copy(dr.mocap_tsv_path("v1s1"), dr.recording_dir("v1s1") / "aligned_v1s1_s_Q.tsv")
    with pytest.raises(AmbiguousFileError):
        dr.mocap_tsv_path("v1s1")


def test_modality_dims(mini_root):
    dr = DataRoot.from_arg(mini_root)
    dims = {m: load_modality(dr, "v1s1", m).data.shape[1]
            for m in ("emg", "imu", "imu_wrist", "imu_finger", "eyetrack", "eyetrack_gaze",
                      "eyetrack_ocular", "pressure", "myo_quat")}
    assert dims == {"emg": 8, "imu": 180, "imu_wrist": 36, "imu_finger": 144, "eyetrack": 24,
                    "eyetrack_gaze": 2, "eyetrack_ocular": 22, "pressure": 50, "myo_quat": 4}
    mo = load_modality(dr, "v1s2", "mocap")  # TSV at the recording root
    n_feat = 7 * 5
    assert mo.data.shape == (240, n_feat + 3 * 5)
    assert np.allclose(mo.data[:, mo.columns.index("Hips_X")], 0)
    assert mo.data.dtype == np.float32


def test_v5_imu_is_missing_modality(mini_root):
    dr = DataRoot.from_arg(mini_root)
    with pytest.raises(ModalityUnavailableError) as e:
        load_modality(dr, "v5s1", "imu")
    assert e.value.reason == "imu_columns_72"
    with pytest.raises(ModalityUnavailableError):
        load_modality(dr, "v5s1", "imu_wrist")


def test_eyetrack_core_columns(mini_root):
    dr = DataRoot.from_arg(mini_root)
    with pytest.raises(ModalityUnavailableError) as e:
        load_modality(dr, "v2s2", "eyetrack")
    assert e.value.reason == "eyetrack_core_columns"


def test_missing_file_raises(mini_root):
    dr = DataRoot.from_arg(mini_root)
    with pytest.raises(MissingFileError):
        load_modality(dr, "v3s1", "pressure")
    with pytest.raises(MissingFileError):
        load_recording(dr, "v5s1", ["emg", "mocap"])


def test_quality_rules():
    with pytest.raises(DataQualityError) as e:
        quality_check(np.array([[0.0, 2e6]]), "x", "imu")
    assert e.value.reason == "extreme_values"
    z = np.zeros((100, 4))
    with pytest.raises(DataQualityError):
        quality_check(z, "x", "imu")
    quality_check(np.zeros((100, 8)), "x", "emg")       # EMG and pressure may rest at 0
    quality_check(np.zeros((100, 50)), "x", "pressure")


def test_load_recording_gate(mini_root):
    dr = DataRoot.from_arg(mini_root)
    rec = load_recording(dr, "v1s1", ["emg", "imu"], gate_modalities=["pressure"])
    assert set(rec.arrays) == {"emg", "imu"} and rec.concat().shape == (240, 188)
    with pytest.raises(MissingFileError):
        load_recording(dr, "v3s1", ["emg"], gate_modalities=["pressure"])


def _availability(mini_root, tmp_path):
    dr = DataRoot.from_arg(mini_root)
    table = build_availability(dr)
    p = tmp_path / "availability.json"
    p.write_text(json.dumps(table), encoding="utf-8")
    return Availability.load(p), table


def test_availability_statuses(mini_root, tmp_path):
    av, table = _availability(mini_root, tmp_path)
    assert av.status("v5s1", "imu") == "unavailable:imu_columns_72"
    assert av.status("v5s1", "mocap") == "absent"
    assert av.status("v3s1", "pressure") == "absent"
    assert av.status("v2s2", "eyetrack") == "unavailable:eyetrack_core_columns"
    assert av.status("v1s2", "mocap") == "ok"
    assert table["summary"]["mocap_tsv_locations"] == {"raw": 8, "recording_root": 1}
    assert table["summary"]["errors"] == []
    assert not av.has_annotation("v1s9") and av.has_annotation("v1s1")


def test_recording_without_any_data(mini_root, tmp_path):
    import csv
    with open(mini_root / "metadata" / "recordings.csv", "a", newline="", encoding="utf-8") as f:
        csv.writer(f).writerow(["v6s3", "v6", "s3", "", "", "", "emg|imu", 0, "train"])
    av, table = _availability(mini_root, tmp_path)
    assert table["recordings"]["v6s3"]["no_data"] is True
    assert table["summary"]["recordings_without_data"] == ["v6s3"]
    assert av.status("v6s3", "emg") == "absent" and table["summary"]["errors"] == []


def test_availability_flags_missing_listed_file(mini_root, tmp_path):
    os.remove(mini_root / "data" / "v6" / "s1" / "aligned_pressure_100hz.csv")
    av, table = _availability(mini_root, tmp_path)
    assert av.status("v6s1", "pressure") == "missing_file"
    assert any("v6s1" in e for e in table["summary"]["errors"])
    sp = load_split(mini_split(tmp_path / "split.json"))
    with pytest.raises(ManifestMismatchError):
        expected_manifest(av, sp.fold(2), ["emg", "pressure"])


def test_manifest_and_verify(mini_root, tmp_path):
    av, _ = _availability(mini_root, tmp_path)
    sp = load_split(mini_split(tmp_path / "split.json"))
    man = expected_manifest(av, sp.fold(1), ["emg", "mocap"])
    assert man.partitions["test"] == ["v2s1", "v2s2"]       # v5 has no MoCap
    assert man.excluded["test"] == {"v5s1": "mocap:absent", "v5s2": "mocap:absent"}
    assert man.partitions["train"] == ["v1s1", "v1s2", "v3s1", "v3s2"]
    m9 = expected_manifest(av, sp.fold(0), ["emg"], scenarios=["s9"])
    assert m9.partitions["test"] == ["v1s9"]
    lab = expected_manifest(av, sp.fold(0), ["emg"], scenarios=["s1", "s2", "s9"],
                            require_annotation=True)
    assert "v1s9" in lab.excluded["test"]
    gate = expected_manifest(av, sp.fold(2), ["emg"], gate_modalities=["pressure"])
    assert gate.partitions["test"] == ["v6s1", "v6s2"]
    verify_loaded(man, {p: list(v) for p, v in man.partitions.items()})
    bad = {p: list(v) for p, v in man.partitions.items()}
    bad["test"] = bad["test"][:1]
    with pytest.raises(ManifestMismatchError, match="expected but not loaded"):
        verify_loaded(man, bad)
    bad["test"] = man.partitions["test"] + ["v5s1"]
    with pytest.raises(ManifestMismatchError, match="loaded but not expected"):
        verify_loaded(man, bad)
    assert man.md5 == expected_manifest(av, sp.fold(1), ["emg", "mocap"]).md5


def test_registry_covers_benchmark_modalities():
    for m in M.BENCHMARK_MODALITIES:
        assert m in M.MODALITIES