import json import os import shutil import numpy as np import pytest from pulse.data import modalities as M from pulse.data.availability import Availability, build_availability from pulse.data.io import load_modality, load_recording, quality_check from pulse.data.manifest import expected_manifest, verify_loaded from pulse.errors import (AmbiguousFileError, DataQualityError, DataRootError, ManifestMismatchError, MissingFileError, ModalityUnavailableError) from pulse.paths import DataRoot, resolve_root from pulse.splits import load_split from conftest import mini_split def test_root_resolution(tmp_path, mini_root, monkeypatch): monkeypatch.delenv("PULSE_ROOT", raising=False) with pytest.raises(DataRootError): resolve_root(None) with pytest.raises(DataRootError): resolve_root(tmp_path / "nope") monkeypatch.setenv("PULSE_ROOT", str(mini_root)) assert resolve_root(None) == mini_root.resolve() monkeypatch.setenv("PULSE_ROOT", "$UNDEFINED_PULSE_VAR/x") with pytest.raises(DataRootError): resolve_root(None) def test_tsv_locations(mini_root): dr = DataRoot.from_arg(mini_root) assert dr.mocap_tsv_path("v1s1").parent.name == "raw" assert dr.mocap_tsv_path("v1s2").parent.name == "s2" # recording root with pytest.raises(MissingFileError): dr.mocap_tsv_path("v5s1") shutil.copy(dr.mocap_tsv_path("v1s1"), dr.recording_dir("v1s1") / "aligned_v1s1_s_Q.tsv") with pytest.raises(AmbiguousFileError): dr.mocap_tsv_path("v1s1") def test_modality_dims(mini_root): dr = DataRoot.from_arg(mini_root) dims = {m: load_modality(dr, "v1s1", m).data.shape[1] for m in ("emg", "imu", "imu_wrist", "imu_finger", "eyetrack", "eyetrack_gaze", "eyetrack_ocular", "pressure", "myo_quat")} assert dims == {"emg": 8, "imu": 180, "imu_wrist": 36, "imu_finger": 144, "eyetrack": 24, "eyetrack_gaze": 2, "eyetrack_ocular": 22, "pressure": 50, "myo_quat": 4} mo = load_modality(dr, "v1s2", "mocap") # TSV at the recording root n_feat = 7 * 5 assert mo.data.shape == (240, n_feat + 3 * 5) assert np.allclose(mo.data[:, mo.columns.index("Hips_X")], 0) assert mo.data.dtype == np.float32 def test_v5_imu_is_missing_modality(mini_root): dr = DataRoot.from_arg(mini_root) with pytest.raises(ModalityUnavailableError) as e: load_modality(dr, "v5s1", "imu") assert e.value.reason == "imu_columns_72" with pytest.raises(ModalityUnavailableError): load_modality(dr, "v5s1", "imu_wrist") def test_eyetrack_core_columns(mini_root): dr = DataRoot.from_arg(mini_root) with pytest.raises(ModalityUnavailableError) as e: load_modality(dr, "v2s2", "eyetrack") assert e.value.reason == "eyetrack_core_columns" def test_missing_file_raises(mini_root): dr = DataRoot.from_arg(mini_root) with pytest.raises(MissingFileError): load_modality(dr, "v3s1", "pressure") with pytest.raises(MissingFileError): load_recording(dr, "v5s1", ["emg", "mocap"]) def test_quality_rules(): with pytest.raises(DataQualityError) as e: quality_check(np.array([[0.0, 2e6]]), "x", "imu") assert e.value.reason == "extreme_values" z = np.zeros((100, 4)) with pytest.raises(DataQualityError): quality_check(z, "x", "imu") quality_check(np.zeros((100, 8)), "x", "emg") # EMG and pressure may rest at 0 quality_check(np.zeros((100, 50)), "x", "pressure") def test_load_recording_gate(mini_root): dr = DataRoot.from_arg(mini_root) rec = load_recording(dr, "v1s1", ["emg", "imu"], gate_modalities=["pressure"]) assert set(rec.arrays) == {"emg", "imu"} and rec.concat().shape == (240, 188) with pytest.raises(MissingFileError): load_recording(dr, "v3s1", ["emg"], gate_modalities=["pressure"]) def _availability(mini_root, tmp_path): dr = DataRoot.from_arg(mini_root) table = build_availability(dr) p = tmp_path / "availability.json" p.write_text(json.dumps(table), encoding="utf-8") return Availability.load(p), table def test_availability_statuses(mini_root, tmp_path): av, table = _availability(mini_root, tmp_path) assert av.status("v5s1", "imu") == "unavailable:imu_columns_72" assert av.status("v5s1", "mocap") == "absent" assert av.status("v3s1", "pressure") == "absent" assert av.status("v2s2", "eyetrack") == "unavailable:eyetrack_core_columns" assert av.status("v1s2", "mocap") == "ok" assert table["summary"]["mocap_tsv_locations"] == {"raw": 8, "recording_root": 1} assert table["summary"]["errors"] == [] assert not av.has_annotation("v1s9") and av.has_annotation("v1s1") def test_recording_without_any_data(mini_root, tmp_path): import csv with open(mini_root / "metadata" / "recordings.csv", "a", newline="", encoding="utf-8") as f: csv.writer(f).writerow(["v6s3", "v6", "s3", "", "", "", "emg|imu", 0, "train"]) av, table = _availability(mini_root, tmp_path) assert table["recordings"]["v6s3"]["no_data"] is True assert table["summary"]["recordings_without_data"] == ["v6s3"] assert av.status("v6s3", "emg") == "absent" and table["summary"]["errors"] == [] def test_availability_flags_missing_listed_file(mini_root, tmp_path): os.remove(mini_root / "data" / "v6" / "s1" / "aligned_pressure_100hz.csv") av, table = _availability(mini_root, tmp_path) assert av.status("v6s1", "pressure") == "missing_file" assert any("v6s1" in e for e in table["summary"]["errors"]) sp = load_split(mini_split(tmp_path / "split.json")) with pytest.raises(ManifestMismatchError): expected_manifest(av, sp.fold(2), ["emg", "pressure"]) def test_manifest_and_verify(mini_root, tmp_path): av, _ = _availability(mini_root, tmp_path) sp = load_split(mini_split(tmp_path / "split.json")) man = expected_manifest(av, sp.fold(1), ["emg", "mocap"]) assert man.partitions["test"] == ["v2s1", "v2s2"] # v5 has no MoCap assert man.excluded["test"] == {"v5s1": "mocap:absent", "v5s2": "mocap:absent"} assert man.partitions["train"] == ["v1s1", "v1s2", "v3s1", "v3s2"] m9 = expected_manifest(av, sp.fold(0), ["emg"], scenarios=["s9"]) assert m9.partitions["test"] == ["v1s9"] lab = expected_manifest(av, sp.fold(0), ["emg"], scenarios=["s1", "s2", "s9"], require_annotation=True) assert "v1s9" in lab.excluded["test"] gate = expected_manifest(av, sp.fold(2), ["emg"], gate_modalities=["pressure"]) assert gate.partitions["test"] == ["v6s1", "v6s2"] verify_loaded(man, {p: list(v) for p, v in man.partitions.items()}) bad = {p: list(v) for p, v in man.partitions.items()} bad["test"] = bad["test"][:1] with pytest.raises(ManifestMismatchError, match="expected but not loaded"): verify_loaded(man, bad) bad["test"] = man.partitions["test"] + ["v5s1"] with pytest.raises(ManifestMismatchError, match="loaded but not expected"): verify_loaded(man, bad) assert man.md5 == expected_manifest(av, sp.fold(1), ["emg", "mocap"]).md5 def test_registry_covers_benchmark_modalities(): for m in M.BENCHMARK_MODALITIES: assert m in M.MODALITIES