Download tests/test_data_layer.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 7.29 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/tests/test_data_layer.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/tests/test_data_layer.py
-
curl -L -o test_data_layer.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/tests/test_data_layer.py
7.29 kB
| import json | |
| import os | |
| import shutil | |
| import numpy as np | |
| import pytest | |
| from pulse.data import modalities as M | |
| from pulse.data.availability import Availability, build_availability | |
| from pulse.data.io import load_modality, load_recording, quality_check | |
| from pulse.data.manifest import expected_manifest, verify_loaded | |
| from pulse.errors import (AmbiguousFileError, DataQualityError, DataRootError, | |
| ManifestMismatchError, MissingFileError, ModalityUnavailableError) | |
| from pulse.paths import DataRoot, resolve_root | |
| from pulse.splits import load_split | |
| from conftest import mini_split | |
| def test_root_resolution(tmp_path, mini_root, monkeypatch): | |
| monkeypatch.delenv("PULSE_ROOT", raising=False) | |
| with pytest.raises(DataRootError): | |
| resolve_root(None) | |
| with pytest.raises(DataRootError): | |
| resolve_root(tmp_path / "nope") | |
| monkeypatch.setenv("PULSE_ROOT", str(mini_root)) | |
| assert resolve_root(None) == mini_root.resolve() | |
| monkeypatch.setenv("PULSE_ROOT", "$UNDEFINED_PULSE_VAR/x") | |
| with pytest.raises(DataRootError): | |
| resolve_root(None) | |
| def test_tsv_locations(mini_root): | |
| dr = DataRoot.from_arg(mini_root) | |
| assert dr.mocap_tsv_path("v1s1").parent.name == "raw" | |
| assert dr.mocap_tsv_path("v1s2").parent.name == "s2" # recording root | |
| with pytest.raises(MissingFileError): | |
| dr.mocap_tsv_path("v5s1") | |
| shutil.copy(dr.mocap_tsv_path("v1s1"), dr.recording_dir("v1s1") / "aligned_v1s1_s_Q.tsv") | |
| with pytest.raises(AmbiguousFileError): | |
| dr.mocap_tsv_path("v1s1") | |
| def test_modality_dims(mini_root): | |
| dr = DataRoot.from_arg(mini_root) | |
| dims = {m: load_modality(dr, "v1s1", m).data.shape[1] | |
| for m in ("emg", "imu", "imu_wrist", "imu_finger", "eyetrack", "eyetrack_gaze", | |
| "eyetrack_ocular", "pressure", "myo_quat")} | |
| assert dims == {"emg": 8, "imu": 180, "imu_wrist": 36, "imu_finger": 144, "eyetrack": 24, | |
| "eyetrack_gaze": 2, "eyetrack_ocular": 22, "pressure": 50, "myo_quat": 4} | |
| mo = load_modality(dr, "v1s2", "mocap") # TSV at the recording root | |
| n_feat = 7 * 5 | |
| assert mo.data.shape == (240, n_feat + 3 * 5) | |
| assert np.allclose(mo.data[:, mo.columns.index("Hips_X")], 0) | |
| assert mo.data.dtype == np.float32 | |
| def test_v5_imu_is_missing_modality(mini_root): | |
| dr = DataRoot.from_arg(mini_root) | |
| with pytest.raises(ModalityUnavailableError) as e: | |
| load_modality(dr, "v5s1", "imu") | |
| assert e.value.reason == "imu_columns_72" | |
| with pytest.raises(ModalityUnavailableError): | |
| load_modality(dr, "v5s1", "imu_wrist") | |
| def test_eyetrack_core_columns(mini_root): | |
| dr = DataRoot.from_arg(mini_root) | |
| with pytest.raises(ModalityUnavailableError) as e: | |
| load_modality(dr, "v2s2", "eyetrack") | |
| assert e.value.reason == "eyetrack_core_columns" | |
| def test_missing_file_raises(mini_root): | |
| dr = DataRoot.from_arg(mini_root) | |
| with pytest.raises(MissingFileError): | |
| load_modality(dr, "v3s1", "pressure") | |
| with pytest.raises(MissingFileError): | |
| load_recording(dr, "v5s1", ["emg", "mocap"]) | |
| def test_quality_rules(): | |
| with pytest.raises(DataQualityError) as e: | |
| quality_check(np.array([[0.0, 2e6]]), "x", "imu") | |
| assert e.value.reason == "extreme_values" | |
| z = np.zeros((100, 4)) | |
| with pytest.raises(DataQualityError): | |
| quality_check(z, "x", "imu") | |
| quality_check(np.zeros((100, 8)), "x", "emg") # EMG and pressure may rest at 0 | |
| quality_check(np.zeros((100, 50)), "x", "pressure") | |
| def test_load_recording_gate(mini_root): | |
| dr = DataRoot.from_arg(mini_root) | |
| rec = load_recording(dr, "v1s1", ["emg", "imu"], gate_modalities=["pressure"]) | |
| assert set(rec.arrays) == {"emg", "imu"} and rec.concat().shape == (240, 188) | |
| with pytest.raises(MissingFileError): | |
| load_recording(dr, "v3s1", ["emg"], gate_modalities=["pressure"]) | |
| def _availability(mini_root, tmp_path): | |
| dr = DataRoot.from_arg(mini_root) | |
| table = build_availability(dr) | |
| p = tmp_path / "availability.json" | |
| p.write_text(json.dumps(table), encoding="utf-8") | |
| return Availability.load(p), table | |
| def test_availability_statuses(mini_root, tmp_path): | |
| av, table = _availability(mini_root, tmp_path) | |
| assert av.status("v5s1", "imu") == "unavailable:imu_columns_72" | |
| assert av.status("v5s1", "mocap") == "absent" | |
| assert av.status("v3s1", "pressure") == "absent" | |
| assert av.status("v2s2", "eyetrack") == "unavailable:eyetrack_core_columns" | |
| assert av.status("v1s2", "mocap") == "ok" | |
| assert table["summary"]["mocap_tsv_locations"] == {"raw": 8, "recording_root": 1} | |
| assert table["summary"]["errors"] == [] | |
| assert not av.has_annotation("v1s9") and av.has_annotation("v1s1") | |
| def test_recording_without_any_data(mini_root, tmp_path): | |
| import csv | |
| with open(mini_root / "metadata" / "recordings.csv", "a", newline="", encoding="utf-8") as f: | |
| csv.writer(f).writerow(["v6s3", "v6", "s3", "", "", "", "emg|imu", 0, "train"]) | |
| av, table = _availability(mini_root, tmp_path) | |
| assert table["recordings"]["v6s3"]["no_data"] is True | |
| assert table["summary"]["recordings_without_data"] == ["v6s3"] | |
| assert av.status("v6s3", "emg") == "absent" and table["summary"]["errors"] == [] | |
| def test_availability_flags_missing_listed_file(mini_root, tmp_path): | |
| os.remove(mini_root / "data" / "v6" / "s1" / "aligned_pressure_100hz.csv") | |
| av, table = _availability(mini_root, tmp_path) | |
| assert av.status("v6s1", "pressure") == "missing_file" | |
| assert any("v6s1" in e for e in table["summary"]["errors"]) | |
| sp = load_split(mini_split(tmp_path / "split.json")) | |
| with pytest.raises(ManifestMismatchError): | |
| expected_manifest(av, sp.fold(2), ["emg", "pressure"]) | |
| def test_manifest_and_verify(mini_root, tmp_path): | |
| av, _ = _availability(mini_root, tmp_path) | |
| sp = load_split(mini_split(tmp_path / "split.json")) | |
| man = expected_manifest(av, sp.fold(1), ["emg", "mocap"]) | |
| assert man.partitions["test"] == ["v2s1", "v2s2"] # v5 has no MoCap | |
| assert man.excluded["test"] == {"v5s1": "mocap:absent", "v5s2": "mocap:absent"} | |
| assert man.partitions["train"] == ["v1s1", "v1s2", "v3s1", "v3s2"] | |
| m9 = expected_manifest(av, sp.fold(0), ["emg"], scenarios=["s9"]) | |
| assert m9.partitions["test"] == ["v1s9"] | |
| lab = expected_manifest(av, sp.fold(0), ["emg"], scenarios=["s1", "s2", "s9"], | |
| require_annotation=True) | |
| assert "v1s9" in lab.excluded["test"] | |
| gate = expected_manifest(av, sp.fold(2), ["emg"], gate_modalities=["pressure"]) | |
| assert gate.partitions["test"] == ["v6s1", "v6s2"] | |
| verify_loaded(man, {p: list(v) for p, v in man.partitions.items()}) | |
| bad = {p: list(v) for p, v in man.partitions.items()} | |
| bad["test"] = bad["test"][:1] | |
| with pytest.raises(ManifestMismatchError, match="expected but not loaded"): | |
| verify_loaded(man, bad) | |
| bad["test"] = man.partitions["test"] + ["v5s1"] | |
| with pytest.raises(ManifestMismatchError, match="loaded but not expected"): | |
| verify_loaded(man, bad) | |
| assert man.md5 == expected_manifest(av, sp.fold(1), ["emg", "mocap"]).md5 | |
| def test_registry_covers_benchmark_modalities(): | |
| for m in M.BENCHMARK_MODALITIES: | |
| assert m in M.MODALITIES | |