PULSE-code / tests /test_t6.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
15.2 kB
"""T6 (cross-modal pressure prediction): signal processing, configs, end to end."""
import json
from pathlib import Path
import numpy as np
import pytest
from conftest import REPO, mini_split, real_root, realdata
from pulse.config import load_config
from pulse.tasks import t6
T6_CONFIGS = sorted(p for p in (REPO / "configs" / "t6").glob("t6*_*.yaml")
if not p.stem.endswith("_base"))
def _torch_numpy_ok():
try:
import torch
torch.from_numpy(np.zeros(2, np.float32))
return True
except Exception: # torch missing or built against another numpy ABI
return False
needs_torch = pytest.mark.skipif(not _torch_numpy_ok(), reason="torch with numpy interop needed")
def test_moving_rms_matches_reference():
rng = np.random.default_rng(0)
x = rng.normal(0, 1, (57, 3))
win = t6.envelope_window(100, 200)
assert win == 21
h = win // 2
xp = np.concatenate([x[:h][::-1], x, x[-h:][::-1]]) # half-sample symmetric
ref = np.sqrt(np.stack([(xp[i:i + win] ** 2).mean(0) for i in range(x.shape[0])]))
assert np.allclose(t6.moving_rms(x, win), ref)
try:
from scipy.ndimage import uniform_filter1d
except Exception:
return
assert np.allclose(t6.moving_rms(x, win),
np.sqrt(uniform_filter1d(x ** 2, size=win, axis=0, mode="reflect")))
def test_dilation_and_segments():
a = np.zeros(30, bool)
a[[3, 10, 29]] = True
d = t6.dilate(a, 2)
ref = np.zeros(30, bool)
for i in np.where(a)[0]:
ref[max(0, i - 2):i + 3] = True
assert np.array_equal(d, ref)
assert np.array_equal(t6.dilate(np.ones(3, bool), 5), np.ones(3, bool)) # shorter than kernel
total = np.zeros(200)
total[50:52] = 500 # dilated by 20 -> [30, 72), padded by 20 -> [10, 92)
total[100:101] = 500 # -> [80, 121) -> [60, 141): overlaps the first -> merged
total[180:181] = 50 # below threshold
segs = t6.active_segments(total, 100.0, 20, 12, 20)
assert segs == [(10, 141)]
assert t6.active_segments(np.zeros(10), 100.0, 20, 12, 20) == []
def test_hand_mocap_features():
cols = ["Hips_X", "Hips_Y", "Hips_Z", "RightHand_X", "RightHand_Y", "RightHand_Z", "RightHand_QX",
"RightInHandIndex_X", "RightInHandIndex_Y", "RightInHandIndex_Z",
"LeftHand_X", "LeftHand_Y", "LeftHand_Z", "LeftForeArm_X", "LeftForeArm_Y",
"LeftForeArm_Z"]
v = np.arange(4 * len(cols), dtype=np.float64).reshape(4, len(cols))
v[2, 7] = np.nan
f = t6.hand_mocap_features(v, cols)
# kept: 13 columns (no Hips), 12 position columns -> +12 velocities
assert f.shape == (4, 13 + 12)
assert np.allclose(f[:, 0:3], v[:, 3:6]) # wrist stays absolute
assert np.allclose(f[[0, 1, 3], 4:7], v[[0, 1, 3], 7:10] - v[[0, 1, 3], 3:6])
assert f[2, 4] == 0.0 # NaN -> 0 after the transform
assert np.allclose(f[:, 10:13], v[:, 13:16] - v[:, 10:13]) # forearm wrist-relative
assert np.all(f[0, 13:] == 0) # first velocity frame
def test_t6_configs_valid():
assert len(T6_CONFIGS) == 14 # 9 T6a + 5 T6b rows
for p in T6_CONFIGS:
cfg = load_config(p)
assert cfg["task"] == "t6", p
t6.check_config(cfg)
assert cfg["preprocess"]["calibration"]["seconds"] == 30
if "g4" in p.stem:
assert len(cfg["gate_modalities"]) == 3
ens = load_config(REPO / "configs" / "t6" / "ensemble.yaml")
assert ens["task"] == "t6ens" and ens["members"]["seeds"] == [42, 43, 44]
def test_t6_tables_registered():
from pulse import aggregate as A
reg = A.load_registry()
t6_tables = {k: v for k, v in reg.items() if v.get("task") in ("t6", "t6ens")}
assert {"11a", "11b", "11a-ens", "11b-ens", "11b-b19"} <= set(t6_tables)
# author decision 2026-10-18: no T6b Transformer row, no B19 hand-MoCap control row
assert [r["key"] for r in reg["11b"]["rows"]] == [
"pham_mocap", "latefusion_emg_mocap", "pham_emg_mocap", "ghorbani_mocap"]
assert [r["key"] for r in reg["11b-b19"]["rows"]] == ["ghorbani_m4"]
assert reg["11b-b19"]["reference_row"] is None
for tid, spec in t6_tables.items():
assert spec["split"] == "splits/pool27.json"
if spec["task"] == "t6ens":
assert spec["seeds"] == ["ens"] and spec["resources"] == "cpu"
members = reg[[x.split("=", 1)[1] for x in spec["rows"][0]["set"]
if x.startswith("members.table=")][0]]
assert {r["key"] for r in spec["rows"]} == {r["key"] for r in members["rows"]}
else:
assert spec["seeds"] == [42, 43, 44]
def _mini_setup(mini_root, tmp_path):
from pulse.data.availability import build_availability
from pulse.paths import DataRoot
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(DataRoot.from_arg(mini_root))), encoding="utf-8")
return av, mini_split(tmp_path / "mini_split.json")
SMALL = ["train.epochs=2", "model.hidden_dim=8", "preprocess.calibration.seconds=0.4",
"active.threshold_g=5", "load_workers=0"]
def _run_t6(mini_root, av, split, cfg, out, seed=42, extra=()):
args = ["--root", str(mini_root), "--split", str(split), "--fold", "0", "--seed", str(seed),
"--config", str(REPO / "configs" / "t6" / cfg), "--availability", str(av),
"--out", str(out), "--device", "cpu"]
for kv in list(SMALL) + list(extra):
args += ["--set", kv]
return t6.main(args)
@needs_torch
@pytest.mark.parametrize("cfg", ["t6a_pham_emg.yaml", "t6b_latefusion_emg_mocap.yaml",
"t6b_ghorbani_mocap.yaml", "t6a_bao_m4.yaml",
"t6a_transformer_emg.yaml"])
def test_t6_end_to_end(mini_root, tmp_path, cfg):
from pulse.provenance import load_results
av, split = _mini_setup(mini_root, tmp_path)
out = tmp_path / "run"
assert _run_t6(mini_root, av, split, cfg, out) == 0
r = load_results(out / "results.json")
prov = r["provenance"]
assert prov["inner_val_volunteers"] == ["v2"] and prov["seed"] == 42
assert prov["loaded_recordings"]["test"] == ["v1s1", "v1s2"]
if "mocap" in cfg or "m4" in cfg:
assert "v5s1" not in prov["loaded_recordings"]["train"] # v5: no MoCap / IMU
else:
assert prov["loaded_recordings"]["train"] == ["v5s1", "v5s2", "v6s1", "v6s2"]
assert set(r["per_recording"]["test"]) == {"v1s1", "v1s2"}
key = "pearson" if cfg.startswith("t6a") else "pearson_max"
assert key in r["metrics"]["test"] and "mae_g" in r["metrics"]["test"]
assert r["selection"]["swa_start_epoch"] == 1 and len(r["selection"]["history"]) == 2
cal = r["extra"]["calibration"]["frames_per_recording"]
assert cal["v1s1"] == 8 and cal["v1s2"] == 0 # 0.4 s at 20 Hz, S1 first
z = np.load(out / "predictions.npz")
assert sorted(set(z["rec_index"].tolist())) == [0, 1]
assert z["frame"][z["rec_index"] == 0].min() >= 8 # calibration frames not scored
assert str(mini_root) not in json.dumps(r)
@needs_torch
def test_t6_manifest_mismatch_refused(mini_root, tmp_path, monkeypatch):
from pulse.errors import ManifestMismatchError
av, split = _mini_setup(mini_root, tmp_path)
real = t6.load_many
def drop_one(dr, rids, *a, **k):
out = real(dr, rids, *a, **k)
out.pop("v6s2", None)
return out
monkeypatch.setattr(t6, "load_many", drop_one)
with pytest.raises(ManifestMismatchError, match="v6s2"):
_run_t6(mini_root, av, split, "t6a_pham_emg.yaml", tmp_path / "run")
assert not (tmp_path / "run" / "results.json").exists()
@needs_torch
def test_t6_seed_ensemble_and_aggregate(mini_root, tmp_path):
from pulse import aggregate as A
from pulse.provenance import load_results
from pulse.tasks import t6ens
av, split = _mini_setup(mini_root, tmp_path)
runs = tmp_path / "runs"
for s in (42, 43):
assert _run_t6(mini_root, av, split, "t6b_pham_mocap.yaml",
A.run_dir(runs, "11b", "pham_mocap", 0, s), seed=s) == 0
out = A.run_dir(runs, "11b-ens", "pham_mocap", 0, "ens")
rc = t6ens.main(["--split", str(split), "--fold", "0", "--seed", "ens", "--config",
str(REPO / "configs" / "t6" / "ensemble.yaml"), "--out", str(out),
"--set", "members.table=11b", "--set", "members.row=pham_mocap",
"--set", "members.seeds=[42, 43]", "--set", "members.allow_subset=true",
"--set", "members.allow_dirty=true"])
assert rc == 0
e = load_results(out / "results.json")
assert e["provenance"]["seed"] == "ens" and e["provenance"]["members"]["seeds"] == [42, 43]
assert e["provenance"]["members"]["all_table_seeds"] is False
# without the explicit opt-in a subset of the table's seeds 42/43/44 is refused
with pytest.raises(ValueError, match="averages all of them"):
t6ens.main(["--split", str(split), "--fold", "0", "--seed", "ens", "--config",
str(REPO / "configs" / "t6" / "ensemble.yaml"), "--out", str(tmp_path / "x"),
"--set", "members.table=11b", "--set", "members.row=pham_mocap",
"--set", "members.seeds=[43, 44]", "--set", "members.allow_dirty=true"])
m = [load_results(A.run_dir(runs, "11b", "pham_mocap", 0, s) / "results.json") for s in (42, 43)]
assert e["provenance"]["loaded_recordings"] == m[0]["provenance"]["loaded_recordings"]
# averaging the predictions: MAE of the mean is at most the mean MAE
assert e["metrics"]["test"]["mae_g"] <= np.mean([x["metrics"]["test"]["mae_g"] for x in m]) + 1e-9
spec = {"id": "11b-ens", "title": "t", "task": "t6ens", "split": str(split), "folds": [0],
"seeds": ["ens"], "reference_row": None,
"metrics": [{"key": "pearson_max", "label": "r", "digits": 3, "better": "higher"}],
"rows": [{"key": "pham_mocap", "label": "Pham"}]}
got, missing = A.collect(spec, runs)
assert not missing
summ = A.aggregate(spec, got, allow_dirty=True)
assert summ["rows"]["pham_mocap"]["own"]["pearson_max"]["n_runs"] == 1
# a member from another commit is refused
p = A.run_dir(runs, "11b", "pham_mocap", 0, 43) / "results.json"
d = json.loads(p.read_text(encoding="utf-8"))
d["provenance"]["git_sha"] = "0" * 40
p.write_text(json.dumps(d), encoding="utf-8")
from pulse.errors import ProvenanceError
with pytest.raises(ProvenanceError, match="git_sha"):
t6ens.main(["--split", str(split), "--fold", "0", "--seed", "ens", "--config",
str(REPO / "configs" / "t6" / "ensemble.yaml"), "--out", str(out),
"--set", "members.table=11b", "--set", "members.row=pham_mocap",
"--set", "members.seeds=[42, 43]", "--set", "members.allow_subset=true",
"--set", "members.allow_dirty=true"])
@needs_torch
def test_t6_ensemble_refuses_foreign_predictions(mini_root, tmp_path):
"""A predictions.npz that does not reproduce its results.json is refused."""
from pulse import aggregate as A
from pulse.errors import ProvenanceError
from pulse.tasks import t6ens
av, split = _mini_setup(mini_root, tmp_path)
runs = tmp_path / "runs"
for s in (42, 43):
assert _run_t6(mini_root, av, split, "t6a_pham_emg.yaml",
A.run_dir(runs, "11a", "pham_emg", 0, s), seed=s) == 0
npz = A.run_dir(runs, "11a", "pham_emg", 0, 43) / "predictions.npz"
z = dict(np.load(npz))
z["pred"] = z["pred"][::-1].copy() # same frames, other predictions
np.savez_compressed(npz, **z)
out = A.run_dir(runs, "11a-ens", "pham_emg", 0, "ens")
with pytest.raises(ProvenanceError, match="stale or foreign"):
t6ens.main(["--split", str(split), "--fold", "0", "--seed", "ens", "--config",
str(REPO / "configs" / "t6" / "ensemble.yaml"), "--out", str(out),
"--set", "members.table=11a", "--set", "members.row=pham_emg",
"--set", "members.seeds=[42, 43]", "--set", "members.allow_subset=true",
"--set", "members.allow_dirty=true"])
assert not (out / "results.json").exists()
@needs_torch
def test_t6_refuses_when_checkout_changes_during_run(mini_root, tmp_path, monkeypatch):
from pulse.errors import ProvenanceError
av, split = _mini_setup(mini_root, tmp_path)
states = iter([{"git_sha": "a" * 40, "git_dirty": False, "git_diff_md5": None},
{"git_sha": "b" * 40, "git_dirty": False, "git_diff_md5": None}])
monkeypatch.setattr(t6, "git_state", lambda: next(states))
with pytest.raises(ProvenanceError, match="repository changed"):
_run_t6(mini_root, av, split, "t6a_pham_emg.yaml", tmp_path / "run")
assert not (tmp_path / "run" / "results.json").exists()
def test_t6_trim_window_refuses_unplaceable_stream(monkeypatch):
"""Re-trimmed (n rows) and full-timeline streams are aligned; a stream of any
other length cannot be placed and raises instead of being cut from the start."""
from pulse.data import io as dio
from pulse.errors import DataQualityError
lengths = {}
def fake_stream(dr, rid, stream):
cols = {"emg": list(t6.M.EMG_COLS), "pressure": list(t6.M.PRESSURE_COLS),
"imu": [f"WT{s}_{f}" for s in range(10) for f in range(18)]}[stream]
vals = np.ones((lengths[stream], len(cols)))
return dio.Stream(rid, stream, f"{stream}.csv", cols, vals, None, cols)
monkeypatch.setattr(dio, "load_stream", fake_stream)
monkeypatch.setattr(dio, "load_alignment_metadata",
lambda dr, rid: {"n_samples": 1000,
"mocap_padding": {"mocap_sq": {"front_samples": 100,
"n_frames": 800}}})
lengths.update(emg=800, pressure=800, imu=1000)
r = t6.load_t6_recording(None, "v13s2", ["emg", "imu"], "hand", ["pressure"], 5)
assert r["offsets"] == {"imu": 100} and r["n100"] == 800
lengths.update(imu=850) # neither 800 nor >= 900 rows
with pytest.raises(DataQualityError, match="cannot align"):
t6.load_t6_recording(None, "v13s2", ["emg", "imu"], "hand", ["pressure"], 5)
@realdata
@pytest.mark.realdata
def test_t6_loader_on_real_data():
"""460 hand-MoCap columns; release trim window for v13s2 (MoCap / IMU sliced)."""
from pulse.paths import DataRoot
dr = DataRoot.from_arg(real_root())
r = t6.load_t6_recording(dr, "v25s1", ["emg", "mocap"], "hand", ["pressure"], 5)
assert r["dims"] == {"emg": 8, "mocap": 460} and r["offsets"] == {}
assert r["pressure"].shape == (int(np.ceil(r["n100"] / 5)), 50)
r = t6.load_t6_recording(dr, "v13s2", ["emg", "mocap", "imu"], "hand", ["pressure"], 5)
assert r["offsets"] == {"mocap": 393, "imu": 393} and r["n100"] == 3790
assert r["emg100"].shape[0] == 3790