PULSE-code / tests /test_t3.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
14.9 kB
"""T3 grasp-onset anticipation: labels, sample identity across rows, trimmed-stream
alignment and an end-to-end CPU run on a synthetic tree."""
import csv
import json
import numpy as np
import pytest
import conftest
from conftest import REPO, mini_split
from pulse import aggregate as A
from pulse.data import modalities as M
from pulse.data.availability import build_availability
from pulse.data.io import load_recording, release_trim_window
from pulse.errors import ManifestMismatchError
from pulse.paths import DataRoot
from pulse.provenance import REQUIRED_PROVENANCE, load_results
from pulse.tasks import t3
LAB = t3.label_params({})
T_LONG = 2400 # 24 s per synthetic recording
CONTACTS = [(400, 500), (1200, 1300), (2000, 2100)] # frames with pressure > 5 g
def contact_trace(n=T_LONG, contacts=CONTACTS, level=20.0):
p = np.zeros(n)
for a, b in contacts:
p[a:b] = level
return p
# ---------------------------------------------------------------------------
# labels
# ---------------------------------------------------------------------------
def test_onsets_and_positive_anchors():
total = contact_trace()
assert t3.detect_onsets(total, LAB) == [400, 1200, 2000]
assert t3.positive_anchors(total, [400, 1200, 2000], LAB) == [400, 1200, 2000]
# contact 1 s before an onset -> not a clean positive (quiet window is 1.5 s)
total2 = contact_trace(contacts=[(400, 500), (550, 600)])
on = t3.detect_onsets(total2, LAB)
assert on == [400, 550]
assert t3.positive_anchors(total2, on, LAB) == [400]
# hysteresis: no new onset until the signal falls below rearm_g (1 g)
total3 = contact_trace(contacts=[(400, 500), (600, 700)])
total3[500:600] = 3.0 # 1 g < P <= 5 g: not re-armed
assert t3.detect_onsets(total3, LAB) == [400]
# an onset within the first 1.5 s cannot be a positive
assert t3.positive_anchors(contact_trace(contacts=[(100, 200)]), [100], LAB) == []
def test_negative_anchors_respect_the_rules():
total = contact_trace()
onsets = t3.detect_onsets(total, LAB)
cands = t3.negative_candidates(total, onsets, LAB)
assert len(cands) > 0
for a in cands:
assert total[a - 150:a + 150].max() <= LAB["threshold_g"] # quiet window..a+1.5 s
assert min(abs(a - o) for o in onsets) >= 200 # >= 2 s from onsets
pos, neg = t3.recording_anchors("v1s1", total[:, None].repeat(50, 1) / 50, T_LONG, LAB)
assert pos == [400, 1200, 2000] and len(neg) == 3
assert all(abs(a - b) >= 100 for a in neg for b in neg if a != b)
assert set(neg) <= set(cands.tolist())
def test_negatives_depend_only_on_recording_and_sampling_seed():
p = contact_trace()[:, None].repeat(50, 1) / 50
a = t3.recording_anchors("v7s3", p, T_LONG, LAB)
assert a == t3.recording_anchors("v7s3", p, T_LONG, LAB)
other = t3.label_params({"sampling_seed": 1})
b = [t3.recording_anchors("v7s3", p, T_LONG, other)[1] for _ in range(2)]
assert b[0] == b[1]
# the horizon cuts anchors that would need frames past it
pos, neg = t3.recording_anchors("v7s3", p, 1900, LAB)
assert pos == [400, 1200] and all(n + 150 <= 1900 for n in neg)
def test_balance_per_recording_drops_surplus_positives():
# 6 clean onsets, but only 1350..1900 is quiet long enough for negatives
contacts = [(100, 150), (400, 450), (700, 750), (1000, 1050), (1300, 1350), (1900, 1950),
(2200, 2250)]
p = contact_trace(contacts=contacts)[:, None].repeat(50, 1) / 50
pos, neg = t3.recording_anchors("v7s3", p, T_LONG, LAB)
assert pos == [400, 700, 1000, 1300, 1900, 2200]
assert 1 <= len(neg) < len(pos) and all(1500 <= a <= 1700 for a in neg)
bal = t3.label_params({"balance_per_recording": True})
bpos, bneg = t3.recording_anchors("v7s3", p, T_LONG, bal)
assert bneg == neg and len(bpos) == len(neg) and set(bpos) <= set(pos)
assert (bpos, bneg) == t3.recording_anchors("v7s3", p, T_LONG, bal) # deterministic
# already balanced recordings are unchanged
q = contact_trace()[:, None].repeat(50, 1) / 50
assert t3.recording_anchors("v1s1", q, T_LONG, bal) == t3.recording_anchors("v1s1", q, T_LONG, LAB)
def test_windows_end_half_a_second_before_the_anchor():
x = np.arange(T_LONG, dtype=np.float32)[:, None]
w = t3.extract_windows(x, [400], LAB, 5)
assert w.shape == (1, 20, 1)
assert w[0, 0, 0] == 250 and w[0, -1, 0] == 345 # frames 250..349, every 5th
def test_config_rejects_bad_label_options():
with pytest.raises(ValueError):
t3.label_params({"quiet_before_sec": 1.0})
with pytest.raises(ValueError):
t3.label_params({"no_such_option": 1})
# ---------------------------------------------------------------------------
# synthetic tree with grasp events
# ---------------------------------------------------------------------------
def _write(path, header, data):
with open(path, "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["time"] + list(header))
for i, row in enumerate(data):
w.writerow([f"{i / 100:.2f}"] + [f"{v:.4f}" for v in row])
def _read(path):
with open(path, newline="", encoding="utf-8") as f:
rows = list(csv.reader(f))
return rows[0], rows[1:]
TRIMMED = "v6s1"
FRONT, N_TRIM = 300, 1900
@pytest.fixture(scope="module")
def grasp_root(tmp_path_factory):
"""conftest's synthetic tree at 24 s per recording, with grasp events in the
pressure files, an EMG channel that rises in the 1.5 s before each onset,
and one recording (v6s1) whose EMG / eye / pressure files are re-trimmed
to frames [300, 2200) like v2s6 / v13s2 / v13s3 in v1.1."""
mp = pytest.MonkeyPatch()
mp.setattr(conftest, "T", T_LONG)
root = conftest.build_tree(tmp_path_factory.mktemp("t3") / "PULSE", seed=3)
mp.undo()
rng = np.random.default_rng(0)
for d in sorted((root / "data").glob("v*/s*")):
rid = d.parent.name + d.name
if not (d / "aligned_pressure_100hz.csv").is_file():
continue
p = np.zeros((T_LONG, 50))
p[:, 0] = contact_trace()
_write(d / "aligned_pressure_100hz.csv", M.PRESSURE_COLS, p)
emg = rng.normal(0, 1, (T_LONG, 8))
for a, _b in CONTACTS:
emg[a - 150:a, 0] += 4.0
_write(d / "aligned_emg_100hz.csv", M.EMG_COLS, emg)
if rid == TRIMMED:
for name in ("aligned_emg_100hz.csv", "aligned_eyetrack_100hz.csv",
"aligned_pressure_100hz.csv"):
header, rows = _read(d / name)
with open(d / name, "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(header)
for i, r in enumerate(rows[FRONT:FRONT + N_TRIM]):
w.writerow([f"{i / 100:.2f}"] + r[1:])
meta = json.loads((d / "alignment_metadata.json").read_text(encoding="utf-8"))
meta["mocap_padding"] = {"mocap_sq": {"front_samples": FRONT, "n_frames": N_TRIM,
"back_samples": T_LONG - FRONT - N_TRIM}}
(d / "alignment_metadata.json").write_text(json.dumps(meta), encoding="utf-8")
return root
def test_release_trim_alignment(grasp_root):
dr = DataRoot.from_arg(grasp_root)
meta = json.loads((dr.recording_dir(TRIMMED) / "alignment_metadata.json").read_text())
assert release_trim_window(meta) == (FRONT, N_TRIM)
assert release_trim_window({"n_samples": 100}) is None
full = load_recording(dr, TRIMMED, ["imu"]).arrays["imu"]
old = load_recording(dr, TRIMMED, ["imu", "pressure"])
new = load_recording(dr, TRIMMED, ["imu", "pressure"], align_trimmed=True)
assert old.n_frames == new.n_frames == N_TRIM and not old.start_offsets
assert np.array_equal(old.arrays["imu"], full[:N_TRIM]) # default: cut from start
assert np.array_equal(new.arrays["imu"], full[FRONT:FRONT + N_TRIM])
assert new.start_offsets == {"imu": FRONT}
# only re-trimmed streams, or only full-timeline streams: nothing is shifted
assert load_recording(dr, TRIMMED, ["emg", "pressure"], align_trimmed=True).start_offsets == {}
assert load_recording(dr, TRIMMED, ["imu", "mocap"], align_trimmed=True).n_frames == T_LONG
def test_release_trim_alignment_refuses_streams_on_neither_timeline(grasp_root, monkeypatch):
"""A stream longer than the re-trimmed window but not covering [front, front + n)
cannot be placed: raise instead of silently leaving it unshifted."""
import dataclasses
from pulse.data import io as dio
from pulse.errors import DataQualityError
dr = DataRoot.from_arg(grasp_root)
real = dio.load_stream
def short_imu(dr_, rid, stream):
st = real(dr_, rid, stream)
if rid == TRIMMED and stream == "imu":
cut = FRONT + N_TRIM - 100
st = dataclasses.replace(st, values=st.values[:cut],
time=None if st.time is None else st.time[:cut])
return st
monkeypatch.setattr(dio, "load_stream", short_imu)
with pytest.raises(DataQualityError, match="neither the re-trimmed window"):
load_recording(dr, TRIMMED, ["imu", "mocap", "pressure"], align_trimmed=True)
# default (no alignment) behaviour is unchanged
assert load_recording(dr, TRIMMED, ["imu", "pressure"]).n_frames == N_TRIM
def _run(root, tmp, row, fold, av, extra=()):
split = mini_split(tmp / "mini_split.json")
out = tmp / "6" / row / f"fold{fold}_seed42"
rc = t3.main(["--root", str(root), "--split", str(split), "--fold", str(fold), "--seed", "42",
"--config", str(REPO / "configs" / "t3" / f"{row}.yaml"),
"--availability", str(av), "--out", str(out), "--device", "cpu",
"--set", "train.epochs=3", "--set", "train.batch_size=4", *extra])
assert rc == 0
return load_results(out / "results.json")
def test_t3_end_to_end(grasp_root, tmp_path):
dr = DataRoot.from_arg(grasp_root)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(dr)), encoding="utf-8")
# fold 1: test v2 + v5, inner_val v6 (v6s1 re-trimmed), train v1 + v3 (v3: no pressure)
r_emg = _run(grasp_root, tmp_path, "emg", 1, av, ["--num-workers", "2"]) # spawned loaders
r_imu = _run(grasp_root, tmp_path, "emg_imu", 1, av, ["--num-workers", "1"])
prov = r_emg["provenance"]
assert set(REQUIRED_PROVENANCE) <= set(prov)
assert prov["loaded_recordings"] == {"train": ["v1s1", "v1s2"], "inner_val": ["v6s1", "v6s2"],
"test": ["v2s1", "v2s2", "v5s1", "v5s2"]}
# v5's 72-column IMU is a missing modality: excluded from the EMG + IMU row
assert r_imu["provenance"]["loaded_recordings"]["test"] == ["v2s1", "v2s2"]
assert r_imu["extra"]["trim_alignment"] == {TRIMMED: {"imu": FRONT}}
assert r_emg["extra"]["trim_alignment"] == {}
# same samples of every shared recording in both rows
for rid in ("v2s1", "v2s2"):
assert (r_emg["extra"]["test_recordings"][rid]["anchors_md5"]
== r_imu["extra"]["test_recordings"][rid]["anchors_md5"])
assert r_emg["extra"]["anchors_md5"]["inner_val"] == r_imu["extra"]["anchors_md5"]["inner_val"]
# v6s1 (re-trimmed, timeline starts at frame 300): its first onset is 1 s into the
# file, too early for a 1.5 s quiet window, so 2 + 3 positives in inner_val
assert r_emg["extra"]["counts"]["inner_val"]["n_pos"] == 3 + 2
te = r_emg["metrics"]["test"]
assert set(te) >= {"accuracy", "f1", "macro_f1", "roc_auc", "average_precision",
"positive_fraction"}
assert te["positive_fraction"] == pytest.approx(0.5)
assert r_emg["selection"]["criterion"] == "inner_val/roc_auc (max)"
assert "ReduceLROnPlateau" in r_emg["selection"]["scheduler"]
assert prov["config"]["label"]["lead_sec"] == 0.5 and prov["config"]["modalities"] == ["emg"]
assert r_emg["n_params"] > 0 and r_emg["selection"]["best_epoch"] >= 1
# per-fold sample report and the table aggregation accept the two rows
from pulse.tasks import t3_report
assert t3_report.main(["--runs", str(tmp_path), "--out", str(tmp_path / "tables"),
"--allow-incomplete"]) == 0
folds = json.loads((tmp_path / "tables" / "tab6_folds.json").read_text(encoding="utf-8"))
assert folds["common"]["1"]["n_recordings"] == 2
assert folds["rows"]["emg"]["1"]["positive_fraction"] == pytest.approx(0.5)
spec = A.table_spec(A.load_registry(), "6")
spec["folds"] = [0, 1]
# only two of the six rows ran, on fold 1 only: a partial table must still build
runs, missing = A.collect(spec, tmp_path)
assert not runs["imu"] and not runs["eyetrack"]
summ = A.aggregate(spec, runs, missing=missing, allow_incomplete=True, allow_dirty=True)
assert summ["common_test_recordings"]["1"] == ["v2s1", "v2s2"]
assert "f1" in summ["rows"]["emg_imu"]["vs_reference"]
assert "vs_reference" not in summ["rows"]["imu"]
A.write_outputs(summ, tmp_path / "agg")
nums = (tmp_path / "agg" / "numbers.tex").read_text(encoding="utf-8")
assert "nan" not in nums # one fold: SDs are "--", not "nan"
assert r"pulse@6/emg_imu/roc_auc/deltasd\endcsname{--}" in nums
def test_t3_refuses_results_when_loaded_recordings_differ(grasp_root, tmp_path, monkeypatch):
"""finish() checks the loaded recordings against the manifest: drop one test
recording on the way and no results.json may be written."""
dr = DataRoot.from_arg(grasp_root)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(dr)), encoding="utf-8")
real = t3.build_partition
def drop_last_test(run, rids, *a, **k):
out = real(run, rids, *a, **k)
return out[:-1] if list(rids) == run.manifests["main"].partitions["test"] else out
monkeypatch.setattr(t3, "build_partition", drop_last_test)
with pytest.raises(ManifestMismatchError, match="expected but not loaded"):
_run(grasp_root, tmp_path, "emg", 1, av, ["--num-workers", "1"])
assert not (tmp_path / "6" / "emg" / "fold1_seed42" / "results.json").exists()
def test_registry_rows_for_table_6():
spec = A.table_spec(A.load_registry(), "6")
assert spec["task"] == "t3" and spec["split"] == "splits/pool27.json"
assert [r["key"] for r in spec["rows"]] == ["emg", "emg_imu", "mocap_emg_imu_eyetrack", "imu",
"mocap", "eyetrack"]
for r in spec["rows"]:
cfg = __import__("pulse.config", fromlist=["load_config"]).load_config(REPO / r["config"])
assert cfg["task"] == "t3" and "pressure" not in cfg["modalities"]
assert cfg["select"]["metric"] == "roc_auc"