PULSE-code / tests /test_t2.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
10.2 kB
"""T2 (fine-grained action recognition): samples, timeline, models, end-to-end run."""
import json
import numpy as np
import pytest
import conftest
from conftest import REPO, mini_split
from pulse.data.availability import Availability, build_availability
from pulse.errors import DataQualityError
from pulse.paths import DataRoot
from pulse.provenance import load_results
from pulse.tasks import t2_data as D
torch = pytest.importorskip("torch")
SIZES = {"verb_fine": 17, "verb_composite": 6, "noun": 33, "hand": 3}
def _tl(n_ds=240, start=0, ds=5):
return D.Timeline("v1s1", start + ds * n_ds, {"emg": 0}, {"emg": start + ds * n_ds}, start,
n_ds, ds)
def test_sliding_centre_labels():
spans = [D.LabelledSpan(200, 300, (2, 1, 0, 1)), D.LabelledSpan(400, 500, (0, 0, 5, 1))]
c = {}
s0, s1, labs = D.sliding_samples(_tl(), spans, 4.0, 1.0, c)
# windows start every 20 frames (1 s); centre = start + 2 s
assert s0 == [0, 40] and s1 == [80, 120]
assert labs == [(2, 1, 0, 1), (0, 0, 5, 1)]
assert c["windows_total"] == 9 and c["windows_unlabelled"] == 7
# a centre exactly on a segment end belongs to the next segment / nothing
s0, _, labs = D.sliding_samples(_tl(), [D.LabelledSpan(100, 200, (1, 1, 1, 1))], 4.0, 1.0, {})
assert s0 == []
def test_segment_samples_crop_and_span():
tl = _tl(n_ds=200, start=1000) # span = timeline frames [1000, 2000)
spans = [D.LabelledSpan(900, 1100, (0,) * 4), # starts before the span
D.LabelledSpan(1100, 1300, (1,) * 4),
D.LabelledSpan(1300, 1310, (2,) * 4), # 2 frames at 20 Hz
D.LabelledSpan(1400, 1990, (3,) * 4)] # 118 frames -> cropped to 100
c = {}
s0, s1, labs = D.segment_samples(tl, spans, max_frames=100, min_frames=4, counts=c)
assert (s0, s1) == ([20, 89], [60, 189]) # [80, 198) -> centre 139 -> [89, 189)
assert [l[0] for l in labs] == [1, 3]
assert c == {"dropped_outside_span": 1, "dropped_too_few_frames": 1, "cropped_long": 1}
def test_timeline_places_retrimmed_streams(tmp_path):
rid = "v2s6"
rec = {"volunteer": "v2", "scenario": "s6", "alignment_n_samples": 7650,
"modalities": {"emg": {"status": "ok", "n_frames": 5701},
"eyetrack": {"status": "ok", "n_frames": 5701},
"pressure": {"status": "ok", "n_frames": 5701},
"imu": {"status": "ok", "n_frames": 7650},
"mocap": {"status": "ok", "n_frames": 7651}}}
av = Availability({"schema": "pulse-availability/1", "recordings": {rid: rec}})
root = tmp_path / "P"
(root / "data" / "v2" / "s6").mkdir(parents=True)
(root / "metadata").mkdir()
meta = {"n_samples": 7650, "mocap_padding": {"mocap": {"front_samples": 1048, "n_frames": 5701}}}
(root / "data" / "v2" / "s6" / "alignment_metadata.json").write_text(json.dumps(meta))
tl = D.recording_timeline(av, DataRoot(root=root), rid, 5)
assert tl.offsets == {"emg": 1048, "eyetrack": 1048, "pressure": 1048, "imu": 0, "mocap": 0}
assert tl.start == 1050 and tl.end <= 1048 + 5701 and tl.n_frames_ds == (6749 - 1050) // 5
rec["modalities"]["emg"]["n_frames"] = 5000
with pytest.raises(DataQualityError, match="cannot place"):
D.recording_timeline(av, DataRoot(root=root), rid, 5)
def test_normalisation_weights_frames_by_sample_coverage():
a = np.arange(10, dtype=np.float32)[:, None]
r = D.RecordingSamples("v1s1", {"emg": a}, np.array([0, 0]), np.array([2, 4]),
np.zeros((2, 4), np.int64), _tl())
mu, sd = D.normalisation_stats([r], ["emg"])["emg"]
w = np.array([2, 2, 1, 1], float)
assert mu[0] == pytest.approx((w * np.arange(4)).sum() / w.sum())
def test_models_build_and_head_sizes():
from pulse.models.t2 import MODELS, build_model, n_params
dims = {"imu": 12, "emg": 8, "mocap": 10}
x = {m: torch.randn(3, 30, d) for m, d in dims.items()}
mask = torch.ones(3, 30, dtype=torch.bool)
mask[1, 20:] = False
for name in MODELS:
m = build_model(name, dims, SIZES)
out = m(x, mask)
assert {h: tuple(v.shape) for h, v in out.items()} == {h: (3, k) for h, k in SIZES.items()}
assert n_params(m) > 0
daf = build_model("dailyactformer_v2", dims, SIZES)
assert tuple(daf.head.compat.shape) == (17, 33)
with pytest.raises(ValueError, match="max_T"):
build_model("futr", dims, SIZES, max_T=16)({m: t for m, t in x.items()}, mask)
def test_registry_t2_tables_use_one_config_per_model():
from pulse.aggregate import load_registry
reg = load_registry()
t4 = {r["key"]: r for r in reg["4"]["rows"]}
seg = {r["key"]: r for r in reg["4seg"]["rows"]}
assert set(t4) == set(seg) and len(t4) == 8
for k, r in t4.items():
assert seg[k]["config"] == r["config"] and seg[k]["set"] == ["protocol=segment"]
for tid in ("4m5", "4imu", "4maj", "4segmaj"):
for r in reg[tid]["rows"]:
if "reuse" in r:
src_t, src_r = r["reuse"].split("/")
src = next(x for x in reg[src_t]["rows"] if x["key"] == src_r)
assert src["config"] == r["config"] and src.get("set", []) == r.get("set", [])
@pytest.fixture()
def long_root(tmp_path, monkeypatch):
monkeypatch.setattr(conftest, "T", 1200) # 12 s recordings
return conftest.build_tree(tmp_path / "PULSE")
@pytest.mark.parametrize("protocol,n_test", [("sliding", 8), ("segment", 16)])
def test_t2_end_to_end(long_root, tmp_path, protocol, n_test):
dr = DataRoot.from_arg(long_root)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(dr)), encoding="utf-8")
split = mini_split(tmp_path / "mini_split.json")
out = tmp_path / "run"
from pulse.tasks import t2
sets = ["modalities=[emg,imu]", f"protocol={protocol}", "train.epochs=2", "train.batch_size=2",
"train.warmup_epochs=1", "model.d_model=16", "model.n_heads=2", "model.n_layers=1",
"model.head_hidden=16", "load_workers=1"]
argv = ["--root", str(long_root), "--split", str(split), "--fold", "2", "--seed", "42",
"--config", str(REPO / "configs" / "t2" / "daf.yaml"), "--availability", str(av),
"--out", str(out), "--device", "cpu"]
for s in sets:
argv += ["--set", s]
assert t2.main(argv) == 0
r = load_results(out / "results.json")
prov = r["provenance"]
# fold 2: test v3, v6; inner_val v1; train v2, v5 (v5 has no usable IMU -> excluded)
assert prov["loaded_recordings"] == {"train": ["v2s1", "v2s2"], "inner_val": ["v1s1", "v1s2"],
"test": ["v3s1", "v3s2", "v6s1", "v6s2"]}
assert prov["taxonomy_md5"] and prov["inner_val_volunteers"] == ["v1"]
assert r["extra"]["data"]["test"]["n_samples"] == n_test
m = r["metrics"]["test"]
assert 0 <= m["act_vn@5"] <= min(m["verb_fine@5"], m["noun@5"]) <= 1
assert m["act_vn@1"] <= m["act_vn@5"]
assert r["selection"]["best_epoch"] in (1, 2) and len(r["selection"]["history"]) == 2
assert set(r["per_recording"]["test"]) == {"v3s1", "v3s2", "v6s1", "v6s2"}
assert r["n_params"] > 0 and str(long_root) not in json.dumps(r)
def _e2e_argv(root, tmp_path, out):
dr = DataRoot.from_arg(root)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(dr)), encoding="utf-8")
split = mini_split(tmp_path / "mini_split.json")
argv = ["--root", str(root), "--split", str(split), "--fold", "2", "--seed", "42",
"--config", str(REPO / "configs" / "t2" / "handformer.yaml"), "--availability", str(av),
"--out", str(out), "--device", "cpu"]
for s in ["modalities=[emg]", "train.epochs=1", "train.batch_size=2", "train.warmup_epochs=0",
"model.d_model=8", "model.n_heads=2", "model.n_layers=1", "model.head_hidden=8",
"load_workers=1"]:
argv += ["--set", s]
return argv
def test_t2_refuses_loaded_set_different_from_manifest(long_root, tmp_path, monkeypatch):
from pulse.errors import ManifestMismatchError
from pulse.tasks import t2
real = D.load_partition
def drop_last(dr, av, tax, rids, *a, **kw):
return real(dr, av, tax, list(rids)[:-1] if len(rids) > 2 else rids, *a, **kw)
monkeypatch.setattr(D, "load_partition", drop_last)
with pytest.raises(ManifestMismatchError, match="expected but not loaded"):
t2.main(_e2e_argv(long_root, tmp_path, tmp_path / "run"))
assert not (tmp_path / "run" / "results.json").exists()
def test_t2_refuses_when_checkout_changes_during_run(long_root, tmp_path, monkeypatch):
from pulse.errors import ProvenanceError
from pulse.tasks import t2
states = iter([{"git_sha": "a" * 40, "git_dirty": False, "git_diff_md5": None},
{"git_sha": "b" * 40, "git_dirty": False, "git_diff_md5": None}])
monkeypatch.setattr(t2, "git_state", lambda: next(states))
with pytest.raises(ProvenanceError, match="repository changed"):
t2.main(_e2e_argv(long_root, tmp_path, tmp_path / "run"))
assert not (tmp_path / "run" / "results.json").exists()
def test_t2_base_config_matches_paper_protocol():
from pulse.config import load_config
cfg = load_config(REPO / "configs" / "t2" / "daf.yaml")
assert cfg["protocol"] == "sliding" and cfg["sliding"] == {"window_sec": 4.0, "stride_sec": 1.0,
"label": "centre"}
tr = cfg["train"]
assert (tr["lr"], tr["weight_decay"], tr["batch_size"], tr["epochs"], tr["warmup_epochs"],
tr["label_smoothing"]) == (3e-4, 1e-4, 32, 40, 5, 0.05)
assert cfg["selection"] == {"metric": "act_vn@5", "mode": "max", "patience": 25}
def test_t2_missing_file_raises(long_root, tmp_path):
from pulse.errors import MissingFileError
from pulse.tasks import t2
argv = _e2e_argv(long_root, tmp_path, tmp_path / "run") # availability built first
(long_root / "data" / "v6" / "s2" / "aligned_emg_100hz.csv").unlink()
with pytest.raises(MissingFileError):
t2.main(argv)