PULSE-code / tests /test_t5.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
13.1 kB
"""T5 (grasp-state recognition): label rules, release re-trim, normalisation,
end-to-end run on the synthetic tree, and the report."""
import csv
import json
import numpy as np
import pytest
import yaml
from conftest import REPO, T, mini_split
from pulse.data import modalities as M
from pulse.data.annotations import Segment
from pulse.errors import ManifestMismatchError
from pulse.taxonomy import Taxonomy, TaxonomyError, default_taxonomy_path
from pulse.tasks import t5_data as D
SPEC = D.LabelSpec()
def test_label_spec_frames():
assert (SPEC.sr, SPEC.t_obs, SPEC.t_fut, SPEC.stride, SPEC.sustained_frames) == (20, 20, 10, 5, 6)
with pytest.raises(ValueError):
D.LabelSpec.from_config({"t_obs": 1})
D.LabelSpec().check_against(Taxonomy.load(default_taxonomy_path()))
def test_longest_true_run():
w = np.array([[1, 1, 0, 1, 1, 1, 0, 0, 0, 0], [0] * 10, [1] * 10], dtype=bool)
assert D.longest_true_run(w).tolist() == [3, 0, 10]
def test_derive_anchors_rules():
n = 60
mask = np.zeros(n, bool)
contact = np.zeros(n, bool)
mask[20:30] = True # anchor 20: future fully grasp, no contact -> Attempted
mask[25:35] = True
contact[25:31] = True # anchor 25: run of 6 frames (0.3 s) -> Sustained
mask[40:45] = True # anchor 40: exactly 50 % grasp -> grasp; contact run 5 -> Attempted
contact[40:45] = True
anchors, labels, events = D.derive_anchors(mask, contact, SPEC)
assert anchors.tolist() == [20, 25, 30, 35, 40, 45, 50]
lab = dict(zip(anchors.tolist(), labels.tolist()))
assert lab[20] == 1 and lab[25] == 2 and lab[40] == 1 and lab[45] == 0 and lab[50] == 0
ev = dict(zip(anchors.tolist(), events.tolist()))
# anchor 25: past [5,25) no contact, future [25,35) 6/10 contact -> pre-contact
assert ev[25] == 1
# anchor 30: past [10,30) 5/20 contact (not >50 %), future 1/10 -> non-contact
assert ev[30] == 0
# sustained anchors only in pre-contact / steady-grip
assert set(events[labels == 2].tolist()) <= {1, 2}
a, l, e = D.derive_anchors(np.zeros(25, bool), np.zeros(25, bool), SPEC)
assert a.size == 0
def test_steady_grip_and_release():
n = 80
mask = np.ones(n, bool)
contact = np.zeros(n, bool)
contact[0:50] = True
anchors, labels, events = D.derive_anchors(mask, contact, SPEC)
ev = dict(zip(anchors.tolist(), events.tolist()))
assert ev[30] == 2 and ev[50] == 3 # steady-grip, then release
assert dict(zip(anchors.tolist(), labels.tolist()))[30] == 2
def _seg(i, a, b, verb):
return Segment("v1s1", i, a, b, verb, "cup", "right", {})
def test_grasp_mask_offset_clipping_and_unknown_verbs():
tax = Taxonomy.load(default_taxonomy_path())
segs = [_seg(0, 1.0, 2.0, "grasp"), _seg(1, 2.0, 3.0, "adjust"), _seg(2, 2.5, 9.0, "hold"),
_seg(3, 0.0, 1.0, "support")]
m, st = D.grasp_mask(segs, 400, SPEC.grasp_verbs, tax=tax)
assert m[100:200].all() and not m[200:250].any() and m[250:400].all() and not m[:100].any()
assert st["grasp_segments"] == 2 and st["grasp_clipped"] == 1
m2, st2 = D.grasp_mask(segs, 400, SPEC.grasp_verbs, offset_frames=150, tax=tax)
assert m2[0:50].all() and not m2[50:100].any() and m2[100:400].all()
assert st2["grasp_clipped"] == 2
with pytest.raises(TaxonomyError):
D.grasp_mask([_seg(0, 0, 1, "juggle")], 100, SPEC.grasp_verbs, tax=tax)
def test_release_trim_alignment():
meta = {"n_samples": 1000, "mocap_padding": {"mocap_sq": {"front_samples": 100,
"n_frames": 700}}}
assert D.release_trim_window(meta) == (100, 700)
full = np.arange(1000, dtype=np.float32)[:, None]
trimmed = full[100:800].copy()
out, off = D.align_release_trim("v2s6", {"imu": full, "emg": trimmed, "pressure": trimmed}, meta)
assert off == 100 and np.array_equal(out["imu"], trimmed)
out, off = D.align_release_trim("v2s6", {"imu": full}, meta) # nothing trimmed
assert off == 0 and out["imu"].shape[0] == 1000
out, off = D.align_release_trim("v2s6", {"emg": trimmed, "pressure": trimmed}, meta)
assert off == 100 # labels must shift
assert D.release_trim_window({"n_samples": 1000}) is None
assert D.release_trim_window({"n_samples": 700, "mocap_padding": {
"mocap_sq": {"front_samples": 0, "n_frames": 700}}}) is None
def test_window_stats_match_bruteforce():
rng = np.random.default_rng(0)
rec = D.RecordingSamples("v1s1", {"emg": rng.normal(3, 2, (60, 2)).astype(np.float32)},
np.array([20, 25, 30, 50]), np.array([0, 1, 2, 0]),
np.array([0, 1, 2, 3]), 300, 0, {}, "x")
p = D.build_partition("train", [rec], ["emg"], SPEC)
mu, sd = D.window_stats(p)["emg"]
wins = np.concatenate([rec.arrays["emg"][a - 20:a] for a in rec.anchors])
assert np.allclose(mu, wins.mean(0), atol=1e-5) and np.allclose(sd, wins.std(0), atol=1e-5)
q, info = D.cap_per_class(p, 1, seed=1)
assert q.class_counts() == [1, 1, 1] and info["before"] == [2, 1, 1]
w = q.window_index(np.arange(q.n))
assert w.shape == (3, 20)
# ---------------------------------------------------------------------------
# end to end on the synthetic tree
# ---------------------------------------------------------------------------
def _write_pressure(path, contact_from, contact_to):
p = np.zeros((T, 50))
p[contact_from:contact_to, :6] = 25.0
with open(path, "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["time"] + list(M.PRESSURE_COLS))
for i, row in enumerate(p):
w.writerow([f"{i / 100:.2f}"] + [f"{x:.3f}" for x in row])
def _prepare(mini_root, tmp_path):
from pulse.data.availability import build_availability
from pulse.paths import DataRoot
for d in sorted((mini_root / "data").glob("v*/s*")):
if (d / "aligned_pressure_100hz.csv").is_file():
_write_pressure(d / "aligned_pressure_100hz.csv", 130, 230)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(DataRoot.from_arg(mini_root))), encoding="utf-8")
return av, mini_split(tmp_path / "split.json")
SMOKE = ["--set", "train.epochs=2", "--set", "train.batch_size=4", "--set", "model.d_model=16"]
def _run_t5(mini_root, av, split, cfg, fold, out, seed=42, extra=()):
from pulse.tasks import t5
return t5.main(["--root", str(mini_root), "--split", str(split), "--fold", str(fold),
"--seed", str(seed), "--config", str(REPO / "configs" / "t5" / cfg),
"--availability", str(av), "--out", str(out), "--device", "cpu",
*SMOKE, *extra])
def test_t5_end_to_end(mini_root, tmp_path):
pytest.importorskip("torch")
from pulse.provenance import load_results
from pulse.metrics import compute_metrics
av, split = _prepare(mini_root, tmp_path)
out = tmp_path / "run"
assert _run_t5(mini_root, av, split, "daf_emg.yaml", 0, out) == 0
r = load_results(out / "results.json")
prov = r["provenance"]
# fold 0: test v1, inner-val v2, train v3 (no pressure -> excluded), v5, v6
assert prov["loaded_recordings"] == {"train": ["v5s1", "v5s2", "v6s1", "v6s2"],
"inner_val": ["v2s1", "v2s2"], "test": ["v1s1", "v1s2"]}
assert prov["inner_val_volunteers"] == ["v2"] and prov["taxonomy_md5"]
assert prov["config"]["inputs"] == ["emg"] and r["selection"]["best_epoch"] in (1, 2)
assert set(r["per_recording"]["test"]["v1s1"]) == {"confusion"} | {
f"confusion:{e}" for e in D.EVENT_NAMES}
assert compute_metrics(r) == pytest.approx(r["metrics"]["test"], nan_ok=True)
assert r["extra"]["anchors"]["test"]["classes"] == [0, 2, 6]
# pressure-only row: same anchors
out2 = tmp_path / "run_p"
assert _run_t5(mini_root, av, split, "daf_pressure.yaml", 0, out2) == 0
r2 = load_results(out2 / "results.json")
assert r2["extra"]["anchors"] == r["extra"]["anchors"]
for rid in ("v1s1", "v1s2"):
a = np.asarray(r["per_recording"]["test"][rid]["confusion"]).sum(1)
b = np.asarray(r2["per_recording"]["test"][rid]["confusion"]).sum(1)
assert a.tolist() == b.tolist()
def test_t5_manifest_check_refuses_missing_recording(mini_root, tmp_path, monkeypatch):
pytest.importorskip("torch")
from pulse.tasks import t5
av, split = _prepare(mini_root, tmp_path)
real = t5.load_recording_samples
def drop_one(dr, rid, *a, **k):
r = real(dr, rid, *a, **k)
if rid == "v6s2":
r.rid = "v6s1" # pretend a recording silently went missing
return r
monkeypatch.setattr(t5, "load_recording_samples", drop_one)
with pytest.raises(ManifestMismatchError):
_run_t5(mini_root, av, split, "daf_emg.yaml", 0, tmp_path / "bad")
assert not (tmp_path / "bad" / "results.json").exists()
def _fake_run(inputs, cm, spec=None, overrides=(), lr=3e-4):
cfg = {"task": "t5", "inputs": list(inputs), "sample_streams": ["emg", "pressure"],
"model": {"backbone": "daf"}, "train": {"lr": lr}}
per = {"v1s1": {"confusion": cm, **{f"confusion:{e}": cm for e in D.EVENT_NAMES}}}
return {"provenance": {"config": cfg, "overrides": list(overrides)},
"per_recording": {"test": per},
"extra": {"sample_streams": ["emg", "pressure"], "label_spec": spec or {"a": 1},
"annotations_md5": "x", "class_names": list(D.CLASS_NAMES),
"event_names": list(D.EVENT_NAMES),
"per_class_cap": {"per_class_max": 10000}}}
def test_t5_report_refuses_mixed_labels_and_unpaired_configs():
from pulse.errors import ProvenanceError
from pulse.tasks import t5_report as R
cm = [[3, 1, 0], [0, 2, 1], [1, 0, 4]]
ctrl = {(0, 42): _fake_run(["emg"], cm)}
treat = {(0, 42): _fake_run(["emg", "pressure"], [[4, 0, 0], [1, 2, 0], [0, 0, 5]])}
R.check_label_provenance({"c": ctrl, "t": treat})
R.check_pair("t", "c", treat, ctrl)
assert R.anchor_counts({"c": ctrl, "t": treat}, {0: ["v1s1"]})["overall"] == {0: 12}
# same anchor total, different true classes (e.g. another sustained_min_sec)
relab = {(0, 42): _fake_run(["emg", "pressure"], [[4, 0, 0], [0, 2, 0], [0, 1, 5]])}
with pytest.raises(ProvenanceError):
R.anchor_counts({"c": ctrl, "t": relab}, {0: ["v1s1"]})
with pytest.raises(ProvenanceError):
R.check_label_provenance({"c": ctrl, "t": {(0, 42): _fake_run(["emg"], cm, spec={"a": 2})}})
with pytest.raises(ProvenanceError):
R.check_pair("t", "c", {(0, 42): _fake_run(["emg", "pressure"], cm, lr=1e-4)}, ctrl)
with pytest.raises(ProvenanceError):
R.check_pair("t", "c", {(0, 42): _fake_run(["emg", "pressure"], cm,
overrides=["train.epochs=2"])}, ctrl)
with pytest.raises(ProvenanceError):
R.check_pair("t", "c", {(0, 42): _fake_run(["pressure"], cm)}, ctrl)
def test_t5_report(mini_root, tmp_path):
pytest.importorskip("torch")
pytest.importorskip("scipy.stats")
from pulse.tasks import t5_report
av, split = _prepare(mini_root, tmp_path)
reg = yaml.safe_load((REPO / "configs" / "tables.yaml").read_text(encoding="utf-8"))
spec = reg["tables"]["8"]
spec["split"] = str(split)
spec["seeds"] = [42]
spec["folds"] = [0, 1, 2]
spec["t5_pairs"] = {"daf": ["daf_with_pressure", "daf_no_pressure"],
"futr": ["futr_with_pressure", "futr_no_pressure"]}
keep = {"daf_no_pressure", "daf_with_pressure", "futr_no_pressure", "futr_with_pressure",
"daf_pressure_only"}
spec["rows"] = [r for r in spec["rows"] if r["key"] in keep]
regf = tmp_path / "tables.yaml"
regf.write_text(yaml.safe_dump({"tables": {"8": spec}}), encoding="utf-8")
runs = tmp_path / "runs"
for row in spec["rows"]:
for k in (0, 1, 2):
out = runs / "8" / row["key"] / f"fold{k}_seed42"
assert _run_t5(mini_root, av, split, row["config"].split("/")[-1], k, out) == 0
rc = t5_report.main(["--table", "8", "--registry", str(regf), "--runs", str(runs),
"--out", str(tmp_path / "tables"), "--allow-dirty"])
assert rc == 0
rep = json.loads((tmp_path / "tables" / "t5_tab8.json").read_text(encoding="utf-8"))
assert set(rep["groups"]) == {"daf", "futr", "all"}
allm = rep["groups"]["all"]["metrics"]["f1_sustained"]
assert allm["pair_level"]["n_units"] == 6 and allm["fold_level"]["n_units"] == 3
assert rep["pressure_only"]["metrics"]["macro_f1"]["vs_daf_with_pressure"]
for name in ("lift", "classes", "events", "backbones", "tests"):
assert (tmp_path / "tables" / f"t5_tab8_{name}.tex").is_file()
nums = (tmp_path / "tables" / "numbers_t5_8.tex").read_text(encoding="utf-8")
assert r"pulse@t5/8/all/f1_sustained/liftpm" in nums and r"pulse@t5/8/ntest/steady-grip" in nums