PULSE-code / tests /test_t7.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
13.4 kB
"""T7 (EMG -> right-hand pose): target definition, metrics, alignment, end-to-end run."""
import json
import numpy as np
import pytest
from conftest import REPO, build_tree, mini_split, realdata, real_root
from pulse.paths import DataRoot
from pulse.provenance import REQUIRED_PROVENANCE, load_results
from pulse.tasks import t7_pose as P
torch = pytest.importorskip("torch")
def _rand_quat(rng, n):
q = rng.normal(size=(n, 4))
return q / np.linalg.norm(q, axis=1, keepdims=True)
def write_hand_tsv(path, emg, rng):
"""Skeleton TSV whose right-hand pose follows the EMG (all 21-joint models' columns)."""
T = emg.shape[0]
joints = sorted({j for hm in P.HAND_MODELS.values() for j in hm.joints}
| {"Hips", "Spine", "LeftHand"})
wrist = np.cumsum(rng.normal(0, 1, (T, 3)), 0) + 500.0
q = _rand_quat(rng, T)
R = P.quat_to_matrix(q)
flex = np.tanh(emg[:, :1] - emg[:, :1].mean()) # (T, 1) driven by channel 1
cols, vals = ["Frame", "Time"], [np.arange(T)[:, None], (np.arange(T) / 100)[:, None]]
for k, j in enumerate(joints):
off = np.array([10.0 * (k + 1), 5.0 * np.sin(k), 3.0]) + flex * np.array([0, 20.0, 0]) * (k % 3)
p = wrist if j == "RightHand" else wrist + np.einsum("tij,tj->ti", R, off)
cols += [f"{j}_{a}" for a in ("X", "Y", "Z", "QX", "QY", "QZ", "QW")]
vals += [p, q if j == "RightHand" else _rand_quat(rng, T)]
a = np.concatenate(vals, axis=1)
a[7, :] = np.nan # one NaN frame
a[7, :2] = [7, 0.07]
qi = cols.index("RightHand_QX")
a[11, qi:qi + 4] = 0.0 # one dropout frame (zero quaternion)
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write("\t".join(cols) + "\n")
for row in a:
f.write("\t".join("" if np.isnan(v) else f"{v:.5f}" for v in row) + "\n")
@pytest.fixture()
def hand_root(tmp_path):
root = build_tree(tmp_path / "PULSE")
rng = np.random.default_rng(1)
for tsv in sorted(root.glob("data/v*/s*/**/aligned_*_s_Q.tsv")):
emg = np.loadtxt(tsv.parents[1 if tsv.parent.name == "raw" else 0] / "aligned_emg_100hz.csv",
delimiter=",", skiprows=1)[:, 1:]
write_hand_tsv(tsv, emg, rng)
return root
def test_hand_models():
hm = P.get_hand_model("standard21")
assert hm.n_joints == 21 and hm.dim == 63
assert [hm.joints[i] for i in hm.tips] == [f"Right{f}Tip" for f in P.FINGERS]
assert len(hm.angle_bones) == 20 # paper: 20 hand-skeleton bones
assert sum(p == 0 for p, _ in hm.angle_bones) == 5 # 5 palm bones
assert len(hm.finger_bones) == 15 and all(p != 0 for p, _ in hm.finger_bones)
old = P.get_hand_model("inhand21")
assert "RightInHand" not in old.joints and "RightInHandThumb" in old.joints
assert len(set(old.columns())) == 63
def test_wrist_frame_matches_scipy():
Rot = pytest.importorskip("scipy.spatial.transform").Rotation
rng = np.random.default_rng(0)
hm = P.get_hand_model("standard21")
T = 50
q = _rand_quat(rng, T)
pos = rng.normal(0, 100, (T, 21, 3))
vals = np.concatenate([pos.reshape(T, 63), q], axis=1)
vals[3, 70 - 7:] = 0.0 # zero quaternion -> invalid
vals[5, 10] = np.nan # NaN position -> invalid
pose, valid = P.hand_pose_from_table(vals, P.needed_columns(hm), hm)
assert valid.sum() == T - 2 and not valid[3] and not valid[5]
ref = np.stack([Rot.from_quat(q[t]).inv().apply(pos[t] - pos[t, 0]) for t in range(T)])
np.testing.assert_allclose(pose.reshape(T, 21, 3)[valid], ref[valid], atol=1e-3)
assert np.all(pose[~valid] == 0) and np.all(pose[:, :3] == 0)
def test_metrics_definitions():
hm = P.get_hand_model("standard21")
rng = np.random.default_rng(0)
t = rng.normal(0, 30, (40, 63))
t[:, :3] = 0
assert P.bone_angle_errors(t, t, hm).max() < 1e-6
p = t.copy()
p[:, 3:] += 2.0 # every non-wrist joint off by (2,2,2)
st = P.recording_stats(p, t, np.ones(40, bool), hm)
assert st["sum_count:mpjpe"]["sum"] / st["sum_count:mpjpe"]["count"] == pytest.approx(
20 / 21 * 2 * np.sqrt(3))
assert st["sum_count:tip"]["sum"] / st["sum_count:tip"]["count"] == pytest.approx(2 * np.sqrt(3))
# bone angle: a 90 degree turn of one finger bone
u = np.zeros((1, 63))
hm_b = hm.finger_bones[0]
u[0, 3 * hm_b[1]:3 * hm_b[1] + 3] = [1.0, 0, 0]
v = u.copy()
v[0, 3 * hm_b[1]:3 * hm_b[1] + 3] = [0, 1.0, 0]
assert P.bone_angle_errors(v, u, hm, hm.finger_bones)[0, 0] == pytest.approx(90.0)
# rotating every joint about the wrist by 90 deg turns all 20 bones by 90 deg
t2 = t[:1].reshape(21, 3).copy()
rz = np.array([[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]])
t2[:, 2] = 0.0 # bones in the rotation plane
r2 = (t2 @ rz.T).reshape(1, 63)
st = P.recording_stats(r2, t2.reshape(1, 63), np.ones(1, bool), hm)
assert st["sum_count:angle"]["count"] == 20 and st["sum_count:angle_finger"]["count"] == 15
assert st["sum_count:angle"]["sum"] == pytest.approx(20 * 90.0)
with pytest.raises(Exception):
P.recording_stats(p, t, np.zeros(40, bool), hm)
def test_normalizer_constant_target_coordinates():
"""Coordinates that are constant up to TSV rounding are centred, not scaled."""
from pulse.tasks.t7 import MIN_TARGET_STD_MM, Normalizer, T7Recording
rng = np.random.default_rng(0)
pose = rng.normal(0, 20, (200, 63)).astype(np.float32)
pose[:, :3] = 0.0 # wrist
pose[:, 5] = rng.normal(0, 4e-4, 200) # rounding-level spread
rec = T7Recording("v1s1", rng.normal(size=(200, 8)).astype(np.float32), pose,
np.ones(200, bool))
n = Normalizer.fit([rec])
assert np.all(n.y_std[:3] == 1.0) and n.y_std[5] == 1.0 and MIN_TARGET_STD_MM > 4e-4
assert np.abs(n.y(pose)[:, 5]).max() < 0.01 # stays ~0, not unit variance
assert np.all(n.y_std[6:] > 1.0)
def test_mpjpe_loss_is_weighted_mm_error():
from pulse.tasks.t7 import masked_loss
scale = torch.full((63,), 10.0) # 1 z-unit = 10 mm
w = torch.ones(21)
w[[4, 8, 12, 16, 20]] = 2.0
pred = torch.zeros(1, 3, 63)
tgt = torch.zeros(1, 3, 63)
tgt[0, :, 3 * 4] = 1.0 # thumb tip off by 10 mm
valid = torch.tensor([[True, True, False]])
tgt[0, 2] = 100.0 # invalid frame ignored
loss = masked_loss(pred, tgt, valid, "mpjpe", 1.0, scale, w)
assert float(loss) == pytest.approx(10.0 * 2.0 / float(w.sum()), rel=1e-4)
def test_alignment_rules():
from pulse.errors import DataQualityError
from pulse.tasks.t7 import align_emg_pose
meta = {"n_samples": 100, "mocap_padding": {"mocap_sq": {"front_samples": 7, "n_frames": 80}}}
emg = np.zeros((80, 8))
se, sp, how = align_emg_pose("v2s6", emg, 100, meta)
assert (se, sp) == (slice(0, 80), slice(7, 87)) and how.startswith("tsv_window")
se, sp, _ = align_emg_pose("x", np.zeros((99, 8)), 100, {"n_samples": 99})
assert se == sp == slice(0, 99)
with pytest.raises(DataQualityError):
align_emg_pose("x", np.zeros((90, 8)), 100, {"n_samples": 100})
def test_masked_batchnorm_uses_real_frames_only():
from pulse.models.t7.pose_models import MaskedBatchNorm
g = torch.Generator().manual_seed(0)
x = torch.randn(3, 4, 10, generator=g)
mask = torch.ones(3, 10, dtype=torch.bool)
mask[1, 6:] = False
mask[2, 3:] = False
x[~mask.unsqueeze(1).expand_as(x)] = 0.0
bn, ref = MaskedBatchNorm(4).train(), torch.nn.BatchNorm1d(4).train()
out = bn(x, mask)
real = x.transpose(1, 2)[mask] # (n_real, C)
exp = ref(real) # BatchNorm over the real frames only
assert torch.allclose(out.transpose(1, 2)[mask], exp, atol=1e-5)
assert torch.allclose(bn.running_mean, ref.running_mean, atol=1e-6)
assert torch.allclose(bn.running_var, ref.running_var, atol=1e-5)
bn2 = MaskedBatchNorm(4).train()
x4 = x.unsqueeze(-1).repeat(1, 1, 1, 2) # (B, C, T, W)
assert torch.allclose(bn2(x4, mask)[..., 0], out, atol=1e-5)
@pytest.mark.parametrize("name", ["mscnn_transformer", "neuropose"])
def test_padding_does_not_change_real_frames(name):
"""Training mode: a recording padded inside a batch gives the same output as alone."""
from pulse.models.t7 import build_model
torch.manual_seed(0)
kw = {"use_velocity": False} if name == "mscnn_transformer" else {}
model = build_model(name, 8, 63, dropout=0.0, **kw).train()
a = torch.randn(1, 50, 8)
b = torch.randn(1, 90, 8)
x = torch.zeros(2, 90, 8)
x[0, :50], x[1] = a[0], b[0]
mask = torch.zeros(2, 90, dtype=torch.bool)
mask[0, :50] = mask[1] = True
batched = model(x, mask)
# same real frames, more padding (90 -> 160 frames; NeuroPose pads 90 to 120 itself)
x2 = torch.zeros(2, 160, 8)
x2[0, :50], x2[1, :90] = a[0], b[0]
mask2 = torch.zeros(2, 160, dtype=torch.bool)
mask2[0, :50] = mask2[1, :90] = True
batched2 = model(x2, mask2)
assert torch.allclose(batched[0, :50], batched2[0, :50], atol=1e-4)
assert torch.allclose(batched[1], batched2[1, :90], atol=1e-4)
@pytest.mark.parametrize("cfg", ["mean_pose", "transformer", "mscnn_transformer",
"mscnn_transformer_velocity", "neuropose", "tds", "tds+huber"])
def test_t7_end_to_end(hand_root, tmp_path, cfg):
cfg, _, loss = cfg.partition("+")
extra = ["--set", f"train.loss={loss}"] if loss else []
from pulse.data.availability import build_availability
from pulse.tasks import t7
dr = DataRoot.from_arg(hand_root)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(dr)), encoding="utf-8")
split = mini_split(tmp_path / "mini_split.json")
out = tmp_path / "run"
rc = t7.main(["--root", str(hand_root), "--split", str(split), "--fold", "0", "--seed", "42",
"--config", str(REPO / "configs" / "t7" / f"{cfg}.yaml"), "--availability", str(av),
"--out", str(out), "--device", "cpu", "--set", "train.epochs=2",
"--set", "train.batch_size=2", *extra])
assert rc == 0
r = load_results(out / "results.json")
prov = r["provenance"]
assert set(REQUIRED_PROVENANCE) <= set(prov)
# fold 0: test v1 (s1 + root-level TSV s2; s9 excluded), inner-val v2, train v3 + v6 (v5: no MoCap)
assert prov["loaded_recordings"] == {"train": ["v3s1", "v3s2", "v6s1", "v6s2"],
"inner_val": ["v2s1", "v2s2"], "test": ["v1s1", "v1s2"]}
assert prov["inner_val_volunteers"] == ["v2"]
m = r["metrics"]["test"]
assert set(m) == set(P.METRIC_DEFS) and m["mpjpe_mm"] > 0 and 0 <= m["angle_err_deg"] <= 180
assert r["selection"]["test_evaluations"] == 1
assert r["extra"]["recordings"]["v1s1"]["invalid_frames"] >= 0
if cfg == "mean_pose":
assert r["n_params"] == 0
else:
assert r["selection"]["epochs_run"] == 2 and r["selection"]["best_epoch"] in (1, 2)
def test_t7_manifest_mismatch_refused(hand_root, tmp_path):
"""A recording the availability table expects but that cannot be loaded raises."""
from pulse.data.availability import build_availability
from pulse.tasks import t7
dr = DataRoot.from_arg(hand_root)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(dr)), encoding="utf-8")
(hand_root / "data" / "v6" / "s2" / "raw" / "aligned_v6s2_s_Q.tsv").unlink()
split = mini_split(tmp_path / "mini_split.json")
from pulse.errors import MissingFileError
with pytest.raises(MissingFileError):
t7.main(["--root", str(hand_root), "--split", str(split), "--fold", "0", "--seed", "42",
"--config", str(REPO / "configs" / "t7" / "mean_pose.yaml"),
"--availability", str(av), "--out", str(tmp_path / "run"), "--device", "cpu"])
@realdata
def test_t7_real_columns_and_trim_window():
"""All pool33 recordings provide the 21-joint columns; the three re-trimmed ones align."""
import pandas as pd
from pulse.data.availability import Availability, default_availability_path
from pulse.splits import load_split
from pulse.tasks.t7 import load_t7_recording
dr = DataRoot.from_arg(real_root())
av = Availability.load(default_availability_path())
vols = load_split(REPO / "splits" / "pool33.json").volunteers
hm = P.get_hand_model("standard21")
need = set(P.needed_columns(hm))
for rid in av.rids(volunteers=vols, scenarios=[f"s{i}" for i in range(1, 9)]):
if av.ok(rid, "emg") and av.ok(rid, "mocap"):
cols = pd.read_csv(dr.mocap_tsv_path(rid), sep="\t", nrows=0).columns
assert need <= set(cols), rid
for rid in ("v2s6", "v13s2", "v13s3"):
rec = load_t7_recording(dr, rid, hm, 5, "rms")
assert rec.info["alignment"].startswith("tsv_window"), rec.info