"""T7 (EMG -> right-hand pose): target definition, metrics, alignment, end-to-end run.""" import json import numpy as np import pytest from conftest import REPO, build_tree, mini_split, realdata, real_root from pulse.paths import DataRoot from pulse.provenance import REQUIRED_PROVENANCE, load_results from pulse.tasks import t7_pose as P torch = pytest.importorskip("torch") def _rand_quat(rng, n): q = rng.normal(size=(n, 4)) return q / np.linalg.norm(q, axis=1, keepdims=True) def write_hand_tsv(path, emg, rng): """Skeleton TSV whose right-hand pose follows the EMG (all 21-joint models' columns).""" T = emg.shape[0] joints = sorted({j for hm in P.HAND_MODELS.values() for j in hm.joints} | {"Hips", "Spine", "LeftHand"}) wrist = np.cumsum(rng.normal(0, 1, (T, 3)), 0) + 500.0 q = _rand_quat(rng, T) R = P.quat_to_matrix(q) flex = np.tanh(emg[:, :1] - emg[:, :1].mean()) # (T, 1) driven by channel 1 cols, vals = ["Frame", "Time"], [np.arange(T)[:, None], (np.arange(T) / 100)[:, None]] for k, j in enumerate(joints): off = np.array([10.0 * (k + 1), 5.0 * np.sin(k), 3.0]) + flex * np.array([0, 20.0, 0]) * (k % 3) p = wrist if j == "RightHand" else wrist + np.einsum("tij,tj->ti", R, off) cols += [f"{j}_{a}" for a in ("X", "Y", "Z", "QX", "QY", "QZ", "QW")] vals += [p, q if j == "RightHand" else _rand_quat(rng, T)] a = np.concatenate(vals, axis=1) a[7, :] = np.nan # one NaN frame a[7, :2] = [7, 0.07] qi = cols.index("RightHand_QX") a[11, qi:qi + 4] = 0.0 # one dropout frame (zero quaternion) path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w", encoding="utf-8") as f: f.write("\t".join(cols) + "\n") for row in a: f.write("\t".join("" if np.isnan(v) else f"{v:.5f}" for v in row) + "\n") @pytest.fixture() def hand_root(tmp_path): root = build_tree(tmp_path / "PULSE") rng = np.random.default_rng(1) for tsv in sorted(root.glob("data/v*/s*/**/aligned_*_s_Q.tsv")): emg = np.loadtxt(tsv.parents[1 if tsv.parent.name == "raw" else 0] / "aligned_emg_100hz.csv", delimiter=",", skiprows=1)[:, 1:] write_hand_tsv(tsv, emg, rng) return root def test_hand_models(): hm = P.get_hand_model("standard21") assert hm.n_joints == 21 and hm.dim == 63 assert [hm.joints[i] for i in hm.tips] == [f"Right{f}Tip" for f in P.FINGERS] assert len(hm.angle_bones) == 20 # paper: 20 hand-skeleton bones assert sum(p == 0 for p, _ in hm.angle_bones) == 5 # 5 palm bones assert len(hm.finger_bones) == 15 and all(p != 0 for p, _ in hm.finger_bones) old = P.get_hand_model("inhand21") assert "RightInHand" not in old.joints and "RightInHandThumb" in old.joints assert len(set(old.columns())) == 63 def test_wrist_frame_matches_scipy(): Rot = pytest.importorskip("scipy.spatial.transform").Rotation rng = np.random.default_rng(0) hm = P.get_hand_model("standard21") T = 50 q = _rand_quat(rng, T) pos = rng.normal(0, 100, (T, 21, 3)) vals = np.concatenate([pos.reshape(T, 63), q], axis=1) vals[3, 70 - 7:] = 0.0 # zero quaternion -> invalid vals[5, 10] = np.nan # NaN position -> invalid pose, valid = P.hand_pose_from_table(vals, P.needed_columns(hm), hm) assert valid.sum() == T - 2 and not valid[3] and not valid[5] ref = np.stack([Rot.from_quat(q[t]).inv().apply(pos[t] - pos[t, 0]) for t in range(T)]) np.testing.assert_allclose(pose.reshape(T, 21, 3)[valid], ref[valid], atol=1e-3) assert np.all(pose[~valid] == 0) and np.all(pose[:, :3] == 0) def test_metrics_definitions(): hm = P.get_hand_model("standard21") rng = np.random.default_rng(0) t = rng.normal(0, 30, (40, 63)) t[:, :3] = 0 assert P.bone_angle_errors(t, t, hm).max() < 1e-6 p = t.copy() p[:, 3:] += 2.0 # every non-wrist joint off by (2,2,2) st = P.recording_stats(p, t, np.ones(40, bool), hm) assert st["sum_count:mpjpe"]["sum"] / st["sum_count:mpjpe"]["count"] == pytest.approx( 20 / 21 * 2 * np.sqrt(3)) assert st["sum_count:tip"]["sum"] / st["sum_count:tip"]["count"] == pytest.approx(2 * np.sqrt(3)) # bone angle: a 90 degree turn of one finger bone u = np.zeros((1, 63)) hm_b = hm.finger_bones[0] u[0, 3 * hm_b[1]:3 * hm_b[1] + 3] = [1.0, 0, 0] v = u.copy() v[0, 3 * hm_b[1]:3 * hm_b[1] + 3] = [0, 1.0, 0] assert P.bone_angle_errors(v, u, hm, hm.finger_bones)[0, 0] == pytest.approx(90.0) # rotating every joint about the wrist by 90 deg turns all 20 bones by 90 deg t2 = t[:1].reshape(21, 3).copy() rz = np.array([[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]) t2[:, 2] = 0.0 # bones in the rotation plane r2 = (t2 @ rz.T).reshape(1, 63) st = P.recording_stats(r2, t2.reshape(1, 63), np.ones(1, bool), hm) assert st["sum_count:angle"]["count"] == 20 and st["sum_count:angle_finger"]["count"] == 15 assert st["sum_count:angle"]["sum"] == pytest.approx(20 * 90.0) with pytest.raises(Exception): P.recording_stats(p, t, np.zeros(40, bool), hm) def test_normalizer_constant_target_coordinates(): """Coordinates that are constant up to TSV rounding are centred, not scaled.""" from pulse.tasks.t7 import MIN_TARGET_STD_MM, Normalizer, T7Recording rng = np.random.default_rng(0) pose = rng.normal(0, 20, (200, 63)).astype(np.float32) pose[:, :3] = 0.0 # wrist pose[:, 5] = rng.normal(0, 4e-4, 200) # rounding-level spread rec = T7Recording("v1s1", rng.normal(size=(200, 8)).astype(np.float32), pose, np.ones(200, bool)) n = Normalizer.fit([rec]) assert np.all(n.y_std[:3] == 1.0) and n.y_std[5] == 1.0 and MIN_TARGET_STD_MM > 4e-4 assert np.abs(n.y(pose)[:, 5]).max() < 0.01 # stays ~0, not unit variance assert np.all(n.y_std[6:] > 1.0) def test_mpjpe_loss_is_weighted_mm_error(): from pulse.tasks.t7 import masked_loss scale = torch.full((63,), 10.0) # 1 z-unit = 10 mm w = torch.ones(21) w[[4, 8, 12, 16, 20]] = 2.0 pred = torch.zeros(1, 3, 63) tgt = torch.zeros(1, 3, 63) tgt[0, :, 3 * 4] = 1.0 # thumb tip off by 10 mm valid = torch.tensor([[True, True, False]]) tgt[0, 2] = 100.0 # invalid frame ignored loss = masked_loss(pred, tgt, valid, "mpjpe", 1.0, scale, w) assert float(loss) == pytest.approx(10.0 * 2.0 / float(w.sum()), rel=1e-4) def test_alignment_rules(): from pulse.errors import DataQualityError from pulse.tasks.t7 import align_emg_pose meta = {"n_samples": 100, "mocap_padding": {"mocap_sq": {"front_samples": 7, "n_frames": 80}}} emg = np.zeros((80, 8)) se, sp, how = align_emg_pose("v2s6", emg, 100, meta) assert (se, sp) == (slice(0, 80), slice(7, 87)) and how.startswith("tsv_window") se, sp, _ = align_emg_pose("x", np.zeros((99, 8)), 100, {"n_samples": 99}) assert se == sp == slice(0, 99) with pytest.raises(DataQualityError): align_emg_pose("x", np.zeros((90, 8)), 100, {"n_samples": 100}) def test_masked_batchnorm_uses_real_frames_only(): from pulse.models.t7.pose_models import MaskedBatchNorm g = torch.Generator().manual_seed(0) x = torch.randn(3, 4, 10, generator=g) mask = torch.ones(3, 10, dtype=torch.bool) mask[1, 6:] = False mask[2, 3:] = False x[~mask.unsqueeze(1).expand_as(x)] = 0.0 bn, ref = MaskedBatchNorm(4).train(), torch.nn.BatchNorm1d(4).train() out = bn(x, mask) real = x.transpose(1, 2)[mask] # (n_real, C) exp = ref(real) # BatchNorm over the real frames only assert torch.allclose(out.transpose(1, 2)[mask], exp, atol=1e-5) assert torch.allclose(bn.running_mean, ref.running_mean, atol=1e-6) assert torch.allclose(bn.running_var, ref.running_var, atol=1e-5) bn2 = MaskedBatchNorm(4).train() x4 = x.unsqueeze(-1).repeat(1, 1, 1, 2) # (B, C, T, W) assert torch.allclose(bn2(x4, mask)[..., 0], out, atol=1e-5) @pytest.mark.parametrize("name", ["mscnn_transformer", "neuropose"]) def test_padding_does_not_change_real_frames(name): """Training mode: a recording padded inside a batch gives the same output as alone.""" from pulse.models.t7 import build_model torch.manual_seed(0) kw = {"use_velocity": False} if name == "mscnn_transformer" else {} model = build_model(name, 8, 63, dropout=0.0, **kw).train() a = torch.randn(1, 50, 8) b = torch.randn(1, 90, 8) x = torch.zeros(2, 90, 8) x[0, :50], x[1] = a[0], b[0] mask = torch.zeros(2, 90, dtype=torch.bool) mask[0, :50] = mask[1] = True batched = model(x, mask) # same real frames, more padding (90 -> 160 frames; NeuroPose pads 90 to 120 itself) x2 = torch.zeros(2, 160, 8) x2[0, :50], x2[1, :90] = a[0], b[0] mask2 = torch.zeros(2, 160, dtype=torch.bool) mask2[0, :50] = mask2[1, :90] = True batched2 = model(x2, mask2) assert torch.allclose(batched[0, :50], batched2[0, :50], atol=1e-4) assert torch.allclose(batched[1], batched2[1, :90], atol=1e-4) @pytest.mark.parametrize("cfg", ["mean_pose", "transformer", "mscnn_transformer", "mscnn_transformer_velocity", "neuropose", "tds", "tds+huber"]) def test_t7_end_to_end(hand_root, tmp_path, cfg): cfg, _, loss = cfg.partition("+") extra = ["--set", f"train.loss={loss}"] if loss else [] from pulse.data.availability import build_availability from pulse.tasks import t7 dr = DataRoot.from_arg(hand_root) av = tmp_path / "availability.json" av.write_text(json.dumps(build_availability(dr)), encoding="utf-8") split = mini_split(tmp_path / "mini_split.json") out = tmp_path / "run" rc = t7.main(["--root", str(hand_root), "--split", str(split), "--fold", "0", "--seed", "42", "--config", str(REPO / "configs" / "t7" / f"{cfg}.yaml"), "--availability", str(av), "--out", str(out), "--device", "cpu", "--set", "train.epochs=2", "--set", "train.batch_size=2", *extra]) assert rc == 0 r = load_results(out / "results.json") prov = r["provenance"] assert set(REQUIRED_PROVENANCE) <= set(prov) # fold 0: test v1 (s1 + root-level TSV s2; s9 excluded), inner-val v2, train v3 + v6 (v5: no MoCap) assert prov["loaded_recordings"] == {"train": ["v3s1", "v3s2", "v6s1", "v6s2"], "inner_val": ["v2s1", "v2s2"], "test": ["v1s1", "v1s2"]} assert prov["inner_val_volunteers"] == ["v2"] m = r["metrics"]["test"] assert set(m) == set(P.METRIC_DEFS) and m["mpjpe_mm"] > 0 and 0 <= m["angle_err_deg"] <= 180 assert r["selection"]["test_evaluations"] == 1 assert r["extra"]["recordings"]["v1s1"]["invalid_frames"] >= 0 if cfg == "mean_pose": assert r["n_params"] == 0 else: assert r["selection"]["epochs_run"] == 2 and r["selection"]["best_epoch"] in (1, 2) def test_t7_manifest_mismatch_refused(hand_root, tmp_path): """A recording the availability table expects but that cannot be loaded raises.""" from pulse.data.availability import build_availability from pulse.tasks import t7 dr = DataRoot.from_arg(hand_root) av = tmp_path / "availability.json" av.write_text(json.dumps(build_availability(dr)), encoding="utf-8") (hand_root / "data" / "v6" / "s2" / "raw" / "aligned_v6s2_s_Q.tsv").unlink() split = mini_split(tmp_path / "mini_split.json") from pulse.errors import MissingFileError with pytest.raises(MissingFileError): t7.main(["--root", str(hand_root), "--split", str(split), "--fold", "0", "--seed", "42", "--config", str(REPO / "configs" / "t7" / "mean_pose.yaml"), "--availability", str(av), "--out", str(tmp_path / "run"), "--device", "cpu"]) @realdata def test_t7_real_columns_and_trim_window(): """All pool33 recordings provide the 21-joint columns; the three re-trimmed ones align.""" import pandas as pd from pulse.data.availability import Availability, default_availability_path from pulse.splits import load_split from pulse.tasks.t7 import load_t7_recording dr = DataRoot.from_arg(real_root()) av = Availability.load(default_availability_path()) vols = load_split(REPO / "splits" / "pool33.json").volunteers hm = P.get_hand_model("standard21") need = set(P.needed_columns(hm)) for rid in av.rids(volunteers=vols, scenarios=[f"s{i}" for i in range(1, 9)]): if av.ok(rid, "emg") and av.ok(rid, "mocap"): cols = pd.read_csv(dr.mocap_tsv_path(rid), sep="\t", nrows=0).columns assert need <= set(cols), rid for rid in ("v2s6", "v13s2", "v13s3"): rec = load_t7_recording(dr, rid, hm, 5, "rms") assert rec.info["alignment"].startswith("tsv_window"), rec.info