PULSE-code / tests /test_f5.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
11.6 kB
"""F5 (S9 pre-training -> T2 fine-tuning): labels, subsets, transfer, cache, end to end."""
import json
import numpy as np
import pytest
torch = pytest.importorskip("torch")
from conftest import REPO # noqa: E402
from pulse.data.annotations import load_segments_file # noqa: E402
from pulse.data.availability import build_availability # noqa: E402
from pulse.models.f5 import pretrain as P # noqa: E402
from pulse.models.f5 import s9 as S9 # noqa: E402
from pulse.models.f5 import subset as SUB # noqa: E402
from pulse.paths import DataRoot # noqa: E402
from pulse.provenance import REQUIRED_PROVENANCE, load_results # noqa: E402
from pulse.taxonomy import build_taxonomy # noqa: E402
DAF = {"d_model": 16, "n_layers": 2, "n_heads": 2, "dropout": 0.0, "head_hidden": 8, "max_T": 64,
"mod_dropout_p": 0.3, "fusion_type": "concat", "pool_type": "query", "joint_head": True}
SIZES = {"verb_fine": 17, "verb_composite": 6, "noun": 5, "hand": 3}
def test_s9_decomposition_is_closed():
assert len(S9.S9_DECOMPOSITION) == 16
assert len(S9.S9_VERBS) == 12 and len(S9.S9_NOUNS) == 10
with pytest.raises(S9.S9LabelError):
S9.check_disjoint(["v9", "v16"], ["v1", "v16"])
S9.check_disjoint(["v9", "v16", "v17", "v18", "v19", "v20", "v21"],
json.loads((REPO / "splits" / "pool27.json").read_text())["volunteers"])
def test_backbone_transfer_by_name():
from pulse.models.t2 import build_model
dims = {"emg": 8, "imu": 180}
s9 = build_model("dailyactformer_v2", dims, P.S9_SIZES, **DAF)
t2 = build_model("dailyactformer_v2", dims, SIZES, **DAF)
state = {k: v for k, v in s9.state_dict().items() if not k.startswith("head.")}
keys = P.load_backbone(t2, state)
assert keys and all(torch.equal(t2.state_dict()[k], state[k]) for k in keys)
assert any(k.startswith("stems.emg.") for k in keys) and "pos_embed" in keys
with pytest.raises(ValueError, match="keys differ"):
P.load_backbone(t2, {k: v for k, v in state.items() if k != "pos_embed"})
with pytest.raises(ValueError, match="shapes"):
P.load_backbone(build_model("dailyactformer_v2", {"emg": 8, "imu": 36}, SIZES, **DAF), state)
g = P.backbone_param_groups(t2, 1e-3, 0.1)
assert [x["lr"] for x in g] == [1e-4, 1e-3]
n_head = sum(p.numel() for n, p in t2.named_parameters() if n.startswith("head."))
assert sum(p.numel() for p in g[1]["params"]) == n_head
assert sum(p.numel() for x in g for p in x["params"]) == sum(p.numel() for p in t2.parameters())
def test_subsets_nested_and_deterministic():
keys = [(f"v{i}s1", 100 * k) for i in range(1, 9) for k in range(25)]
subs = {f: SUB.subset_segments(keys, f, 42, 0) for f in (0.05, 0.1, 0.25, 0.5, 1.0)}
assert [len(subs[f]) for f in sorted(subs)] == [10, 20, 50, 100, 200]
for a, b in ((0.05, 0.1), (0.1, 0.25), (0.25, 0.5), (0.5, 1.0)):
assert set(subs[a]) <= set(subs[b])
assert SUB.subset_segments(keys, 0.25, 42, 0) == subs[0.25]
assert SUB.subset_segments(keys, 0.25, 43, 0) != subs[0.25]
assert SUB.subset_segments(keys, 0.25, 42, 1) != subs[0.25]
idx = SUB.subset_index([keys[0], keys[1], keys[0]], [keys[0]])
assert idx.tolist() == [0, 2]
def test_encoder_cache_builds_once(tmp_path):
calls = []
def build():
calls.append(1)
return {"w": torch.ones(2)}, {"recordings": ["v1s9"]}
s1, r1, b1 = P.ensure_encoder(tmp_path, "k" * 32, {"a": 1}, build, poll_sec=0.01)
s2, r2, b2 = P.ensure_encoder(tmp_path, "k" * 32, {"a": 1}, build, poll_sec=0.01)
assert calls == [1] and b1 and not b2 and r1["encoder_md5"] == r2["encoder_md5"]
assert torch.equal(s1["w"], s2["w"])
(tmp_path / ("j" * 32 + ".lock")).mkdir()
with pytest.raises(TimeoutError):
P.ensure_encoder(tmp_path, "j" * 32, {}, build, timeout_sec=0.05, poll_sec=0.01)
# ---------------------------------------------------------------------------
# end to end on the synthetic tree
# ---------------------------------------------------------------------------
def _setup(mini_root, tmp_path):
dr = DataRoot.from_arg(mini_root)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(dr)), encoding="utf-8")
segs = []
for p in sorted((mini_root / "annotations").glob("v*/s*.json")):
rid = p.parent.name + p.stem
segs += [(rid, s.verb, s.noun, s.hand) for s in load_segments_file(p, rid)]
taxp = tmp_path / "taxonomy.json"
taxp.write_text(json.dumps(build_taxonomy(segs, threshold=1)), encoding="utf-8")
split = tmp_path / "split.json" # pool without v1, whose S9 is the pre-training data
split.write_text(json.dumps({"schema": "pulse-split/1", "name": "mini4",
"volunteers": ["v2", "v3", "v5", "v6"],
"folds": [{"fold": 0, "test": ["v6"], "inner_val": ["v2"],
"train": ["v3", "v5"]},
{"fold": 1, "test": ["v2", "v3", "v5"],
"inner_val": ["v6"], "train": []}]}),
encoding="utf-8")
s9dir = tmp_path / "s9labels"
(s9dir / "v1").mkdir(parents=True)
names = sorted(S9.S9_DECOMPOSITION)
(s9dir / "v1" / "s9.json").write_text(json.dumps({"segments": [
{"timestamp": f"00:00:0{i % 2}-00:00:0{i % 2 + 1}",
"action_annotation": {"action_name": names[i % 16], "object_name": "cup",
"hand_type": "right"}} for i in range(12)]}), encoding="utf-8")
return av, taxp, split, s9dir
def _run(mini_root, av, taxp, split, s9dir, out, cfg, extra_sets=()):
from pulse.tasks import f5
sets = ["train.epochs=2", "train.batch_size=2", "pretrain.train.epochs=2",
"pretrain.train.batch_size=4", "pretrain.min_frames=4",
f"pretrain.cache_dir={(out.parent / 'cache').as_posix()}", "pretrain.volunteers=[v1]",
"model.d_model=16", "model.n_heads=2", "model.head_hidden=8", "load_workers=1",
"sliding.window_sec=1.0", "sliding.stride_sec=0.5", *extra_sets]
argv = ["--root", str(mini_root), "--split", str(split), "--fold", "0", "--seed", "42",
"--config", str(REPO / cfg), "--availability", str(av), "--taxonomy", str(taxp),
"--s9-annotations", str(s9dir), "--out", str(out), "--device", "cpu"]
for s in sets:
argv += ["--set", s]
assert f5.main(argv) == 0
return load_results(out / "results.json")
def test_f5_end_to_end(mini_root, tmp_path):
av, taxp, split, s9dir = _setup(mini_root, tmp_path)
r1 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "a", "configs/f5/sliding/s9pre_f050.yaml")
prov = r1["provenance"]
assert set(REQUIRED_PROVENANCE) <= set(prov)
assert prov["loaded_recordings"] == {"train": ["v3s1", "v3s2"], "inner_val": ["v2s1", "v2s2"],
"test": ["v6s1", "v6s2"]} # v5: 72-column IMU
assert prov["inner_val_volunteers"] == ["v2"] and "s9_pretrain" in prov["manifests"]
assert prov["taxonomy_md5"] and prov["config"]["init"] == "s9"
ex = r1["extra"]
assert ex["s9_pretrain"]["built_by_this_run"] and ex["fraction"] == 0.5
lb = ex["label_budget"]
assert lb["segments_total"] == 6 and lb["segments_used"] == 3 # 0-1, 1-2, 2-3 s overlap the span
assert set(r1["per_recording"]["test"]) == {"v6s1", "v6s2"}
assert r1["selection"]["criterion"].startswith("inner_val/verb_composite@1")
assert "verb_composite@1" in r1["metrics"]["test"]
assert str(s9dir) not in json.dumps(r1) and str(mini_root) not in json.dumps(r1)
# same seed: the encoder is reused; the scratch run never touches S9
r2 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "b", "configs/f5/sliding/s9pre_f100.yaml")
assert not r2["extra"]["s9_pretrain"]["built_by_this_run"]
assert r2["extra"]["s9_pretrain"]["encoder_md5"] == ex["s9_pretrain"]["encoder_md5"]
assert r2["extra"]["label_budget"]["samples_used"] == r2["extra"]["label_budget"]["samples_total"]
r3 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "c", "configs/f5/segment/scratch_f100.yaml")
assert r3["extra"]["s9_pretrain"] is None and r3["extra"]["protocol"] == "segment"
assert r3["extra"]["data"]["test"]["n_samples"] == 4 # 2 segments in the span x 2
def test_f5_missing_s9_labels_raise(mini_root, tmp_path):
from pulse.errors import MissingFileError
av, taxp, split, s9dir = _setup(mini_root, tmp_path)
(s9dir / "v1" / "s9.json").unlink()
with pytest.raises(MissingFileError):
_run(mini_root, av, taxp, split, s9dir, tmp_path / "a", "configs/f5/sliding/s9pre_f050.yaml")
def test_f5_registry_rows():
from pulse.aggregate import load_registry, table_spec
from pulse.config import load_config
reg = load_registry()
for t, proto in (("5", "sliding"), ("5seg", "segment")):
spec = table_spec(reg, t)
keys = {r["key"] for r in spec["rows"]}
assert len(keys) == 10 and spec["status"] == "ready" and spec["task"] == "f5"
# the paper's sweep (results.tex, Table 5): 5 / 10 / 25 / 50 / 100 %
assert sorted({int(k[-3:]) for k in keys}) == [5, 10, 25, 50, 100]
for r in spec["rows"]:
cfg = load_config(REPO / r["config"])
assert cfg["task"] == "f5" and cfg["protocol"] == proto
assert cfg["modalities"] == ["emg", "imu"] and cfg["model"]["name"] == "dailyactformer_v2"
assert cfg["init"] == ("s9" if r["key"].startswith("s9pre") else "scratch")
assert round(100 * cfg["fraction"]) == int(r["key"][-3:])
if r["key"].startswith("s9pre"):
assert r["reference"] == r["key"].replace("s9pre", "scratch") and r["reference"] in keys
def test_aggregate_per_row_reference(tmp_path):
from test_pipeline import _fake
from pulse import aggregate as A
from pulse.provenance import write_results
spec = {"id": "98", "title": "synthetic", "task": "tx", "split": "s", "folds": list(range(5)),
"seeds": [42], "metrics": [{"key": "macro_f1", "label": "F1", "digits": 3,
"better": "higher"}],
"primary_metric": "macro_f1", "reference_row": None,
"rows": [{"key": "s10", "label": "S 10"}, {"key": "p10", "label": "P 10", "reference": "s10"},
{"key": "s50", "label": "S 50"}, {"key": "p50", "label": "P 50", "reference": "s50"}]}
rng = np.random.default_rng(1)
for row, acc in (("s10", 0.3), ("p10", 0.6), ("s50", 0.5), ("p50", 0.5)):
for k in range(5):
write_results(A.run_dir(tmp_path / "runs", "98", row, k, 42),
_fake("tx", k, 42, [f"v{k + 1}s{i}" for i in range(1, 5)], rng, acc=acc))
runs, missing = A.collect(spec, tmp_path / "runs")
summ = A.aggregate(spec, runs, missing=missing)
assert summ["rows"]["p10"]["reference"] == "s10" and summ["rows"]["p50"]["reference"] == "s50"
assert "vs_reference" not in summ["rows"]["s10"]
d = summ["rows"]["p10"]["vs_reference"]["macro_f1"]
own = summ["rows"]
assert d["delta_mean"] == pytest.approx(own["p10"]["common"]["macro_f1"]["mean"]
- own["s10"]["common"]["macro_f1"]["mean"])
files = {p.name for p in A.write_outputs(summ, tmp_path / "tables")}
assert "tab98_paired.tex" in files
assert "P 10 vs S 10" in (tmp_path / "tables" / "tab98_paired.tex").read_text(encoding="utf-8")