File size: 11,590 Bytes
dffbc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 | """F5 (S9 pre-training -> T2 fine-tuning): labels, subsets, transfer, cache, end to end."""
import json
import numpy as np
import pytest
torch = pytest.importorskip("torch")
from conftest import REPO # noqa: E402
from pulse.data.annotations import load_segments_file # noqa: E402
from pulse.data.availability import build_availability # noqa: E402
from pulse.models.f5 import pretrain as P # noqa: E402
from pulse.models.f5 import s9 as S9 # noqa: E402
from pulse.models.f5 import subset as SUB # noqa: E402
from pulse.paths import DataRoot # noqa: E402
from pulse.provenance import REQUIRED_PROVENANCE, load_results # noqa: E402
from pulse.taxonomy import build_taxonomy # noqa: E402
DAF = {"d_model": 16, "n_layers": 2, "n_heads": 2, "dropout": 0.0, "head_hidden": 8, "max_T": 64,
"mod_dropout_p": 0.3, "fusion_type": "concat", "pool_type": "query", "joint_head": True}
SIZES = {"verb_fine": 17, "verb_composite": 6, "noun": 5, "hand": 3}
def test_s9_decomposition_is_closed():
assert len(S9.S9_DECOMPOSITION) == 16
assert len(S9.S9_VERBS) == 12 and len(S9.S9_NOUNS) == 10
with pytest.raises(S9.S9LabelError):
S9.check_disjoint(["v9", "v16"], ["v1", "v16"])
S9.check_disjoint(["v9", "v16", "v17", "v18", "v19", "v20", "v21"],
json.loads((REPO / "splits" / "pool27.json").read_text())["volunteers"])
def test_backbone_transfer_by_name():
from pulse.models.t2 import build_model
dims = {"emg": 8, "imu": 180}
s9 = build_model("dailyactformer_v2", dims, P.S9_SIZES, **DAF)
t2 = build_model("dailyactformer_v2", dims, SIZES, **DAF)
state = {k: v for k, v in s9.state_dict().items() if not k.startswith("head.")}
keys = P.load_backbone(t2, state)
assert keys and all(torch.equal(t2.state_dict()[k], state[k]) for k in keys)
assert any(k.startswith("stems.emg.") for k in keys) and "pos_embed" in keys
with pytest.raises(ValueError, match="keys differ"):
P.load_backbone(t2, {k: v for k, v in state.items() if k != "pos_embed"})
with pytest.raises(ValueError, match="shapes"):
P.load_backbone(build_model("dailyactformer_v2", {"emg": 8, "imu": 36}, SIZES, **DAF), state)
g = P.backbone_param_groups(t2, 1e-3, 0.1)
assert [x["lr"] for x in g] == [1e-4, 1e-3]
n_head = sum(p.numel() for n, p in t2.named_parameters() if n.startswith("head."))
assert sum(p.numel() for p in g[1]["params"]) == n_head
assert sum(p.numel() for x in g for p in x["params"]) == sum(p.numel() for p in t2.parameters())
def test_subsets_nested_and_deterministic():
keys = [(f"v{i}s1", 100 * k) for i in range(1, 9) for k in range(25)]
subs = {f: SUB.subset_segments(keys, f, 42, 0) for f in (0.05, 0.1, 0.25, 0.5, 1.0)}
assert [len(subs[f]) for f in sorted(subs)] == [10, 20, 50, 100, 200]
for a, b in ((0.05, 0.1), (0.1, 0.25), (0.25, 0.5), (0.5, 1.0)):
assert set(subs[a]) <= set(subs[b])
assert SUB.subset_segments(keys, 0.25, 42, 0) == subs[0.25]
assert SUB.subset_segments(keys, 0.25, 43, 0) != subs[0.25]
assert SUB.subset_segments(keys, 0.25, 42, 1) != subs[0.25]
idx = SUB.subset_index([keys[0], keys[1], keys[0]], [keys[0]])
assert idx.tolist() == [0, 2]
def test_encoder_cache_builds_once(tmp_path):
calls = []
def build():
calls.append(1)
return {"w": torch.ones(2)}, {"recordings": ["v1s9"]}
s1, r1, b1 = P.ensure_encoder(tmp_path, "k" * 32, {"a": 1}, build, poll_sec=0.01)
s2, r2, b2 = P.ensure_encoder(tmp_path, "k" * 32, {"a": 1}, build, poll_sec=0.01)
assert calls == [1] and b1 and not b2 and r1["encoder_md5"] == r2["encoder_md5"]
assert torch.equal(s1["w"], s2["w"])
(tmp_path / ("j" * 32 + ".lock")).mkdir()
with pytest.raises(TimeoutError):
P.ensure_encoder(tmp_path, "j" * 32, {}, build, timeout_sec=0.05, poll_sec=0.01)
# ---------------------------------------------------------------------------
# end to end on the synthetic tree
# ---------------------------------------------------------------------------
def _setup(mini_root, tmp_path):
dr = DataRoot.from_arg(mini_root)
av = tmp_path / "availability.json"
av.write_text(json.dumps(build_availability(dr)), encoding="utf-8")
segs = []
for p in sorted((mini_root / "annotations").glob("v*/s*.json")):
rid = p.parent.name + p.stem
segs += [(rid, s.verb, s.noun, s.hand) for s in load_segments_file(p, rid)]
taxp = tmp_path / "taxonomy.json"
taxp.write_text(json.dumps(build_taxonomy(segs, threshold=1)), encoding="utf-8")
split = tmp_path / "split.json" # pool without v1, whose S9 is the pre-training data
split.write_text(json.dumps({"schema": "pulse-split/1", "name": "mini4",
"volunteers": ["v2", "v3", "v5", "v6"],
"folds": [{"fold": 0, "test": ["v6"], "inner_val": ["v2"],
"train": ["v3", "v5"]},
{"fold": 1, "test": ["v2", "v3", "v5"],
"inner_val": ["v6"], "train": []}]}),
encoding="utf-8")
s9dir = tmp_path / "s9labels"
(s9dir / "v1").mkdir(parents=True)
names = sorted(S9.S9_DECOMPOSITION)
(s9dir / "v1" / "s9.json").write_text(json.dumps({"segments": [
{"timestamp": f"00:00:0{i % 2}-00:00:0{i % 2 + 1}",
"action_annotation": {"action_name": names[i % 16], "object_name": "cup",
"hand_type": "right"}} for i in range(12)]}), encoding="utf-8")
return av, taxp, split, s9dir
def _run(mini_root, av, taxp, split, s9dir, out, cfg, extra_sets=()):
from pulse.tasks import f5
sets = ["train.epochs=2", "train.batch_size=2", "pretrain.train.epochs=2",
"pretrain.train.batch_size=4", "pretrain.min_frames=4",
f"pretrain.cache_dir={(out.parent / 'cache').as_posix()}", "pretrain.volunteers=[v1]",
"model.d_model=16", "model.n_heads=2", "model.head_hidden=8", "load_workers=1",
"sliding.window_sec=1.0", "sliding.stride_sec=0.5", *extra_sets]
argv = ["--root", str(mini_root), "--split", str(split), "--fold", "0", "--seed", "42",
"--config", str(REPO / cfg), "--availability", str(av), "--taxonomy", str(taxp),
"--s9-annotations", str(s9dir), "--out", str(out), "--device", "cpu"]
for s in sets:
argv += ["--set", s]
assert f5.main(argv) == 0
return load_results(out / "results.json")
def test_f5_end_to_end(mini_root, tmp_path):
av, taxp, split, s9dir = _setup(mini_root, tmp_path)
r1 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "a", "configs/f5/sliding/s9pre_f050.yaml")
prov = r1["provenance"]
assert set(REQUIRED_PROVENANCE) <= set(prov)
assert prov["loaded_recordings"] == {"train": ["v3s1", "v3s2"], "inner_val": ["v2s1", "v2s2"],
"test": ["v6s1", "v6s2"]} # v5: 72-column IMU
assert prov["inner_val_volunteers"] == ["v2"] and "s9_pretrain" in prov["manifests"]
assert prov["taxonomy_md5"] and prov["config"]["init"] == "s9"
ex = r1["extra"]
assert ex["s9_pretrain"]["built_by_this_run"] and ex["fraction"] == 0.5
lb = ex["label_budget"]
assert lb["segments_total"] == 6 and lb["segments_used"] == 3 # 0-1, 1-2, 2-3 s overlap the span
assert set(r1["per_recording"]["test"]) == {"v6s1", "v6s2"}
assert r1["selection"]["criterion"].startswith("inner_val/verb_composite@1")
assert "verb_composite@1" in r1["metrics"]["test"]
assert str(s9dir) not in json.dumps(r1) and str(mini_root) not in json.dumps(r1)
# same seed: the encoder is reused; the scratch run never touches S9
r2 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "b", "configs/f5/sliding/s9pre_f100.yaml")
assert not r2["extra"]["s9_pretrain"]["built_by_this_run"]
assert r2["extra"]["s9_pretrain"]["encoder_md5"] == ex["s9_pretrain"]["encoder_md5"]
assert r2["extra"]["label_budget"]["samples_used"] == r2["extra"]["label_budget"]["samples_total"]
r3 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "c", "configs/f5/segment/scratch_f100.yaml")
assert r3["extra"]["s9_pretrain"] is None and r3["extra"]["protocol"] == "segment"
assert r3["extra"]["data"]["test"]["n_samples"] == 4 # 2 segments in the span x 2
def test_f5_missing_s9_labels_raise(mini_root, tmp_path):
from pulse.errors import MissingFileError
av, taxp, split, s9dir = _setup(mini_root, tmp_path)
(s9dir / "v1" / "s9.json").unlink()
with pytest.raises(MissingFileError):
_run(mini_root, av, taxp, split, s9dir, tmp_path / "a", "configs/f5/sliding/s9pre_f050.yaml")
def test_f5_registry_rows():
from pulse.aggregate import load_registry, table_spec
from pulse.config import load_config
reg = load_registry()
for t, proto in (("5", "sliding"), ("5seg", "segment")):
spec = table_spec(reg, t)
keys = {r["key"] for r in spec["rows"]}
assert len(keys) == 10 and spec["status"] == "ready" and spec["task"] == "f5"
# the paper's sweep (results.tex, Table 5): 5 / 10 / 25 / 50 / 100 %
assert sorted({int(k[-3:]) for k in keys}) == [5, 10, 25, 50, 100]
for r in spec["rows"]:
cfg = load_config(REPO / r["config"])
assert cfg["task"] == "f5" and cfg["protocol"] == proto
assert cfg["modalities"] == ["emg", "imu"] and cfg["model"]["name"] == "dailyactformer_v2"
assert cfg["init"] == ("s9" if r["key"].startswith("s9pre") else "scratch")
assert round(100 * cfg["fraction"]) == int(r["key"][-3:])
if r["key"].startswith("s9pre"):
assert r["reference"] == r["key"].replace("s9pre", "scratch") and r["reference"] in keys
def test_aggregate_per_row_reference(tmp_path):
from test_pipeline import _fake
from pulse import aggregate as A
from pulse.provenance import write_results
spec = {"id": "98", "title": "synthetic", "task": "tx", "split": "s", "folds": list(range(5)),
"seeds": [42], "metrics": [{"key": "macro_f1", "label": "F1", "digits": 3,
"better": "higher"}],
"primary_metric": "macro_f1", "reference_row": None,
"rows": [{"key": "s10", "label": "S 10"}, {"key": "p10", "label": "P 10", "reference": "s10"},
{"key": "s50", "label": "S 50"}, {"key": "p50", "label": "P 50", "reference": "s50"}]}
rng = np.random.default_rng(1)
for row, acc in (("s10", 0.3), ("p10", 0.6), ("s50", 0.5), ("p50", 0.5)):
for k in range(5):
write_results(A.run_dir(tmp_path / "runs", "98", row, k, 42),
_fake("tx", k, 42, [f"v{k + 1}s{i}" for i in range(1, 5)], rng, acc=acc))
runs, missing = A.collect(spec, tmp_path / "runs")
summ = A.aggregate(spec, runs, missing=missing)
assert summ["rows"]["p10"]["reference"] == "s10" and summ["rows"]["p50"]["reference"] == "s50"
assert "vs_reference" not in summ["rows"]["s10"]
d = summ["rows"]["p10"]["vs_reference"]["macro_f1"]
own = summ["rows"]
assert d["delta_mean"] == pytest.approx(own["p10"]["common"]["macro_f1"]["mean"]
- own["s10"]["common"]["macro_f1"]["mean"])
files = {p.name for p in A.write_outputs(summ, tmp_path / "tables")}
assert "tab98_paired.tex" in files
assert "P 10 vs S 10" in (tmp_path / "tables" / "tab98_paired.tex").read_text(encoding="utf-8")
|