"""F5 (S9 pre-training -> T2 fine-tuning): labels, subsets, transfer, cache, end to end.""" import json import numpy as np import pytest torch = pytest.importorskip("torch") from conftest import REPO # noqa: E402 from pulse.data.annotations import load_segments_file # noqa: E402 from pulse.data.availability import build_availability # noqa: E402 from pulse.models.f5 import pretrain as P # noqa: E402 from pulse.models.f5 import s9 as S9 # noqa: E402 from pulse.models.f5 import subset as SUB # noqa: E402 from pulse.paths import DataRoot # noqa: E402 from pulse.provenance import REQUIRED_PROVENANCE, load_results # noqa: E402 from pulse.taxonomy import build_taxonomy # noqa: E402 DAF = {"d_model": 16, "n_layers": 2, "n_heads": 2, "dropout": 0.0, "head_hidden": 8, "max_T": 64, "mod_dropout_p": 0.3, "fusion_type": "concat", "pool_type": "query", "joint_head": True} SIZES = {"verb_fine": 17, "verb_composite": 6, "noun": 5, "hand": 3} def test_s9_decomposition_is_closed(): assert len(S9.S9_DECOMPOSITION) == 16 assert len(S9.S9_VERBS) == 12 and len(S9.S9_NOUNS) == 10 with pytest.raises(S9.S9LabelError): S9.check_disjoint(["v9", "v16"], ["v1", "v16"]) S9.check_disjoint(["v9", "v16", "v17", "v18", "v19", "v20", "v21"], json.loads((REPO / "splits" / "pool27.json").read_text())["volunteers"]) def test_backbone_transfer_by_name(): from pulse.models.t2 import build_model dims = {"emg": 8, "imu": 180} s9 = build_model("dailyactformer_v2", dims, P.S9_SIZES, **DAF) t2 = build_model("dailyactformer_v2", dims, SIZES, **DAF) state = {k: v for k, v in s9.state_dict().items() if not k.startswith("head.")} keys = P.load_backbone(t2, state) assert keys and all(torch.equal(t2.state_dict()[k], state[k]) for k in keys) assert any(k.startswith("stems.emg.") for k in keys) and "pos_embed" in keys with pytest.raises(ValueError, match="keys differ"): P.load_backbone(t2, {k: v for k, v in state.items() if k != "pos_embed"}) with pytest.raises(ValueError, match="shapes"): P.load_backbone(build_model("dailyactformer_v2", {"emg": 8, "imu": 36}, SIZES, **DAF), state) g = P.backbone_param_groups(t2, 1e-3, 0.1) assert [x["lr"] for x in g] == [1e-4, 1e-3] n_head = sum(p.numel() for n, p in t2.named_parameters() if n.startswith("head.")) assert sum(p.numel() for p in g[1]["params"]) == n_head assert sum(p.numel() for x in g for p in x["params"]) == sum(p.numel() for p in t2.parameters()) def test_subsets_nested_and_deterministic(): keys = [(f"v{i}s1", 100 * k) for i in range(1, 9) for k in range(25)] subs = {f: SUB.subset_segments(keys, f, 42, 0) for f in (0.05, 0.1, 0.25, 0.5, 1.0)} assert [len(subs[f]) for f in sorted(subs)] == [10, 20, 50, 100, 200] for a, b in ((0.05, 0.1), (0.1, 0.25), (0.25, 0.5), (0.5, 1.0)): assert set(subs[a]) <= set(subs[b]) assert SUB.subset_segments(keys, 0.25, 42, 0) == subs[0.25] assert SUB.subset_segments(keys, 0.25, 43, 0) != subs[0.25] assert SUB.subset_segments(keys, 0.25, 42, 1) != subs[0.25] idx = SUB.subset_index([keys[0], keys[1], keys[0]], [keys[0]]) assert idx.tolist() == [0, 2] def test_encoder_cache_builds_once(tmp_path): calls = [] def build(): calls.append(1) return {"w": torch.ones(2)}, {"recordings": ["v1s9"]} s1, r1, b1 = P.ensure_encoder(tmp_path, "k" * 32, {"a": 1}, build, poll_sec=0.01) s2, r2, b2 = P.ensure_encoder(tmp_path, "k" * 32, {"a": 1}, build, poll_sec=0.01) assert calls == [1] and b1 and not b2 and r1["encoder_md5"] == r2["encoder_md5"] assert torch.equal(s1["w"], s2["w"]) (tmp_path / ("j" * 32 + ".lock")).mkdir() with pytest.raises(TimeoutError): P.ensure_encoder(tmp_path, "j" * 32, {}, build, timeout_sec=0.05, poll_sec=0.01) # --------------------------------------------------------------------------- # end to end on the synthetic tree # --------------------------------------------------------------------------- def _setup(mini_root, tmp_path): dr = DataRoot.from_arg(mini_root) av = tmp_path / "availability.json" av.write_text(json.dumps(build_availability(dr)), encoding="utf-8") segs = [] for p in sorted((mini_root / "annotations").glob("v*/s*.json")): rid = p.parent.name + p.stem segs += [(rid, s.verb, s.noun, s.hand) for s in load_segments_file(p, rid)] taxp = tmp_path / "taxonomy.json" taxp.write_text(json.dumps(build_taxonomy(segs, threshold=1)), encoding="utf-8") split = tmp_path / "split.json" # pool without v1, whose S9 is the pre-training data split.write_text(json.dumps({"schema": "pulse-split/1", "name": "mini4", "volunteers": ["v2", "v3", "v5", "v6"], "folds": [{"fold": 0, "test": ["v6"], "inner_val": ["v2"], "train": ["v3", "v5"]}, {"fold": 1, "test": ["v2", "v3", "v5"], "inner_val": ["v6"], "train": []}]}), encoding="utf-8") s9dir = tmp_path / "s9labels" (s9dir / "v1").mkdir(parents=True) names = sorted(S9.S9_DECOMPOSITION) (s9dir / "v1" / "s9.json").write_text(json.dumps({"segments": [ {"timestamp": f"00:00:0{i % 2}-00:00:0{i % 2 + 1}", "action_annotation": {"action_name": names[i % 16], "object_name": "cup", "hand_type": "right"}} for i in range(12)]}), encoding="utf-8") return av, taxp, split, s9dir def _run(mini_root, av, taxp, split, s9dir, out, cfg, extra_sets=()): from pulse.tasks import f5 sets = ["train.epochs=2", "train.batch_size=2", "pretrain.train.epochs=2", "pretrain.train.batch_size=4", "pretrain.min_frames=4", f"pretrain.cache_dir={(out.parent / 'cache').as_posix()}", "pretrain.volunteers=[v1]", "model.d_model=16", "model.n_heads=2", "model.head_hidden=8", "load_workers=1", "sliding.window_sec=1.0", "sliding.stride_sec=0.5", *extra_sets] argv = ["--root", str(mini_root), "--split", str(split), "--fold", "0", "--seed", "42", "--config", str(REPO / cfg), "--availability", str(av), "--taxonomy", str(taxp), "--s9-annotations", str(s9dir), "--out", str(out), "--device", "cpu"] for s in sets: argv += ["--set", s] assert f5.main(argv) == 0 return load_results(out / "results.json") def test_f5_end_to_end(mini_root, tmp_path): av, taxp, split, s9dir = _setup(mini_root, tmp_path) r1 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "a", "configs/f5/sliding/s9pre_f050.yaml") prov = r1["provenance"] assert set(REQUIRED_PROVENANCE) <= set(prov) assert prov["loaded_recordings"] == {"train": ["v3s1", "v3s2"], "inner_val": ["v2s1", "v2s2"], "test": ["v6s1", "v6s2"]} # v5: 72-column IMU assert prov["inner_val_volunteers"] == ["v2"] and "s9_pretrain" in prov["manifests"] assert prov["taxonomy_md5"] and prov["config"]["init"] == "s9" ex = r1["extra"] assert ex["s9_pretrain"]["built_by_this_run"] and ex["fraction"] == 0.5 lb = ex["label_budget"] assert lb["segments_total"] == 6 and lb["segments_used"] == 3 # 0-1, 1-2, 2-3 s overlap the span assert set(r1["per_recording"]["test"]) == {"v6s1", "v6s2"} assert r1["selection"]["criterion"].startswith("inner_val/verb_composite@1") assert "verb_composite@1" in r1["metrics"]["test"] assert str(s9dir) not in json.dumps(r1) and str(mini_root) not in json.dumps(r1) # same seed: the encoder is reused; the scratch run never touches S9 r2 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "b", "configs/f5/sliding/s9pre_f100.yaml") assert not r2["extra"]["s9_pretrain"]["built_by_this_run"] assert r2["extra"]["s9_pretrain"]["encoder_md5"] == ex["s9_pretrain"]["encoder_md5"] assert r2["extra"]["label_budget"]["samples_used"] == r2["extra"]["label_budget"]["samples_total"] r3 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "c", "configs/f5/segment/scratch_f100.yaml") assert r3["extra"]["s9_pretrain"] is None and r3["extra"]["protocol"] == "segment" assert r3["extra"]["data"]["test"]["n_samples"] == 4 # 2 segments in the span x 2 def test_f5_missing_s9_labels_raise(mini_root, tmp_path): from pulse.errors import MissingFileError av, taxp, split, s9dir = _setup(mini_root, tmp_path) (s9dir / "v1" / "s9.json").unlink() with pytest.raises(MissingFileError): _run(mini_root, av, taxp, split, s9dir, tmp_path / "a", "configs/f5/sliding/s9pre_f050.yaml") def test_f5_registry_rows(): from pulse.aggregate import load_registry, table_spec from pulse.config import load_config reg = load_registry() for t, proto in (("5", "sliding"), ("5seg", "segment")): spec = table_spec(reg, t) keys = {r["key"] for r in spec["rows"]} assert len(keys) == 10 and spec["status"] == "ready" and spec["task"] == "f5" # the paper's sweep (results.tex, Table 5): 5 / 10 / 25 / 50 / 100 % assert sorted({int(k[-3:]) for k in keys}) == [5, 10, 25, 50, 100] for r in spec["rows"]: cfg = load_config(REPO / r["config"]) assert cfg["task"] == "f5" and cfg["protocol"] == proto assert cfg["modalities"] == ["emg", "imu"] and cfg["model"]["name"] == "dailyactformer_v2" assert cfg["init"] == ("s9" if r["key"].startswith("s9pre") else "scratch") assert round(100 * cfg["fraction"]) == int(r["key"][-3:]) if r["key"].startswith("s9pre"): assert r["reference"] == r["key"].replace("s9pre", "scratch") and r["reference"] in keys def test_aggregate_per_row_reference(tmp_path): from test_pipeline import _fake from pulse import aggregate as A from pulse.provenance import write_results spec = {"id": "98", "title": "synthetic", "task": "tx", "split": "s", "folds": list(range(5)), "seeds": [42], "metrics": [{"key": "macro_f1", "label": "F1", "digits": 3, "better": "higher"}], "primary_metric": "macro_f1", "reference_row": None, "rows": [{"key": "s10", "label": "S 10"}, {"key": "p10", "label": "P 10", "reference": "s10"}, {"key": "s50", "label": "S 50"}, {"key": "p50", "label": "P 50", "reference": "s50"}]} rng = np.random.default_rng(1) for row, acc in (("s10", 0.3), ("p10", 0.6), ("s50", 0.5), ("p50", 0.5)): for k in range(5): write_results(A.run_dir(tmp_path / "runs", "98", row, k, 42), _fake("tx", k, 42, [f"v{k + 1}s{i}" for i in range(1, 5)], rng, acc=acc)) runs, missing = A.collect(spec, tmp_path / "runs") summ = A.aggregate(spec, runs, missing=missing) assert summ["rows"]["p10"]["reference"] == "s10" and summ["rows"]["p50"]["reference"] == "s50" assert "vs_reference" not in summ["rows"]["s10"] d = summ["rows"]["p10"]["vs_reference"]["macro_f1"] own = summ["rows"] assert d["delta_mean"] == pytest.approx(own["p10"]["common"]["macro_f1"]["mean"] - own["s10"]["common"]["macro_f1"]["mean"]) files = {p.name for p in A.write_outputs(summ, tmp_path / "tables")} assert "tab98_paired.tex" in files assert "P 10 vs S 10" in (tmp_path / "tables" / "tab98_paired.tex").read_text(encoding="utf-8")