Download tests/test_f5.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/tests/test_f5.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/tests/test_f5.py
-
curl -L -o test_f5.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/tests/test_f5.py
11.6 kB
| """F5 (S9 pre-training -> T2 fine-tuning): labels, subsets, transfer, cache, end to end.""" | |
| import json | |
| import numpy as np | |
| import pytest | |
| torch = pytest.importorskip("torch") | |
| from conftest import REPO # noqa: E402 | |
| from pulse.data.annotations import load_segments_file # noqa: E402 | |
| from pulse.data.availability import build_availability # noqa: E402 | |
| from pulse.models.f5 import pretrain as P # noqa: E402 | |
| from pulse.models.f5 import s9 as S9 # noqa: E402 | |
| from pulse.models.f5 import subset as SUB # noqa: E402 | |
| from pulse.paths import DataRoot # noqa: E402 | |
| from pulse.provenance import REQUIRED_PROVENANCE, load_results # noqa: E402 | |
| from pulse.taxonomy import build_taxonomy # noqa: E402 | |
| DAF = {"d_model": 16, "n_layers": 2, "n_heads": 2, "dropout": 0.0, "head_hidden": 8, "max_T": 64, | |
| "mod_dropout_p": 0.3, "fusion_type": "concat", "pool_type": "query", "joint_head": True} | |
| SIZES = {"verb_fine": 17, "verb_composite": 6, "noun": 5, "hand": 3} | |
| def test_s9_decomposition_is_closed(): | |
| assert len(S9.S9_DECOMPOSITION) == 16 | |
| assert len(S9.S9_VERBS) == 12 and len(S9.S9_NOUNS) == 10 | |
| with pytest.raises(S9.S9LabelError): | |
| S9.check_disjoint(["v9", "v16"], ["v1", "v16"]) | |
| S9.check_disjoint(["v9", "v16", "v17", "v18", "v19", "v20", "v21"], | |
| json.loads((REPO / "splits" / "pool27.json").read_text())["volunteers"]) | |
| def test_backbone_transfer_by_name(): | |
| from pulse.models.t2 import build_model | |
| dims = {"emg": 8, "imu": 180} | |
| s9 = build_model("dailyactformer_v2", dims, P.S9_SIZES, **DAF) | |
| t2 = build_model("dailyactformer_v2", dims, SIZES, **DAF) | |
| state = {k: v for k, v in s9.state_dict().items() if not k.startswith("head.")} | |
| keys = P.load_backbone(t2, state) | |
| assert keys and all(torch.equal(t2.state_dict()[k], state[k]) for k in keys) | |
| assert any(k.startswith("stems.emg.") for k in keys) and "pos_embed" in keys | |
| with pytest.raises(ValueError, match="keys differ"): | |
| P.load_backbone(t2, {k: v for k, v in state.items() if k != "pos_embed"}) | |
| with pytest.raises(ValueError, match="shapes"): | |
| P.load_backbone(build_model("dailyactformer_v2", {"emg": 8, "imu": 36}, SIZES, **DAF), state) | |
| g = P.backbone_param_groups(t2, 1e-3, 0.1) | |
| assert [x["lr"] for x in g] == [1e-4, 1e-3] | |
| n_head = sum(p.numel() for n, p in t2.named_parameters() if n.startswith("head.")) | |
| assert sum(p.numel() for p in g[1]["params"]) == n_head | |
| assert sum(p.numel() for x in g for p in x["params"]) == sum(p.numel() for p in t2.parameters()) | |
| def test_subsets_nested_and_deterministic(): | |
| keys = [(f"v{i}s1", 100 * k) for i in range(1, 9) for k in range(25)] | |
| subs = {f: SUB.subset_segments(keys, f, 42, 0) for f in (0.05, 0.1, 0.25, 0.5, 1.0)} | |
| assert [len(subs[f]) for f in sorted(subs)] == [10, 20, 50, 100, 200] | |
| for a, b in ((0.05, 0.1), (0.1, 0.25), (0.25, 0.5), (0.5, 1.0)): | |
| assert set(subs[a]) <= set(subs[b]) | |
| assert SUB.subset_segments(keys, 0.25, 42, 0) == subs[0.25] | |
| assert SUB.subset_segments(keys, 0.25, 43, 0) != subs[0.25] | |
| assert SUB.subset_segments(keys, 0.25, 42, 1) != subs[0.25] | |
| idx = SUB.subset_index([keys[0], keys[1], keys[0]], [keys[0]]) | |
| assert idx.tolist() == [0, 2] | |
| def test_encoder_cache_builds_once(tmp_path): | |
| calls = [] | |
| def build(): | |
| calls.append(1) | |
| return {"w": torch.ones(2)}, {"recordings": ["v1s9"]} | |
| s1, r1, b1 = P.ensure_encoder(tmp_path, "k" * 32, {"a": 1}, build, poll_sec=0.01) | |
| s2, r2, b2 = P.ensure_encoder(tmp_path, "k" * 32, {"a": 1}, build, poll_sec=0.01) | |
| assert calls == [1] and b1 and not b2 and r1["encoder_md5"] == r2["encoder_md5"] | |
| assert torch.equal(s1["w"], s2["w"]) | |
| (tmp_path / ("j" * 32 + ".lock")).mkdir() | |
| with pytest.raises(TimeoutError): | |
| P.ensure_encoder(tmp_path, "j" * 32, {}, build, timeout_sec=0.05, poll_sec=0.01) | |
| # --------------------------------------------------------------------------- | |
| # end to end on the synthetic tree | |
| # --------------------------------------------------------------------------- | |
| def _setup(mini_root, tmp_path): | |
| dr = DataRoot.from_arg(mini_root) | |
| av = tmp_path / "availability.json" | |
| av.write_text(json.dumps(build_availability(dr)), encoding="utf-8") | |
| segs = [] | |
| for p in sorted((mini_root / "annotations").glob("v*/s*.json")): | |
| rid = p.parent.name + p.stem | |
| segs += [(rid, s.verb, s.noun, s.hand) for s in load_segments_file(p, rid)] | |
| taxp = tmp_path / "taxonomy.json" | |
| taxp.write_text(json.dumps(build_taxonomy(segs, threshold=1)), encoding="utf-8") | |
| split = tmp_path / "split.json" # pool without v1, whose S9 is the pre-training data | |
| split.write_text(json.dumps({"schema": "pulse-split/1", "name": "mini4", | |
| "volunteers": ["v2", "v3", "v5", "v6"], | |
| "folds": [{"fold": 0, "test": ["v6"], "inner_val": ["v2"], | |
| "train": ["v3", "v5"]}, | |
| {"fold": 1, "test": ["v2", "v3", "v5"], | |
| "inner_val": ["v6"], "train": []}]}), | |
| encoding="utf-8") | |
| s9dir = tmp_path / "s9labels" | |
| (s9dir / "v1").mkdir(parents=True) | |
| names = sorted(S9.S9_DECOMPOSITION) | |
| (s9dir / "v1" / "s9.json").write_text(json.dumps({"segments": [ | |
| {"timestamp": f"00:00:0{i % 2}-00:00:0{i % 2 + 1}", | |
| "action_annotation": {"action_name": names[i % 16], "object_name": "cup", | |
| "hand_type": "right"}} for i in range(12)]}), encoding="utf-8") | |
| return av, taxp, split, s9dir | |
| def _run(mini_root, av, taxp, split, s9dir, out, cfg, extra_sets=()): | |
| from pulse.tasks import f5 | |
| sets = ["train.epochs=2", "train.batch_size=2", "pretrain.train.epochs=2", | |
| "pretrain.train.batch_size=4", "pretrain.min_frames=4", | |
| f"pretrain.cache_dir={(out.parent / 'cache').as_posix()}", "pretrain.volunteers=[v1]", | |
| "model.d_model=16", "model.n_heads=2", "model.head_hidden=8", "load_workers=1", | |
| "sliding.window_sec=1.0", "sliding.stride_sec=0.5", *extra_sets] | |
| argv = ["--root", str(mini_root), "--split", str(split), "--fold", "0", "--seed", "42", | |
| "--config", str(REPO / cfg), "--availability", str(av), "--taxonomy", str(taxp), | |
| "--s9-annotations", str(s9dir), "--out", str(out), "--device", "cpu"] | |
| for s in sets: | |
| argv += ["--set", s] | |
| assert f5.main(argv) == 0 | |
| return load_results(out / "results.json") | |
| def test_f5_end_to_end(mini_root, tmp_path): | |
| av, taxp, split, s9dir = _setup(mini_root, tmp_path) | |
| r1 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "a", "configs/f5/sliding/s9pre_f050.yaml") | |
| prov = r1["provenance"] | |
| assert set(REQUIRED_PROVENANCE) <= set(prov) | |
| assert prov["loaded_recordings"] == {"train": ["v3s1", "v3s2"], "inner_val": ["v2s1", "v2s2"], | |
| "test": ["v6s1", "v6s2"]} # v5: 72-column IMU | |
| assert prov["inner_val_volunteers"] == ["v2"] and "s9_pretrain" in prov["manifests"] | |
| assert prov["taxonomy_md5"] and prov["config"]["init"] == "s9" | |
| ex = r1["extra"] | |
| assert ex["s9_pretrain"]["built_by_this_run"] and ex["fraction"] == 0.5 | |
| lb = ex["label_budget"] | |
| assert lb["segments_total"] == 6 and lb["segments_used"] == 3 # 0-1, 1-2, 2-3 s overlap the span | |
| assert set(r1["per_recording"]["test"]) == {"v6s1", "v6s2"} | |
| assert r1["selection"]["criterion"].startswith("inner_val/verb_composite@1") | |
| assert "verb_composite@1" in r1["metrics"]["test"] | |
| assert str(s9dir) not in json.dumps(r1) and str(mini_root) not in json.dumps(r1) | |
| # same seed: the encoder is reused; the scratch run never touches S9 | |
| r2 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "b", "configs/f5/sliding/s9pre_f100.yaml") | |
| assert not r2["extra"]["s9_pretrain"]["built_by_this_run"] | |
| assert r2["extra"]["s9_pretrain"]["encoder_md5"] == ex["s9_pretrain"]["encoder_md5"] | |
| assert r2["extra"]["label_budget"]["samples_used"] == r2["extra"]["label_budget"]["samples_total"] | |
| r3 = _run(mini_root, av, taxp, split, s9dir, tmp_path / "c", "configs/f5/segment/scratch_f100.yaml") | |
| assert r3["extra"]["s9_pretrain"] is None and r3["extra"]["protocol"] == "segment" | |
| assert r3["extra"]["data"]["test"]["n_samples"] == 4 # 2 segments in the span x 2 | |
| def test_f5_missing_s9_labels_raise(mini_root, tmp_path): | |
| from pulse.errors import MissingFileError | |
| av, taxp, split, s9dir = _setup(mini_root, tmp_path) | |
| (s9dir / "v1" / "s9.json").unlink() | |
| with pytest.raises(MissingFileError): | |
| _run(mini_root, av, taxp, split, s9dir, tmp_path / "a", "configs/f5/sliding/s9pre_f050.yaml") | |
| def test_f5_registry_rows(): | |
| from pulse.aggregate import load_registry, table_spec | |
| from pulse.config import load_config | |
| reg = load_registry() | |
| for t, proto in (("5", "sliding"), ("5seg", "segment")): | |
| spec = table_spec(reg, t) | |
| keys = {r["key"] for r in spec["rows"]} | |
| assert len(keys) == 10 and spec["status"] == "ready" and spec["task"] == "f5" | |
| # the paper's sweep (results.tex, Table 5): 5 / 10 / 25 / 50 / 100 % | |
| assert sorted({int(k[-3:]) for k in keys}) == [5, 10, 25, 50, 100] | |
| for r in spec["rows"]: | |
| cfg = load_config(REPO / r["config"]) | |
| assert cfg["task"] == "f5" and cfg["protocol"] == proto | |
| assert cfg["modalities"] == ["emg", "imu"] and cfg["model"]["name"] == "dailyactformer_v2" | |
| assert cfg["init"] == ("s9" if r["key"].startswith("s9pre") else "scratch") | |
| assert round(100 * cfg["fraction"]) == int(r["key"][-3:]) | |
| if r["key"].startswith("s9pre"): | |
| assert r["reference"] == r["key"].replace("s9pre", "scratch") and r["reference"] in keys | |
| def test_aggregate_per_row_reference(tmp_path): | |
| from test_pipeline import _fake | |
| from pulse import aggregate as A | |
| from pulse.provenance import write_results | |
| spec = {"id": "98", "title": "synthetic", "task": "tx", "split": "s", "folds": list(range(5)), | |
| "seeds": [42], "metrics": [{"key": "macro_f1", "label": "F1", "digits": 3, | |
| "better": "higher"}], | |
| "primary_metric": "macro_f1", "reference_row": None, | |
| "rows": [{"key": "s10", "label": "S 10"}, {"key": "p10", "label": "P 10", "reference": "s10"}, | |
| {"key": "s50", "label": "S 50"}, {"key": "p50", "label": "P 50", "reference": "s50"}]} | |
| rng = np.random.default_rng(1) | |
| for row, acc in (("s10", 0.3), ("p10", 0.6), ("s50", 0.5), ("p50", 0.5)): | |
| for k in range(5): | |
| write_results(A.run_dir(tmp_path / "runs", "98", row, k, 42), | |
| _fake("tx", k, 42, [f"v{k + 1}s{i}" for i in range(1, 5)], rng, acc=acc)) | |
| runs, missing = A.collect(spec, tmp_path / "runs") | |
| summ = A.aggregate(spec, runs, missing=missing) | |
| assert summ["rows"]["p10"]["reference"] == "s10" and summ["rows"]["p50"]["reference"] == "s50" | |
| assert "vs_reference" not in summ["rows"]["s10"] | |
| d = summ["rows"]["p10"]["vs_reference"]["macro_f1"] | |
| own = summ["rows"] | |
| assert d["delta_mean"] == pytest.approx(own["p10"]["common"]["macro_f1"]["mean"] | |
| - own["s10"]["common"]["macro_f1"]["mean"]) | |
| files = {p.name for p in A.write_outputs(summ, tmp_path / "tables")} | |
| assert "tab98_paired.tex" in files | |
| assert "P 10 vs S 10" in (tmp_path / "tables" / "tab98_paired.tex").read_text(encoding="utf-8") | |