PULSE-code / tests /test_splits_taxonomy.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
4.95 kB
import json
from collections import Counter
import pytest
from conftest import REPO
from pulse.split_builder import allocate, choose_inner_val, round_half_up
from pulse.splits import SplitError, load_split, validate_split
from pulse.taxonomy import (FINE_TO_COMPOSITE, VERB_FINE, Taxonomy, TaxonomyError,
build_taxonomy)
# Test folds of the source files (results/t1_cv/cv_folds.json md5 8f20127a...,
# results/t2_cv/cv_folds.json md5 3b93df5d...), copied here to pin them.
SOURCE_FOLDS = {
"pool33": [["v1", "v12", "v19", "v24", "v29", "v34", "v39"],
["v2", "v13", "v20", "v25", "v30", "v35", "v40"],
["v3", "v15", "v21", "v26", "v31", "v36", "v41"],
["v9", "v16", "v22", "v27", "v32", "v37"],
["v11", "v17", "v23", "v28", "v33", "v38"]],
"pool27": [["v25", "v28", "v11", "v38", "v5", "v35"],
["v3", "v32", "v2", "v13", "v36", "v23"],
["v26", "v33", "v22", "v41", "v24"],
["v37", "v29", "v12", "v39", "v27"],
["v1", "v40", "v31", "v34", "v30"]],
}
SOURCE_MD5 = {"pool33": "8f20127a120ad516bedc52c983e82eb2",
"pool27": "3b93df5dcefc2f74d444cda3f5bdd997"}
N_VAL = {"pool33": 4, "pool27": 3}
NO_PRESSURE_33 = {"v9", "v15", "v16", "v17", "v19", "v20", "v21"}
@pytest.mark.parametrize("name", ["pool33", "pool27"])
def test_committed_split(name):
sp = load_split(REPO / "splits" / f"{name}.json")
assert sp.name == name and sp.n_folds == 5
assert [list(sp.fold(k).test) for k in range(5)] == SOURCE_FOLDS[name]
assert sp.data["source"]["md5"] == SOURCE_MD5[name]
used = Counter()
for k in range(5):
f = sp.fold(k)
assert len(f.inner_val) == N_VAL[name]
assert set(f.inner_val).isdisjoint(f.test) and set(f.inner_val).isdisjoint(f.train)
for v in f.inner_val:
assert sp.data["volunteer_profiles"][v]["eligible"]
used.update(f.inner_val)
if name == "pool33":
assert len(set(f.inner_val) & NO_PRESSURE_33) == 1
assert max(used.values()) == 1 # every volunteer is inner-val at most once
# regenerate from the recorded profiles: identical
folds = choose_inner_val(name, sp.volunteers, SOURCE_FOLDS[name],
sp.data["volunteer_profiles"], sp.data["inner_val_rule"]["fraction"])
assert [f["inner_val"] for f in folds] == [list(sp.fold(k).inner_val) for k in range(5)]
text = (REPO / "splits" / f"{name}.json").read_text(encoding="utf-8")
assert '"data_root"' not in text and '"/' not in text # no machine paths
def test_split_validation_rejects_overlap():
d = json.loads((REPO / "splits" / "pool33.json").read_text(encoding="utf-8"))
d["folds"][0]["train"].append(d["folds"][0]["test"][0])
with pytest.raises(SplitError):
validate_split(d)
def test_rounding_and_allocation():
assert round_half_up(3.9) == 4 and round_half_up(4.05) == 4 and round_half_up(3.3) == 3
assert allocate(4, {"none": 6, "pressure": 20}) == {"none": 1, "pressure": 3}
assert sum(allocate(3, {"a": 1, "b": 1, "c": 1, "d": 1}).values()) == 3
def _segs():
s = []
for i in range(60):
s.append((f"v{i % 3 + 1}s1", "grasp", "cup", "right"))
for i in range(55):
s.append((f"v{i % 3 + 1}s2", "move", "mouse", "both"))
s.append(("v1s3", "support", "cup", "left"))
s.append(("v1s3", "place", "rare thing", "left"))
return s
def test_build_taxonomy_rules():
tax = build_taxonomy(_segs(), threshold=50, pool_volunteers={"p": ["v1", "v2"]})
assert tax["verbs"] == list(VERB_FINE) and len(VERB_FINE) == 17
assert tax["nouns"] == ["cup", "wired mouse"] # cup kept as written; alias applied
assert tax["n_segments_dropped_verb"] == 1 and tax["counts"]["dropped_verbs"] == {"support": 1}
assert tax["n_segments_dropped_noun"] == 1 and tax["n_segments_kept"] == 115
assert tax["chance"]["uniform"]["act_vn@5"] == pytest.approx(min(5, 17) * 2 / (17 * 2))
with pytest.raises(TaxonomyError):
build_taxonomy(_segs() + [("v1s1", "juggle", "cup", "left")])
with pytest.raises(TaxonomyError):
build_taxonomy(_segs() + [("v1s1", "grasp", "cup", "b")])
def test_committed_taxonomy():
tax = Taxonomy.load(REPO / "taxonomy" / "taxonomy_v1.1.json")
assert tax.verbs == list(VERB_FINE)
assert tax.f2c == FINE_TO_COMPOSITE and len(tax.verb_composite) == 6
assert "cup" in tax.nouns and "water cup" not in tax.nouns
assert all(c >= 50 for c in tax.d["counts"]["noun"].values())
assert tax.d["n_annotated_volunteers"] == 36
lab, why = tax.classify("support", "cup", "left")
assert lab is None and why == "dropped_verb:support"
lab, why = tax.classify("grasp", "cup", "both")
assert lab["verb_composite"] == tax.verb_composite.index("grasp-family")
assert tax.classify("grasp", "food", "left")[1] == "rare_noun"