import json from collections import Counter import pytest from conftest import REPO from pulse.split_builder import allocate, choose_inner_val, round_half_up from pulse.splits import SplitError, load_split, validate_split from pulse.taxonomy import (FINE_TO_COMPOSITE, VERB_FINE, Taxonomy, TaxonomyError, build_taxonomy) # Test folds of the source files (results/t1_cv/cv_folds.json md5 8f20127a..., # results/t2_cv/cv_folds.json md5 3b93df5d...), copied here to pin them. SOURCE_FOLDS = { "pool33": [["v1", "v12", "v19", "v24", "v29", "v34", "v39"], ["v2", "v13", "v20", "v25", "v30", "v35", "v40"], ["v3", "v15", "v21", "v26", "v31", "v36", "v41"], ["v9", "v16", "v22", "v27", "v32", "v37"], ["v11", "v17", "v23", "v28", "v33", "v38"]], "pool27": [["v25", "v28", "v11", "v38", "v5", "v35"], ["v3", "v32", "v2", "v13", "v36", "v23"], ["v26", "v33", "v22", "v41", "v24"], ["v37", "v29", "v12", "v39", "v27"], ["v1", "v40", "v31", "v34", "v30"]], } SOURCE_MD5 = {"pool33": "8f20127a120ad516bedc52c983e82eb2", "pool27": "3b93df5dcefc2f74d444cda3f5bdd997"} N_VAL = {"pool33": 4, "pool27": 3} NO_PRESSURE_33 = {"v9", "v15", "v16", "v17", "v19", "v20", "v21"} @pytest.mark.parametrize("name", ["pool33", "pool27"]) def test_committed_split(name): sp = load_split(REPO / "splits" / f"{name}.json") assert sp.name == name and sp.n_folds == 5 assert [list(sp.fold(k).test) for k in range(5)] == SOURCE_FOLDS[name] assert sp.data["source"]["md5"] == SOURCE_MD5[name] used = Counter() for k in range(5): f = sp.fold(k) assert len(f.inner_val) == N_VAL[name] assert set(f.inner_val).isdisjoint(f.test) and set(f.inner_val).isdisjoint(f.train) for v in f.inner_val: assert sp.data["volunteer_profiles"][v]["eligible"] used.update(f.inner_val) if name == "pool33": assert len(set(f.inner_val) & NO_PRESSURE_33) == 1 assert max(used.values()) == 1 # every volunteer is inner-val at most once # regenerate from the recorded profiles: identical folds = choose_inner_val(name, sp.volunteers, SOURCE_FOLDS[name], sp.data["volunteer_profiles"], sp.data["inner_val_rule"]["fraction"]) assert [f["inner_val"] for f in folds] == [list(sp.fold(k).inner_val) for k in range(5)] text = (REPO / "splits" / f"{name}.json").read_text(encoding="utf-8") assert '"data_root"' not in text and '"/' not in text # no machine paths def test_split_validation_rejects_overlap(): d = json.loads((REPO / "splits" / "pool33.json").read_text(encoding="utf-8")) d["folds"][0]["train"].append(d["folds"][0]["test"][0]) with pytest.raises(SplitError): validate_split(d) def test_rounding_and_allocation(): assert round_half_up(3.9) == 4 and round_half_up(4.05) == 4 and round_half_up(3.3) == 3 assert allocate(4, {"none": 6, "pressure": 20}) == {"none": 1, "pressure": 3} assert sum(allocate(3, {"a": 1, "b": 1, "c": 1, "d": 1}).values()) == 3 def _segs(): s = [] for i in range(60): s.append((f"v{i % 3 + 1}s1", "grasp", "cup", "right")) for i in range(55): s.append((f"v{i % 3 + 1}s2", "move", "mouse", "both")) s.append(("v1s3", "support", "cup", "left")) s.append(("v1s3", "place", "rare thing", "left")) return s def test_build_taxonomy_rules(): tax = build_taxonomy(_segs(), threshold=50, pool_volunteers={"p": ["v1", "v2"]}) assert tax["verbs"] == list(VERB_FINE) and len(VERB_FINE) == 17 assert tax["nouns"] == ["cup", "wired mouse"] # cup kept as written; alias applied assert tax["n_segments_dropped_verb"] == 1 and tax["counts"]["dropped_verbs"] == {"support": 1} assert tax["n_segments_dropped_noun"] == 1 and tax["n_segments_kept"] == 115 assert tax["chance"]["uniform"]["act_vn@5"] == pytest.approx(min(5, 17) * 2 / (17 * 2)) with pytest.raises(TaxonomyError): build_taxonomy(_segs() + [("v1s1", "juggle", "cup", "left")]) with pytest.raises(TaxonomyError): build_taxonomy(_segs() + [("v1s1", "grasp", "cup", "b")]) def test_committed_taxonomy(): tax = Taxonomy.load(REPO / "taxonomy" / "taxonomy_v1.1.json") assert tax.verbs == list(VERB_FINE) assert tax.f2c == FINE_TO_COMPOSITE and len(tax.verb_composite) == 6 assert "cup" in tax.nouns and "water cup" not in tax.nouns assert all(c >= 50 for c in tax.d["counts"]["noun"].values()) assert tax.d["n_annotated_volunteers"] == 36 lab, why = tax.classify("support", "cup", "left") assert lab is None and why == "dropped_verb:support" lab, why = tax.classify("grasp", "cup", "both") assert lab["verb_composite"] == tax.verb_composite.index("grasp-family") assert tax.classify("grasp", "food", "left")[1] == "rare_noun"