Spaces:
Running
Running
Download tests/test_sample.py from dynatrace-oss/TurnGEO: direct link, hf CLI and curl.
- Browser
- Download file 14.1 kB
-
https://huggingface.co/spaces/dynatrace-oss/TurnGEO/resolve/main/tests/test_sample.py
- Command line
-
hf download hf://spaces/dynatrace-oss/TurnGEO/tests/test_sample.py
-
curl -L -o test_sample.py https://huggingface.co/spaces/dynatrace-oss/TurnGEO/resolve/main/tests/test_sample.py
14.1 kB
| """The sampler: the eligibility filters, the seed-cluster split, the representative instance, | |
| stratified allocation with capping, `STRATA`, `sample_id`, and the excluded-document swap.""" | |
| import pytest | |
| from conftest import SCENARIO_DOC_FLAGS, doc_url, make_scenario_corpus | |
| from turngeo import config, gate, sample | |
| def _small_horizon(monkeypatch): | |
| monkeypatch.setattr(config, "REPLAY_HORIZON_CAP", 2) | |
| monkeypatch.setattr(config, "TRAIN_SEEDS", 2) | |
| def _gate_tables() -> tuple[list[dict], dict[str, dict]]: | |
| return gate.build(make_scenario_corpus()) | |
| # --------------------------------------------------------------------------- # | |
| # The four eligibility filters, over the hand-derived scenario in CONVERSATIONS | |
| # --------------------------------------------------------------------------- # | |
| def test_funnel_matches_the_fixtures_hand_derived_counts(): | |
| corpus = make_scenario_corpus() | |
| result = sample.build_sample(corpus, "test", n_pairs=1) | |
| funnel = result.sample_json["eligibility_funnel"] | |
| assert funnel["pairs_total"] == 14 | |
| assert funnel["pairs_after_horizon_filter"] == 12 # excludes c1c/d7 (L=3), c2/d3 (L=4) | |
| assert funnel["pairs_after_gate"] == 10 # additionally excludes c3/d4 (doc_class=broken_wall), c9/d9 (thin) | |
| assert funnel["pairs_after_page_type_filter"] == 9 # additionally excludes c10/d10 (page_type=social) | |
| assert funnel["unique_eligible_documents_pre_multiseed"] == 7 # d1,d2,d7,d5,d6,d8,d11 | |
| assert funnel["multi_seed_documents_excluded"] == 1 # d5 spans s4 and s5 | |
| assert funnel["unique_eligible_documents"] == 6 # d1,d2,d7,d6,d8,d11 | |
| def test_low_prose_is_labelled_not_gated(): | |
| """d11 has stopword_rate=0.1 (< 0.18, low_prose) but is otherwise fully eligible -- | |
| it must survive the gate and appear in the drawn pool with `low_prose=True`: a | |
| reported dimension, not a filter.""" | |
| all_pairs = sample._build_pairs(_gate_tables()[0]) | |
| docs = _gate_tables()[1] | |
| funnel = sample._eligibility_funnel(all_pairs, docs) | |
| eligible_doc_ids = {p.doc_id for p in funnel["eligible_pairs"]} | |
| assert doc_url("d11") in eligible_doc_ids | |
| pool = sample._build_doc_pool(funnel["eligible_pairs"], docs, {"s6"}) | |
| d11 = next(c for c in pool if c.doc_id == doc_url("d11")) | |
| assert d11.low_prose is True | |
| def test_filter_2_excludes_l_at_least_2_never_cited_counts_as_l_zero(): | |
| """L=4 and the boundary L=2 are excluded; L=1 and a never-cited pair (L=0) survive.""" | |
| corpus = make_scenario_corpus( | |
| {"cL1": ("s1", 6, 3, {doc_url("dL1"): (1, [2])}), "cL2": ("s1", 7, 3, {doc_url("dL2"): (1, [3])})}, | |
| {doc_url("dL1"): SCENARIO_DOC_FLAGS[doc_url("d6")], doc_url("dL2"): SCENARIO_DOC_FLAGS[doc_url("d6")]}, | |
| ) | |
| gate_pairs = sample._build_pairs(gate.build(corpus)[0]) | |
| assert (gate_pairs[("c2", doc_url("d3"))].L, gate_pairs[("c1", doc_url("d2"))].L) == (4, 0) | |
| assert (gate_pairs[("cL1", doc_url("dL1"))].L, gate_pairs[("cL2", doc_url("dL2"))].L) == (1, 2) | |
| funnel = sample._eligibility_funnel(gate_pairs, gate.build(corpus)[1]) | |
| eligible = {p.doc_id for p in funnel["eligible_pairs"]} | |
| assert {doc_url("dL1"), doc_url("d2")} <= eligible | |
| assert not {doc_url("dL2"), doc_url("d3")} & eligible | |
| def test_representative_conversation_is_always_a_surviving_instance(): | |
| """dx's earlier instance (t_enter 1, L=3) would win the tie-break but is past the horizon, | |
| so the drawn pair must be the later, eligible one.""" | |
| corpus = make_scenario_corpus( | |
| {"cx_early": ("s4", 1, 5, {doc_url("dx"): (1, [4])}), "cx_late": ("s4", 2, 3, {doc_url("dx"): (2, [2])})}, | |
| {doc_url("dx"): SCENARIO_DOC_FLAGS[doc_url("d5")]}, | |
| ) | |
| drawn = [p for side in ("test", "train") for p in sample.build_sample(corpus, side, n_pairs=100).pairs] | |
| dx = next(p for p in drawn if p.doc_id == doc_url("dx")) | |
| assert (dx.conversation_id, dx.t_enter, dx.L) == ("cx_late", 2, 0) | |
| def test_representative_tie_break_is_lowest_t_enter_then_lowest_conversation_id(): | |
| """Lowest t_enter wins (d8: `zzz_conv` over `aaa_conv`); on equal t_enter the lower id wins (dT).""" | |
| all_pairs = sample._build_pairs(_gate_tables()[0]) | |
| docs = _gate_tables()[1] | |
| funnel = sample._eligibility_funnel(all_pairs, docs) | |
| pool = sample._build_doc_pool(funnel["eligible_pairs"], docs, {"s1"}) | |
| d8 = next(c for c in pool if c.doc_id == doc_url("d8")) | |
| assert d8.conversation_id == "zzz_conv" | |
| assert d8.t_enter == 1 | |
| tied = make_scenario_corpus( | |
| {"b_tie": ("s1", 6, 2, {doc_url("dT"): (1, [1])}), "a_tie": ("s1", 7, 2, {doc_url("dT"): (1, [1])})}, | |
| {doc_url("dT"): SCENARIO_DOC_FLAGS[doc_url("d6")]}, | |
| ) | |
| tied_pairs, tied_docs = gate.build(tied) | |
| funnel = sample._eligibility_funnel(sample._build_pairs(tied_pairs), tied_docs) | |
| for eligible in (funnel["eligible_pairs"], funnel["eligible_pairs"][::-1]): # the input order must not matter | |
| pool = sample._build_doc_pool(eligible, tied_docs, {"s1"}) | |
| dt = next(c for c in pool if c.doc_id == doc_url("dT")) | |
| assert (dt.conversation_id, dt.t_enter) == ("a_tie", 1) | |
| # --------------------------------------------------------------------------- # | |
| # Split | |
| # --------------------------------------------------------------------------- # | |
| def test_split_is_disjoint_and_covers_every_seed(): | |
| seed_ids = [f"s{i}" for i in range(1, 11)] | |
| test_seeds, train_seeds = sample.seed_split(seed_ids, split_seed=42, train_seeds=3) | |
| assert test_seeds & train_seeds == set() | |
| assert test_seeds | train_seeds == set(seed_ids) | |
| assert len(train_seeds) == 3 | |
| def test_split_refuses_an_unusable_train_count(n): | |
| with pytest.raises(RuntimeError, match="not a usable split"): | |
| sample.seed_split([f"s{i}" for i in range(10)], split_seed=42, train_seeds=n) | |
| def test_strata_decide_the_allocation(monkeypatch): | |
| corpus = make_scenario_corpus() | |
| default = sample.build_sample(corpus, "test", n_pairs=4) | |
| assert all(len(k.split(" | ")) == 3 for k in default.sample_json["allocation"]) | |
| monkeypatch.setattr(config, "STRATA", ("page_type",)) | |
| by_type = sample.build_sample(corpus, "test", n_pairs=4) | |
| assert set(by_type.sample_json["allocation"]) <= {"earned", "brand", "marketplace"} | |
| assert set(by_type.sample_json["balance"]) == {"page_type"} | |
| assert by_type.sample_id != default.sample_id | |
| def test_sample_seed_never_moves_the_split(monkeypatch): | |
| """`--seed` sets SAMPLE_SEED only -- the seed_id split depends solely on | |
| SPLIT_SEED/TRAIN_SEEDS, both untouched by sample_seed.""" | |
| corpus = make_scenario_corpus() | |
| monkeypatch.setattr(config, "SPLIT_SEED", 1234) | |
| monkeypatch.setattr(config, "TRAIN_SEEDS", 3) | |
| seed_ids = sorted({c.seed_id for c in corpus.conversations.values()}) | |
| for side, side_seeds in zip(("test", "train"), sample.seed_split(seed_ids, 1234, 3)): | |
| for sample_seed in (1, 999): | |
| drawn = sample.build_sample(corpus, side, n_pairs=100, sample_seed=sample_seed).pairs | |
| assert drawn and {p.seed_id for p in drawn} <= side_seeds | |
| def test_no_document_spans_both_sides(): | |
| corpus = make_scenario_corpus() | |
| test_result = sample.build_sample(corpus, "test", n_pairs=10) | |
| train_result = sample.build_sample(corpus, "train", n_pairs=10) | |
| test_docs = {p.doc_id for p in test_result.pairs} | |
| train_docs = {p.doc_id for p in train_result.pairs} | |
| assert test_docs & train_docs == set() | |
| test_seeds = {p.seed_id for p in test_result.pairs} | |
| train_seeds = {p.seed_id for p in train_result.pairs} | |
| assert test_seeds & train_seeds == set() | |
| # --------------------------------------------------------------------------- # | |
| # sample_id | |
| # --------------------------------------------------------------------------- # | |
| def test_the_draw_on_the_scenario_corpus_is_pinned(): | |
| # Pinned: a change to the draw algorithm moves this id (see test_paper_sample.py). | |
| r = sample.build_sample(make_scenario_corpus(), "test", n_pairs=3) | |
| assert [p.pair_id for p in r.pairs] == ["c1__0003", "c1b__0008", "zzz_conv__0009"] | |
| assert r.sample_id == "51e95eb3dfe706948bb994890d4efe4c9835ecf1e3660e29b546e0a0dcc52dd4" | |
| # --------------------------------------------------------------------------- # | |
| # Stratum allocation: cap + redistribute, and the whole-pool-smaller-than-N shortfall | |
| # --------------------------------------------------------------------------- # | |
| def test_allocate_with_cap_matches_uncapped_largest_remainder_when_pool_is_ample(): | |
| assert sample.allocate_with_cap({"a": 10, "b": 10, "c": 10}, 15) == ({"a": 5, "b": 5, "c": 5}, {}, 0) | |
| # exact shares 1.17 / 2.33 / 3.5: floors 1+2+3, the one leftover goes to the largest remainder (c) | |
| sizes = {"a": 10, "b": 20, "c": 30} | |
| assert sample.allocate_with_cap(sizes, 7) == ({"a": 1, "b": 2, "c": 4}, {}, 0) | |
| assert sample.allocate_with_cap(sizes, 7)[0] == sample._largest_remainder(sizes, 7) | |
| # shares .9 / .8 / .7 / 7.6: the three leftovers go to the small strata, not the largest one | |
| assert sample.allocate_with_cap({"a": 9, "b": 8, "c": 7, "d": 76}, 10) == ({"a": 1, "b": 1, "c": 1, "d": 7}, {}, 0) | |
| def test_allocate_with_cap_caps_an_exhausted_stratum_and_redistributes_over_two_rounds(): | |
| """sizes={"A":1,"B":3,"C":50} (pool=54), total=60 (exceeds the whole pool by 6). | |
| Round 1 caps only C (its naive largest-remainder share would be 56 > 50). Freeing | |
| C's slot still leaves 10 requested against a 4-document remainder (A+B), so round 2 | |
| caps both A and B too. Every capped stratum is recorded, and the final shortfall | |
| (60 - 54 = 6) is exact.""" | |
| sizes = {"A": 1, "B": 3, "C": 50} | |
| alloc, capped, shortfall = sample.allocate_with_cap(sizes, 60) | |
| assert alloc == {"A": 1, "B": 3, "C": 50} # the entire pool, nothing more | |
| assert capped == {"A": 1, "B": 3, "C": 50} # every stratum was exhausted | |
| assert shortfall == 6 | |
| assert sum(alloc.values()) == sum(sizes.values()) | |
| def test_allocate_with_cap_never_over_allocates_when_the_request_fits_the_pool(): | |
| """Largest-remainder apportionment has a quota property: whenever the requested | |
| total is within the combined pool, no single stratum's share can exceed its own | |
| pool size (`exact[k] = sizes[k] * total/pool_total <= sizes[k]` whenever | |
| `total <= pool_total`) -- so a tiny stratum survives unscathed alongside a huge one, | |
| and nothing is ever capped, even at the exact boundary `total == pool_total`.""" | |
| sizes = {"A": 1, "B": 100} | |
| alloc, capped, shortfall = sample.allocate_with_cap(sizes, 50) | |
| assert capped == {} | |
| assert shortfall == 0 | |
| assert alloc["A"] <= 1 and alloc["B"] <= 100 | |
| assert sum(alloc.values()) == 50 | |
| # even at the exact boundary (total == the whole pool), nothing is capped -- every | |
| # stratum's exact allocation lands precisely at its own size. | |
| alloc_full, capped_full, shortfall_full = sample.allocate_with_cap(sizes, 101) | |
| assert alloc_full == sizes | |
| assert capped_full == {} | |
| assert shortfall_full == 0 | |
| def test_stratified_draw_records_shortfall_when_the_whole_side_pool_is_too_small(): | |
| corpus = make_scenario_corpus() | |
| result = sample.build_sample(corpus, "test", n_pairs=1000) # far more than the tiny fixture holds | |
| # the test side holds 4 eligible documents (d1, d2, d7, d8), each taken once: 1000 - 4 short | |
| assert sorted(p.doc_id for p in result.pairs) == [doc_url(d) for d in ("d1", "d2", "d7", "d8")] | |
| assert result.sample_json["shortfall"] == 996 | |
| assert sum(result.sample_json["allocation"].values()) == 4 | |
| def _cand(doc_id: str, stratum_axis: str = "a"): | |
| from turngeo.sample import DocCandidate | |
| return DocCandidate( | |
| doc_id=doc_id, conversation_id=f"c-{doc_id}", seed_id="s", persona_index=0, t_enter=1, n_turns=3, n_rem=2, n_r=2, | |
| L=1, page_type=stratum_axis, cited_label="cited", reuse_bucket="1", n_conversations=1, thin=False, | |
| low_prose=False, doc_class="prose", | |
| ) | |
| def test_substitute_excluded_swaps_only_the_excluded_document_within_its_stratum(): | |
| from turngeo.sample import substitute_excluded | |
| pool = [_cand(f"a{i}", "a") for i in range(6)] + [_cand(f"b{i}", "b") for i in range(3)] | |
| drawn = [pool[0], pool[1], pool[6]] # a0, a1, b0 | |
| out, swaps = substitute_excluded(drawn, pool, ["a1"], sample_seed=7) | |
| assert [c.doc_id for c in out if c.doc_id != swaps["a1"]] == ["a0", "b0"] # nothing else moved | |
| new = next(c for c in out if c.doc_id == swaps["a1"]) | |
| assert new.stratum == pool[1].stratum and new.doc_id not in {"a0", "a1", "b0"} | |
| assert substitute_excluded(drawn, pool, ["a1"], sample_seed=7)[1] == swaps # deterministic | |
| assert substitute_excluded(drawn, pool, [], sample_seed=7) == (drawn, {}) # no exclusions, no change | |
| picks = {substitute_excluded(drawn, pool, ["a1"], sample_seed=seed)[1]["a1"] for seed in range(30)} | |
| assert picks <= {"a2", "a3", "a4", "a5"} and len(picks) > 1 # in-stratum, and the sample seed moves it | |
| def test_substitute_excluded_never_picks_another_excluded_document(): | |
| from turngeo.sample import substitute_excluded | |
| pool = [_cand(f"a{i}", "a") for i in range(4)] | |
| for seed in range(20): # a1 is the only allowed pick, whatever the seed draws | |
| assert substitute_excluded([pool[0]], pool, ["a0", "a2", "a3"], sample_seed=seed)[1] == {"a0": "a1"} | |
| def test_a_document_whose_other_seed_instance_was_horizon_filtered_is_still_multi_seed(): | |
| def pair(conv, seed, L): | |
| return sample._PairRow( | |
| conversation_id=conv, doc_id="d", seed_id=seed, persona_index=0, t_enter=1, n_turns=4, n_rem=4, L=L, n_r=2, | |
| ) | |
| pairs = {("c1", "d"): pair("c1", "s1", L=0), ("c2", "d"): pair("c2", "s2", L=5)} # c2 is past the horizon (2) | |
| docs = {"d": dict(doc_class="ok", thin=False, wall_suspect=False, prose_sentences=9, page_type="article")} | |
| funnel = sample._eligibility_funnel(pairs, docs) | |
| assert funnel["pairs_after_horizon_filter"] == 1 | |
| assert funnel["multi_seed_documents_excluded"] == 1 | |
| assert funnel["eligible_pairs"] == [] | |