"""The sampler: the eligibility filters, the seed-cluster split, the representative instance, stratified allocation with capping, `STRATA`, `sample_id`, and the excluded-document swap.""" import pytest from conftest import SCENARIO_DOC_FLAGS, doc_url, make_scenario_corpus from turngeo import config, gate, sample @pytest.fixture(autouse=True) def _small_horizon(monkeypatch): monkeypatch.setattr(config, "REPLAY_HORIZON_CAP", 2) monkeypatch.setattr(config, "TRAIN_SEEDS", 2) def _gate_tables() -> tuple[list[dict], dict[str, dict]]: return gate.build(make_scenario_corpus()) # --------------------------------------------------------------------------- # # The four eligibility filters, over the hand-derived scenario in CONVERSATIONS # --------------------------------------------------------------------------- # def test_funnel_matches_the_fixtures_hand_derived_counts(): corpus = make_scenario_corpus() result = sample.build_sample(corpus, "test", n_pairs=1) funnel = result.sample_json["eligibility_funnel"] assert funnel["pairs_total"] == 14 assert funnel["pairs_after_horizon_filter"] == 12 # excludes c1c/d7 (L=3), c2/d3 (L=4) assert funnel["pairs_after_gate"] == 10 # additionally excludes c3/d4 (doc_class=broken_wall), c9/d9 (thin) assert funnel["pairs_after_page_type_filter"] == 9 # additionally excludes c10/d10 (page_type=social) assert funnel["unique_eligible_documents_pre_multiseed"] == 7 # d1,d2,d7,d5,d6,d8,d11 assert funnel["multi_seed_documents_excluded"] == 1 # d5 spans s4 and s5 assert funnel["unique_eligible_documents"] == 6 # d1,d2,d7,d6,d8,d11 def test_low_prose_is_labelled_not_gated(): """d11 has stopword_rate=0.1 (< 0.18, low_prose) but is otherwise fully eligible -- it must survive the gate and appear in the drawn pool with `low_prose=True`: a reported dimension, not a filter.""" all_pairs = sample._build_pairs(_gate_tables()[0]) docs = _gate_tables()[1] funnel = sample._eligibility_funnel(all_pairs, docs) eligible_doc_ids = {p.doc_id for p in funnel["eligible_pairs"]} assert doc_url("d11") in eligible_doc_ids pool = sample._build_doc_pool(funnel["eligible_pairs"], docs, {"s6"}) d11 = next(c for c in pool if c.doc_id == doc_url("d11")) assert d11.low_prose is True def test_filter_2_excludes_l_at_least_2_never_cited_counts_as_l_zero(): """L=4 and the boundary L=2 are excluded; L=1 and a never-cited pair (L=0) survive.""" corpus = make_scenario_corpus( {"cL1": ("s1", 6, 3, {doc_url("dL1"): (1, [2])}), "cL2": ("s1", 7, 3, {doc_url("dL2"): (1, [3])})}, {doc_url("dL1"): SCENARIO_DOC_FLAGS[doc_url("d6")], doc_url("dL2"): SCENARIO_DOC_FLAGS[doc_url("d6")]}, ) gate_pairs = sample._build_pairs(gate.build(corpus)[0]) assert (gate_pairs[("c2", doc_url("d3"))].L, gate_pairs[("c1", doc_url("d2"))].L) == (4, 0) assert (gate_pairs[("cL1", doc_url("dL1"))].L, gate_pairs[("cL2", doc_url("dL2"))].L) == (1, 2) funnel = sample._eligibility_funnel(gate_pairs, gate.build(corpus)[1]) eligible = {p.doc_id for p in funnel["eligible_pairs"]} assert {doc_url("dL1"), doc_url("d2")} <= eligible assert not {doc_url("dL2"), doc_url("d3")} & eligible def test_representative_conversation_is_always_a_surviving_instance(): """dx's earlier instance (t_enter 1, L=3) would win the tie-break but is past the horizon, so the drawn pair must be the later, eligible one.""" corpus = make_scenario_corpus( {"cx_early": ("s4", 1, 5, {doc_url("dx"): (1, [4])}), "cx_late": ("s4", 2, 3, {doc_url("dx"): (2, [2])})}, {doc_url("dx"): SCENARIO_DOC_FLAGS[doc_url("d5")]}, ) drawn = [p for side in ("test", "train") for p in sample.build_sample(corpus, side, n_pairs=100).pairs] dx = next(p for p in drawn if p.doc_id == doc_url("dx")) assert (dx.conversation_id, dx.t_enter, dx.L) == ("cx_late", 2, 0) def test_representative_tie_break_is_lowest_t_enter_then_lowest_conversation_id(): """Lowest t_enter wins (d8: `zzz_conv` over `aaa_conv`); on equal t_enter the lower id wins (dT).""" all_pairs = sample._build_pairs(_gate_tables()[0]) docs = _gate_tables()[1] funnel = sample._eligibility_funnel(all_pairs, docs) pool = sample._build_doc_pool(funnel["eligible_pairs"], docs, {"s1"}) d8 = next(c for c in pool if c.doc_id == doc_url("d8")) assert d8.conversation_id == "zzz_conv" assert d8.t_enter == 1 tied = make_scenario_corpus( {"b_tie": ("s1", 6, 2, {doc_url("dT"): (1, [1])}), "a_tie": ("s1", 7, 2, {doc_url("dT"): (1, [1])})}, {doc_url("dT"): SCENARIO_DOC_FLAGS[doc_url("d6")]}, ) tied_pairs, tied_docs = gate.build(tied) funnel = sample._eligibility_funnel(sample._build_pairs(tied_pairs), tied_docs) for eligible in (funnel["eligible_pairs"], funnel["eligible_pairs"][::-1]): # the input order must not matter pool = sample._build_doc_pool(eligible, tied_docs, {"s1"}) dt = next(c for c in pool if c.doc_id == doc_url("dT")) assert (dt.conversation_id, dt.t_enter) == ("a_tie", 1) # --------------------------------------------------------------------------- # # Split # --------------------------------------------------------------------------- # def test_split_is_disjoint_and_covers_every_seed(): seed_ids = [f"s{i}" for i in range(1, 11)] test_seeds, train_seeds = sample.seed_split(seed_ids, split_seed=42, train_seeds=3) assert test_seeds & train_seeds == set() assert test_seeds | train_seeds == set(seed_ids) assert len(train_seeds) == 3 @pytest.mark.parametrize("n", [0, -1, 10]) def test_split_refuses_an_unusable_train_count(n): with pytest.raises(RuntimeError, match="not a usable split"): sample.seed_split([f"s{i}" for i in range(10)], split_seed=42, train_seeds=n) def test_strata_decide_the_allocation(monkeypatch): corpus = make_scenario_corpus() default = sample.build_sample(corpus, "test", n_pairs=4) assert all(len(k.split(" | ")) == 3 for k in default.sample_json["allocation"]) monkeypatch.setattr(config, "STRATA", ("page_type",)) by_type = sample.build_sample(corpus, "test", n_pairs=4) assert set(by_type.sample_json["allocation"]) <= {"earned", "brand", "marketplace"} assert set(by_type.sample_json["balance"]) == {"page_type"} assert by_type.sample_id != default.sample_id def test_sample_seed_never_moves_the_split(monkeypatch): """`--seed` sets SAMPLE_SEED only -- the seed_id split depends solely on SPLIT_SEED/TRAIN_SEEDS, both untouched by sample_seed.""" corpus = make_scenario_corpus() monkeypatch.setattr(config, "SPLIT_SEED", 1234) monkeypatch.setattr(config, "TRAIN_SEEDS", 3) seed_ids = sorted({c.seed_id for c in corpus.conversations.values()}) for side, side_seeds in zip(("test", "train"), sample.seed_split(seed_ids, 1234, 3)): for sample_seed in (1, 999): drawn = sample.build_sample(corpus, side, n_pairs=100, sample_seed=sample_seed).pairs assert drawn and {p.seed_id for p in drawn} <= side_seeds def test_no_document_spans_both_sides(): corpus = make_scenario_corpus() test_result = sample.build_sample(corpus, "test", n_pairs=10) train_result = sample.build_sample(corpus, "train", n_pairs=10) test_docs = {p.doc_id for p in test_result.pairs} train_docs = {p.doc_id for p in train_result.pairs} assert test_docs & train_docs == set() test_seeds = {p.seed_id for p in test_result.pairs} train_seeds = {p.seed_id for p in train_result.pairs} assert test_seeds & train_seeds == set() # --------------------------------------------------------------------------- # # sample_id # --------------------------------------------------------------------------- # def test_the_draw_on_the_scenario_corpus_is_pinned(): # Pinned: a change to the draw algorithm moves this id (see test_paper_sample.py). r = sample.build_sample(make_scenario_corpus(), "test", n_pairs=3) assert [p.pair_id for p in r.pairs] == ["c1__0003", "c1b__0008", "zzz_conv__0009"] assert r.sample_id == "51e95eb3dfe706948bb994890d4efe4c9835ecf1e3660e29b546e0a0dcc52dd4" # --------------------------------------------------------------------------- # # Stratum allocation: cap + redistribute, and the whole-pool-smaller-than-N shortfall # --------------------------------------------------------------------------- # def test_allocate_with_cap_matches_uncapped_largest_remainder_when_pool_is_ample(): assert sample.allocate_with_cap({"a": 10, "b": 10, "c": 10}, 15) == ({"a": 5, "b": 5, "c": 5}, {}, 0) # exact shares 1.17 / 2.33 / 3.5: floors 1+2+3, the one leftover goes to the largest remainder (c) sizes = {"a": 10, "b": 20, "c": 30} assert sample.allocate_with_cap(sizes, 7) == ({"a": 1, "b": 2, "c": 4}, {}, 0) assert sample.allocate_with_cap(sizes, 7)[0] == sample._largest_remainder(sizes, 7) # shares .9 / .8 / .7 / 7.6: the three leftovers go to the small strata, not the largest one assert sample.allocate_with_cap({"a": 9, "b": 8, "c": 7, "d": 76}, 10) == ({"a": 1, "b": 1, "c": 1, "d": 7}, {}, 0) def test_allocate_with_cap_caps_an_exhausted_stratum_and_redistributes_over_two_rounds(): """sizes={"A":1,"B":3,"C":50} (pool=54), total=60 (exceeds the whole pool by 6). Round 1 caps only C (its naive largest-remainder share would be 56 > 50). Freeing C's slot still leaves 10 requested against a 4-document remainder (A+B), so round 2 caps both A and B too. Every capped stratum is recorded, and the final shortfall (60 - 54 = 6) is exact.""" sizes = {"A": 1, "B": 3, "C": 50} alloc, capped, shortfall = sample.allocate_with_cap(sizes, 60) assert alloc == {"A": 1, "B": 3, "C": 50} # the entire pool, nothing more assert capped == {"A": 1, "B": 3, "C": 50} # every stratum was exhausted assert shortfall == 6 assert sum(alloc.values()) == sum(sizes.values()) def test_allocate_with_cap_never_over_allocates_when_the_request_fits_the_pool(): """Largest-remainder apportionment has a quota property: whenever the requested total is within the combined pool, no single stratum's share can exceed its own pool size (`exact[k] = sizes[k] * total/pool_total <= sizes[k]` whenever `total <= pool_total`) -- so a tiny stratum survives unscathed alongside a huge one, and nothing is ever capped, even at the exact boundary `total == pool_total`.""" sizes = {"A": 1, "B": 100} alloc, capped, shortfall = sample.allocate_with_cap(sizes, 50) assert capped == {} assert shortfall == 0 assert alloc["A"] <= 1 and alloc["B"] <= 100 assert sum(alloc.values()) == 50 # even at the exact boundary (total == the whole pool), nothing is capped -- every # stratum's exact allocation lands precisely at its own size. alloc_full, capped_full, shortfall_full = sample.allocate_with_cap(sizes, 101) assert alloc_full == sizes assert capped_full == {} assert shortfall_full == 0 def test_stratified_draw_records_shortfall_when_the_whole_side_pool_is_too_small(): corpus = make_scenario_corpus() result = sample.build_sample(corpus, "test", n_pairs=1000) # far more than the tiny fixture holds # the test side holds 4 eligible documents (d1, d2, d7, d8), each taken once: 1000 - 4 short assert sorted(p.doc_id for p in result.pairs) == [doc_url(d) for d in ("d1", "d2", "d7", "d8")] assert result.sample_json["shortfall"] == 996 assert sum(result.sample_json["allocation"].values()) == 4 def _cand(doc_id: str, stratum_axis: str = "a"): from turngeo.sample import DocCandidate return DocCandidate( doc_id=doc_id, conversation_id=f"c-{doc_id}", seed_id="s", persona_index=0, t_enter=1, n_turns=3, n_rem=2, n_r=2, L=1, page_type=stratum_axis, cited_label="cited", reuse_bucket="1", n_conversations=1, thin=False, low_prose=False, doc_class="prose", ) def test_substitute_excluded_swaps_only_the_excluded_document_within_its_stratum(): from turngeo.sample import substitute_excluded pool = [_cand(f"a{i}", "a") for i in range(6)] + [_cand(f"b{i}", "b") for i in range(3)] drawn = [pool[0], pool[1], pool[6]] # a0, a1, b0 out, swaps = substitute_excluded(drawn, pool, ["a1"], sample_seed=7) assert [c.doc_id for c in out if c.doc_id != swaps["a1"]] == ["a0", "b0"] # nothing else moved new = next(c for c in out if c.doc_id == swaps["a1"]) assert new.stratum == pool[1].stratum and new.doc_id not in {"a0", "a1", "b0"} assert substitute_excluded(drawn, pool, ["a1"], sample_seed=7)[1] == swaps # deterministic assert substitute_excluded(drawn, pool, [], sample_seed=7) == (drawn, {}) # no exclusions, no change picks = {substitute_excluded(drawn, pool, ["a1"], sample_seed=seed)[1]["a1"] for seed in range(30)} assert picks <= {"a2", "a3", "a4", "a5"} and len(picks) > 1 # in-stratum, and the sample seed moves it def test_substitute_excluded_never_picks_another_excluded_document(): from turngeo.sample import substitute_excluded pool = [_cand(f"a{i}", "a") for i in range(4)] for seed in range(20): # a1 is the only allowed pick, whatever the seed draws assert substitute_excluded([pool[0]], pool, ["a0", "a2", "a3"], sample_seed=seed)[1] == {"a0": "a1"} def test_a_document_whose_other_seed_instance_was_horizon_filtered_is_still_multi_seed(): def pair(conv, seed, L): return sample._PairRow( conversation_id=conv, doc_id="d", seed_id=seed, persona_index=0, t_enter=1, n_turns=4, n_rem=4, L=L, n_r=2, ) pairs = {("c1", "d"): pair("c1", "s1", L=0), ("c2", "d"): pair("c2", "s2", L=5)} # c2 is past the horizon (2) docs = {"d": dict(doc_class="ok", thin=False, wall_suspect=False, prose_sentences=9, page_type="article")} funnel = sample._eligibility_funnel(pairs, docs) assert funnel["pairs_after_horizon_filter"] == 1 assert funnel["multi_seed_documents_excluded"] == 1 assert funnel["eligible_pairs"] == []