TurnGEO / tests /test_sample.py
janwikadyn's picture
Initial commit: TurnGEO evaluation harness
d710584
Raw History Blame Contribute Delete
14.1 kB
"""The sampler: the eligibility filters, the seed-cluster split, the representative instance,
stratified allocation with capping, `STRATA`, `sample_id`, and the excluded-document swap."""
import pytest
from conftest import SCENARIO_DOC_FLAGS, doc_url, make_scenario_corpus
from turngeo import config, gate, sample
@pytest.fixture(autouse=True)
def _small_horizon(monkeypatch):
monkeypatch.setattr(config, "REPLAY_HORIZON_CAP", 2)
monkeypatch.setattr(config, "TRAIN_SEEDS", 2)
def _gate_tables() -> tuple[list[dict], dict[str, dict]]:
return gate.build(make_scenario_corpus())
# --------------------------------------------------------------------------- #
# The four eligibility filters, over the hand-derived scenario in CONVERSATIONS
# --------------------------------------------------------------------------- #
def test_funnel_matches_the_fixtures_hand_derived_counts():
corpus = make_scenario_corpus()
result = sample.build_sample(corpus, "test", n_pairs=1)
funnel = result.sample_json["eligibility_funnel"]
assert funnel["pairs_total"] == 14
assert funnel["pairs_after_horizon_filter"] == 12 # excludes c1c/d7 (L=3), c2/d3 (L=4)
assert funnel["pairs_after_gate"] == 10 # additionally excludes c3/d4 (doc_class=broken_wall), c9/d9 (thin)
assert funnel["pairs_after_page_type_filter"] == 9 # additionally excludes c10/d10 (page_type=social)
assert funnel["unique_eligible_documents_pre_multiseed"] == 7 # d1,d2,d7,d5,d6,d8,d11
assert funnel["multi_seed_documents_excluded"] == 1 # d5 spans s4 and s5
assert funnel["unique_eligible_documents"] == 6 # d1,d2,d7,d6,d8,d11
def test_low_prose_is_labelled_not_gated():
"""d11 has stopword_rate=0.1 (< 0.18, low_prose) but is otherwise fully eligible --
it must survive the gate and appear in the drawn pool with `low_prose=True`: a
reported dimension, not a filter."""
all_pairs = sample._build_pairs(_gate_tables()[0])
docs = _gate_tables()[1]
funnel = sample._eligibility_funnel(all_pairs, docs)
eligible_doc_ids = {p.doc_id for p in funnel["eligible_pairs"]}
assert doc_url("d11") in eligible_doc_ids
pool = sample._build_doc_pool(funnel["eligible_pairs"], docs, {"s6"})
d11 = next(c for c in pool if c.doc_id == doc_url("d11"))
assert d11.low_prose is True
def test_filter_2_excludes_l_at_least_2_never_cited_counts_as_l_zero():
"""L=4 and the boundary L=2 are excluded; L=1 and a never-cited pair (L=0) survive."""
corpus = make_scenario_corpus(
{"cL1": ("s1", 6, 3, {doc_url("dL1"): (1, [2])}), "cL2": ("s1", 7, 3, {doc_url("dL2"): (1, [3])})},
{doc_url("dL1"): SCENARIO_DOC_FLAGS[doc_url("d6")], doc_url("dL2"): SCENARIO_DOC_FLAGS[doc_url("d6")]},
)
gate_pairs = sample._build_pairs(gate.build(corpus)[0])
assert (gate_pairs[("c2", doc_url("d3"))].L, gate_pairs[("c1", doc_url("d2"))].L) == (4, 0)
assert (gate_pairs[("cL1", doc_url("dL1"))].L, gate_pairs[("cL2", doc_url("dL2"))].L) == (1, 2)
funnel = sample._eligibility_funnel(gate_pairs, gate.build(corpus)[1])
eligible = {p.doc_id for p in funnel["eligible_pairs"]}
assert {doc_url("dL1"), doc_url("d2")} <= eligible
assert not {doc_url("dL2"), doc_url("d3")} & eligible
def test_representative_conversation_is_always_a_surviving_instance():
"""dx's earlier instance (t_enter 1, L=3) would win the tie-break but is past the horizon,
so the drawn pair must be the later, eligible one."""
corpus = make_scenario_corpus(
{"cx_early": ("s4", 1, 5, {doc_url("dx"): (1, [4])}), "cx_late": ("s4", 2, 3, {doc_url("dx"): (2, [2])})},
{doc_url("dx"): SCENARIO_DOC_FLAGS[doc_url("d5")]},
)
drawn = [p for side in ("test", "train") for p in sample.build_sample(corpus, side, n_pairs=100).pairs]
dx = next(p for p in drawn if p.doc_id == doc_url("dx"))
assert (dx.conversation_id, dx.t_enter, dx.L) == ("cx_late", 2, 0)
def test_representative_tie_break_is_lowest_t_enter_then_lowest_conversation_id():
"""Lowest t_enter wins (d8: `zzz_conv` over `aaa_conv`); on equal t_enter the lower id wins (dT)."""
all_pairs = sample._build_pairs(_gate_tables()[0])
docs = _gate_tables()[1]
funnel = sample._eligibility_funnel(all_pairs, docs)
pool = sample._build_doc_pool(funnel["eligible_pairs"], docs, {"s1"})
d8 = next(c for c in pool if c.doc_id == doc_url("d8"))
assert d8.conversation_id == "zzz_conv"
assert d8.t_enter == 1
tied = make_scenario_corpus(
{"b_tie": ("s1", 6, 2, {doc_url("dT"): (1, [1])}), "a_tie": ("s1", 7, 2, {doc_url("dT"): (1, [1])})},
{doc_url("dT"): SCENARIO_DOC_FLAGS[doc_url("d6")]},
)
tied_pairs, tied_docs = gate.build(tied)
funnel = sample._eligibility_funnel(sample._build_pairs(tied_pairs), tied_docs)
for eligible in (funnel["eligible_pairs"], funnel["eligible_pairs"][::-1]): # the input order must not matter
pool = sample._build_doc_pool(eligible, tied_docs, {"s1"})
dt = next(c for c in pool if c.doc_id == doc_url("dT"))
assert (dt.conversation_id, dt.t_enter) == ("a_tie", 1)
# --------------------------------------------------------------------------- #
# Split
# --------------------------------------------------------------------------- #
def test_split_is_disjoint_and_covers_every_seed():
seed_ids = [f"s{i}" for i in range(1, 11)]
test_seeds, train_seeds = sample.seed_split(seed_ids, split_seed=42, train_seeds=3)
assert test_seeds & train_seeds == set()
assert test_seeds | train_seeds == set(seed_ids)
assert len(train_seeds) == 3
@pytest.mark.parametrize("n", [0, -1, 10])
def test_split_refuses_an_unusable_train_count(n):
with pytest.raises(RuntimeError, match="not a usable split"):
sample.seed_split([f"s{i}" for i in range(10)], split_seed=42, train_seeds=n)
def test_strata_decide_the_allocation(monkeypatch):
corpus = make_scenario_corpus()
default = sample.build_sample(corpus, "test", n_pairs=4)
assert all(len(k.split(" | ")) == 3 for k in default.sample_json["allocation"])
monkeypatch.setattr(config, "STRATA", ("page_type",))
by_type = sample.build_sample(corpus, "test", n_pairs=4)
assert set(by_type.sample_json["allocation"]) <= {"earned", "brand", "marketplace"}
assert set(by_type.sample_json["balance"]) == {"page_type"}
assert by_type.sample_id != default.sample_id
def test_sample_seed_never_moves_the_split(monkeypatch):
"""`--seed` sets SAMPLE_SEED only -- the seed_id split depends solely on
SPLIT_SEED/TRAIN_SEEDS, both untouched by sample_seed."""
corpus = make_scenario_corpus()
monkeypatch.setattr(config, "SPLIT_SEED", 1234)
monkeypatch.setattr(config, "TRAIN_SEEDS", 3)
seed_ids = sorted({c.seed_id for c in corpus.conversations.values()})
for side, side_seeds in zip(("test", "train"), sample.seed_split(seed_ids, 1234, 3)):
for sample_seed in (1, 999):
drawn = sample.build_sample(corpus, side, n_pairs=100, sample_seed=sample_seed).pairs
assert drawn and {p.seed_id for p in drawn} <= side_seeds
def test_no_document_spans_both_sides():
corpus = make_scenario_corpus()
test_result = sample.build_sample(corpus, "test", n_pairs=10)
train_result = sample.build_sample(corpus, "train", n_pairs=10)
test_docs = {p.doc_id for p in test_result.pairs}
train_docs = {p.doc_id for p in train_result.pairs}
assert test_docs & train_docs == set()
test_seeds = {p.seed_id for p in test_result.pairs}
train_seeds = {p.seed_id for p in train_result.pairs}
assert test_seeds & train_seeds == set()
# --------------------------------------------------------------------------- #
# sample_id
# --------------------------------------------------------------------------- #
def test_the_draw_on_the_scenario_corpus_is_pinned():
# Pinned: a change to the draw algorithm moves this id (see test_paper_sample.py).
r = sample.build_sample(make_scenario_corpus(), "test", n_pairs=3)
assert [p.pair_id for p in r.pairs] == ["c1__0003", "c1b__0008", "zzz_conv__0009"]
assert r.sample_id == "51e95eb3dfe706948bb994890d4efe4c9835ecf1e3660e29b546e0a0dcc52dd4"
# --------------------------------------------------------------------------- #
# Stratum allocation: cap + redistribute, and the whole-pool-smaller-than-N shortfall
# --------------------------------------------------------------------------- #
def test_allocate_with_cap_matches_uncapped_largest_remainder_when_pool_is_ample():
assert sample.allocate_with_cap({"a": 10, "b": 10, "c": 10}, 15) == ({"a": 5, "b": 5, "c": 5}, {}, 0)
# exact shares 1.17 / 2.33 / 3.5: floors 1+2+3, the one leftover goes to the largest remainder (c)
sizes = {"a": 10, "b": 20, "c": 30}
assert sample.allocate_with_cap(sizes, 7) == ({"a": 1, "b": 2, "c": 4}, {}, 0)
assert sample.allocate_with_cap(sizes, 7)[0] == sample._largest_remainder(sizes, 7)
# shares .9 / .8 / .7 / 7.6: the three leftovers go to the small strata, not the largest one
assert sample.allocate_with_cap({"a": 9, "b": 8, "c": 7, "d": 76}, 10) == ({"a": 1, "b": 1, "c": 1, "d": 7}, {}, 0)
def test_allocate_with_cap_caps_an_exhausted_stratum_and_redistributes_over_two_rounds():
"""sizes={"A":1,"B":3,"C":50} (pool=54), total=60 (exceeds the whole pool by 6).
Round 1 caps only C (its naive largest-remainder share would be 56 > 50). Freeing
C's slot still leaves 10 requested against a 4-document remainder (A+B), so round 2
caps both A and B too. Every capped stratum is recorded, and the final shortfall
(60 - 54 = 6) is exact."""
sizes = {"A": 1, "B": 3, "C": 50}
alloc, capped, shortfall = sample.allocate_with_cap(sizes, 60)
assert alloc == {"A": 1, "B": 3, "C": 50} # the entire pool, nothing more
assert capped == {"A": 1, "B": 3, "C": 50} # every stratum was exhausted
assert shortfall == 6
assert sum(alloc.values()) == sum(sizes.values())
def test_allocate_with_cap_never_over_allocates_when_the_request_fits_the_pool():
"""Largest-remainder apportionment has a quota property: whenever the requested
total is within the combined pool, no single stratum's share can exceed its own
pool size (`exact[k] = sizes[k] * total/pool_total <= sizes[k]` whenever
`total <= pool_total`) -- so a tiny stratum survives unscathed alongside a huge one,
and nothing is ever capped, even at the exact boundary `total == pool_total`."""
sizes = {"A": 1, "B": 100}
alloc, capped, shortfall = sample.allocate_with_cap(sizes, 50)
assert capped == {}
assert shortfall == 0
assert alloc["A"] <= 1 and alloc["B"] <= 100
assert sum(alloc.values()) == 50
# even at the exact boundary (total == the whole pool), nothing is capped -- every
# stratum's exact allocation lands precisely at its own size.
alloc_full, capped_full, shortfall_full = sample.allocate_with_cap(sizes, 101)
assert alloc_full == sizes
assert capped_full == {}
assert shortfall_full == 0
def test_stratified_draw_records_shortfall_when_the_whole_side_pool_is_too_small():
corpus = make_scenario_corpus()
result = sample.build_sample(corpus, "test", n_pairs=1000) # far more than the tiny fixture holds
# the test side holds 4 eligible documents (d1, d2, d7, d8), each taken once: 1000 - 4 short
assert sorted(p.doc_id for p in result.pairs) == [doc_url(d) for d in ("d1", "d2", "d7", "d8")]
assert result.sample_json["shortfall"] == 996
assert sum(result.sample_json["allocation"].values()) == 4
def _cand(doc_id: str, stratum_axis: str = "a"):
from turngeo.sample import DocCandidate
return DocCandidate(
doc_id=doc_id, conversation_id=f"c-{doc_id}", seed_id="s", persona_index=0, t_enter=1, n_turns=3, n_rem=2, n_r=2,
L=1, page_type=stratum_axis, cited_label="cited", reuse_bucket="1", n_conversations=1, thin=False,
low_prose=False, doc_class="prose",
)
def test_substitute_excluded_swaps_only_the_excluded_document_within_its_stratum():
from turngeo.sample import substitute_excluded
pool = [_cand(f"a{i}", "a") for i in range(6)] + [_cand(f"b{i}", "b") for i in range(3)]
drawn = [pool[0], pool[1], pool[6]] # a0, a1, b0
out, swaps = substitute_excluded(drawn, pool, ["a1"], sample_seed=7)
assert [c.doc_id for c in out if c.doc_id != swaps["a1"]] == ["a0", "b0"] # nothing else moved
new = next(c for c in out if c.doc_id == swaps["a1"])
assert new.stratum == pool[1].stratum and new.doc_id not in {"a0", "a1", "b0"}
assert substitute_excluded(drawn, pool, ["a1"], sample_seed=7)[1] == swaps # deterministic
assert substitute_excluded(drawn, pool, [], sample_seed=7) == (drawn, {}) # no exclusions, no change
picks = {substitute_excluded(drawn, pool, ["a1"], sample_seed=seed)[1]["a1"] for seed in range(30)}
assert picks <= {"a2", "a3", "a4", "a5"} and len(picks) > 1 # in-stratum, and the sample seed moves it
def test_substitute_excluded_never_picks_another_excluded_document():
from turngeo.sample import substitute_excluded
pool = [_cand(f"a{i}", "a") for i in range(4)]
for seed in range(20): # a1 is the only allowed pick, whatever the seed draws
assert substitute_excluded([pool[0]], pool, ["a0", "a2", "a3"], sample_seed=seed)[1] == {"a0": "a1"}
def test_a_document_whose_other_seed_instance_was_horizon_filtered_is_still_multi_seed():
def pair(conv, seed, L):
return sample._PairRow(
conversation_id=conv, doc_id="d", seed_id=seed, persona_index=0, t_enter=1, n_turns=4, n_rem=4, L=L, n_r=2,
)
pairs = {("c1", "d"): pair("c1", "s1", L=0), ("c2", "d"): pair("c2", "s2", L=5)} # c2 is past the horizon (2)
docs = {"d": dict(doc_class="ok", thin=False, wall_suspect=False, prose_sentences=9, page_type="article")}
funnel = sample._eligibility_funnel(pairs, docs)
assert funnel["pairs_after_horizon_filter"] == 1
assert funnel["multi_seed_documents_excluded"] == 1
assert funnel["eligible_pairs"] == []