Blame for code/tests/test_audit_g6_eval_scripts_eval_checkpoint.py

"""U11-15: --quality-gate must not write the DPO-unlocking receipt from a subset eval."""
import importlib.util
import os
import types

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


def _load():
    spec = importlib.util.spec_from_file_location(
        "eval_checkpoint_under_test", os.path.join(REPO, "scripts", "eval_checkpoint.py")
    )
    m = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(m)
    return m


CFG = types.SimpleNamespace(eval=types.SimpleNamespace(math500_n=500, gsm8k_n=1319))


def test_subset_sizes_are_refused_for_the_gate():
    m = _load()
    assert m.quality_gate_size_errors(CFG, 5, 5)
    assert m.quality_gate_size_errors(CFG, 500, 0)  # a skipped set cannot certify the gate
    assert m.quality_gate_size_errors(CFG, 0, 1319)
    assert m.quality_gate_size_errors(CFG, 500, 100)


def test_full_sizes_pass_the_precheck():
    m = _load()
    assert m.quality_gate_size_errors(CFG, 500, 1319) == []


def test_measured_n_must_match_too():
    m = _load()
    per_set = {"math500": {"n": 500}, "gsm8k": {"n": 12}}  # dataset shorter than configured
    errs = m.quality_gate_size_errors(CFG, 500, 1319, per_set=per_set)
    assert errs and "gsm8k" in errs[0]
    ok = {"math500": {"n": 500}, "gsm8k": {"n": 1319}}
    assert m.quality_gate_size_errors(CFG, 500, 1319, per_set=ok) == []