| """U11-15: --quality-gate must not write the DPO-unlocking receipt from a subset eval.""" |
| import importlib.util |
| import os |
| import types |
|
|
| REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
|
|
|
|
| def _load(): |
| spec = importlib.util.spec_from_file_location( |
| "eval_checkpoint_under_test", os.path.join(REPO, "scripts", "eval_checkpoint.py") |
| ) |
| m = importlib.util.module_from_spec(spec) |
| spec.loader.exec_module(m) |
| return m |
|
|
|
|
| CFG = types.SimpleNamespace(eval=types.SimpleNamespace(math500_n=500, gsm8k_n=1319)) |
|
|
|
|
| def test_subset_sizes_are_refused_for_the_gate(): |
| m = _load() |
| assert m.quality_gate_size_errors(CFG, 5, 5) |
| assert m.quality_gate_size_errors(CFG, 500, 0) |
| assert m.quality_gate_size_errors(CFG, 0, 1319) |
| assert m.quality_gate_size_errors(CFG, 500, 100) |
|
|
|
|
| def test_full_sizes_pass_the_precheck(): |
| m = _load() |
| assert m.quality_gate_size_errors(CFG, 500, 1319) == [] |
|
|
|
|
| def test_measured_n_must_match_too(): |
| m = _load() |
| per_set = {"math500": {"n": 500}, "gsm8k": {"n": 12}} |
| errs = m.quality_gate_size_errors(CFG, 500, 1319, per_set=per_set) |
| assert errs and "gsm8k" in errs[0] |
| ok = {"math500": {"n": 500}, "gsm8k": {"n": 1319}} |
| assert m.quality_gate_size_errors(CFG, 500, 1319, per_set=ok) == [] |
|
|
