File size: 968 Bytes
201dc77 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | """U10-5: assert_meets_golden's tolerance must be explicit and tightenable."""
import pytest
from align.eval.harness import assert_meets_golden
GOLDEN = {"gsm8k": 0.258}
def _rep(acc, n=500):
return {"per_set": {"gsm8k": {"n": n, "pass@1": acc}}}
def test_replay_gate_can_be_tight():
# Default tolerance is the target's own Wilson CI (noisy re-training): a
# 3pp drop passes. A deterministic replay opts into a tight drop bound.
assert_meets_golden(_rep(0.225), GOLDEN) # documented, loose default
with pytest.raises(RuntimeError, match="max_drop"):
assert_meets_golden(_rep(0.225), GOLDEN, max_drop=0.01)
assert_meets_golden(_rep(0.25), GOLDEN, max_drop=0.01) # within 1pp
assert_meets_golden(_rep(0.30), GOLDEN, max_drop=0.01) # above is fine
def test_docstring_states_what_is_actually_checked():
doc = assert_meets_golden.__doc__
assert "LOWER BOUND of the TARGET" in doc.replace("\n", " ").replace(" ", " ")
|