| """Smoke #8: composition math (gap, recovery) with fake callbacks.""" |
| from mathcompose.eval.compose import evaluate_composition, _weighted_vote |
|
|
|
|
| def test_weighted_vote_rescues_minority_correct(): |
| answers = ["4", "4", "7", "7", "7"] |
| correct = [True, True, False, False, False] |
| |
| assert _weighted_vote(answers, [1] * 5, correct) is False |
| |
| assert _weighted_vote(answers, [0.9, 0.9, 0.1, 0.1, 0.1], correct) is True |
|
|
|
|
| def test_composition_gap_and_recovery(): |
| samples = [r"\boxed{4} #0", r"\boxed{4} #1", r"\boxed{7} #2", r"\boxed{7} #3", r"\boxed{7} #4"] |
| vmap = {0: 0.9, 1: 0.9, 2: 0.1, 3: 0.1, 4: 0.1} |
|
|
| def solve_fn(_): |
| return samples |
|
|
| def v_score_fn(_, steps): |
| joined = " ".join(steps) |
| for i in range(len(samples)): |
| if f"#{i}" in joined: |
| return vmap[i] |
| return 0.0 |
|
|
| res = evaluate_composition([("q", "4")], solve_fn, v_score_fn, progress=False) |
| assert res["maj@n"] == 0.0 |
| assert res["oracle"] == 1.0 |
| assert res["weighted_maj"] == 1.0 |
| assert res["best_of_n"] == 1.0 |
| assert res["generator_verifier_gap"] == 1.0 |
| assert res["gap_recovered_by_V"] == 1.0 |
|
|
