File size: 1,757 Bytes
014b14c 6f2d0b4 1c0986b 6f2d0b4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | """Smoke #7: full eval plumbing load -> generate -> parse -> metric with a tiny
model on 3 synthetic examples (output is garbage; the pipeline is what's tested).
Requires network (tiny model download)."""
import pytest
from mathcompose.eval.processbench import score_subset
from mathcompose.eval.parse import majority_index
TINY = "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5"
@pytest.mark.slow
@pytest.mark.network
def test_eval_pipeline_runs():
pytest.importorskip("torch")
from mathcompose.eval.runners import HFRunner, make_verify_fn
runner = HFRunner(TINY, max_context=512)
verify_fn = make_verify_fn(runner, n=1, max_new_tokens=8)
examples = [
("What is 2+2?", ["We add.", "2+2=5."], 1),
("What is 3*3?", ["Multiply.", "3*3=9."], -1),
("Derivative of x^2?", ["Power rule.", "= x."], 1),
]
golds, preds = [], []
for problem, steps, gold in examples:
preds.append(majority_index(verify_fn(problem, steps)))
golds.append(gold)
s = score_subset("smoke", golds, preds) # tiny model -> garbage preds, but scoring runs
assert 0.0 <= s.f1 <= 1.0
assert s.n_error + s.n_correct == 3
@pytest.mark.slow
@pytest.mark.network
def test_batched_processbench_eval_runs():
"""The fast batched path: load -> chat_many -> parse -> score over a tiny slice."""
pytest.importorskip("torch")
from mathcompose.eval.runners import HFRunner
from mathcompose.eval.processbench import evaluate_batched
runner = HFRunner(TINY, max_context=512)
res = evaluate_batched(runner, splits=["gsm8k"], limit=4, n=1, batch_size=2, max_new_tokens=6)
assert 0.0 <= res["average_f1"] <= 100.0 # now reported on the 0-100 scale
assert "gsm8k" in res["subsets"]
|