Spaces:
Running
Running
File size: 1,793 Bytes
c96d602 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | """dm-bench CLI: build -> verify -> solve -> eval round trip (fast methods)."""
import json
from src.bench.__main__ import main
def test_cli_round_trip(tmp_path, monkeypatch, capsys):
monkeypatch.delenv("DM_BENCH_TEST_SECRET", raising=False)
rel, sols = str(tmp_path / "rel"), str(tmp_path / "sol")
assert main(["build", "--out", rel, "--tiers", "easy", "--splits", "val", "test",
"--limit-docs", "1"]) == 0
assert "skipping split 'test'" in capsys.readouterr().err
assert main(["verify", rel]) == 0
assert main(["solve", "--release", rel, "--out", sols + "/oracle", "--method", "oracle",
"--tiers", "easy", "--splits", "val"]) == 0
assert main(["solve", "--release", rel, "--out", sols + "/random", "--method", "random",
"--tiers", "easy", "--splits", "val"]) == 0
assert main(["solve", "--release", rel, "--out", sols + "/x", "--method", "oracle",
"--tiers", "easy", "--splits", "test"]) == 2 # oracle refused on test
for method, expected in (("oracle", 1.0), ("random", 0.0)):
out = str(tmp_path / f"{method}.json")
assert main(["eval", "--release", rel, "--solutions", f"{sols}/{method}",
"--tiers", "easy", "--splits", "val", "--out", out]) == 0
with open(out) as fh:
res = json.load(fh)
assert res["eval_version"] == "1.0"
assert res["benchmark"] == "dm-bench@0.1.0"
assert res["solvers"] == [f"{method}@0.1"]
assert len(res["release_sha256"]) == 64
assert res["scores"]["easy"]["val"]["direct_acc"]["mean"] == expected
def test_solve_fails_on_missing_release(tmp_path):
assert main(["solve", "--release", str(tmp_path / "nope"), "--out", str(tmp_path / "s")]) == 2
|