File size: 1,313 Bytes
79df115 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | import json
from pathlib import Path
import pytest
from evaluation import CASES, grade
from app import create_demo, save_report
from test_engine import Tokenizer, FakeModel
def test_app_builds_without_downloading_weights():
demo = create_demo(FakeModel(), Tokenizer(), {"test_only": True})
config = demo.get_config_file()
assert config["mode"] == "blocks"
assert any(d.get("api_name") == "generate" for d in config["dependencies"])
def test_reports_are_unique_valid_json():
a = save_report({"text": "hello λ"})
b = save_report({"text": "other"})
assert a != b
assert json.loads(Path(a).read_text())["text"] == "hello λ"
@pytest.mark.parametrize("case", CASES)
def test_grading(case):
if case["kind"] == "manual":
assert grade("anything", case)["status"] == "manual review"
else:
text = json.dumps(case["expected"]) if case["kind"] == "json" else case["expected"]
assert grade(text, case)["status"] == "pass"
assert grade("wrong answer", case)["status"] == "fail"
def test_json_grading_rejects_extra_keys_and_fences():
c = next(c for c in CASES if c["kind"] == "json")
assert grade('{"name":"Ada","count":3,"extra":true}', c)["status"] == "fail"
assert grade('```json\n{"name":"Ada","count":3}\n```', c)["status"] == "fail"
|