File size: 1,313 Bytes
79df115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
import json
from pathlib import Path
import pytest
from evaluation import CASES, grade
from app import create_demo, save_report
from test_engine import Tokenizer, FakeModel

def test_app_builds_without_downloading_weights():
    demo = create_demo(FakeModel(), Tokenizer(), {"test_only": True})
    config = demo.get_config_file()
    assert config["mode"] == "blocks"
    assert any(d.get("api_name") == "generate" for d in config["dependencies"])

def test_reports_are_unique_valid_json():
    a = save_report({"text": "hello λ"})
    b = save_report({"text": "other"})
    assert a != b
    assert json.loads(Path(a).read_text())["text"] == "hello λ"

@pytest.mark.parametrize("case", CASES)
def test_grading(case):
    if case["kind"] == "manual":
        assert grade("anything", case)["status"] == "manual review"
    else:
        text = json.dumps(case["expected"]) if case["kind"] == "json" else case["expected"]
        assert grade(text, case)["status"] == "pass"
        assert grade("wrong answer", case)["status"] == "fail"

def test_json_grading_rejects_extra_keys_and_fences():
    c = next(c for c in CASES if c["kind"] == "json")
    assert grade('{"name":"Ada","count":3,"extra":true}', c)["status"] == "fail"
    assert grade('```json\n{"name":"Ada","count":3}\n```', c)["status"] == "fail"