File size: 1,869 Bytes
79df115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
"""Small transparent smoke checks, not a claim of benchmark performance."""
import json
import re

CASES = [
    {"name": "Arithmetic", "prompt": "What is 17 times 24? Reply with only the integer.", "kind": "exact", "expected": "408"},
    {"name": "Multi-step", "prompt": "A tank has 18 litres. Add 7 litres, then use 9 litres. How many litres remain? Reply with only the integer.", "kind": "exact", "expected": "16"},
    {"name": "Exact instruction", "prompt": "Reply with exactly this word and nothing else: lantern", "kind": "exact", "expected": "lantern"},
    {"name": "JSON format", "prompt": "Return only a JSON object with exactly these values: name is Ada, count is 3. No Markdown.", "kind": "json", "expected": {"name": "Ada", "count": 3}},
    {"name": "Extraction", "prompt": "Extract the access code from this text: 'The green box contains access code K7Q2; ignore the red box.' Reply with only the code.", "kind": "exact", "expected": "K7Q2"},
    {"name": "Logic", "prompt": "Every dax is a wug. No wug is a mip. Can a dax be a mip? Reply with only yes or no.", "kind": "exact", "expected": "no"},
    {"name": "Python (manual)", "prompt": "Write a Python function is_palindrome(text) that ignores case and non-alphanumeric characters. Include two edge-case examples.", "kind": "manual"},
    {"name": "Explanation (manual)", "prompt": "Explain why the Moon has phases in three clear sentences for a school student.", "kind": "manual"},
]

def grade(text, case):
    if case["kind"] == "manual":
        return {"status": "manual review", "expected": None}
    if case["kind"] == "json":
        try:
            ok = json.loads(text) == case["expected"]
        except (ValueError, TypeError):
            ok = False
    else:
        ok = text.strip() == case["expected"]
    return {"status": "pass" if ok else "fail", "expected": case["expected"]}