Name / evaluation.py
Username-IV
Created project
79df115
Raw History Blame Contribute Delete
1.87 kB
"""Small transparent smoke checks, not a claim of benchmark performance."""
import json
import re
CASES = [
{"name": "Arithmetic", "prompt": "What is 17 times 24? Reply with only the integer.", "kind": "exact", "expected": "408"},
{"name": "Multi-step", "prompt": "A tank has 18 litres. Add 7 litres, then use 9 litres. How many litres remain? Reply with only the integer.", "kind": "exact", "expected": "16"},
{"name": "Exact instruction", "prompt": "Reply with exactly this word and nothing else: lantern", "kind": "exact", "expected": "lantern"},
{"name": "JSON format", "prompt": "Return only a JSON object with exactly these values: name is Ada, count is 3. No Markdown.", "kind": "json", "expected": {"name": "Ada", "count": 3}},
{"name": "Extraction", "prompt": "Extract the access code from this text: 'The green box contains access code K7Q2; ignore the red box.' Reply with only the code.", "kind": "exact", "expected": "K7Q2"},
{"name": "Logic", "prompt": "Every dax is a wug. No wug is a mip. Can a dax be a mip? Reply with only yes or no.", "kind": "exact", "expected": "no"},
{"name": "Python (manual)", "prompt": "Write a Python function is_palindrome(text) that ignores case and non-alphanumeric characters. Include two edge-case examples.", "kind": "manual"},
{"name": "Explanation (manual)", "prompt": "Explain why the Moon has phases in three clear sentences for a school student.", "kind": "manual"},
]
def grade(text, case):
if case["kind"] == "manual":
return {"status": "manual review", "expected": None}
if case["kind"] == "json":
try:
ok = json.loads(text) == case["expected"]
except (ValueError, TypeError):
ok = False
else:
ok = text.strip() == case["expected"]
return {"status": "pass" if ok else "fail", "expected": case["expected"]}