"""Small transparent smoke checks, not a claim of benchmark performance.""" import json import re CASES = [ {"name": "Arithmetic", "prompt": "What is 17 times 24? Reply with only the integer.", "kind": "exact", "expected": "408"}, {"name": "Multi-step", "prompt": "A tank has 18 litres. Add 7 litres, then use 9 litres. How many litres remain? Reply with only the integer.", "kind": "exact", "expected": "16"}, {"name": "Exact instruction", "prompt": "Reply with exactly this word and nothing else: lantern", "kind": "exact", "expected": "lantern"}, {"name": "JSON format", "prompt": "Return only a JSON object with exactly these values: name is Ada, count is 3. No Markdown.", "kind": "json", "expected": {"name": "Ada", "count": 3}}, {"name": "Extraction", "prompt": "Extract the access code from this text: 'The green box contains access code K7Q2; ignore the red box.' Reply with only the code.", "kind": "exact", "expected": "K7Q2"}, {"name": "Logic", "prompt": "Every dax is a wug. No wug is a mip. Can a dax be a mip? Reply with only yes or no.", "kind": "exact", "expected": "no"}, {"name": "Python (manual)", "prompt": "Write a Python function is_palindrome(text) that ignores case and non-alphanumeric characters. Include two edge-case examples.", "kind": "manual"}, {"name": "Explanation (manual)", "prompt": "Explain why the Moon has phases in three clear sentences for a school student.", "kind": "manual"}, ] def grade(text, case): if case["kind"] == "manual": return {"status": "manual review", "expected": None} if case["kind"] == "json": try: ok = json.loads(text) == case["expected"] except (ValueError, TypeError): ok = False else: ok = text.strip() == case["expected"] return {"status": "pass" if ok else "fail", "expected": case["expected"]}