"""What a task looks like, plus the helpers its check uses.""" import re TASKS = [] def add(id, title, difficulty, kind, prompt, files, check, reference, max_turns=20, answers=None, alternatives=None): """files: dict of path -> text, or a function returning one. check(sandbox, answer) -> (ok, detail). reference: the steps of a correct solution, used by selftest. Each step is a list of (tool, args) calls, or ("final", answer) for the last message. answers: for tasks judged on the final answer, (text, should_pass) pairs that selftest feeds to the check. alternatives: other correct solutions (step lists like reference) that selftest must also see pass, for tasks with more than one valid way to finish.""" TASKS.append(dict(id=id, title=title, difficulty=difficulty, kind=kind, prompt=prompt, files=files, check=check, reference=reference, max_turns=max_turns, answers=answers or [], alternatives=alternatives or [])) def node(sb, script, *args): return sb.exec("node", [script, *args], timeout=60) def py(sb, script, *args): return sb.exec("python", [script, *args], timeout=60) def why(rc, out): """A short reason from a failed script's output (its error line), or nothing when the script succeeded.""" if rc == 0: return "" lines = [l.strip() for l in out.splitlines() if l.strip()] hit = next((l for l in lines if re.match(r"\w*(Error|Exception)\b", l)), next((l for l in lines if re.search(r"error:|FAIL|failed", l)), lines[-1] if lines else "no output")) return f" ({hit[:140]})" def unchanged(sb, *names): """True when the named files are still exactly as the task gave them.""" return all(sb.get(n) == sb.files[n] for n in names)