sebika123123's picture
Initial release: settings, results, eval
869d7f8 verified
Raw History Blame Contribute Delete
1.76 kB
"""What a task looks like, plus the helpers its check uses."""
import re
TASKS = []
def add(id, title, difficulty, kind, prompt, files, check, reference, max_turns=20, answers=None, alternatives=None):
"""files: dict of path -> text, or a function returning one. check(sandbox, answer) -> (ok, detail).
reference: the steps of a correct solution, used by selftest. Each step is a list of (tool, args) calls,
or ("final", answer) for the last message. answers: for tasks judged on the final answer, (text, should_pass)
pairs that selftest feeds to the check. alternatives: other correct solutions (step lists like reference) that
selftest must also see pass, for tasks with more than one valid way to finish."""
TASKS.append(dict(id=id, title=title, difficulty=difficulty, kind=kind, prompt=prompt, files=files, check=check,
reference=reference, max_turns=max_turns, answers=answers or [], alternatives=alternatives or []))
def node(sb, script, *args):
return sb.exec("node", [script, *args], timeout=60)
def py(sb, script, *args):
return sb.exec("python", [script, *args], timeout=60)
def why(rc, out):
"""A short reason from a failed script's output (its error line), or nothing when the script succeeded."""
if rc == 0:
return ""
lines = [l.strip() for l in out.splitlines() if l.strip()]
hit = next((l for l in lines if re.match(r"\w*(Error|Exception)\b", l)),
next((l for l in lines if re.search(r"error:|FAIL|failed", l)), lines[-1] if lines else "no output"))
return f" ({hit[:140]})"
def unchanged(sb, *names):
"""True when the named files are still exactly as the task gave them."""
return all(sb.get(n) == sb.files[n] for n in names)