File size: 4,621 Bytes
b445c46 d82d9c4 b445c46 d82d9c4 b445c46 d82d9c4 b445c46 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | """Capability: execute the Python code supplied in the request (sandboxed, see sandbox.py).
- output selection: the request carries Python code that defines a function and an input (argument list), and
the options are Python literals -> run f(input) and pick the option(s) equal to the value.
- test passing: the request carries code and tests -> run the tests (a `check(candidate)` function, if defined, is
called on the entry function; bare asserts run as they are) and answer the yes/no question.
Detection is by content (a parseable `def` in a code field or a fenced block), never by benchmark name.
"""
import ast
import re
import warnings
from prumo_rules import sandbox
from prumo_rules.common import bool_answer, crit_text, peaked, yes_no_keys
from prumo_rules.params import P as PRM
CODE_KEYS = ("code", "function", "program", "source", "snippet")
TEST_KEYS = ("tests", "test", "test_code", "unit_tests", "test_cases")
INPUT_KEYS = ("input", "inputs", "args", "arguments", "call_args")
FENCE = re.compile(r"```(?:python|py)?\s*\n(.*?)```", re.S)
def _defs(code):
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
tree = ast.parse(code)
except (SyntaxError, ValueError):
return []
return [n.name for n in tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))]
def _get(state, keys):
if not isinstance(state, dict):
return None
for k in keys:
v = state.get(k)
if isinstance(v, str) and v.strip():
return v
return None
def _code_in_text(text):
m = FENCE.search(text)
if m and "def " in m.group(1):
return m.group(1)
i = text.find("def ")
if i < 0 or (i > 0 and text[i - 1] not in "\n"):
return None
lines, out = text[i:].split("\n"), []
for j, ln in enumerate(lines):
if j and ln.strip() and not ln.startswith((" ", "\t", "def ", "class ", "@", "import ", "from ")):
break
out.append(ln)
return "\n".join(out)
def _call_in_text(text, fname):
m = re.search(r"\b" + re.escape(fname) + r"\((.*?)\)\s*(?:==|returns?\b|\?|$)", text, re.M)
return m.group(1) if m else None
def _is_literal(s):
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
ast.literal_eval(s.strip())
return True
except Exception:
return False
def solve(state, questions):
code = _get(state, CODE_KEYS)
texts = [state] if isinstance(state, str) else [v for v in state.values() if isinstance(v, str)] if isinstance(state, dict) else []
texts += [crit_text(q.get("instructions")) for q in questions.values()]
if not code:
for t in texts:
c = _code_in_text(t) if "def " in t else None
if c and _defs(c):
code = c
break
if not code or "def " not in code:
return None
defs = _defs(code)
if not defs:
return None
out = {}
tests = _get(state, TEST_KEYS)
inp = _get(state, INPUT_KEYS)
fname = "f" if "f" in defs else defs[0]
if inp is None:
for t in texts:
inp = _call_in_text(t.replace(code, ""), fname)
if inp is not None:
break
for key, q in questions.items():
if tests:
if q["type"] != "noul" and not yes_no_keys(q):
continue
entry = state.get("entry_point") if isinstance(state, dict) and isinstance(state.get("entry_point"), str) else None
if not entry:
called = set(re.findall(r"\b([A-Za-z_]\w*)\s*\(", tests))
cands = [d for d in defs if d in called] or defs
entry = cands[-1]
r = sandbox.run({"mode": "tests", "code": code, "tests": tests, "entry": entry})
if not r.get("ok"):
continue
ans = bool_answer(q, bool(r.get("passes")), PRM["code.mass"])
if ans:
out[key] = ans
elif inp is not None and q["type"] == "choice":
keys = list(q["criteria"])
cands = [crit_text(q["criteria"][k]) for k in keys]
if not all(_is_literal(c) for c in cands):
continue
r = sandbox.run({"mode": "output", "code": code, "call": f"{fname}({inp})", "candidates": cands})
if not r.get("ok"):
continue
hits = [k for k, m in zip(keys, r["matches"]) if m]
if hits and len(hits) < len(keys):
out[key] = peaked(keys, hits, PRM["code.mass"])
return out or None
|