"""Restricted Python executor: the tool the ReVeR judge calls for mechanical checks (counting, string and format tests). The code runs in a separate process with a time limit, a memory limit, a whitelist of builtins and modules, and the task (instruction, response, rubrics) bound as variables. The reply is a JSON string with the keys ok, stdout, result and error; `result` is the repr of a variable named `result`, if the code sets one.""" from __future__ import annotations import ast import contextlib import json import multiprocessing as mp import re import reprlib import signal import time import traceback from dataclasses import dataclass from multiprocessing.connection import Connection from typing import Any DISALLOWED_NODES = ( ast.Global, ast.Nonlocal, ast.ClassDef, ) ALLOWED_IMPORTS = { "collections", "datetime", "itertools", "json", "math", "re", "statistics", "string", "textwrap", "unicodedata", } DISALLOWED_NAMES = { "__import__", "breakpoint", "compile", "delattr", "dir", "eval", "exec", "exit", "getattr", "globals", "help", "input", "locals", "open", "quit", "setattr", "vars", } @dataclass(frozen=True) class ToolExecutionResult: ok: bool stdout: str = "" result: str | None = None error: str | None = None def to_json(self, output_limit: int) -> str: obj = { "ok": self.ok, "stdout": self.stdout[:output_limit], "result": self.result[:output_limit] if self.result is not None else None, "error": self.error[:output_limit] if self.error is not None else None, } return json.dumps(obj, ensure_ascii=False) class _BoundedTextBuffer: """A minimal text stream that never retains more than ``limit`` chars.""" def __init__(self, limit: int) -> None: self.limit = max(0, limit) self._parts: list[str] = [] self._length = 0 def write(self, value: str) -> int: text = str(value) remaining = self.limit - self._length if remaining > 0: chunk = text[:remaining] self._parts.append(chunk) self._length += len(chunk) return len(text) def flush(self) -> None: return None def getvalue(self) -> str: return "".join(self._parts) def _bounded_repr(value: Any, output_limit: int) -> str: formatter = reprlib.Repr() formatter.maxlevel = 4 formatter.maxdict = 20 formatter.maxlist = 20 formatter.maxtuple = 20 formatter.maxset = 20 formatter.maxfrozenset = 20 formatter.maxdeque = 20 formatter.maxarray = 20 formatter.maxstring = output_limit formatter.maxlong = output_limit formatter.maxother = output_limit return formatter.repr(value)[:output_limit] def _configure_worker(memory_limit_mb: int) -> None: """Install worker-only signal and address-space limits.""" signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) import resource memory_limit_bytes = memory_limit_mb * 1024 * 1024 resource.setrlimit(resource.RLIMIT_AS, (memory_limit_bytes, memory_limit_bytes)) def _send_worker_result(connection: Connection, result: ToolExecutionResult, output_limit: int) -> None: try: connection.send( { "ok": result.ok, "stdout": result.stdout[:output_limit], "result": result.result[:output_limit] if result.result is not None else None, "error": result.error[:output_limit] if result.error is not None else None, } ) except (BrokenPipeError, EOFError, OSError): pass finally: connection.close() def strip_code_fence(code: str) -> str: code = (code or "").strip() code = re.sub(r"^\s*```(?:python)?\s*", "", code, flags=re.IGNORECASE) code = re.sub(r"\s*```\s*$", "", code) return code.strip() def validate_code(code: str) -> None: tree = ast.parse(code, mode="exec") for node in ast.walk(tree): if isinstance(node, DISALLOWED_NODES): raise ValueError(f"Disallowed Python syntax: {node.__class__.__name__}") if isinstance(node, ast.Import): for alias in node.names: root = alias.name.split(".", 1)[0] if alias.name not in ALLOWED_IMPORTS or root not in ALLOWED_IMPORTS: raise ValueError(f"Disallowed import: {alias.name}") if alias.asname and (alias.asname in DISALLOWED_NAMES or alias.asname.startswith("__")): raise ValueError(f"Disallowed import alias: {alias.asname}") if isinstance(node, ast.ImportFrom): module = node.module or "" if node.level != 0 or module not in ALLOWED_IMPORTS: raise ValueError(f"Disallowed import: {module}") for alias in node.names: if alias.name == "*" or alias.name.startswith("__"): raise ValueError(f"Disallowed import name: {alias.name}") if alias.asname and (alias.asname in DISALLOWED_NAMES or alias.asname.startswith("__")): raise ValueError(f"Disallowed import alias: {alias.asname}") if isinstance(node, ast.Name): if node.id in DISALLOWED_NAMES or node.id.startswith("__"): raise ValueError(f"Disallowed name: {node.id}") if isinstance(node, ast.Attribute) and node.attr.startswith("__"): raise ValueError(f"Disallowed attribute: {node.attr}") def _safe_worker( code: str, context: dict[str, Any], connection: Connection, output_limit: int, memory_limit_mb: int, ) -> None: import collections import itertools import math import re as re_module import statistics try: _configure_worker(memory_limit_mb) except Exception: _send_worker_result( connection, ToolExecutionResult( ok=False, error="failed to configure Python tool process limits:\n" + traceback.format_exc(limit=4), ), output_limit, ) return safe_modules = { "collections": collections, "itertools": itertools, "json": json, "math": math, "re": re_module, "statistics": statistics, } def restricted_import(name, globals=None, locals=None, fromlist=(), level=0): if level != 0: raise ImportError("relative imports are not allowed") if name not in safe_modules: raise ImportError(f"import of {name!r} is not allowed") return safe_modules[name] safe_builtins = { "abs": abs, "all": all, "any": any, "bool": bool, "dict": dict, "enumerate": enumerate, "filter": filter, "float": float, "int": int, "isinstance": isinstance, "len": len, "list": list, "map": map, "max": max, "min": min, "print": print, "range": range, "repr": repr, "reversed": reversed, "round": round, "set": set, "sorted": sorted, "str": str, "sum": sum, "tuple": tuple, "zip": zip, "__import__": restricted_import, } namespace = { "__builtins__": safe_builtins, "collections": collections, "itertools": itertools, "json": json, "math": math, "re": re_module, "statistics": statistics, } namespace.update(context) stdout = _BoundedTextBuffer(output_limit) try: validate_code(code) compiled = compile(code, "", "exec") with contextlib.redirect_stdout(stdout): exec(compiled, namespace, namespace) result = None if "result" in namespace: result = _bounded_repr(namespace["result"], output_limit) payload = ToolExecutionResult(ok=True, stdout=stdout.getvalue(), result=result) except MemoryError: payload = ToolExecutionResult( ok=False, stdout=stdout.getvalue(), error=f"memory limit exceeded ({memory_limit_mb} MiB)", ) except Exception: payload = ToolExecutionResult( ok=False, stdout=stdout.getvalue(), error=traceback.format_exc(limit=4), ) _send_worker_result(connection, payload, output_limit) def _terminate_process(process: mp.Process, grace_seconds: float) -> str: """Stop and reap a worker, escalating from SIGTERM to SIGKILL.""" process.terminate() process.join(grace_seconds) if not process.is_alive(): return "SIGTERM" process.kill() process.join(grace_seconds) if process.is_alive(): raise RuntimeError(f"failed to reap Python tool process pid={process.pid} after SIGKILL") return "SIGKILL" class RestrictedPythonExecutor: def __init__( self, *, timeout_seconds: float = 3.0, output_limit: int = 4000, memory_limit_mb: int = 512, termination_grace_seconds: float = 1.0, ) -> None: if timeout_seconds <= 0: raise ValueError("timeout_seconds must be positive") if output_limit <= 0: raise ValueError("output_limit must be positive") if type(memory_limit_mb) is not int or memory_limit_mb <= 0: raise ValueError("memory_limit_mb must be a positive integer") if termination_grace_seconds <= 0: raise ValueError("termination_grace_seconds must be positive") self.timeout_seconds = timeout_seconds self.output_limit = output_limit self.memory_limit_mb = memory_limit_mb self.termination_grace_seconds = termination_grace_seconds def run(self, code: str, context: dict[str, Any]) -> str: code = strip_code_fence(code) if not code: return ToolExecutionResult(ok=False, error="empty code").to_json(self.output_limit) if len(code) > 12000: return ToolExecutionResult(ok=False, error="code is too long").to_json(self.output_limit) ctx = mp.get_context("spawn") receive_connection, send_connection = ctx.Pipe(duplex=False) process = ctx.Process( target=_safe_worker, args=(code, context, send_connection, self.output_limit, self.memory_limit_mb), ) try: process.start() send_connection.close() process.join(self.timeout_seconds) if process.is_alive(): signal_used = _terminate_process(process, self.termination_grace_seconds) return ToolExecutionResult( ok=False, error=f"execution timed out after {self.timeout_seconds}s; worker reaped with {signal_used}", ).to_json(self.output_limit) if receive_connection.poll(0.1): data = receive_connection.recv() return ToolExecutionResult(**data).to_json(self.output_limit) if process.exitcode is not None and process.exitcode < 0: signal_name = signal.Signals(-process.exitcode).name return ToolExecutionResult( ok=False, error=f"execution terminated by signal {signal_name}", ).to_json(self.output_limit) return ToolExecutionResult( ok=False, error=f"execution failed without returning a result (exitcode={process.exitcode})", ).to_json(self.output_limit) finally: send_connection.close() receive_connection.close() if process.is_alive(): _terminate_process(process, self.termination_grace_seconds) if process.exitcode is not None: process.close()