auto / language_core /codegen_contracts.py
rsnarsna
feat(security,codegen): Wave 2-5 node standardization, SSRF & AST hardening (DEC-015)
a724795
Raw History Blame Contribute Delete
27 kB
"""
ALLI Code Generation Contract Specification -- Phase 7, Step 7.1 & Step 7.2
Standardizes the schema-driven DAG-to-Python synthesis pipeline.
Provides Pydantic models, validation utilities, literal formatters,
dependency rules, and deterministic SHA-256 artifact fingerprinting.
"""
import ast
import hashlib
import json
import keyword
import math
import re
from typing import Dict, Any, List, Optional, Set, Tuple
from pydantic import BaseModel, Field
# ─── 1. DEPENDENCY RULES & BANNED MODULES ─────────────────────────────────────
# Standard library modules that must NEVER appear in requirements.txt
STANDARD_LIBS: Set[str] = {
"abc", "argparse", "array", "ast", "asyncio", "base64", "binascii", "bisect",
"builtins", "calendar", "cmath", "collections", "colorsys", "concurrent",
"contextlib", "contextvars", "copy", "csv", "ctypes", "datetime", "decimal",
"difflib", "dis", "doctest", "email", "enum", "errno", "faulthandler",
"fcntl", "filecmp", "fileinput", "fnmatch", "fractions", "functools",
"gc", "getopt", "getpass", "gettext", "glob", "graphlib", "gzip", "hashlib",
"heapq", "hmac", "html", "http", "imaplib", "imghdr", "importlib", "inspect",
"io", "ipaddress", "itertools", "json", "keyword", "linecache", "locale",
"logging", "lzma", "mailbox", "mailcap", "marshal", "math", "mimetypes",
"mmap", "modulefinder", "msilib", "msvcrt", "multiprocessing", "netrc",
"nntplib", "numbers", "operator", "optparse", "os", "pathlib", "pdb",
"pickle", "pickletools", "pkgutil", "platform", "plistlib", "poplib",
"posix", "pprint", "profile", "pstats", "pty", "pwd", "py_compile",
"pyclbr", "pydoc", "queue", "quopri", "random", "re", "readline",
"reprlib", "resource", "rlcompleter", "sched", "secrets", "select",
"selectors", "shelve", "shlex", "shutil", "signal", "site", "smtpd",
"smtplib", "sndhdr", "socket", "socketserver", "spwd", "sqlite3", "ssl",
"stat", "statistics", "string", "stringprep", "struct", "subprocess",
"sunau", "symtable", "sys", "sysconfig", "syslog", "tarfile", "telnetlib",
"tempfile", "termios", "test", "textwrap", "threading", "time", "timeit",
"tkinter", "token", "tokenize", "tomllib", "trace", "traceback", "tracemalloc",
"tty", "turtle", "turtledemo", "types", "typing", "unicodedata", "unittest",
"urllib", "uu", "uuid", "venv", "warnings", "wave", "weakref", "webbrowser",
"winreg", "winsound", "wsgiref", "xdrlib", "xml", "xmlrpc", "zipapp",
"zipfile", "zipimport", "zlib", "zoneinfo"
}
# Internal ALLI namespaces that must NEVER be imported as Python modules
BANNED_IMPORTS: Set[str] = {
"core", "control", "data", "primitive", "function", "system",
"language_core", "lambdas", "library_inspector", "database", "api", "workflow", "trigger"
}
# Allowlisted standard library modules safe for user-selected lang.import & staged execution.
SAFE_RUNTIME_STDLIB_MODULES: Set[str] = {
"math", "statistics", "json", "datetime", "re", "random", "itertools",
"functools", "collections", "decimal", "fractions", "uuid", "hashlib",
"base64", "urllib.parse", "string", "csv",
}
# Restricted host/OS modules blocked from lang.import, staged binding, and dotted fallback.
RESTRICTED_HOST_MODULES: Set[str] = {
"os", "sys", "subprocess", "socket", "shutil", "importlib",
"builtins", "ctypes", "pickle", "marshal", "pty", "pathlib", "posix",
} | BANNED_IMPORTS
# Authoritative pinned packages for external dependencies
PINNED_PACKAGE_VERSIONS: Dict[str, str] = {
"requests": "2.31.0",
"urllib3": "2.2.1",
"pandas": "2.2.2",
"numpy": "1.26.4",
"scipy": "1.13.0",
"pydantic": "2.7.0",
"httpx": "0.27.0",
"fastapi": "0.110.0",
"langchain": "0.1.0",
"openai": "1.0.0",
"torch": "2.2.2",
"scikit-learn": "1.4.2",
"matplotlib": "3.8.4",
"sympy": "1.12",
}
# Dangerous callables and modules blocked by AST safety inspection (Phase 7 Gate)
DANGEROUS_CALLABLES: Set[str] = {
"eval", "exec", "compile", "__import__", "globals", "locals",
"vars", "dir", "breakpoint",
}
DANGEROUS_ATTRIBUTES: Set[Tuple[str, str]] = {
("os", "system"), ("os", "popen"), ("os", "spawn"), ("os", "spawnl"),
("subprocess", "Popen"), ("subprocess", "call"), ("subprocess", "run"),
("subprocess", "check_output"), ("subprocess", "check_call"),
("shutil", "rmtree"), ("posix", "system"), ("pty", "spawn")
}
DANGEROUS_SPECIAL_ATTRS: Set[str] = {
"__builtins__", "__globals__", "__subclasses__", "__bases__", "__class__",
"__dict__", "__code__", "__closure__", "__mro__", "__qualname__",
"gi_frame", "f_globals", "f_locals", "f_builtins", "f_code", "tb_frame",
}
DANGEROUS_IMPORTS: Set[str] = {
"ctypes", "pickle", "socket", "code", "pdb",
"subprocess", "shutil", "importlib", "pty", "posix", "marshal", "builtins",
"sqlite3", "inspect", "multiprocessing", "threading", "asyncio", "webbrowser", "signal",
} | BANNED_IMPORTS
# ─── 2. CONTRACT MODELS ───────────────────────────────────────────────────────
class CodegenDiagnostic(BaseModel):
"""Structured diagnostic reported during code generation preflight or synthesis."""
code: str = Field(..., description="Canonical diagnostic code (e.g. 'DAG-001', 'CODEGEN-UNSUPPORTED-NODE')")
message: str = Field(..., description="Human-readable description of the condition")
severity: str = Field(default="error", description="Diagnostic severity: 'error', 'warning', or 'info'")
node_id: Optional[str] = Field(default=None, description="Graph node identifier")
port_id: Optional[str] = Field(default=None, description="Input or output port handle identifier")
fix_suggestion: Optional[str] = Field(default=None, description="Remediation guidance")
class SourceMapEntry(BaseModel):
"""Maps a generated Python statement range back to a canvas node."""
node_id: str = Field(..., description="Unique graph node ID")
line_start: int = Field(..., description="1-based starting line of generated statement")
line_end: int = Field(..., description="1-based ending line of generated statement")
target_var: Optional[str] = Field(default=None, description="Variable assigned by this step")
statement_label: Optional[str] = Field(default=None, description="Human readable node label")
class CodegenRequest(BaseModel):
"""Payload contract for Python code generation."""
nodes: List[Dict[str, Any]] = Field(default_factory=list, description="Visual graph nodes")
edges: List[Dict[str, Any]] = Field(default_factory=list, description="Visual graph edge connections")
graph: Optional[Dict[str, Any]] = Field(default=None, description="Alternative graph bundle payload")
name: Optional[str] = Field(default="alli_workflow", description="Workflow/file project name")
execution_order: Optional[List[str]] = Field(default=None, description="Topological execution order override")
dependencies: Optional[List[str]] = Field(default_factory=list, description="Explicit package dependencies")
pin_exact: bool = Field(default=True, description="Enforce exact pinned versions (==) in requirements.txt")
class CodegenArtifacts(BaseModel):
"""Comprehensive generated artifacts returned by the backend codegen engine."""
success: bool = Field(..., description="Whether code generation succeeded without blocking errors")
python_code: str = Field(..., description="PEP 8 compliant, standalone Python source code")
requirements_txt: str = Field(..., description="Pinned pip requirements.txt file contents")
dockerfile: str = Field(..., description="Production Dockerfile container specification")
filename: str = Field(..., description="Target Python filename (e.g. 'workflow.py')")
dependencies: List[str] = Field(default_factory=list, description="Discovered external pip package names")
execution_order: List[str] = Field(default_factory=list, description="Topological execution order of nodes")
diagnostics: List[Dict[str, Any]] = Field(default_factory=list, description="List of diagnostics/errors")
source_map: List[Dict[str, Any]] = Field(default_factory=list, description="Source mapping entries to nodes")
generator_version: str = Field(default="1.0.0", description="Authoritative generator version")
graph_fingerprint: Optional[str] = Field(default=None, description="64-char SHA-256 graph fingerprint")
metadata: Dict[str, Any] = Field(default_factory=dict, description="Metadata including SHA-256 hashes")
# ─── 3. SANITIZATION & LITERAL FORMATTING UTILITIES ───────────────────────────
def sanitize_identifier(name: Any, fallback: str = "var") -> str:
"""
Sanitizes user-provided or node-derived identifier into a valid Python variable name.
Preserves alphanumeric characters and underscores, avoids keywords and numeric prefixes.
"""
if not name or not isinstance(name, str):
return fallback
# Replace invalid characters with underscore
cleaned = re.sub(r"[^a-zA-Z0-9_]", "_", name.strip())
# Prepend underscore if identifier starts with a digit
if re.match(r"^[0-9]", cleaned):
cleaned = f"_{cleaned}"
# Avoid Python reserved keywords (e.g. 'def', 'class', 'import', 'for')
if keyword.iskeyword(cleaned):
cleaned = f"{cleaned}_val"
# Avoid builtins conflict
if cleaned in ("print", "input", "sum", "min", "max", "id", "type", "format", "open"):
cleaned = f"{cleaned}_val"
return cleaned if cleaned else fallback
def format_literal(val: Any, data_type: str = "any") -> str:
"""
Formats a raw value into a safe, deterministic Python literal string.
Ensures safe escaping for strings, quotes, booleans, None, and collections.
Rejects or safely handles non-finite numbers (inf, nan).
"""
if val is None:
return "None"
dt = str(data_type).lower().strip()
# 1. Explicit integer type
if dt in ("int", "integer"):
try:
return str(int(val))
except (ValueError, TypeError):
return "0"
# 2. Explicit float/number type
if dt in ("float", "number"):
try:
f = float(val)
if math.isnan(f) or math.isinf(f):
raise ValueError("Non-finite float values are not supported in code generation")
# Preserve float decimal format
s = str(f)
return s if "." in s or "e" in s.lower() else f"{s}.0"
except (ValueError, TypeError) as e:
if "Non-finite" in str(e):
raise
return "0.0"
# 3. Explicit boolean type
if dt in ("bool", "boolean"):
if isinstance(val, str):
return "True" if val.strip().lower() in ("true", "1", "yes") else "False"
return "True" if bool(val) else "False"
# 4. Explicit string/text type (Deterministic JSON quoting handles quotes, newlines, Unicode)
if dt in ("str", "string", "text"):
return json.dumps(str(val), ensure_ascii=False)
# 5. Collections: List
if dt == "list":
if isinstance(val, list):
formatted_elements = [format_literal(elem) for elem in val]
return f"[{', '.join(formatted_elements)}]"
if isinstance(val, str):
s = val.strip()
if s.startswith("[") and s.endswith("]"):
return s
return "[]"
# 6. Collections: Dict
if dt == "dict":
if isinstance(val, dict):
formatted_entries = [
f"{json.dumps(str(k), ensure_ascii=False)}: {format_literal(v)}"
for k, v in val.items()
]
return f"{{{', '.join(formatted_entries)}}}"
if isinstance(val, str):
s = val.strip()
if s.startswith("{") and s.endswith("}"):
return s
return "{}"
# 7. Type-inferred formatting fallback
if isinstance(val, bool):
return "True" if val else "False"
if isinstance(val, int):
return str(val)
if isinstance(val, float):
if math.isnan(val) or math.isinf(val):
raise ValueError("Non-finite float values are not supported in code generation")
s = str(val)
return s if "." in s or "e" in s.lower() else f"{s}.0"
if isinstance(val, list):
formatted_elements = [format_literal(elem) for elem in val]
return f"[{', '.join(formatted_elements)}]"
if isinstance(val, dict):
formatted_entries = [
f"{json.dumps(str(k), ensure_ascii=False)}: {format_literal(v)}"
for k, v in val.items()
]
return f"{{{', '.join(formatted_entries)}}}"
# String heuristic
if isinstance(val, str):
s = val.strip()
if s.lower() == "true":
return "True"
if s.lower() == "false":
return "False"
if s.lower() in ("none", "null"):
return "None"
# Check numeric literal strings
try:
if "." in s:
f = float(s)
if not (math.isnan(f) or math.isinf(f)):
return s
else:
int(s)
return s
except ValueError:
pass
return json.dumps(val, ensure_ascii=False)
return repr(val)
# ─── 4. HASHING & AST VALIDATION WITH SECURITY INSPECTION ──────────────────
def compute_codegen_hash(artifacts: Dict[str, Any]) -> str:
"""
Computes a deterministic 64-character SHA-256 hex digest for generated artifacts.
Ensures identical code generations produce identical hashes.
"""
canon = {
"python_code": artifacts.get("python_code", ""),
"requirements_txt": artifacts.get("requirements_txt", ""),
"dockerfile": artifacts.get("dockerfile", ""),
}
encoded = json.dumps(canon, sort_keys=True, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
RESTRICTED_HOST_MODULE_ATTRS: Set[str] = {
"os", "sys", "subprocess", "shutil", "importlib",
"ctypes", "pickle", "marshal", "pty", "builtins", "posix",
}
from language_core.workspace_env_contracts import validate_env_key
_BLOCKED_PATH_PREFIXES: Tuple[str, ...] = ("/etc", "/proc", "/sys", "/dev", "/root", "/home")
_BLOCKED_PATH_SEGMENTS: Tuple[str, ...] = (".git", ".ssh", ".aws", ".gnupg")
_BLOCKED_FILENAMES: Tuple[str, ...] = (".env", "id_rsa", "id_ed25519", "authorized_keys")
def _is_unsafe_literal_path(raw_path: str) -> bool:
norm = raw_path.replace("\\", "/").strip()
if "\x00" in norm or norm.startswith("//") or norm.startswith("~"):
return True
parts = [pt for pt in norm.split("/") if pt]
lower_parts = [pt.lower() for pt in parts]
if ".." in parts:
return True
if any(lp in _BLOCKED_PATH_SEGMENTS or lp in _BLOCKED_FILENAMES or lp.startswith(".env.") for lp in lower_parts):
return True
lowered = norm.lower()
if any(lowered == pf or lowered.startswith(pf + "/") for pf in _BLOCKED_PATH_PREFIXES):
return True
return len(norm) >= 2 and norm[1] == ":" and norm[0].isalpha()
class SecurityASTVisitor(ast.NodeVisitor):
"""
Static Abstract Syntax Tree security visitor.
Inspects synthesized code for dangerous callables (eval, exec, __import__),
arbitrary subprocess/system invocations, restricted paths, and introspection tampering.
"""
def __init__(self):
self.diagnostics: List[Dict[str, Any]] = []
self._allowed_os_environ_attrs: Set[int] = set()
def visit_Assign(self, node: ast.Assign):
for target in node.targets:
if (
isinstance(target, ast.Subscript)
and isinstance(target.value, ast.Attribute)
and isinstance(target.value.value, ast.Name)
and target.value.value.id == "os"
and target.value.attr == "environ"
):
if isinstance(target.slice, ast.Constant) and isinstance(target.slice.value, str):
ok_k, code_k, err_k = validate_env_key(target.slice.value)
if ok_k:
self._allowed_os_environ_attrs.add(id(target.value))
else:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Restricted environment key '{target.slice.value}' in os.environ assignment: {err_k} (line {node.lineno}).",
"fix_suggestion": f"Avoid writing reserved environment variable ({code_k}).",
})
self.generic_visit(node)
def visit_Call(self, node: ast.Call):
if isinstance(node.func, ast.Name):
if node.func.id in DANGEROUS_CALLABLES:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Dangerous callable '{node.func.id}()' is prohibited in generated code (line {node.lineno}).",
"fix_suggestion": "Remove unsafe dynamic evaluation or execution calls.",
})
elif node.func.id in ("getattr", "setattr", "delattr"):
if len(node.args) >= 2:
if not isinstance(node.args[1], ast.Constant) or not isinstance(node.args[1].value, str):
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Dynamic reflection via '{node.func.id}()' with non-literal attribute is prohibited in generated code (line {node.lineno}).",
"fix_suggestion": "Use explicit static attribute access.",
})
elif node.args[1].value.startswith("__") or node.args[1].value in DANGEROUS_SPECIAL_ATTRS:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Dunder reflection via '{node.func.id}()' is prohibited in generated code (line {node.lineno}).",
"fix_suggestion": "Remove dunder attribute introspection calls.",
})
elif node.func.id == "open" and node.args and isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str):
if _is_unsafe_literal_path(node.args[0].value):
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Restricted or traversal file path in open() is prohibited (line {node.lineno}).",
"fix_suggestion": "Use a relative filename inside the working directory.",
})
elif isinstance(node.func, ast.Attribute):
attr_name = node.func.attr
if isinstance(node.func.value, ast.Name):
mod_name = node.func.value.id
if (mod_name, attr_name) in DANGEROUS_ATTRIBUTES:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Dangerous method '{mod_name}.{attr_name}()' is prohibited in generated code (line {node.lineno}).",
"fix_suggestion": "Remove arbitrary system command execution or shell invocations.",
})
elif mod_name == "os" and attr_name == "getenv" and node.args and isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str):
ok_k, code_k, err_k = validate_env_key(node.args[0].value)
if not ok_k:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Restricted environment key '{node.args[0].value}' in os.getenv(): {err_k} (line {node.lineno}).",
"fix_suggestion": f"Use an application env key instead of reserved/invalid key ({code_k}).",
})
self.generic_visit(node)
def visit_Attribute(self, node: ast.Attribute):
if node.attr in RESTRICTED_HOST_MODULE_ATTRS:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Restricted host module attribute '.{node.attr}' is prohibited (line {node.lineno}).",
"fix_suggestion": "Remove access to host OS or runtime reflection modules.",
})
if (
isinstance(node.value, ast.Name)
and node.value.id == "os"
and node.attr not in {"getenv", "path", "makedirs"}
and not (node.attr == "environ" and id(node) in self._allowed_os_environ_attrs)
):
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Restricted OS access 'os.{node.attr}' is prohibited (line {node.lineno}).",
"fix_suggestion": "Remove restricted os attributes.",
})
if isinstance(node.value, ast.Name) and node.value.id == "sys" and node.attr != "stdin":
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Restricted sys access 'sys.{node.attr}' is prohibited; only sys.stdin is allowed (line {node.lineno}).",
"fix_suggestion": "Remove restricted sys attributes.",
})
if isinstance(node.value, ast.Attribute) and (node.value.attr, node.attr) in DANGEROUS_ATTRIBUTES:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Chained dangerous attribute '{node.value.attr}.{node.attr}' is prohibited (line {node.lineno}).",
"fix_suggestion": "Remove unsafe chained system execution calls.",
})
if node.attr in DANGEROUS_SPECIAL_ATTRS:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Prohibited access to internal introspection attribute '{node.attr}' (line {node.lineno}).",
"fix_suggestion": "Avoid dunder attribute reflection exploits.",
})
self.generic_visit(node)
def visit_Import(self, node: ast.Import):
for alias in node.names:
root_mod = alias.name.split(".")[0]
if root_mod in ("os", "sys") and alias.asname:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Aliased import of restricted module '{alias.name}' is prohibited (line {node.lineno}).",
"fix_suggestion": "Remove aliased import of 'os'.",
})
if root_mod in DANGEROUS_IMPORTS:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Import of restricted or dangerous module '{alias.name}' is prohibited (line {node.lineno}).",
"fix_suggestion": f"Remove import of '{alias.name}'.",
})
self.generic_visit(node)
def visit_ImportFrom(self, node: ast.ImportFrom):
if node.module:
root_mod = node.module.split(".")[0]
if root_mod in ("os", "sys"):
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Import from restricted module '{node.module}' is prohibited (line {node.lineno}).",
"fix_suggestion": f"Remove import from '{node.module}'.",
})
if root_mod in DANGEROUS_IMPORTS:
self.diagnostics.append({
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"Import from restricted or dangerous module '{node.module}' is prohibited (line {node.lineno}).",
"fix_suggestion": f"Remove import from '{node.module}'.",
})
self.generic_visit(node)
def inspect_generated_ast_security(python_code: str) -> List[Dict[str, Any]]:
"""
Inspects synthesized Python AST for security violations (banned modules, dangerous callables,
introspection tampering). Returns list of CodegenDiagnostic dictionary objects.
"""
try:
tree = ast.parse(python_code)
visitor = SecurityASTVisitor()
visitor.visit(tree)
return visitor.diagnostics
except SyntaxError as e:
return [{
"code": "SYNTAX-ERROR",
"severity": "error",
"message": f"Syntax error in synthesized code: {str(e)}",
"fix_suggestion": "Ensure generated Python code has valid syntax.",
}]
except Exception as exc:
return [{
"code": "CODEGEN-SECURITY-VIOLATION",
"severity": "error",
"message": f"AST parsing failed: {str(exc)}",
"fix_suggestion": "Verify code validity.",
}]
def validate_generated_ast(python_code: str, check_security: bool = True) -> Tuple[bool, Optional[str]]:
"""
Validates that synthesized Python code compiles into a valid Abstract Syntax Tree
and passes security AST inspection (rejecting eval, exec, os.system, etc.).
Never executes the code; only inspects AST structures statically.
Returns (True, None) if valid and safe, or (False, error_description) if invalid or unsafe.
"""
try:
tree = ast.parse(python_code)
if check_security:
visitor = SecurityASTVisitor()
visitor.visit(tree)
if visitor.diagnostics:
first_diag = visitor.diagnostics[0]
return False, f"[{first_diag['code']}] {first_diag['message']}"
return True, None
except SyntaxError as e:
return False, f"SyntaxError at line {e.lineno}, col {e.offset}: {e.msg}"
except Exception as e:
return False, f"AST Parse Error: {str(e)}"