""" ALLI Code Generation Contract Specification -- Phase 7, Step 7.1 & Step 7.2 Standardizes the schema-driven DAG-to-Python synthesis pipeline. Provides Pydantic models, validation utilities, literal formatters, dependency rules, and deterministic SHA-256 artifact fingerprinting. """ import ast import hashlib import json import keyword import math import re from typing import Dict, Any, List, Optional, Set, Tuple from pydantic import BaseModel, Field # ─── 1. DEPENDENCY RULES & BANNED MODULES ───────────────────────────────────── # Standard library modules that must NEVER appear in requirements.txt STANDARD_LIBS: Set[str] = { "abc", "argparse", "array", "ast", "asyncio", "base64", "binascii", "bisect", "builtins", "calendar", "cmath", "collections", "colorsys", "concurrent", "contextlib", "contextvars", "copy", "csv", "ctypes", "datetime", "decimal", "difflib", "dis", "doctest", "email", "enum", "errno", "faulthandler", "fcntl", "filecmp", "fileinput", "fnmatch", "fractions", "functools", "gc", "getopt", "getpass", "gettext", "glob", "graphlib", "gzip", "hashlib", "heapq", "hmac", "html", "http", "imaplib", "imghdr", "importlib", "inspect", "io", "ipaddress", "itertools", "json", "keyword", "linecache", "locale", "logging", "lzma", "mailbox", "mailcap", "marshal", "math", "mimetypes", "mmap", "modulefinder", "msilib", "msvcrt", "multiprocessing", "netrc", "nntplib", "numbers", "operator", "optparse", "os", "pathlib", "pdb", "pickle", "pickletools", "pkgutil", "platform", "plistlib", "poplib", "posix", "pprint", "profile", "pstats", "pty", "pwd", "py_compile", "pyclbr", "pydoc", "queue", "quopri", "random", "re", "readline", "reprlib", "resource", "rlcompleter", "sched", "secrets", "select", "selectors", "shelve", "shlex", "shutil", "signal", "site", "smtpd", "smtplib", "sndhdr", "socket", "socketserver", "spwd", "sqlite3", "ssl", "stat", "statistics", "string", "stringprep", "struct", "subprocess", "sunau", "symtable", "sys", "sysconfig", "syslog", "tarfile", "telnetlib", "tempfile", "termios", "test", "textwrap", "threading", "time", "timeit", "tkinter", "token", "tokenize", "tomllib", "trace", "traceback", "tracemalloc", "tty", "turtle", "turtledemo", "types", "typing", "unicodedata", "unittest", "urllib", "uu", "uuid", "venv", "warnings", "wave", "weakref", "webbrowser", "winreg", "winsound", "wsgiref", "xdrlib", "xml", "xmlrpc", "zipapp", "zipfile", "zipimport", "zlib", "zoneinfo" } # Internal ALLI namespaces that must NEVER be imported as Python modules BANNED_IMPORTS: Set[str] = { "core", "control", "data", "primitive", "function", "system", "language_core", "lambdas", "library_inspector", "database", "api", "workflow", "trigger" } # Allowlisted standard library modules safe for user-selected lang.import & staged execution. SAFE_RUNTIME_STDLIB_MODULES: Set[str] = { "math", "statistics", "json", "datetime", "re", "random", "itertools", "functools", "collections", "decimal", "fractions", "uuid", "hashlib", "base64", "urllib.parse", "string", "csv", } # Restricted host/OS modules blocked from lang.import, staged binding, and dotted fallback. RESTRICTED_HOST_MODULES: Set[str] = { "os", "sys", "subprocess", "socket", "shutil", "importlib", "builtins", "ctypes", "pickle", "marshal", "pty", "pathlib", "posix", } | BANNED_IMPORTS # Authoritative pinned packages for external dependencies PINNED_PACKAGE_VERSIONS: Dict[str, str] = { "requests": "2.31.0", "urllib3": "2.2.1", "pandas": "2.2.2", "numpy": "1.26.4", "scipy": "1.13.0", "pydantic": "2.7.0", "httpx": "0.27.0", "fastapi": "0.110.0", "langchain": "0.1.0", "openai": "1.0.0", "torch": "2.2.2", "scikit-learn": "1.4.2", "matplotlib": "3.8.4", "sympy": "1.12", } # Dangerous callables and modules blocked by AST safety inspection (Phase 7 Gate) DANGEROUS_CALLABLES: Set[str] = { "eval", "exec", "compile", "__import__", "globals", "locals", "vars", "dir", "breakpoint", } DANGEROUS_ATTRIBUTES: Set[Tuple[str, str]] = { ("os", "system"), ("os", "popen"), ("os", "spawn"), ("os", "spawnl"), ("subprocess", "Popen"), ("subprocess", "call"), ("subprocess", "run"), ("subprocess", "check_output"), ("subprocess", "check_call"), ("shutil", "rmtree"), ("posix", "system"), ("pty", "spawn") } DANGEROUS_SPECIAL_ATTRS: Set[str] = { "__builtins__", "__globals__", "__subclasses__", "__bases__", "__class__", "__dict__", "__code__", "__closure__", "__mro__", "__qualname__", "gi_frame", "f_globals", "f_locals", "f_builtins", "f_code", "tb_frame", } DANGEROUS_IMPORTS: Set[str] = { "ctypes", "pickle", "socket", "code", "pdb", "subprocess", "shutil", "importlib", "pty", "posix", "marshal", "builtins", "sqlite3", "inspect", "multiprocessing", "threading", "asyncio", "webbrowser", "signal", } | BANNED_IMPORTS # ─── 2. CONTRACT MODELS ─────────────────────────────────────────────────────── class CodegenDiagnostic(BaseModel): """Structured diagnostic reported during code generation preflight or synthesis.""" code: str = Field(..., description="Canonical diagnostic code (e.g. 'DAG-001', 'CODEGEN-UNSUPPORTED-NODE')") message: str = Field(..., description="Human-readable description of the condition") severity: str = Field(default="error", description="Diagnostic severity: 'error', 'warning', or 'info'") node_id: Optional[str] = Field(default=None, description="Graph node identifier") port_id: Optional[str] = Field(default=None, description="Input or output port handle identifier") fix_suggestion: Optional[str] = Field(default=None, description="Remediation guidance") class SourceMapEntry(BaseModel): """Maps a generated Python statement range back to a canvas node.""" node_id: str = Field(..., description="Unique graph node ID") line_start: int = Field(..., description="1-based starting line of generated statement") line_end: int = Field(..., description="1-based ending line of generated statement") target_var: Optional[str] = Field(default=None, description="Variable assigned by this step") statement_label: Optional[str] = Field(default=None, description="Human readable node label") class CodegenRequest(BaseModel): """Payload contract for Python code generation.""" nodes: List[Dict[str, Any]] = Field(default_factory=list, description="Visual graph nodes") edges: List[Dict[str, Any]] = Field(default_factory=list, description="Visual graph edge connections") graph: Optional[Dict[str, Any]] = Field(default=None, description="Alternative graph bundle payload") name: Optional[str] = Field(default="alli_workflow", description="Workflow/file project name") execution_order: Optional[List[str]] = Field(default=None, description="Topological execution order override") dependencies: Optional[List[str]] = Field(default_factory=list, description="Explicit package dependencies") pin_exact: bool = Field(default=True, description="Enforce exact pinned versions (==) in requirements.txt") class CodegenArtifacts(BaseModel): """Comprehensive generated artifacts returned by the backend codegen engine.""" success: bool = Field(..., description="Whether code generation succeeded without blocking errors") python_code: str = Field(..., description="PEP 8 compliant, standalone Python source code") requirements_txt: str = Field(..., description="Pinned pip requirements.txt file contents") dockerfile: str = Field(..., description="Production Dockerfile container specification") filename: str = Field(..., description="Target Python filename (e.g. 'workflow.py')") dependencies: List[str] = Field(default_factory=list, description="Discovered external pip package names") execution_order: List[str] = Field(default_factory=list, description="Topological execution order of nodes") diagnostics: List[Dict[str, Any]] = Field(default_factory=list, description="List of diagnostics/errors") source_map: List[Dict[str, Any]] = Field(default_factory=list, description="Source mapping entries to nodes") generator_version: str = Field(default="1.0.0", description="Authoritative generator version") graph_fingerprint: Optional[str] = Field(default=None, description="64-char SHA-256 graph fingerprint") metadata: Dict[str, Any] = Field(default_factory=dict, description="Metadata including SHA-256 hashes") # ─── 3. SANITIZATION & LITERAL FORMATTING UTILITIES ─────────────────────────── def sanitize_identifier(name: Any, fallback: str = "var") -> str: """ Sanitizes user-provided or node-derived identifier into a valid Python variable name. Preserves alphanumeric characters and underscores, avoids keywords and numeric prefixes. """ if not name or not isinstance(name, str): return fallback # Replace invalid characters with underscore cleaned = re.sub(r"[^a-zA-Z0-9_]", "_", name.strip()) # Prepend underscore if identifier starts with a digit if re.match(r"^[0-9]", cleaned): cleaned = f"_{cleaned}" # Avoid Python reserved keywords (e.g. 'def', 'class', 'import', 'for') if keyword.iskeyword(cleaned): cleaned = f"{cleaned}_val" # Avoid builtins conflict if cleaned in ("print", "input", "sum", "min", "max", "id", "type", "format", "open"): cleaned = f"{cleaned}_val" return cleaned if cleaned else fallback def format_literal(val: Any, data_type: str = "any") -> str: """ Formats a raw value into a safe, deterministic Python literal string. Ensures safe escaping for strings, quotes, booleans, None, and collections. Rejects or safely handles non-finite numbers (inf, nan). """ if val is None: return "None" dt = str(data_type).lower().strip() # 1. Explicit integer type if dt in ("int", "integer"): try: return str(int(val)) except (ValueError, TypeError): return "0" # 2. Explicit float/number type if dt in ("float", "number"): try: f = float(val) if math.isnan(f) or math.isinf(f): raise ValueError("Non-finite float values are not supported in code generation") # Preserve float decimal format s = str(f) return s if "." in s or "e" in s.lower() else f"{s}.0" except (ValueError, TypeError) as e: if "Non-finite" in str(e): raise return "0.0" # 3. Explicit boolean type if dt in ("bool", "boolean"): if isinstance(val, str): return "True" if val.strip().lower() in ("true", "1", "yes") else "False" return "True" if bool(val) else "False" # 4. Explicit string/text type (Deterministic JSON quoting handles quotes, newlines, Unicode) if dt in ("str", "string", "text"): return json.dumps(str(val), ensure_ascii=False) # 5. Collections: List if dt == "list": if isinstance(val, list): formatted_elements = [format_literal(elem) for elem in val] return f"[{', '.join(formatted_elements)}]" if isinstance(val, str): s = val.strip() if s.startswith("[") and s.endswith("]"): return s return "[]" # 6. Collections: Dict if dt == "dict": if isinstance(val, dict): formatted_entries = [ f"{json.dumps(str(k), ensure_ascii=False)}: {format_literal(v)}" for k, v in val.items() ] return f"{{{', '.join(formatted_entries)}}}" if isinstance(val, str): s = val.strip() if s.startswith("{") and s.endswith("}"): return s return "{}" # 7. Type-inferred formatting fallback if isinstance(val, bool): return "True" if val else "False" if isinstance(val, int): return str(val) if isinstance(val, float): if math.isnan(val) or math.isinf(val): raise ValueError("Non-finite float values are not supported in code generation") s = str(val) return s if "." in s or "e" in s.lower() else f"{s}.0" if isinstance(val, list): formatted_elements = [format_literal(elem) for elem in val] return f"[{', '.join(formatted_elements)}]" if isinstance(val, dict): formatted_entries = [ f"{json.dumps(str(k), ensure_ascii=False)}: {format_literal(v)}" for k, v in val.items() ] return f"{{{', '.join(formatted_entries)}}}" # String heuristic if isinstance(val, str): s = val.strip() if s.lower() == "true": return "True" if s.lower() == "false": return "False" if s.lower() in ("none", "null"): return "None" # Check numeric literal strings try: if "." in s: f = float(s) if not (math.isnan(f) or math.isinf(f)): return s else: int(s) return s except ValueError: pass return json.dumps(val, ensure_ascii=False) return repr(val) # ─── 4. HASHING & AST VALIDATION WITH SECURITY INSPECTION ────────────────── def compute_codegen_hash(artifacts: Dict[str, Any]) -> str: """ Computes a deterministic 64-character SHA-256 hex digest for generated artifacts. Ensures identical code generations produce identical hashes. """ canon = { "python_code": artifacts.get("python_code", ""), "requirements_txt": artifacts.get("requirements_txt", ""), "dockerfile": artifacts.get("dockerfile", ""), } encoded = json.dumps(canon, sort_keys=True, separators=(",", ":")).encode("utf-8") return hashlib.sha256(encoded).hexdigest() RESTRICTED_HOST_MODULE_ATTRS: Set[str] = { "os", "sys", "subprocess", "shutil", "importlib", "ctypes", "pickle", "marshal", "pty", "builtins", "posix", } from language_core.workspace_env_contracts import validate_env_key _BLOCKED_PATH_PREFIXES: Tuple[str, ...] = ("/etc", "/proc", "/sys", "/dev", "/root", "/home") _BLOCKED_PATH_SEGMENTS: Tuple[str, ...] = (".git", ".ssh", ".aws", ".gnupg") _BLOCKED_FILENAMES: Tuple[str, ...] = (".env", "id_rsa", "id_ed25519", "authorized_keys") def _is_unsafe_literal_path(raw_path: str) -> bool: norm = raw_path.replace("\\", "/").strip() if "\x00" in norm or norm.startswith("//") or norm.startswith("~"): return True parts = [pt for pt in norm.split("/") if pt] lower_parts = [pt.lower() for pt in parts] if ".." in parts: return True if any(lp in _BLOCKED_PATH_SEGMENTS or lp in _BLOCKED_FILENAMES or lp.startswith(".env.") for lp in lower_parts): return True lowered = norm.lower() if any(lowered == pf or lowered.startswith(pf + "/") for pf in _BLOCKED_PATH_PREFIXES): return True return len(norm) >= 2 and norm[1] == ":" and norm[0].isalpha() class SecurityASTVisitor(ast.NodeVisitor): """ Static Abstract Syntax Tree security visitor. Inspects synthesized code for dangerous callables (eval, exec, __import__), arbitrary subprocess/system invocations, restricted paths, and introspection tampering. """ def __init__(self): self.diagnostics: List[Dict[str, Any]] = [] self._allowed_os_environ_attrs: Set[int] = set() def visit_Assign(self, node: ast.Assign): for target in node.targets: if ( isinstance(target, ast.Subscript) and isinstance(target.value, ast.Attribute) and isinstance(target.value.value, ast.Name) and target.value.value.id == "os" and target.value.attr == "environ" ): if isinstance(target.slice, ast.Constant) and isinstance(target.slice.value, str): ok_k, code_k, err_k = validate_env_key(target.slice.value) if ok_k: self._allowed_os_environ_attrs.add(id(target.value)) else: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Restricted environment key '{target.slice.value}' in os.environ assignment: {err_k} (line {node.lineno}).", "fix_suggestion": f"Avoid writing reserved environment variable ({code_k}).", }) self.generic_visit(node) def visit_Call(self, node: ast.Call): if isinstance(node.func, ast.Name): if node.func.id in DANGEROUS_CALLABLES: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Dangerous callable '{node.func.id}()' is prohibited in generated code (line {node.lineno}).", "fix_suggestion": "Remove unsafe dynamic evaluation or execution calls.", }) elif node.func.id in ("getattr", "setattr", "delattr"): if len(node.args) >= 2: if not isinstance(node.args[1], ast.Constant) or not isinstance(node.args[1].value, str): self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Dynamic reflection via '{node.func.id}()' with non-literal attribute is prohibited in generated code (line {node.lineno}).", "fix_suggestion": "Use explicit static attribute access.", }) elif node.args[1].value.startswith("__") or node.args[1].value in DANGEROUS_SPECIAL_ATTRS: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Dunder reflection via '{node.func.id}()' is prohibited in generated code (line {node.lineno}).", "fix_suggestion": "Remove dunder attribute introspection calls.", }) elif node.func.id == "open" and node.args and isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str): if _is_unsafe_literal_path(node.args[0].value): self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Restricted or traversal file path in open() is prohibited (line {node.lineno}).", "fix_suggestion": "Use a relative filename inside the working directory.", }) elif isinstance(node.func, ast.Attribute): attr_name = node.func.attr if isinstance(node.func.value, ast.Name): mod_name = node.func.value.id if (mod_name, attr_name) in DANGEROUS_ATTRIBUTES: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Dangerous method '{mod_name}.{attr_name}()' is prohibited in generated code (line {node.lineno}).", "fix_suggestion": "Remove arbitrary system command execution or shell invocations.", }) elif mod_name == "os" and attr_name == "getenv" and node.args and isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str): ok_k, code_k, err_k = validate_env_key(node.args[0].value) if not ok_k: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Restricted environment key '{node.args[0].value}' in os.getenv(): {err_k} (line {node.lineno}).", "fix_suggestion": f"Use an application env key instead of reserved/invalid key ({code_k}).", }) self.generic_visit(node) def visit_Attribute(self, node: ast.Attribute): if node.attr in RESTRICTED_HOST_MODULE_ATTRS: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Restricted host module attribute '.{node.attr}' is prohibited (line {node.lineno}).", "fix_suggestion": "Remove access to host OS or runtime reflection modules.", }) if ( isinstance(node.value, ast.Name) and node.value.id == "os" and node.attr not in {"getenv", "path", "makedirs"} and not (node.attr == "environ" and id(node) in self._allowed_os_environ_attrs) ): self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Restricted OS access 'os.{node.attr}' is prohibited (line {node.lineno}).", "fix_suggestion": "Remove restricted os attributes.", }) if isinstance(node.value, ast.Name) and node.value.id == "sys" and node.attr != "stdin": self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Restricted sys access 'sys.{node.attr}' is prohibited; only sys.stdin is allowed (line {node.lineno}).", "fix_suggestion": "Remove restricted sys attributes.", }) if isinstance(node.value, ast.Attribute) and (node.value.attr, node.attr) in DANGEROUS_ATTRIBUTES: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Chained dangerous attribute '{node.value.attr}.{node.attr}' is prohibited (line {node.lineno}).", "fix_suggestion": "Remove unsafe chained system execution calls.", }) if node.attr in DANGEROUS_SPECIAL_ATTRS: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Prohibited access to internal introspection attribute '{node.attr}' (line {node.lineno}).", "fix_suggestion": "Avoid dunder attribute reflection exploits.", }) self.generic_visit(node) def visit_Import(self, node: ast.Import): for alias in node.names: root_mod = alias.name.split(".")[0] if root_mod in ("os", "sys") and alias.asname: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Aliased import of restricted module '{alias.name}' is prohibited (line {node.lineno}).", "fix_suggestion": "Remove aliased import of 'os'.", }) if root_mod in DANGEROUS_IMPORTS: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Import of restricted or dangerous module '{alias.name}' is prohibited (line {node.lineno}).", "fix_suggestion": f"Remove import of '{alias.name}'.", }) self.generic_visit(node) def visit_ImportFrom(self, node: ast.ImportFrom): if node.module: root_mod = node.module.split(".")[0] if root_mod in ("os", "sys"): self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Import from restricted module '{node.module}' is prohibited (line {node.lineno}).", "fix_suggestion": f"Remove import from '{node.module}'.", }) if root_mod in DANGEROUS_IMPORTS: self.diagnostics.append({ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"Import from restricted or dangerous module '{node.module}' is prohibited (line {node.lineno}).", "fix_suggestion": f"Remove import from '{node.module}'.", }) self.generic_visit(node) def inspect_generated_ast_security(python_code: str) -> List[Dict[str, Any]]: """ Inspects synthesized Python AST for security violations (banned modules, dangerous callables, introspection tampering). Returns list of CodegenDiagnostic dictionary objects. """ try: tree = ast.parse(python_code) visitor = SecurityASTVisitor() visitor.visit(tree) return visitor.diagnostics except SyntaxError as e: return [{ "code": "SYNTAX-ERROR", "severity": "error", "message": f"Syntax error in synthesized code: {str(e)}", "fix_suggestion": "Ensure generated Python code has valid syntax.", }] except Exception as exc: return [{ "code": "CODEGEN-SECURITY-VIOLATION", "severity": "error", "message": f"AST parsing failed: {str(exc)}", "fix_suggestion": "Verify code validity.", }] def validate_generated_ast(python_code: str, check_security: bool = True) -> Tuple[bool, Optional[str]]: """ Validates that synthesized Python code compiles into a valid Abstract Syntax Tree and passes security AST inspection (rejecting eval, exec, os.system, etc.). Never executes the code; only inspects AST structures statically. Returns (True, None) if valid and safe, or (False, error_description) if invalid or unsafe. """ try: tree = ast.parse(python_code) if check_security: visitor = SecurityASTVisitor() visitor.visit(tree) if visitor.diagnostics: first_diag = visitor.diagnostics[0] return False, f"[{first_diag['code']}] {first_diag['message']}" return True, None except SyntaxError as e: return False, f"SyntaxError at line {e.lineno}, col {e.offset}: {e.msg}" except Exception as e: return False, f"AST Parse Error: {str(e)}"