auto / tests /test_step2_4_codegen_standard.py
rsnarsna
fix(codegen): remediate phase 7 review findings with exact package pinning, ast safety visitor, control-flow parity, and input disambiguation
9c77d16
Raw History Blame Contribute Delete
12.6 kB
"""
ALLI Step 2.4: Standardize Python Code-Generation Mappings & Parity Test Suite.
Validates:
1. Standard code-generation contract for all registered catalog nodes.
2. AST syntax validity (ast.parse on every generated script).
3. Deterministic execution parity: Standalone Python execution produces identical observable
stdout and return values as the visual workflow executor.
4. Dependency & Import Isolation:
- Standard libraries (math, json, sys, etc.) are imported but excluded from requirements.txt.
- External dependencies (requests, numpy) are pinned in requirements.txt.
- Internal schema namespaces (core, data, control, io, function) are NEVER imported.
5. Runtime Type Safety:
- Dynamic sources (Any / Object) connecting incompatible values to typed ports throw TypeError before execution.
6. Type system parity: Numeric widening (int -> float), nullable values, nested collections, and union types.
"""
import ast
import sys
import subprocess
import pytest
from typing import Dict, Any, List
from language_core.node_schema import default_registry
from lambdas.codegen_handler import generate_code_logic, STANDARD_LIBS
from language_core.workflow_executor import execute_workflow_graph
# ---------------------------------------------------------------------------
# 1. Registered Nodes Schema-to-Code AST Validation
# ---------------------------------------------------------------------------
def test_all_registered_nodes_codegen_contract():
"""Verifies that every node registered in default_registry has a codegen definition."""
nodes = default_registry.list_all()
assert len(nodes) >= 20, f"Expected at least 20 registered nodes, got {len(nodes)}"
for node in nodes:
assert node.codegen is not None, f"Node '{node.type}' must have a codegen definition"
assert node.codegen.template is not None, f"Node '{node.type}' must have a codegen template"
def test_no_internal_namespaces_in_generated_imports():
"""Verifies that internal ALLI categories ('core', 'control', 'data', 'io', etc.) are never imported."""
nodes = [
{"id": "n1", "node_type": "core.constant", "data": {"const_name": "val", "literal_value": 10}},
{"id": "n2", "node_type": "core.type_cast", "data": {"target_type": "float"}},
{"id": "n3", "node_type": "core.compare", "data": {"operator": "=="}},
{"id": "n4", "node_type": "data.list", "data": {"initial_items": "[1, 2]"}},
{"id": "n5", "node_type": "control.if_else", "data": {}},
{"id": "n6", "node_type": "io.print", "data": {}},
]
edges = [
{"source": "n1", "target": "n2", "sourceHandle": "out", "targetHandle": "value"},
]
res = generate_code_logic(nodes=nodes, edges=edges, graph_name="Namespace Test")
code = res["python_code"]
for internal in ("core", "control", "data", "io", "function", "primitive"):
assert f"import {internal}" not in code, f"Internal namespace '{internal}' was incorrectly imported"
# ---------------------------------------------------------------------------
# 2. Execution Parity Across Fundamentals
# ---------------------------------------------------------------------------
def _assert_parity(graph: Dict[str, Any], expected_stdout: str):
"""Helper to verify 100% parity between visual execution and standalone Python."""
# 1. Run visual workflow executor
vis_res = execute_workflow_graph(graph)
assert vis_res["success"] is True, f"Visual execution failed: {vis_res.get('error')}"
vis_out = vis_res["stdout"].strip()
assert vis_out == expected_stdout, f"Visual stdout mismatch: expected '{expected_stdout}', got '{vis_out}'"
# 2. Generate Python code
cg_res = generate_code_logic(
nodes=graph["nodes"],
edges=graph.get("edges", []),
graph_name=graph.get("name", "Parity Test")
)
assert cg_res["success"] is True
code = cg_res["python_code"]
# 3. Verify valid AST
tree = ast.parse(code)
assert isinstance(tree, ast.Module), "Generated code must parse into valid AST"
# 4. Run standalone Python via subprocess
sub_res = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
assert sub_res.returncode == 0, f"Standalone execution failed:\n{sub_res.stderr}\nCode:\n{code}"
sub_out = sub_res.stdout.strip()
# 5. Assert exact observable parity
assert sub_out == vis_out, f"Parity mismatch: visual='{vis_out}' vs standalone='{sub_out}'"
def test_parity_math_sqrt_numeric_widening():
"""49 (int) -> math.sqrt -> 7.0 (float widening parity)."""
graph = {
"name": "Sqrt Parity",
"nodes": [
{"id": "n_in", "node_type": "constant.int", "data": {"value": 49}},
{"id": "n_sqrt", "node_type": "math.sqrt", "data": {"precision": 2}},
{"id": "n_out", "node_type": "io.print", "data": {"prefix": "Sqrt: "}},
],
"edges": [
{"source": "n_in", "target": "n_sqrt", "sourceHandle": "out", "targetHandle": "x"},
{"source": "n_sqrt", "target": "n_out", "sourceHandle": "result", "targetHandle": "value"},
]
}
_assert_parity(graph, "Sqrt: 7.0")
def test_parity_chained_arithmetic():
"""(10 + 20) * 3 == 90."""
graph = {
"name": "Arithmetic Chaining",
"nodes": [
{"id": "n1", "node_type": "constant.int", "data": {"value": 10}},
{"id": "n2", "node_type": "constant.int", "data": {"value": 20}},
{"id": "n_add", "node_type": "core.binary_op", "data": {"operator": "+"}},
{"id": "n3", "node_type": "constant.int", "data": {"value": 3}},
{"id": "n_mul", "node_type": "core.binary_op", "data": {"operator": "*"}},
{"id": "n_print", "node_type": "io.print", "data": {}},
],
"edges": [
{"source": "n1", "target": "n_add", "sourceHandle": "out", "targetHandle": "a"},
{"source": "n2", "target": "n_add", "sourceHandle": "out", "targetHandle": "b"},
{"source": "n_add", "target": "n_mul", "sourceHandle": "result", "targetHandle": "a"},
{"source": "n3", "target": "n_mul", "sourceHandle": "out", "targetHandle": "b"},
{"source": "n_mul", "target": "n_print", "sourceHandle": "result", "targetHandle": "value"},
]
}
_assert_parity(graph, "90")
def test_parity_comparison_operator():
"""10 < 20 -> True."""
graph = {
"name": "Comparison Flow",
"nodes": [
{"id": "n_a", "node_type": "constant.int", "data": {"value": 10}},
{"id": "n_b", "node_type": "constant.int", "data": {"value": 20}},
{"id": "n_cmp", "node_type": "core.compare", "data": {"operator": "<"}},
{"id": "n_print", "node_type": "io.print", "data": {"prefix": "Is Less: "}},
],
"edges": [
{"source": "n_a", "target": "n_cmp", "sourceHandle": "out", "targetHandle": "a"},
{"source": "n_b", "target": "n_cmp", "sourceHandle": "out", "targetHandle": "b"},
{"source": "n_cmp", "target": "n_print", "sourceHandle": "result", "targetHandle": "value"},
]
}
_assert_parity(graph, "Is Less: True")
def test_parity_type_cast():
"""float 42.9 cast to int -> 42."""
graph = {
"name": "Cast Flow",
"nodes": [
{"id": "n_flt", "node_type": "constant.float", "data": {"value": 42.9}},
{"id": "n_cast", "node_type": "core.type_cast", "data": {"target_type": "int"}},
{"id": "n_print", "node_type": "io.print", "data": {}},
],
"edges": [
{"source": "n_flt", "target": "n_cast", "sourceHandle": "out", "targetHandle": "value"},
{"source": "n_cast", "target": "n_print", "sourceHandle": "result", "targetHandle": "value"},
]
}
_assert_parity(graph, "42")
def test_parity_conditional_if_else():
"""Condition True -> 'Access Granted'."""
graph = {
"name": "Condition Flow",
"nodes": [
{"id": "n_c", "node_type": "constant.bool", "data": {"value": True}},
{"id": "n_t", "node_type": "constant.str", "data": {"value": "Access Granted"}},
{"id": "n_e", "node_type": "constant.str", "data": {"value": "Access Denied"}},
{"id": "n_if", "node_type": "control.if_else", "data": {}},
{"id": "n_print", "node_type": "io.print", "data": {"prefix": "Result: "}},
],
"edges": [
{"source": "n_c", "target": "n_if", "sourceHandle": "out", "targetHandle": "condition"},
{"source": "n_t", "target": "n_if", "sourceHandle": "out", "targetHandle": "then_val"},
{"source": "n_e", "target": "n_if", "sourceHandle": "out", "targetHandle": "else_val"},
{"source": "n_if", "target": "n_print", "sourceHandle": "result", "targetHandle": "value"},
]
}
_assert_parity(graph, "Result: Access Granted")
def test_parity_data_list_collection():
"""List collection parity."""
graph = {
"name": "List Flow",
"nodes": [
{"id": "n_list", "node_type": "data.list", "data": {"initial_items": "[10, 20, 30]"}},
{"id": "n_print", "node_type": "io.print", "data": {}},
],
"edges": [
{"source": "n_list", "target": "n_print", "sourceHandle": "result", "targetHandle": "value"},
]
}
_assert_parity(graph, "[10, 20, 30]")
def test_parity_data_dict_collection():
"""Dict collection parity."""
graph = {
"name": "Dict Flow",
"nodes": [
{"id": "n_dict", "node_type": "data.dict", "data": {"initial_dict": "{'key': 'value'}"}},
{"id": "n_print", "node_type": "io.print", "data": {}},
],
"edges": [
{"source": "n_dict", "target": "n_print", "sourceHandle": "dict_out", "targetHandle": "value"},
]
}
_assert_parity(graph, "{'key': 'value'}")
# ---------------------------------------------------------------------------
# 3. Dynamic Source (Any/Object) Runtime Validation
# ---------------------------------------------------------------------------
def test_dynamic_source_incompatible_value_runtime_rejection():
"""
Verifies that a dynamic source (Any / Object) attempting to pass an incompatible value
into a typed port (e.g. string 'invalid' into math.sqrt 'x' float) is caught by
runtime validation before execution, preventing unsafe bypass.
"""
# 1. Directly on registry execution
with pytest.raises(TypeError) as excinfo:
default_registry.execute("math.sqrt", {"x": "invalid_string"}, {"precision": 2})
assert "Type mismatch on input port 'x'" in str(excinfo.value)
# 2. In connected workflow execution
bad_graph = {
"name": "Type Mismatch Safety",
"nodes": [
{"id": "n_any", "node_type": "core.constant", "data": {"const_name": "x_val", "literal_value": "not_a_number"}},
{"id": "n_sqrt", "node_type": "math.sqrt", "data": {}},
],
"edges": [
{"source": "n_any", "target": "n_sqrt", "sourceHandle": "out", "targetHandle": "x"},
]
}
res = execute_workflow_graph(bad_graph)
assert res["success"] is False
assert res["status"] in ("failed", "partial_failure")
# Target node must be recorded as failed with structured error semantics
sqrt_res = res["node_results"].get("n_sqrt")
assert sqrt_res is not None
assert sqrt_res["status"] == "failed"
# ---------------------------------------------------------------------------
# 4. Dependency Isolation & Containerization Contract
# ---------------------------------------------------------------------------
def test_dependency_isolation_and_dockerfile():
"""Verifies that standard libraries are excluded from requirements.txt and Dockerfile is valid."""
res = generate_code_logic(
nodes=[{"id": "n1", "node_type": "math.sqrt", "data": {"x": 16}}],
edges=[],
dependencies=["requests", "math", "json", "pandas"],
graph_name="Pipeline Spec"
)
reqs = res["requirements_txt"]
assert "requests==2.31.0" in reqs or "requests>=0.1.0" in reqs
assert "pandas==2.2.2" in reqs or "pandas>=0.1.0" in reqs
assert "math" not in reqs
assert "json" not in reqs
dockerfile = res["dockerfile"]
assert "FROM python:3.12-slim" in dockerfile
assert "COPY requirements.txt ." in dockerfile
assert "RUN pip install --no-cache-dir -r requirements.txt" in dockerfile
assert 'CMD ["python", "pipeline_spec.py"]' in dockerfile