rsnarsna
fix(codegen): remediate phase 7 review findings with exact package pinning, ast safety visitor, control-flow parity, and input disambiguation
9c77d16 Download tests/test_step2_4_codegen_standard.py from Rsnarsna/auto: direct link, hf CLI and curl.
- Browser
- Download file 12.6 kB
-
https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/tests/test_step2_4_codegen_standard.py
- Command line
-
hf download hf://spaces/Rsnarsna/auto/tests/test_step2_4_codegen_standard.py
-
curl -L -o test_step2_4_codegen_standard.py https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/tests/test_step2_4_codegen_standard.py
12.6 kB
| """ | |
| ALLI Step 2.4: Standardize Python Code-Generation Mappings & Parity Test Suite. | |
| Validates: | |
| 1. Standard code-generation contract for all registered catalog nodes. | |
| 2. AST syntax validity (ast.parse on every generated script). | |
| 3. Deterministic execution parity: Standalone Python execution produces identical observable | |
| stdout and return values as the visual workflow executor. | |
| 4. Dependency & Import Isolation: | |
| - Standard libraries (math, json, sys, etc.) are imported but excluded from requirements.txt. | |
| - External dependencies (requests, numpy) are pinned in requirements.txt. | |
| - Internal schema namespaces (core, data, control, io, function) are NEVER imported. | |
| 5. Runtime Type Safety: | |
| - Dynamic sources (Any / Object) connecting incompatible values to typed ports throw TypeError before execution. | |
| 6. Type system parity: Numeric widening (int -> float), nullable values, nested collections, and union types. | |
| """ | |
| import ast | |
| import sys | |
| import subprocess | |
| import pytest | |
| from typing import Dict, Any, List | |
| from language_core.node_schema import default_registry | |
| from lambdas.codegen_handler import generate_code_logic, STANDARD_LIBS | |
| from language_core.workflow_executor import execute_workflow_graph | |
| # --------------------------------------------------------------------------- | |
| # 1. Registered Nodes Schema-to-Code AST Validation | |
| # --------------------------------------------------------------------------- | |
| def test_all_registered_nodes_codegen_contract(): | |
| """Verifies that every node registered in default_registry has a codegen definition.""" | |
| nodes = default_registry.list_all() | |
| assert len(nodes) >= 20, f"Expected at least 20 registered nodes, got {len(nodes)}" | |
| for node in nodes: | |
| assert node.codegen is not None, f"Node '{node.type}' must have a codegen definition" | |
| assert node.codegen.template is not None, f"Node '{node.type}' must have a codegen template" | |
| def test_no_internal_namespaces_in_generated_imports(): | |
| """Verifies that internal ALLI categories ('core', 'control', 'data', 'io', etc.) are never imported.""" | |
| nodes = [ | |
| {"id": "n1", "node_type": "core.constant", "data": {"const_name": "val", "literal_value": 10}}, | |
| {"id": "n2", "node_type": "core.type_cast", "data": {"target_type": "float"}}, | |
| {"id": "n3", "node_type": "core.compare", "data": {"operator": "=="}}, | |
| {"id": "n4", "node_type": "data.list", "data": {"initial_items": "[1, 2]"}}, | |
| {"id": "n5", "node_type": "control.if_else", "data": {}}, | |
| {"id": "n6", "node_type": "io.print", "data": {}}, | |
| ] | |
| edges = [ | |
| {"source": "n1", "target": "n2", "sourceHandle": "out", "targetHandle": "value"}, | |
| ] | |
| res = generate_code_logic(nodes=nodes, edges=edges, graph_name="Namespace Test") | |
| code = res["python_code"] | |
| for internal in ("core", "control", "data", "io", "function", "primitive"): | |
| assert f"import {internal}" not in code, f"Internal namespace '{internal}' was incorrectly imported" | |
| # --------------------------------------------------------------------------- | |
| # 2. Execution Parity Across Fundamentals | |
| # --------------------------------------------------------------------------- | |
| def _assert_parity(graph: Dict[str, Any], expected_stdout: str): | |
| """Helper to verify 100% parity between visual execution and standalone Python.""" | |
| # 1. Run visual workflow executor | |
| vis_res = execute_workflow_graph(graph) | |
| assert vis_res["success"] is True, f"Visual execution failed: {vis_res.get('error')}" | |
| vis_out = vis_res["stdout"].strip() | |
| assert vis_out == expected_stdout, f"Visual stdout mismatch: expected '{expected_stdout}', got '{vis_out}'" | |
| # 2. Generate Python code | |
| cg_res = generate_code_logic( | |
| nodes=graph["nodes"], | |
| edges=graph.get("edges", []), | |
| graph_name=graph.get("name", "Parity Test") | |
| ) | |
| assert cg_res["success"] is True | |
| code = cg_res["python_code"] | |
| # 3. Verify valid AST | |
| tree = ast.parse(code) | |
| assert isinstance(tree, ast.Module), "Generated code must parse into valid AST" | |
| # 4. Run standalone Python via subprocess | |
| sub_res = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) | |
| assert sub_res.returncode == 0, f"Standalone execution failed:\n{sub_res.stderr}\nCode:\n{code}" | |
| sub_out = sub_res.stdout.strip() | |
| # 5. Assert exact observable parity | |
| assert sub_out == vis_out, f"Parity mismatch: visual='{vis_out}' vs standalone='{sub_out}'" | |
| def test_parity_math_sqrt_numeric_widening(): | |
| """49 (int) -> math.sqrt -> 7.0 (float widening parity).""" | |
| graph = { | |
| "name": "Sqrt Parity", | |
| "nodes": [ | |
| {"id": "n_in", "node_type": "constant.int", "data": {"value": 49}}, | |
| {"id": "n_sqrt", "node_type": "math.sqrt", "data": {"precision": 2}}, | |
| {"id": "n_out", "node_type": "io.print", "data": {"prefix": "Sqrt: "}}, | |
| ], | |
| "edges": [ | |
| {"source": "n_in", "target": "n_sqrt", "sourceHandle": "out", "targetHandle": "x"}, | |
| {"source": "n_sqrt", "target": "n_out", "sourceHandle": "result", "targetHandle": "value"}, | |
| ] | |
| } | |
| _assert_parity(graph, "Sqrt: 7.0") | |
| def test_parity_chained_arithmetic(): | |
| """(10 + 20) * 3 == 90.""" | |
| graph = { | |
| "name": "Arithmetic Chaining", | |
| "nodes": [ | |
| {"id": "n1", "node_type": "constant.int", "data": {"value": 10}}, | |
| {"id": "n2", "node_type": "constant.int", "data": {"value": 20}}, | |
| {"id": "n_add", "node_type": "core.binary_op", "data": {"operator": "+"}}, | |
| {"id": "n3", "node_type": "constant.int", "data": {"value": 3}}, | |
| {"id": "n_mul", "node_type": "core.binary_op", "data": {"operator": "*"}}, | |
| {"id": "n_print", "node_type": "io.print", "data": {}}, | |
| ], | |
| "edges": [ | |
| {"source": "n1", "target": "n_add", "sourceHandle": "out", "targetHandle": "a"}, | |
| {"source": "n2", "target": "n_add", "sourceHandle": "out", "targetHandle": "b"}, | |
| {"source": "n_add", "target": "n_mul", "sourceHandle": "result", "targetHandle": "a"}, | |
| {"source": "n3", "target": "n_mul", "sourceHandle": "out", "targetHandle": "b"}, | |
| {"source": "n_mul", "target": "n_print", "sourceHandle": "result", "targetHandle": "value"}, | |
| ] | |
| } | |
| _assert_parity(graph, "90") | |
| def test_parity_comparison_operator(): | |
| """10 < 20 -> True.""" | |
| graph = { | |
| "name": "Comparison Flow", | |
| "nodes": [ | |
| {"id": "n_a", "node_type": "constant.int", "data": {"value": 10}}, | |
| {"id": "n_b", "node_type": "constant.int", "data": {"value": 20}}, | |
| {"id": "n_cmp", "node_type": "core.compare", "data": {"operator": "<"}}, | |
| {"id": "n_print", "node_type": "io.print", "data": {"prefix": "Is Less: "}}, | |
| ], | |
| "edges": [ | |
| {"source": "n_a", "target": "n_cmp", "sourceHandle": "out", "targetHandle": "a"}, | |
| {"source": "n_b", "target": "n_cmp", "sourceHandle": "out", "targetHandle": "b"}, | |
| {"source": "n_cmp", "target": "n_print", "sourceHandle": "result", "targetHandle": "value"}, | |
| ] | |
| } | |
| _assert_parity(graph, "Is Less: True") | |
| def test_parity_type_cast(): | |
| """float 42.9 cast to int -> 42.""" | |
| graph = { | |
| "name": "Cast Flow", | |
| "nodes": [ | |
| {"id": "n_flt", "node_type": "constant.float", "data": {"value": 42.9}}, | |
| {"id": "n_cast", "node_type": "core.type_cast", "data": {"target_type": "int"}}, | |
| {"id": "n_print", "node_type": "io.print", "data": {}}, | |
| ], | |
| "edges": [ | |
| {"source": "n_flt", "target": "n_cast", "sourceHandle": "out", "targetHandle": "value"}, | |
| {"source": "n_cast", "target": "n_print", "sourceHandle": "result", "targetHandle": "value"}, | |
| ] | |
| } | |
| _assert_parity(graph, "42") | |
| def test_parity_conditional_if_else(): | |
| """Condition True -> 'Access Granted'.""" | |
| graph = { | |
| "name": "Condition Flow", | |
| "nodes": [ | |
| {"id": "n_c", "node_type": "constant.bool", "data": {"value": True}}, | |
| {"id": "n_t", "node_type": "constant.str", "data": {"value": "Access Granted"}}, | |
| {"id": "n_e", "node_type": "constant.str", "data": {"value": "Access Denied"}}, | |
| {"id": "n_if", "node_type": "control.if_else", "data": {}}, | |
| {"id": "n_print", "node_type": "io.print", "data": {"prefix": "Result: "}}, | |
| ], | |
| "edges": [ | |
| {"source": "n_c", "target": "n_if", "sourceHandle": "out", "targetHandle": "condition"}, | |
| {"source": "n_t", "target": "n_if", "sourceHandle": "out", "targetHandle": "then_val"}, | |
| {"source": "n_e", "target": "n_if", "sourceHandle": "out", "targetHandle": "else_val"}, | |
| {"source": "n_if", "target": "n_print", "sourceHandle": "result", "targetHandle": "value"}, | |
| ] | |
| } | |
| _assert_parity(graph, "Result: Access Granted") | |
| def test_parity_data_list_collection(): | |
| """List collection parity.""" | |
| graph = { | |
| "name": "List Flow", | |
| "nodes": [ | |
| {"id": "n_list", "node_type": "data.list", "data": {"initial_items": "[10, 20, 30]"}}, | |
| {"id": "n_print", "node_type": "io.print", "data": {}}, | |
| ], | |
| "edges": [ | |
| {"source": "n_list", "target": "n_print", "sourceHandle": "result", "targetHandle": "value"}, | |
| ] | |
| } | |
| _assert_parity(graph, "[10, 20, 30]") | |
| def test_parity_data_dict_collection(): | |
| """Dict collection parity.""" | |
| graph = { | |
| "name": "Dict Flow", | |
| "nodes": [ | |
| {"id": "n_dict", "node_type": "data.dict", "data": {"initial_dict": "{'key': 'value'}"}}, | |
| {"id": "n_print", "node_type": "io.print", "data": {}}, | |
| ], | |
| "edges": [ | |
| {"source": "n_dict", "target": "n_print", "sourceHandle": "dict_out", "targetHandle": "value"}, | |
| ] | |
| } | |
| _assert_parity(graph, "{'key': 'value'}") | |
| # --------------------------------------------------------------------------- | |
| # 3. Dynamic Source (Any/Object) Runtime Validation | |
| # --------------------------------------------------------------------------- | |
| def test_dynamic_source_incompatible_value_runtime_rejection(): | |
| """ | |
| Verifies that a dynamic source (Any / Object) attempting to pass an incompatible value | |
| into a typed port (e.g. string 'invalid' into math.sqrt 'x' float) is caught by | |
| runtime validation before execution, preventing unsafe bypass. | |
| """ | |
| # 1. Directly on registry execution | |
| with pytest.raises(TypeError) as excinfo: | |
| default_registry.execute("math.sqrt", {"x": "invalid_string"}, {"precision": 2}) | |
| assert "Type mismatch on input port 'x'" in str(excinfo.value) | |
| # 2. In connected workflow execution | |
| bad_graph = { | |
| "name": "Type Mismatch Safety", | |
| "nodes": [ | |
| {"id": "n_any", "node_type": "core.constant", "data": {"const_name": "x_val", "literal_value": "not_a_number"}}, | |
| {"id": "n_sqrt", "node_type": "math.sqrt", "data": {}}, | |
| ], | |
| "edges": [ | |
| {"source": "n_any", "target": "n_sqrt", "sourceHandle": "out", "targetHandle": "x"}, | |
| ] | |
| } | |
| res = execute_workflow_graph(bad_graph) | |
| assert res["success"] is False | |
| assert res["status"] in ("failed", "partial_failure") | |
| # Target node must be recorded as failed with structured error semantics | |
| sqrt_res = res["node_results"].get("n_sqrt") | |
| assert sqrt_res is not None | |
| assert sqrt_res["status"] == "failed" | |
| # --------------------------------------------------------------------------- | |
| # 4. Dependency Isolation & Containerization Contract | |
| # --------------------------------------------------------------------------- | |
| def test_dependency_isolation_and_dockerfile(): | |
| """Verifies that standard libraries are excluded from requirements.txt and Dockerfile is valid.""" | |
| res = generate_code_logic( | |
| nodes=[{"id": "n1", "node_type": "math.sqrt", "data": {"x": 16}}], | |
| edges=[], | |
| dependencies=["requests", "math", "json", "pandas"], | |
| graph_name="Pipeline Spec" | |
| ) | |
| reqs = res["requirements_txt"] | |
| assert "requests==2.31.0" in reqs or "requests>=0.1.0" in reqs | |
| assert "pandas==2.2.2" in reqs or "pandas>=0.1.0" in reqs | |
| assert "math" not in reqs | |
| assert "json" not in reqs | |
| dockerfile = res["dockerfile"] | |
| assert "FROM python:3.12-slim" in dockerfile | |
| assert "COPY requirements.txt ." in dockerfile | |
| assert "RUN pip install --no-cache-dir -r requirements.txt" in dockerfile | |
| assert 'CMD ["python", "pipeline_spec.py"]' in dockerfile | |