auto / scripts /verify_gate4_python_parity.py
rsnarsna
feat(phase-18): complete Gate 4 python parity and Gate 5 security audit verification
9326c18
Raw History Blame Contribute Delete
13.9 kB
"""
ALLI Phase 18: Gate 4 — Python Code Generation & Standalone Execution Parity
Verifies byte-for-byte stdout and numeric parity between visual workflow execution
and standalone Python scripts across all 5 Beginner Scenarios and standard topologies.
"""
import sys
import ast
import subprocess
from pathlib import Path
# Add backend root directory to sys.path
sys.path.insert(0, str(Path(__file__).parent.parent))
from lambdas.codegen_handler import generate_code_logic
from language_core.workflow_executor import execute_workflow_graph
def run_gate4_parity_verification():
print("=" * 80)
print("ALLI PHASE 18: GATE 4 — PYTHON CODE GENERATION & EXECUTION PARITY")
print("=" * 80)
passed = 0
total = 0
def assert_test(condition: bool, msg: str):
nonlocal passed, total
total += 1
if not condition:
print(f" [FAIL] {msg}")
raise AssertionError(f"Gate 4 Assertion Failed: {msg}")
passed += 1
print(f" [PASS] {msg}")
# =========================================================================
# SCENARIO 1: Add Two Numbers (10 + 20 = 30)
# =========================================================================
print("\n--- Scenario 1: Add Two Numbers (10 + 20 = 30) ---")
s1_nodes = [
{"id": "s1_a", "node_type": "constant.int", "data": {"label": "Number A", "value": 10}},
{"id": "s1_b", "node_type": "constant.int", "data": {"label": "Number B", "value": 20}},
{"id": "s1_add", "node_type": "core.binary_op", "data": {"label": "Add", "operator": "+"}},
{"id": "s1_print", "node_type": "io.print", "data": {"label": "Print Sum", "prefix": "Sum Result: "}},
]
s1_edges = [
{"source": "s1_a", "target": "s1_add", "sourceHandle": "val", "targetHandle": "a"},
{"source": "s1_b", "target": "s1_add", "sourceHandle": "val", "targetHandle": "b"},
{"source": "s1_add", "target": "s1_print", "sourceHandle": "result", "targetHandle": "value"},
]
s1_graph = {"name": "Scenario 1 Add", "nodes": s1_nodes, "edges": s1_edges}
# 1. Visual Execution
s1_vis = execute_workflow_graph(s1_graph)
assert_test(s1_vis["success"], "Visual execution for Scenario 1 succeeded")
assert_test(s1_vis["node_results"]["s1_add"]["outputs"]["result"] == 30, "Visual add result is 30")
assert_test(s1_vis["stdout"].strip() == "Sum Result: 30", "Visual stdout is 'Sum Result: 30'")
# 2. Codegen & Standalone Execution
s1_cg = generate_code_logic(nodes=s1_nodes, edges=s1_edges, graph_name="Scenario 1 Add")
assert_test(s1_cg["success"], "Scenario 1 code generation succeeded")
ast.parse(s1_cg["python_code"]) # Verify AST parse
s1_sub = subprocess.run([sys.executable, "-c", s1_cg["python_code"]], capture_output=True, text=True)
assert_test(s1_sub.returncode == 0, "Scenario 1 standalone Python process exited with code 0")
assert_test(
s1_sub.stdout.strip() == s1_vis["stdout"].strip(),
f"Scenario 1 Byte-for-Byte Parity Matched: visual={s1_vis['stdout'].strip()!r} vs standalone={s1_sub.stdout.strip()!r}"
)
# =========================================================================
# SCENARIO 2: Square Root (sqrt(49) = 7.0)
# =========================================================================
print("\n--- Scenario 2: Calculate Square Root (sqrt(49) = 7.0) ---")
s2_nodes = [
{"id": "s2_input", "node_type": "constant.float", "data": {"label": "Input Number", "value": 49.0}},
{"id": "s2_sqrt", "node_type": "math.sqrt", "data": {"label": "Square Root"}},
{"id": "s2_print", "node_type": "io.print", "data": {"label": "Print Output", "prefix": "Result: "}},
]
s2_edges = [
{"source": "s2_input", "target": "s2_sqrt", "sourceHandle": "val", "targetHandle": "x"},
{"source": "s2_sqrt", "target": "s2_print", "sourceHandle": "result", "targetHandle": "value"},
]
s2_graph = {"name": "Scenario 2 Sqrt", "nodes": s2_nodes, "edges": s2_edges}
s2_vis = execute_workflow_graph(s2_graph)
assert_test(s2_vis["success"], "Visual execution for Scenario 2 succeeded")
assert_test(s2_vis["node_results"]["s2_sqrt"]["outputs"]["result"] == 7.0, "Visual sqrt result is 7.0")
assert_test(s2_vis["stdout"].strip() == "Result: 7.0", "Visual stdout is 'Result: 7.0'")
s2_cg = generate_code_logic(nodes=s2_nodes, edges=s2_edges, graph_name="Scenario 2 Sqrt")
assert_test(s2_cg["success"], "Scenario 2 code generation succeeded")
ast.parse(s2_cg["python_code"])
s2_sub = subprocess.run([sys.executable, "-c", s2_cg["python_code"]], capture_output=True, text=True)
assert_test(s2_sub.returncode == 0, "Scenario 2 standalone Python process exited with code 0")
assert_test(
s2_sub.stdout.strip() == s2_vis["stdout"].strip(),
f"Scenario 2 Byte-for-Byte Parity Matched: visual={s2_vis['stdout'].strip()!r} vs standalone={s2_sub.stdout.strip()!r}"
)
# =========================================================================
# SCENARIO 3: Conditional Branching (Adult vs Minor)
# =========================================================================
print("\n--- Scenario 3: Conditional Branching Parity (Age 20 vs Age 15) ---")
# Pathway A: Age 20 -> Adult
s3a_nodes = [
{"id": "s3_cond", "node_type": "constant.bool", "data": {"label": "Condition True", "value": True}},
{"id": "s3_then", "node_type": "constant.str", "data": {"label": "Branch True", "value": "Adult"}},
{"id": "s3_else", "node_type": "constant.str", "data": {"label": "Branch False", "value": "Minor"}},
{"id": "s3_branch", "node_type": "control.if_else", "data": {"label": "Age Check"}},
{"id": "s3_print", "node_type": "io.print", "data": {"label": "Print Status", "prefix": "Access: "}},
]
s3a_edges = [
{"source": "s3_cond", "target": "s3_branch", "sourceHandle": "val", "targetHandle": "condition"},
{"source": "s3_then", "target": "s3_branch", "sourceHandle": "val", "targetHandle": "then_val"},
{"source": "s3_else", "target": "s3_branch", "sourceHandle": "val", "targetHandle": "else_val"},
{"source": "s3_branch", "target": "s3_print", "sourceHandle": "result", "targetHandle": "value"},
]
s3a_vis = execute_workflow_graph({"name": "Scenario 3 Adult", "nodes": s3a_nodes, "edges": s3a_edges})
assert_test(s3a_vis["success"], "Visual execution for Adult pathway succeeded")
assert_test(s3a_vis["stdout"].strip() == "Access: Adult", "Visual stdout is 'Access: Adult'")
s3a_cg = generate_code_logic(nodes=s3a_nodes, edges=s3a_edges, graph_name="Scenario 3 Adult")
s3a_sub = subprocess.run([sys.executable, "-c", s3a_cg["python_code"]], capture_output=True, text=True)
assert_test(s3a_sub.returncode == 0, "Adult standalone script succeeded")
assert_test(
s3a_sub.stdout.strip() == s3a_vis["stdout"].strip(),
f"Adult Pathway Parity Matched: {s3a_sub.stdout.strip()!r}"
)
# Pathway B: Age 15 -> Minor
s3b_nodes = [
{"id": "s3_cond", "node_type": "constant.bool", "data": {"label": "Condition False", "value": False}},
{"id": "s3_then", "node_type": "constant.str", "data": {"label": "Branch True", "value": "Adult"}},
{"id": "s3_else", "node_type": "constant.str", "data": {"label": "Branch False", "value": "Minor"}},
{"id": "s3_branch", "node_type": "control.if_else", "data": {"label": "Age Check"}},
{"id": "s3_print", "node_type": "io.print", "data": {"label": "Print Status", "prefix": "Access: "}},
]
s3b_vis = execute_workflow_graph({"name": "Scenario 3 Minor", "nodes": s3b_nodes, "edges": s3a_edges})
assert_test(s3b_vis["success"], "Visual execution for Minor pathway succeeded")
assert_test(s3b_vis["stdout"].strip() == "Access: Minor", "Visual stdout is 'Access: Minor'")
s3b_cg = generate_code_logic(nodes=s3b_nodes, edges=s3a_edges, graph_name="Scenario 3 Minor")
s3b_sub = subprocess.run([sys.executable, "-c", s3b_cg["python_code"]], capture_output=True, text=True)
assert_test(s3b_sub.returncode == 0, "Minor standalone script succeeded")
assert_test(
s3b_sub.stdout.strip() == s3b_vis["stdout"].strip(),
f"Minor Pathway Parity Matched: {s3b_sub.stdout.strip()!r}"
)
# =========================================================================
# SCENARIO 4: Value Reuse ((10 + 20) * 2 = 60)
# =========================================================================
print("\n--- Scenario 4: Value Reuse ((10 + 20) * 2 = 60) ---")
s4_nodes = [
{"id": "s4_a", "node_type": "constant.int", "data": {"label": "Base A", "value": 10}},
{"id": "s4_b", "node_type": "constant.int", "data": {"label": "Base B", "value": 20}},
{"id": "s4_add", "node_type": "core.binary_op", "data": {"label": "Add Sum", "operator": "+"}},
{"id": "s4_factor", "node_type": "constant.int", "data": {"label": "Factor 2", "value": 2}},
{"id": "s4_mul", "node_type": "core.binary_op", "data": {"label": "Multiply", "operator": "*"}},
{"id": "s4_print", "node_type": "io.print", "data": {"label": "Print Reused", "prefix": "Final Result: "}},
]
s4_edges = [
{"source": "s4_a", "target": "s4_add", "sourceHandle": "val", "targetHandle": "a"},
{"source": "s4_b", "target": "s4_add", "sourceHandle": "val", "targetHandle": "b"},
{"source": "s4_add", "target": "s4_mul", "sourceHandle": "result", "targetHandle": "a"},
{"source": "s4_factor", "target": "s4_mul", "sourceHandle": "val", "targetHandle": "b"},
{"source": "s4_mul", "target": "s4_print", "sourceHandle": "result", "targetHandle": "value"},
]
s4_graph = {"name": "Scenario 4 Reuse", "nodes": s4_nodes, "edges": s4_edges}
s4_vis = execute_workflow_graph(s4_graph)
assert_test(s4_vis["success"], "Visual execution for Scenario 4 succeeded")
assert_test(s4_vis["node_results"]["s4_mul"]["outputs"]["result"] == 60, "Multiplied output is 60")
assert_test(s4_vis["stdout"].strip() == "Final Result: 60", "Visual stdout is 'Final Result: 60'")
s4_cg = generate_code_logic(nodes=s4_nodes, edges=s4_edges, graph_name="Scenario 4 Reuse")
assert_test(s4_cg["success"], "Scenario 4 code generation succeeded")
ast.parse(s4_cg["python_code"])
s4_sub = subprocess.run([sys.executable, "-c", s4_cg["python_code"]], capture_output=True, text=True)
assert_test(s4_sub.returncode == 0, "Scenario 4 standalone Python process exited with code 0")
assert_test(
s4_sub.stdout.strip() == s4_vis["stdout"].strip(),
f"Scenario 4 Byte-for-Byte Parity Matched: visual={s4_vis['stdout'].strip()!r} vs standalone={s4_sub.stdout.strip()!r}"
)
# =========================================================================
# SCENARIO 5: Custom Message
# =========================================================================
print("\n--- Scenario 5: Custom Message ---")
s5_nodes = [
{"id": "s5_msg", "node_type": "core.variable", "data": {"label": "Message", "var_name": "greeting", "literal_value": "Hello, ALLI!"}},
{"id": "s5_print", "node_type": "io.print", "data": {"label": "Print Banner", "prefix": "[ALLI STUDIO]: "}},
]
s5_edges = [
{"source": "s5_msg", "target": "s5_print", "sourceHandle": "out_val", "targetHandle": "value"},
]
s5_graph = {"name": "Scenario 5 Custom Message", "nodes": s5_nodes, "edges": s5_edges}
s5_vis = execute_workflow_graph(s5_graph)
assert_test(s5_vis["success"], "Visual execution for Scenario 5 succeeded")
assert_test(s5_vis["stdout"].strip() == "[ALLI STUDIO]: Hello, ALLI!", "Visual stdout is '[ALLI STUDIO]: Hello, ALLI!'")
s5_cg = generate_code_logic(nodes=s5_nodes, edges=s5_edges, graph_name="Scenario 5 Custom Message")
assert_test(s5_cg["success"], "Scenario 5 code generation succeeded")
ast.parse(s5_cg["python_code"])
s5_sub = subprocess.run([sys.executable, "-c", s5_cg["python_code"]], capture_output=True, text=True)
assert_test(s5_sub.returncode == 0, "Scenario 5 standalone Python process exited with code 0")
assert_test(
s5_sub.stdout.strip() == s5_vis["stdout"].strip(),
f"Scenario 5 Byte-for-Byte Parity Matched: visual={s5_vis['stdout'].strip()!r} vs standalone={s5_sub.stdout.strip()!r}"
)
# =========================================================================
# DEPENDENCY ISOLATION & DOCKERFILE CONTRACT
# =========================================================================
print("\n--- Dependency Isolation & Container Contract ---")
manifest = generate_code_logic(
nodes=s1_nodes,
edges=s1_edges,
dependencies=["math", "requests", "json", "pandas"],
graph_name="Contract Test"
)
reqs = manifest["requirements_txt"]
dockerfile = manifest["dockerfile"]
assert_test("requests>=0.1.0" in reqs, "3rd-party 'requests' pinned in requirements.txt")
assert_test("pandas>=0.1.0" in reqs, "3rd-party 'pandas' pinned in requirements.txt")
assert_test("math" not in reqs, "Standard library 'math' excluded from requirements.txt")
assert_test("json" not in reqs, "Standard library 'json' excluded from requirements.txt")
assert_test("FROM python:3.12-slim" in dockerfile, "Dockerfile uses python:3.12-slim base image")
assert_test("WORKDIR /app" in dockerfile, "Dockerfile configures /app working directory")
assert_test('CMD ["python", "contract_test.py"]' in dockerfile, "Dockerfile specifies sanitized entrypoint command")
print("\n" + "=" * 80)
print(f"GATE 4 VERIFICATION: ALL {passed}/{total} ASSERTIONS PASSED (100% OK)")
print("=" * 80)
if __name__ == "__main__":
run_gate4_parity_verification()