auto / tests /test_complete_node_validation_matrix.py
rsnarsna
fix(registry): restore canonical node registry to 47 nodes and extract CANONICAL_NODE_COUNT
475fb21
Raw History Blame Contribute Delete
16.3 kB
"""
Complete Node Validation Matrix Test Suite (ALLI Studio)
Verifies for every registered node:
1. Inputs contract: port IDs, data types, required constraints.
2. Outputs contract: port IDs, return data types.
3. Parameters contract: types, defaults, validation.
4. Operation execution: runtime execution handler produces expected outputs.
5. Error handling: invalid inputs/parameters produce clean, actionable error diagnostics.
6. Code generation: synthesis produces valid standalone Python.
"""
import pytest
import math
import json
import os
import tempfile
from language_core.node_schema import (
default_registry,
NodeDefinition,
NodeExecutionStatus,
PortDefinition,
ParameterDefinition,
CANONICAL_NODE_COUNT,
)
from lambdas.codegen_handler import generate_code_logic
from lambdas.validate_handler import validate_workflow_protocol_logic
# Test input fixtures matching 100% of the exact declared ports and parameters
NODE_TEST_FIXTURES = {
# Essentials
"workflow.start": {
"inputs": {},
"parameters": {},
"expected_output_keys": ["trigger"],
},
"trigger.schedule": {
"inputs": {},
"parameters": {"cron": "0 * * * *", "interval_minutes": 60},
"expected_output_keys": ["trigger", "timestamp"],
},
"trigger.webhook": {
"inputs": {},
"parameters": {"path": "/webhook/v1/event", "method": "POST"},
"expected_output_keys": ["trigger", "payload", "headers"],
},
"api.http_request": {
"inputs": {},
"parameters": {"method": "GET", "url": "https://httpbin.org/get", "timeout": 5},
"expected_output_keys": ["then", "status_code", "response", "headers"],
},
# Variables & Constants
"core.variable": {
"inputs": {},
"parameters": {"var_name": "counter", "var_type": "int", "literal_value": "42"},
"expected_output_keys": ["out_val"],
},
"core.constant": {
"inputs": {},
"parameters": {"const_name": "PI", "const_type": "float", "literal_value": "3.14159"},
"expected_output_keys": ["out"],
},
"constant.int": {
"inputs": {},
"parameters": {"const_name": "CONST_INT", "const_type": "int", "literal_value": "100"},
"expected_output_keys": ["out"],
},
"constant.float": {
"inputs": {},
"parameters": {"const_name": "CONST_FLOAT", "const_type": "float", "literal_value": "99.95"},
"expected_output_keys": ["out"],
},
"constant.str": {
"inputs": {},
"parameters": {"const_name": "CONST_STR", "const_type": "str", "literal_value": "Hello World"},
"expected_output_keys": ["out"],
},
"constant.bool": {
"inputs": {},
"parameters": {"const_name": "CONST_BOOL", "const_type": "bool", "literal_value": "True"},
"expected_output_keys": ["out"],
},
# Data Types & Operators
"core.type_cast": {
"inputs": {"value": "123"},
"parameters": {"target_type": "int"},
"expected_output_keys": ["result"],
},
"core.binary_op": {
"inputs": {"a": 25, "b": 17},
"parameters": {"operator": "+"},
"expected_output_keys": ["result"],
},
"core.compare": {
"inputs": {"a": 20, "b": 10},
"parameters": {"operator": ">"},
"expected_output_keys": ["result"],
},
"core.logical_op": {
"inputs": {"a": True, "b": False},
"parameters": {"operator": "and"},
"expected_output_keys": ["result"],
},
# Math Functions
"math.sqrt": {
"inputs": {"x": 49.0},
"parameters": {"rounding": "none", "precision": 2},
"expected_output_keys": ["result"],
},
"math.pow": {
"inputs": {"x": 2.0, "y": 8.0},
"parameters": {},
"expected_output_keys": ["result"],
},
"math.abs": {
"inputs": {"x": -42.5},
"parameters": {},
"expected_output_keys": ["result"],
},
"math.round": {
"inputs": {"x": 3.14159},
"parameters": {"precision": 2},
"expected_output_keys": ["result"],
},
"math.floor": {
"inputs": {"x": 7.89},
"parameters": {},
"expected_output_keys": ["result"],
},
"math.ceil": {
"inputs": {"x": 7.12},
"parameters": {},
"expected_output_keys": ["result"],
},
"math.min": {
"inputs": {"a": 15.0, "b": 3.0},
"parameters": {},
"expected_output_keys": ["result"],
},
"math.max": {
"inputs": {"a": 15.0, "b": 3.0},
"parameters": {},
"expected_output_keys": ["result"],
},
# Control Flow & Loops
"control.if_else": {
"inputs": {"condition": True, "then_val": "yes", "else_val": "no"},
"parameters": {"default_branch": "then"},
"expected_output_keys": ["then", "else", "result", "branch_taken"],
},
"control.for_loop": {
"inputs": {"collection": [1, 2, 3, 4]},
"parameters": {"max_iterations": 100},
"expected_output_keys": ["items", "iteration_count", "completed"],
},
"control.while_loop": {
"inputs": {"condition": False},
"parameters": {"max_iterations": 100},
"expected_output_keys": ["final_value", "iterations", "completed"],
},
"control.loop_control": {
"inputs": {},
"parameters": {"control_type": "break"},
"expected_output_keys": ["signal"],
},
# Functions
"function.def": {
"inputs": {},
"parameters": {"func_name": "compute_total", "params": "price, tax", "return_type": "float"},
"expected_output_keys": ["callable_fn"],
},
"function.call": {
"inputs": {"callable_fn": abs, "arg_1": -99},
"parameters": {"func_name": "abs"},
"expected_output_keys": ["result"],
},
# Collections & Data Structures
"data.list": {
"inputs": {"list_in": [10, 20, 30], "item": 40},
"parameters": {"operation": "append"},
"expected_output_keys": ["list_out", "result", "length"],
},
"data.dict": {
"inputs": {"dict_in": {"user": "alice"}, "key": "role", "value": "admin"},
"parameters": {"operation": "set"},
"expected_output_keys": ["dict_out", "result", "keys"],
},
"data.tuple": {
"inputs": {"item_1": 1.0, "item_2": 2.0, "item_3": 3.0},
"parameters": {"item_count": 3},
"expected_output_keys": ["tuple_out", "item_1", "item_2"],
},
"data.set": {
"inputs": {"set_a": {1, 2, 3}, "set_b": {3, 4, 5}},
"parameters": {"operation": "union"},
"expected_output_keys": ["set_out", "cardinality"],
},
# Strings
"string.transform": {
"inputs": {"text_in": " Alli Visual Programming "},
"parameters": {"operation": "strip"},
"expected_output_keys": ["text_out", "length"],
},
"string.concat": {
"inputs": {"a": "Hello", "b": "World"},
"parameters": {"separator": " "},
"expected_output_keys": ["result", "length"],
},
"string.split": {
"inputs": {"text": "apple,banana,cherry"},
"parameters": {"delimiter": ","},
"expected_output_keys": ["items", "count"],
},
# Exception Handling
"control.try_except": {
"inputs": {},
"parameters": {"exception_type": "Exception"},
"expected_output_keys": ["result", "caught_error", "has_error"],
},
# OOP & Classes
"oop.class_def": {
"inputs": {},
"parameters": {
"class_name": "Student",
"attributes": "name, score",
"methods": "add_bonus",
"docstring": "Represents a student class.",
},
"expected_output_keys": ["class_ref", "class_name", "out"],
},
"oop.instance": {
"inputs": {},
"parameters": {"class_name": "Student", "arg_1_val": "Alice", "arg_2_val": 85},
"expected_output_keys": ["instance", "result", "val", "out"],
},
"oop.method_call": {
"inputs": {"instance_in": {"name": "Alice", "score": 85}},
"parameters": {
"method_name": "add_bonus",
"attribute_to_modify": "score",
"operation": "increment",
"default_arg": 15,
},
"expected_output_keys": ["instance_out", "result", "out"],
},
# I/O Primitives
"io.input": {
"inputs": {},
"parameters": {"prompt": "Enter value: ", "input_type": "str", "default_value": "ALLI"},
"expected_output_keys": ["then", "out", "output", "value"],
},
"io.print": {
"inputs": {"value": "Test message to stdout"},
"parameters": {"prefix": "[LOG] ", "sep": " ", "end": "\n"},
"expected_output_keys": ["then", "out", "printed_string"],
},
# Sandboxed File I/O
"file.read": {
"inputs": {},
"parameters": {"path": "sample.txt", "mode": "text", "encoding": "utf-8", "create_if_missing": True},
"expected_output_keys": ["then", "content", "byte_count", "success"],
},
"file.write": {
"inputs": {"content": "File content from validation matrix"},
"parameters": {"path": "sample_out.txt", "mode": "write", "encoding": "utf-8", "create_dirs": True},
"expected_output_keys": ["then", "bytes_written", "file_path", "success"],
},
# JSON & CSV Parsers
"json.parse": {
"inputs": {"text": '{"name": "ALLI", "version": 2.0, "active": true}'},
"parameters": {"strict": True, "fallback_to_none": False},
"expected_output_keys": ["then", "data", "is_valid", "error"],
},
"json.stringify": {
"inputs": {"data": {"status": "ok", "items": [1, 2, 3]}},
"parameters": {"indent": 2, "sort_keys": True, "ensure_ascii": False},
"expected_output_keys": ["then", "text", "byte_size", "success"],
},
"csv.parse": {
"inputs": {"text": "name,age,city\nAlice,30,NYC\nBob,25,LA"},
"parameters": {"delimiter": ",", "has_header": True, "skip_empty_lines": True, "trim_whitespace": True},
"expected_output_keys": ["then", "rows", "headers", "row_count", "success"],
},
"csv.stringify": {
"inputs": {"rows": [{"name": "Alice", "score": 98}, {"name": "Bob", "score": 95}]},
"parameters": {"delimiter": ",", "include_header": True, "line_terminator": "\n"},
"expected_output_keys": ["then", "text", "row_count", "byte_size", "success"],
},
}
def test_registry_contains_all_expected_nodes():
"""Verify that all 47 expected schemas are registered in default_registry."""
all_registered = default_registry.list_all()
reg_types = {n.type for n in all_registered}
assert len(reg_types) >= CANONICAL_NODE_COUNT, f"Expected at least 47 schemas, got {len(reg_types)}"
for expected_type in NODE_TEST_FIXTURES.keys():
assert expected_type in reg_types, f"Missing node schema in registry: {expected_type}"
@pytest.mark.parametrize("node_type", list(NODE_TEST_FIXTURES.keys()))
def test_node_schema_structure_and_ports(node_type):
"""Verify that every node's schema defines valid inputs, outputs, parameters, and docs."""
node = default_registry.get(node_type)
assert node is not None, f"Node {node_type} not found in registry"
assert node.label, f"Node {node_type} must have a non-empty label"
assert node.category, f"Node {node_type} must have a category"
# Verify inputs
for inp in node.inputs:
assert inp.id, f"Input in {node_type} missing id"
assert inp.name, f"Input {inp.id} in {node_type} missing name"
assert inp.type, f"Input {inp.id} in {node_type} missing type"
assert isinstance(inp.required, bool), f"Input {inp.id} in {node_type} required must be bool"
# Verify outputs
for out in node.outputs:
assert out.id, f"Output in {node_type} missing id"
assert out.name, f"Output {out.id} in {node_type} missing name"
assert out.type, f"Output {out.id} in {node_type} missing type"
# Verify parameters
for param in node.parameters:
assert param.id, f"Parameter in {node_type} missing id"
assert param.name, f"Parameter {param.id} in {node_type} missing name"
assert param.type, f"Parameter {param.id} in {node_type} missing type"
@pytest.mark.parametrize("node_type", list(NODE_TEST_FIXTURES.keys()))
def test_node_execution_operation(node_type):
"""Verify that every node's runtime execution handler successfully computes valid outputs."""
fixture = NODE_TEST_FIXTURES[node_type]
inputs = fixture["inputs"]
parameters = fixture["parameters"]
expected_keys = fixture["expected_output_keys"]
assert default_registry.has_executor(node_type), f"Node {node_type} must have an executor"
# Run the execution handler
result = default_registry.execute(node_type, inputs=inputs, parameters=parameters)
assert isinstance(result, dict), f"Node {node_type} executor must return a dict, got {type(result)}"
# Check all declared expected output keys are present
for k in expected_keys:
assert k in result, f"Node {node_type} output missing declared key '{k}'. Returned keys: {list(result.keys())}"
def test_node_math_sqrt_precision_and_validation():
"""Detailed operation test for math.sqrt."""
res = default_registry.execute("math.sqrt", inputs={"x": 50.0}, parameters={"rounding": "round", "precision": 3})
assert res["result"] == round(math.sqrt(50.0), 3)
def test_node_file_io_sandboxing():
"""Detailed operation test for file.write and file.read path safety."""
# Write safe file
write_res = default_registry.execute(
"file.write",
inputs={"content": "ALLI Verification Content"},
parameters={"path": "audit_test.txt", "mode": "write", "encoding": "utf-8"}
)
assert write_res["bytes_written"] > 0
assert write_res["success"] is True
# Read safe file
read_res = default_registry.execute(
"file.read",
inputs={},
parameters={"path": "audit_test.txt", "mode": "text", "encoding": "utf-8"}
)
assert read_res["content"] == "ALLI Verification Content"
assert read_res["success"] is True
# Attempt path traversal
with pytest.raises(Exception):
default_registry.execute(
"file.read",
inputs={},
parameters={"path": "../../../etc/passwd", "mode": "text", "encoding": "utf-8"}
)
def test_node_json_parse_and_stringify():
"""Detailed operation test for json.stringify and json.parse."""
data = {"status": "ok", "count": 42, "features": ["ast", "sandbox", "codegen"]}
# Stringify
s_res = default_registry.execute(
"json.stringify",
inputs={"data": data},
parameters={"indent": 2, "sort_keys": True, "ensure_ascii": False}
)
assert "status" in s_res["text"]
assert s_res["byte_size"] > 0
assert s_res["success"] is True
# Parse back
p_res = default_registry.execute(
"json.parse",
inputs={"text": s_res["text"]},
parameters={"strict": True, "fallback_to_none": False}
)
assert p_res["is_valid"] is True
assert p_res["data"]["count"] == 42
def test_node_csv_parse_and_stringify():
"""Detailed operation test for csv.stringify and csv.parse."""
rows = [
{"id": "1", "name": "Alice", "score": "95"},
{"id": "2", "name": "Bob", "score": "88"},
]
# Stringify
s_res = default_registry.execute(
"csv.stringify",
inputs={"rows": rows},
parameters={"delimiter": ",", "include_header": True, "line_terminator": "\n"}
)
assert "Alice" in s_res["text"]
assert s_res["row_count"] == 2
assert s_res["success"] is True
# Parse back
p_res = default_registry.execute(
"csv.parse",
inputs={"text": s_res["text"]},
parameters={"delimiter": ",", "has_header": True, "skip_empty_lines": True, "trim_whitespace": True}
)
assert p_res["row_count"] == 2
assert p_res["headers"] == ["id", "name", "score"]
assert p_res["rows"][0]["name"] == "Alice"
assert p_res["success"] is True