""" Complete Node Validation Matrix Test Suite (ALLI Studio) Verifies for every registered node: 1. Inputs contract: port IDs, data types, required constraints. 2. Outputs contract: port IDs, return data types. 3. Parameters contract: types, defaults, validation. 4. Operation execution: runtime execution handler produces expected outputs. 5. Error handling: invalid inputs/parameters produce clean, actionable error diagnostics. 6. Code generation: synthesis produces valid standalone Python. """ import pytest import math import json import os import tempfile from language_core.node_schema import ( default_registry, NodeDefinition, NodeExecutionStatus, PortDefinition, ParameterDefinition, CANONICAL_NODE_COUNT, ) from lambdas.codegen_handler import generate_code_logic from lambdas.validate_handler import validate_workflow_protocol_logic # Test input fixtures matching 100% of the exact declared ports and parameters NODE_TEST_FIXTURES = { # Essentials "workflow.start": { "inputs": {}, "parameters": {}, "expected_output_keys": ["trigger"], }, "trigger.schedule": { "inputs": {}, "parameters": {"cron": "0 * * * *", "interval_minutes": 60}, "expected_output_keys": ["trigger", "timestamp"], }, "trigger.webhook": { "inputs": {}, "parameters": {"path": "/webhook/v1/event", "method": "POST"}, "expected_output_keys": ["trigger", "payload", "headers"], }, "api.http_request": { "inputs": {}, "parameters": {"method": "GET", "url": "https://httpbin.org/get", "timeout": 5}, "expected_output_keys": ["then", "status_code", "response", "headers"], }, # Variables & Constants "core.variable": { "inputs": {}, "parameters": {"var_name": "counter", "var_type": "int", "literal_value": "42"}, "expected_output_keys": ["out_val"], }, "core.constant": { "inputs": {}, "parameters": {"const_name": "PI", "const_type": "float", "literal_value": "3.14159"}, "expected_output_keys": ["out"], }, "constant.int": { "inputs": {}, "parameters": {"const_name": "CONST_INT", "const_type": "int", "literal_value": "100"}, "expected_output_keys": ["out"], }, "constant.float": { "inputs": {}, "parameters": {"const_name": "CONST_FLOAT", "const_type": "float", "literal_value": "99.95"}, "expected_output_keys": ["out"], }, "constant.str": { "inputs": {}, "parameters": {"const_name": "CONST_STR", "const_type": "str", "literal_value": "Hello World"}, "expected_output_keys": ["out"], }, "constant.bool": { "inputs": {}, "parameters": {"const_name": "CONST_BOOL", "const_type": "bool", "literal_value": "True"}, "expected_output_keys": ["out"], }, # Data Types & Operators "core.type_cast": { "inputs": {"value": "123"}, "parameters": {"target_type": "int"}, "expected_output_keys": ["result"], }, "core.binary_op": { "inputs": {"a": 25, "b": 17}, "parameters": {"operator": "+"}, "expected_output_keys": ["result"], }, "core.compare": { "inputs": {"a": 20, "b": 10}, "parameters": {"operator": ">"}, "expected_output_keys": ["result"], }, "core.logical_op": { "inputs": {"a": True, "b": False}, "parameters": {"operator": "and"}, "expected_output_keys": ["result"], }, # Math Functions "math.sqrt": { "inputs": {"x": 49.0}, "parameters": {"rounding": "none", "precision": 2}, "expected_output_keys": ["result"], }, "math.pow": { "inputs": {"x": 2.0, "y": 8.0}, "parameters": {}, "expected_output_keys": ["result"], }, "math.abs": { "inputs": {"x": -42.5}, "parameters": {}, "expected_output_keys": ["result"], }, "math.round": { "inputs": {"x": 3.14159}, "parameters": {"precision": 2}, "expected_output_keys": ["result"], }, "math.floor": { "inputs": {"x": 7.89}, "parameters": {}, "expected_output_keys": ["result"], }, "math.ceil": { "inputs": {"x": 7.12}, "parameters": {}, "expected_output_keys": ["result"], }, "math.min": { "inputs": {"a": 15.0, "b": 3.0}, "parameters": {}, "expected_output_keys": ["result"], }, "math.max": { "inputs": {"a": 15.0, "b": 3.0}, "parameters": {}, "expected_output_keys": ["result"], }, # Control Flow & Loops "control.if_else": { "inputs": {"condition": True, "then_val": "yes", "else_val": "no"}, "parameters": {"default_branch": "then"}, "expected_output_keys": ["then", "else", "result", "branch_taken"], }, "control.for_loop": { "inputs": {"collection": [1, 2, 3, 4]}, "parameters": {"max_iterations": 100}, "expected_output_keys": ["items", "iteration_count", "completed"], }, "control.while_loop": { "inputs": {"condition": False}, "parameters": {"max_iterations": 100}, "expected_output_keys": ["final_value", "iterations", "completed"], }, "control.loop_control": { "inputs": {}, "parameters": {"control_type": "break"}, "expected_output_keys": ["signal"], }, # Functions "function.def": { "inputs": {}, "parameters": {"func_name": "compute_total", "params": "price, tax", "return_type": "float"}, "expected_output_keys": ["callable_fn"], }, "function.call": { "inputs": {"callable_fn": abs, "arg_1": -99}, "parameters": {"func_name": "abs"}, "expected_output_keys": ["result"], }, # Collections & Data Structures "data.list": { "inputs": {"list_in": [10, 20, 30], "item": 40}, "parameters": {"operation": "append"}, "expected_output_keys": ["list_out", "result", "length"], }, "data.dict": { "inputs": {"dict_in": {"user": "alice"}, "key": "role", "value": "admin"}, "parameters": {"operation": "set"}, "expected_output_keys": ["dict_out", "result", "keys"], }, "data.tuple": { "inputs": {"item_1": 1.0, "item_2": 2.0, "item_3": 3.0}, "parameters": {"item_count": 3}, "expected_output_keys": ["tuple_out", "item_1", "item_2"], }, "data.set": { "inputs": {"set_a": {1, 2, 3}, "set_b": {3, 4, 5}}, "parameters": {"operation": "union"}, "expected_output_keys": ["set_out", "cardinality"], }, # Strings "string.transform": { "inputs": {"text_in": " Alli Visual Programming "}, "parameters": {"operation": "strip"}, "expected_output_keys": ["text_out", "length"], }, "string.concat": { "inputs": {"a": "Hello", "b": "World"}, "parameters": {"separator": " "}, "expected_output_keys": ["result", "length"], }, "string.split": { "inputs": {"text": "apple,banana,cherry"}, "parameters": {"delimiter": ","}, "expected_output_keys": ["items", "count"], }, # Exception Handling "control.try_except": { "inputs": {}, "parameters": {"exception_type": "Exception"}, "expected_output_keys": ["result", "caught_error", "has_error"], }, # OOP & Classes "oop.class_def": { "inputs": {}, "parameters": { "class_name": "Student", "attributes": "name, score", "methods": "add_bonus", "docstring": "Represents a student class.", }, "expected_output_keys": ["class_ref", "class_name", "out"], }, "oop.instance": { "inputs": {}, "parameters": {"class_name": "Student", "arg_1_val": "Alice", "arg_2_val": 85}, "expected_output_keys": ["instance", "result", "val", "out"], }, "oop.method_call": { "inputs": {"instance_in": {"name": "Alice", "score": 85}}, "parameters": { "method_name": "add_bonus", "attribute_to_modify": "score", "operation": "increment", "default_arg": 15, }, "expected_output_keys": ["instance_out", "result", "out"], }, # I/O Primitives "io.input": { "inputs": {}, "parameters": {"prompt": "Enter value: ", "input_type": "str", "default_value": "ALLI"}, "expected_output_keys": ["then", "out", "output", "value"], }, "io.print": { "inputs": {"value": "Test message to stdout"}, "parameters": {"prefix": "[LOG] ", "sep": " ", "end": "\n"}, "expected_output_keys": ["then", "out", "printed_string"], }, # Sandboxed File I/O "file.read": { "inputs": {}, "parameters": {"path": "sample.txt", "mode": "text", "encoding": "utf-8", "create_if_missing": True}, "expected_output_keys": ["then", "content", "byte_count", "success"], }, "file.write": { "inputs": {"content": "File content from validation matrix"}, "parameters": {"path": "sample_out.txt", "mode": "write", "encoding": "utf-8", "create_dirs": True}, "expected_output_keys": ["then", "bytes_written", "file_path", "success"], }, # JSON & CSV Parsers "json.parse": { "inputs": {"text": '{"name": "ALLI", "version": 2.0, "active": true}'}, "parameters": {"strict": True, "fallback_to_none": False}, "expected_output_keys": ["then", "data", "is_valid", "error"], }, "json.stringify": { "inputs": {"data": {"status": "ok", "items": [1, 2, 3]}}, "parameters": {"indent": 2, "sort_keys": True, "ensure_ascii": False}, "expected_output_keys": ["then", "text", "byte_size", "success"], }, "csv.parse": { "inputs": {"text": "name,age,city\nAlice,30,NYC\nBob,25,LA"}, "parameters": {"delimiter": ",", "has_header": True, "skip_empty_lines": True, "trim_whitespace": True}, "expected_output_keys": ["then", "rows", "headers", "row_count", "success"], }, "csv.stringify": { "inputs": {"rows": [{"name": "Alice", "score": 98}, {"name": "Bob", "score": 95}]}, "parameters": {"delimiter": ",", "include_header": True, "line_terminator": "\n"}, "expected_output_keys": ["then", "text", "row_count", "byte_size", "success"], }, } def test_registry_contains_all_expected_nodes(): """Verify that all 47 expected schemas are registered in default_registry.""" all_registered = default_registry.list_all() reg_types = {n.type for n in all_registered} assert len(reg_types) >= CANONICAL_NODE_COUNT, f"Expected at least 47 schemas, got {len(reg_types)}" for expected_type in NODE_TEST_FIXTURES.keys(): assert expected_type in reg_types, f"Missing node schema in registry: {expected_type}" @pytest.mark.parametrize("node_type", list(NODE_TEST_FIXTURES.keys())) def test_node_schema_structure_and_ports(node_type): """Verify that every node's schema defines valid inputs, outputs, parameters, and docs.""" node = default_registry.get(node_type) assert node is not None, f"Node {node_type} not found in registry" assert node.label, f"Node {node_type} must have a non-empty label" assert node.category, f"Node {node_type} must have a category" # Verify inputs for inp in node.inputs: assert inp.id, f"Input in {node_type} missing id" assert inp.name, f"Input {inp.id} in {node_type} missing name" assert inp.type, f"Input {inp.id} in {node_type} missing type" assert isinstance(inp.required, bool), f"Input {inp.id} in {node_type} required must be bool" # Verify outputs for out in node.outputs: assert out.id, f"Output in {node_type} missing id" assert out.name, f"Output {out.id} in {node_type} missing name" assert out.type, f"Output {out.id} in {node_type} missing type" # Verify parameters for param in node.parameters: assert param.id, f"Parameter in {node_type} missing id" assert param.name, f"Parameter {param.id} in {node_type} missing name" assert param.type, f"Parameter {param.id} in {node_type} missing type" @pytest.mark.parametrize("node_type", list(NODE_TEST_FIXTURES.keys())) def test_node_execution_operation(node_type): """Verify that every node's runtime execution handler successfully computes valid outputs.""" fixture = NODE_TEST_FIXTURES[node_type] inputs = fixture["inputs"] parameters = fixture["parameters"] expected_keys = fixture["expected_output_keys"] assert default_registry.has_executor(node_type), f"Node {node_type} must have an executor" # Run the execution handler result = default_registry.execute(node_type, inputs=inputs, parameters=parameters) assert isinstance(result, dict), f"Node {node_type} executor must return a dict, got {type(result)}" # Check all declared expected output keys are present for k in expected_keys: assert k in result, f"Node {node_type} output missing declared key '{k}'. Returned keys: {list(result.keys())}" def test_node_math_sqrt_precision_and_validation(): """Detailed operation test for math.sqrt.""" res = default_registry.execute("math.sqrt", inputs={"x": 50.0}, parameters={"rounding": "round", "precision": 3}) assert res["result"] == round(math.sqrt(50.0), 3) def test_node_file_io_sandboxing(): """Detailed operation test for file.write and file.read path safety.""" # Write safe file write_res = default_registry.execute( "file.write", inputs={"content": "ALLI Verification Content"}, parameters={"path": "audit_test.txt", "mode": "write", "encoding": "utf-8"} ) assert write_res["bytes_written"] > 0 assert write_res["success"] is True # Read safe file read_res = default_registry.execute( "file.read", inputs={}, parameters={"path": "audit_test.txt", "mode": "text", "encoding": "utf-8"} ) assert read_res["content"] == "ALLI Verification Content" assert read_res["success"] is True # Attempt path traversal with pytest.raises(Exception): default_registry.execute( "file.read", inputs={}, parameters={"path": "../../../etc/passwd", "mode": "text", "encoding": "utf-8"} ) def test_node_json_parse_and_stringify(): """Detailed operation test for json.stringify and json.parse.""" data = {"status": "ok", "count": 42, "features": ["ast", "sandbox", "codegen"]} # Stringify s_res = default_registry.execute( "json.stringify", inputs={"data": data}, parameters={"indent": 2, "sort_keys": True, "ensure_ascii": False} ) assert "status" in s_res["text"] assert s_res["byte_size"] > 0 assert s_res["success"] is True # Parse back p_res = default_registry.execute( "json.parse", inputs={"text": s_res["text"]}, parameters={"strict": True, "fallback_to_none": False} ) assert p_res["is_valid"] is True assert p_res["data"]["count"] == 42 def test_node_csv_parse_and_stringify(): """Detailed operation test for csv.stringify and csv.parse.""" rows = [ {"id": "1", "name": "Alice", "score": "95"}, {"id": "2", "name": "Bob", "score": "88"}, ] # Stringify s_res = default_registry.execute( "csv.stringify", inputs={"rows": rows}, parameters={"delimiter": ",", "include_header": True, "line_terminator": "\n"} ) assert "Alice" in s_res["text"] assert s_res["row_count"] == 2 assert s_res["success"] is True # Parse back p_res = default_registry.execute( "csv.parse", inputs={"text": s_res["text"]}, parameters={"delimiter": ",", "has_header": True, "skip_empty_lines": True, "trim_whitespace": True} ) assert p_res["row_count"] == 2 assert p_res["headers"] == ["id", "name", "score"] assert p_res["rows"][0]["name"] == "Alice" assert p_res["success"] is True