rsnarsna
fix(registry): restore canonical node registry to 47 nodes and extract CANONICAL_NODE_COUNT
475fb21 Download tests/test_complete_node_validation_matrix.py from Rsnarsna/auto: direct link, hf CLI and curl.
- Browser
- Download file 16.3 kB
-
https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/tests/test_complete_node_validation_matrix.py
- Command line
-
hf download hf://spaces/Rsnarsna/auto/tests/test_complete_node_validation_matrix.py
-
curl -L -o test_complete_node_validation_matrix.py https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/tests/test_complete_node_validation_matrix.py
16.3 kB
| """ | |
| Complete Node Validation Matrix Test Suite (ALLI Studio) | |
| Verifies for every registered node: | |
| 1. Inputs contract: port IDs, data types, required constraints. | |
| 2. Outputs contract: port IDs, return data types. | |
| 3. Parameters contract: types, defaults, validation. | |
| 4. Operation execution: runtime execution handler produces expected outputs. | |
| 5. Error handling: invalid inputs/parameters produce clean, actionable error diagnostics. | |
| 6. Code generation: synthesis produces valid standalone Python. | |
| """ | |
| import pytest | |
| import math | |
| import json | |
| import os | |
| import tempfile | |
| from language_core.node_schema import ( | |
| default_registry, | |
| NodeDefinition, | |
| NodeExecutionStatus, | |
| PortDefinition, | |
| ParameterDefinition, | |
| CANONICAL_NODE_COUNT, | |
| ) | |
| from lambdas.codegen_handler import generate_code_logic | |
| from lambdas.validate_handler import validate_workflow_protocol_logic | |
| # Test input fixtures matching 100% of the exact declared ports and parameters | |
| NODE_TEST_FIXTURES = { | |
| # Essentials | |
| "workflow.start": { | |
| "inputs": {}, | |
| "parameters": {}, | |
| "expected_output_keys": ["trigger"], | |
| }, | |
| "trigger.schedule": { | |
| "inputs": {}, | |
| "parameters": {"cron": "0 * * * *", "interval_minutes": 60}, | |
| "expected_output_keys": ["trigger", "timestamp"], | |
| }, | |
| "trigger.webhook": { | |
| "inputs": {}, | |
| "parameters": {"path": "/webhook/v1/event", "method": "POST"}, | |
| "expected_output_keys": ["trigger", "payload", "headers"], | |
| }, | |
| "api.http_request": { | |
| "inputs": {}, | |
| "parameters": {"method": "GET", "url": "https://httpbin.org/get", "timeout": 5}, | |
| "expected_output_keys": ["then", "status_code", "response", "headers"], | |
| }, | |
| # Variables & Constants | |
| "core.variable": { | |
| "inputs": {}, | |
| "parameters": {"var_name": "counter", "var_type": "int", "literal_value": "42"}, | |
| "expected_output_keys": ["out_val"], | |
| }, | |
| "core.constant": { | |
| "inputs": {}, | |
| "parameters": {"const_name": "PI", "const_type": "float", "literal_value": "3.14159"}, | |
| "expected_output_keys": ["out"], | |
| }, | |
| "constant.int": { | |
| "inputs": {}, | |
| "parameters": {"const_name": "CONST_INT", "const_type": "int", "literal_value": "100"}, | |
| "expected_output_keys": ["out"], | |
| }, | |
| "constant.float": { | |
| "inputs": {}, | |
| "parameters": {"const_name": "CONST_FLOAT", "const_type": "float", "literal_value": "99.95"}, | |
| "expected_output_keys": ["out"], | |
| }, | |
| "constant.str": { | |
| "inputs": {}, | |
| "parameters": {"const_name": "CONST_STR", "const_type": "str", "literal_value": "Hello World"}, | |
| "expected_output_keys": ["out"], | |
| }, | |
| "constant.bool": { | |
| "inputs": {}, | |
| "parameters": {"const_name": "CONST_BOOL", "const_type": "bool", "literal_value": "True"}, | |
| "expected_output_keys": ["out"], | |
| }, | |
| # Data Types & Operators | |
| "core.type_cast": { | |
| "inputs": {"value": "123"}, | |
| "parameters": {"target_type": "int"}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| "core.binary_op": { | |
| "inputs": {"a": 25, "b": 17}, | |
| "parameters": {"operator": "+"}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| "core.compare": { | |
| "inputs": {"a": 20, "b": 10}, | |
| "parameters": {"operator": ">"}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| "core.logical_op": { | |
| "inputs": {"a": True, "b": False}, | |
| "parameters": {"operator": "and"}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| # Math Functions | |
| "math.sqrt": { | |
| "inputs": {"x": 49.0}, | |
| "parameters": {"rounding": "none", "precision": 2}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| "math.pow": { | |
| "inputs": {"x": 2.0, "y": 8.0}, | |
| "parameters": {}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| "math.abs": { | |
| "inputs": {"x": -42.5}, | |
| "parameters": {}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| "math.round": { | |
| "inputs": {"x": 3.14159}, | |
| "parameters": {"precision": 2}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| "math.floor": { | |
| "inputs": {"x": 7.89}, | |
| "parameters": {}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| "math.ceil": { | |
| "inputs": {"x": 7.12}, | |
| "parameters": {}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| "math.min": { | |
| "inputs": {"a": 15.0, "b": 3.0}, | |
| "parameters": {}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| "math.max": { | |
| "inputs": {"a": 15.0, "b": 3.0}, | |
| "parameters": {}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| # Control Flow & Loops | |
| "control.if_else": { | |
| "inputs": {"condition": True, "then_val": "yes", "else_val": "no"}, | |
| "parameters": {"default_branch": "then"}, | |
| "expected_output_keys": ["then", "else", "result", "branch_taken"], | |
| }, | |
| "control.for_loop": { | |
| "inputs": {"collection": [1, 2, 3, 4]}, | |
| "parameters": {"max_iterations": 100}, | |
| "expected_output_keys": ["items", "iteration_count", "completed"], | |
| }, | |
| "control.while_loop": { | |
| "inputs": {"condition": False}, | |
| "parameters": {"max_iterations": 100}, | |
| "expected_output_keys": ["final_value", "iterations", "completed"], | |
| }, | |
| "control.loop_control": { | |
| "inputs": {}, | |
| "parameters": {"control_type": "break"}, | |
| "expected_output_keys": ["signal"], | |
| }, | |
| # Functions | |
| "function.def": { | |
| "inputs": {}, | |
| "parameters": {"func_name": "compute_total", "params": "price, tax", "return_type": "float"}, | |
| "expected_output_keys": ["callable_fn"], | |
| }, | |
| "function.call": { | |
| "inputs": {"callable_fn": abs, "arg_1": -99}, | |
| "parameters": {"func_name": "abs"}, | |
| "expected_output_keys": ["result"], | |
| }, | |
| # Collections & Data Structures | |
| "data.list": { | |
| "inputs": {"list_in": [10, 20, 30], "item": 40}, | |
| "parameters": {"operation": "append"}, | |
| "expected_output_keys": ["list_out", "result", "length"], | |
| }, | |
| "data.dict": { | |
| "inputs": {"dict_in": {"user": "alice"}, "key": "role", "value": "admin"}, | |
| "parameters": {"operation": "set"}, | |
| "expected_output_keys": ["dict_out", "result", "keys"], | |
| }, | |
| "data.tuple": { | |
| "inputs": {"item_1": 1.0, "item_2": 2.0, "item_3": 3.0}, | |
| "parameters": {"item_count": 3}, | |
| "expected_output_keys": ["tuple_out", "item_1", "item_2"], | |
| }, | |
| "data.set": { | |
| "inputs": {"set_a": {1, 2, 3}, "set_b": {3, 4, 5}}, | |
| "parameters": {"operation": "union"}, | |
| "expected_output_keys": ["set_out", "cardinality"], | |
| }, | |
| # Strings | |
| "string.transform": { | |
| "inputs": {"text_in": " Alli Visual Programming "}, | |
| "parameters": {"operation": "strip"}, | |
| "expected_output_keys": ["text_out", "length"], | |
| }, | |
| "string.concat": { | |
| "inputs": {"a": "Hello", "b": "World"}, | |
| "parameters": {"separator": " "}, | |
| "expected_output_keys": ["result", "length"], | |
| }, | |
| "string.split": { | |
| "inputs": {"text": "apple,banana,cherry"}, | |
| "parameters": {"delimiter": ","}, | |
| "expected_output_keys": ["items", "count"], | |
| }, | |
| # Exception Handling | |
| "control.try_except": { | |
| "inputs": {}, | |
| "parameters": {"exception_type": "Exception"}, | |
| "expected_output_keys": ["result", "caught_error", "has_error"], | |
| }, | |
| # OOP & Classes | |
| "oop.class_def": { | |
| "inputs": {}, | |
| "parameters": { | |
| "class_name": "Student", | |
| "attributes": "name, score", | |
| "methods": "add_bonus", | |
| "docstring": "Represents a student class.", | |
| }, | |
| "expected_output_keys": ["class_ref", "class_name", "out"], | |
| }, | |
| "oop.instance": { | |
| "inputs": {}, | |
| "parameters": {"class_name": "Student", "arg_1_val": "Alice", "arg_2_val": 85}, | |
| "expected_output_keys": ["instance", "result", "val", "out"], | |
| }, | |
| "oop.method_call": { | |
| "inputs": {"instance_in": {"name": "Alice", "score": 85}}, | |
| "parameters": { | |
| "method_name": "add_bonus", | |
| "attribute_to_modify": "score", | |
| "operation": "increment", | |
| "default_arg": 15, | |
| }, | |
| "expected_output_keys": ["instance_out", "result", "out"], | |
| }, | |
| # I/O Primitives | |
| "io.input": { | |
| "inputs": {}, | |
| "parameters": {"prompt": "Enter value: ", "input_type": "str", "default_value": "ALLI"}, | |
| "expected_output_keys": ["then", "out", "output", "value"], | |
| }, | |
| "io.print": { | |
| "inputs": {"value": "Test message to stdout"}, | |
| "parameters": {"prefix": "[LOG] ", "sep": " ", "end": "\n"}, | |
| "expected_output_keys": ["then", "out", "printed_string"], | |
| }, | |
| # Sandboxed File I/O | |
| "file.read": { | |
| "inputs": {}, | |
| "parameters": {"path": "sample.txt", "mode": "text", "encoding": "utf-8", "create_if_missing": True}, | |
| "expected_output_keys": ["then", "content", "byte_count", "success"], | |
| }, | |
| "file.write": { | |
| "inputs": {"content": "File content from validation matrix"}, | |
| "parameters": {"path": "sample_out.txt", "mode": "write", "encoding": "utf-8", "create_dirs": True}, | |
| "expected_output_keys": ["then", "bytes_written", "file_path", "success"], | |
| }, | |
| # JSON & CSV Parsers | |
| "json.parse": { | |
| "inputs": {"text": '{"name": "ALLI", "version": 2.0, "active": true}'}, | |
| "parameters": {"strict": True, "fallback_to_none": False}, | |
| "expected_output_keys": ["then", "data", "is_valid", "error"], | |
| }, | |
| "json.stringify": { | |
| "inputs": {"data": {"status": "ok", "items": [1, 2, 3]}}, | |
| "parameters": {"indent": 2, "sort_keys": True, "ensure_ascii": False}, | |
| "expected_output_keys": ["then", "text", "byte_size", "success"], | |
| }, | |
| "csv.parse": { | |
| "inputs": {"text": "name,age,city\nAlice,30,NYC\nBob,25,LA"}, | |
| "parameters": {"delimiter": ",", "has_header": True, "skip_empty_lines": True, "trim_whitespace": True}, | |
| "expected_output_keys": ["then", "rows", "headers", "row_count", "success"], | |
| }, | |
| "csv.stringify": { | |
| "inputs": {"rows": [{"name": "Alice", "score": 98}, {"name": "Bob", "score": 95}]}, | |
| "parameters": {"delimiter": ",", "include_header": True, "line_terminator": "\n"}, | |
| "expected_output_keys": ["then", "text", "row_count", "byte_size", "success"], | |
| }, | |
| } | |
| def test_registry_contains_all_expected_nodes(): | |
| """Verify that all 47 expected schemas are registered in default_registry.""" | |
| all_registered = default_registry.list_all() | |
| reg_types = {n.type for n in all_registered} | |
| assert len(reg_types) >= CANONICAL_NODE_COUNT, f"Expected at least 47 schemas, got {len(reg_types)}" | |
| for expected_type in NODE_TEST_FIXTURES.keys(): | |
| assert expected_type in reg_types, f"Missing node schema in registry: {expected_type}" | |
| def test_node_schema_structure_and_ports(node_type): | |
| """Verify that every node's schema defines valid inputs, outputs, parameters, and docs.""" | |
| node = default_registry.get(node_type) | |
| assert node is not None, f"Node {node_type} not found in registry" | |
| assert node.label, f"Node {node_type} must have a non-empty label" | |
| assert node.category, f"Node {node_type} must have a category" | |
| # Verify inputs | |
| for inp in node.inputs: | |
| assert inp.id, f"Input in {node_type} missing id" | |
| assert inp.name, f"Input {inp.id} in {node_type} missing name" | |
| assert inp.type, f"Input {inp.id} in {node_type} missing type" | |
| assert isinstance(inp.required, bool), f"Input {inp.id} in {node_type} required must be bool" | |
| # Verify outputs | |
| for out in node.outputs: | |
| assert out.id, f"Output in {node_type} missing id" | |
| assert out.name, f"Output {out.id} in {node_type} missing name" | |
| assert out.type, f"Output {out.id} in {node_type} missing type" | |
| # Verify parameters | |
| for param in node.parameters: | |
| assert param.id, f"Parameter in {node_type} missing id" | |
| assert param.name, f"Parameter {param.id} in {node_type} missing name" | |
| assert param.type, f"Parameter {param.id} in {node_type} missing type" | |
| def test_node_execution_operation(node_type): | |
| """Verify that every node's runtime execution handler successfully computes valid outputs.""" | |
| fixture = NODE_TEST_FIXTURES[node_type] | |
| inputs = fixture["inputs"] | |
| parameters = fixture["parameters"] | |
| expected_keys = fixture["expected_output_keys"] | |
| assert default_registry.has_executor(node_type), f"Node {node_type} must have an executor" | |
| # Run the execution handler | |
| result = default_registry.execute(node_type, inputs=inputs, parameters=parameters) | |
| assert isinstance(result, dict), f"Node {node_type} executor must return a dict, got {type(result)}" | |
| # Check all declared expected output keys are present | |
| for k in expected_keys: | |
| assert k in result, f"Node {node_type} output missing declared key '{k}'. Returned keys: {list(result.keys())}" | |
| def test_node_math_sqrt_precision_and_validation(): | |
| """Detailed operation test for math.sqrt.""" | |
| res = default_registry.execute("math.sqrt", inputs={"x": 50.0}, parameters={"rounding": "round", "precision": 3}) | |
| assert res["result"] == round(math.sqrt(50.0), 3) | |
| def test_node_file_io_sandboxing(): | |
| """Detailed operation test for file.write and file.read path safety.""" | |
| # Write safe file | |
| write_res = default_registry.execute( | |
| "file.write", | |
| inputs={"content": "ALLI Verification Content"}, | |
| parameters={"path": "audit_test.txt", "mode": "write", "encoding": "utf-8"} | |
| ) | |
| assert write_res["bytes_written"] > 0 | |
| assert write_res["success"] is True | |
| # Read safe file | |
| read_res = default_registry.execute( | |
| "file.read", | |
| inputs={}, | |
| parameters={"path": "audit_test.txt", "mode": "text", "encoding": "utf-8"} | |
| ) | |
| assert read_res["content"] == "ALLI Verification Content" | |
| assert read_res["success"] is True | |
| # Attempt path traversal | |
| with pytest.raises(Exception): | |
| default_registry.execute( | |
| "file.read", | |
| inputs={}, | |
| parameters={"path": "../../../etc/passwd", "mode": "text", "encoding": "utf-8"} | |
| ) | |
| def test_node_json_parse_and_stringify(): | |
| """Detailed operation test for json.stringify and json.parse.""" | |
| data = {"status": "ok", "count": 42, "features": ["ast", "sandbox", "codegen"]} | |
| # Stringify | |
| s_res = default_registry.execute( | |
| "json.stringify", | |
| inputs={"data": data}, | |
| parameters={"indent": 2, "sort_keys": True, "ensure_ascii": False} | |
| ) | |
| assert "status" in s_res["text"] | |
| assert s_res["byte_size"] > 0 | |
| assert s_res["success"] is True | |
| # Parse back | |
| p_res = default_registry.execute( | |
| "json.parse", | |
| inputs={"text": s_res["text"]}, | |
| parameters={"strict": True, "fallback_to_none": False} | |
| ) | |
| assert p_res["is_valid"] is True | |
| assert p_res["data"]["count"] == 42 | |
| def test_node_csv_parse_and_stringify(): | |
| """Detailed operation test for csv.stringify and csv.parse.""" | |
| rows = [ | |
| {"id": "1", "name": "Alice", "score": "95"}, | |
| {"id": "2", "name": "Bob", "score": "88"}, | |
| ] | |
| # Stringify | |
| s_res = default_registry.execute( | |
| "csv.stringify", | |
| inputs={"rows": rows}, | |
| parameters={"delimiter": ",", "include_header": True, "line_terminator": "\n"} | |
| ) | |
| assert "Alice" in s_res["text"] | |
| assert s_res["row_count"] == 2 | |
| assert s_res["success"] is True | |
| # Parse back | |
| p_res = default_registry.execute( | |
| "csv.parse", | |
| inputs={"text": s_res["text"]}, | |
| parameters={"delimiter": ",", "has_header": True, "skip_empty_lines": True, "trim_whitespace": True} | |
| ) | |
| assert p_res["row_count"] == 2 | |
| assert p_res["headers"] == ["id", "name", "score"] | |
| assert p_res["rows"][0]["name"] == "Alice" | |
| assert p_res["success"] is True | |