rsnarsna
feat(phase12): cross-tier node contract conformance and execution parity test suite
7d53bd3 Download tests/test_phase12_node_conformance_parity.py from Rsnarsna/auto: direct link, hf CLI and curl.
- Browser
- Download file 11.7 kB
-
https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/tests/test_phase12_node_conformance_parity.py
- Command line
-
hf download hf://spaces/Rsnarsna/auto/tests/test_phase12_node_conformance_parity.py
-
curl -L -o test_phase12_node_conformance_parity.py https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/tests/test_phase12_node_conformance_parity.py
11.7 kB
| """ | |
| ALLI Phase 12: Node Contract Conformance & Execution Parity Cross-Tier Test Suite. | |
| Verifies: | |
| 1. Complete mathematical node parity (math.sqrt, math.pow, math.abs, math.ceil, math.floor, math.round, math.max, math.min) | |
| across backend execution, runtime evaluation, and synthesized Python code. | |
| 2. Data structure nodes (data.list, data.dict, data.tuple, data.set) operations, immutability, and alias parity. | |
| 3. Control flow nodes (control.if_else, control.for_loop, control.while_loop, control.try_except). | |
| 4. File I/O security (file.read, file.write): traversal rejection (../, ..\\), Windows reserved names (CON, PRN, NUL), byte quotas. | |
| 5. JSON & CSV processing (json.parse, json.stringify, csv.parse, csv.stringify) round-trip fidelity. | |
| 6. HTTP Request SSRF security (api.http_request): 127.0.0.1, 169.254.169.254, private subnet blocking, redirect validation. | |
| 7. Output truncation quota: 1MB limits enforce safe truncation without poisoning downstream nodes. | |
| """ | |
| import ast | |
| import json | |
| import math | |
| import os | |
| import shutil | |
| import tempfile | |
| import pytest | |
| from language_core.node_schema import ( | |
| default_registry, | |
| resolve_sandboxed_path, | |
| MAX_FILE_BYTES_QUOTA, | |
| ) | |
| from language_core.workflow_executor import ( | |
| execute_workflow_graph, | |
| MAX_OUTPUT_QUOTA_BYTES, | |
| measure_output_port_quota, | |
| ) | |
| from lambdas.codegen_handler import generate_code_logic | |
| from security.ssrf_validator import validate_target_url | |
| def clean_sandbox(): | |
| """Ensure clean sandbox directory for file tests.""" | |
| test_sandbox = tempfile.mkdtemp(prefix="alli_p12_sandbox_") | |
| orig_sandbox = os.environ.get("ALLI_SANDBOX_DIR") | |
| os.environ["ALLI_SANDBOX_DIR"] = test_sandbox | |
| yield test_sandbox | |
| shutil.rmtree(test_sandbox, ignore_errors=True) | |
| if orig_sandbox: | |
| os.environ["ALLI_SANDBOX_DIR"] = orig_sandbox | |
| elif "ALLI_SANDBOX_DIR" in os.environ: | |
| del os.environ["ALLI_SANDBOX_DIR"] | |
| # ─── 1. MATHEMATICAL NODES PARITY ───────────────────────────────────────────── | |
| def test_math_nodes_execution_and_codegen_parity(node_type, inputs, params, expected_val): | |
| """Verify math node parity between registry executor and Python codegen.""" | |
| assert default_registry.has(node_type) | |
| # 1. Direct registry execution | |
| reg_out = default_registry.execute(node_type, inputs, params) | |
| assert math.isclose(reg_out["result"], expected_val, rel_tol=1e-5) or reg_out["result"] == expected_val | |
| # 2. Python AST Codegen execution | |
| wf = { | |
| "nodes": [ | |
| { | |
| "id": "math_node", | |
| "type": node_type, | |
| "data": {**inputs, **params}, | |
| "inputs": inputs, | |
| "parameters": params, | |
| } | |
| ], | |
| "edges": [] | |
| } | |
| codegen_result = generate_code_logic(wf) | |
| py_code = codegen_result["code"] if isinstance(codegen_result, dict) else str(codegen_result) | |
| assert "def main():" in py_code | |
| # Execute synthesized Python in restricted namespace | |
| ns = {} | |
| exec(py_code, ns) | |
| res = ns["main"]() | |
| assert math.isclose(res, expected_val, rel_tol=1e-5) or res == expected_val | |
| # ─── 2. DATA STRUCTURE NODES PARITY ─────────────────────────────────────────── | |
| def test_data_list_parity(): | |
| """Verify data.list creation, append, and get_index operations.""" | |
| out_create = default_registry.execute("data.list", {}, {"Operation": "create", "Initial Items": "[10, 20, 30]"}) | |
| assert out_create["result"] == [10, 20, 30] | |
| assert out_create["length"] == 3 | |
| out_append = default_registry.execute("data.list", {"items": [10, 20], "item": 30}, {"Operation": "append"}) | |
| assert out_append["result"] == [10, 20, 30] | |
| out_get = default_registry.execute("data.list", {"items": [10, 20, 30]}, {"Operation": "get_index", "Index": 1}) | |
| assert out_get["result"] == 20 | |
| def test_data_dict_parity(): | |
| """Verify data.dict creation, set, and lookup operations.""" | |
| out_create = default_registry.execute("data.dict", {}, {"operation": "create", "initial_dict": '{"a": 1, "b": 2}'}) | |
| assert out_create["result"] == {"a": 1, "b": 2} | |
| assert "a" in out_create["keys"] | |
| out_set = default_registry.execute("data.dict", {"dict_in": {"a": 1}, "key": "b", "value": 2}, {"operation": "set"}) | |
| assert out_set["result"] == {"a": 1, "b": 2} | |
| out_get = default_registry.execute("data.dict", {"dict_in": {"a": 100}, "key": "a"}, {"operation": "get"}) | |
| assert out_get["result"] == 100 | |
| def test_data_tuple_immutability(): | |
| """Verify data.tuple produces an immutable tuple sequence.""" | |
| out = default_registry.execute("data.tuple", {"item_1": "x", "item_2": "y", "item_3": "z"}, {}) | |
| assert isinstance(out["tuple_out"], tuple) | |
| assert out["tuple_out"] == ("x", "y", "z") | |
| def test_data_set_operations(): | |
| """Verify data.set union, intersection, and difference.""" | |
| out_union = default_registry.execute("data.set", {"set_a": {1, 2}, "set_b": {2, 3}}, {"Operation": "union"}) | |
| assert out_union["set_out"] == {1, 2, 3} | |
| out_inter = default_registry.execute("data.set", {"set_a": {1, 2}, "set_b": {2, 3}}, {"Operation": "intersection"}) | |
| assert out_inter["set_out"] == {2} | |
| # ─── 3. CONTROL FLOW NODES PARITY ───────────────────────────────────────────── | |
| def test_control_if_else_branching(): | |
| """Verify control.if_else evaluates condition and takes appropriate branch.""" | |
| out_true = default_registry.execute("control.if_else", {"condition": True, "then_val": "yes", "else_val": "no"}, {}) | |
| assert out_true["result"] == "yes" | |
| assert out_true["branch_taken"] == "then" | |
| assert out_true["then"] == "yes" | |
| assert out_true["else"] == "no" | |
| out_false = default_registry.execute("control.if_else", {"condition": False, "then_val": "yes", "else_val": "no"}, {}) | |
| assert out_false["result"] == "no" | |
| assert out_false["branch_taken"] == "else" | |
| assert out_false["then"] == "yes" | |
| assert out_false["else"] == "no" | |
| def test_control_try_except_error_capture(): | |
| """Verify control.try_except catches errors and emits fallback.""" | |
| out_ok = default_registry.execute("control.try_except", {"try_val": 42, "fallback_val": 0}, {}) | |
| assert out_ok["result"] == 42 | |
| assert not out_ok["has_error"] | |
| out_err = default_registry.execute("control.try_except", {"try_val": ZeroDivisionError("division by zero"), "fallback_val": 99}, {}) | |
| assert out_err["result"] == 99 | |
| assert out_err["has_error"] is True | |
| # ─── 4. FILE I/O SECURITY & TRAVERSAL ───────────────────────────────────────── | |
| def test_file_io_path_containment_and_traversal(clean_sandbox): | |
| """Verify path traversal attacks and absolute paths outside sandbox are strictly rejected.""" | |
| sandbox = clean_sandbox | |
| traversal_paths = [ | |
| "../secrets.txt", | |
| "..\\secrets.txt", | |
| "sub/../../passwords.txt", | |
| "/etc/passwd", | |
| "\\Windows\\System32\\cmd.exe", | |
| ] | |
| for path in traversal_paths: | |
| with pytest.raises(PermissionError) as exc_info: | |
| resolve_sandboxed_path(path, base_dir=sandbox) | |
| assert "access denied" in str(exc_info.value).lower() or "traversal" in str(exc_info.value).lower() | |
| outside_abs = os.path.abspath(os.path.join(sandbox, "..", "outside_target.txt")) | |
| with pytest.raises(PermissionError) as exc_info: | |
| resolve_sandboxed_path(outside_abs, base_dir=sandbox) | |
| assert "access denied" in str(exc_info.value).lower() | |
| # Null byte check | |
| with pytest.raises(ValueError) as exc_info: | |
| resolve_sandboxed_path("file\0null.txt", base_dir=sandbox) | |
| assert "null byte" in str(exc_info.value).lower() | |
| def test_file_io_windows_reserved_device_names(clean_sandbox): | |
| """Verify Windows reserved device names (CON, PRN, AUX, NUL) are rejected.""" | |
| sandbox = clean_sandbox | |
| reserved_names = ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1", "con.txt", "nul.dat"] | |
| for name in reserved_names: | |
| with pytest.raises(PermissionError) as exc_info: | |
| resolve_sandboxed_path(name, base_dir=sandbox) | |
| assert "reserved" in str(exc_info.value).lower() or "device" in str(exc_info.value).lower() | |
| def test_file_io_quota_limits(): | |
| """Verify write byte quota rejects payloads > 1MB.""" | |
| oversized = "A" * (MAX_FILE_BYTES_QUOTA + 1024) | |
| with pytest.raises(ValueError) as exc_info: | |
| default_registry.execute("file.write", {"content": oversized, "file_path": "big.txt"}, {}) | |
| assert "quota" in str(exc_info.value).lower() or "exceeds" in str(exc_info.value).lower() | |
| # ─── 5. JSON & CSV ROUND-TRIP FIDELITY ──────────────────────────────────────── | |
| def test_json_round_trip(): | |
| """Verify JSON parse and stringify round-trip fidelity.""" | |
| data = {"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "active": True} | |
| str_out = default_registry.execute("json.stringify", {"data": data}, {}) | |
| json_str = str_out["text"] | |
| parse_out = default_registry.execute("json.parse", {"text": json_str}, {}) | |
| assert parse_out["result"] == data | |
| def test_csv_round_trip(): | |
| """Verify CSV stringify and parse round-trip fidelity.""" | |
| rows = [{"id": "1", "name": "Alice"}, {"id": "2", "name": "Bob"}] | |
| str_out = default_registry.execute("csv.stringify", {"rows": rows}, {}) | |
| csv_str = str_out["text"] | |
| parse_out = default_registry.execute("csv.parse", {"text": csv_str}, {}) | |
| assert len(parse_out["rows"]) == 2 | |
| assert parse_out["rows"][0]["name"] == "Alice" | |
| # ─── 6. HTTP REQUEST SSRF PROTECTIONS ───────────────────────────────────────── | |
| def test_http_ssrf_blocked_ips(): | |
| """Verify SSRF validation blocks loopback, cloud metadata, and link-local addresses.""" | |
| blocked_urls = [ | |
| "http://127.0.0.1:8080/admin", | |
| "http://127.0.0.1:22", | |
| "http://169.254.169.254/latest/meta-data/", | |
| "http://[::1]:8080", | |
| "http://10.0.0.1/private", | |
| "http://192.168.1.1/router", | |
| "http://172.16.0.1/corp", | |
| ] | |
| for url in blocked_urls: | |
| res = validate_target_url(url, resolve_dns=False) | |
| assert not res.allowed, f"Expected SSRF URL '{url}' to be blocked" | |
| assert res.blocked_reason is not None | |
| # ─── 7. OUTPUT TRUNCATION (1MB QUOTA) ───────────────────────────────────────── | |
| def test_output_truncation_quota(): | |
| """Verify output ports measure size and safely truncate payloads > 1MB.""" | |
| oversized_data = {"chunk": "X" * (MAX_OUTPUT_QUOTA_BYTES + 2048)} | |
| size_bytes, truncated, preview = measure_output_port_quota(oversized_data, MAX_OUTPUT_QUOTA_BYTES) | |
| assert truncated is True | |
| assert size_bytes > MAX_OUTPUT_QUOTA_BYTES | |
| assert "[TRUNCATED:" in preview | |