auto / tests /test_phase12_node_conformance_parity.py
rsnarsna
feat(phase12): cross-tier node contract conformance and execution parity test suite
7d53bd3
Raw History Blame Contribute Delete
11.7 kB
"""
ALLI Phase 12: Node Contract Conformance & Execution Parity Cross-Tier Test Suite.
Verifies:
1. Complete mathematical node parity (math.sqrt, math.pow, math.abs, math.ceil, math.floor, math.round, math.max, math.min)
across backend execution, runtime evaluation, and synthesized Python code.
2. Data structure nodes (data.list, data.dict, data.tuple, data.set) operations, immutability, and alias parity.
3. Control flow nodes (control.if_else, control.for_loop, control.while_loop, control.try_except).
4. File I/O security (file.read, file.write): traversal rejection (../, ..\\), Windows reserved names (CON, PRN, NUL), byte quotas.
5. JSON & CSV processing (json.parse, json.stringify, csv.parse, csv.stringify) round-trip fidelity.
6. HTTP Request SSRF security (api.http_request): 127.0.0.1, 169.254.169.254, private subnet blocking, redirect validation.
7. Output truncation quota: 1MB limits enforce safe truncation without poisoning downstream nodes.
"""
import ast
import json
import math
import os
import shutil
import tempfile
import pytest
from language_core.node_schema import (
default_registry,
resolve_sandboxed_path,
MAX_FILE_BYTES_QUOTA,
)
from language_core.workflow_executor import (
execute_workflow_graph,
MAX_OUTPUT_QUOTA_BYTES,
measure_output_port_quota,
)
from lambdas.codegen_handler import generate_code_logic
from security.ssrf_validator import validate_target_url
@pytest.fixture
def clean_sandbox():
"""Ensure clean sandbox directory for file tests."""
test_sandbox = tempfile.mkdtemp(prefix="alli_p12_sandbox_")
orig_sandbox = os.environ.get("ALLI_SANDBOX_DIR")
os.environ["ALLI_SANDBOX_DIR"] = test_sandbox
yield test_sandbox
shutil.rmtree(test_sandbox, ignore_errors=True)
if orig_sandbox:
os.environ["ALLI_SANDBOX_DIR"] = orig_sandbox
elif "ALLI_SANDBOX_DIR" in os.environ:
del os.environ["ALLI_SANDBOX_DIR"]
# ─── 1. MATHEMATICAL NODES PARITY ─────────────────────────────────────────────
@pytest.mark.parametrize("node_type,inputs,params,expected_val", [
("math.sqrt", {"x": 49.0}, {}, 7.0),
("math.pow", {"x": 2.0, "y": 3.0}, {}, 8.0),
("math.abs", {"x": -42.5}, {}, 42.5),
("math.ceil", {"x": 3.14}, {}, 4),
("math.floor", {"x": 3.99}, {}, 3),
("math.round", {"x": 3.14159}, {"Decimal Places": 2, "precision": 2}, 3.14),
("math.max", {"a": 10.5, "b": 20.2}, {}, 20.2),
("math.min", {"a": 10.5, "b": 20.2}, {}, 10.5),
])
def test_math_nodes_execution_and_codegen_parity(node_type, inputs, params, expected_val):
"""Verify math node parity between registry executor and Python codegen."""
assert default_registry.has(node_type)
# 1. Direct registry execution
reg_out = default_registry.execute(node_type, inputs, params)
assert math.isclose(reg_out["result"], expected_val, rel_tol=1e-5) or reg_out["result"] == expected_val
# 2. Python AST Codegen execution
wf = {
"nodes": [
{
"id": "math_node",
"type": node_type,
"data": {**inputs, **params},
"inputs": inputs,
"parameters": params,
}
],
"edges": []
}
codegen_result = generate_code_logic(wf)
py_code = codegen_result["code"] if isinstance(codegen_result, dict) else str(codegen_result)
assert "def main():" in py_code
# Execute synthesized Python in restricted namespace
ns = {}
exec(py_code, ns)
res = ns["main"]()
assert math.isclose(res, expected_val, rel_tol=1e-5) or res == expected_val
# ─── 2. DATA STRUCTURE NODES PARITY ───────────────────────────────────────────
def test_data_list_parity():
"""Verify data.list creation, append, and get_index operations."""
out_create = default_registry.execute("data.list", {}, {"Operation": "create", "Initial Items": "[10, 20, 30]"})
assert out_create["result"] == [10, 20, 30]
assert out_create["length"] == 3
out_append = default_registry.execute("data.list", {"items": [10, 20], "item": 30}, {"Operation": "append"})
assert out_append["result"] == [10, 20, 30]
out_get = default_registry.execute("data.list", {"items": [10, 20, 30]}, {"Operation": "get_index", "Index": 1})
assert out_get["result"] == 20
def test_data_dict_parity():
"""Verify data.dict creation, set, and lookup operations."""
out_create = default_registry.execute("data.dict", {}, {"operation": "create", "initial_dict": '{"a": 1, "b": 2}'})
assert out_create["result"] == {"a": 1, "b": 2}
assert "a" in out_create["keys"]
out_set = default_registry.execute("data.dict", {"dict_in": {"a": 1}, "key": "b", "value": 2}, {"operation": "set"})
assert out_set["result"] == {"a": 1, "b": 2}
out_get = default_registry.execute("data.dict", {"dict_in": {"a": 100}, "key": "a"}, {"operation": "get"})
assert out_get["result"] == 100
def test_data_tuple_immutability():
"""Verify data.tuple produces an immutable tuple sequence."""
out = default_registry.execute("data.tuple", {"item_1": "x", "item_2": "y", "item_3": "z"}, {})
assert isinstance(out["tuple_out"], tuple)
assert out["tuple_out"] == ("x", "y", "z")
def test_data_set_operations():
"""Verify data.set union, intersection, and difference."""
out_union = default_registry.execute("data.set", {"set_a": {1, 2}, "set_b": {2, 3}}, {"Operation": "union"})
assert out_union["set_out"] == {1, 2, 3}
out_inter = default_registry.execute("data.set", {"set_a": {1, 2}, "set_b": {2, 3}}, {"Operation": "intersection"})
assert out_inter["set_out"] == {2}
# ─── 3. CONTROL FLOW NODES PARITY ─────────────────────────────────────────────
def test_control_if_else_branching():
"""Verify control.if_else evaluates condition and takes appropriate branch."""
out_true = default_registry.execute("control.if_else", {"condition": True, "then_val": "yes", "else_val": "no"}, {})
assert out_true["result"] == "yes"
assert out_true["branch_taken"] == "then"
assert out_true["then"] == "yes"
assert out_true["else"] == "no"
out_false = default_registry.execute("control.if_else", {"condition": False, "then_val": "yes", "else_val": "no"}, {})
assert out_false["result"] == "no"
assert out_false["branch_taken"] == "else"
assert out_false["then"] == "yes"
assert out_false["else"] == "no"
def test_control_try_except_error_capture():
"""Verify control.try_except catches errors and emits fallback."""
out_ok = default_registry.execute("control.try_except", {"try_val": 42, "fallback_val": 0}, {})
assert out_ok["result"] == 42
assert not out_ok["has_error"]
out_err = default_registry.execute("control.try_except", {"try_val": ZeroDivisionError("division by zero"), "fallback_val": 99}, {})
assert out_err["result"] == 99
assert out_err["has_error"] is True
# ─── 4. FILE I/O SECURITY & TRAVERSAL ─────────────────────────────────────────
def test_file_io_path_containment_and_traversal(clean_sandbox):
"""Verify path traversal attacks and absolute paths outside sandbox are strictly rejected."""
sandbox = clean_sandbox
traversal_paths = [
"../secrets.txt",
"..\\secrets.txt",
"sub/../../passwords.txt",
"/etc/passwd",
"\\Windows\\System32\\cmd.exe",
]
for path in traversal_paths:
with pytest.raises(PermissionError) as exc_info:
resolve_sandboxed_path(path, base_dir=sandbox)
assert "access denied" in str(exc_info.value).lower() or "traversal" in str(exc_info.value).lower()
outside_abs = os.path.abspath(os.path.join(sandbox, "..", "outside_target.txt"))
with pytest.raises(PermissionError) as exc_info:
resolve_sandboxed_path(outside_abs, base_dir=sandbox)
assert "access denied" in str(exc_info.value).lower()
# Null byte check
with pytest.raises(ValueError) as exc_info:
resolve_sandboxed_path("file\0null.txt", base_dir=sandbox)
assert "null byte" in str(exc_info.value).lower()
def test_file_io_windows_reserved_device_names(clean_sandbox):
"""Verify Windows reserved device names (CON, PRN, AUX, NUL) are rejected."""
sandbox = clean_sandbox
reserved_names = ["CON", "PRN", "AUX", "NUL", "COM1", "LPT1", "con.txt", "nul.dat"]
for name in reserved_names:
with pytest.raises(PermissionError) as exc_info:
resolve_sandboxed_path(name, base_dir=sandbox)
assert "reserved" in str(exc_info.value).lower() or "device" in str(exc_info.value).lower()
def test_file_io_quota_limits():
"""Verify write byte quota rejects payloads > 1MB."""
oversized = "A" * (MAX_FILE_BYTES_QUOTA + 1024)
with pytest.raises(ValueError) as exc_info:
default_registry.execute("file.write", {"content": oversized, "file_path": "big.txt"}, {})
assert "quota" in str(exc_info.value).lower() or "exceeds" in str(exc_info.value).lower()
# ─── 5. JSON & CSV ROUND-TRIP FIDELITY ────────────────────────────────────────
def test_json_round_trip():
"""Verify JSON parse and stringify round-trip fidelity."""
data = {"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "active": True}
str_out = default_registry.execute("json.stringify", {"data": data}, {})
json_str = str_out["text"]
parse_out = default_registry.execute("json.parse", {"text": json_str}, {})
assert parse_out["result"] == data
def test_csv_round_trip():
"""Verify CSV stringify and parse round-trip fidelity."""
rows = [{"id": "1", "name": "Alice"}, {"id": "2", "name": "Bob"}]
str_out = default_registry.execute("csv.stringify", {"rows": rows}, {})
csv_str = str_out["text"]
parse_out = default_registry.execute("csv.parse", {"text": csv_str}, {})
assert len(parse_out["rows"]) == 2
assert parse_out["rows"][0]["name"] == "Alice"
# ─── 6. HTTP REQUEST SSRF PROTECTIONS ─────────────────────────────────────────
def test_http_ssrf_blocked_ips():
"""Verify SSRF validation blocks loopback, cloud metadata, and link-local addresses."""
blocked_urls = [
"http://127.0.0.1:8080/admin",
"http://127.0.0.1:22",
"http://169.254.169.254/latest/meta-data/",
"http://[::1]:8080",
"http://10.0.0.1/private",
"http://192.168.1.1/router",
"http://172.16.0.1/corp",
]
for url in blocked_urls:
res = validate_target_url(url, resolve_dns=False)
assert not res.allowed, f"Expected SSRF URL '{url}' to be blocked"
assert res.blocked_reason is not None
# ─── 7. OUTPUT TRUNCATION (1MB QUOTA) ─────────────────────────────────────────
def test_output_truncation_quota():
"""Verify output ports measure size and safely truncate payloads > 1MB."""
oversized_data = {"chunk": "X" * (MAX_OUTPUT_QUOTA_BYTES + 2048)}
size_bytes, truncated, preview = measure_output_port_quota(oversized_data, MAX_OUTPUT_QUOTA_BYTES)
assert truncated is True
assert size_bytes > MAX_OUTPUT_QUOTA_BYTES
assert "[TRUNCATED:" in preview