auto / tests /test_blueprint_to_dynamic_node_integration.py
rsnarsna
feat(blueprint): implement 7-step blueprint-to-node integration, 4 capability states, and security bounds
848f240
Raw History Blame Contribute Delete
21 kB
"""
ALLI Phase 10 / Milestone: Package Blueprint -> Dynamic Node Integration
Complete 7-Step Workflow & 4 Explicit Capability States Verification Suite.
Tests:
1. Package & exact version selection (requests==2.31.0, version='2.31.0', requests@2.31.0).
2. Blueprint extraction and MongoDB Atlas composite key saving (f'{pkg}@{version}', SHA-256 content_hash).
3. Cache hit verification: querying same package/version returns cached blueprint with 0 downloads/network calls.
4. Function parameter & return inspection (required, defaults, types: float, int, str, dict, list, any).
5. Dynamic node generation satisfying Node Standard v1 (ports, parameters, codegen template).
6. Distinct capability states (extracted, node_generated, runtime_supported, codegen_supported) & execution diagnostics (EXEC_UNSUPPORTED, CODEGEN_UNSUPPORTED).
7. Security bounds & hazardous construct rejection (ASTInspectorGuard, eval/exec/subprocess/os.system, 512KB bound).
"""
import os
import sys
import json
import pytest
from unittest.mock import patch, MagicMock
from library_inspector.blueprint_models import (
LibraryBlueprint,
PackageMeta,
ExtractionMeta,
BlueprintModule,
BlueprintFunction,
BlueprintParameter,
BlueprintReturns,
compute_sha256_hash,
build_library_blueprint_from_extracted,
)
from library_inspector.node_synthesizer import (
DynamicNodeSynthesizer,
TypeNormalizer,
ASTInspectorGuard,
StagingNodeRegistry,
MAX_SOURCE_SIZE,
)
from language_core.node_schema import (
NodeDefinition,
PortDefinition,
ParameterDefinition,
ParameterType,
default_registry,
)
from database.mongo import workflow_repo
from lambdas.inspect_handler import introspect_package_logic
from lambdas.validate_handler import validate_workflow_protocol_logic
from lambdas.run_handler import execute_workflow_logic
from lambdas.codegen_handler import generate_code_logic
# ─── TEST 1: PACKAGE & EXACT VERSION SELECTION ────────────────────────────────
def test_package_and_version_selection():
"""
Step 1: Verify selecting package name and explicit version across all specifier formats:
- pip standard: 'requests==2.31.0'
- explicit kwargs: package_name='requests', version='2.31.0'
- composite uri: 'requests@2.31.0'
- version bounds: 'requests>=2.31.0'
"""
# 1. Pip equality format
base, ver, comp = workflow_repo._parse_package_and_version("requests==2.31.0")
assert base == "requests"
assert ver == "2.31.0"
assert comp == "requests@2.31.0"
# 2. Explicit kwarg version
base, ver, comp = workflow_repo._parse_package_and_version("requests", version="2.31.0")
assert base == "requests"
assert ver == "2.31.0"
assert comp == "requests@2.31.0"
# 3. Composite @ format
base, ver, comp = workflow_repo._parse_package_and_version("requests@2.31.0")
assert base == "requests"
assert ver == "2.31.0"
assert comp == "requests@2.31.0"
# 4. In inspect_handler, verify package specifier parsing
with patch("lambdas.inspect_handler.workflow_repo.get_blueprint", return_value=None):
with patch("lambdas.inspect_handler.download_package_to_sandbox") as mock_dl:
mock_dl.side_effect = RuntimeError("Mocked stop")
try:
introspect_package_logic("test_spec_pkg==2.0.7")
except RuntimeError:
pass
# download_package_to_sandbox was called with "test_spec_pkg==2.0.7"
assert mock_dl.called
call_spec = mock_dl.call_args[0][0]
assert call_spec == "test_spec_pkg==2.0.7"
# ─── TEST 2: EXTRACTION & PERSISTENCE WITH COMPOSITE KEY & SHA-256 ───────────
def test_blueprint_extraction_and_composite_key_persistence():
"""
Step 2: Extract AST blueprint using DynamicNodeSynthesizer and save in MongoDB Atlas
with composite key f'{pkg}@{version}' and SHA-256 content_hash.
"""
pkg_name = "dataprocessor"
pkg_ver = "1.5.0"
composite_key = f"{pkg_name}@{pkg_ver}"
content_data = b"def transform(data: dict) -> list: return list(data.values())"
sha256_hash = compute_sha256_hash(content_data)
assert len(sha256_hash) == 64
func_symbol = {
"name": "transform",
"docstring": "Transforms input dictionary values into a list.",
"inputs": [{"name": "data", "type": "dict", "required": True}],
"output": {"type": "list"},
"body_code": "return list(data.values())",
}
# Extract blueprint
bp_model = build_library_blueprint_from_extracted(
base_name=pkg_name,
pkg_meta_dict={"name": pkg_name, "version": pkg_ver, "summary": "Data processing library"},
extracted_data={"functions": [func_symbol], "classes": []},
content_hash=sha256_hash,
source_type="wheel",
)
bp_dict = bp_model.model_dump()
# Synthesize node
node_def = DynamicNodeSynthesizer.synthesize_node(
func_symbol=func_symbol,
module_name=pkg_name,
package_name=pkg_name,
package_version=pkg_ver,
)
assert node_def.type == f"{pkg_name}.transform"
bp_dict["nodes"] = [node_def.model_dump()]
bp_dict["blueprint_id"] = composite_key
# Save under composite key
workflow_repo.save_blueprint(pkg_name, bp_dict, version=pkg_ver)
# Verify retrieval via composite key
fetched = workflow_repo.get_blueprint(composite_key)
assert fetched is not None
assert fetched["blueprint_id"] == composite_key
assert fetched["extraction"]["content_hash"] == sha256_hash
assert fetched["package"]["version"] == pkg_ver
assert len(fetched["nodes"]) == 1
assert fetched["nodes"][0]["type"] == f"{pkg_name}.transform"
# ─── TEST 3: CACHE HIT VERIFICATION (0 NETWORK CALLS / DOWNLOADS) ─────────────
def test_blueprint_cache_hit_zero_downloads():
"""
Step 3: Querying the same package and version again MUST return the cached blueprint
immediately with 0 downloads/network calls and matching content hash.
"""
pkg_name = "networkcached"
pkg_ver = "3.2.1"
composite_key = f"{pkg_name}@{pkg_ver}"
expected_hash = compute_sha256_hash(b"cached_payload_v3_2_1")
cached_schema = {
"blueprint_id": composite_key,
"blueprint_version": "1.0.0",
"package": {"name": pkg_name, "version": pkg_ver, "summary": "Cached network package"},
"extraction": {
"extracted_at": "2026-10-22T10:00:00Z",
"extractor_version": "1.0.0",
"content_hash": expected_hash,
"source_type": "wheel",
"total_functions": 1,
"total_classes": 0,
"confidence_score": 1.0,
},
"modules": [],
"initialization": [],
"confidence_scores": {},
"nodes": [
{
"type": f"{pkg_name}.fetch",
"label": "Fetch",
"category": pkg_name.capitalize(),
"inputs": [{"id": "url", "name": "url", "type": "str", "required": True}],
"outputs": [{"id": "result", "name": "Result", "type": "str"}],
"parameters": [],
"description": "Fetches remote URL",
}
],
"package_name": pkg_name,
"version": pkg_ver,
"node_count": 1,
"categories": [pkg_name.capitalize()],
}
workflow_repo.save_blueprint(pkg_name, cached_schema, version=pkg_ver)
with patch("lambdas.inspect_handler.download_package_to_sandbox") as mock_dl:
# 1. Query with package + version
res1 = introspect_package_logic(pkg_name, version=pkg_ver)
assert mock_dl.call_count == 0
assert res1["blueprint_id"] == composite_key
assert res1["extraction"]["content_hash"] == expected_hash
# 2. Query with composite key string
res2 = introspect_package_logic(composite_key)
assert mock_dl.call_count == 0
assert res2["blueprint_id"] == composite_key
assert res2["extraction"]["content_hash"] == expected_hash
# 3. Query with pip equality format
res3 = introspect_package_logic(f"{pkg_name}=={pkg_ver}")
assert mock_dl.call_count == 0
assert res3["blueprint_id"] == composite_key
assert res3["extraction"]["content_hash"] == expected_hash
# ─── TEST 4: FUNCTION PARAMETER & RETURN INSPECTION ──────────────────────────
def test_function_parameter_and_return_inspection():
"""
Step 4: Accurately parse required parameters, default values, normalized input types
(float, int, str, dict, list, any), and return type.
"""
func_symbol = {
"name": "calculate_metrics",
"docstring": "Computes weighted metrics.\n:raises ValueError: If weights are invalid",
"inputs": [
{"name": "score", "type": "float", "required": True, "default": None},
{"name": "count", "type": "int", "required": True, "default": None},
{"name": "label", "type": "str", "required": False, "default": "default_metric"},
{"name": "config", "type": "dict", "required": False, "default": {"mode": "strict"}},
{"name": "factors", "type": "list", "required": False, "default": [1.0, 2.0]},
{"name": "raw_payload", "type": "Any", "required": False, "default": None},
],
"output": {"type": "float"},
"body_code": "return score * count",
}
node_def = DynamicNodeSynthesizer.synthesize_node(
func_symbol=func_symbol,
module_name="analytics",
package_name="analytics",
package_version="2.1.0",
)
# 1. Verify input ports mapping and normalized types
inputs_by_id = {p.id: p for p in node_def.inputs}
assert "score" in inputs_by_id
assert inputs_by_id["score"].type == "float"
assert inputs_by_id["score"].required is True
assert "count" in inputs_by_id
assert inputs_by_id["count"].type == "int"
assert inputs_by_id["count"].required is True
assert "label" in inputs_by_id
assert inputs_by_id["label"].type == "str"
assert inputs_by_id["label"].required is False
assert inputs_by_id["label"].default_value == "default_metric"
assert "config" in inputs_by_id
assert inputs_by_id["config"].type == "dict"
assert inputs_by_id["config"].required is False
assert "factors" in inputs_by_id
assert inputs_by_id["factors"].type == "list"
assert inputs_by_id["factors"].required is False
assert "raw_payload" in inputs_by_id
assert inputs_by_id["raw_payload"].type == "any"
assert inputs_by_id["raw_payload"].required is False
# 2. Verify output port mapping and normalized return type
assert len(node_def.outputs) == 1
assert node_def.outputs[0].id == "result"
assert node_def.outputs[0].type == "float"
# 3. Verify parameters drawer contains optional parameters with defaults
params_by_id = {p.id: p for p in node_def.parameters}
assert "label" in params_by_id
assert params_by_id["label"].default == "default_metric"
assert params_by_id["label"].type == ParameterType.STRING
# ─── TEST 5: DYNAMIC NODE GENERATION (NODE STANDARD V1) ──────────────────────
def test_dynamic_node_generation_node_standard_v1():
"""
Step 5: Synthesize a standard NodeDefinition satisfying Node Standard v1:
matching input ports, output ports, parameter definitions, and codegen template.
"""
func_symbol = {
"name": "generate_report",
"docstring": "Generates a formatted summary report.",
"inputs": [
{"name": "title", "type": "str", "required": True},
{"name": "items", "type": "list", "required": True},
{"name": "limit", "type": "int", "required": False, "default": 50},
],
"output": {"type": "dict"},
"body_code": "return {'title': title, 'count': len(items[:limit])}",
}
node = DynamicNodeSynthesizer.synthesize_node(
func_symbol=func_symbol,
module_name="reporting",
package_name="reportlab",
package_version="4.0.0",
)
# Standard fields
assert node.type == "reporting.generate_report"
assert node.label == "Generate Report"
assert node.category == "Reporting"
assert node.version == "4.0.0"
assert node.description == "Generates a formatted summary report."
assert node.icon == "Box"
# Codegen contract
assert node.codegen is not None
assert "{result} = reporting.generate_report(title={title}, items={items}, limit={limit})" in node.codegen.template
assert "reporting" in node.codegen.imports
assert "reportlab==4.0.0" in node.codegen.requirements
# Verification metadata
assert node.verification is not None
assert node.verification.node_kind == "value_producer"
# ─── TEST 6: CAPABILITY STATES SEPARATION & GRAPH EXECUTION DIAGNOSTICS ───────
def test_capability_states_separation_and_execution_diagnostics():
"""
Step 6: Clearly delineate and surface 4 capability states:
- extracted: bool
- node_generated: bool
- runtime_supported: bool
- codegen_supported: bool
In graph validation and execution:
- runtime_supported: false emits structured diagnostic EXEC_UNSUPPORTED
- codegen_supported: false emits structured diagnostic CODEGEN_UNSUPPORTED
"""
# 1. Stdlib node capability states: fully supported
stdlib_sym = {
"name": "sqrt",
"docstring": "Square root",
"inputs": [{"name": "x", "type": "float", "required": True}],
"output": {"type": "float"},
}
stdlib_node = DynamicNodeSynthesizer.synthesize_node(stdlib_sym, module_name="math", package_name="math")
assert stdlib_node.capability_states == {
"extracted": True,
"node_generated": True,
"runtime_supported": True,
"codegen_supported": True,
}
# 2. Unsupported runtime node (e.g. external package without local runtime executor)
ext_sym = {
"name": "remote_predict",
"docstring": "External inference callable",
"inputs": [{"name": "data", "type": "dict", "required": True}],
"output": {"type": "dict"},
"runtime_supported": False,
"codegen_supported": True,
}
unsupported_runtime_node = DynamicNodeSynthesizer.synthesize_node(
ext_sym,
module_name="external_ml",
package_name="external_ml",
package_version="1.0.0",
runtime_supported=False,
)
assert unsupported_runtime_node.capability_states["extracted"] is True
assert unsupported_runtime_node.capability_states["node_generated"] is True
assert unsupported_runtime_node.capability_states["runtime_supported"] is False
assert unsupported_runtime_node.capability_states["codegen_supported"] is True
# 3. Test graph validation rejection with EXEC_UNSUPPORTED
graph_unsupported = {
"nodes": [
{
"id": "start_1",
"type": "workflow.start",
"data": {"label": "Start"},
},
{
"id": "node_ml",
"type": "external_ml.remote_predict",
"data": {
"label": "Remote Predict",
"capability_states": unsupported_runtime_node.capability_states,
},
},
],
"edges": [
{"id": "e1", "source": "start_1", "target": "node_ml", "sourceHandle": "out", "targetHandle": "data"}
],
}
val_res = validate_workflow_protocol_logic(graph_unsupported)
assert val_res["valid"] is False
assert val_res["can_execute"] is False
exec_unsupported_diags = [
d for d in val_res["diagnostics"]
if d.get("code") == "EXEC_UNSUPPORTED" or "VAL-UNSUPPORTED-RUNTIME" in str(d.get("message", ""))
]
assert len(exec_unsupported_diags) > 0
assert exec_unsupported_diags[0]["severity"] == "error"
assert exec_unsupported_diags[0]["node_id"] == "node_ml"
# 4. Test run_handler execution handles unsupported node gracefully
exec_res = execute_workflow_logic(graph_unsupported)
assert exec_res["success"] is False
assert "node_ml" in exec_res["node_results"]
node_res = exec_res["node_results"]["node_ml"]
assert node_res["status"] == "failed"
assert node_res["error_semantics"]["code"] == "EXEC_UNSUPPORTED"
# 5. Unsupported codegen node emits CODEGEN_UNSUPPORTED
graph_unsupported_codegen = {
"name": "unsupported_codegen_wf",
"nodes": [
{
"id": "start_2",
"type": "workflow.start",
"data": {"label": "Start"},
},
{
"id": "node_bad_codegen",
"type": "special.dynamic_call",
"data": {
"label": "Dynamic Call",
"capability_states": {
"extracted": True,
"node_generated": True,
"runtime_supported": False,
"codegen_supported": False,
},
},
},
],
"edges": [
{"id": "e2", "source": "start_2", "target": "node_bad_codegen", "sourceHandle": "out", "targetHandle": "input"}
],
}
codegen_res = generate_code_logic(graph_unsupported_codegen, validate=False)
assert codegen_res["success"] is False
codegen_diags = [d for d in codegen_res["diagnostics"] if d.get("code") == "CODEGEN_UNSUPPORTED"]
assert len(codegen_diags) > 0
assert codegen_diags[0]["severity"] == "error"
assert codegen_diags[0]["node_id"] == "node_bad_codegen"
# ─── TEST 7: SECURITY BOUNDS & HAZARDOUS CONSTRUCT REJECTION ─────────────────
def test_security_bounds_and_hazardous_constructs():
"""
Step 7: Block dangerous calls (eval, exec, os.system, subprocess) via ASTInspectorGuard.
Prevent package metadata from bypassing authentication, size bounds (512 KB), or AST limits.
"""
# 1. Prohibited eval call
code_eval = "def unsafe_eval(x): return eval(x)"
is_safe, warnings, _ = ASTInspectorGuard.inspect_and_guard(code_eval)
assert is_safe is False
assert any("eval" in w for w in warnings)
# 2. Prohibited exec call
code_exec = "def unsafe_exec(x): exec(x)"
is_safe, warnings, _ = ASTInspectorGuard.inspect_and_guard(code_exec)
assert is_safe is False
assert any("exec" in w for w in warnings)
# 3. Prohibited os.system call
code_os = "import os\ndef unsafe_cmd(cmd): os.system(cmd)"
is_safe, warnings, _ = ASTInspectorGuard.inspect_and_guard(code_os)
assert is_safe is False
assert any("os.system" in w for w in warnings)
# 4. Prohibited subprocess call
code_sub = "import subprocess\ndef unsafe_proc(): subprocess.Popen(['ls'])"
is_safe, warnings, _ = ASTInspectorGuard.inspect_and_guard(code_sub)
assert is_safe is False
assert any("subprocess.Popen" in w for w in warnings)
# 5. Resource bound: Source code size > 512 KB
oversized_code = "x = 1\n" * ((MAX_SOURCE_SIZE // 6) + 100)
assert len(oversized_code.encode("utf-8")) > MAX_SOURCE_SIZE
with pytest.raises(ValueError, match="exceeds limit"):
ASTInspectorGuard.inspect_and_guard(oversized_code)
# 6. Synthesizer automatically detects hazardous body_code and marks capability states
hazardous_sym = {
"name": "dangerous_action",
"docstring": "Executes shell command",
"inputs": [{"name": "cmd", "type": "str", "required": True}],
"output": {"type": "Any"},
"body_code": "import os\nos.system(cmd)",
}
haz_node = DynamicNodeSynthesizer.synthesize_node(
func_symbol=hazardous_sym,
module_name="shelltools",
package_name="shelltools",
)
assert haz_node.capability_states["runtime_supported"] is False
assert haz_node.capability_states["codegen_supported"] is False
# 7. Isolated staging registry marks hazardous candidate as rejected
staging = StagingNodeRegistry()
ok, errs = staging.register_candidate(
haz_node,
source_module="shelltools",
is_safe=False,
security_warnings=["Hazardous os.system call detected"],
)
assert ok is True
record = staging.get_record("shelltools.dangerous_action")
assert record is not None
assert record.is_safe is False
assert record.approval_status == "rejected"
# Attempted promotion of rejected node is strictly blocked
promoted, msg = staging.promote_to_target("shelltools.dangerous_action", default_registry)
assert promoted is False
assert "previously rejected" in msg