rsnarsna
feat(blueprint): implement 7-step blueprint-to-node integration, 4 capability states, and security bounds
848f240 Download tests/test_blueprint_to_dynamic_node_integration.py from Rsnarsna/auto: direct link, hf CLI and curl.
- Browser
- Download file 21 kB
-
https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/tests/test_blueprint_to_dynamic_node_integration.py
- Command line
-
hf download hf://spaces/Rsnarsna/auto/tests/test_blueprint_to_dynamic_node_integration.py
-
curl -L -o test_blueprint_to_dynamic_node_integration.py https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/tests/test_blueprint_to_dynamic_node_integration.py
21 kB
| """ | |
| ALLI Phase 10 / Milestone: Package Blueprint -> Dynamic Node Integration | |
| Complete 7-Step Workflow & 4 Explicit Capability States Verification Suite. | |
| Tests: | |
| 1. Package & exact version selection (requests==2.31.0, version='2.31.0', requests@2.31.0). | |
| 2. Blueprint extraction and MongoDB Atlas composite key saving (f'{pkg}@{version}', SHA-256 content_hash). | |
| 3. Cache hit verification: querying same package/version returns cached blueprint with 0 downloads/network calls. | |
| 4. Function parameter & return inspection (required, defaults, types: float, int, str, dict, list, any). | |
| 5. Dynamic node generation satisfying Node Standard v1 (ports, parameters, codegen template). | |
| 6. Distinct capability states (extracted, node_generated, runtime_supported, codegen_supported) & execution diagnostics (EXEC_UNSUPPORTED, CODEGEN_UNSUPPORTED). | |
| 7. Security bounds & hazardous construct rejection (ASTInspectorGuard, eval/exec/subprocess/os.system, 512KB bound). | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import pytest | |
| from unittest.mock import patch, MagicMock | |
| from library_inspector.blueprint_models import ( | |
| LibraryBlueprint, | |
| PackageMeta, | |
| ExtractionMeta, | |
| BlueprintModule, | |
| BlueprintFunction, | |
| BlueprintParameter, | |
| BlueprintReturns, | |
| compute_sha256_hash, | |
| build_library_blueprint_from_extracted, | |
| ) | |
| from library_inspector.node_synthesizer import ( | |
| DynamicNodeSynthesizer, | |
| TypeNormalizer, | |
| ASTInspectorGuard, | |
| StagingNodeRegistry, | |
| MAX_SOURCE_SIZE, | |
| ) | |
| from language_core.node_schema import ( | |
| NodeDefinition, | |
| PortDefinition, | |
| ParameterDefinition, | |
| ParameterType, | |
| default_registry, | |
| ) | |
| from database.mongo import workflow_repo | |
| from lambdas.inspect_handler import introspect_package_logic | |
| from lambdas.validate_handler import validate_workflow_protocol_logic | |
| from lambdas.run_handler import execute_workflow_logic | |
| from lambdas.codegen_handler import generate_code_logic | |
| # ─── TEST 1: PACKAGE & EXACT VERSION SELECTION ──────────────────────────────── | |
| def test_package_and_version_selection(): | |
| """ | |
| Step 1: Verify selecting package name and explicit version across all specifier formats: | |
| - pip standard: 'requests==2.31.0' | |
| - explicit kwargs: package_name='requests', version='2.31.0' | |
| - composite uri: 'requests@2.31.0' | |
| - version bounds: 'requests>=2.31.0' | |
| """ | |
| # 1. Pip equality format | |
| base, ver, comp = workflow_repo._parse_package_and_version("requests==2.31.0") | |
| assert base == "requests" | |
| assert ver == "2.31.0" | |
| assert comp == "requests@2.31.0" | |
| # 2. Explicit kwarg version | |
| base, ver, comp = workflow_repo._parse_package_and_version("requests", version="2.31.0") | |
| assert base == "requests" | |
| assert ver == "2.31.0" | |
| assert comp == "requests@2.31.0" | |
| # 3. Composite @ format | |
| base, ver, comp = workflow_repo._parse_package_and_version("requests@2.31.0") | |
| assert base == "requests" | |
| assert ver == "2.31.0" | |
| assert comp == "requests@2.31.0" | |
| # 4. In inspect_handler, verify package specifier parsing | |
| with patch("lambdas.inspect_handler.workflow_repo.get_blueprint", return_value=None): | |
| with patch("lambdas.inspect_handler.download_package_to_sandbox") as mock_dl: | |
| mock_dl.side_effect = RuntimeError("Mocked stop") | |
| try: | |
| introspect_package_logic("test_spec_pkg==2.0.7") | |
| except RuntimeError: | |
| pass | |
| # download_package_to_sandbox was called with "test_spec_pkg==2.0.7" | |
| assert mock_dl.called | |
| call_spec = mock_dl.call_args[0][0] | |
| assert call_spec == "test_spec_pkg==2.0.7" | |
| # ─── TEST 2: EXTRACTION & PERSISTENCE WITH COMPOSITE KEY & SHA-256 ─────────── | |
| def test_blueprint_extraction_and_composite_key_persistence(): | |
| """ | |
| Step 2: Extract AST blueprint using DynamicNodeSynthesizer and save in MongoDB Atlas | |
| with composite key f'{pkg}@{version}' and SHA-256 content_hash. | |
| """ | |
| pkg_name = "dataprocessor" | |
| pkg_ver = "1.5.0" | |
| composite_key = f"{pkg_name}@{pkg_ver}" | |
| content_data = b"def transform(data: dict) -> list: return list(data.values())" | |
| sha256_hash = compute_sha256_hash(content_data) | |
| assert len(sha256_hash) == 64 | |
| func_symbol = { | |
| "name": "transform", | |
| "docstring": "Transforms input dictionary values into a list.", | |
| "inputs": [{"name": "data", "type": "dict", "required": True}], | |
| "output": {"type": "list"}, | |
| "body_code": "return list(data.values())", | |
| } | |
| # Extract blueprint | |
| bp_model = build_library_blueprint_from_extracted( | |
| base_name=pkg_name, | |
| pkg_meta_dict={"name": pkg_name, "version": pkg_ver, "summary": "Data processing library"}, | |
| extracted_data={"functions": [func_symbol], "classes": []}, | |
| content_hash=sha256_hash, | |
| source_type="wheel", | |
| ) | |
| bp_dict = bp_model.model_dump() | |
| # Synthesize node | |
| node_def = DynamicNodeSynthesizer.synthesize_node( | |
| func_symbol=func_symbol, | |
| module_name=pkg_name, | |
| package_name=pkg_name, | |
| package_version=pkg_ver, | |
| ) | |
| assert node_def.type == f"{pkg_name}.transform" | |
| bp_dict["nodes"] = [node_def.model_dump()] | |
| bp_dict["blueprint_id"] = composite_key | |
| # Save under composite key | |
| workflow_repo.save_blueprint(pkg_name, bp_dict, version=pkg_ver) | |
| # Verify retrieval via composite key | |
| fetched = workflow_repo.get_blueprint(composite_key) | |
| assert fetched is not None | |
| assert fetched["blueprint_id"] == composite_key | |
| assert fetched["extraction"]["content_hash"] == sha256_hash | |
| assert fetched["package"]["version"] == pkg_ver | |
| assert len(fetched["nodes"]) == 1 | |
| assert fetched["nodes"][0]["type"] == f"{pkg_name}.transform" | |
| # ─── TEST 3: CACHE HIT VERIFICATION (0 NETWORK CALLS / DOWNLOADS) ───────────── | |
| def test_blueprint_cache_hit_zero_downloads(): | |
| """ | |
| Step 3: Querying the same package and version again MUST return the cached blueprint | |
| immediately with 0 downloads/network calls and matching content hash. | |
| """ | |
| pkg_name = "networkcached" | |
| pkg_ver = "3.2.1" | |
| composite_key = f"{pkg_name}@{pkg_ver}" | |
| expected_hash = compute_sha256_hash(b"cached_payload_v3_2_1") | |
| cached_schema = { | |
| "blueprint_id": composite_key, | |
| "blueprint_version": "1.0.0", | |
| "package": {"name": pkg_name, "version": pkg_ver, "summary": "Cached network package"}, | |
| "extraction": { | |
| "extracted_at": "2026-10-22T10:00:00Z", | |
| "extractor_version": "1.0.0", | |
| "content_hash": expected_hash, | |
| "source_type": "wheel", | |
| "total_functions": 1, | |
| "total_classes": 0, | |
| "confidence_score": 1.0, | |
| }, | |
| "modules": [], | |
| "initialization": [], | |
| "confidence_scores": {}, | |
| "nodes": [ | |
| { | |
| "type": f"{pkg_name}.fetch", | |
| "label": "Fetch", | |
| "category": pkg_name.capitalize(), | |
| "inputs": [{"id": "url", "name": "url", "type": "str", "required": True}], | |
| "outputs": [{"id": "result", "name": "Result", "type": "str"}], | |
| "parameters": [], | |
| "description": "Fetches remote URL", | |
| } | |
| ], | |
| "package_name": pkg_name, | |
| "version": pkg_ver, | |
| "node_count": 1, | |
| "categories": [pkg_name.capitalize()], | |
| } | |
| workflow_repo.save_blueprint(pkg_name, cached_schema, version=pkg_ver) | |
| with patch("lambdas.inspect_handler.download_package_to_sandbox") as mock_dl: | |
| # 1. Query with package + version | |
| res1 = introspect_package_logic(pkg_name, version=pkg_ver) | |
| assert mock_dl.call_count == 0 | |
| assert res1["blueprint_id"] == composite_key | |
| assert res1["extraction"]["content_hash"] == expected_hash | |
| # 2. Query with composite key string | |
| res2 = introspect_package_logic(composite_key) | |
| assert mock_dl.call_count == 0 | |
| assert res2["blueprint_id"] == composite_key | |
| assert res2["extraction"]["content_hash"] == expected_hash | |
| # 3. Query with pip equality format | |
| res3 = introspect_package_logic(f"{pkg_name}=={pkg_ver}") | |
| assert mock_dl.call_count == 0 | |
| assert res3["blueprint_id"] == composite_key | |
| assert res3["extraction"]["content_hash"] == expected_hash | |
| # ─── TEST 4: FUNCTION PARAMETER & RETURN INSPECTION ────────────────────────── | |
| def test_function_parameter_and_return_inspection(): | |
| """ | |
| Step 4: Accurately parse required parameters, default values, normalized input types | |
| (float, int, str, dict, list, any), and return type. | |
| """ | |
| func_symbol = { | |
| "name": "calculate_metrics", | |
| "docstring": "Computes weighted metrics.\n:raises ValueError: If weights are invalid", | |
| "inputs": [ | |
| {"name": "score", "type": "float", "required": True, "default": None}, | |
| {"name": "count", "type": "int", "required": True, "default": None}, | |
| {"name": "label", "type": "str", "required": False, "default": "default_metric"}, | |
| {"name": "config", "type": "dict", "required": False, "default": {"mode": "strict"}}, | |
| {"name": "factors", "type": "list", "required": False, "default": [1.0, 2.0]}, | |
| {"name": "raw_payload", "type": "Any", "required": False, "default": None}, | |
| ], | |
| "output": {"type": "float"}, | |
| "body_code": "return score * count", | |
| } | |
| node_def = DynamicNodeSynthesizer.synthesize_node( | |
| func_symbol=func_symbol, | |
| module_name="analytics", | |
| package_name="analytics", | |
| package_version="2.1.0", | |
| ) | |
| # 1. Verify input ports mapping and normalized types | |
| inputs_by_id = {p.id: p for p in node_def.inputs} | |
| assert "score" in inputs_by_id | |
| assert inputs_by_id["score"].type == "float" | |
| assert inputs_by_id["score"].required is True | |
| assert "count" in inputs_by_id | |
| assert inputs_by_id["count"].type == "int" | |
| assert inputs_by_id["count"].required is True | |
| assert "label" in inputs_by_id | |
| assert inputs_by_id["label"].type == "str" | |
| assert inputs_by_id["label"].required is False | |
| assert inputs_by_id["label"].default_value == "default_metric" | |
| assert "config" in inputs_by_id | |
| assert inputs_by_id["config"].type == "dict" | |
| assert inputs_by_id["config"].required is False | |
| assert "factors" in inputs_by_id | |
| assert inputs_by_id["factors"].type == "list" | |
| assert inputs_by_id["factors"].required is False | |
| assert "raw_payload" in inputs_by_id | |
| assert inputs_by_id["raw_payload"].type == "any" | |
| assert inputs_by_id["raw_payload"].required is False | |
| # 2. Verify output port mapping and normalized return type | |
| assert len(node_def.outputs) == 1 | |
| assert node_def.outputs[0].id == "result" | |
| assert node_def.outputs[0].type == "float" | |
| # 3. Verify parameters drawer contains optional parameters with defaults | |
| params_by_id = {p.id: p for p in node_def.parameters} | |
| assert "label" in params_by_id | |
| assert params_by_id["label"].default == "default_metric" | |
| assert params_by_id["label"].type == ParameterType.STRING | |
| # ─── TEST 5: DYNAMIC NODE GENERATION (NODE STANDARD V1) ────────────────────── | |
| def test_dynamic_node_generation_node_standard_v1(): | |
| """ | |
| Step 5: Synthesize a standard NodeDefinition satisfying Node Standard v1: | |
| matching input ports, output ports, parameter definitions, and codegen template. | |
| """ | |
| func_symbol = { | |
| "name": "generate_report", | |
| "docstring": "Generates a formatted summary report.", | |
| "inputs": [ | |
| {"name": "title", "type": "str", "required": True}, | |
| {"name": "items", "type": "list", "required": True}, | |
| {"name": "limit", "type": "int", "required": False, "default": 50}, | |
| ], | |
| "output": {"type": "dict"}, | |
| "body_code": "return {'title': title, 'count': len(items[:limit])}", | |
| } | |
| node = DynamicNodeSynthesizer.synthesize_node( | |
| func_symbol=func_symbol, | |
| module_name="reporting", | |
| package_name="reportlab", | |
| package_version="4.0.0", | |
| ) | |
| # Standard fields | |
| assert node.type == "reporting.generate_report" | |
| assert node.label == "Generate Report" | |
| assert node.category == "Reporting" | |
| assert node.version == "4.0.0" | |
| assert node.description == "Generates a formatted summary report." | |
| assert node.icon == "Box" | |
| # Codegen contract | |
| assert node.codegen is not None | |
| assert "{result} = reporting.generate_report(title={title}, items={items}, limit={limit})" in node.codegen.template | |
| assert "reporting" in node.codegen.imports | |
| assert "reportlab==4.0.0" in node.codegen.requirements | |
| # Verification metadata | |
| assert node.verification is not None | |
| assert node.verification.node_kind == "value_producer" | |
| # ─── TEST 6: CAPABILITY STATES SEPARATION & GRAPH EXECUTION DIAGNOSTICS ─────── | |
| def test_capability_states_separation_and_execution_diagnostics(): | |
| """ | |
| Step 6: Clearly delineate and surface 4 capability states: | |
| - extracted: bool | |
| - node_generated: bool | |
| - runtime_supported: bool | |
| - codegen_supported: bool | |
| In graph validation and execution: | |
| - runtime_supported: false emits structured diagnostic EXEC_UNSUPPORTED | |
| - codegen_supported: false emits structured diagnostic CODEGEN_UNSUPPORTED | |
| """ | |
| # 1. Stdlib node capability states: fully supported | |
| stdlib_sym = { | |
| "name": "sqrt", | |
| "docstring": "Square root", | |
| "inputs": [{"name": "x", "type": "float", "required": True}], | |
| "output": {"type": "float"}, | |
| } | |
| stdlib_node = DynamicNodeSynthesizer.synthesize_node(stdlib_sym, module_name="math", package_name="math") | |
| assert stdlib_node.capability_states == { | |
| "extracted": True, | |
| "node_generated": True, | |
| "runtime_supported": True, | |
| "codegen_supported": True, | |
| } | |
| # 2. Unsupported runtime node (e.g. external package without local runtime executor) | |
| ext_sym = { | |
| "name": "remote_predict", | |
| "docstring": "External inference callable", | |
| "inputs": [{"name": "data", "type": "dict", "required": True}], | |
| "output": {"type": "dict"}, | |
| "runtime_supported": False, | |
| "codegen_supported": True, | |
| } | |
| unsupported_runtime_node = DynamicNodeSynthesizer.synthesize_node( | |
| ext_sym, | |
| module_name="external_ml", | |
| package_name="external_ml", | |
| package_version="1.0.0", | |
| runtime_supported=False, | |
| ) | |
| assert unsupported_runtime_node.capability_states["extracted"] is True | |
| assert unsupported_runtime_node.capability_states["node_generated"] is True | |
| assert unsupported_runtime_node.capability_states["runtime_supported"] is False | |
| assert unsupported_runtime_node.capability_states["codegen_supported"] is True | |
| # 3. Test graph validation rejection with EXEC_UNSUPPORTED | |
| graph_unsupported = { | |
| "nodes": [ | |
| { | |
| "id": "start_1", | |
| "type": "workflow.start", | |
| "data": {"label": "Start"}, | |
| }, | |
| { | |
| "id": "node_ml", | |
| "type": "external_ml.remote_predict", | |
| "data": { | |
| "label": "Remote Predict", | |
| "capability_states": unsupported_runtime_node.capability_states, | |
| }, | |
| }, | |
| ], | |
| "edges": [ | |
| {"id": "e1", "source": "start_1", "target": "node_ml", "sourceHandle": "out", "targetHandle": "data"} | |
| ], | |
| } | |
| val_res = validate_workflow_protocol_logic(graph_unsupported) | |
| assert val_res["valid"] is False | |
| assert val_res["can_execute"] is False | |
| exec_unsupported_diags = [ | |
| d for d in val_res["diagnostics"] | |
| if d.get("code") == "EXEC_UNSUPPORTED" or "VAL-UNSUPPORTED-RUNTIME" in str(d.get("message", "")) | |
| ] | |
| assert len(exec_unsupported_diags) > 0 | |
| assert exec_unsupported_diags[0]["severity"] == "error" | |
| assert exec_unsupported_diags[0]["node_id"] == "node_ml" | |
| # 4. Test run_handler execution handles unsupported node gracefully | |
| exec_res = execute_workflow_logic(graph_unsupported) | |
| assert exec_res["success"] is False | |
| assert "node_ml" in exec_res["node_results"] | |
| node_res = exec_res["node_results"]["node_ml"] | |
| assert node_res["status"] == "failed" | |
| assert node_res["error_semantics"]["code"] == "EXEC_UNSUPPORTED" | |
| # 5. Unsupported codegen node emits CODEGEN_UNSUPPORTED | |
| graph_unsupported_codegen = { | |
| "name": "unsupported_codegen_wf", | |
| "nodes": [ | |
| { | |
| "id": "start_2", | |
| "type": "workflow.start", | |
| "data": {"label": "Start"}, | |
| }, | |
| { | |
| "id": "node_bad_codegen", | |
| "type": "special.dynamic_call", | |
| "data": { | |
| "label": "Dynamic Call", | |
| "capability_states": { | |
| "extracted": True, | |
| "node_generated": True, | |
| "runtime_supported": False, | |
| "codegen_supported": False, | |
| }, | |
| }, | |
| }, | |
| ], | |
| "edges": [ | |
| {"id": "e2", "source": "start_2", "target": "node_bad_codegen", "sourceHandle": "out", "targetHandle": "input"} | |
| ], | |
| } | |
| codegen_res = generate_code_logic(graph_unsupported_codegen, validate=False) | |
| assert codegen_res["success"] is False | |
| codegen_diags = [d for d in codegen_res["diagnostics"] if d.get("code") == "CODEGEN_UNSUPPORTED"] | |
| assert len(codegen_diags) > 0 | |
| assert codegen_diags[0]["severity"] == "error" | |
| assert codegen_diags[0]["node_id"] == "node_bad_codegen" | |
| # ─── TEST 7: SECURITY BOUNDS & HAZARDOUS CONSTRUCT REJECTION ───────────────── | |
| def test_security_bounds_and_hazardous_constructs(): | |
| """ | |
| Step 7: Block dangerous calls (eval, exec, os.system, subprocess) via ASTInspectorGuard. | |
| Prevent package metadata from bypassing authentication, size bounds (512 KB), or AST limits. | |
| """ | |
| # 1. Prohibited eval call | |
| code_eval = "def unsafe_eval(x): return eval(x)" | |
| is_safe, warnings, _ = ASTInspectorGuard.inspect_and_guard(code_eval) | |
| assert is_safe is False | |
| assert any("eval" in w for w in warnings) | |
| # 2. Prohibited exec call | |
| code_exec = "def unsafe_exec(x): exec(x)" | |
| is_safe, warnings, _ = ASTInspectorGuard.inspect_and_guard(code_exec) | |
| assert is_safe is False | |
| assert any("exec" in w for w in warnings) | |
| # 3. Prohibited os.system call | |
| code_os = "import os\ndef unsafe_cmd(cmd): os.system(cmd)" | |
| is_safe, warnings, _ = ASTInspectorGuard.inspect_and_guard(code_os) | |
| assert is_safe is False | |
| assert any("os.system" in w for w in warnings) | |
| # 4. Prohibited subprocess call | |
| code_sub = "import subprocess\ndef unsafe_proc(): subprocess.Popen(['ls'])" | |
| is_safe, warnings, _ = ASTInspectorGuard.inspect_and_guard(code_sub) | |
| assert is_safe is False | |
| assert any("subprocess.Popen" in w for w in warnings) | |
| # 5. Resource bound: Source code size > 512 KB | |
| oversized_code = "x = 1\n" * ((MAX_SOURCE_SIZE // 6) + 100) | |
| assert len(oversized_code.encode("utf-8")) > MAX_SOURCE_SIZE | |
| with pytest.raises(ValueError, match="exceeds limit"): | |
| ASTInspectorGuard.inspect_and_guard(oversized_code) | |
| # 6. Synthesizer automatically detects hazardous body_code and marks capability states | |
| hazardous_sym = { | |
| "name": "dangerous_action", | |
| "docstring": "Executes shell command", | |
| "inputs": [{"name": "cmd", "type": "str", "required": True}], | |
| "output": {"type": "Any"}, | |
| "body_code": "import os\nos.system(cmd)", | |
| } | |
| haz_node = DynamicNodeSynthesizer.synthesize_node( | |
| func_symbol=hazardous_sym, | |
| module_name="shelltools", | |
| package_name="shelltools", | |
| ) | |
| assert haz_node.capability_states["runtime_supported"] is False | |
| assert haz_node.capability_states["codegen_supported"] is False | |
| # 7. Isolated staging registry marks hazardous candidate as rejected | |
| staging = StagingNodeRegistry() | |
| ok, errs = staging.register_candidate( | |
| haz_node, | |
| source_module="shelltools", | |
| is_safe=False, | |
| security_warnings=["Hazardous os.system call detected"], | |
| ) | |
| assert ok is True | |
| record = staging.get_record("shelltools.dangerous_action") | |
| assert record is not None | |
| assert record.is_safe is False | |
| assert record.approval_status == "rejected" | |
| # Attempted promotion of rejected node is strictly blocked | |
| promoted, msg = staging.promote_to_target("shelltools.dangerous_action", default_registry) | |
| assert promoted is False | |
| assert "previously rejected" in msg | |