Download node_validation_lab/tests/test_inventory_integrity.py from Rsnarsna/auto: direct link, hf CLI and curl.
- Browser
- Download file 13.9 kB
-
https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/node_validation_lab/tests/test_inventory_integrity.py
- Command line
-
hf download hf://spaces/Rsnarsna/auto/node_validation_lab/tests/test_inventory_integrity.py
-
curl -L -o test_inventory_integrity.py https://huggingface.co/spaces/Rsnarsna/auto/resolve/main/node_validation_lab/tests/test_inventory_integrity.py
13.9 kB
| """Unit and integration test suite for Node Inventory Integrity (Phase 2). | |
| Validates: | |
| 1. Inventory JSON loading & structural validity. | |
| 2. Canonical node uniqueness (exactly 59 nodes, 0 duplicate IDs). | |
| 3. Frontend/backend registration parity (47 backend, 59 frontend). | |
| 4. Expected frontend-only registrations (12 identified, 0 backend-only). | |
| 5. Schema mismatch reporting (15 full parity, 32 schema mismatches documented). | |
| 6. Independent runtime vs. codegen capability reporting. | |
| 7. Deterministic JSON report generation. | |
| 8. Correct exit codes for inventory errors (0 for success, 1 for integrity failure). | |
| """ | |
| from __future__ import annotations | |
| import copy | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| import tempfile | |
| from pathlib import Path | |
| from typing import Any, Dict | |
| import pytest | |
| from language_core.node_schema import default_registry | |
| from node_validation_lab.fixtures.inventory_loader import ( | |
| get_inventory_path, | |
| load_complete_inventory_bundle, | |
| load_inventory_contracts, | |
| load_inventory_summary, | |
| load_raw_inventory, | |
| ) | |
| from node_validation_lab.run_validation import ( | |
| EXPECTED_FRONTEND_ONLY_TYPES, | |
| EXPECTED_FULL_PARITY_TYPES, | |
| execute_inventory_integrity_validation, | |
| main, | |
| perform_inventory_integrity_checks, | |
| ) | |
| from node_validation_lab.schemas.models import ( | |
| InventoryIntegrityReport, | |
| InventorySummaryCounts, | |
| NodeValidationContract, | |
| ValidationStatus, | |
| ) | |
| def inventory_bundle(): | |
| """Module-level fixture loading raw data, summary, and contracts.""" | |
| raw_data, summary, contracts = load_complete_inventory_bundle() | |
| return raw_data, summary, contracts | |
| # ─── TEST 1: Inventory JSON Loading & Structural Validity ─────────────────────── | |
| def test_inventory_loading_and_structural_validity(inventory_bundle): | |
| """Test 1: Verify inventory JSON loads deterministically and adheres to schema.""" | |
| raw_data, summary, contracts = inventory_bundle | |
| # 1.1 Root keys | |
| assert isinstance(raw_data, dict), "Root must be a JSON dictionary" | |
| assert "audit_metadata" in raw_data, "Missing 'audit_metadata' key" | |
| assert "inventory" in raw_data, "Missing 'inventory' key" | |
| assert isinstance(raw_data["inventory"], list), "'inventory' must be a list" | |
| # 1.2 Audit metadata | |
| meta = raw_data["audit_metadata"] | |
| assert "timestamp" in meta, "Missing timestamp in audit metadata" | |
| assert "phase" in meta, "Missing phase in audit metadata" | |
| assert "summary_counts" in meta, "Missing summary_counts in audit metadata" | |
| # 1.3 Typed contracts | |
| assert len(contracts) == 59, f"Expected 59 contracts, got {len(contracts)}" | |
| for contract in contracts: | |
| assert isinstance(contract, NodeValidationContract) | |
| assert isinstance(contract.type, str) and len(contract.type) > 0 | |
| assert contract.canonical_node_type == contract.type | |
| assert isinstance(contract.status, str) | |
| assert contract.capabilities is not None | |
| assert contract.validation_status in ( | |
| ValidationStatus.PASS, | |
| ValidationStatus.UNSUPPORTED, | |
| ValidationStatus.FAIL, | |
| ValidationStatus.NOT_TESTED, | |
| ) | |
| # ─── TEST 2: Canonical Node Uniqueness (Exactly 59 Nodes, 0 Duplicate IDs) ───── | |
| def test_canonical_node_uniqueness(inventory_bundle): | |
| """Test 2: Verify exactly 59 canonical nodes exist with 0 duplicate IDs.""" | |
| _, summary, contracts = inventory_bundle | |
| all_types = [c.type for c in contracts] | |
| unique_types = set(all_types) | |
| assert len(all_types) == 59, f"Total node count must be exactly 59, got {len(all_types)}" | |
| assert len(unique_types) == 59, f"Duplicate node types detected! Unique count is {len(unique_types)}" | |
| assert summary.total_unique_node_types == 59 | |
| # Verify ID format (dotted category.action or valid namespace) | |
| for node_type in unique_types: | |
| assert "." in node_type, f"Node type '{node_type}' must be namespaced with a dot" | |
| category, name = node_type.split(".", 1) | |
| assert len(category) > 0 and len(name) > 0, f"Malformed node type: '{node_type}'" | |
| # ─── TEST 3: Frontend/Backend Registration Parity (47 BE, 59 FE) ─────────────── | |
| def test_registration_parity_counts(inventory_bundle): | |
| """Test 3: Verify registration parity (47 backend registered, 59 frontend canonical).""" | |
| _, summary, contracts = inventory_bundle | |
| be_registered = [c for c in contracts if c.is_backend_registered] | |
| fe_registered = [c for c in contracts if c.is_frontend_registered] | |
| assert len(be_registered) == 47, f"Expected 47 backend registered nodes, got {len(be_registered)}" | |
| assert len(fe_registered) == 59, f"Expected 59 frontend registered nodes, got {len(fe_registered)}" | |
| assert summary.backend_registered_count == 47 | |
| assert summary.frontend_canonical_count == 59 | |
| # Cross-reference with live backend schema registry | |
| backend_schemas = default_registry.list_all() | |
| assert len(backend_schemas) == 47, f"default_registry must have exactly 47 nodes, got {len(backend_schemas)}" | |
| backend_schema_types = {s.type for s in backend_schemas} | |
| be_contract_types = {c.type for c in be_registered} | |
| assert backend_schema_types == be_contract_types, ( | |
| f"Mismatch between default_registry types and backend-registered contracts! " | |
| f"Diff: {backend_schema_types ^ be_contract_types}" | |
| ) | |
| # ─── TEST 4: Expected Frontend-Only Registrations (12 Identified, 0 BE-Only) ─── | |
| def test_expected_frontend_only_registrations(inventory_bundle): | |
| """Test 4: Verify exactly 12 expected frontend-only nodes and 0 backend-only nodes.""" | |
| _, summary, contracts = inventory_bundle | |
| fe_only_nodes = {c.type for c in contracts if c.status == "FRONTEND_ONLY"} | |
| be_only_nodes = {c.type for c in contracts if c.status == "BACKEND_ONLY"} | |
| assert len(fe_only_nodes) == 12, f"Expected 12 frontend-only nodes, found {len(fe_only_nodes)}" | |
| assert fe_only_nodes == EXPECTED_FRONTEND_ONLY_TYPES, ( | |
| f"Unexpected frontend-only nodes set! Diff: {fe_only_nodes ^ EXPECTED_FRONTEND_ONLY_TYPES}" | |
| ) | |
| assert len(be_only_nodes) == 0, f"Found unexpected backend-only nodes: {be_only_nodes}" | |
| assert summary.frontend_only_nodes == 12 | |
| assert summary.backend_only_nodes == 0 | |
| # Ensure all frontend-only nodes are not registered in backend default_registry | |
| for fe_type in fe_only_nodes: | |
| assert not default_registry.has(fe_type), ( | |
| f"Frontend-only node '{fe_type}' should NOT be in backend default_registry" | |
| ) | |
| # ─── TEST 5: Schema Mismatch Reporting (15 Parity, 32 Documented Mismatches) ──── | |
| def test_schema_mismatch_reporting(inventory_bundle): | |
| """Test 5: Verify 15 full parity nodes and 32 schema mismatch nodes with documented gaps.""" | |
| _, summary, contracts = inventory_bundle | |
| full_parity_nodes = {c.type for c in contracts if c.status == "FULL_PARITY"} | |
| mismatch_nodes = {c.type for c in contracts if c.status == "SCHEMA_MISMATCH"} | |
| assert len(full_parity_nodes) == 15, f"Expected 15 full parity nodes, got {len(full_parity_nodes)}" | |
| assert full_parity_nodes == EXPECTED_FULL_PARITY_TYPES, ( | |
| f"Unexpected full parity nodes! Diff: {full_parity_nodes ^ EXPECTED_FULL_PARITY_TYPES}" | |
| ) | |
| assert len(mismatch_nodes) == 32, f"Expected 32 schema mismatch nodes, got {len(mismatch_nodes)}" | |
| assert summary.full_parity_nodes == 15 | |
| assert summary.schema_mismatch_nodes == 32 | |
| # Verify that total nodes sum exactly: 15 + 32 + 12 = 59 | |
| assert len(full_parity_nodes) + len(mismatch_nodes) + summary.frontend_only_nodes == 59 | |
| # Verify that all 32 schema mismatch nodes document their discrepancies | |
| for contract in contracts: | |
| if contract.status == "SCHEMA_MISMATCH": | |
| has_input_diff = bool(contract.ports and contract.ports.input_mismatches) | |
| has_output_diff = bool(contract.ports and contract.ports.output_mismatches) | |
| has_param_diff = bool(contract.parameters and contract.parameters.mismatches) | |
| has_gap_notes = bool(contract.gap_notes) | |
| assert (has_input_diff or has_output_diff or has_param_diff or has_gap_notes), ( | |
| f"Node '{contract.type}' marked SCHEMA_MISMATCH but has no documented discrepancies!" | |
| ) | |
| # ─── TEST 6: Independent Runtime vs. Codegen Capability Reporting ────────────── | |
| def test_independent_capability_reporting(inventory_bundle): | |
| """Test 6: Verify decoupled runtime execution vs. code generation capability reporting.""" | |
| _, summary, contracts = inventory_bundle | |
| be_exec_counts = {} | |
| be_codegen_counts = {} | |
| fe_exec_counts = {} | |
| fe_codegen_counts = {} | |
| for c in contracts: | |
| be_exec_counts[c.capabilities.backend_execution] = ( | |
| be_exec_counts.get(c.capabilities.backend_execution, 0) + 1 | |
| ) | |
| be_codegen_counts[c.capabilities.backend_codegen] = ( | |
| be_codegen_counts.get(c.capabilities.backend_codegen, 0) + 1 | |
| ) | |
| fe_exec_counts[c.capabilities.frontend_execution] = ( | |
| fe_exec_counts.get(c.capabilities.frontend_execution, 0) + 1 | |
| ) | |
| fe_codegen_counts[c.capabilities.frontend_codegen] = ( | |
| fe_codegen_counts.get(c.capabilities.frontend_codegen, 0) + 1 | |
| ) | |
| # 6.1 Backend Execution: 47 PASS, 12 UNSUPPORTED | |
| assert be_exec_counts.get(ValidationStatus.PASS) == 47 | |
| assert be_exec_counts.get(ValidationStatus.UNSUPPORTED) == 12 | |
| # 6.2 Backend Codegen: 58 PASS, 1 FAIL (lang.import) | |
| assert be_codegen_counts.get(ValidationStatus.PASS) == 58 | |
| assert be_codegen_counts.get(ValidationStatus.FAIL) == 1 | |
| lang_import = next(c for c in contracts if c.type == "lang.import") | |
| assert lang_import.capabilities.backend_codegen == ValidationStatus.FAIL | |
| # 6.3 Frontend Execution: 51 PASS, 8 UNSUPPORTED (6 agent + 2 external libs) | |
| assert fe_exec_counts.get(ValidationStatus.PASS) == 51 | |
| assert fe_exec_counts.get(ValidationStatus.UNSUPPORTED) == 8 | |
| # 6.4 Frontend Codegen: 59 PASS | |
| assert fe_codegen_counts.get(ValidationStatus.PASS) == 59 | |
| # 6.5 Verify independence: agent nodes can generate code even though execution is unsupported | |
| agent_nodes = [c for c in contracts if c.type.startswith("agent.")] | |
| assert len(agent_nodes) == 6 | |
| for ag in agent_nodes: | |
| assert ag.capabilities.backend_execution == ValidationStatus.UNSUPPORTED | |
| assert ag.capabilities.frontend_execution == ValidationStatus.UNSUPPORTED | |
| assert ag.capabilities.backend_codegen == ValidationStatus.PASS | |
| assert ag.capabilities.frontend_codegen == ValidationStatus.PASS | |
| # ─── TEST 7: Deterministic JSON Report Generation ────────────────────────────── | |
| def test_deterministic_report_generation(tmp_path): | |
| """Test 7: Verify report generation produces deterministic, valid JSON output.""" | |
| report_file = tmp_path / "test_report.json" | |
| exit_code, report = execute_inventory_integrity_validation( | |
| report_path=str(report_file), | |
| quiet=True, | |
| ) | |
| assert exit_code == 0, f"Validation returned failure exit code: {exit_code}" | |
| assert report_file.is_file(), "Report file was not created" | |
| with open(report_file, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| # Verify structural integrity of generated report | |
| assert data["phase"] == "Phase 2: Project Structure & Inventory Integrity" | |
| assert data["summary_counts"]["total_unique_node_types"] == 59 | |
| assert len(data["nodes"]) == 59 | |
| assert all(data["integrity_checks"].values()), "Not all integrity checks passed in report" | |
| # Verify no credential leaks in report | |
| report_text = report_file.read_text(encoding="utf-8") | |
| for secret_keyword in ["AWS_SECRET", "SUPABASE_KEY", "MONGODB_URI", "PASSWORD"]: | |
| assert secret_keyword not in report_text, f"Potential credential leak detected: {secret_keyword}" | |
| # ─── TEST 8: Correct Exit Codes for Inventory Errors ─────────────────────────── | |
| def test_exit_codes_for_inventory_errors(tmp_path): | |
| """Test 8: Verify exit codes: 0 for success, 1 for integrity failure or corrupted file.""" | |
| # 8.1 Valid run -> exit code 0 | |
| valid_report = tmp_path / "valid_report.json" | |
| exit_code, _ = execute_inventory_integrity_validation( | |
| report_path=str(valid_report), | |
| quiet=True, | |
| strict=False, | |
| ) | |
| assert exit_code == 0 | |
| # 8.2 Tampered inventory with duplicate node type -> exit code 1 | |
| raw_data = load_raw_inventory() | |
| tampered_data = copy.deepcopy(raw_data) | |
| # Duplicate first node with existing ID | |
| duplicate_node = copy.deepcopy(tampered_data["inventory"][0]) | |
| tampered_data["inventory"].append(duplicate_node) | |
| tampered_file = tmp_path / "tampered_inventory.json" | |
| tampered_file.write_text(json.dumps(tampered_data), encoding="utf-8") | |
| tampered_report = tmp_path / "tampered_report.json" | |
| exit_code, _ = execute_inventory_integrity_validation( | |
| report_path=str(tampered_report), | |
| inventory_path=str(tampered_file), | |
| quiet=True, | |
| strict=False, | |
| ) | |
| assert exit_code == 1, "Expected exit code 1 for duplicate node types" | |
| # 8.3 CLI main function invocation test | |
| cli_code = main(["--quiet", "--report-path", str(tmp_path / "cli_report.json")]) | |
| assert cli_code == 0, f"CLI main returned non-zero exit code: {cli_code}" | |
| # 8.4 Strict mode failure test (strict mode fails because 32 schema mismatches exist) | |
| strict_code = main(["--quiet", "--strict", "--report-path", str(tmp_path / "strict_report.json")]) | |
| assert strict_code == 1, "Strict mode must exit with code 1 when schema mismatches exist" | |