auto / node_validation_lab /tests /test_inventory_integrity.py
rsnarsna
feat(lab): implement Phase 2 project structure and inventory integrity test suite
e06cbf7
Raw History Blame Contribute Delete
13.9 kB
"""Unit and integration test suite for Node Inventory Integrity (Phase 2).
Validates:
1. Inventory JSON loading & structural validity.
2. Canonical node uniqueness (exactly 59 nodes, 0 duplicate IDs).
3. Frontend/backend registration parity (47 backend, 59 frontend).
4. Expected frontend-only registrations (12 identified, 0 backend-only).
5. Schema mismatch reporting (15 full parity, 32 schema mismatches documented).
6. Independent runtime vs. codegen capability reporting.
7. Deterministic JSON report generation.
8. Correct exit codes for inventory errors (0 for success, 1 for integrity failure).
"""
from __future__ import annotations
import copy
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict
import pytest
from language_core.node_schema import default_registry
from node_validation_lab.fixtures.inventory_loader import (
get_inventory_path,
load_complete_inventory_bundle,
load_inventory_contracts,
load_inventory_summary,
load_raw_inventory,
)
from node_validation_lab.run_validation import (
EXPECTED_FRONTEND_ONLY_TYPES,
EXPECTED_FULL_PARITY_TYPES,
execute_inventory_integrity_validation,
main,
perform_inventory_integrity_checks,
)
from node_validation_lab.schemas.models import (
InventoryIntegrityReport,
InventorySummaryCounts,
NodeValidationContract,
ValidationStatus,
)
@pytest.fixture(scope="module")
def inventory_bundle():
"""Module-level fixture loading raw data, summary, and contracts."""
raw_data, summary, contracts = load_complete_inventory_bundle()
return raw_data, summary, contracts
# ─── TEST 1: Inventory JSON Loading & Structural Validity ───────────────────────
def test_inventory_loading_and_structural_validity(inventory_bundle):
"""Test 1: Verify inventory JSON loads deterministically and adheres to schema."""
raw_data, summary, contracts = inventory_bundle
# 1.1 Root keys
assert isinstance(raw_data, dict), "Root must be a JSON dictionary"
assert "audit_metadata" in raw_data, "Missing 'audit_metadata' key"
assert "inventory" in raw_data, "Missing 'inventory' key"
assert isinstance(raw_data["inventory"], list), "'inventory' must be a list"
# 1.2 Audit metadata
meta = raw_data["audit_metadata"]
assert "timestamp" in meta, "Missing timestamp in audit metadata"
assert "phase" in meta, "Missing phase in audit metadata"
assert "summary_counts" in meta, "Missing summary_counts in audit metadata"
# 1.3 Typed contracts
assert len(contracts) == 59, f"Expected 59 contracts, got {len(contracts)}"
for contract in contracts:
assert isinstance(contract, NodeValidationContract)
assert isinstance(contract.type, str) and len(contract.type) > 0
assert contract.canonical_node_type == contract.type
assert isinstance(contract.status, str)
assert contract.capabilities is not None
assert contract.validation_status in (
ValidationStatus.PASS,
ValidationStatus.UNSUPPORTED,
ValidationStatus.FAIL,
ValidationStatus.NOT_TESTED,
)
# ─── TEST 2: Canonical Node Uniqueness (Exactly 59 Nodes, 0 Duplicate IDs) ─────
def test_canonical_node_uniqueness(inventory_bundle):
"""Test 2: Verify exactly 59 canonical nodes exist with 0 duplicate IDs."""
_, summary, contracts = inventory_bundle
all_types = [c.type for c in contracts]
unique_types = set(all_types)
assert len(all_types) == 59, f"Total node count must be exactly 59, got {len(all_types)}"
assert len(unique_types) == 59, f"Duplicate node types detected! Unique count is {len(unique_types)}"
assert summary.total_unique_node_types == 59
# Verify ID format (dotted category.action or valid namespace)
for node_type in unique_types:
assert "." in node_type, f"Node type '{node_type}' must be namespaced with a dot"
category, name = node_type.split(".", 1)
assert len(category) > 0 and len(name) > 0, f"Malformed node type: '{node_type}'"
# ─── TEST 3: Frontend/Backend Registration Parity (47 BE, 59 FE) ───────────────
def test_registration_parity_counts(inventory_bundle):
"""Test 3: Verify registration parity (47 backend registered, 59 frontend canonical)."""
_, summary, contracts = inventory_bundle
be_registered = [c for c in contracts if c.is_backend_registered]
fe_registered = [c for c in contracts if c.is_frontend_registered]
assert len(be_registered) == 47, f"Expected 47 backend registered nodes, got {len(be_registered)}"
assert len(fe_registered) == 59, f"Expected 59 frontend registered nodes, got {len(fe_registered)}"
assert summary.backend_registered_count == 47
assert summary.frontend_canonical_count == 59
# Cross-reference with live backend schema registry
backend_schemas = default_registry.list_all()
assert len(backend_schemas) == 47, f"default_registry must have exactly 47 nodes, got {len(backend_schemas)}"
backend_schema_types = {s.type for s in backend_schemas}
be_contract_types = {c.type for c in be_registered}
assert backend_schema_types == be_contract_types, (
f"Mismatch between default_registry types and backend-registered contracts! "
f"Diff: {backend_schema_types ^ be_contract_types}"
)
# ─── TEST 4: Expected Frontend-Only Registrations (12 Identified, 0 BE-Only) ───
def test_expected_frontend_only_registrations(inventory_bundle):
"""Test 4: Verify exactly 12 expected frontend-only nodes and 0 backend-only nodes."""
_, summary, contracts = inventory_bundle
fe_only_nodes = {c.type for c in contracts if c.status == "FRONTEND_ONLY"}
be_only_nodes = {c.type for c in contracts if c.status == "BACKEND_ONLY"}
assert len(fe_only_nodes) == 12, f"Expected 12 frontend-only nodes, found {len(fe_only_nodes)}"
assert fe_only_nodes == EXPECTED_FRONTEND_ONLY_TYPES, (
f"Unexpected frontend-only nodes set! Diff: {fe_only_nodes ^ EXPECTED_FRONTEND_ONLY_TYPES}"
)
assert len(be_only_nodes) == 0, f"Found unexpected backend-only nodes: {be_only_nodes}"
assert summary.frontend_only_nodes == 12
assert summary.backend_only_nodes == 0
# Ensure all frontend-only nodes are not registered in backend default_registry
for fe_type in fe_only_nodes:
assert not default_registry.has(fe_type), (
f"Frontend-only node '{fe_type}' should NOT be in backend default_registry"
)
# ─── TEST 5: Schema Mismatch Reporting (15 Parity, 32 Documented Mismatches) ────
def test_schema_mismatch_reporting(inventory_bundle):
"""Test 5: Verify 15 full parity nodes and 32 schema mismatch nodes with documented gaps."""
_, summary, contracts = inventory_bundle
full_parity_nodes = {c.type for c in contracts if c.status == "FULL_PARITY"}
mismatch_nodes = {c.type for c in contracts if c.status == "SCHEMA_MISMATCH"}
assert len(full_parity_nodes) == 15, f"Expected 15 full parity nodes, got {len(full_parity_nodes)}"
assert full_parity_nodes == EXPECTED_FULL_PARITY_TYPES, (
f"Unexpected full parity nodes! Diff: {full_parity_nodes ^ EXPECTED_FULL_PARITY_TYPES}"
)
assert len(mismatch_nodes) == 32, f"Expected 32 schema mismatch nodes, got {len(mismatch_nodes)}"
assert summary.full_parity_nodes == 15
assert summary.schema_mismatch_nodes == 32
# Verify that total nodes sum exactly: 15 + 32 + 12 = 59
assert len(full_parity_nodes) + len(mismatch_nodes) + summary.frontend_only_nodes == 59
# Verify that all 32 schema mismatch nodes document their discrepancies
for contract in contracts:
if contract.status == "SCHEMA_MISMATCH":
has_input_diff = bool(contract.ports and contract.ports.input_mismatches)
has_output_diff = bool(contract.ports and contract.ports.output_mismatches)
has_param_diff = bool(contract.parameters and contract.parameters.mismatches)
has_gap_notes = bool(contract.gap_notes)
assert (has_input_diff or has_output_diff or has_param_diff or has_gap_notes), (
f"Node '{contract.type}' marked SCHEMA_MISMATCH but has no documented discrepancies!"
)
# ─── TEST 6: Independent Runtime vs. Codegen Capability Reporting ──────────────
def test_independent_capability_reporting(inventory_bundle):
"""Test 6: Verify decoupled runtime execution vs. code generation capability reporting."""
_, summary, contracts = inventory_bundle
be_exec_counts = {}
be_codegen_counts = {}
fe_exec_counts = {}
fe_codegen_counts = {}
for c in contracts:
be_exec_counts[c.capabilities.backend_execution] = (
be_exec_counts.get(c.capabilities.backend_execution, 0) + 1
)
be_codegen_counts[c.capabilities.backend_codegen] = (
be_codegen_counts.get(c.capabilities.backend_codegen, 0) + 1
)
fe_exec_counts[c.capabilities.frontend_execution] = (
fe_exec_counts.get(c.capabilities.frontend_execution, 0) + 1
)
fe_codegen_counts[c.capabilities.frontend_codegen] = (
fe_codegen_counts.get(c.capabilities.frontend_codegen, 0) + 1
)
# 6.1 Backend Execution: 47 PASS, 12 UNSUPPORTED
assert be_exec_counts.get(ValidationStatus.PASS) == 47
assert be_exec_counts.get(ValidationStatus.UNSUPPORTED) == 12
# 6.2 Backend Codegen: 58 PASS, 1 FAIL (lang.import)
assert be_codegen_counts.get(ValidationStatus.PASS) == 58
assert be_codegen_counts.get(ValidationStatus.FAIL) == 1
lang_import = next(c for c in contracts if c.type == "lang.import")
assert lang_import.capabilities.backend_codegen == ValidationStatus.FAIL
# 6.3 Frontend Execution: 51 PASS, 8 UNSUPPORTED (6 agent + 2 external libs)
assert fe_exec_counts.get(ValidationStatus.PASS) == 51
assert fe_exec_counts.get(ValidationStatus.UNSUPPORTED) == 8
# 6.4 Frontend Codegen: 59 PASS
assert fe_codegen_counts.get(ValidationStatus.PASS) == 59
# 6.5 Verify independence: agent nodes can generate code even though execution is unsupported
agent_nodes = [c for c in contracts if c.type.startswith("agent.")]
assert len(agent_nodes) == 6
for ag in agent_nodes:
assert ag.capabilities.backend_execution == ValidationStatus.UNSUPPORTED
assert ag.capabilities.frontend_execution == ValidationStatus.UNSUPPORTED
assert ag.capabilities.backend_codegen == ValidationStatus.PASS
assert ag.capabilities.frontend_codegen == ValidationStatus.PASS
# ─── TEST 7: Deterministic JSON Report Generation ──────────────────────────────
def test_deterministic_report_generation(tmp_path):
"""Test 7: Verify report generation produces deterministic, valid JSON output."""
report_file = tmp_path / "test_report.json"
exit_code, report = execute_inventory_integrity_validation(
report_path=str(report_file),
quiet=True,
)
assert exit_code == 0, f"Validation returned failure exit code: {exit_code}"
assert report_file.is_file(), "Report file was not created"
with open(report_file, "r", encoding="utf-8") as f:
data = json.load(f)
# Verify structural integrity of generated report
assert data["phase"] == "Phase 2: Project Structure & Inventory Integrity"
assert data["summary_counts"]["total_unique_node_types"] == 59
assert len(data["nodes"]) == 59
assert all(data["integrity_checks"].values()), "Not all integrity checks passed in report"
# Verify no credential leaks in report
report_text = report_file.read_text(encoding="utf-8")
for secret_keyword in ["AWS_SECRET", "SUPABASE_KEY", "MONGODB_URI", "PASSWORD"]:
assert secret_keyword not in report_text, f"Potential credential leak detected: {secret_keyword}"
# ─── TEST 8: Correct Exit Codes for Inventory Errors ───────────────────────────
def test_exit_codes_for_inventory_errors(tmp_path):
"""Test 8: Verify exit codes: 0 for success, 1 for integrity failure or corrupted file."""
# 8.1 Valid run -> exit code 0
valid_report = tmp_path / "valid_report.json"
exit_code, _ = execute_inventory_integrity_validation(
report_path=str(valid_report),
quiet=True,
strict=False,
)
assert exit_code == 0
# 8.2 Tampered inventory with duplicate node type -> exit code 1
raw_data = load_raw_inventory()
tampered_data = copy.deepcopy(raw_data)
# Duplicate first node with existing ID
duplicate_node = copy.deepcopy(tampered_data["inventory"][0])
tampered_data["inventory"].append(duplicate_node)
tampered_file = tmp_path / "tampered_inventory.json"
tampered_file.write_text(json.dumps(tampered_data), encoding="utf-8")
tampered_report = tmp_path / "tampered_report.json"
exit_code, _ = execute_inventory_integrity_validation(
report_path=str(tampered_report),
inventory_path=str(tampered_file),
quiet=True,
strict=False,
)
assert exit_code == 1, "Expected exit code 1 for duplicate node types"
# 8.3 CLI main function invocation test
cli_code = main(["--quiet", "--report-path", str(tmp_path / "cli_report.json")])
assert cli_code == 0, f"CLI main returned non-zero exit code: {cli_code}"
# 8.4 Strict mode failure test (strict mode fails because 32 schema mismatches exist)
strict_code = main(["--quiet", "--strict", "--report-path", str(tmp_path / "strict_report.json")])
assert strict_code == 1, "Strict mode must exit with code 1 when schema mismatches exist"