File size: 13,936 Bytes
e06cbf7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
"""Unit and integration test suite for Node Inventory Integrity (Phase 2).

Validates:
1. Inventory JSON loading & structural validity.
2. Canonical node uniqueness (exactly 59 nodes, 0 duplicate IDs).
3. Frontend/backend registration parity (47 backend, 59 frontend).
4. Expected frontend-only registrations (12 identified, 0 backend-only).
5. Schema mismatch reporting (15 full parity, 32 schema mismatches documented).
6. Independent runtime vs. codegen capability reporting.
7. Deterministic JSON report generation.
8. Correct exit codes for inventory errors (0 for success, 1 for integrity failure).
"""

from __future__ import annotations

import copy
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict

import pytest

from language_core.node_schema import default_registry
from node_validation_lab.fixtures.inventory_loader import (
    get_inventory_path,
    load_complete_inventory_bundle,
    load_inventory_contracts,
    load_inventory_summary,
    load_raw_inventory,
)
from node_validation_lab.run_validation import (
    EXPECTED_FRONTEND_ONLY_TYPES,
    EXPECTED_FULL_PARITY_TYPES,
    execute_inventory_integrity_validation,
    main,
    perform_inventory_integrity_checks,
)
from node_validation_lab.schemas.models import (
    InventoryIntegrityReport,
    InventorySummaryCounts,
    NodeValidationContract,
    ValidationStatus,
)


@pytest.fixture(scope="module")
def inventory_bundle():
    """Module-level fixture loading raw data, summary, and contracts."""
    raw_data, summary, contracts = load_complete_inventory_bundle()
    return raw_data, summary, contracts


# ─── TEST 1: Inventory JSON Loading & Structural Validity ───────────────────────

def test_inventory_loading_and_structural_validity(inventory_bundle):
    """Test 1: Verify inventory JSON loads deterministically and adheres to schema."""
    raw_data, summary, contracts = inventory_bundle

    # 1.1 Root keys
    assert isinstance(raw_data, dict), "Root must be a JSON dictionary"
    assert "audit_metadata" in raw_data, "Missing 'audit_metadata' key"
    assert "inventory" in raw_data, "Missing 'inventory' key"
    assert isinstance(raw_data["inventory"], list), "'inventory' must be a list"

    # 1.2 Audit metadata
    meta = raw_data["audit_metadata"]
    assert "timestamp" in meta, "Missing timestamp in audit metadata"
    assert "phase" in meta, "Missing phase in audit metadata"
    assert "summary_counts" in meta, "Missing summary_counts in audit metadata"

    # 1.3 Typed contracts
    assert len(contracts) == 59, f"Expected 59 contracts, got {len(contracts)}"
    for contract in contracts:
        assert isinstance(contract, NodeValidationContract)
        assert isinstance(contract.type, str) and len(contract.type) > 0
        assert contract.canonical_node_type == contract.type
        assert isinstance(contract.status, str)
        assert contract.capabilities is not None
        assert contract.validation_status in (
            ValidationStatus.PASS,
            ValidationStatus.UNSUPPORTED,
            ValidationStatus.FAIL,
            ValidationStatus.NOT_TESTED,
        )


# ─── TEST 2: Canonical Node Uniqueness (Exactly 59 Nodes, 0 Duplicate IDs) ─────

def test_canonical_node_uniqueness(inventory_bundle):
    """Test 2: Verify exactly 59 canonical nodes exist with 0 duplicate IDs."""
    _, summary, contracts = inventory_bundle

    all_types = [c.type for c in contracts]
    unique_types = set(all_types)

    assert len(all_types) == 59, f"Total node count must be exactly 59, got {len(all_types)}"
    assert len(unique_types) == 59, f"Duplicate node types detected! Unique count is {len(unique_types)}"
    assert summary.total_unique_node_types == 59

    # Verify ID format (dotted category.action or valid namespace)
    for node_type in unique_types:
        assert "." in node_type, f"Node type '{node_type}' must be namespaced with a dot"
        category, name = node_type.split(".", 1)
        assert len(category) > 0 and len(name) > 0, f"Malformed node type: '{node_type}'"


# ─── TEST 3: Frontend/Backend Registration Parity (47 BE, 59 FE) ───────────────

def test_registration_parity_counts(inventory_bundle):
    """Test 3: Verify registration parity (47 backend registered, 59 frontend canonical)."""
    _, summary, contracts = inventory_bundle

    be_registered = [c for c in contracts if c.is_backend_registered]
    fe_registered = [c for c in contracts if c.is_frontend_registered]

    assert len(be_registered) == 47, f"Expected 47 backend registered nodes, got {len(be_registered)}"
    assert len(fe_registered) == 59, f"Expected 59 frontend registered nodes, got {len(fe_registered)}"

    assert summary.backend_registered_count == 47
    assert summary.frontend_canonical_count == 59

    # Cross-reference with live backend schema registry
    backend_schemas = default_registry.list_all()
    assert len(backend_schemas) == 47, f"default_registry must have exactly 47 nodes, got {len(backend_schemas)}"

    backend_schema_types = {s.type for s in backend_schemas}
    be_contract_types = {c.type for c in be_registered}
    assert backend_schema_types == be_contract_types, (
        f"Mismatch between default_registry types and backend-registered contracts! "
        f"Diff: {backend_schema_types ^ be_contract_types}"
    )


# ─── TEST 4: Expected Frontend-Only Registrations (12 Identified, 0 BE-Only) ───

def test_expected_frontend_only_registrations(inventory_bundle):
    """Test 4: Verify exactly 12 expected frontend-only nodes and 0 backend-only nodes."""
    _, summary, contracts = inventory_bundle

    fe_only_nodes = {c.type for c in contracts if c.status == "FRONTEND_ONLY"}
    be_only_nodes = {c.type for c in contracts if c.status == "BACKEND_ONLY"}

    assert len(fe_only_nodes) == 12, f"Expected 12 frontend-only nodes, found {len(fe_only_nodes)}"
    assert fe_only_nodes == EXPECTED_FRONTEND_ONLY_TYPES, (
        f"Unexpected frontend-only nodes set! Diff: {fe_only_nodes ^ EXPECTED_FRONTEND_ONLY_TYPES}"
    )

    assert len(be_only_nodes) == 0, f"Found unexpected backend-only nodes: {be_only_nodes}"
    assert summary.frontend_only_nodes == 12
    assert summary.backend_only_nodes == 0

    # Ensure all frontend-only nodes are not registered in backend default_registry
    for fe_type in fe_only_nodes:
        assert not default_registry.has(fe_type), (
            f"Frontend-only node '{fe_type}' should NOT be in backend default_registry"
        )


# ─── TEST 5: Schema Mismatch Reporting (15 Parity, 32 Documented Mismatches) ────

def test_schema_mismatch_reporting(inventory_bundle):
    """Test 5: Verify 15 full parity nodes and 32 schema mismatch nodes with documented gaps."""
    _, summary, contracts = inventory_bundle

    full_parity_nodes = {c.type for c in contracts if c.status == "FULL_PARITY"}
    mismatch_nodes = {c.type for c in contracts if c.status == "SCHEMA_MISMATCH"}

    assert len(full_parity_nodes) == 15, f"Expected 15 full parity nodes, got {len(full_parity_nodes)}"
    assert full_parity_nodes == EXPECTED_FULL_PARITY_TYPES, (
        f"Unexpected full parity nodes! Diff: {full_parity_nodes ^ EXPECTED_FULL_PARITY_TYPES}"
    )

    assert len(mismatch_nodes) == 32, f"Expected 32 schema mismatch nodes, got {len(mismatch_nodes)}"
    assert summary.full_parity_nodes == 15
    assert summary.schema_mismatch_nodes == 32

    # Verify that total nodes sum exactly: 15 + 32 + 12 = 59
    assert len(full_parity_nodes) + len(mismatch_nodes) + summary.frontend_only_nodes == 59

    # Verify that all 32 schema mismatch nodes document their discrepancies
    for contract in contracts:
        if contract.status == "SCHEMA_MISMATCH":
            has_input_diff = bool(contract.ports and contract.ports.input_mismatches)
            has_output_diff = bool(contract.ports and contract.ports.output_mismatches)
            has_param_diff = bool(contract.parameters and contract.parameters.mismatches)
            has_gap_notes = bool(contract.gap_notes)

            assert (has_input_diff or has_output_diff or has_param_diff or has_gap_notes), (
                f"Node '{contract.type}' marked SCHEMA_MISMATCH but has no documented discrepancies!"
            )


# ─── TEST 6: Independent Runtime vs. Codegen Capability Reporting ──────────────

def test_independent_capability_reporting(inventory_bundle):
    """Test 6: Verify decoupled runtime execution vs. code generation capability reporting."""
    _, summary, contracts = inventory_bundle

    be_exec_counts = {}
    be_codegen_counts = {}
    fe_exec_counts = {}
    fe_codegen_counts = {}

    for c in contracts:
        be_exec_counts[c.capabilities.backend_execution] = (
            be_exec_counts.get(c.capabilities.backend_execution, 0) + 1
        )
        be_codegen_counts[c.capabilities.backend_codegen] = (
            be_codegen_counts.get(c.capabilities.backend_codegen, 0) + 1
        )
        fe_exec_counts[c.capabilities.frontend_execution] = (
            fe_exec_counts.get(c.capabilities.frontend_execution, 0) + 1
        )
        fe_codegen_counts[c.capabilities.frontend_codegen] = (
            fe_codegen_counts.get(c.capabilities.frontend_codegen, 0) + 1
        )

    # 6.1 Backend Execution: 47 PASS, 12 UNSUPPORTED
    assert be_exec_counts.get(ValidationStatus.PASS) == 47
    assert be_exec_counts.get(ValidationStatus.UNSUPPORTED) == 12

    # 6.2 Backend Codegen: 58 PASS, 1 FAIL (lang.import)
    assert be_codegen_counts.get(ValidationStatus.PASS) == 58
    assert be_codegen_counts.get(ValidationStatus.FAIL) == 1
    lang_import = next(c for c in contracts if c.type == "lang.import")
    assert lang_import.capabilities.backend_codegen == ValidationStatus.FAIL

    # 6.3 Frontend Execution: 51 PASS, 8 UNSUPPORTED (6 agent + 2 external libs)
    assert fe_exec_counts.get(ValidationStatus.PASS) == 51
    assert fe_exec_counts.get(ValidationStatus.UNSUPPORTED) == 8

    # 6.4 Frontend Codegen: 59 PASS
    assert fe_codegen_counts.get(ValidationStatus.PASS) == 59

    # 6.5 Verify independence: agent nodes can generate code even though execution is unsupported
    agent_nodes = [c for c in contracts if c.type.startswith("agent.")]
    assert len(agent_nodes) == 6
    for ag in agent_nodes:
        assert ag.capabilities.backend_execution == ValidationStatus.UNSUPPORTED
        assert ag.capabilities.frontend_execution == ValidationStatus.UNSUPPORTED
        assert ag.capabilities.backend_codegen == ValidationStatus.PASS
        assert ag.capabilities.frontend_codegen == ValidationStatus.PASS


# ─── TEST 7: Deterministic JSON Report Generation ──────────────────────────────

def test_deterministic_report_generation(tmp_path):
    """Test 7: Verify report generation produces deterministic, valid JSON output."""
    report_file = tmp_path / "test_report.json"
    exit_code, report = execute_inventory_integrity_validation(
        report_path=str(report_file),
        quiet=True,
    )

    assert exit_code == 0, f"Validation returned failure exit code: {exit_code}"
    assert report_file.is_file(), "Report file was not created"

    with open(report_file, "r", encoding="utf-8") as f:
        data = json.load(f)

    # Verify structural integrity of generated report
    assert data["phase"] == "Phase 2: Project Structure & Inventory Integrity"
    assert data["summary_counts"]["total_unique_node_types"] == 59
    assert len(data["nodes"]) == 59
    assert all(data["integrity_checks"].values()), "Not all integrity checks passed in report"

    # Verify no credential leaks in report
    report_text = report_file.read_text(encoding="utf-8")
    for secret_keyword in ["AWS_SECRET", "SUPABASE_KEY", "MONGODB_URI", "PASSWORD"]:
        assert secret_keyword not in report_text, f"Potential credential leak detected: {secret_keyword}"


# ─── TEST 8: Correct Exit Codes for Inventory Errors ───────────────────────────

def test_exit_codes_for_inventory_errors(tmp_path):
    """Test 8: Verify exit codes: 0 for success, 1 for integrity failure or corrupted file."""
    # 8.1 Valid run -> exit code 0
    valid_report = tmp_path / "valid_report.json"
    exit_code, _ = execute_inventory_integrity_validation(
        report_path=str(valid_report),
        quiet=True,
        strict=False,
    )
    assert exit_code == 0

    # 8.2 Tampered inventory with duplicate node type -> exit code 1
    raw_data = load_raw_inventory()
    tampered_data = copy.deepcopy(raw_data)
    # Duplicate first node with existing ID
    duplicate_node = copy.deepcopy(tampered_data["inventory"][0])
    tampered_data["inventory"].append(duplicate_node)

    tampered_file = tmp_path / "tampered_inventory.json"
    tampered_file.write_text(json.dumps(tampered_data), encoding="utf-8")

    tampered_report = tmp_path / "tampered_report.json"
    exit_code, _ = execute_inventory_integrity_validation(
        report_path=str(tampered_report),
        inventory_path=str(tampered_file),
        quiet=True,
        strict=False,
    )
    assert exit_code == 1, "Expected exit code 1 for duplicate node types"

    # 8.3 CLI main function invocation test
    cli_code = main(["--quiet", "--report-path", str(tmp_path / "cli_report.json")])
    assert cli_code == 0, f"CLI main returned non-zero exit code: {cli_code}"

    # 8.4 Strict mode failure test (strict mode fails because 32 schema mismatches exist)
    strict_code = main(["--quiet", "--strict", "--report-path", str(tmp_path / "strict_report.json")])
    assert strict_code == 1, "Strict mode must exit with code 1 when schema mismatches exist"