VibeCoding / src /modules /skill /tests /test_skill.py
dellweb3's picture
Deploy verified conversational workbench and isolated Java application previews
caf8059 verified
Raw History Blame Contribute Delete
15.9 kB
"""Business invariant tests; no model calls, Java runs or source-download execution."""
import copy
import tempfile
import unittest
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
from pydantic import ValidationError
from src.adapters.persistence.skill_sqlite import SqliteSkillRepository
from src.modules.skill.application.service import SkillService
from src.modules.skill.atomic_registry.catalog import builtin_operations, check_handler
from src.modules.skill.domain.models import Actor, Budget, Draft, Framework, Operation, SkillError, fits
from src.modules.skill.graph.compiler import compile_graph
from src.modules.skill.packs.examples import all_examples, framework_candidate
ADMIN = Actor(tenant_id="test-tenant", user_id="author", roles=["admin"])
REVIEWER = Actor(tenant_id="test-tenant", user_id="reviewer", roles=["reviewer"])
USER = Actor(tenant_id="test-tenant", user_id="developer", roles=[])
OTHER = Actor(tenant_id="another-tenant", user_id="author", roles=["admin"])
REVISIONS = {c: "test-revision-1" for c in ["java-basics", "java-patterns", "erp", "wms", "equipment-ops", "java-frameworks", "web-development", "java-persistence", "java-testing", "java-integration"]}
class KnowledgeDouble:
def __init__(self):
self.allowed = True
self.revision = "test-revision-1"
def __call__(self, actor, binding):
if not self.allowed:
raise SkillError("KNOWLEDGE_NOT_FOUND", "test denied", 404)
if binding.revision != self.revision:
raise SkillError("KNOWLEDGE_REVISION_MISMATCH", "test changed", 409)
return {"id": binding.collection_id, "revision": self.revision, "digest": "a" * 64, "scope": binding.usage}
def approved_framework():
return framework_candidate().model_copy(update={"state": "approved", "evidence_refs": ["unit-test-fixture-not-production-certification"]})
class CompilerTests(unittest.TestCase):
def setUp(self):
self.draft = all_examples(REVISIONS)[0]
self.ops = {o.operation_id + "@" + o.version: o for o in builtin_operations()}
def report(self, data=None):
return compile_graph(Draft.model_validate(data) if data else self.draft, self.ops, approved_framework())
def codes(self, data):
return {x.code for x in self.report(data).issues}
def test_three_domain_flows_compile_and_share_operations(self):
for draft in all_examples(REVISIONS):
report = compile_graph(draft, self.ops, approved_framework())
self.assertTrue(report.valid, report.issues)
self.assertEqual(report.model_calls, 2)
self.assertEqual(len(report.execution_order), 16)
def test_cycle_rejected(self):
data = self.draft.model_dump()
data["graph"]["control_edges"].append({"source": "end", "target": "start"})
self.assertIn("CONTROL_CYCLE", self.codes(data))
def test_duplicate_node_rejected(self):
data = self.draft.model_dump()
data["graph"]["nodes"].append(data["graph"]["nodes"][0])
self.assertIn("DUPLICATE_NODE", self.codes(data))
def test_unknown_node_and_duplicate_edge_rejected(self):
data = self.draft.model_dump()
data["graph"]["control_edges"] += [{"source": "unknown", "target": "end"}, data["graph"]["control_edges"][0]]
self.assertTrue({"UNKNOWN_NODE", "DUPLICATE_EDGE"}.issubset(self.codes(data)))
def test_disconnected_node_rejected(self):
data = self.draft.model_dump()
data["graph"]["nodes"].append({"id": "orphan", "kind": "approval", "approval_scope": "design"})
self.assertIn("DISCONNECTED_NODE", self.codes(data))
def test_type_mismatch_and_reverse_data_rejected(self):
data = self.draft.model_dump()
data["graph"]["data_edges"][0]["source_port"] = "query"
data["graph"]["data_edges"][0]["source"] = "end"
self.assertTrue({"UNKNOWN_PORT", "DATA_ORDER"}.issubset(self.codes(data)))
data = self.draft.model_dump()
data["graph"]["data_edges"][0].update(source="requirements", source_port="document", target="knowledge", target_port="query")
self.assertIn("PORT_TYPE_MISMATCH", self.codes(data))
def test_unknown_operation_and_framework_rejected(self):
data = self.draft.model_dump()
data["graph"]["nodes"][1]["operation_ref"] = "unapproved@1.0.0"
self.assertIn("OPERATION_UNAVAILABLE", self.codes(data))
self.assertIn("FRAMEWORK_UNAVAILABLE", {x.code for x in compile_graph(self.draft, self.ops, framework_candidate()).issues})
def test_missing_input_and_duplicate_binding_rejected(self):
data = self.draft.model_dump()
data["graph"]["data_edges"].pop(0)
self.assertIn("REQUIRED_INPUT", self.codes(data))
data = self.draft.model_dump()
data["graph"]["data_edges"].append(copy.deepcopy(data["graph"]["data_edges"][0]))
self.assertIn("MULTIPLE_INPUT_SOURCES", self.codes(data))
def test_literal_cannot_forge_test_evidence(self):
data = self.draft.model_dump()
data["graph"]["data_edges"] = [e for e in data["graph"]["data_edges"] if not(e["target"] == "preview" and e["target_port"] == "test_evidence")]
next(n for n in data["graph"]["nodes"] if n["id"] == "preview")["inputs"] = {"test_evidence": {"passed": True}}
self.assertIn("UNTRUSTED_STAGE_INPUT", self.codes(data))
def test_object_port_cannot_substitute_for_test_evidence(self):
data = self.draft.model_dump()
edge = next(e for e in data["graph"]["data_edges"] if e["target"] == "preview" and e["target_port"] == "test_evidence")
edge.update(source="design", source_port="design")
self.assertIn("UNTRUSTED_STAGE_INPUT", self.codes(data))
def test_design_and_execution_gates_must_dominate_all_paths(self):
data = self.draft.model_dump()
data["graph"]["control_edges"].append({"source": "design", "target": "patch"})
self.assertTrue({"DESIGN_GATE_BYPASS", "EXECUTION_GATE_BYPASS"}.issubset(self.codes(data)))
def test_test_gate_must_dominate_preview(self):
data = self.draft.model_dump()
data["graph"]["control_edges"].append({"source": "build", "target": "preview"})
self.assertIn("QUALITY_GATE_BYPASS", self.codes(data))
def test_acceptance_cannot_be_bypassed(self):
data = self.draft.model_dump()
data["graph"]["control_edges"].append({"source": "preview", "target": "evidence"})
self.assertIn("ACCEPTANCE_GATE_BYPASS", self.codes(data))
def test_incomplete_flow_and_forged_end_manifest_rejected(self):
data = self.draft.model_dump()
data["graph"]["control_edges"].append({"source": "start", "target": "end"})
self.assertIn("INCOMPLETE_CODING_FLOW", self.codes(data))
data = self.draft.model_dump()
data["graph"]["data_edges"][-1].update(source="design", source_port="design")
self.assertIn("UNTRUSTED_MANIFEST", self.codes(data))
def test_resources_cannot_reference_unbound_collection(self):
data = self.draft.model_dump()
data["graph"]["resource_edges"][0]["resource"] = "secret"
self.assertIn("RESOURCE_REFERENCE", self.codes(data))
def test_budget_must_include_shared_repair_reserve(self):
data = self.draft.model_dump()
data["budget"]["max_repairs"] = 2
self.assertIn("BUDGET_UNREACHABLE", self.codes(data))
self.assertEqual(Budget().max_model_calls,3)
data['budget']=Budget(max_model_calls=6,max_tokens=120000,max_repairs=2).model_dump()
self.assertNotIn('BUDGET_UNREACHABLE',self.codes(data))
with self.assertRaises(ValidationError):
Budget(max_model_calls=7)
def test_unknown_executable_fields_and_weak_handler_contract_rejected(self):
data = self.draft.model_dump()
data["graph"]["nodes"][1]["script"] = "arbitrary code"
with self.assertRaises(ValidationError):
Draft.model_validate(data)
operation = builtin_operations()[5].model_copy(update={"permissions": []})
with self.assertRaises(SkillError):
check_handler(operation)
def test_boolean_is_not_integer_and_unsafe_framework_paths_rejected(self):
self.assertFalse(fits(True, "integer"))
self.assertFalse(fits(float("nan"), "number"))
for path in ["../", "/etc/", "runner/", "src/main/../../"]:
with self.subTest(path=path), self.assertRaises(ValidationError):
Framework(profile_id="unsafe", version="0.1.0", allowed_paths=[path])
class ServiceTests(unittest.TestCase):
def setUp(self):
folder = Path(__file__).resolve().parents[4] / ".local/test-runs/skill" / uuid.uuid4().hex
folder.mkdir(parents=True)
self.tmp = SimpleNamespace(name=str(folder))
self.repo = SqliteSkillRepository(Path(self.tmp.name) / "skill.sqlite")
self.knowledge = KnowledgeDouble()
self.access = lambda actor, pid: self.assertEqual(pid, "project-one")
self.service = SkillService(self.repo, self.knowledge, self.access)
self.service.bootstrap(ADMIN)
self.service.add_framework(ADMIN, approved_framework())
self.draft = all_examples(REVISIONS)[0]
self.saved = self.service.save_draft(ADMIN, self.draft)
def publish(self):
report = self.service.validate(ADMIN, self.draft.skill_id)
approval = self.service.approve(REVIEWER, self.draft.skill_id, 1, report.digest, ["unit-test-evidence"])
return self.service.publish(ADMIN, self.draft.skill_id, "0.1.0", 1, approval["key"])
def test_draft_cas_does_not_overwrite(self):
changed = self.draft.model_copy(update={"description": "updated"})
result = self.service.save_draft(ADMIN, changed, 1)
self.assertEqual(result["revision"], 2)
with self.assertRaises(SkillError) as exc:
self.service.save_draft(ADMIN, self.draft, 1)
self.assertEqual(exc.exception.code, "REVISION_CONFLICT")
def test_concurrent_cas_has_one_winner(self):
def update(label):
try:
self.service.save_draft(ADMIN, self.draft.model_copy(update={"description": label}), 1)
return "won"
except SkillError as exc:
return exc.code
with ThreadPoolExecutor(max_workers=2) as pool:
results = list(pool.map(update, ["one", "two"]))
self.assertCountEqual(results, ["won", "REVISION_CONFLICT"])
def test_cross_tenant_and_regular_user_denied(self):
with self.assertRaises(SkillError) as exc:
self.service.read_draft(OTHER, self.draft.skill_id)
self.assertEqual(exc.exception.status, 404)
with self.assertRaises(SkillError) as exc:
self.service.save_draft(USER, self.draft)
self.assertEqual(exc.exception.status, 403)
def test_author_cannot_approve_own_revision(self):
report = self.service.validate(ADMIN, self.draft.skill_id)
with self.assertRaises(SkillError) as exc:
self.service.approve(ADMIN, self.draft.skill_id, 1, report.digest, ["test"])
self.assertEqual(exc.exception.code, "INDEPENDENT_REVIEW_REQUIRED")
def test_last_editor_cannot_approve_others_original_draft(self):
other = Actor(tenant_id=ADMIN.tenant_id, user_id="editor-two", roles=["admin"])
self.service.save_draft(other, self.draft, 1)
report = self.service.validate(other, self.draft.skill_id)
with self.assertRaises(SkillError) as exc:
self.service.approve(other, self.draft.skill_id, 2, report.digest, ["test"])
self.assertEqual(exc.exception.code, "INDEPENDENT_REVIEW_REQUIRED")
def test_approval_invalidated_by_draft_change(self):
report = self.service.validate(ADMIN, self.draft.skill_id)
approval = self.service.approve(REVIEWER, self.draft.skill_id, 1, report.digest, ["test"])
self.service.save_draft(ADMIN, self.draft.model_copy(update={"description": "changed"}), 1)
with self.assertRaises(SkillError) as exc:
self.service.publish(ADMIN, self.draft.skill_id, "0.1.0", 2, approval["key"])
self.assertEqual(exc.exception.code, "APPROVAL_TARGET_CHANGED")
def test_releases_are_immutable_and_new_drafts_do_not_change_them(self):
release = self.publish()
self.service.save_draft(ADMIN, self.draft.model_copy(update={"description": "new design"}), 1)
self.assertEqual(self.service.releases(USER)[0]["digest"], release["digest"])
with self.repo.transaction(write=True) as tx, self.assertRaises(SkillError):
tx.update(ADMIN.tenant_id, "release", release["key"], {}, 1)
def test_duplicate_release_cannot_replace_history(self):
release = self.publish()
with self.repo.transaction(write=True) as tx, self.assertRaises(SkillError) as exc:
tx.create(ADMIN.tenant_id, "release", release["key"], {})
self.assertEqual(exc.exception.code, "VERSION_EXISTS")
def test_stop_new_snapshots_for_retired_dependencies(self):
release = self.publish()
self.service.retire(ADMIN, "operation", "AO09@0.1.0", "test retirement")
with self.assertRaises(SkillError) as exc:
self.service.prepare(USER, release["key"], "project-one", "a" * 64, "request-one")
self.assertEqual(exc.exception.code, "DEPENDENCY_RETIRED")
def test_knowledge_revocation_is_rechecked_at_start(self):
release = self.publish()
self.knowledge.allowed = False
with self.assertRaises(SkillError) as exc:
self.service.prepare(USER, release["key"], "project-one", "a" * 64, "request-one")
self.assertEqual(exc.exception.status, 404)
def test_snapshot_idempotency_and_source_binding(self):
release = self.publish()
first = self.service.prepare(USER, release["key"], "project-one", "a" * 64, "request-one")
replay = self.service.prepare(USER, release["key"], "project-one", "a" * 64, "request-one")
self.assertEqual(first, replay)
self.assertEqual(first["execution_status"], "not_started")
with self.assertRaises(SkillError) as exc:
self.service.prepare(USER, release["key"], "project-one", "b" * 64, "request-one")
self.assertEqual(exc.exception.code, "IDEMPOTENCY_CONFLICT")
def test_restart_keeps_data_and_does_not_execute(self):
release = self.publish()
restarted = SkillService(SqliteSkillRepository(self.repo.path), self.knowledge, self.access)
self.assertEqual(restarted.releases(USER)[0]["key"], release["key"])
with self.repo.transaction() as tx:
self.assertEqual(tx.listing(ADMIN.tenant_id, "snapshot"), [])
def test_invisible_knowledge_fails_validation(self):
self.knowledge.allowed = False
report = self.service.validate(ADMIN, self.draft.skill_id)
self.assertFalse(report.valid)
self.assertIn("KNOWLEDGE_NOT_FOUND", {i.code for i in report.issues})
def test_mutable_framework_dependencies_rejected(self):
for version in ["latest", "[1.0,2.0)", "1.0-SNAPSHOT", "approved-version"]:
framework = approved_framework().model_copy(update={"version": "0.2.0", "dependencies": {"example:library": version}})
with self.subTest(version=version), self.assertRaises(SkillError):
self.service.add_framework(ADMIN, framework)
def test_retirement_does_not_erase_release_evidence(self):
release = self.publish()
self.service.retire(ADMIN, "release", release["key"], "disabled for test")
self.assertEqual(self.service.releases(USER), [])
with self.repo.transaction() as tx:
self.assertIsNotNone(tx.get(ADMIN.tenant_id, "release", release["key"]))
if __name__ == "__main__":
unittest.main()