Spaces:
Sleeping
Sleeping
File size: 15,937 Bytes
caf8059 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 | """Business invariant tests; no model calls, Java runs or source-download execution."""
import copy
import tempfile
import unittest
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
from pydantic import ValidationError
from src.adapters.persistence.skill_sqlite import SqliteSkillRepository
from src.modules.skill.application.service import SkillService
from src.modules.skill.atomic_registry.catalog import builtin_operations, check_handler
from src.modules.skill.domain.models import Actor, Budget, Draft, Framework, Operation, SkillError, fits
from src.modules.skill.graph.compiler import compile_graph
from src.modules.skill.packs.examples import all_examples, framework_candidate
ADMIN = Actor(tenant_id="test-tenant", user_id="author", roles=["admin"])
REVIEWER = Actor(tenant_id="test-tenant", user_id="reviewer", roles=["reviewer"])
USER = Actor(tenant_id="test-tenant", user_id="developer", roles=[])
OTHER = Actor(tenant_id="another-tenant", user_id="author", roles=["admin"])
REVISIONS = {c: "test-revision-1" for c in ["java-basics", "java-patterns", "erp", "wms", "equipment-ops", "java-frameworks", "web-development", "java-persistence", "java-testing", "java-integration"]}
class KnowledgeDouble:
def __init__(self):
self.allowed = True
self.revision = "test-revision-1"
def __call__(self, actor, binding):
if not self.allowed:
raise SkillError("KNOWLEDGE_NOT_FOUND", "test denied", 404)
if binding.revision != self.revision:
raise SkillError("KNOWLEDGE_REVISION_MISMATCH", "test changed", 409)
return {"id": binding.collection_id, "revision": self.revision, "digest": "a" * 64, "scope": binding.usage}
def approved_framework():
return framework_candidate().model_copy(update={"state": "approved", "evidence_refs": ["unit-test-fixture-not-production-certification"]})
class CompilerTests(unittest.TestCase):
def setUp(self):
self.draft = all_examples(REVISIONS)[0]
self.ops = {o.operation_id + "@" + o.version: o for o in builtin_operations()}
def report(self, data=None):
return compile_graph(Draft.model_validate(data) if data else self.draft, self.ops, approved_framework())
def codes(self, data):
return {x.code for x in self.report(data).issues}
def test_three_domain_flows_compile_and_share_operations(self):
for draft in all_examples(REVISIONS):
report = compile_graph(draft, self.ops, approved_framework())
self.assertTrue(report.valid, report.issues)
self.assertEqual(report.model_calls, 2)
self.assertEqual(len(report.execution_order), 16)
def test_cycle_rejected(self):
data = self.draft.model_dump()
data["graph"]["control_edges"].append({"source": "end", "target": "start"})
self.assertIn("CONTROL_CYCLE", self.codes(data))
def test_duplicate_node_rejected(self):
data = self.draft.model_dump()
data["graph"]["nodes"].append(data["graph"]["nodes"][0])
self.assertIn("DUPLICATE_NODE", self.codes(data))
def test_unknown_node_and_duplicate_edge_rejected(self):
data = self.draft.model_dump()
data["graph"]["control_edges"] += [{"source": "unknown", "target": "end"}, data["graph"]["control_edges"][0]]
self.assertTrue({"UNKNOWN_NODE", "DUPLICATE_EDGE"}.issubset(self.codes(data)))
def test_disconnected_node_rejected(self):
data = self.draft.model_dump()
data["graph"]["nodes"].append({"id": "orphan", "kind": "approval", "approval_scope": "design"})
self.assertIn("DISCONNECTED_NODE", self.codes(data))
def test_type_mismatch_and_reverse_data_rejected(self):
data = self.draft.model_dump()
data["graph"]["data_edges"][0]["source_port"] = "query"
data["graph"]["data_edges"][0]["source"] = "end"
self.assertTrue({"UNKNOWN_PORT", "DATA_ORDER"}.issubset(self.codes(data)))
data = self.draft.model_dump()
data["graph"]["data_edges"][0].update(source="requirements", source_port="document", target="knowledge", target_port="query")
self.assertIn("PORT_TYPE_MISMATCH", self.codes(data))
def test_unknown_operation_and_framework_rejected(self):
data = self.draft.model_dump()
data["graph"]["nodes"][1]["operation_ref"] = "unapproved@1.0.0"
self.assertIn("OPERATION_UNAVAILABLE", self.codes(data))
self.assertIn("FRAMEWORK_UNAVAILABLE", {x.code for x in compile_graph(self.draft, self.ops, framework_candidate()).issues})
def test_missing_input_and_duplicate_binding_rejected(self):
data = self.draft.model_dump()
data["graph"]["data_edges"].pop(0)
self.assertIn("REQUIRED_INPUT", self.codes(data))
data = self.draft.model_dump()
data["graph"]["data_edges"].append(copy.deepcopy(data["graph"]["data_edges"][0]))
self.assertIn("MULTIPLE_INPUT_SOURCES", self.codes(data))
def test_literal_cannot_forge_test_evidence(self):
data = self.draft.model_dump()
data["graph"]["data_edges"] = [e for e in data["graph"]["data_edges"] if not(e["target"] == "preview" and e["target_port"] == "test_evidence")]
next(n for n in data["graph"]["nodes"] if n["id"] == "preview")["inputs"] = {"test_evidence": {"passed": True}}
self.assertIn("UNTRUSTED_STAGE_INPUT", self.codes(data))
def test_object_port_cannot_substitute_for_test_evidence(self):
data = self.draft.model_dump()
edge = next(e for e in data["graph"]["data_edges"] if e["target"] == "preview" and e["target_port"] == "test_evidence")
edge.update(source="design", source_port="design")
self.assertIn("UNTRUSTED_STAGE_INPUT", self.codes(data))
def test_design_and_execution_gates_must_dominate_all_paths(self):
data = self.draft.model_dump()
data["graph"]["control_edges"].append({"source": "design", "target": "patch"})
self.assertTrue({"DESIGN_GATE_BYPASS", "EXECUTION_GATE_BYPASS"}.issubset(self.codes(data)))
def test_test_gate_must_dominate_preview(self):
data = self.draft.model_dump()
data["graph"]["control_edges"].append({"source": "build", "target": "preview"})
self.assertIn("QUALITY_GATE_BYPASS", self.codes(data))
def test_acceptance_cannot_be_bypassed(self):
data = self.draft.model_dump()
data["graph"]["control_edges"].append({"source": "preview", "target": "evidence"})
self.assertIn("ACCEPTANCE_GATE_BYPASS", self.codes(data))
def test_incomplete_flow_and_forged_end_manifest_rejected(self):
data = self.draft.model_dump()
data["graph"]["control_edges"].append({"source": "start", "target": "end"})
self.assertIn("INCOMPLETE_CODING_FLOW", self.codes(data))
data = self.draft.model_dump()
data["graph"]["data_edges"][-1].update(source="design", source_port="design")
self.assertIn("UNTRUSTED_MANIFEST", self.codes(data))
def test_resources_cannot_reference_unbound_collection(self):
data = self.draft.model_dump()
data["graph"]["resource_edges"][0]["resource"] = "secret"
self.assertIn("RESOURCE_REFERENCE", self.codes(data))
def test_budget_must_include_shared_repair_reserve(self):
data = self.draft.model_dump()
data["budget"]["max_repairs"] = 2
self.assertIn("BUDGET_UNREACHABLE", self.codes(data))
self.assertEqual(Budget().max_model_calls,3)
data['budget']=Budget(max_model_calls=6,max_tokens=120000,max_repairs=2).model_dump()
self.assertNotIn('BUDGET_UNREACHABLE',self.codes(data))
with self.assertRaises(ValidationError):
Budget(max_model_calls=7)
def test_unknown_executable_fields_and_weak_handler_contract_rejected(self):
data = self.draft.model_dump()
data["graph"]["nodes"][1]["script"] = "arbitrary code"
with self.assertRaises(ValidationError):
Draft.model_validate(data)
operation = builtin_operations()[5].model_copy(update={"permissions": []})
with self.assertRaises(SkillError):
check_handler(operation)
def test_boolean_is_not_integer_and_unsafe_framework_paths_rejected(self):
self.assertFalse(fits(True, "integer"))
self.assertFalse(fits(float("nan"), "number"))
for path in ["../", "/etc/", "runner/", "src/main/../../"]:
with self.subTest(path=path), self.assertRaises(ValidationError):
Framework(profile_id="unsafe", version="0.1.0", allowed_paths=[path])
class ServiceTests(unittest.TestCase):
def setUp(self):
folder = Path(__file__).resolve().parents[4] / ".local/test-runs/skill" / uuid.uuid4().hex
folder.mkdir(parents=True)
self.tmp = SimpleNamespace(name=str(folder))
self.repo = SqliteSkillRepository(Path(self.tmp.name) / "skill.sqlite")
self.knowledge = KnowledgeDouble()
self.access = lambda actor, pid: self.assertEqual(pid, "project-one")
self.service = SkillService(self.repo, self.knowledge, self.access)
self.service.bootstrap(ADMIN)
self.service.add_framework(ADMIN, approved_framework())
self.draft = all_examples(REVISIONS)[0]
self.saved = self.service.save_draft(ADMIN, self.draft)
def publish(self):
report = self.service.validate(ADMIN, self.draft.skill_id)
approval = self.service.approve(REVIEWER, self.draft.skill_id, 1, report.digest, ["unit-test-evidence"])
return self.service.publish(ADMIN, self.draft.skill_id, "0.1.0", 1, approval["key"])
def test_draft_cas_does_not_overwrite(self):
changed = self.draft.model_copy(update={"description": "updated"})
result = self.service.save_draft(ADMIN, changed, 1)
self.assertEqual(result["revision"], 2)
with self.assertRaises(SkillError) as exc:
self.service.save_draft(ADMIN, self.draft, 1)
self.assertEqual(exc.exception.code, "REVISION_CONFLICT")
def test_concurrent_cas_has_one_winner(self):
def update(label):
try:
self.service.save_draft(ADMIN, self.draft.model_copy(update={"description": label}), 1)
return "won"
except SkillError as exc:
return exc.code
with ThreadPoolExecutor(max_workers=2) as pool:
results = list(pool.map(update, ["one", "two"]))
self.assertCountEqual(results, ["won", "REVISION_CONFLICT"])
def test_cross_tenant_and_regular_user_denied(self):
with self.assertRaises(SkillError) as exc:
self.service.read_draft(OTHER, self.draft.skill_id)
self.assertEqual(exc.exception.status, 404)
with self.assertRaises(SkillError) as exc:
self.service.save_draft(USER, self.draft)
self.assertEqual(exc.exception.status, 403)
def test_author_cannot_approve_own_revision(self):
report = self.service.validate(ADMIN, self.draft.skill_id)
with self.assertRaises(SkillError) as exc:
self.service.approve(ADMIN, self.draft.skill_id, 1, report.digest, ["test"])
self.assertEqual(exc.exception.code, "INDEPENDENT_REVIEW_REQUIRED")
def test_last_editor_cannot_approve_others_original_draft(self):
other = Actor(tenant_id=ADMIN.tenant_id, user_id="editor-two", roles=["admin"])
self.service.save_draft(other, self.draft, 1)
report = self.service.validate(other, self.draft.skill_id)
with self.assertRaises(SkillError) as exc:
self.service.approve(other, self.draft.skill_id, 2, report.digest, ["test"])
self.assertEqual(exc.exception.code, "INDEPENDENT_REVIEW_REQUIRED")
def test_approval_invalidated_by_draft_change(self):
report = self.service.validate(ADMIN, self.draft.skill_id)
approval = self.service.approve(REVIEWER, self.draft.skill_id, 1, report.digest, ["test"])
self.service.save_draft(ADMIN, self.draft.model_copy(update={"description": "changed"}), 1)
with self.assertRaises(SkillError) as exc:
self.service.publish(ADMIN, self.draft.skill_id, "0.1.0", 2, approval["key"])
self.assertEqual(exc.exception.code, "APPROVAL_TARGET_CHANGED")
def test_releases_are_immutable_and_new_drafts_do_not_change_them(self):
release = self.publish()
self.service.save_draft(ADMIN, self.draft.model_copy(update={"description": "new design"}), 1)
self.assertEqual(self.service.releases(USER)[0]["digest"], release["digest"])
with self.repo.transaction(write=True) as tx, self.assertRaises(SkillError):
tx.update(ADMIN.tenant_id, "release", release["key"], {}, 1)
def test_duplicate_release_cannot_replace_history(self):
release = self.publish()
with self.repo.transaction(write=True) as tx, self.assertRaises(SkillError) as exc:
tx.create(ADMIN.tenant_id, "release", release["key"], {})
self.assertEqual(exc.exception.code, "VERSION_EXISTS")
def test_stop_new_snapshots_for_retired_dependencies(self):
release = self.publish()
self.service.retire(ADMIN, "operation", "AO09@0.1.0", "test retirement")
with self.assertRaises(SkillError) as exc:
self.service.prepare(USER, release["key"], "project-one", "a" * 64, "request-one")
self.assertEqual(exc.exception.code, "DEPENDENCY_RETIRED")
def test_knowledge_revocation_is_rechecked_at_start(self):
release = self.publish()
self.knowledge.allowed = False
with self.assertRaises(SkillError) as exc:
self.service.prepare(USER, release["key"], "project-one", "a" * 64, "request-one")
self.assertEqual(exc.exception.status, 404)
def test_snapshot_idempotency_and_source_binding(self):
release = self.publish()
first = self.service.prepare(USER, release["key"], "project-one", "a" * 64, "request-one")
replay = self.service.prepare(USER, release["key"], "project-one", "a" * 64, "request-one")
self.assertEqual(first, replay)
self.assertEqual(first["execution_status"], "not_started")
with self.assertRaises(SkillError) as exc:
self.service.prepare(USER, release["key"], "project-one", "b" * 64, "request-one")
self.assertEqual(exc.exception.code, "IDEMPOTENCY_CONFLICT")
def test_restart_keeps_data_and_does_not_execute(self):
release = self.publish()
restarted = SkillService(SqliteSkillRepository(self.repo.path), self.knowledge, self.access)
self.assertEqual(restarted.releases(USER)[0]["key"], release["key"])
with self.repo.transaction() as tx:
self.assertEqual(tx.listing(ADMIN.tenant_id, "snapshot"), [])
def test_invisible_knowledge_fails_validation(self):
self.knowledge.allowed = False
report = self.service.validate(ADMIN, self.draft.skill_id)
self.assertFalse(report.valid)
self.assertIn("KNOWLEDGE_NOT_FOUND", {i.code for i in report.issues})
def test_mutable_framework_dependencies_rejected(self):
for version in ["latest", "[1.0,2.0)", "1.0-SNAPSHOT", "approved-version"]:
framework = approved_framework().model_copy(update={"version": "0.2.0", "dependencies": {"example:library": version}})
with self.subTest(version=version), self.assertRaises(SkillError):
self.service.add_framework(ADMIN, framework)
def test_retirement_does_not_erase_release_evidence(self):
release = self.publish()
self.service.retire(ADMIN, "release", release["key"], "disabled for test")
self.assertEqual(self.service.releases(USER), [])
with self.repo.transaction() as tx:
self.assertIsNotNone(tx.get(ADMIN.tenant_id, "release", release["key"]))
if __name__ == "__main__":
unittest.main()
|