Buckets:

download
raw
26.9 kB
from __future__ import annotations
# ruff: noqa: I001 — import order is load-bearing: message_processor must
# precede persistence.executor/repository so it resolves the persistence
# circular-import chain (see block comment below).
"""ACGS-2 Enhanced Agent Bus API Application.
Constitutional Hash: 608508a9bd224290
"""
import inspect
import os
from collections.abc import Awaitable, Callable
from contextlib import asynccontextmanager
from importlib import import_module
from typing import TYPE_CHECKING, Any, cast
import pybreaker
from fastapi import APIRouter, FastAPI
from fastapi.responses import ORJSONResponse
from enhanced_agent_bus.exceptions import (
AgentBusError,
AgentError,
BusNotStartedError,
BusOperationError,
ConstitutionalError,
MACIError,
MessageError,
MessageTimeoutError,
OPAConnectionError,
PolicyError,
)
from ..api_exceptions import (
agent_bus_error_handler,
agent_error_handler,
bus_not_started_handler,
bus_operation_error_handler,
constitutional_error_handler,
global_exception_handler,
maci_error_handler,
message_error_handler,
message_timeout_handler,
opa_connection_handler,
policy_error_handler,
rate_limit_exceeded_handler,
)
# message_processor must load FIRST: it is the only guard that pre-loads
# persistence.repository to resolve the circular import chain
# persistence/__init__.py -> executor -> models -> data_flywheel
# -> dataset_builder -> persistence.repository -> models.
# Reordering this block breaks startup (cold import regresses ~1500ms).
from ..message_processor import MessageProcessor
from ..maci_enforcement import MACIEnforcer, MACIRoleRegistry
from ..persistence.executor import DurableWorkflowExecutor, WorkflowContext
from ..persistence.repository import InMemoryWorkflowRepository
if TYPE_CHECKING:
from ..batch_processor import BatchMessageProcessor
from ..persistence.postgres_repository import PostgresWorkflowRepository
from .config import (
API_VERSION,
BATCH_PROCESSOR_ITEM_TIMEOUT_SECONDS,
BATCH_PROCESSOR_MAX_CONCURRENCY,
BATCH_PROCESSOR_SLOW_ITEM_THRESHOLD_SECONDS,
CIRCUIT_BREAKER_FAIL_MAX,
CIRCUIT_BREAKER_RESET_TIMEOUT_SECONDS,
DEFAULT_API_PORT,
DEFAULT_WORKERS,
)
from .dependencies import get_agent_bus, get_batch_processor
from .middleware import (
correlation_id_middleware,
logger,
setup_all_middleware,
)
from .rate_limiting import (
RATE_LIMITING_AVAILABLE,
RateLimitExceeded,
_rate_limit_exceeded_handler,
limiter,
require_rate_limiting_dependencies,
)
from .message_lifecycle import InMemoryMessageStatusStore
from .runtime_profile import (
allow_fallback,
is_dev,
is_demo,
is_prod,
require_auth_secret,
require_no_degraded_dependencies,
require_prod_dependency,
)
_API_APP_OPERATION_ERRORS = (
RuntimeError,
ValueError,
TypeError,
AttributeError,
LookupError,
OSError,
TimeoutError,
ConnectionError,
)
def _normalize_workflow_dsn(db_url: str) -> str:
"""Normalize workflow DB DSN for asyncpg pool compatibility."""
if db_url.startswith("postgresql+asyncpg://"):
return db_url.replace("postgresql+asyncpg://", "postgresql://")
return db_url
def _load_visual_studio_router() -> APIRouter | None:
"""Load optional Visual Studio router when available."""
try:
module = import_module("..visual_studio.api", package=__package__)
except ImportError:
return None
router = getattr(module, "router", None)
return router if isinstance(router, APIRouter) else None
def _load_copilot_router() -> APIRouter | None:
"""Load optional Policy Copilot router when available."""
try:
module = import_module("..policy_copilot.api", package=__package__)
except ImportError:
return None
router = getattr(module, "router", None)
return router if isinstance(router, APIRouter) else None
def _load_sessions_module() -> Any | None:
"""Load optional sessions module when available."""
try:
return import_module("..routes.sessions", package=__package__)
except ImportError:
return None
# pybreaker is a required dependency — always available
CIRCUIT_BREAKER_AVAILABLE = True
# Global state
agent_bus: MessageProcessor | dict[str, Any] | None = None
batch_processor: BatchMessageProcessor | None = None
message_circuit_breaker: pybreaker.CircuitBreaker | None = None
workflow_executor: DurableWorkflowExecutor | None = None
workflow_repository: Any = None
_CORE_ROUTE_MODULES: tuple[str, ...] = (
".routes.health",
".routes.agent_health",
".routes.messages",
".routes.batch",
".routes.policies",
".routes.governance",
".routes.public_v1",
".routes.badge",
".routes.signup",
".routes.stats",
".routes.usage",
".routes.widget_js",
".routes.workflows",
".routes.z3",
)
def _register_builtin_workflows(executor: DurableWorkflowExecutor) -> DurableWorkflowExecutor:
"""Register stable built-in workflows required by the HTTP API."""
@executor.workflow("builtin.echo")
async def builtin_echo_workflow(ctx: WorkflowContext) -> dict[str, Any]:
payload = ctx.input or {}
return {
"echo": payload,
"workflow_id": ctx.workflow_id,
"tenant_id": ctx.tenant_id,
}
return executor
def _is_development_like_environment() -> bool:
"""Return whether current environment is development-like."""
return is_dev()
def _build_governance_redis_client() -> Any:
"""Create the shared Redis client used by MACI and PQC API state."""
import redis.asyncio as aioredis
redis_url = os.getenv("REDIS_URL", "redis://localhost:6379")
return aioredis.from_url(redis_url, decode_responses=True)
def _initialize_agent_bus_state(
*,
workflow_repository: PostgresWorkflowRepository | InMemoryWorkflowRepository | None = None,
) -> MessageProcessor | dict[str, Any]:
"""Initialize primary message processor, with mock fallback in development."""
logger.info("Initializing Enhanced Agent Bus Message Processor...")
try:
bus = MessageProcessor(workflow_repository=workflow_repository)
logger.info("Enhanced Agent Bus initialized successfully")
return bus
except _API_APP_OPERATION_ERRORS as e:
logger.error(f"Failed to initialize agent bus: {e}")
if allow_fallback("agent_bus_mock"):
environment = os.environ.get("ENVIRONMENT", "").lower()
logger.warning(
f"MessageProcessor not available, using mock mode (ENVIRONMENT={environment})",
)
return {"status": "mock_initialized", "mode": "development"}
raise
async def _initialize_workflow_components(
app: FastAPI,
) -> tuple[DurableWorkflowExecutor | None, Any]:
"""Initialize durable workflow repository and executor."""
logger.info("Initializing Durable Workflow Executor...")
try:
from ..persistence.postgres_repository import PostgresWorkflowRepository
db_url = os.environ.get(
"DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/postgres"
)
normalized_dsn = _normalize_workflow_dsn(db_url)
repository: Any = PostgresWorkflowRepository(dsn=normalized_dsn)
await repository.initialize()
pool = getattr(repository, "_pool", None)
if pool is not None:
async with pool.acquire() as conn:
await conn.execute("SELECT 1")
executor = _register_builtin_workflows(DurableWorkflowExecutor(repository=repository))
app.state.workflow_executor = executor
app.state.workflow_repository = repository
logger.info("Durable Workflow Executor initialized successfully")
return executor, repository
except ImportError:
if allow_fallback("workflow_repository"):
logger.warning(
"asyncpg not installed, using in-memory Workflow Executor outside production"
)
repository = InMemoryWorkflowRepository()
executor = _register_builtin_workflows(DurableWorkflowExecutor(repository=repository))
app.state.workflow_executor = executor
app.state.workflow_repository = repository
return executor, None
raise RuntimeError("asyncpg is required for production workflow persistence") from None
except Exception as e:
if allow_fallback("workflow_repository"):
logger.warning(
"Workflow repository unavailable, using in-memory Workflow Executor in "
f"non-production profile: {e}"
)
repository = InMemoryWorkflowRepository()
executor = _register_builtin_workflows(DurableWorkflowExecutor(repository=repository))
app.state.workflow_executor = executor
return executor, None
logger.error(f"Failed to initialize Durable Workflow Executor: {e}")
raise
return None, None
def _initialize_batch_processor_state(
message_processor: MessageProcessor | dict[str, Any],
) -> BatchMessageProcessor | None:
"""Initialize batch message processor with defensive error handling."""
from ..batch_processor import BatchMessageProcessor
logger.info("Initializing Batch Message Processor...")
try:
processor = BatchMessageProcessor(
message_processor=cast(Any, message_processor),
max_concurrency=BATCH_PROCESSOR_MAX_CONCURRENCY,
enable_deduplication=True,
cache_results=True,
item_timeout=BATCH_PROCESSOR_ITEM_TIMEOUT_SECONDS,
slow_item_threshold=BATCH_PROCESSOR_SLOW_ITEM_THRESHOLD_SECONDS,
)
logger.info("Batch Message Processor initialized successfully")
return processor
except _API_APP_OPERATION_ERRORS as e:
logger.warning(f"BatchMessageProcessor not available: {e}")
return None
async def _initialize_session_manager_if_available() -> None:
"""Initialize session manager when router dependency is installed."""
sessions_module = _load_sessions_module()
if sessions_module is None:
logger.debug("Session Governance API router not available")
return
init_session_manager = getattr(sessions_module, "init_session_manager", None)
if init_session_manager is None:
logger.debug("Session Governance API router not available")
return
redis_url = os.getenv("REDIS_URL", "redis://localhost:6379")
if await init_session_manager(redis_url=redis_url):
logger.info(f"Session context manager connected to Redis at {redis_url}")
else:
logger.warning("Session context manager failed to connect to Redis")
async def _stop_cache_warmer_if_running() -> None:
"""Stop cache warmer on shutdown when available and active."""
try:
from .. import _ext_cache_warming as cache_warming
if (
cache_warming.CACHE_WARMING_AVAILABLE
and (get_cache_warmer := getattr(cache_warming, "get_cache_warmer", None)) is not None
and (warmer := get_cache_warmer()).is_warming
):
logger.info("Cancelling ongoing cache warming...")
warmer.cancel()
except _API_APP_OPERATION_ERRORS as e:
logger.error(f"Failed to stop cache warmer: {e}")
async def _shutdown_session_manager_if_available() -> None:
"""Shutdown session manager when dependency is installed."""
sessions_module = _load_sessions_module()
if sessions_module is None:
logger.debug("Session Governance API router not available on shutdown")
return
shutdown_session_manager = getattr(sessions_module, "shutdown_session_manager", None)
if shutdown_session_manager is None:
logger.debug("Session Governance API router not available on shutdown")
return
try:
await shutdown_session_manager()
logger.info("Session context manager disconnected")
except ImportError:
logger.debug("Session Governance API router not available on shutdown")
async def _close_workflow_repository_if_available(
repository: Any,
) -> None:
"""Close workflow repository during shutdown when initialized."""
try:
if repository is not None and hasattr(repository, "close"):
await repository.close()
logger.info("Workflow Repository closed")
except Exception as e:
logger.error(f"Error closing Workflow Repository: {e}")
async def _close_governance_redis_if_available(app: FastAPI) -> None:
"""Close the shared governance Redis client when initialized."""
client = getattr(app.state, "governance_redis_client", None)
if client is None:
return
close_method = getattr(client, "aclose", None) or getattr(client, "close", None)
if close_method is None:
return
try:
result = close_method()
if inspect.isawaitable(result):
await result
except Exception as e:
logger.error(f"Error closing governance Redis client: {e}")
def _bind_runtime_state(app: FastAPI, *, bus: MessageProcessor | dict[str, Any]) -> None:
"""Bind initialized runtime dependencies onto FastAPI state."""
global agent_bus, batch_processor, message_circuit_breaker
agent_bus = bus
app.state.agent_bus = bus
batch_processor = _initialize_batch_processor_state(bus)
app.state.batch_processor = batch_processor
message_circuit_breaker = pybreaker.CircuitBreaker(
fail_max=CIRCUIT_BREAKER_FAIL_MAX,
reset_timeout=CIRCUIT_BREAKER_RESET_TIMEOUT_SECONDS,
name="message_processing",
)
app.state.message_circuit_breaker = message_circuit_breaker
logger.info("Circuit breaker enabled for message processing")
def _configure_in_memory_governance_state(application: FastAPI) -> None:
"""Wire development/test-only in-memory governance backends."""
from ..pqc_enforcement_config import EnforcementModeConfigService
from .routes.governance import InMemoryPQCConfigBackend, MACIRecordStore
application.state.governance_redis_client = None
application.state.maci_record_store = MACIRecordStore()
application.state.maci_role_registry = MACIRoleRegistry()
application.state.maci_enforcer = MACIEnforcer(
registry=application.state.maci_role_registry,
strict_mode=True,
)
application.state.pqc_enforcement_service = EnforcementModeConfigService(
redis_client=InMemoryPQCConfigBackend()
)
def _configure_shared_governance_state(
application: FastAPI,
*,
redis_client: Any,
) -> None:
"""Wire shared Redis-backed governance backends."""
from ..pqc_enforcement_config import EnforcementModeConfigService
from .routes.governance import RedisMACIRecordStore, RedisMACIRoleRegistry
registry = RedisMACIRoleRegistry(redis_client=redis_client)
application.state.governance_redis_client = redis_client
application.state.maci_record_store = RedisMACIRecordStore(redis_client=redis_client)
application.state.maci_role_registry = registry
application.state.maci_enforcer = MACIEnforcer(
registry=registry,
strict_mode=True,
)
application.state.pqc_enforcement_service = EnforcementModeConfigService(
redis_client=redis_client
)
def _governance_state_is_shared(application: FastAPI) -> bool:
"""Return whether the app already uses shared governance backends."""
from .routes.governance import RedisMACIRecordStore, RedisMACIRoleRegistry
app_state = application.state
return isinstance(getattr(app_state, "maci_record_store", None), RedisMACIRecordStore) and (
isinstance(getattr(app_state, "maci_role_registry", None), RedisMACIRoleRegistry)
)
def _ensure_governance_state(
application: FastAPI,
*,
allow_in_memory_fallback: bool,
) -> None:
"""Ensure MACI and PQC services use shared backends whenever available."""
if _governance_state_is_shared(application):
return
try:
redis_client = _build_governance_redis_client()
_configure_shared_governance_state(application, redis_client=redis_client)
except Exception as e:
if not allow_in_memory_fallback:
raise
logger.warning(f"Shared governance backends unavailable, using in-memory fallback: {e}")
_configure_in_memory_governance_state(application)
async def _verify_governance_redis_ready(application: FastAPI) -> None:
"""Verify the shared Redis governance backend is reachable in production."""
client = getattr(application.state, "governance_redis_client", None)
require_prod_dependency("governance_redis_client", client is not None)
if client is None:
return
try:
ping_result = client.ping()
if inspect.isawaitable(ping_result):
await ping_result
except Exception as e:
if is_prod():
raise RuntimeError("Production governance Redis readiness check failed") from e
logger.warning(f"Governance Redis readiness check failed outside production: {e}")
async def _verify_policy_backend_ready() -> None:
"""Verify configured policy backend availability."""
registry_url = os.environ.get("POLICY_REGISTRY_URL") or os.environ.get(
"EAB_POLICY_REGISTRY_URL"
)
required = is_prod() or os.environ.get("EAB_POLICY_BACKEND_REQUIRED") == "1"
if registry_url and "mock" in registry_url.lower():
require_prod_dependency("policy_backend_non_mock", False)
if not required and not registry_url:
return
require_prod_dependency("policy_backend_url", registry_url)
from ..policy_client import PolicyRegistryClient
client = PolicyRegistryClient(registry_url=registry_url or "http://localhost:8000")
try:
health = await client.health_check()
finally:
await client.close()
if str(health.get("status", "")).lower() not in {"healthy", "ready", "ok"}:
require_prod_dependency("policy_backend_health", False)
async def _verify_audit_backend_ready() -> None:
"""Verify configured audit backend availability."""
service_url = os.environ.get("AUDIT_SERVICE_URL") or os.environ.get("EAB_AUDIT_SERVICE_URL")
required = is_prod() or os.environ.get("EAB_AUDIT_BACKEND_REQUIRED") == "1"
if not required and not service_url:
return
require_prod_dependency("audit_backend_url", service_url)
from ..audit_client import AuditClient, AuditClientConfig
client = AuditClient(config=AuditClientConfig(service_url=service_url or ""))
try:
health = await client.health_check()
finally:
await client.close()
if str(health.get("status", "")).lower() != "healthy":
require_prod_dependency("audit_backend_health", False)
async def _verify_startup_dependencies(application: FastAPI) -> None:
"""Run production fail-closed startup dependency checks."""
require_auth_secret()
await _verify_governance_redis_ready(application)
if is_prod() or is_demo():
await _verify_policy_backend_ready()
await _verify_audit_backend_ready()
# Checked last: the named gates above report a specific missing backend,
# which is the more actionable error when both are unsatisfied.
require_no_degraded_dependencies()
def _attach_pqc_postgres_fallback(
application: FastAPI,
repository: Any,
) -> None:
"""Attach PostgreSQL fallback storage to the shared PQC service when available."""
from ..pqc_enforcement_config import EnforcementModeConfigService
service = getattr(application.state, "pqc_enforcement_service", None)
if not isinstance(service, EnforcementModeConfigService):
return
pool = getattr(repository, "_pool", None) if repository is not None else None
if pool is not None:
service._pg = pool
async def _startup_runtime(app: FastAPI) -> None:
"""Initialize runtime state required during application startup."""
global workflow_executor, workflow_repository
_ensure_governance_state(
app,
allow_in_memory_fallback=allow_fallback("governance_state"),
)
await _verify_governance_redis_ready(app)
workflow_executor, workflow_repository = await _initialize_workflow_components(app)
bus = _initialize_agent_bus_state(workflow_repository=workflow_repository)
_bind_runtime_state(app, bus=bus)
_attach_pqc_postgres_fallback(app, workflow_repository)
await _verify_startup_dependencies(app)
await _initialize_session_manager_if_available()
async def _shutdown_runtime(app: FastAPI) -> None:
"""Shutdown runtime state in the inverse order of startup."""
await _stop_cache_warmer_if_running()
await _shutdown_session_manager_if_available()
await _close_workflow_repository_if_available(workflow_repository)
await _close_governance_redis_if_available(app)
logger.info("Enhanced Agent Bus stopped")
def _configure_application_state(application: FastAPI) -> None:
"""Initialize non-runtime FastAPI application state."""
application.state.limiter = limiter
application.state.failed_tasks = []
if allow_fallback("message_status_store"):
application.state.message_status_store = InMemoryMessageStatusStore()
_ensure_governance_state(
application,
allow_in_memory_fallback=allow_fallback("governance_state"),
)
def _register_core_routers(application: FastAPI) -> None:
"""Register the stable built-in router set (lazy imports at call time)."""
for module_path in _CORE_ROUTE_MODULES:
module = import_module(module_path, package=__package__)
router = getattr(module, "router", None)
if isinstance(router, APIRouter):
application.include_router(router)
def _register_feature_routers(application: FastAPI) -> None:
"""Register optional feature routers when present."""
if (vsr := _load_visual_studio_router()) is not None:
application.include_router(vsr)
logger.info("Visual Studio API router registered")
if (cr := _load_copilot_router()) is not None:
application.include_router(cr)
logger.info("Policy Copilot API router registered")
_register_optional_routers(application)
def _configure_application(application: FastAPI) -> FastAPI:
"""Apply state, middleware, exception handlers, and routers to the app."""
_configure_application_state(application)
setup_all_middleware(application)
_register_exception_handlers(application)
application.middleware("http")(correlation_id_middleware)
application.exception_handler(Exception)(global_exception_handler)
_register_core_routers(application)
_register_feature_routers(application)
return application
@asynccontextmanager
async def _lifespan_context(app: FastAPI) -> Any:
"""Manage application startup and shutdown lifecycle."""
global workflow_executor, workflow_repository
await _startup_runtime(app)
yield
await _shutdown_runtime(app)
def _register_exception_handlers(app: FastAPI) -> None:
"""Register all exception handlers."""
rate_limit_handler = (
_rate_limit_exceeded_handler if RATE_LIMITING_AVAILABLE else rate_limit_exceeded_handler
)
app.add_exception_handler(RateLimitExceeded, cast(Any, rate_limit_handler))
handlers: list[tuple[type[Exception], Callable[[Any, Exception], Awaitable[Any] | Any]]] = [
(MessageTimeoutError, message_timeout_handler),
(BusNotStartedError, bus_not_started_handler),
(OPAConnectionError, opa_connection_handler),
(ConstitutionalError, constitutional_error_handler),
(MACIError, maci_error_handler),
(PolicyError, policy_error_handler),
(AgentError, agent_error_handler),
(MessageError, message_error_handler),
(BusOperationError, bus_operation_error_handler),
(AgentBusError, agent_bus_error_handler),
]
for exc_class, handler in handlers:
app.add_exception_handler(exc_class, cast(Any, handler))
if RATE_LIMITING_AVAILABLE:
logger.info("Rate limiting enabled: 60/minute per client")
else:
logger.info("Rate limiting not available (slowapi not installed)")
def _register_optional_routers(application: FastAPI) -> None:
"""Register optional routers that may not be available."""
try:
from ..constitutional.review_api import router as constitutional_review_router
application.include_router(constitutional_review_router)
logger.info("Constitutional Review API router registered")
except ImportError:
logger.debug("Constitutional Review API router not available")
try:
from ..circuit_breaker import create_circuit_health_router
if circuit_health_router := create_circuit_health_router():
application.include_router(circuit_health_router)
logger.info("Circuit Breaker Health router registered")
except ImportError:
logger.debug("Circuit Breaker Health router not available")
sessions_module = _load_sessions_module()
sessions_router = getattr(sessions_module, "router", None) if sessions_module else None
if isinstance(sessions_router, APIRouter):
application.include_router(sessions_router)
logger.info("Session Governance API router registered")
else:
logger.debug("Session Governance API router not available")
def create_app() -> FastAPI:
"""Create and configure the FastAPI application."""
require_rate_limiting_dependencies()
application = FastAPI(
title="ACGS-2 Enhanced Agent Bus API",
description="API for the ACGS-2 Enhanced Agent Bus with Constitutional Compliance",
version=API_VERSION,
default_response_class=ORJSONResponse,
lifespan=_lifespan_context,
)
return _configure_application(application)
_module_app: FastAPI | None = None
def __getattr__(name: str) -> Any:
global _module_app
if name == "app":
if _module_app is None:
_module_app = create_app()
return _module_app
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
if __name__ == "__main__":
import uvicorn
uvicorn.run(
create_app(),
host="127.0.0.1", # nosec B104 - Intentional for container deployment
port=DEFAULT_API_PORT,
reload=False,
log_level="warning",
workers=DEFAULT_WORKERS,
loop="uvloop",
http="httptools",
access_log=False,
)
__all__ = [ # noqa: F822 — "app" provided lazily via module __getattr__ (PEP 562)
"agent_bus",
"app",
"batch_processor",
"create_app",
"get_agent_bus",
"get_batch_processor",
"message_circuit_breaker",
]

Xet Storage Details

Size:
26.9 kB
·
Xet hash:
2f0558de463245f92706597f3f10adef5e1e3a17cae3ee62c80e28f9f66266f7

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.