Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /dislovelhl_enhanced-agent-bus /decision_store.py
| """ | |
| ACGS-2 Decision Store - Redis-backed Decision Explanation Storage | |
| Constitutional Hash: 608508a9bd224290 | |
| Implements FR-12 Decision Explanation API storage requirements: | |
| - Redis-backed storage with 24-hour TTL | |
| - Async CRUD operations | |
| - Tenant isolation via key prefixes | |
| - Integration with audit trail | |
| Performance Targets: | |
| - P99 latency <5ms for single operations | |
| - Support for high-throughput batch operations | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import json | |
| import time | |
| from datetime import UTC, datetime | |
| from typing import TYPE_CHECKING, Any, TypeAlias, cast | |
| from typing_extensions import TypedDict | |
| from enhanced_agent_bus.observability.structured_logging import get_logger | |
| class _DecisionStoreMetrics(TypedDict, total=False): | |
| """Type definition for decision store metrics.""" | |
| total_stores: int | |
| total_retrievals: int | |
| total_deletes: int | |
| cache_hits: int | |
| cache_misses: int | |
| failed_operations: int | |
| total_latency_ms: float | |
| memory_fallback_active: bool | |
| created_at: str | |
| cache_hit_rate: float | |
| avg_latency_ms: float | |
| constitutional_hash: str | |
| try: | |
| from enhanced_agent_bus._compat.constants import CONSTITUTIONAL_HASH | |
| except ImportError: | |
| CONSTITUTIONAL_HASH = "standalone" | |
| if TYPE_CHECKING: | |
| from enhanced_agent_bus._compat.types import JSONDict | |
| else: | |
| try: | |
| from enhanced_agent_bus._compat.types import JSONDict | |
| except ImportError: | |
| JSONDict: TypeAlias = dict[str, Any] | |
| if TYPE_CHECKING: | |
| from enhanced_agent_bus._compat.event_schemas.decision_explanation import ( | |
| DecisionExplanationV1, | |
| create_decision_explanation, | |
| ) | |
| else: | |
| try: | |
| from enhanced_agent_bus._compat.event_schemas.decision_explanation import ( | |
| DecisionExplanationV1, | |
| create_decision_explanation, | |
| ) | |
| except ImportError: | |
| DecisionExplanationV1 = None | |
| create_decision_explanation = None | |
| from .redis_pool import ( | |
| REDIS_AVAILABLE, | |
| RedisConnectionPool, | |
| get_shared_pool, | |
| ) | |
| logger = get_logger(__name__) | |
| # Default configuration | |
| DEFAULT_TTL_SECONDS = 900 # 15 minutes | |
| DEFAULT_KEY_PREFIX = "acgs:decision" | |
| DEFAULT_INDEX_PREFIX = "acgs:decision:idx" | |
| class DecisionStore: | |
| """ | |
| Redis-backed storage for governance decision explanations. | |
| Features: | |
| - 24-hour TTL for decision explanations | |
| - Tenant isolation via key prefixes | |
| - Message ID indexing for lookup | |
| - Batch operations with pipelining | |
| - Metrics collection for observability | |
| - Audit trail integration | |
| Key Schema: | |
| - Decision: {prefix}:{tenant_id}:{decision_id} | |
| - Message Index: {index_prefix}:msg:{tenant_id}:{message_id} | |
| - Time Index: {index_prefix}:time:{tenant_id}:{timestamp} | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| def __init__( | |
| self, | |
| redis_pool: RedisConnectionPool | None = None, | |
| redis_url: str = "redis://localhost:6379", | |
| ttl_seconds: int = DEFAULT_TTL_SECONDS, | |
| key_prefix: str = DEFAULT_KEY_PREFIX, | |
| index_prefix: str = DEFAULT_INDEX_PREFIX, | |
| enable_metrics: bool = True, | |
| ): | |
| """ | |
| Initialize the Decision Store. | |
| Args: | |
| redis_pool: Optional shared Redis pool (creates new if None) | |
| redis_url: Redis connection URL (only used if pool not provided) | |
| ttl_seconds: TTL for stored decisions (default 24 hours) | |
| key_prefix: Key prefix for decision storage | |
| index_prefix: Key prefix for indexes | |
| enable_metrics: Whether to collect operation metrics | |
| """ | |
| self._pool = redis_pool | |
| self._redis_url = redis_url | |
| self._ttl_seconds = ttl_seconds | |
| self._key_prefix = key_prefix | |
| self._index_prefix = index_prefix | |
| self._enable_metrics = enable_metrics | |
| self.constitutional_hash = CONSTITUTIONAL_HASH | |
| self._initialized = False | |
| self._lock = asyncio.Lock() | |
| # In-memory fallback when Redis unavailable | |
| self._memory_store: dict[str, str] = {} | |
| self._memory_indexes: dict[str, str] = {} | |
| self._use_memory_fallback = False | |
| # Metrics tracking | |
| self._metrics: _DecisionStoreMetrics = { | |
| "total_stores": 0, | |
| "total_retrievals": 0, | |
| "total_deletes": 0, | |
| "cache_hits": 0, | |
| "cache_misses": 0, | |
| "failed_operations": 0, | |
| "total_latency_ms": 0.0, | |
| "memory_fallback_active": False, | |
| "created_at": datetime.now(UTC).isoformat(), | |
| } | |
| def _require_pool(self) -> RedisConnectionPool: | |
| if self._pool is None: | |
| raise RuntimeError("Redis connection pool unavailable") | |
| return self._pool | |
| async def initialize(self) -> bool: | |
| """ | |
| Initialize the decision store and Redis connection. | |
| Returns: | |
| True if initialization successful, False otherwise | |
| """ | |
| if self._initialized: | |
| return True | |
| async with self._lock: | |
| if self._initialized: | |
| return True | |
| if not REDIS_AVAILABLE: | |
| logger.warning( | |
| f"[{CONSTITUTIONAL_HASH}] Redis not available, using memory fallback" | |
| ) | |
| self._use_memory_fallback = True | |
| self._metrics["memory_fallback_active"] = True | |
| self._initialized = True | |
| return True | |
| try: | |
| if self._pool is None: | |
| self._pool = await get_shared_pool(redis_url=self._redis_url) | |
| # Verify connection | |
| health = await self._pool.health_check() | |
| if not health.get("healthy"): | |
| logger.warning( | |
| f"[{CONSTITUTIONAL_HASH}] Redis unhealthy, using memory fallback: " | |
| f"{health.get('error', 'unknown error')}" | |
| ) | |
| self._use_memory_fallback = True | |
| self._metrics["memory_fallback_active"] = True | |
| else: | |
| self._use_memory_fallback = False | |
| self._metrics["memory_fallback_active"] = False | |
| self._initialized = True | |
| logger.info( | |
| f"[{CONSTITUTIONAL_HASH}] DecisionStore initialized: " | |
| f"redis={not self._use_memory_fallback}, ttl={self._ttl_seconds}s" | |
| ) | |
| return True | |
| except (OSError, ConnectionError, ValueError, TypeError, RuntimeError) as e: | |
| logger.warning( | |
| f"[{CONSTITUTIONAL_HASH}] Redis init failed, using memory fallback: {e}" | |
| ) | |
| self._use_memory_fallback = True | |
| self._metrics["memory_fallback_active"] = True | |
| self._initialized = True | |
| return True | |
| def _make_key(self, tenant_id: str, decision_id: str) -> str: | |
| """Generate storage key for a decision.""" | |
| safe_tenant = tenant_id.replace(":", "_") if tenant_id else "default" | |
| return f"{self._key_prefix}:{safe_tenant}:{decision_id}" | |
| def _make_message_index_key(self, tenant_id: str, message_id: str) -> str: | |
| """Generate index key for message ID lookup.""" | |
| safe_tenant = tenant_id.replace(":", "_") if tenant_id else "default" | |
| return f"{self._index_prefix}:msg:{safe_tenant}:{message_id}" | |
| def _make_time_index_key(self, tenant_id: str, timestamp: str) -> str: | |
| """Generate index key for time-based lookup.""" | |
| safe_tenant = tenant_id.replace(":", "_") if tenant_id else "default" | |
| return f"{self._index_prefix}:time:{safe_tenant}:{timestamp}" | |
| async def store( | |
| self, | |
| explanation: "DecisionExplanationV1", | |
| ttl_seconds: int | None = None, | |
| ) -> bool: | |
| """ | |
| Store a decision explanation. | |
| Args: | |
| explanation: DecisionExplanationV1 instance to store | |
| ttl_seconds: Optional custom TTL (defaults to store TTL) | |
| Returns: | |
| True if stored successfully, False otherwise | |
| """ | |
| if not self._initialized: | |
| await self.initialize() | |
| start_time = time.perf_counter() | |
| ttl = ttl_seconds or self._ttl_seconds | |
| try: | |
| # Serialize explanation | |
| json_data = explanation.model_dump_json() | |
| tenant_id = explanation.tenant_id or "default" | |
| decision_id = explanation.decision_id | |
| key = self._make_key(tenant_id, decision_id) | |
| if self._use_memory_fallback: | |
| self._memory_store[key] = json_data | |
| # Create message index if message_id present | |
| if explanation.message_id: | |
| msg_key = self._make_message_index_key(tenant_id, explanation.message_id) | |
| self._memory_indexes[msg_key] = decision_id | |
| success = True | |
| else: | |
| async with self._require_pool().acquire() as conn: | |
| # Store explanation with TTL | |
| await conn.setex(key, ttl, json_data) | |
| # Create message index if message_id present | |
| if explanation.message_id: | |
| msg_key = self._make_message_index_key(tenant_id, explanation.message_id) | |
| await conn.setex(msg_key, ttl, decision_id) | |
| success = True | |
| elapsed_ms = (time.perf_counter() - start_time) * 1000 | |
| self._metrics["total_stores"] += 1 | |
| self._metrics["total_latency_ms"] += elapsed_ms | |
| logger.debug( | |
| f"[{CONSTITUTIONAL_HASH}] Stored decision {decision_id} " | |
| f"for tenant {tenant_id} (latency={elapsed_ms:.2f}ms)" | |
| ) | |
| return success | |
| except ( | |
| OSError, | |
| ConnectionError, | |
| ValueError, | |
| TypeError, | |
| RuntimeError, | |
| json.JSONDecodeError, | |
| ) as e: | |
| self._metrics["failed_operations"] += 1 | |
| logger.error(f"[{CONSTITUTIONAL_HASH}] Failed to store decision: {e}") | |
| return False | |
| async def get( | |
| self, | |
| decision_id: str, | |
| tenant_id: str = "default", | |
| ) -> "DecisionExplanationV1" | None: | |
| """ | |
| Retrieve a decision explanation by ID. | |
| Args: | |
| decision_id: Unique decision identifier | |
| tenant_id: Tenant identifier | |
| Returns: | |
| DecisionExplanationV1 if found, None otherwise | |
| """ | |
| if not self._initialized: | |
| await self.initialize() | |
| start_time = time.perf_counter() | |
| key = self._make_key(tenant_id, decision_id) | |
| try: | |
| json_data: str | None | |
| if self._use_memory_fallback: | |
| json_data = self._memory_store.get(key) | |
| else: | |
| async with self._require_pool().acquire() as conn: | |
| json_data = cast(str | None, await conn.get(key)) | |
| elapsed_ms = (time.perf_counter() - start_time) * 1000 | |
| self._metrics["total_retrievals"] += 1 | |
| self._metrics["total_latency_ms"] += elapsed_ms | |
| if json_data: | |
| self._metrics["cache_hits"] += 1 | |
| if DecisionExplanationV1 is not None: | |
| return DecisionExplanationV1.model_validate_json(json_data) | |
| else: | |
| # Return raw dict if schema not available | |
| return json.loads(json_data) | |
| else: | |
| self._metrics["cache_misses"] += 1 | |
| return None | |
| except ( | |
| OSError, | |
| ConnectionError, | |
| ValueError, | |
| TypeError, | |
| RuntimeError, | |
| json.JSONDecodeError, | |
| ) as e: | |
| self._metrics["failed_operations"] += 1 | |
| logger.error(f"[{CONSTITUTIONAL_HASH}] Failed to get decision {decision_id}: {e}") | |
| return None | |
| async def get_by_message_id( | |
| self, | |
| message_id: str, | |
| tenant_id: str = "default", | |
| ) -> "DecisionExplanationV1" | None: | |
| """ | |
| Retrieve a decision explanation by message ID. | |
| Args: | |
| message_id: Message identifier | |
| tenant_id: Tenant identifier | |
| Returns: | |
| DecisionExplanationV1 if found, None otherwise | |
| """ | |
| if not self._initialized: | |
| await self.initialize() | |
| try: | |
| msg_key = self._make_message_index_key(tenant_id, message_id) | |
| decision_id: str | None | |
| if self._use_memory_fallback: | |
| decision_id = self._memory_indexes.get(msg_key) | |
| else: | |
| async with self._require_pool().acquire() as conn: | |
| decision_id = cast(str | None, await conn.get(msg_key)) | |
| if decision_id: | |
| return await self.get(decision_id, tenant_id) | |
| return None | |
| except ( | |
| OSError, | |
| ConnectionError, | |
| ValueError, | |
| TypeError, | |
| RuntimeError, | |
| json.JSONDecodeError, | |
| ) as e: | |
| self._metrics["failed_operations"] += 1 | |
| logger.error( | |
| f"[{CONSTITUTIONAL_HASH}] Failed to get decision by message {message_id}: {e}" | |
| ) | |
| return None | |
| async def delete( | |
| self, | |
| decision_id: str, | |
| tenant_id: str = "default", | |
| ) -> bool: | |
| """ | |
| Delete a decision explanation. | |
| Args: | |
| decision_id: Unique decision identifier | |
| tenant_id: Tenant identifier | |
| Returns: | |
| True if deleted, False otherwise | |
| """ | |
| if not self._initialized: | |
| await self.initialize() | |
| start_time = time.perf_counter() | |
| key = self._make_key(tenant_id, decision_id) | |
| try: | |
| if self._use_memory_fallback: | |
| if key in self._memory_store: | |
| del self._memory_store[key] | |
| # Clean up any message indexes pointing to this decision | |
| for idx_key, idx_val in list(self._memory_indexes.items()): | |
| if idx_val == decision_id: | |
| del self._memory_indexes[idx_key] | |
| deleted = True | |
| else: | |
| deleted = False | |
| else: | |
| async with self._require_pool().acquire() as conn: | |
| # Get the explanation first to find message_id | |
| json_data = await conn.get(key) | |
| if isinstance(json_data, str): | |
| try: | |
| data = json.loads(json_data) | |
| message_id = data.get("message_id") | |
| if message_id: | |
| msg_key = self._make_message_index_key(tenant_id, message_id) | |
| await conn.delete(msg_key) | |
| except json.JSONDecodeError: | |
| pass | |
| result = await conn.delete(key) | |
| deleted = result > 0 | |
| elapsed_ms = (time.perf_counter() - start_time) * 1000 | |
| self._metrics["total_deletes"] += 1 | |
| self._metrics["total_latency_ms"] += elapsed_ms | |
| return deleted | |
| except ( | |
| OSError, | |
| ConnectionError, | |
| ValueError, | |
| TypeError, | |
| RuntimeError, | |
| json.JSONDecodeError, | |
| ) as e: | |
| self._metrics["failed_operations"] += 1 | |
| logger.error(f"[{CONSTITUTIONAL_HASH}] Failed to delete decision {decision_id}: {e}") | |
| return False | |
| async def list_decisions( | |
| self, | |
| tenant_id: str = "default", | |
| limit: int = 100, | |
| offset: int = 0, | |
| ) -> list[str]: | |
| """ | |
| List decision IDs for a tenant. | |
| Args: | |
| tenant_id: Tenant identifier | |
| limit: Maximum number of results | |
| offset: Starting offset | |
| Returns: | |
| List of decision IDs | |
| """ | |
| if not self._initialized: | |
| await self.initialize() | |
| try: | |
| pattern = self._make_key(tenant_id, "*") | |
| if self._use_memory_fallback: | |
| matching_keys = [k for k in self._memory_store.keys() if k.startswith(pattern[:-1])] | |
| matching_keys = matching_keys[offset : offset + limit] | |
| # Extract decision IDs from keys | |
| prefix_len = len(self._make_key(tenant_id, "")) | |
| return [k[prefix_len:] for k in matching_keys] | |
| else: | |
| async with self._require_pool().acquire() as conn: | |
| cursor = 0 | |
| decision_ids = [] | |
| count = 0 | |
| max_scan_iterations = 10_000 | |
| for _ in range(max_scan_iterations): | |
| cursor, keys = await conn.scan(cursor=cursor, match=pattern, count=100) | |
| for key in keys: | |
| if count >= offset: | |
| # Extract decision ID from key | |
| parts = key.split(":") | |
| if len(parts) >= 3: | |
| decision_ids.append(parts[-1]) | |
| count += 1 | |
| if len(decision_ids) >= limit: | |
| return decision_ids | |
| if cursor == 0: | |
| break | |
| return decision_ids | |
| except (OSError, ConnectionError, ValueError, TypeError, RuntimeError) as e: | |
| self._metrics["failed_operations"] += 1 | |
| logger.error(f"[{CONSTITUTIONAL_HASH}] Failed to list decisions: {e}") | |
| return [] | |
| async def exists(self, decision_id: str, tenant_id: str = "default") -> bool: | |
| """ | |
| Check if a decision exists. | |
| Args: | |
| decision_id: Unique decision identifier | |
| tenant_id: Tenant identifier | |
| Returns: | |
| True if exists, False otherwise | |
| """ | |
| if not self._initialized: | |
| await self.initialize() | |
| key = self._make_key(tenant_id, decision_id) | |
| try: | |
| if self._use_memory_fallback: | |
| return key in self._memory_store | |
| else: | |
| async with self._require_pool().acquire() as conn: | |
| return bool(await conn.exists(key) > 0) | |
| except (OSError, ConnectionError, ValueError, TypeError, RuntimeError) as e: | |
| self._metrics["failed_operations"] += 1 | |
| logger.error(f"[{CONSTITUTIONAL_HASH}] Failed to check decision exists: {e}") | |
| return False | |
| async def get_ttl(self, decision_id: str, tenant_id: str = "default") -> int: | |
| """ | |
| Get remaining TTL for a decision. | |
| Args: | |
| decision_id: Unique decision identifier | |
| tenant_id: Tenant identifier | |
| Returns: | |
| TTL in seconds, -1 if no expiry, -2 if key doesn't exist | |
| """ | |
| if not self._initialized: | |
| await self.initialize() | |
| key = self._make_key(tenant_id, decision_id) | |
| try: | |
| if self._use_memory_fallback: | |
| # Memory fallback doesn't support TTL | |
| return self._ttl_seconds if key in self._memory_store else -2 | |
| else: | |
| async with self._require_pool().acquire() as conn: | |
| return int(await conn.ttl(key)) | |
| except (OSError, ConnectionError, ValueError, TypeError, RuntimeError) as e: | |
| self._metrics["failed_operations"] += 1 | |
| logger.error(f"[{CONSTITUTIONAL_HASH}] Failed to get TTL: {e}") | |
| return -2 | |
| async def extend_ttl( | |
| self, | |
| decision_id: str, | |
| tenant_id: str = "default", | |
| ttl_seconds: int | None = None, | |
| ) -> bool: | |
| """ | |
| Extend TTL for a decision. | |
| Args: | |
| decision_id: Unique decision identifier | |
| tenant_id: Tenant identifier | |
| ttl_seconds: New TTL (defaults to store TTL) | |
| Returns: | |
| True if extended, False otherwise | |
| """ | |
| if not self._initialized: | |
| await self.initialize() | |
| key = self._make_key(tenant_id, decision_id) | |
| ttl = ttl_seconds or self._ttl_seconds | |
| try: | |
| if self._use_memory_fallback: | |
| return key in self._memory_store | |
| else: | |
| async with self._require_pool().acquire() as conn: | |
| return bool(await conn.expire(key, ttl)) | |
| except (OSError, ConnectionError, ValueError, TypeError, RuntimeError) as e: | |
| self._metrics["failed_operations"] += 1 | |
| logger.error(f"[{CONSTITUTIONAL_HASH}] Failed to extend TTL: {e}") | |
| return False | |
| def get_metrics(self) -> JSONDict: | |
| """ | |
| Get store operation metrics. | |
| Returns: | |
| Dictionary with store metrics | |
| """ | |
| metrics: JSONDict = dict(self._metrics) | |
| if metrics["total_retrievals"] > 0: | |
| metrics["cache_hit_rate"] = (metrics["cache_hits"] / metrics["total_retrievals"]) * 100 | |
| else: | |
| metrics["cache_hit_rate"] = 0.0 | |
| total_ops = metrics["total_stores"] + metrics["total_retrievals"] + metrics["total_deletes"] | |
| if total_ops > 0: | |
| metrics["avg_latency_ms"] = metrics["total_latency_ms"] / total_ops | |
| else: | |
| metrics["avg_latency_ms"] = 0.0 | |
| metrics["constitutional_hash"] = self.constitutional_hash | |
| return metrics | |
| async def health_check(self) -> JSONDict: | |
| """ | |
| Check health of the decision store. | |
| Returns: | |
| Health status dictionary | |
| """ | |
| health = { | |
| "healthy": True, | |
| "constitutional_hash": self.constitutional_hash, | |
| "timestamp": datetime.now(UTC).isoformat(), | |
| "using_memory_fallback": self._use_memory_fallback, | |
| } | |
| if not self._use_memory_fallback and self._pool: | |
| pool_health = await self._pool.health_check() | |
| health["redis_healthy"] = pool_health.get("healthy", False) | |
| if not health["redis_healthy"]: | |
| health["redis_error"] = pool_health.get("error") | |
| return health | |
| async def close(self) -> None: | |
| """Close the decision store (does not close shared pool).""" | |
| self._initialized = False | |
| self._memory_store.clear() | |
| self._memory_indexes.clear() | |
| logger.info(f"[{CONSTITUTIONAL_HASH}] DecisionStore closed") | |
| # Singleton instance | |
| _decision_store: DecisionStore | None = None | |
| _decision_store_lock = asyncio.Lock() | |
| async def get_decision_store( | |
| redis_url: str = "redis://localhost:6379", | |
| ttl_seconds: int = DEFAULT_TTL_SECONDS, | |
| ) -> DecisionStore: | |
| """ | |
| Get or create shared DecisionStore singleton. | |
| Args: | |
| redis_url: Redis connection URL (only used for first call) | |
| ttl_seconds: TTL for decisions (only used for first call) | |
| Returns: | |
| Shared DecisionStore instance | |
| """ | |
| global _decision_store | |
| if _decision_store is not None and _decision_store._initialized: | |
| return _decision_store | |
| async with _decision_store_lock: | |
| if _decision_store is None or not _decision_store._initialized: | |
| _decision_store = DecisionStore( | |
| redis_url=redis_url, | |
| ttl_seconds=ttl_seconds, | |
| ) | |
| await _decision_store.initialize() | |
| return _decision_store | |
| async def reset_decision_store() -> None: | |
| """Reset the decision store singleton (for testing).""" | |
| global _decision_store | |
| async with _decision_store_lock: | |
| if _decision_store is not None: | |
| await _decision_store.close() | |
| _decision_store = None | |
| __all__ = [ | |
| "DEFAULT_KEY_PREFIX", | |
| "DEFAULT_TTL_SECONDS", | |
| "DecisionStore", | |
| "get_decision_store", | |
| "reset_decision_store", | |
| ] | |
Xet Storage Details
- Size:
- 24.2 kB
- Xet hash:
- 457c66b6e79d5ef2657b22cbd26d445d512c1efb2feafed88b561ff4587bc2cb
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.