Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /dislovelhl_enhanced-agent-bus /context_optimization.py
| """ | |
| ACGS-2 Enhanced Agent Bus - Context Window Optimization | |
| Constitutional Hash: 608508a9bd224290 | |
| Phase 4 implementation providing: | |
| - SpecDeltaCompressor: 60-80% payload reduction via delta compression | |
| - CachedGovernanceValidator: 90%+ cache hit rate for OPA decisions | |
| - OptimizedAgentBus: Topic-partitioned pub/sub for efficient coordination | |
| Part of Agent Orchestration Improvements Phases 4-7. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import hashlib | |
| import json | |
| import time | |
| from abc import ABC, abstractmethod | |
| from collections import OrderedDict | |
| from collections.abc import Callable, Coroutine | |
| from copy import deepcopy | |
| from dataclasses import dataclass, field | |
| from datetime import datetime, timedelta | |
| from enum import Enum | |
| from typing import ( | |
| TYPE_CHECKING, | |
| Any, | |
| ClassVar, | |
| Literal, | |
| TypeAlias, | |
| TypeVar, | |
| ) | |
| try: | |
| from enhanced_agent_bus._compat.constants import CONSTITUTIONAL_HASH | |
| except ImportError: | |
| CONSTITUTIONAL_HASH = "standalone" | |
| if TYPE_CHECKING: | |
| from enhanced_agent_bus._compat.types import JSONDict | |
| else: | |
| try: | |
| from enhanced_agent_bus._compat.types import JSONDict | |
| except ImportError: | |
| JSONDict: TypeAlias = dict[str, Any] | |
| from enhanced_agent_bus.observability.structured_logging import get_logger | |
| logger = get_logger(__name__) | |
| T = TypeVar("T") | |
| DEFAULT_CACHE_HASH_MODE = "sha256" | |
| _CACHE_HASH_MODES = {"sha256", "fast"} | |
| try: | |
| from acgs2_perf import fast_hash | |
| FAST_HASH_AVAILABLE = True | |
| except ImportError: | |
| FAST_HASH_AVAILABLE = False | |
| # Feature flag for context optimization | |
| CONTEXT_OPTIMIZATION_AVAILABLE = True | |
| # ============================================================================= | |
| # Task 4.1: Spec Delta Compression | |
| # ============================================================================= | |
| class CompressionStrategy(str, Enum): | |
| """Compression strategy for spec delta encoding.""" | |
| FULL = "full" # Send complete spec | |
| DELTA = "delta" # Send only changed fields | |
| INCREMENTAL = "incremental" # Incremental updates with version tracking | |
| class CompressionResult: | |
| """Result of spec compression operation. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| strategy: CompressionStrategy | |
| original_size: int | |
| compressed_size: int | |
| compression_ratio: float | |
| payload: JSONDict | |
| checksum: str | |
| constitutional_hash: str = field(default=CONSTITUTIONAL_HASH) | |
| def bytes_saved(self) -> int: | |
| """Calculate bytes saved through compression.""" | |
| return self.original_size - self.compressed_size | |
| class SpecBaseline: | |
| """Baseline spec for delta computation. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| spec_id: str | |
| spec_data: JSONDict | |
| checksum: str | |
| created_at: datetime = field(default_factory=lambda: datetime.now()) | |
| version: int = 1 | |
| constitutional_hash: str = field(default=CONSTITUTIONAL_HASH) | |
| class SpecDeltaCompressor: | |
| """ | |
| Transmit only changed fields after initial sync. | |
| Achieves 60-80% reduction in inter-agent communication payload | |
| by maintaining baseline specs and computing deltas. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| def __init__( | |
| self, | |
| max_baselines: int = 1000, | |
| baseline_ttl_seconds: int = 3600, | |
| enable_incremental: bool = True, | |
| ): | |
| self._baselines: OrderedDict[str, SpecBaseline] = OrderedDict() | |
| self._max_baselines = max_baselines | |
| self._baseline_ttl = timedelta(seconds=baseline_ttl_seconds) | |
| self._enable_incremental = enable_incremental | |
| self._lock = asyncio.Lock() | |
| self._stats = { | |
| "compressions": 0, | |
| "full_sends": 0, | |
| "delta_sends": 0, | |
| "bytes_saved": 0, | |
| } | |
| self.constitutional_hash = CONSTITUTIONAL_HASH | |
| def _compute_checksum(self, data: JSONDict) -> str: | |
| """Compute checksum including constitutional hash for integrity.""" | |
| content = json.dumps(data, sort_keys=True) + CONSTITUTIONAL_HASH | |
| if FAST_HASH_AVAILABLE: | |
| return f"{fast_hash(content):016x}" | |
| return hashlib.sha256(content.encode()).hexdigest()[:16] | |
| def _compute_delta(self, current: JSONDict, baseline: JSONDict) -> tuple[JSONDict, bool]: | |
| """ | |
| Compute delta between current and baseline specs. | |
| Returns tuple of (delta_dict, has_changes). | |
| """ | |
| delta: JSONDict = {} | |
| has_changes = False | |
| missing = object() | |
| for key, value in current.items(): | |
| baseline_value = baseline.get(key, missing) | |
| if baseline_value is missing or value != baseline_value: | |
| delta[key] = value | |
| has_changes = True | |
| # Check for removed keys | |
| for key in baseline: | |
| if key not in current: | |
| delta[f"__removed__{key}"] = None | |
| has_changes = True | |
| return delta, has_changes | |
| async def compress(self, spec_id: str, spec_data: JSONDict) -> CompressionResult: | |
| """ | |
| Compress spec data using delta encoding. | |
| First call for a spec_id sends full data and establishes baseline. | |
| Subsequent calls send only changed fields. | |
| """ | |
| async with self._lock: | |
| self._stats["compressions"] += 1 | |
| original_json = json.dumps(spec_data, sort_keys=True) | |
| original_size = len(original_json.encode()) | |
| # Evict old baselines if needed | |
| await self._evict_stale_baselines() | |
| baseline = self._baselines.get(spec_id) | |
| if baseline is None: | |
| # No baseline - send full spec | |
| self._stats["full_sends"] += 1 | |
| checksum = self._compute_checksum(spec_data) | |
| # Store as new baseline | |
| self._baselines[spec_id] = SpecBaseline( | |
| spec_id=spec_id, | |
| spec_data=deepcopy(spec_data), | |
| checksum=checksum, | |
| ) | |
| self._baselines.move_to_end(spec_id) | |
| return CompressionResult( | |
| strategy=CompressionStrategy.FULL, | |
| original_size=original_size, | |
| compressed_size=original_size, | |
| compression_ratio=1.0, | |
| payload={"full": True, "spec": spec_data}, | |
| checksum=checksum, | |
| ) | |
| # Compute delta from baseline | |
| delta, has_changes = self._compute_delta(spec_data, baseline.spec_data) | |
| if not has_changes: | |
| # No changes - send minimal acknowledgment | |
| compressed_payload: JSONDict = { | |
| "full": False, | |
| "delta": {}, | |
| "checksum": baseline.checksum, | |
| "version": baseline.version, | |
| } | |
| compressed_json = json.dumps(compressed_payload) | |
| compressed_size = len(compressed_json.encode()) | |
| return CompressionResult( | |
| strategy=CompressionStrategy.DELTA, | |
| original_size=original_size, | |
| compressed_size=compressed_size, | |
| compression_ratio=compressed_size / original_size, | |
| payload=compressed_payload, | |
| checksum=baseline.checksum, | |
| ) | |
| # Has changes - send delta | |
| self._stats["delta_sends"] += 1 | |
| new_checksum = self._compute_checksum(spec_data) | |
| compressed_payload = { | |
| "full": False, | |
| "delta": delta, | |
| "checksum": new_checksum, | |
| "baseline_checksum": baseline.checksum, | |
| "version": baseline.version + 1, | |
| } | |
| compressed_json = json.dumps(compressed_payload) | |
| compressed_size = len(compressed_json.encode()) | |
| # Update baseline | |
| baseline.spec_data = deepcopy(spec_data) | |
| baseline.checksum = new_checksum | |
| baseline.version += 1 | |
| baseline.created_at = datetime.now() | |
| self._baselines.move_to_end(spec_id) | |
| bytes_saved = original_size - compressed_size | |
| self._stats["bytes_saved"] += bytes_saved | |
| return CompressionResult( | |
| strategy=CompressionStrategy.DELTA, | |
| original_size=original_size, | |
| compressed_size=compressed_size, | |
| compression_ratio=compressed_size / original_size, | |
| payload=compressed_payload, | |
| checksum=new_checksum, | |
| ) | |
| async def decompress(self, spec_id: str, compressed: JSONDict) -> tuple[JSONDict, bool]: | |
| """ | |
| Decompress a compressed spec payload. | |
| Returns tuple of (full_spec, success). | |
| """ | |
| async with self._lock: | |
| if compressed.get("full", True): | |
| # Full spec - store as baseline and return | |
| spec_data = compressed.get("spec", {}) | |
| checksum = self._compute_checksum(spec_data) | |
| self._baselines[spec_id] = SpecBaseline( | |
| spec_id=spec_id, | |
| spec_data=deepcopy(spec_data), | |
| checksum=checksum, | |
| ) | |
| return spec_data, True | |
| # Delta payload - need baseline | |
| baseline = self._baselines.get(spec_id) | |
| if baseline is None: | |
| logger.warning( | |
| f"[{CONSTITUTIONAL_HASH}] No baseline for delta decompression: {spec_id}" | |
| ) | |
| return {}, False | |
| # Verify baseline checksum | |
| baseline_checksum = compressed.get("baseline_checksum") | |
| if baseline_checksum and baseline_checksum != baseline.checksum: | |
| logger.warning(f"[{CONSTITUTIONAL_HASH}] Checksum mismatch for {spec_id}") | |
| return {}, False | |
| # Apply delta to baseline | |
| delta = compressed.get("delta", {}) | |
| reconstructed = deepcopy(baseline.spec_data) | |
| for key, value in delta.items(): | |
| if key.startswith("__removed__"): | |
| actual_key = key[11:] # Remove prefix | |
| reconstructed.pop(actual_key, None) | |
| else: | |
| reconstructed[key] = value | |
| # Update baseline | |
| baseline.spec_data = deepcopy(reconstructed) | |
| baseline.checksum = compressed.get("checksum", self._compute_checksum(reconstructed)) | |
| baseline.version = compressed.get("version", baseline.version + 1) | |
| return reconstructed, True | |
| async def _evict_stale_baselines(self) -> None: | |
| """Evict stale baselines beyond TTL or max size.""" | |
| now = datetime.now() | |
| # Evict by TTL | |
| stale_keys = [ | |
| key | |
| for key, baseline in self._baselines.items() | |
| if now - baseline.created_at > self._baseline_ttl | |
| ] | |
| for key in stale_keys: | |
| del self._baselines[key] | |
| # Evict by size (LRU) | |
| while len(self._baselines) > self._max_baselines: | |
| self._baselines.popitem(last=False) | |
| def get_stats(self) -> JSONDict: | |
| """Get compression statistics.""" | |
| return { | |
| **self._stats, | |
| "baselines_cached": len(self._baselines), | |
| "compression_rate": (self._stats["delta_sends"] / max(1, self._stats["compressions"])), | |
| "constitutional_hash": CONSTITUTIONAL_HASH, | |
| } | |
| async def clear(self) -> None: | |
| """Clear all baselines.""" | |
| async with self._lock: | |
| self._baselines.clear() | |
| # ============================================================================= | |
| # Task 4.2: Cached Governance Validator | |
| # ============================================================================= | |
| class GovernanceDecision: | |
| """Cached governance decision. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| allowed: bool | |
| reason: str | |
| cached_at: datetime | |
| expires_at: datetime | |
| cache_key: str | |
| policy_version: str | None = None | |
| constitutional_hash: str = field(default=CONSTITUTIONAL_HASH) | |
| def is_expired(self) -> bool: | |
| """Check if decision has expired.""" | |
| return datetime.now() > self.expires_at | |
| class ValidationContext: | |
| """Context for governance validation. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| action: str | |
| resource: str | |
| agent_id: str | |
| tenant_id: str | None = None | |
| metadata: JSONDict = field(default_factory=dict) | |
| constitutional_hash: str = field(default=CONSTITUTIONAL_HASH) | |
| class GovernanceValidatorProtocol(ABC): | |
| """Protocol for governance validators.""" | |
| async def validate(self, context: ValidationContext) -> GovernanceDecision: | |
| """Validate a governance action.""" | |
| ... | |
| class CachedGovernanceValidator: | |
| """ | |
| Governance decision caching with constitutional hash validation. | |
| Achieves 90%+ cache hit rate for repeated governance queries | |
| by caching OPA decisions with configurable TTL. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| def __init__( | |
| self, | |
| upstream_validator: GovernanceValidatorProtocol | None = None, | |
| cache_ttl_seconds: int = 60, | |
| max_cache_size: int = 10000, | |
| enable_negative_caching: bool = True, | |
| negative_cache_ttl_seconds: int = 30, | |
| cache_hash_mode: Literal["sha256", "fast"] = "sha256", | |
| ): | |
| self._upstream = upstream_validator | |
| self._cache_ttl = timedelta(seconds=cache_ttl_seconds) | |
| self._negative_cache_ttl = timedelta(seconds=negative_cache_ttl_seconds) | |
| self._max_cache_size = max_cache_size | |
| self._enable_negative_caching = enable_negative_caching | |
| if cache_hash_mode not in _CACHE_HASH_MODES: | |
| raise ValueError(f"Invalid cache_hash_mode: {cache_hash_mode}") | |
| self._cache_hash_mode = cache_hash_mode | |
| self._cache: OrderedDict[str, GovernanceDecision] = OrderedDict() | |
| self._lock = asyncio.Lock() | |
| self._stats = { | |
| "hits": 0, | |
| "misses": 0, | |
| "evictions": 0, | |
| "upstream_calls": 0, | |
| } | |
| self.constitutional_hash = CONSTITUTIONAL_HASH | |
| if self._cache_hash_mode == "fast" and not FAST_HASH_AVAILABLE: | |
| logger.warning( | |
| "cache_hash_mode=fast requested but acgs2_perf.fast_hash unavailable; " | |
| "falling back to sha256" | |
| ) | |
| def _cache_key(self, context: ValidationContext) -> str: | |
| """ | |
| Generate cache key including constitutional hash for integrity. | |
| The constitutional hash ensures cached decisions are invalidated | |
| if governance rules change. | |
| """ | |
| key_data = { | |
| "action": context.action, | |
| "resource": context.resource, | |
| "agent_id": context.agent_id, | |
| "tenant_id": context.tenant_id, | |
| "constitutional_hash": CONSTITUTIONAL_HASH, | |
| } | |
| content = json.dumps(key_data, sort_keys=True) | |
| if self._cache_hash_mode == "fast" and FAST_HASH_AVAILABLE: | |
| return f"fast:{fast_hash(content):016x}" | |
| return hashlib.sha256(content.encode()).hexdigest() | |
| async def validate(self, context: ValidationContext) -> GovernanceDecision: | |
| """ | |
| Validate a governance action with caching. | |
| Cache lookup is O(1), with LRU eviction for memory management. | |
| """ | |
| cache_key = self._cache_key(context) | |
| async with self._lock: | |
| # Check cache | |
| if cache_key in self._cache: | |
| decision = self._cache[cache_key] | |
| if not decision.is_expired: | |
| self._stats["hits"] += 1 | |
| self._cache.move_to_end(cache_key) | |
| return decision | |
| else: | |
| # Expired - remove from cache | |
| del self._cache[cache_key] | |
| self._stats["misses"] += 1 | |
| # Cache miss - call upstream | |
| decision = await self._call_upstream(context, cache_key) | |
| async with self._lock: | |
| # Cache the decision | |
| await self._cache_decision(cache_key, decision) | |
| return decision | |
| async def _call_upstream( | |
| self, context: ValidationContext, cache_key: str | |
| ) -> GovernanceDecision: | |
| """Call upstream validator or return default decision.""" | |
| self._stats["upstream_calls"] += 1 | |
| now = datetime.now() | |
| if self._upstream is not None: | |
| try: | |
| return await self._upstream.validate(context) | |
| except Exception: | |
| logger.error( | |
| "[%s] Upstream validation failed", | |
| CONSTITUTIONAL_HASH, | |
| exc_info=True, | |
| ) | |
| # Fail-closed: deny on error | |
| return GovernanceDecision( | |
| allowed=False, | |
| reason="Upstream validation unavailable — fail-closed", | |
| cached_at=now, | |
| expires_at=now + self._negative_cache_ttl, | |
| cache_key=cache_key, | |
| ) | |
| # No upstream - simulate OPA call (for testing/standalone mode) | |
| return GovernanceDecision( | |
| allowed=True, | |
| reason="No upstream validator configured - allowing by default", | |
| cached_at=now, | |
| expires_at=now + self._cache_ttl, | |
| cache_key=cache_key, | |
| ) | |
| async def _cache_decision(self, cache_key: str, decision: GovernanceDecision) -> None: | |
| """Cache a decision with LRU eviction.""" | |
| # Don't cache denied decisions if negative caching disabled | |
| if not decision.allowed and not self._enable_negative_caching: | |
| return | |
| # Evict if at capacity | |
| while len(self._cache) >= self._max_cache_size: | |
| self._cache.popitem(last=False) | |
| self._stats["evictions"] += 1 | |
| self._cache[cache_key] = decision | |
| self._cache.move_to_end(cache_key) | |
| async def invalidate(self, pattern: str | None = None) -> int: | |
| """ | |
| Invalidate cached decisions. | |
| Args: | |
| pattern: If provided, invalidate keys matching pattern. | |
| If None, invalidate all. | |
| Returns: | |
| Number of invalidated entries. | |
| """ | |
| async with self._lock: | |
| if pattern is None: | |
| count = len(self._cache) | |
| self._cache.clear() | |
| return count | |
| # Pattern-based invalidation | |
| keys_to_remove = [key for key in self._cache if pattern in key] | |
| for key in keys_to_remove: | |
| del self._cache[key] | |
| return len(keys_to_remove) | |
| def get_stats(self) -> JSONDict: | |
| """Get cache statistics.""" | |
| total = self._stats["hits"] + self._stats["misses"] | |
| return { | |
| **self._stats, | |
| "cache_size": len(self._cache), | |
| "hit_rate": self._stats["hits"] / max(1, total), | |
| "constitutional_hash": CONSTITUTIONAL_HASH, | |
| } | |
| # ============================================================================= | |
| # Task 4.3: Optimized Agent Bus (Topic Partitioning) | |
| # ============================================================================= | |
| class TopicPriority(str, Enum): | |
| """Priority levels for bus topics.""" | |
| LOW = "low" | |
| NORMAL = "normal" | |
| HIGH = "high" | |
| CRITICAL = "critical" | |
| class TopicConfig: | |
| """Configuration for a bus topic. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| name: str | |
| partitions: int = 4 | |
| priority: TopicPriority = TopicPriority.NORMAL | |
| max_batch_size: int = 100 | |
| retention_seconds: int = 3600 | |
| constitutional_hash: str = field(default=CONSTITUTIONAL_HASH) | |
| class PartitionedMessage: | |
| """Message with partition routing. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| topic: str | |
| partition: int | |
| payload: JSONDict | |
| partition_key: str | |
| timestamp: datetime = field(default_factory=lambda: datetime.now()) | |
| message_id: str = field( | |
| default_factory=lambda: hashlib.sha256(str(time.time_ns()).encode()).hexdigest()[:12] | |
| ) | |
| constitutional_hash: str = field(default=CONSTITUTIONAL_HASH) | |
| EventHandler = Callable[[PartitionedMessage], Coroutine[Any, Any, None]] | |
| class PartitionBroker: | |
| """ | |
| Broker for a single partition. | |
| Handles message buffering and subscriber notification. | |
| """ | |
| def __init__(self, topic: str, partition: int, max_buffer: int = 1000): | |
| self.topic = topic | |
| self.partition = partition | |
| self._buffer: list[PartitionedMessage] = [] | |
| self._max_buffer = max_buffer | |
| self._subscribers: list[EventHandler] = [] | |
| self._lock = asyncio.Lock() | |
| self._stats = {"published": 0, "delivered": 0, "dropped": 0} | |
| async def publish(self, message: PartitionedMessage) -> bool: | |
| """Publish a message to this partition.""" | |
| async with self._lock: | |
| if len(self._buffer) >= self._max_buffer: | |
| self._stats["dropped"] += 1 | |
| logger.warning( | |
| f"[{CONSTITUTIONAL_HASH}] Partition {self.topic}:{self.partition} " | |
| f"buffer full, dropping message" | |
| ) | |
| return False | |
| self._buffer.append(message) | |
| self._stats["published"] += 1 | |
| # Notify subscribers outside lock | |
| await self._notify_subscribers(message) | |
| return True | |
| async def subscribe(self, handler: EventHandler) -> None: | |
| """Subscribe to this partition.""" | |
| async with self._lock: | |
| self._subscribers.append(handler) | |
| async def unsubscribe(self, handler: EventHandler) -> None: | |
| """Unsubscribe from this partition.""" | |
| async with self._lock: | |
| if handler in self._subscribers: | |
| self._subscribers.remove(handler) | |
| async def _notify_subscribers(self, message: PartitionedMessage) -> None: | |
| """Notify all subscribers of a new message.""" | |
| for handler in self._subscribers: | |
| try: | |
| await handler(message) | |
| self._stats["delivered"] += 1 | |
| except Exception as e: | |
| logger.error( | |
| f"[{CONSTITUTIONAL_HASH}] Handler error in {self.topic}:{self.partition}: {e}" | |
| ) | |
| def get_stats(self) -> JSONDict: | |
| """Get partition statistics.""" | |
| return { | |
| "topic": self.topic, | |
| "partition": self.partition, | |
| "buffer_size": len(self._buffer), | |
| "subscribers": len(self._subscribers), | |
| **self._stats, | |
| } | |
| class OptimizedAgentBus: | |
| """ | |
| Partitioned event bus for efficient multi-agent coordination. | |
| Uses topic partitioning to reduce contention and enable | |
| parallel processing across agents. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| # Default topic configurations | |
| DEFAULT_TOPIC_PARTITIONS: ClassVar[dict[str, int]] = { | |
| "health": 4, # Health events partitioned by tenant | |
| "deployment": 8, # Deployment events partitioned by agent type | |
| "governance": 2, # Governance events (low volume, high priority) | |
| "audit": 4, # Audit events partitioned by component | |
| "metrics": 4, # Metrics events | |
| "alerts": 2, # Alert events (high priority) | |
| } | |
| def __init__( | |
| self, | |
| topic_configs: dict[str, TopicConfig] | None = None, | |
| default_partitions: int = 4, | |
| ): | |
| self._topic_configs: dict[str, TopicConfig] = {} | |
| self._brokers: dict[str, list[PartitionBroker]] = {} | |
| self._default_partitions = default_partitions | |
| self._lock = asyncio.Lock() | |
| self._stats = {"topics_created": 0, "total_published": 0} | |
| self.constitutional_hash = CONSTITUTIONAL_HASH | |
| # Initialize with provided configs | |
| if topic_configs: | |
| for _name, config in topic_configs.items(): | |
| self._create_topic_sync(config) | |
| def _create_topic_sync(self, config: TopicConfig) -> None: | |
| """Create a topic synchronously (for init).""" | |
| self._topic_configs[config.name] = config | |
| self._brokers[config.name] = [ | |
| PartitionBroker(config.name, i) for i in range(config.partitions) | |
| ] | |
| self._stats["topics_created"] += 1 | |
| async def create_topic(self, config: TopicConfig) -> None: | |
| """Create a new topic with specified configuration.""" | |
| async with self._lock: | |
| if config.name in self._brokers: | |
| logger.warning(f"[{CONSTITUTIONAL_HASH}] Topic {config.name} already exists") | |
| return | |
| self._create_topic_sync(config) | |
| logger.info( | |
| f"[{CONSTITUTIONAL_HASH}] Created topic {config.name} " | |
| f"with {config.partitions} partitions" | |
| ) | |
| def _get_partition(self, topic: str, partition_key: str) -> int: | |
| """Determine partition for a message based on key hash.""" | |
| config = self._topic_configs.get(topic) | |
| num_partitions = config.partitions if config else self._default_partitions | |
| return hash(partition_key) % num_partitions | |
| async def publish( | |
| self, | |
| topic: str, | |
| payload: JSONDict, | |
| partition_key: str, | |
| ) -> bool: | |
| """ | |
| Publish a message to a topic. | |
| The partition is determined by hashing the partition_key, | |
| ensuring consistent routing for related messages. | |
| """ | |
| # Auto-create topic if needed | |
| if topic not in self._brokers: | |
| num_partitions = self.DEFAULT_TOPIC_PARTITIONS.get(topic, self._default_partitions) | |
| await self.create_topic(TopicConfig(name=topic, partitions=num_partitions)) | |
| partition = self._get_partition(topic, partition_key) | |
| message = PartitionedMessage( | |
| topic=topic, | |
| partition=partition, | |
| payload=payload, | |
| partition_key=partition_key, | |
| ) | |
| broker = self._brokers[topic][partition] | |
| success = await broker.publish(message) | |
| if success: | |
| self._stats["total_published"] += 1 | |
| return success | |
| async def subscribe( | |
| self, | |
| topic: str, | |
| handler: EventHandler, | |
| partitions: list[int] | None = None, | |
| ) -> None: | |
| """ | |
| Subscribe to a topic. | |
| Args: | |
| topic: Topic name to subscribe to | |
| handler: Async callback for message handling | |
| partitions: Specific partitions to subscribe to. | |
| If None, subscribes to all partitions. | |
| """ | |
| if topic not in self._brokers: | |
| logger.warning(f"[{CONSTITUTIONAL_HASH}] Topic {topic} doesn't exist, creating") | |
| num_partitions = self.DEFAULT_TOPIC_PARTITIONS.get(topic, self._default_partitions) | |
| await self.create_topic(TopicConfig(name=topic, partitions=num_partitions)) | |
| brokers = self._brokers[topic] | |
| target_partitions = partitions or range(len(brokers)) | |
| for p in target_partitions: | |
| if p < len(brokers): | |
| await brokers[p].subscribe(handler) | |
| else: | |
| logger.warning(f"[{CONSTITUTIONAL_HASH}] Invalid partition {p} for topic {topic}") | |
| async def unsubscribe( | |
| self, | |
| topic: str, | |
| handler: EventHandler, | |
| partitions: list[int] | None = None, | |
| ) -> None: | |
| """Unsubscribe from a topic.""" | |
| if topic not in self._brokers: | |
| return | |
| brokers = self._brokers[topic] | |
| target_partitions = partitions or range(len(brokers)) | |
| for p in target_partitions: | |
| if p < len(brokers): | |
| await brokers[p].unsubscribe(handler) | |
| def get_stats(self) -> JSONDict: | |
| """Get bus statistics.""" | |
| topic_stats = {} | |
| for topic, brokers in self._brokers.items(): | |
| topic_stats[topic] = { | |
| "partitions": len(brokers), | |
| "partition_stats": [b.get_stats() for b in brokers], | |
| } | |
| return { | |
| **self._stats, | |
| "topics": topic_stats, | |
| "constitutional_hash": CONSTITUTIONAL_HASH, | |
| } | |
| # ============================================================================= | |
| # Factory Functions | |
| # ============================================================================= | |
| def create_spec_compressor( | |
| max_baselines: int = 1000, | |
| baseline_ttl_seconds: int = 3600, | |
| ) -> SpecDeltaCompressor: | |
| """ | |
| Create a configured SpecDeltaCompressor. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| return SpecDeltaCompressor( | |
| max_baselines=max_baselines, | |
| baseline_ttl_seconds=baseline_ttl_seconds, | |
| ) | |
| def create_cached_validator( | |
| upstream_validator: GovernanceValidatorProtocol | None = None, | |
| cache_ttl_seconds: int = 60, | |
| max_cache_size: int = 10000, | |
| cache_hash_mode: Literal["sha256", "fast"] = "sha256", | |
| ) -> CachedGovernanceValidator: | |
| """ | |
| Create a configured CachedGovernanceValidator. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| return CachedGovernanceValidator( | |
| upstream_validator=upstream_validator, | |
| cache_ttl_seconds=cache_ttl_seconds, | |
| max_cache_size=max_cache_size, | |
| cache_hash_mode=cache_hash_mode, | |
| ) | |
| def create_optimized_bus( | |
| topic_configs: dict[str, TopicConfig] | None = None, | |
| ) -> OptimizedAgentBus: | |
| """ | |
| Create a configured OptimizedAgentBus. | |
| Constitutional Hash: 608508a9bd224290 | |
| """ | |
| return OptimizedAgentBus(topic_configs=topic_configs) | |
| # ============================================================================= | |
| # Module Exports | |
| # ============================================================================= | |
| __all__ = [ | |
| # Feature flag | |
| "CONTEXT_OPTIMIZATION_AVAILABLE", | |
| "CachedGovernanceValidator", | |
| "CompressionResult", | |
| # Task 4.1: Spec Delta Compression | |
| "CompressionStrategy", | |
| # Task 4.2: Cached Governance Validator | |
| "GovernanceDecision", | |
| "GovernanceValidatorProtocol", | |
| "OptimizedAgentBus", | |
| "PartitionBroker", | |
| "PartitionedMessage", | |
| "SpecBaseline", | |
| "SpecDeltaCompressor", | |
| "TopicConfig", | |
| # Task 4.3: Optimized Agent Bus | |
| "TopicPriority", | |
| "ValidationContext", | |
| "create_cached_validator", | |
| "create_optimized_bus", | |
| "create_spec_compressor", | |
| ] | |
Xet Storage Details
- Size:
- 30.7 kB
- Xet hash:
- 25028d4e4d6132cf45efa4f550047c7290bd32e0639560973e39705ad27026d2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.