Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /dislovelhl_enhanced-agent-bus /siem_integration.py
| """ | |
| ACGS-2 Enhanced Agent Bus - SIEM Integration | |
| Constitutional Hash: 608508a9bd224290 | |
| Lightweight SIEM formatting, alerting, and buffering support. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import hashlib | |
| import inspect | |
| import json | |
| from collections import defaultdict, deque | |
| from collections.abc import Callable | |
| from dataclasses import dataclass | |
| from datetime import UTC, datetime | |
| from enum import Enum, IntEnum | |
| from functools import wraps | |
| from typing import TYPE_CHECKING, Any, TypeAlias | |
| if TYPE_CHECKING: | |
| from enhanced_agent_bus._compat.types import JSONDict | |
| else: | |
| JSONDict: TypeAlias = dict[str, Any] | |
| from .runtime_security import SecurityEvent, SecurityEventType, SecuritySeverity | |
| try: | |
| from .rust.fast_hash import fast_hash | |
| FAST_HASH_AVAILABLE = True | |
| except ImportError: # pragma: no cover - optional acceleration | |
| fast_hash = None | |
| FAST_HASH_AVAILABLE = False | |
| class SIEMFormat(str, Enum): | |
| JSON = "json" | |
| CEF = "cef" | |
| LEEF = "leef" | |
| SYSLOG = "syslog" | |
| class AlertLevel(IntEnum): | |
| NOTIFY = 1 | |
| PAGE = 2 | |
| ESCALATE = 3 | |
| CRITICAL = 4 | |
| class AlertThreshold: | |
| event_type: SecurityEventType | |
| count_threshold: int | |
| time_window_seconds: int | |
| alert_level: AlertLevel | |
| cooldown_seconds: int = 300 | |
| escalation_multiplier: int = 2 | |
| DEFAULT_ALERT_THRESHOLDS = [ | |
| AlertThreshold( | |
| event_type=SecurityEventType.CONSTITUTIONAL_HASH_MISMATCH, | |
| count_threshold=1, | |
| time_window_seconds=60, | |
| alert_level=AlertLevel.CRITICAL, | |
| cooldown_seconds=0, | |
| ), | |
| AlertThreshold( | |
| event_type=SecurityEventType.PROMPT_INJECTION_ATTEMPT, | |
| count_threshold=3, | |
| time_window_seconds=60, | |
| alert_level=AlertLevel.PAGE, | |
| ), | |
| AlertThreshold( | |
| event_type=SecurityEventType.AUTHENTICATION_FAILURE, | |
| count_threshold=5, | |
| time_window_seconds=60, | |
| alert_level=AlertLevel.NOTIFY, | |
| ), | |
| ] | |
| class SIEMConfig: | |
| format: SIEMFormat = SIEMFormat.JSON | |
| endpoint_url: str | None = None | |
| enable_alerting: bool = True | |
| include_constitutional_hash: bool = True | |
| max_queue_size: int = 1000 | |
| drop_on_overflow: bool = True | |
| flush_interval_seconds: float = 1.0 | |
| correlation_window_seconds: int = 60 | |
| alert_callback: Callable[[AlertLevel, str, JSONDict], Any] | None = None | |
| class SIEMEventFormatter: | |
| def __init__(self, format_type: SIEMFormat = SIEMFormat.JSON) -> None: | |
| self.format_type = format_type | |
| def format(self, event: SecurityEvent, correlation_id: str | None = None) -> str: | |
| if self.format_type == SIEMFormat.JSON: | |
| return self._format_json(event, correlation_id) | |
| if self.format_type == SIEMFormat.CEF: | |
| return self._format_cef(event, correlation_id) | |
| if self.format_type == SIEMFormat.LEEF: | |
| return self._format_leef(event, correlation_id) | |
| return self._format_syslog(event, correlation_id) | |
| def _format_json(self, event: SecurityEvent, correlation_id: str | None) -> str: | |
| payload = event.to_dict() | |
| if correlation_id is not None: | |
| payload["correlation_id"] = correlation_id | |
| payload["_siem"] = {"vendor": "ACGS-2", "product": "EnhancedAgentBus", "version": "2.4.0"} | |
| return json.dumps(payload, default=str) | |
| def _format_cef(self, event: SecurityEvent, correlation_id: str | None) -> str: | |
| ext = { | |
| "msg": self._escape(event.message), | |
| "TenantID": event.tenant_id or "", | |
| "AgentID": event.agent_id or "", | |
| "ConstitutionalHash": event.constitutional_hash, | |
| } | |
| if correlation_id is not None: | |
| ext["CorrelationID"] = correlation_id | |
| extension = " ".join(f"{key}={value}" for key, value in ext.items()) | |
| return ( | |
| f"CEF:0|ACGS-2|EnhancedAgentBus|2.4.0|{event.event_type.value}|" | |
| f"{event.event_type.value}|{self._cef_severity(event.severity)}|{extension}" | |
| ) | |
| def _format_leef(self, event: SecurityEvent, correlation_id: str | None) -> str: | |
| values = { | |
| "eventType": event.event_type.value, | |
| "severity": event.severity.value, | |
| "message": self._escape(event.message), | |
| "tenantId": event.tenant_id or "", | |
| "agentId": event.agent_id or "", | |
| "constitutionalHash": event.constitutional_hash, | |
| } | |
| if correlation_id is not None: | |
| values["correlationId"] = correlation_id | |
| body = "\t".join(f"{key}={value}" for key, value in values.items()) | |
| return f"LEEF:2.0|ACGS-2|EnhancedAgentBus|2.4.0|{body}" | |
| def _format_syslog(self, event: SecurityEvent, correlation_id: str | None) -> str: | |
| structured = [ | |
| "[acgs2@12345", | |
| f'event_type="{event.event_type.value}"', | |
| f'severity="{event.severity.value}"', | |
| f'constitutional_hash="{event.constitutional_hash}"', | |
| ] | |
| if correlation_id is not None: | |
| structured.append(f'correlation_id="{correlation_id}"') | |
| structured.append("]") | |
| ts = event.timestamp.astimezone(UTC).isoformat() | |
| return f"<134>1 {ts} EnhancedAgentBus - - - {' '.join(structured)} {event.message}" | |
| def _escape(value: str) -> str: | |
| return value.replace("\\", "\\\\").replace("|", "\\|").replace("=", "\\=") | |
| def _cef_severity(severity: SecuritySeverity) -> int: | |
| mapping = { | |
| "info": 1, | |
| "low": 3, | |
| "medium": 5, | |
| "high": 8, | |
| "critical": 10, | |
| } | |
| return mapping.get(severity.value, 5) | |
| class AlertManager: | |
| def __init__( | |
| self, | |
| thresholds: list[AlertThreshold] | None = None, | |
| callback: Callable[[AlertLevel, str, JSONDict], Any] | None = None, | |
| ) -> None: | |
| self.thresholds = thresholds or list(DEFAULT_ALERT_THRESHOLDS) | |
| self.callback = callback | |
| self._events: dict[str, deque[datetime]] = defaultdict(deque) | |
| self._last_alert_at: dict[str, datetime] = {} | |
| self._states: dict[str, JSONDict] = {} | |
| async def process_event(self, event: SecurityEvent) -> AlertLevel | None: | |
| threshold = self._find_threshold(event.event_type) | |
| if threshold is None and event.severity.value == "critical": | |
| threshold = AlertThreshold( | |
| event.event_type, 1, 60, AlertLevel.CRITICAL, cooldown_seconds=0 | |
| ) | |
| if threshold is None: | |
| return None | |
| key = event.event_type.value | |
| now = event.timestamp.astimezone(UTC) | |
| events = self._events[key] | |
| events.append(now) | |
| cutoff = now.timestamp() - threshold.time_window_seconds | |
| while events and events[0].timestamp() < cutoff: | |
| events.popleft() | |
| count = len(events) | |
| last_alert = self._last_alert_at.get(key) | |
| if last_alert is not None and threshold.cooldown_seconds > 0: | |
| if (now - last_alert).total_seconds() < threshold.cooldown_seconds: | |
| self._record_state(event.event_type, count, None) | |
| return None | |
| if count < threshold.count_threshold: | |
| self._record_state(event.event_type, count, None) | |
| return None | |
| level = self._escalate_level(threshold, count) | |
| self._last_alert_at[key] = now | |
| context: JSONDict = { | |
| "event_type": event.event_type.value, | |
| "severity": event.severity.value, | |
| "constitutional_hash": event.constitutional_hash, | |
| "tenant_id": event.tenant_id, | |
| "agent_id": event.agent_id, | |
| "metadata": event.metadata, | |
| "event_count": count, | |
| } | |
| self._record_state(event.event_type, count, level) | |
| alert_message = f"[{event.event_type.value}] {event.message}" | |
| await self._invoke_callback(level, alert_message, context) | |
| return level | |
| def get_alert_states(self) -> dict[str, JSONDict]: | |
| return {key: dict(value) for key, value in self._states.items()} | |
| def reset_alert_state(self, event_type: SecurityEventType) -> None: | |
| self._events[event_type.value].clear() | |
| self._last_alert_at.pop(event_type.value, None) | |
| self._states[event_type.value] = {"event_count": 0, "current_level": None} | |
| def _find_threshold(self, event_type: SecurityEventType) -> AlertThreshold | None: | |
| for threshold in self.thresholds: | |
| if threshold.event_type.value == event_type.value: | |
| return threshold | |
| return None | |
| def _escalate_level(self, threshold: AlertThreshold, count: int) -> AlertLevel: | |
| level = threshold.alert_level | |
| if threshold.escalation_multiplier <= 1: | |
| return level | |
| steps = max(0, count // threshold.count_threshold - 1) | |
| return AlertLevel(min(AlertLevel.CRITICAL.value, level.value + steps)) | |
| async def _invoke_callback(self, level: AlertLevel, message: str, context: JSONDict) -> None: | |
| if self.callback is None: | |
| return | |
| result = self.callback(level, message, context) | |
| if inspect.isawaitable(result): | |
| await result | |
| def _record_state( | |
| self, event_type: SecurityEventType, count: int, level: AlertLevel | None | |
| ) -> None: | |
| self._states[event_type.value] = { | |
| "event_count": count, | |
| "current_level": level.name if level is not None else None, | |
| } | |
| class EventCorrelator: | |
| def __init__(self, window_seconds: int = 60) -> None: | |
| self.window_seconds = window_seconds | |
| self._events: list[SecurityEvent] = [] | |
| self._correlations: dict[str, list[SecurityEvent]] = {} | |
| async def add_event(self, event: SecurityEvent) -> str | None: | |
| self._events.append(event) | |
| now = event.timestamp.astimezone(UTC) | |
| self._events = [ | |
| item | |
| for item in self._events | |
| if (now - item.timestamp.astimezone(UTC)).total_seconds() <= self.window_seconds | |
| ] | |
| same_tenant = [ | |
| item | |
| for item in self._events | |
| if item.tenant_id | |
| and item.tenant_id == event.tenant_id | |
| and item.severity.value in {"high", "critical"} | |
| ] | |
| if event.tenant_id and len(same_tenant) >= 3: | |
| correlation_id = self._generate_correlation_id("tenant_attack", event.tenant_id) | |
| self._correlations[correlation_id] = same_tenant | |
| return correlation_id | |
| matching_type = [ | |
| item for item in self._events if item.event_type.value == event.event_type.value | |
| ] | |
| unique_agents = {item.agent_id for item in matching_type if item.agent_id} | |
| if len(unique_agents) >= 3: | |
| correlation_id = self._generate_correlation_id( | |
| "distributed_attack", event.event_type.value | |
| ) | |
| self._correlations[correlation_id] = matching_type | |
| return correlation_id | |
| return None | |
| def get_correlated_events(self, correlation_id: str) -> list[SecurityEvent]: | |
| return list(self._correlations.get(correlation_id, [])) | |
| def _generate_correlation_id(self, pattern: str, discriminator: str) -> str: | |
| value = f"{pattern}:{discriminator}" | |
| if FAST_HASH_AVAILABLE and fast_hash is not None: | |
| return f"{fast_hash(value):016x}" | |
| return hashlib.sha256(value.encode("utf-8")).hexdigest()[:16] | |
| class SIEMIntegration: | |
| def __init__(self, config: SIEMConfig | None = None) -> None: | |
| self.config = config or SIEMConfig() | |
| self.formatter = SIEMEventFormatter(self.config.format) | |
| self.alert_manager = AlertManager(callback=self.config.alert_callback) | |
| self.correlator = EventCorrelator(window_seconds=self.config.correlation_window_seconds) | |
| self._queue: asyncio.Queue[tuple[SecurityEvent, str | None]] = asyncio.Queue( | |
| maxsize=self.config.max_queue_size | |
| ) | |
| self._running = False | |
| self._worker: asyncio.Task[None] | None = None | |
| self._metrics: dict[str, int | bool] = { | |
| "events_logged": 0, | |
| "events_dropped": 0, | |
| "events_shipped": 0, | |
| "alerts_triggered": 0, | |
| "correlations_detected": 0, | |
| "running": False, | |
| } | |
| async def start(self) -> None: | |
| if self._running: | |
| return | |
| self._running = True | |
| self._metrics["running"] = True | |
| self._worker = asyncio.create_task(self._flush_loop()) | |
| async def stop(self) -> None: | |
| if not self._running: | |
| return | |
| self._running = False | |
| self._metrics["running"] = False | |
| if self._worker is not None: | |
| await self._queue.join() | |
| self._worker.cancel() | |
| try: | |
| await self._worker | |
| except asyncio.CancelledError: | |
| pass | |
| self._worker = None | |
| async def log_event(self, event: SecurityEvent) -> None: | |
| correlation_id = await self.correlator.add_event(event) | |
| if correlation_id is not None: | |
| self._metrics["correlations_detected"] += 1 | |
| if self.config.enable_alerting: | |
| level = await self.alert_manager.process_event(event) | |
| if level is not None: | |
| self._metrics["alerts_triggered"] += 1 | |
| try: | |
| self._queue.put_nowait((event, correlation_id)) | |
| self._metrics["events_logged"] += 1 | |
| except asyncio.QueueFull: | |
| if self.config.drop_on_overflow: | |
| self._metrics["events_dropped"] += 1 | |
| return | |
| await self._queue.put((event, correlation_id)) | |
| self._metrics["events_logged"] += 1 | |
| def get_metrics(self) -> dict[str, int | bool]: | |
| return { | |
| **self._metrics, | |
| "queue_size": self._queue.qsize(), | |
| } | |
| def get_alert_states(self) -> dict[str, JSONDict]: | |
| return self.alert_manager.get_alert_states() | |
| async def _flush_loop(self) -> None: | |
| try: | |
| while True: | |
| try: | |
| event, correlation_id = await asyncio.wait_for( | |
| self._queue.get(), timeout=self.config.flush_interval_seconds | |
| ) | |
| except asyncio.TimeoutError: | |
| if not self._running and self._queue.empty(): | |
| return | |
| continue | |
| _payload = self.formatter.format(event, correlation_id) | |
| self._metrics["events_shipped"] += 1 | |
| self._queue.task_done() | |
| except asyncio.CancelledError: | |
| raise | |
| _GLOBAL_SIEM: SIEMIntegration | None = None | |
| async def initialize_siem(config: SIEMConfig | None = None) -> SIEMIntegration: | |
| global _GLOBAL_SIEM | |
| if _GLOBAL_SIEM is None: | |
| _GLOBAL_SIEM = SIEMIntegration(config or SIEMConfig()) | |
| await _GLOBAL_SIEM.start() | |
| return _GLOBAL_SIEM | |
| def get_siem_integration() -> SIEMIntegration | None: | |
| return _GLOBAL_SIEM | |
| async def close_siem() -> None: | |
| global _GLOBAL_SIEM | |
| if _GLOBAL_SIEM is not None: | |
| await _GLOBAL_SIEM.stop() | |
| _GLOBAL_SIEM = None | |
| async def log_security_event( | |
| event_type: SecurityEventType, | |
| severity: SecuritySeverity, | |
| message: str, | |
| tenant_id: str | None = None, | |
| agent_id: str | None = None, | |
| metadata: JSONDict | None = None, | |
| ) -> None: | |
| siem = get_siem_integration() | |
| if siem is None: | |
| return | |
| await siem.log_event( | |
| SecurityEvent( | |
| event_type=event_type, | |
| severity=severity, | |
| message=message, | |
| tenant_id=tenant_id, | |
| agent_id=agent_id, | |
| metadata=metadata or {}, | |
| ) | |
| ) | |
| def security_audit( | |
| event_type: SecurityEventType, | |
| severity: SecuritySeverity, | |
| ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: | |
| def decorator(func: Callable[..., Any]) -> Callable[..., Any]: | |
| async def wrapper(*args: Any, **kwargs: Any) -> Any: | |
| try: | |
| result = await func(*args, **kwargs) | |
| await log_security_event( | |
| event_type=event_type, | |
| severity=severity, | |
| message=f"{func.__name__} succeeded", | |
| ) | |
| return result | |
| except Exception as exc: | |
| await log_security_event( | |
| event_type=event_type, | |
| severity=SecuritySeverity.HIGH, | |
| message=f"{func.__name__} failed: {exc}", | |
| ) | |
| raise RuntimeError(f"{func.__name__} failed: {exc}") from exc | |
| return wrapper | |
| return decorator | |
| __all__ = [ | |
| "DEFAULT_ALERT_THRESHOLDS", | |
| "FAST_HASH_AVAILABLE", | |
| "AlertLevel", | |
| "AlertManager", | |
| "AlertThreshold", | |
| "EventCorrelator", | |
| "SIEMConfig", | |
| "SIEMEventFormatter", | |
| "SIEMFormat", | |
| "SIEMIntegration", | |
| "close_siem", | |
| "fast_hash", | |
| "get_siem_integration", | |
| "initialize_siem", | |
| "log_security_event", | |
| "security_audit", | |
| ] | |
Xet Storage Details
- Size:
- 17.2 kB
- Xet hash:
- 1c412ecd9a118c2f221908a3932415e415124dc40359f1c5b422acc8d86a655e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.