""" ALLI Unified Runtime Event & Feedback Contracts -- Phase 8 Implementation & Hardening Provides versioned runtime event definitions, deterministic sequence ordering, strict credential/token redaction, and a thread-safe bounded in-memory event store with recovery behavior for ring buffer retention gaps. """ import re import threading import time from typing import Dict, Any, List, Optional, Tuple, Set, Union from pydantic import BaseModel, Field EVENT_CONTRACT_VERSION = "1.0.0" MAX_EVENTS_PER_RUN = 1000 # Bounded ring-buffer capacity per run to prevent memory leaks # Canonical Event Types EVENT_RUN_STARTED = "run_started" EVENT_RUN_COMPLETED = "run_completed" EVENT_RUN_FAILED = "run_failed" EVENT_RUN_CANCELLED = "run_cancelled" EVENT_NODE_QUEUED = "node_queued" EVENT_NODE_STARTED = "node_started" EVENT_NODE_COMPLETED = "node_completed" EVENT_NODE_FAILED = "node_failed" EVENT_NODE_SKIPPED = "node_skipped" EVENT_NODE_CANCELLED = "node_cancelled" EVENT_STDOUT_CHUNK = "stdout_chunk" EVENT_STDERR_CHUNK = "stderr_chunk" EVENT_NODE_OUTPUT = "node_output" EVENT_DIAGNOSTIC = "diagnostic" EVENT_TIMEOUT = "timeout" EVENT_QUOTA_EXCEEDED = "quota_exceeded" EVENT_VALIDATION_ERROR = "validation_error" # ─── 1. SENSITIVE CREDENTIAL & TOKEN SCRUBBING ──────────────────────────────── REDACTION_PATTERNS = [ # AWS Access Key ID (re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "[REDACTED_AWS_KEY]"), # AWS Secret Key / Session Token (re.compile(r"(?i)(aws_secret_access_key|aws_session_token)\s*[:=]\s*['\"]?[a-zA-Z0-9/+=]{30,}['\"]?"), r"\1=[REDACTED]"), # Supabase Keys / JWTs (re.compile(r"eyJ[a-zA-Z0-9_-]{10,}\.eyJ[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,}"), "[REDACTED_JWT]"), (re.compile(r"\bsbp_[a-zA-Z0-9]{20,}\b"), "[REDACTED_SUPABASE_TOKEN]"), # Database / Protocol Connection Strings with Passwords (any scheme: mongodb, postgres, mysql, redis, amqp, http, etc.) (re.compile(r"([a-zA-Z0-9+.-]+://[^:\s/]+:)([^@\s]+)(@[^\s]+)"), r"\1[REDACTED_PASSWORD]\3"), # Bearer tokens (opaque and standard) (re.compile(r"(?i)\bbearer\s+[a-zA-Z0-9_\-\.]{8,}\b"), "Bearer [REDACTED_BEARER_TOKEN]"), # HTTP Basic Auth header (re.compile(r"(?i)\bbasic\s+[a-zA-Z0-9+/=]{12,}={0,2}"), "Basic [REDACTED_BASIC_AUTH]"), # Generic Secret / Key assignments in multiline code, configs, or tracebacks (re.compile(r"(?i)(api[_-]?key|secret[_-]?key|access[_-]?token|private[_-]?key|client[_-]?secret|auth[_-]?token|webhook[_-]?secret|password|passwd)\s*[:=]\s*['\"][^'\"]{6,}['\"]"), r"\1='[REDACTED]'"), (re.compile(r"(?i)(api[_-]?key|secret[_-]?key|access[_-]?token|private[_-]?key|client[_-]?secret|auth[_-]?token|webhook[_-]?secret|password|passwd)\s*[:=]\s*([^\s,;'\"}\]\n]{6,})"), r"\1=[REDACTED]"), # Phase 15 Hardening: Explicit prefix and token pattern matching (excluding already-redacted tags) (re.compile(r"(?i)\btoken\s*[:=]\s*['\"]?(?!\[REDACTED)[^\s,;'\"}\]\n]{3,}['\"]?"), "token=[REDACTED]"), (re.compile(r"(?i)\b(supabase|mongodb|aws)_[a-zA-Z0-9_]*\s*[:=]\s*['\"]?(?!\[REDACTED)[^\s,;'\"}\]\n]{3,}['\"]?"), r"\1=[REDACTED]"), ] def redact_sensitive_text(text: str) -> str: """Scrubs credentials, JWT tokens, connection strings, and API keys from text.""" if not isinstance(text, str) or not text: return text from runtime_environment.env_injection import redact_secrets_from_text clean = redact_secrets_from_text(text) for pattern, replacement in REDACTION_PATTERNS: clean = pattern.sub(replacement, clean) return clean SENSITIVE_KEY_PATTERN = re.compile(r"(?i)(api[_-]?key|secret|token|password|passwd|auth|credential)") def redact_sensitive_data(val: Any, _depth: int = 0) -> Any: """Recursively redacts sensitive tokens and secrets from arbitrary nested data structures.""" if _depth > 50: return val if isinstance(val, str): return redact_sensitive_text(val) if isinstance(val, dict): res = {} for k, v in val.items(): k_clean = redact_sensitive_text(k) if isinstance(k, str) else k if isinstance(k, str) and isinstance(v, str): v_clean = redact_sensitive_text(v) if v_clean != v: res[k_clean] = v_clean elif SENSITIVE_KEY_PATTERN.search(k): res[k_clean] = "[REDACTED]" else: res[k_clean] = v_clean else: res[k_clean] = redact_sensitive_data(v, _depth + 1) return res if isinstance(val, list): return [redact_sensitive_data(x, _depth + 1) for x in val] if isinstance(val, tuple): return tuple(redact_sensitive_data(x, _depth + 1) for x in val) if isinstance(val, set): return {redact_sensitive_data(x, _depth + 1) for x in val} if hasattr(val, "__dict__"): try: return redact_sensitive_data(vars(val), _depth + 1) except Exception: return redact_sensitive_text(str(val)) return val scrub_sensitive_string = redact_sensitive_text sanitize_event_payload = redact_sensitive_data # ─── 2. RUNTIME EVENT MODELS ────────────────────────────────────────────────── class RuntimeEvent(BaseModel): """Authoritative, versioned runtime telemetry event model.""" version: str = Field(default=EVENT_CONTRACT_VERSION, description="Contract schema version") run_id: str = Field(..., description="Unique workflow execution identifier") graph_revision: str = Field(..., description="64-character SHA-256 graph revision fingerprint") seq: int = Field(..., description="1-based monotonically increasing sequence number") event_type: str = Field(..., description="Canonical event category identifier") timestamp: str = Field(..., description="ISO 8601 UTC timestamp") node_id: Optional[str] = Field(default=None, description="Graph node identifier") attempt: Optional[int] = Field(default=None, description="Execution attempt number") status: Optional[str] = Field(default=None, description="Current lifecycle or node status") severity: Optional[str] = Field(default="info", description="Severity level: 'info', 'warning', 'error'") code: Optional[str] = Field(default=None, description="Stable canonical diagnostic error code") message: str = Field(default="", description="Human-readable safe, credential-redacted message") payload: Dict[str, Any] = Field(default_factory=dict, description="Structured event payload data") class RuntimeFeedbackSnapshot(BaseModel): """Comprehensive snapshot of an execution run's current state and accumulated telemetry.""" run_id: str = Field(..., description="Unique run identifier") graph_revision: str = Field(..., description="SHA-256 graph revision fingerprint") lifecycle_status: str = Field(..., description="Workflow lifecycle status: idle, running, success, failed, etc.") events: List[RuntimeEvent] = Field(default_factory=list, description="Ordered runtime events") node_states: Dict[str, Dict[str, Any]] = Field(default_factory=dict, description="Per-node execution states") stdout: str = Field(default="", description="Scrubbed standard output") stderr: str = Field(default="", description="Scrubbed standard error") diagnostics: List[Dict[str, Any]] = Field(default_factory=list, description="Structured diagnostics") duration_ms: float = Field(default=0.0, description="Total execution duration in milliseconds") cursor: int = Field(default=0, description="Latest event sequence number") has_more: bool = Field(default=False, description="Whether more events exist beyond cursor") is_complete: bool = Field(default=False, description="Whether run execution is fully terminal") owner_user_id: Optional[str] = Field(default=None, description="Owner user ID for multi-tenant isolation") class EventsResult(tuple): """ 3-element tuple (events, next_cursor, has_more) preserving backward compatibility, augmented with recovery properties for ring buffer retention gaps. """ def __new__(cls, events: List[RuntimeEvent], next_cursor: int, has_more: bool, events_truncated: bool = False, oldest_retained_seq: int = 0): obj = super().__new__(cls, (events, next_cursor, has_more)) obj.events = events obj.next_cursor = next_cursor obj.has_more = has_more obj.events_truncated = events_truncated obj.oldest_retained_seq = oldest_retained_seq return obj # ─── 3. THREAD-SAFE BOUNDED EVENT STORE ─────────────────────────────────────── class RuntimeEventStore: """ Thread-safe, bounded memory event registry with monotonic sequencing, cursor pagination, retention gap recovery, and automatic TTL expiration. """ def __init__(self, max_events_per_run: int = MAX_EVENTS_PER_RUN): self._max_events = max_events_per_run self._events: Dict[str, List[RuntimeEvent]] = {} self._seq_counters: Dict[str, int] = {} self._meta: Dict[str, Dict[str, Any]] = {} self._lock = threading.RLock() def record_event( self, run_id: str, graph_revision: str, event_type: str, message: str = "", node_id: Optional[str] = None, status: Optional[str] = None, severity: str = "info", code: Optional[str] = None, payload: Optional[Dict[str, Any]] = None, attempt: Optional[int] = None, owner_user_id: Optional[str] = None, ) -> RuntimeEvent: """ Appends an event to the run's ordered event log with monotonic sequence numbers. Scrubs sensitive credentials and enforces ring-buffer retention limit. """ with self._lock: if run_id not in self._seq_counters: self._seq_counters[run_id] = 0 self._events[run_id] = [] self._meta[run_id] = { "graph_revision": graph_revision, "created_at": time.time(), "last_accessed": time.time(), "lifecycle_status": "running" if event_type == EVENT_RUN_STARTED else "idle", "owner_user_id": owner_user_id, "is_complete": False, } self._seq_counters[run_id] += 1 seq = self._seq_counters[run_id] now_iso = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) safe_message = redact_sensitive_text(message) safe_payload = redact_sensitive_data(payload or {}) event = RuntimeEvent( run_id=run_id, graph_revision=graph_revision, seq=seq, event_type=event_type, timestamp=now_iso, node_id=node_id, attempt=attempt, status=status, severity=severity, code=code, message=safe_message, payload=safe_payload, ) event_list = self._events[run_id] event_list.append(event) if len(event_list) > self._max_events: # Ring-buffer trimming to maintain bounded memory self._events[run_id] = event_list[-self._max_events:] # Update lifecycle status if terminal event meta = self._meta[run_id] meta["last_accessed"] = time.time() if owner_user_id: meta["owner_user_id"] = owner_user_id if event_type == EVENT_RUN_COMPLETED: meta["lifecycle_status"] = "success" meta["is_complete"] = True elif event_type == EVENT_RUN_FAILED: meta["lifecycle_status"] = "failed" meta["is_complete"] = True elif event_type == EVENT_RUN_CANCELLED: meta["lifecycle_status"] = "cancelled" meta["is_complete"] = True elif event_type == EVENT_RUN_STARTED: meta["lifecycle_status"] = "running" meta["is_complete"] = False return event def get_events( self, run_id: str, since_seq: int = 0, limit: int = 100, ) -> EventsResult: """ Retrieves ordered events strictly greater than since_seq up to limit. Defines recovery behavior when since_seq is behind the ring buffer's oldest retained event. Returns: EventsResult(events, next_cursor, has_more, events_truncated, oldest_retained_seq) """ with self._lock: if run_id not in self._events: return EventsResult([], since_seq, False, False, 0) self._meta[run_id]["last_accessed"] = time.time() all_events = self._events[run_id] if not all_events: return EventsResult([], since_seq, False, False, 0) oldest_retained_seq = all_events[0].seq events_truncated = False # Check if requested cursor is behind the ring-buffer retention window if since_seq < oldest_retained_seq - 1: events_truncated = True eligible = [e for e in all_events if e.seq > since_seq] slice_events = eligible[:limit] has_more = len(eligible) > limit next_cursor = slice_events[-1].seq if slice_events else since_seq return EventsResult(slice_events, next_cursor, has_more, events_truncated, oldest_retained_seq) def get_snapshot(self, run_id: str) -> Optional[Dict[str, Any]]: """Constructs an authoritative RuntimeFeedbackSnapshot dictionary.""" with self._lock: if run_id not in self._events and run_id not in self._meta: return None self._meta[run_id]["last_accessed"] = time.time() meta = self._meta[run_id] events = self._events.get(run_id, []) latest_cursor = self._seq_counters.get(run_id, 0) return { "run_id": run_id, "graph_revision": meta.get("graph_revision", ""), "lifecycle_status": meta.get("lifecycle_status", "idle"), "events": [e.model_dump() if hasattr(e, "model_dump") else e.dict() for e in events], "cursor": latest_cursor, "has_more": False, "is_complete": meta.get("is_complete", False), "owner_user_id": meta.get("owner_user_id"), } def set_run_metadata(self, run_id: str, **kwargs): """Updates internal run metadata (e.g. owner_user_id, lifecycle_status).""" with self._lock: if run_id not in self._meta: self._meta[run_id] = { "graph_revision": "", "created_at": time.time(), "last_accessed": time.time(), "lifecycle_status": "idle", "owner_user_id": None, "is_complete": False, } self._meta[run_id].update(kwargs) def get_run_metadata(self, run_id: str) -> Optional[Dict[str, Any]]: """Returns metadata for a given run ID.""" with self._lock: return dict(self._meta[run_id]) if run_id in self._meta else None def clear_expired_runs(self, ttl_seconds: float = 3600.0) -> int: """Purges execution runs that have been idle past the configured TTL.""" with self._lock: now = time.time() expired = [ rid for rid, m in self._meta.items() if (now - m.get("last_accessed", now)) > ttl_seconds ] for rid in expired: self._events.pop(rid, None) self._seq_counters.pop(rid, None) self._meta.pop(rid, None) return len(expired) # Global Singleton Event Registry GLOBAL_EVENT_STORE = RuntimeEventStore()