Spaces:
Sleeping
Sleeping
File size: 11,957 Bytes
dd63eee | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 | """
Local event queue for graceful degradation when Kafka is unavailable.
Provides in-memory event buffering with periodic retry to Kafka.
Events are held for up to 5 minutes before being discarded.
"""
import asyncio
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any
from dataclasses import dataclass, field
from collections import deque
import json
import logging
logger = logging.getLogger(__name__)
@dataclass
class QueuedEvent:
"""Represents an event in the local queue."""
event_data: Dict[str, Any]
topic: str
timestamp: datetime
retry_count: int = 0
max_retries: int = 10
def is_expired(self, max_age_seconds: int = 300) -> bool:
"""Check if event has exceeded maximum age (default 5 minutes)."""
age = (datetime.utcnow() - self.timestamp).total_seconds()
return age > max_age_seconds
def should_retry(self) -> bool:
"""Check if event should be retried."""
return self.retry_count < self.max_retries and not self.is_expired()
class LocalEventQueue:
"""
Local in-memory event queue for graceful degradation.
When Kafka is unavailable, events are queued locally and retried
periodically. Events older than 5 minutes are discarded.
Usage:
queue = LocalEventQueue(max_size=1000, retry_interval=30)
await queue.start()
# Queue event when Kafka is down
await queue.enqueue(event_data, topic)
# Stop background retry task
await queue.stop()
"""
def __init__(
self,
max_size: int = 1000,
retry_interval: int = 30,
max_age_seconds: int = 300
):
self.max_size = max_size
self.retry_interval = retry_interval
self.max_age_seconds = max_age_seconds
self.queue: deque[QueuedEvent] = deque(maxlen=max_size)
self.retry_task: Optional[asyncio.Task] = None
self.is_running = False
self._lock = asyncio.Lock()
# Statistics
self.stats = {
"total_queued": 0,
"total_retried": 0,
"total_succeeded": 0,
"total_expired": 0,
"total_dropped": 0,
"current_size": 0
}
async def start(self):
"""Start background retry task."""
if self.is_running:
return
self.is_running = True
self.retry_task = asyncio.create_task(self._retry_loop())
logger.info("Local event queue started")
async def stop(self):
"""Stop background retry task."""
self.is_running = False
if self.retry_task:
self.retry_task.cancel()
try:
await self.retry_task
except asyncio.CancelledError:
pass
logger.info("Local event queue stopped")
async def enqueue(self, event_data: Dict[str, Any], topic: str) -> bool:
"""
Add event to local queue.
Args:
event_data: Event payload
topic: Kafka topic name
Returns:
True if queued successfully, False if queue is full
"""
async with self._lock:
if len(self.queue) >= self.max_size:
self.stats["total_dropped"] += 1
logger.warning(
f"Local event queue full ({self.max_size}), dropping event",
extra={"topic": topic, "event_type": event_data.get("event_type")}
)
return False
queued_event = QueuedEvent(
event_data=event_data,
topic=topic,
timestamp=datetime.utcnow()
)
self.queue.append(queued_event)
self.stats["total_queued"] += 1
self.stats["current_size"] = len(self.queue)
logger.info(
f"Event queued locally (queue size: {len(self.queue)})",
extra={"topic": topic, "event_type": event_data.get("event_type")}
)
return True
async def _retry_loop(self):
"""Background task to retry queued events."""
while self.is_running:
try:
await asyncio.sleep(self.retry_interval)
await self._process_queue()
except asyncio.CancelledError:
break
except Exception as e:
logger.error(f"Error in retry loop: {e}", exc_info=True)
async def _process_queue(self):
"""Process queued events and retry publishing to Kafka."""
if not self.queue:
return
async with self._lock:
events_to_retry = list(self.queue)
self.queue.clear()
succeeded = []
failed = []
expired = []
for event in events_to_retry:
if event.is_expired(self.max_age_seconds):
expired.append(event)
self.stats["total_expired"] += 1
logger.warning(
f"Event expired after {self.max_age_seconds}s, discarding",
extra={
"topic": event.topic,
"event_type": event.event_data.get("event_type"),
"age_seconds": (datetime.utcnow() - event.timestamp).total_seconds()
}
)
continue
if not event.should_retry():
expired.append(event)
self.stats["total_expired"] += 1
logger.warning(
f"Event exceeded max retries ({event.max_retries}), discarding",
extra={
"topic": event.topic,
"event_type": event.event_data.get("event_type"),
"retry_count": event.retry_count
}
)
continue
# Try to publish to Kafka
success = await self._try_publish(event)
if success:
succeeded.append(event)
self.stats["total_succeeded"] += 1
else:
event.retry_count += 1
failed.append(event)
self.stats["total_retried"] += 1
# Re-queue failed events
if failed:
async with self._lock:
for event in failed:
if len(self.queue) < self.max_size:
self.queue.append(event)
self.stats["current_size"] = len(self.queue)
if succeeded or expired:
logger.info(
f"Queue processing complete: {len(succeeded)} succeeded, "
f"{len(failed)} failed, {len(expired)} expired/dropped"
)
async def _try_publish(self, event: QueuedEvent) -> bool:
"""
Try to publish event to Kafka via Dapr.
Args:
event: Queued event to publish
Returns:
True if published successfully, False otherwise
"""
try:
import httpx
async with httpx.AsyncClient() as client:
response = await client.post(
f"http://localhost:3500/v1.0/publish/kafka-pubsub/{event.topic}",
json=event.event_data,
headers={"Content-Type": "application/json"},
timeout=5.0
)
if response.status_code == 200:
logger.info(
f"Successfully published queued event to Kafka",
extra={
"topic": event.topic,
"event_type": event.event_data.get("event_type"),
"retry_count": event.retry_count
}
)
return True
else:
logger.warning(
f"Failed to publish queued event: HTTP {response.status_code}",
extra={
"topic": event.topic,
"event_type": event.event_data.get("event_type"),
"retry_count": event.retry_count
}
)
return False
except Exception as e:
logger.warning(
f"Error publishing queued event: {e}",
extra={
"topic": event.topic,
"event_type": event.event_data.get("event_type"),
"retry_count": event.retry_count
}
)
return False
def get_stats(self) -> Dict[str, Any]:
"""Get queue statistics."""
return {
**self.stats,
"is_running": self.is_running,
"max_size": self.max_size,
"retry_interval": self.retry_interval,
"max_age_seconds": self.max_age_seconds
}
def clear(self):
"""Clear all queued events."""
self.queue.clear()
self.stats["current_size"] = 0
logger.info("Local event queue cleared")
# Global instance
_local_queue: Optional[LocalEventQueue] = None
def get_local_queue() -> LocalEventQueue:
"""Get or create global local event queue instance."""
global _local_queue
if _local_queue is None:
_local_queue = LocalEventQueue()
return _local_queue
async def publish_with_fallback(
event_data: Dict[str, Any],
topic: str,
dapr_http_port: int = 3500
) -> bool:
"""
Publish event to Kafka with local queue fallback.
Tries to publish directly to Kafka. If that fails, queues locally
for retry.
Args:
event_data: Event payload
topic: Kafka topic name
dapr_http_port: Dapr HTTP port
Returns:
True if published or queued successfully
"""
import httpx
try:
# Try direct publish to Kafka
async with httpx.AsyncClient() as client:
response = await client.post(
f"http://localhost:{dapr_http_port}/v1.0/publish/kafka-pubsub/{topic}",
json=event_data,
headers={"Content-Type": "application/json"},
timeout=5.0
)
if response.status_code == 200:
return True
else:
logger.warning(
f"Kafka publish failed with HTTP {response.status_code}, "
f"falling back to local queue"
)
except Exception as e:
logger.warning(
f"Kafka unavailable ({e}), falling back to local queue"
)
# Fallback to local queue
queue = get_local_queue()
return await queue.enqueue(event_data, topic)
# Example usage:
if __name__ == "__main__":
async def main():
# Start local queue
queue = get_local_queue()
await queue.start()
# Simulate event publishing with fallback
event = {
"event_type": "todo.task.created",
"payload": {"task_id": "123", "title": "Test Task"},
"timestamp": datetime.utcnow().isoformat(),
"correlation_id": "test-123"
}
success = await publish_with_fallback(event, "todo.task.events")
print(f"Publish result: {success}")
# Check stats
print(f"Queue stats: {queue.get_stats()}")
# Wait for retry
await asyncio.sleep(35)
# Stop queue
await queue.stop()
asyncio.run(main())
|