backend-todo-app / src /utils /local_event_queue.py
AliAskariFace's picture
Upload folder using huggingface_hub
dd63eee verified
Raw History Blame Contribute Delete
12 kB
"""
Local event queue for graceful degradation when Kafka is unavailable.
Provides in-memory event buffering with periodic retry to Kafka.
Events are held for up to 5 minutes before being discarded.
"""
import asyncio
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any
from dataclasses import dataclass, field
from collections import deque
import json
import logging
logger = logging.getLogger(__name__)
@dataclass
class QueuedEvent:
"""Represents an event in the local queue."""
event_data: Dict[str, Any]
topic: str
timestamp: datetime
retry_count: int = 0
max_retries: int = 10
def is_expired(self, max_age_seconds: int = 300) -> bool:
"""Check if event has exceeded maximum age (default 5 minutes)."""
age = (datetime.utcnow() - self.timestamp).total_seconds()
return age > max_age_seconds
def should_retry(self) -> bool:
"""Check if event should be retried."""
return self.retry_count < self.max_retries and not self.is_expired()
class LocalEventQueue:
"""
Local in-memory event queue for graceful degradation.
When Kafka is unavailable, events are queued locally and retried
periodically. Events older than 5 minutes are discarded.
Usage:
queue = LocalEventQueue(max_size=1000, retry_interval=30)
await queue.start()
# Queue event when Kafka is down
await queue.enqueue(event_data, topic)
# Stop background retry task
await queue.stop()
"""
def __init__(
self,
max_size: int = 1000,
retry_interval: int = 30,
max_age_seconds: int = 300
):
self.max_size = max_size
self.retry_interval = retry_interval
self.max_age_seconds = max_age_seconds
self.queue: deque[QueuedEvent] = deque(maxlen=max_size)
self.retry_task: Optional[asyncio.Task] = None
self.is_running = False
self._lock = asyncio.Lock()
# Statistics
self.stats = {
"total_queued": 0,
"total_retried": 0,
"total_succeeded": 0,
"total_expired": 0,
"total_dropped": 0,
"current_size": 0
}
async def start(self):
"""Start background retry task."""
if self.is_running:
return
self.is_running = True
self.retry_task = asyncio.create_task(self._retry_loop())
logger.info("Local event queue started")
async def stop(self):
"""Stop background retry task."""
self.is_running = False
if self.retry_task:
self.retry_task.cancel()
try:
await self.retry_task
except asyncio.CancelledError:
pass
logger.info("Local event queue stopped")
async def enqueue(self, event_data: Dict[str, Any], topic: str) -> bool:
"""
Add event to local queue.
Args:
event_data: Event payload
topic: Kafka topic name
Returns:
True if queued successfully, False if queue is full
"""
async with self._lock:
if len(self.queue) >= self.max_size:
self.stats["total_dropped"] += 1
logger.warning(
f"Local event queue full ({self.max_size}), dropping event",
extra={"topic": topic, "event_type": event_data.get("event_type")}
)
return False
queued_event = QueuedEvent(
event_data=event_data,
topic=topic,
timestamp=datetime.utcnow()
)
self.queue.append(queued_event)
self.stats["total_queued"] += 1
self.stats["current_size"] = len(self.queue)
logger.info(
f"Event queued locally (queue size: {len(self.queue)})",
extra={"topic": topic, "event_type": event_data.get("event_type")}
)
return True
async def _retry_loop(self):
"""Background task to retry queued events."""
while self.is_running:
try:
await asyncio.sleep(self.retry_interval)
await self._process_queue()
except asyncio.CancelledError:
break
except Exception as e:
logger.error(f"Error in retry loop: {e}", exc_info=True)
async def _process_queue(self):
"""Process queued events and retry publishing to Kafka."""
if not self.queue:
return
async with self._lock:
events_to_retry = list(self.queue)
self.queue.clear()
succeeded = []
failed = []
expired = []
for event in events_to_retry:
if event.is_expired(self.max_age_seconds):
expired.append(event)
self.stats["total_expired"] += 1
logger.warning(
f"Event expired after {self.max_age_seconds}s, discarding",
extra={
"topic": event.topic,
"event_type": event.event_data.get("event_type"),
"age_seconds": (datetime.utcnow() - event.timestamp).total_seconds()
}
)
continue
if not event.should_retry():
expired.append(event)
self.stats["total_expired"] += 1
logger.warning(
f"Event exceeded max retries ({event.max_retries}), discarding",
extra={
"topic": event.topic,
"event_type": event.event_data.get("event_type"),
"retry_count": event.retry_count
}
)
continue
# Try to publish to Kafka
success = await self._try_publish(event)
if success:
succeeded.append(event)
self.stats["total_succeeded"] += 1
else:
event.retry_count += 1
failed.append(event)
self.stats["total_retried"] += 1
# Re-queue failed events
if failed:
async with self._lock:
for event in failed:
if len(self.queue) < self.max_size:
self.queue.append(event)
self.stats["current_size"] = len(self.queue)
if succeeded or expired:
logger.info(
f"Queue processing complete: {len(succeeded)} succeeded, "
f"{len(failed)} failed, {len(expired)} expired/dropped"
)
async def _try_publish(self, event: QueuedEvent) -> bool:
"""
Try to publish event to Kafka via Dapr.
Args:
event: Queued event to publish
Returns:
True if published successfully, False otherwise
"""
try:
import httpx
async with httpx.AsyncClient() as client:
response = await client.post(
f"http://localhost:3500/v1.0/publish/kafka-pubsub/{event.topic}",
json=event.event_data,
headers={"Content-Type": "application/json"},
timeout=5.0
)
if response.status_code == 200:
logger.info(
f"Successfully published queued event to Kafka",
extra={
"topic": event.topic,
"event_type": event.event_data.get("event_type"),
"retry_count": event.retry_count
}
)
return True
else:
logger.warning(
f"Failed to publish queued event: HTTP {response.status_code}",
extra={
"topic": event.topic,
"event_type": event.event_data.get("event_type"),
"retry_count": event.retry_count
}
)
return False
except Exception as e:
logger.warning(
f"Error publishing queued event: {e}",
extra={
"topic": event.topic,
"event_type": event.event_data.get("event_type"),
"retry_count": event.retry_count
}
)
return False
def get_stats(self) -> Dict[str, Any]:
"""Get queue statistics."""
return {
**self.stats,
"is_running": self.is_running,
"max_size": self.max_size,
"retry_interval": self.retry_interval,
"max_age_seconds": self.max_age_seconds
}
def clear(self):
"""Clear all queued events."""
self.queue.clear()
self.stats["current_size"] = 0
logger.info("Local event queue cleared")
# Global instance
_local_queue: Optional[LocalEventQueue] = None
def get_local_queue() -> LocalEventQueue:
"""Get or create global local event queue instance."""
global _local_queue
if _local_queue is None:
_local_queue = LocalEventQueue()
return _local_queue
async def publish_with_fallback(
event_data: Dict[str, Any],
topic: str,
dapr_http_port: int = 3500
) -> bool:
"""
Publish event to Kafka with local queue fallback.
Tries to publish directly to Kafka. If that fails, queues locally
for retry.
Args:
event_data: Event payload
topic: Kafka topic name
dapr_http_port: Dapr HTTP port
Returns:
True if published or queued successfully
"""
import httpx
try:
# Try direct publish to Kafka
async with httpx.AsyncClient() as client:
response = await client.post(
f"http://localhost:{dapr_http_port}/v1.0/publish/kafka-pubsub/{topic}",
json=event_data,
headers={"Content-Type": "application/json"},
timeout=5.0
)
if response.status_code == 200:
return True
else:
logger.warning(
f"Kafka publish failed with HTTP {response.status_code}, "
f"falling back to local queue"
)
except Exception as e:
logger.warning(
f"Kafka unavailable ({e}), falling back to local queue"
)
# Fallback to local queue
queue = get_local_queue()
return await queue.enqueue(event_data, topic)
# Example usage:
if __name__ == "__main__":
async def main():
# Start local queue
queue = get_local_queue()
await queue.start()
# Simulate event publishing with fallback
event = {
"event_type": "todo.task.created",
"payload": {"task_id": "123", "title": "Test Task"},
"timestamp": datetime.utcnow().isoformat(),
"correlation_id": "test-123"
}
success = await publish_with_fallback(event, "todo.task.events")
print(f"Publish result: {success}")
# Check stats
print(f"Queue stats: {queue.get_stats()}")
# Wait for retry
await asyncio.sleep(35)
# Stop queue
await queue.stop()
asyncio.run(main())