File size: 11,957 Bytes
dd63eee
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
"""

Local event queue for graceful degradation when Kafka is unavailable.



Provides in-memory event buffering with periodic retry to Kafka.

Events are held for up to 5 minutes before being discarded.

"""

import asyncio
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any
from dataclasses import dataclass, field
from collections import deque
import json
import logging

logger = logging.getLogger(__name__)


@dataclass
class QueuedEvent:
    """Represents an event in the local queue."""
    event_data: Dict[str, Any]
    topic: str
    timestamp: datetime
    retry_count: int = 0
    max_retries: int = 10

    def is_expired(self, max_age_seconds: int = 300) -> bool:
        """Check if event has exceeded maximum age (default 5 minutes)."""
        age = (datetime.utcnow() - self.timestamp).total_seconds()
        return age > max_age_seconds

    def should_retry(self) -> bool:
        """Check if event should be retried."""
        return self.retry_count < self.max_retries and not self.is_expired()


class LocalEventQueue:
    """

    Local in-memory event queue for graceful degradation.



    When Kafka is unavailable, events are queued locally and retried

    periodically. Events older than 5 minutes are discarded.



    Usage:

        queue = LocalEventQueue(max_size=1000, retry_interval=30)

        await queue.start()



        # Queue event when Kafka is down

        await queue.enqueue(event_data, topic)



        # Stop background retry task

        await queue.stop()

    """

    def __init__(

        self,

        max_size: int = 1000,

        retry_interval: int = 30,

        max_age_seconds: int = 300

    ):
        self.max_size = max_size
        self.retry_interval = retry_interval
        self.max_age_seconds = max_age_seconds
        self.queue: deque[QueuedEvent] = deque(maxlen=max_size)
        self.retry_task: Optional[asyncio.Task] = None
        self.is_running = False
        self._lock = asyncio.Lock()

        # Statistics
        self.stats = {
            "total_queued": 0,
            "total_retried": 0,
            "total_succeeded": 0,
            "total_expired": 0,
            "total_dropped": 0,
            "current_size": 0
        }

    async def start(self):
        """Start background retry task."""
        if self.is_running:
            return

        self.is_running = True
        self.retry_task = asyncio.create_task(self._retry_loop())
        logger.info("Local event queue started")

    async def stop(self):
        """Stop background retry task."""
        self.is_running = False
        if self.retry_task:
            self.retry_task.cancel()
            try:
                await self.retry_task
            except asyncio.CancelledError:
                pass
        logger.info("Local event queue stopped")

    async def enqueue(self, event_data: Dict[str, Any], topic: str) -> bool:
        """

        Add event to local queue.



        Args:

            event_data: Event payload

            topic: Kafka topic name



        Returns:

            True if queued successfully, False if queue is full

        """
        async with self._lock:
            if len(self.queue) >= self.max_size:
                self.stats["total_dropped"] += 1
                logger.warning(
                    f"Local event queue full ({self.max_size}), dropping event",
                    extra={"topic": topic, "event_type": event_data.get("event_type")}
                )
                return False

            queued_event = QueuedEvent(
                event_data=event_data,
                topic=topic,
                timestamp=datetime.utcnow()
            )
            self.queue.append(queued_event)
            self.stats["total_queued"] += 1
            self.stats["current_size"] = len(self.queue)

            logger.info(
                f"Event queued locally (queue size: {len(self.queue)})",
                extra={"topic": topic, "event_type": event_data.get("event_type")}
            )
            return True

    async def _retry_loop(self):
        """Background task to retry queued events."""
        while self.is_running:
            try:
                await asyncio.sleep(self.retry_interval)
                await self._process_queue()
            except asyncio.CancelledError:
                break
            except Exception as e:
                logger.error(f"Error in retry loop: {e}", exc_info=True)

    async def _process_queue(self):
        """Process queued events and retry publishing to Kafka."""
        if not self.queue:
            return

        async with self._lock:
            events_to_retry = list(self.queue)
            self.queue.clear()

        succeeded = []
        failed = []
        expired = []

        for event in events_to_retry:
            if event.is_expired(self.max_age_seconds):
                expired.append(event)
                self.stats["total_expired"] += 1
                logger.warning(
                    f"Event expired after {self.max_age_seconds}s, discarding",
                    extra={
                        "topic": event.topic,
                        "event_type": event.event_data.get("event_type"),
                        "age_seconds": (datetime.utcnow() - event.timestamp).total_seconds()
                    }
                )
                continue

            if not event.should_retry():
                expired.append(event)
                self.stats["total_expired"] += 1
                logger.warning(
                    f"Event exceeded max retries ({event.max_retries}), discarding",
                    extra={
                        "topic": event.topic,
                        "event_type": event.event_data.get("event_type"),
                        "retry_count": event.retry_count
                    }
                )
                continue

            # Try to publish to Kafka
            success = await self._try_publish(event)

            if success:
                succeeded.append(event)
                self.stats["total_succeeded"] += 1
            else:
                event.retry_count += 1
                failed.append(event)
                self.stats["total_retried"] += 1

        # Re-queue failed events
        if failed:
            async with self._lock:
                for event in failed:
                    if len(self.queue) < self.max_size:
                        self.queue.append(event)
                self.stats["current_size"] = len(self.queue)

        if succeeded or expired:
            logger.info(
                f"Queue processing complete: {len(succeeded)} succeeded, "
                f"{len(failed)} failed, {len(expired)} expired/dropped"
            )

    async def _try_publish(self, event: QueuedEvent) -> bool:
        """

        Try to publish event to Kafka via Dapr.



        Args:

            event: Queued event to publish



        Returns:

            True if published successfully, False otherwise

        """
        try:
            import httpx

            async with httpx.AsyncClient() as client:
                response = await client.post(
                    f"http://localhost:3500/v1.0/publish/kafka-pubsub/{event.topic}",
                    json=event.event_data,
                    headers={"Content-Type": "application/json"},
                    timeout=5.0
                )

                if response.status_code == 200:
                    logger.info(
                        f"Successfully published queued event to Kafka",
                        extra={
                            "topic": event.topic,
                            "event_type": event.event_data.get("event_type"),
                            "retry_count": event.retry_count
                        }
                    )
                    return True
                else:
                    logger.warning(
                        f"Failed to publish queued event: HTTP {response.status_code}",
                        extra={
                            "topic": event.topic,
                            "event_type": event.event_data.get("event_type"),
                            "retry_count": event.retry_count
                        }
                    )
                    return False

        except Exception as e:
            logger.warning(
                f"Error publishing queued event: {e}",
                extra={
                    "topic": event.topic,
                    "event_type": event.event_data.get("event_type"),
                    "retry_count": event.retry_count
                }
            )
            return False

    def get_stats(self) -> Dict[str, Any]:
        """Get queue statistics."""
        return {
            **self.stats,
            "is_running": self.is_running,
            "max_size": self.max_size,
            "retry_interval": self.retry_interval,
            "max_age_seconds": self.max_age_seconds
        }

    def clear(self):
        """Clear all queued events."""
        self.queue.clear()
        self.stats["current_size"] = 0
        logger.info("Local event queue cleared")


# Global instance
_local_queue: Optional[LocalEventQueue] = None


def get_local_queue() -> LocalEventQueue:
    """Get or create global local event queue instance."""
    global _local_queue
    if _local_queue is None:
        _local_queue = LocalEventQueue()
    return _local_queue


async def publish_with_fallback(

    event_data: Dict[str, Any],

    topic: str,

    dapr_http_port: int = 3500

) -> bool:
    """

    Publish event to Kafka with local queue fallback.



    Tries to publish directly to Kafka. If that fails, queues locally

    for retry.



    Args:

        event_data: Event payload

        topic: Kafka topic name

        dapr_http_port: Dapr HTTP port



    Returns:

        True if published or queued successfully

    """
    import httpx

    try:
        # Try direct publish to Kafka
        async with httpx.AsyncClient() as client:
            response = await client.post(
                f"http://localhost:{dapr_http_port}/v1.0/publish/kafka-pubsub/{topic}",
                json=event_data,
                headers={"Content-Type": "application/json"},
                timeout=5.0
            )

            if response.status_code == 200:
                return True
            else:
                logger.warning(
                    f"Kafka publish failed with HTTP {response.status_code}, "
                    f"falling back to local queue"
                )

    except Exception as e:
        logger.warning(
            f"Kafka unavailable ({e}), falling back to local queue"
        )

    # Fallback to local queue
    queue = get_local_queue()
    return await queue.enqueue(event_data, topic)


# Example usage:
if __name__ == "__main__":
    async def main():
        # Start local queue
        queue = get_local_queue()
        await queue.start()

        # Simulate event publishing with fallback
        event = {
            "event_type": "todo.task.created",
            "payload": {"task_id": "123", "title": "Test Task"},
            "timestamp": datetime.utcnow().isoformat(),
            "correlation_id": "test-123"
        }

        success = await publish_with_fallback(event, "todo.task.events")
        print(f"Publish result: {success}")

        # Check stats
        print(f"Queue stats: {queue.get_stats()}")

        # Wait for retry
        await asyncio.sleep(35)

        # Stop queue
        await queue.stop()

    asyncio.run(main())