File size: 9,400 Bytes
22087d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
"""
Conversation Analytics Background Worker

Processes queued conversations for LLM analysis.
Runs as an asyncio background task alongside the main API.
"""

import asyncio
from typing import Optional, List, Dict, Any

from core.utils.logger import logger
from core.services.supabase import DBConnection
from core.analytics.conversation_analyzer import analyze_conversation, store_analysis

# Worker configuration
PROCESSING_INTERVAL_SECONDS = 15  # Check queue every 15 seconds
BATCH_SIZE = 15  # Process up to 15 items per batch
MAX_CONCURRENT = 5  # Max concurrent LLM calls (avoid rate limits)
MAX_ATTEMPTS = 3  # Max retries for failed analysis
INITIAL_DELAY_SECONDS = 15  # Wait before starting (let API settle)

# Semaphore for limiting concurrent LLM calls
_concurrency_semaphore: asyncio.Semaphore = None

# Global task handle
_analytics_task: Optional[asyncio.Task] = None


async def claim_pending_queue_items(limit: int = BATCH_SIZE) -> List[Dict[str, Any]]:
    """
    Atomically claim pending items from the analytics queue.

    Uses FOR UPDATE SKIP LOCKED to prevent race conditions where
    multiple workers grab the same items.
    """
    try:
        from core.services.db import execute_mutate, serialize_rows

        # Atomic claim: SELECT + UPDATE in one transaction
        # FOR UPDATE SKIP LOCKED ensures no two workers grab same row
        # NOTE: Must use execute_mutate (not execute) to COMMIT the status change!
        result = await execute_mutate("""
            UPDATE conversation_analytics_queue
            SET status = 'processing'
            WHERE id IN (
                SELECT id
                FROM conversation_analytics_queue
                WHERE status = 'pending' AND attempts < :max_attempts
                ORDER BY created_at
                LIMIT :limit
                FOR UPDATE SKIP LOCKED
            )
            RETURNING id, thread_id, agent_run_id, account_id, attempts
        """, {"limit": limit, "max_attempts": MAX_ATTEMPTS})

        # serialize_rows converts UUID objects to strings for JSON compatibility
        return serialize_rows(result) if result else []

    except Exception as e:
        logger.error(f"[ANALYTICS] Failed to claim queue items: {e}")
        return []


async def update_queue_status(
    queue_id: str,
    status: str,
    error_message: Optional[str] = None,
    increment_attempts: bool = False
) -> None:
    """Update the status of a queue item."""
    try:
        db = DBConnection()
        client = await db.client

        update_data = {'status': status}

        if status == 'completed' or status == 'failed':
            update_data['processed_at'] = 'now()'

        if error_message:
            update_data['error_message'] = error_message

        query = client.from_('conversation_analytics_queue')\
            .update(update_data)\
            .eq('id', queue_id)

        await query.execute()

        # Increment attempts separately if needed
        if increment_attempts:
            await client.rpc('increment_analytics_attempts', {'queue_id': queue_id}).execute()

    except Exception as e:
        logger.warning(f"[ANALYTICS] Failed to update queue status for {queue_id}: {e}")


async def process_queue_item(item: Dict[str, Any]) -> bool:
    """
    Process a single queue item.

    Returns True if successful, False otherwise.
    """
    queue_id = item['id']
    thread_id = item['thread_id']
    agent_run_id = item.get('agent_run_id')
    account_id = item['account_id']

    try:
        # Note: Item already marked as 'processing' by claim_pending_queue_items()

        # Run analysis (filtered by agent run time range)
        analysis = await analyze_conversation(thread_id, agent_run_id)

        if not analysis:
            # No analysis possible (e.g., empty conversation)
            await update_queue_status(queue_id, 'completed')
            return True

        # Get agent run status if available
        agent_run_status = None
        if agent_run_id:
            try:
                db = DBConnection()
                client = await db.client
                run_result = await client.from_('agent_runs')\
                    .select('status')\
                    .eq('id', agent_run_id)\
                    .single()\
                    .execute()
                if run_result.data:
                    agent_run_status = run_result.data.get('status')
            except Exception:
                pass

        # Store results
        success = await store_analysis(
            thread_id=thread_id,
            agent_run_id=agent_run_id,
            account_id=account_id,
            analysis=analysis,
            agent_run_status=agent_run_status
        )

        if success:
            await update_queue_status(queue_id, 'completed')
            logger.info(f"[ANALYTICS] Successfully analyzed thread {thread_id}")
            return True
        else:
            await update_queue_status(
                queue_id, 'pending',
                error_message='Failed to store analysis',
                increment_attempts=True
            )
            return False

    except Exception as e:
        logger.error(f"[ANALYTICS] Error processing queue item {queue_id}: {e}")
        await update_queue_status(
            queue_id, 'pending',
            error_message=str(e)[:500],
            increment_attempts=True
        )
        return False


async def _process_with_semaphore(item: Dict[str, Any], semaphore: asyncio.Semaphore) -> bool:
    """Process a queue item with concurrency limiting."""
    async with semaphore:
        try:
            return await process_queue_item(item)
        except Exception as e:
            logger.error(f"[ANALYTICS] Unexpected error processing item {item.get('id')}: {e}")
            return False


async def _analytics_processing_loop() -> None:
    """
    Main processing loop for conversation analytics.

    Runs continuously, checking the queue and processing items concurrently.
    """
    global _concurrency_semaphore

    logger.info("[ANALYTICS] Starting conversation analytics worker")

    # Initialize semaphore for concurrent processing
    _concurrency_semaphore = asyncio.Semaphore(MAX_CONCURRENT)

    # Initial delay to let the API settle
    await asyncio.sleep(INITIAL_DELAY_SECONDS)

    while True:
        try:
            # Atomically claim pending items (prevents race conditions)
            pending = await claim_pending_queue_items(limit=BATCH_SIZE)

            if pending:
                logger.debug(f"[ANALYTICS] Processing {len(pending)} queued conversations concurrently")

                # Process all items concurrently with semaphore limiting
                tasks = [
                    _process_with_semaphore(item, _concurrency_semaphore)
                    for item in pending
                ]
                results = await asyncio.gather(*tasks, return_exceptions=True)

                # Log results
                success_count = sum(1 for r in results if r is True)
                logger.info(f"[ANALYTICS] Batch complete: {success_count}/{len(pending)} succeeded")

            # Clean up old failed items (mark as failed after max attempts)
            await cleanup_failed_items()

        except asyncio.CancelledError:
            logger.info("[ANALYTICS] Worker cancelled, shutting down gracefully")
            break

        except Exception as e:
            logger.error(f"[ANALYTICS] Error in processing loop: {e}")

        # Wait before next check
        try:
            await asyncio.sleep(PROCESSING_INTERVAL_SECONDS)
        except asyncio.CancelledError:
            logger.info("[ANALYTICS] Worker cancelled during sleep")
            break


async def cleanup_failed_items() -> None:
    """Mark items that have exceeded max attempts as failed."""
    try:
        db = DBConnection()
        client = await db.client

        await client.from_('conversation_analytics_queue')\
            .update({
                'status': 'failed',
                'processed_at': 'now()',
                'error_message': f'Exceeded max attempts ({MAX_ATTEMPTS})'
            })\
            .eq('status', 'pending')\
            .gte('attempts', MAX_ATTEMPTS)\
            .execute()

    except Exception as e:
        logger.warning(f"[ANALYTICS] Failed to cleanup failed items: {e}")


async def start_analytics_worker() -> None:
    """Start the analytics background worker."""
    global _analytics_task

    if _analytics_task and not _analytics_task.done():
        logger.warning("[ANALYTICS] Worker already running")
        return

    _analytics_task = asyncio.create_task(_analytics_processing_loop())
    logger.info("[ANALYTICS] Worker task created")


async def stop_analytics_worker() -> None:
    """Stop the analytics background worker gracefully."""
    global _analytics_task

    if not _analytics_task:
        return

    if not _analytics_task.done():
        logger.info("[ANALYTICS] Stopping worker...")
        _analytics_task.cancel()
        try:
            await _analytics_task
        except asyncio.CancelledError:
            pass

    _analytics_task = None
    logger.info("[ANALYTICS] Worker stopped")


def is_analytics_worker_running() -> bool:
    """Check if the analytics worker is running."""
    return _analytics_task is not None and not _analytics_task.done()