File size: 9,046 Bytes
22087d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
import os
import json
import asyncio
import time
from datetime import datetime, timezone
from typing import Optional, Dict, Any

import structlog

from core.utils.logger import logger
from core.services import redis
from core.services.langfuse import langfuse
from core.utils.tool_output_streaming import (
    set_tool_output_streaming_context,
    clear_tool_output_streaming_context,
)

from core.agents.runner.config import AgentConfig
from core.agents.runner.services import (
    REDIS_STREAM_TTL_SECONDS,
    STOP_CHECK_INTERVAL,
    stream_status_message,
    check_terminating_tool_call,
    update_agent_run_status,
    send_completion_notification,
)

async def execute_agent_run(
    agent_run_id: str,
    thread_id: str,
    project_id: str,
    model_name: str,
    agent_config: dict,
    account_id: str,
    cancellation_event: asyncio.Event,
    is_new_thread: bool = False,
    user_message: Optional[str] = None
) -> None:
    execution_start = time.time()

    structlog.contextvars.clear_contextvars()
    structlog.contextvars.bind_contextvars(agent_run_id=agent_run_id, thread_id=thread_id)

    logger.info(f"πŸš€ Executing agent run: {agent_run_id}")

    stop_checker = None
    final_status = "failed"
    stream_key = f"agent_run:{agent_run_id}:stream"

    try:
        from datetime import datetime, timezone
        start_time = datetime.now(timezone.utc)

        await stream_status_message("initializing", "Starting execution...", stream_key=stream_key)
        await redis.verify_stream_writable(stream_key)

        try:
            await redis.expire(stream_key, REDIS_STREAM_TTL_SECONDS)
        except Exception:
            pass

        from core.ai_models import model_manager
        effective_model = model_manager.resolve_model_id(model_name)

        stop_state = {'received': False, 'reason': None}

        async def check_stop():
            while not stop_state['received']:
                try:
                    if cancellation_event.is_set():
                        stop_state['received'] = True
                        stop_state['reason'] = 'cancellation_event'
                        logger.info(f"πŸ›‘ Stop detected via cancellation_event for {agent_run_id}")
                        break

                    if await redis.check_stop_signal(agent_run_id):
                        stop_state['received'] = True
                        stop_state['reason'] = 'stop_signal'
                        cancellation_event.set()
                        logger.info(f"πŸ›‘ Stop detected via Redis for {agent_run_id}")
                        break

                    await asyncio.sleep(STOP_CHECK_INTERVAL)
                except asyncio.CancelledError:
                    break
                except Exception:
                    await asyncio.sleep(5.0)

        stop_checker = asyncio.create_task(check_stop())

        set_tool_output_streaming_context(agent_run_id=agent_run_id, stream_key=stream_key)

        from core.agents.pipeline.context import PipelineContext
        from core.agents.pipeline.stateless import StatelessCoordinator

        ctx = PipelineContext(
            agent_run_id=agent_run_id,
            thread_id=thread_id,
            project_id=project_id,
            account_id=account_id,
            model_name=effective_model,
            agent_config=agent_config,
            is_new_thread=is_new_thread,
            cancellation_event=cancellation_event,
            stream_key=stream_key,
            user_message=user_message
        )

        coordinator = StatelessCoordinator()

        first_response = False
        complete_tool_called = False
        total_responses = 0
        stream_ttl_set = False
        error_message = None

        async for response in coordinator.execute(ctx):
            if cancellation_event.is_set() or stop_state['received']:
                logger.warning(f"πŸ›‘ Agent run stopped: {stop_state.get('reason', 'cancellation_event')}")
                final_status = "stopped"
                error_message = f"Stopped by {stop_state.get('reason', 'cancellation_event')}"
                break

            if not first_response:
                first_response_time_ms = (time.time() - execution_start) * 1000
                logger.info(f"⏱️ FIRST RESPONSE: {first_response_time_ms:.1f}ms")
                first_response = True

                try:
                    timing_msg = {
                        "type": "timing",
                        "first_response_ms": round(first_response_time_ms, 1),
                        "timestamp": datetime.now(timezone.utc).isoformat(),
                        "pipeline": "stateless"
                    }
                    await redis.stream_add(stream_key, {"data": json.dumps(timing_msg)}, maxlen=200, approximate=True)
                except Exception:
                    pass

            from core.services.db import serialize_row
            if isinstance(response, dict):
                response = serialize_row(response)
                if response.get('type') == 'assistant':
                    metadata = response.get('metadata', '')
                    if isinstance(metadata, str) and 'complete' in metadata:
                        logger.debug(f"[STREAM] Sending assistant complete: message_id={response.get('message_id')}")

            try:
                await redis.stream_add(stream_key, {"data": json.dumps(response)}, maxlen=200, approximate=True)

                if not stream_ttl_set:
                    try:
                        await asyncio.wait_for(redis.expire(stream_key, REDIS_STREAM_TTL_SECONDS), timeout=2.0)
                        stream_ttl_set = True
                    except:
                        pass
            except Exception as e:
                logger.warning(f"Failed to write to stream: {e}")

            total_responses += 1

            terminating = check_terminating_tool_call(response)
            if terminating == 'complete':
                complete_tool_called = True

            if response.get('type') == 'status':
                status = response.get('status')
                if status in ['completed', 'failed', 'stopped', 'error']:
                    final_status = status if status != 'error' else 'failed'
                    if status in ['failed', 'error']:
                        error_message = response.get('message') or response.get('error')
                        logger.error(f"Agent run error: {error_message}")
                    break

            if response.get('type') == 'error':
                final_status = 'failed'
                error_message = response.get('error')
                logger.error(f"Agent run error: {error_message}")
                break

        if final_status == "failed" and not error_message:
            final_status = "completed"
            duration = (datetime.now(timezone.utc) - start_time).total_seconds()
            logger.info(f"Agent run completed (duration: {duration:.2f}s, responses: {total_responses})")

            completion_msg = {"type": "status", "status": "completed", "message": "Completed successfully"}
            try:
                await redis.stream_add(stream_key, {'data': json.dumps(completion_msg)}, maxlen=200, approximate=True)
            except:
                pass

            await send_completion_notification(thread_id, agent_config, complete_tool_called)

        if stop_state['reason']:
            final_status = "stopped"

        await update_agent_run_status(agent_run_id, final_status, error=error_message, account_id=account_id)

        logger.info(f"βœ… Agent run completed: {agent_run_id} | status={final_status}")

    except Exception as e:
        logger.error(f"Error in agent run {agent_run_id}: {e}", exc_info=True)
        await update_agent_run_status(agent_run_id, "failed", error=str(e), account_id=account_id)

    finally:
        from core.utils.lifecycle_tracker import log_cleanup_error
        cleanup_errors = []

        try:
            clear_tool_output_streaming_context()
        except Exception as e:
            log_cleanup_error(agent_run_id, "streaming_context", e)
            cleanup_errors.append(f"streaming_context: {e}")

        if stop_checker and not stop_checker.done():
            try:
                stop_checker.cancel()
                await stop_checker
            except asyncio.CancelledError:
                pass
            except Exception as e:
                log_cleanup_error(agent_run_id, "stop_checker", e)
                cleanup_errors.append(f"stop_checker: {e}")

        try:
            await redis.expire(stream_key, REDIS_STREAM_TTL_SECONDS)
        except Exception as e:
            log_cleanup_error(agent_run_id, "redis_expire", e)
            cleanup_errors.append(f"redis_expire: {e}")

        if cleanup_errors:
            logger.error(
                f"[LIFECYCLE] CLEANUP_ERRORS agent_run={agent_run_id} "
                f"count={len(cleanup_errors)} errors={cleanup_errors}"
            )