File size: 20,802 Bytes
22087d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
from core.agentpress.tool import Tool, ToolResult, openapi_schema, tool_metadata
from core.agentpress.thread_manager import ThreadManager
from typing import List
import json

@tool_metadata(
    display_name="Internal Utilities",
    description="Internal tool loading, MCP integration, and message expansion",
    icon="Maximize",
    color="bg-gray-100 dark:bg-gray-800/50",
    weight=100,
    visible=False,
    usage_guide="""
### INTERNAL UTILITIES

**initialize_tools - Tool Activation:**
- Activates tools on-demand
- Loads detailed usage documentation
- Supports batch loading multiple tools at once (e.g., tool1, tool2, tool3)

**CRITICAL USAGE:**
- Call at the start of every task
- Analyze user request β†’ Determine ALL needed tools β†’ Load in ONE batch

**discover_mcp_tools - MCP Schema Discovery:**
- Get schemas for external MCP tools (Gmail, Twitter, Slack, etc.)
- ALWAYS check conversation history first - schemas persist forever
- Batch discover ALL tools needed in ONE call
- Example: use discover_mcp_tools with filter parameter set to "GMAIL_SEND_MESSAGE,TWITTER_CREATION_OF_A_POST"

**execute_mcp_tool - MCP Tool Execution:**
- Execute external integration tools
- Requires schema in conversation history (discover first if needed)
- Example: use execute_mcp_tool with tool_name parameter "GMAIL_SEND_MESSAGE" and args parameter containing the tool arguments

**expand_message - Message Expansion:**
- View full content of truncated messages
- Use when previous messages were shortened
- Retrieve complete message history
"""
)
class ExpandMessageTool(Tool):
    def __init__(self, thread_id: str, thread_manager: ThreadManager):
        super().__init__()
        self.thread_manager = thread_manager
        self.thread_id = thread_id

    @openapi_schema({
        "type": "function",
        "function": {
            "name": "expand_message",
            "description": "Expand a message from the previous conversation with the user. Use this tool to expand a message that was truncated in the earlier conversation. **🚨 PARAMETER NAMES**: Use EXACTLY this parameter name: `message_id` (REQUIRED).",
            "parameters": {
                "type": "object",
                "properties": {
                    "message_id": {
                        "type": "string",
                        "description": "**REQUIRED** - The ID of the message to expand. Must be a UUID."
                    }
                },
                "required": ["message_id"],
                "additionalProperties": False
            }
        }
    })
    async def expand_message(self, message_id: str) -> ToolResult:
        try:
            client = await self.thread_manager.db.client
            message = await client.table('messages').select('*').eq('message_id', message_id).eq('thread_id', self.thread_id).execute()

            if not message.data or len(message.data) == 0:
                return self.fail_response(f"Message with ID {message_id} not found in thread {self.thread_id}")

            message_data = message.data[0]
            message_content = message_data['content']
            final_content = message_content
            if isinstance(message_content, dict) and 'content' in message_content:
                final_content = message_content['content']
            elif isinstance(message_content, str):
                try:
                    parsed_content = json.loads(message_content)
                    if isinstance(parsed_content, dict) and 'content' in parsed_content:
                        final_content = parsed_content['content']
                except json.JSONDecodeError:
                    pass

            return self.success_response({"status": "Message expanded successfully.", "message": final_content})
        except Exception as e:
            return self.fail_response(f"Error expanding message: {str(e)}")

    @openapi_schema({
        "type": "function", 
        "function": {
            "name": "discover_mcp_tools", 
            "description": "Get schemas for external MCP tools (Gmail, Twitter, Slack, etc.). CRITICAL WORKFLOW: (1) Check conversation history FIRST - if tool schemas already exist, skip discovery! (2) If NOT in history: Discover ALL needed tools in ONE batch call. (3) Schemas are cached in conversation forever - NEVER discover same tools twice! **🚨 PARAMETER NAMES**: Use EXACTLY this parameter name: `filter` (REQUIRED).",
            "parameters": {
                "type": "object",
                "properties": {
                    "filter": {
                        "type": "string",
                        "description": "**REQUIRED** - Comma-separated list of ALL tools needed (e.g., 'GMAIL_SEND_MESSAGE,TWITTER_CREATION_OF_A_POST,SLACK_SEND_MESSAGE') OR toolkit name (e.g., 'gmail'). CRITICAL: List ALL tools in ONE call, never call discover multiple times for the same task!"
                    }
                },
                "required": ["filter"],
                "additionalProperties": False
            }
        }
    }) 
    async def discover_mcp_tools(self, filter: str) -> ToolResult:
        return await self._discover_tools(filter)

    @openapi_schema({
        "type": "function", 
        "function": {
            "name": "execute_mcp_tool", 
            "description": "Execute external MCP tool (Gmail, Twitter, Slack, etc.). PREREQUISITE: Tool schema MUST be in conversation history (use discover_mcp_tools first if needed). Use exact tool name and parameters from the discovered schema. **🚨 PARAMETER NAMES**: Use EXACTLY these parameter names: `tool_name` (REQUIRED), `args` (REQUIRED).",
            "parameters": {
                "type": "object",
                "properties": {
                    "tool_name": {
                        "type": "string",
                        "description": "**REQUIRED** - Exact tool name from discovered schema. Example: 'GMAIL_SEND_MESSAGE', 'TWITTER_CREATION_OF_A_POST'. Must match schema in conversation history."
                    },
                    "args": {
                        "type": "object", 
                        "description": "**REQUIRED** - Arguments matching discovered schema parameters. Use exact parameter names from schema in conversation history. If no parameters are required, provide an empty object {}."
                    }
                },
                "required": ["tool_name", "args"],
                "additionalProperties": False
            }
        }
    }) 
    async def execute_mcp_tool(self, tool_name: str, args: dict) -> ToolResult:
        return await self._call_tool(tool_name, args)

    @openapi_schema({
        "type": "function",
        "function": {
            "name": "initialize_tools",
            "description": "Initialize tools needed for your task. Loads the detailed usage guides and activates the tools so they're ready to use. Call this at the start with ALL tools you'll need. **🚨 PARAMETER NAMES**: Use EXACTLY this parameter name: `tool_names` (REQUIRED).",
            "parameters": {
                "type": "object",
                "properties": {
                    "tool_names": {
                        "oneOf": [
                            {
                                "type": "string",
                                "description": "**REQUIRED** - Single tool name to initialize. Example: 'browser_tool', 'sb_files_tool'"
                            },
                            {
                                "type": "array",
                                "items": {"type": "string"},
                                "description": "**REQUIRED** - Multiple tool names to initialize in one batch call. Example: ['browser_tool', 'sb_files_tool', 'web_search_tool']"
                            }
                        ],
                        "description": "**REQUIRED** - Tool name(s) to initialize. Can be a single string or array of strings."
                    }
                },
                "required": ["tool_names"],
                "additionalProperties": False
            }
        }
    })
    async def initialize_tools(self, tool_names: str | List[str]) -> ToolResult:
        import asyncio
        import time
        from core.tools.tool_guide_registry import get_tool_guide, get_tool_guide_registry
        from core.jit import JITLoader
        from core.utils.logger import logger
        
        start = time.time()
        
        if isinstance(tool_names, str):
            # Handle case where LLM passes JSON-encoded array as string (e.g., '["sb_presentation_tool"]')
            stripped = tool_names.strip()
            if stripped.startswith('[') and stripped.endswith(']'):
                try:
                    parsed = json.loads(stripped)
                    if isinstance(parsed, list):
                        tool_names = parsed
                    else:
                        tool_names = [tool_names]
                except json.JSONDecodeError:
                    tool_names = [tool_names]
            else:
                tool_names = [tool_names]
        
        logger.info(f"πŸ”§ [INIT TOOLS] Initializing tools: {tool_names}")
        
        registry = get_tool_guide_registry()
        not_found = []
        
        valid_tool_names = []
        for tool_name in tool_names:
            if not registry.has_tool(tool_name):
                not_found.append(tool_name)
            else:
                valid_tool_names.append(tool_name)
        
        if not_found:
            available = ", ".join(registry.get_all_tool_names())
            logger.error(f"❌ [INIT TOOLS] Tools not found: {not_found}")
            return self.fail_response(
                f"Tools not found: {', '.join(not_found)}. Available tools: {available}"
            )
        
        project_id = getattr(self.thread_manager, 'project_id', None)
        jit_config = getattr(self.thread_manager, 'jit_config', None)
        
        logger.info(f"⚑ [INIT TOOLS] Parallel activation of {len(valid_tool_names)} tools")
        activation_start = time.time()
        
        activation_tasks = [
            JITLoader.activate_tool(tool_name, self.thread_manager, project_id, jit_config=jit_config)
            for tool_name in valid_tool_names
        ]
        
        activation_results = await asyncio.gather(*activation_tasks, return_exceptions=True)
        logger.info(f"⏱️ [INIT TOOLS] Parallel activation completed in {(time.time() - activation_start) * 1000:.1f}ms")
        
        from core.jit.result_types import ActivationSuccess, ActivationError
        
        guides = []
        activation_failures = []
        
        for tool_name, result in zip(valid_tool_names, activation_results):
            if isinstance(result, Exception):
                activation_failures.append(tool_name)
                logger.warning(f"⚠️  [INIT TOOLS] Failed to activate '{tool_name}': {result}")
            elif isinstance(result, ActivationError):
                activation_failures.append(tool_name)
                logger.warning(f"⚠️  [INIT TOOLS] {result.to_user_message()}")
            elif isinstance(result, ActivationSuccess):
                logger.debug(f"βœ… [INIT TOOLS] {result}")
        
        from core.jit.tool_cache import get_tool_cache
        
        tool_cache = get_tool_cache()
        cached_guides = await tool_cache.get_multiple(valid_tool_names)
        
        guides = []
        guides_to_cache = {}
        
        for tool_name in valid_tool_names:
            cached_guide = cached_guides.get(tool_name)
            if cached_guide:
                guides.append(cached_guide)
                logger.debug(f"βœ… [CACHE HIT] {tool_name}")
            else:
                guide = get_tool_guide(tool_name)
                if guide:
                    guides.append(guide)
                    guides_to_cache[tool_name] = guide
                    logger.debug(f"❌ [CACHE MISS] {tool_name}")
                else:
                    info = registry.get_tool_info(tool_name)
                    logger.warning(f"⚠️  [INIT TOOLS] Tool '{tool_name}' has no detailed guide")
                    fallback_guide = f"## {info[0]}\n\nNo detailed guide available. Basic description: {info[1]}"
                    guides.append(fallback_guide)
                    guides_to_cache[tool_name] = fallback_guide
        
        if guides_to_cache:
            await tool_cache.set_multiple(guides_to_cache)
            logger.info(f"πŸ’Ύ [CACHE STORE] Cached {len(guides_to_cache)} new guides")
        
        if activation_failures:
            logger.error(f"❌ [INIT TOOLS] Failed to activate some tools: {activation_failures}")
        
        successfully_activated = [t for t in valid_tool_names if t not in activation_failures]
        if successfully_activated:
            await self._save_dynamic_tools_to_metadata(successfully_activated)
        
        total_guide_size = sum(len(g) for g in guides)
        total_time = (time.time() - start) * 1000
        logger.info(f"βœ… [INIT TOOLS] Returned {len(guides)} guide(s) in {total_time:.1f}ms, total size: {total_guide_size:,} chars")
        logger.info(f"🎯 [INIT TOOLS] Tools now available for use: {[t for t in valid_tool_names if t not in activation_failures]}")
        
        result = self.success_response({
            "status": "success",
            "message": f"Loaded {len(guides)} tool guide(s). Tools are now available for use.",
            "guides": "\n\n---\n\n".join(guides),
            "activated_tools": [t for t in tool_names if t not in activation_failures],
            "_internal": True
        })
        
        return result

    async def _discover_tools(self, filter: str = None) -> ToolResult:
        from core.agentpress.mcp_registry import get_mcp_registry
        from core.utils.logger import logger
        
        mcp_registry = get_mcp_registry()
        
        mcp_loader = getattr(self.thread_manager, 'mcp_loader', None)
        if mcp_loader:
            loader_tool_count = len(mcp_loader.tool_map) if mcp_loader.tool_map else 0
            registry_tool_count = len(mcp_registry._tools)
            
            if not mcp_registry._initialized or loader_tool_count > registry_tool_count:
                from core.agentpress.mcp_registry import init_mcp_registry_from_loader
                logger.info(f"πŸ”„ [MCP REGISTRY] Syncing registry: loader has {loader_tool_count} tools, registry has {registry_tool_count}")
                init_mcp_registry_from_loader(mcp_loader)
                mcp_registry._initialized = True
                
                account_id = getattr(self.thread_manager, 'account_id', None)
                warmed = await mcp_registry.prewarm_schemas(account_id)
                if warmed > 0:
                    logger.info(f"⚑ [MCP REGISTRY] Pre-warmed {warmed} schemas from Redis cache")
        
        account_id = getattr(self.thread_manager, 'account_id', None)
        discovery_info = await mcp_registry.get_discovery_info(filter, load_schemas=True, account_id=account_id)
        
        logger.info(f"πŸ” [MCP DISCOVERY] Found {discovery_info['total_count']} MCP tools across {len(discovery_info['toolkits'])} toolkits with full schemas")
        
        return self.success_response(discovery_info)

    async def _call_tool(self, tool_name: str, args: dict) -> ToolResult:
        from core.utils.logger import logger
        
        if not tool_name:
            return self.fail_response("tool_name required for call action")
        
        if args is None:
            args = {}
        
        if isinstance(args, str):
            try:
                import json
                args = json.loads(args)
                logger.info(f"πŸ”§ [ARGS FIX] Converted string args to JSON object for {tool_name}")
            except json.JSONDecodeError:
                logger.warning(f"⚠️  [ARGS FIX] Failed to parse args string: {args}")
                args = {}
        
        native_tools = ['web_search', 'image_search', 'create_file', 'read_file', 'edit_file', 'create_slide', 'browser_navigate', 'shell_command', 'scrape_webpage']
        if tool_name in native_tools:
            return self.fail_response(f"Tool '{tool_name}' is a native tool. Use initialize_tools to load {tool_name}_tool first, then use {tool_name} directly.")

        integration_labels = {
            'TWITTER_': 'Accessing Twitter',
            'GMAIL_': 'Accessing Gmail',  
            'SLACK_': 'Accessing Slack',
            'GITHUB_': 'Accessing GitHub',
            'GOOGLESHEETS_': 'Accessing Google Sheets',
            'LINEAR_': 'Accessing Linear',
            'NOTION_': 'Accessing Notion'
        }
        
        friendly_status = f'Executing {tool_name}'
        
        from core.agentpress.mcp_registry import get_mcp_registry
        mcp_registry = get_mcp_registry()
        tool_info = mcp_registry.get_tool_info(tool_name)
        
        if tool_info and tool_info.toolkit_slug:
             toolkit_name = tool_info.toolkit_slug.replace('_', ' ').replace('-', ' ').title()
             friendly_status = f"Accessing {toolkit_name}"
        else:
             integration_labels = {
                'TWITTER_': 'Accessing Twitter',
                'GMAIL_': 'Accessing Gmail',  
                'SLACK_': 'Accessing Slack',
                'GITHUB_': 'Accessing GitHub',
                'GOOGLESHEETS_': 'Accessing Google Sheets',
                'LINEAR_': 'Accessing Linear',
                'NOTION_': 'Accessing Notion'
             }
             for prefix, label in integration_labels.items():
                if tool_name.startswith(prefix):
                    friendly_status = label
                    break
        
        logger.info(f"πŸ”§ [MCP_ACTION] {friendly_status}")
        from core.agentpress.mcp_registry import get_mcp_registry, MCPExecutionContext
        
        mcp_registry = get_mcp_registry()
        mcp_loader = getattr(self.thread_manager, 'mcp_loader', None)
        if mcp_loader:
            loader_tool_count = len(mcp_loader.tool_map) if mcp_loader.tool_map else 0
            registry_tool_count = len(mcp_registry._tools)
            
            if not mcp_registry._initialized or loader_tool_count > registry_tool_count:
                from core.agentpress.mcp_registry import init_mcp_registry_from_loader
                logger.info(f"πŸ”„ [MCP REGISTRY] Syncing registry for execute: loader has {loader_tool_count} tools, registry has {registry_tool_count}")
                init_mcp_registry_from_loader(mcp_loader)
                mcp_registry._initialized = True
                
                account_id = getattr(self.thread_manager, 'account_id', None)
                await mcp_registry.prewarm_schemas(account_id)
        
        execution_context = MCPExecutionContext(self.thread_manager)
        
        return await mcp_registry.execute_tool(tool_name, args, execution_context)

    async def _save_dynamic_tools_to_metadata(self, new_tool_names: List[str]) -> None:
        from core.utils.logger import logger
        
        try:
            client = await self.thread_manager.db.client
            
            result = await client.table('threads')\
                .select('metadata')\
                .eq('thread_id', self.thread_id)\
                .single()\
                .execute()
            
            if not result.data:
                logger.warning(f"⚠️  [DYNAMIC TOOLS] Thread {self.thread_id} not found, cannot save tools")
                return
            
            metadata = result.data.get('metadata') or {}
            
            existing_tools = set(metadata.get('dynamic_tools', []))
            updated_tools = list(existing_tools | set(new_tool_names))
            
            metadata['dynamic_tools'] = updated_tools
            
            await client.table('threads')\
                .update({'metadata': metadata})\
                .eq('thread_id', self.thread_id)\
                .execute()
            
            logger.info(f"πŸ’Ύ [DYNAMIC TOOLS] Saved {len(new_tool_names)} tools to thread metadata (total: {len(updated_tools)})")
            
        except Exception as e:
            logger.error(f"❌ [DYNAMIC TOOLS] Failed to save tools to metadata: {e}", exc_info=True)

if __name__ == "__main__":
    import asyncio

    async def test_expand_message_tool():
        expand_message_tool = ExpandMessageTool()

        expand_message_result = await expand_message_tool.expand_message(
            message_id="004ab969-ef9a-4656-8aba-e392345227cd"
        )
        print("Expand message result:", expand_message_result)

    asyncio.run(test_expand_message_tool())