Download core/utils/message_migration.py from Kyrlos11/nniback: direct link, hf CLI and curl.
- Browser
- Download file 25.3 kB
-
https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/message_migration.py
- Command line
-
hf download hf://spaces/Kyrlos11/nniback/core/utils/message_migration.py
-
curl -L -o message_migration.py https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/utils/message_migration.py
25.3 kB
| """ | |
| Message Migration Utility | |
| Migrates old message formats to the new unified structure: | |
| - Assistant messages: Extract tool_calls and text_content to metadata | |
| - Tool messages: Extract result to metadata | |
| Can be run: | |
| 1. Lazy (on read) - migrate messages when accessed | |
| 2. Bulk - migrate all messages in a thread or batch | |
| 3. One-time - migrate entire database | |
| """ | |
| import json | |
| import uuid | |
| import re | |
| from typing import Dict, Any, List, Optional | |
| from core.utils.logger import logger | |
| from core.agentpress.xml_tool_parser import strip_xml_tool_calls, parse_xml_tool_calls | |
| from core.utils.json_helpers import safe_json_parse | |
| def needs_migration(message: Dict[str, Any]) -> bool: | |
| """Check if a message needs migration to new format.""" | |
| msg_type = message.get('type') | |
| metadata = safe_json_parse(message.get('metadata', '{}'), {}) | |
| content = safe_json_parse(message.get('content', '{}'), {}) | |
| if msg_type == 'assistant': | |
| # Needs migration if missing tool_calls or text_content in metadata | |
| has_tool_calls = 'tool_calls' in metadata | |
| has_text_content = 'text_content' in metadata | |
| # Check if content has XML tool calls that need extraction | |
| content_str = content.get('content', '') if isinstance(content, dict) else str(content) | |
| has_xml_tool_calls = '<function_calls>' in content_str if isinstance(content_str, str) else False | |
| # Needs migration if: | |
| # 1. Missing tool_calls in metadata AND has XML tool calls in content | |
| # 2. Missing text_content in metadata AND has extractable text (not just tool calls) | |
| if has_xml_tool_calls and not has_tool_calls: | |
| return True | |
| # Only need migration for text_content if there's actual text to extract | |
| # (not just XML tool calls that would strip to empty) | |
| if not has_text_content and content_str: | |
| # Check if there's extractable text after stripping tool calls | |
| stripped = strip_xml_tool_calls(content_str) if has_xml_tool_calls else content_str | |
| if stripped.strip(): | |
| return True | |
| # Also check for native tool calls | |
| native_tool_calls = content.get('tool_calls') if isinstance(content, dict) else None | |
| if native_tool_calls and not has_tool_calls: | |
| return True | |
| elif msg_type == 'tool': | |
| # Needs migration if missing result in metadata OR has frontend_content that needs migration | |
| if 'result' not in metadata: | |
| return True | |
| # Check if frontend_content exists but result structure doesn't match new format | |
| frontend_content = metadata.get('frontend_content', {}) | |
| if frontend_content and isinstance(frontend_content, dict): | |
| tool_execution = frontend_content.get('tool_execution') | |
| if tool_execution and 'result' not in metadata: | |
| return True | |
| return False | |
| def migrate_assistant_message(message: Dict[str, Any]) -> Dict[str, Any]: | |
| """ | |
| Migrate an assistant message to new format. | |
| Extracts: | |
| - tool_calls (from XML or native format) -> metadata.tool_calls | |
| - text_content (clean text without tool calls) -> metadata.text_content | |
| Generates deterministic tool_call_ids based on message_id + function_name + index | |
| for better linking with tool messages. | |
| """ | |
| content = safe_json_parse(message.get('content', '{}'), {}) | |
| metadata = safe_json_parse(message.get('metadata', '{}'), {}) | |
| message_id = message.get('message_id', '') | |
| # Get content string | |
| content_str = content.get('content', '') if isinstance(content, dict) else str(content) | |
| if not isinstance(content_str, str): | |
| content_str = str(content_str) | |
| # Extract clean text | |
| text_content = strip_xml_tool_calls(content_str) | |
| if text_content.strip(): | |
| metadata['text_content'] = text_content | |
| # Extract tool calls | |
| unified_tool_calls = [] | |
| # 1. Extract XML tool calls (ALL old messages are XML) | |
| xml_tool_calls = parse_xml_tool_calls(content_str) | |
| for idx, xml_tc in enumerate(xml_tool_calls): | |
| function_name = xml_tc['tool_name'].replace('-', '_') | |
| xml_tag_name = xml_tc['tool_name'] | |
| # Generate tool_call_id matching response_processor.py format EXACTLY | |
| # Format: xml_tool_index{idx}_{message_id} | |
| if message_id: | |
| tool_call_id = f"xml_tool_index{idx}_{message_id}" | |
| else: | |
| # Fallback if no message_id (shouldn't happen) | |
| tool_call_id = f"xml_tool_index{idx}_{str(uuid.uuid4())}" | |
| unified_tool_calls.append({ | |
| "tool_call_id": tool_call_id, | |
| "function_name": function_name, | |
| "xml_tag_name": xml_tag_name, | |
| "arguments": xml_tc['parameters'], | |
| "source": "xml" | |
| }) | |
| # 2. Extract native tool calls (for newer messages that might have native format) | |
| native_tool_calls = content.get('tool_calls') if isinstance(content, dict) else None | |
| if native_tool_calls and isinstance(native_tool_calls, list): | |
| for idx, tc in enumerate(native_tool_calls): | |
| if isinstance(tc, dict): | |
| # Use existing ID if available | |
| tc_id = tc.get('id') | |
| if not tc_id: | |
| # Generate deterministic tool_call_id for native calls | |
| func = tc.get('function', {}) | |
| func_name = func.get('name', 'unknown') | |
| # Use UUID v5 for deterministic generation | |
| namespace = uuid.UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8') | |
| name = f"{message_id}:{func_name}:{len(xml_tool_calls) + idx}" | |
| tc_id = str(uuid.uuid5(namespace, name)) | |
| func = tc.get('function', {}) | |
| func_name = func.get('name', 'unknown') | |
| func_args = func.get('arguments', '{}') | |
| # Parse arguments | |
| parsed_args = safe_json_parse(func_args, {}) if isinstance(func_args, str) else func_args | |
| unified_tool_calls.append({ | |
| "tool_call_id": tc_id, | |
| "function_name": func_name, | |
| "arguments": parsed_args, | |
| "source": "native" | |
| }) | |
| if unified_tool_calls: | |
| metadata['tool_calls'] = unified_tool_calls | |
| return { | |
| **message, | |
| 'metadata': metadata | |
| } | |
| def migrate_tool_message(message: Dict[str, Any], assistant_messages: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]: | |
| """ | |
| Migrate a tool message to new format matching response_processor.py structure exactly. | |
| Extracts: | |
| - result -> metadata.result (ONLY success, output, error - NO function_name or tool_call_id) | |
| - return_format -> metadata.return_format ("xml" for old messages since all are XML) | |
| - tool_call_id -> metadata.tool_call_id (stored separately, NOT in result) | |
| - function_name -> metadata.function_name (stored separately, NOT in result) | |
| - assistant_message_id -> metadata.assistant_message_id (if not already present) | |
| IMPORTANT: All old thread messages are XML format, so return_format should be "xml" | |
| Args: | |
| message: Tool message to migrate | |
| assistant_messages: Optional list of assistant messages to find matching tool_call_id | |
| """ | |
| content = safe_json_parse(message.get('content', '{}'), {}) | |
| metadata = safe_json_parse(message.get('metadata', '{}'), {}) | |
| message_id = message.get('message_id') | |
| # PREFER frontend_content if available (it has both result and arguments nicely structured) | |
| frontend_content = metadata.get('frontend_content', {}) | |
| tool_execution = None | |
| if frontend_content and isinstance(frontend_content, dict): | |
| tool_execution = frontend_content.get('tool_execution') | |
| # Determine return format and extract result | |
| # ALL old messages are XML, so default to "xml" | |
| return_format = "xml" | |
| result_data = None | |
| function_name = None | |
| xml_tag_name = None | |
| if tool_execution and isinstance(tool_execution, dict): | |
| # Use frontend_content.tool_execution - this is the cleanest source | |
| result_obj = tool_execution.get('result', {}) | |
| function_name = tool_execution.get('function_name', 'unknown') | |
| xml_tag_name = tool_execution.get('xml_tag_name') | |
| tool_call_id_from_exec = tool_execution.get('tool_call_id') | |
| # Determine return format based on xml_tag_name presence (old messages are XML) | |
| if xml_tag_name: | |
| return_format = "xml" | |
| else: | |
| # If no xml_tag_name but we're migrating old message, assume XML | |
| return_format = "xml" | |
| # Build result structure matching response_processor.py format EXACTLY | |
| # ONLY success, output, error - NO function_name or tool_call_id | |
| result_data = { | |
| "success": result_obj.get('success', True), | |
| "output": result_obj.get('output', ''), | |
| "error": result_obj.get('error') | |
| } | |
| # Store tool_call_id separately in metadata (NOT in result) | |
| if tool_call_id_from_exec: | |
| metadata['tool_call_id'] = tool_call_id_from_exec | |
| elif isinstance(content, dict): | |
| # Check if it's native format (has tool_call_id in content) | |
| if 'tool_call_id' in content or (content.get('role') == 'tool' and 'tool_call_id' in content): | |
| # This is native format, but for old messages we still treat as XML | |
| return_format = "xml" # Old messages are XML | |
| function_name = content.get('name', 'unknown') | |
| content_value = content.get('content', '') | |
| # Parse content if it's a JSON string | |
| if isinstance(content_value, str): | |
| try: | |
| parsed_content = safe_json_parse(content_value, content_value) | |
| if isinstance(parsed_content, dict) and 'tool_execution' in parsed_content: | |
| # Extract from tool_execution structure | |
| tool_exec = parsed_content['tool_execution'] | |
| result_obj = tool_exec.get('result', {}) | |
| function_name = tool_exec.get('function_name', function_name) | |
| xml_tag_name = tool_exec.get('xml_tag_name') | |
| result_data = { | |
| "success": result_obj.get('success', True), | |
| "output": result_obj.get('output', ''), | |
| "error": result_obj.get('error') | |
| } | |
| else: | |
| result_data = { | |
| "success": True, | |
| "output": parsed_content, | |
| "error": None | |
| } | |
| except: | |
| result_data = { | |
| "success": True, | |
| "output": content_value, | |
| "error": None | |
| } | |
| else: | |
| result_data = { | |
| "success": True, | |
| "output": content_value, | |
| "error": None | |
| } | |
| # Store tool_call_id separately if available | |
| if content.get('tool_call_id'): | |
| metadata['tool_call_id'] = content.get('tool_call_id') | |
| else: | |
| # XML format (role="user" with content string containing tool_execution JSON) | |
| return_format = "xml" | |
| content_str = content.get('content', '') if isinstance(content, dict) else str(content) | |
| try: | |
| parsed = safe_json_parse(content_str, {}) | |
| if isinstance(parsed, dict) and 'tool_execution' in parsed: | |
| tool_exec = parsed['tool_execution'] | |
| function_name = tool_exec.get('function_name', 'unknown') | |
| xml_tag_name = tool_exec.get('xml_tag_name') | |
| result_obj = tool_exec.get('result', {}) | |
| # Build result matching response_processor.py format EXACTLY | |
| result_data = { | |
| "success": result_obj.get('success', True), | |
| "output": result_obj.get('output', ''), | |
| "error": result_obj.get('error') | |
| } | |
| if tool_exec.get('tool_call_id'): | |
| metadata['tool_call_id'] = tool_exec.get('tool_call_id') | |
| else: | |
| # Fallback: use parsed content directly as output | |
| function_name = parsed.get('function_name', 'unknown') | |
| result_data = { | |
| "success": True, | |
| "output": parsed, | |
| "error": None | |
| } | |
| except: | |
| # Fallback: use content as-is | |
| function_name = "unknown" | |
| result_data = { | |
| "success": True, | |
| "output": content_str, | |
| "error": None | |
| } | |
| else: | |
| # Content is string or other type - treat as XML (old messages are XML) | |
| return_format = "xml" | |
| function_name = "unknown" | |
| result_data = { | |
| "success": True, | |
| "output": str(content), | |
| "error": None | |
| } | |
| # Store function_name separately in metadata (NOT in result) | |
| if function_name and function_name != 'unknown': | |
| metadata['function_name'] = function_name | |
| # Try to find matching tool_call_id from assistant message | |
| assistant_message_id = metadata.get('assistant_message_id') | |
| # If we don't have assistant_message_id, try to find it by looking for nearby assistant messages | |
| if not assistant_message_id and assistant_messages: | |
| # Find the most recent assistant message before this tool message | |
| message_created_at = message.get('created_at') | |
| if message_created_at: | |
| # Sort assistant messages by created_at descending | |
| sorted_assistants = sorted( | |
| [m for m in assistant_messages if m.get('created_at')], | |
| key=lambda x: x.get('created_at', ''), | |
| reverse=True | |
| ) | |
| # Find the first assistant message created before this tool message | |
| for ass_msg in sorted_assistants: | |
| if ass_msg.get('created_at', '') < message_created_at: | |
| assistant_message_id = ass_msg.get('message_id') | |
| metadata['assistant_message_id'] = assistant_message_id | |
| break | |
| # Link tool_call_id from assistant message if we have assistant_message_id | |
| if assistant_message_id and assistant_messages: | |
| # Find the assistant message | |
| assistant_msg = next((m for m in assistant_messages if m.get('message_id') == assistant_message_id), None) | |
| if assistant_msg: | |
| assistant_metadata = safe_json_parse(assistant_msg.get('metadata', '{}'), {}) | |
| tool_calls = assistant_metadata.get('tool_calls') or [] | |
| if not isinstance(tool_calls, list): | |
| tool_calls = [] | |
| # If we don't have tool_call_id yet, try to match by function name | |
| if not metadata.get('tool_call_id') and function_name and function_name != 'unknown': | |
| # Try exact function_name match first | |
| matching_tc = next(( | |
| tc for tc in tool_calls | |
| if tc.get('function_name') == function_name | |
| ), None) | |
| # If no exact match, try xml_tag_name match (for XML tool calls) | |
| if not matching_tc and xml_tag_name: | |
| matching_tc = next(( | |
| tc for tc in tool_calls | |
| if tc.get('xml_tag_name') == xml_tag_name or | |
| tc.get('xml_tag_name') == xml_tag_name.replace('_', '-') or | |
| tc.get('function_name') == xml_tag_name.replace('-', '_') | |
| ), None) | |
| if matching_tc: | |
| tool_call_id = matching_tc.get('tool_call_id') | |
| metadata['tool_call_id'] = tool_call_id | |
| # If still no tool_call_id and there's only one tool call, use it | |
| if not metadata.get('tool_call_id') and len(tool_calls) == 1: | |
| tool_call_id = tool_calls[0].get('tool_call_id') | |
| if tool_call_id: | |
| metadata['tool_call_id'] = tool_call_id | |
| # If still no match and we have multiple tool calls with same function_name, use first match | |
| if not metadata.get('tool_call_id') and function_name and function_name != 'unknown': | |
| # Find all tool calls matching function_name | |
| matching_tool_calls = [ | |
| tc for tc in tool_calls | |
| if tc.get('function_name') == function_name or | |
| (xml_tag_name and tc.get('xml_tag_name') == xml_tag_name) | |
| ] | |
| if len(matching_tool_calls) == 1: | |
| # Only one match, use it | |
| tool_call_id = matching_tool_calls[0].get('tool_call_id') | |
| if tool_call_id: | |
| metadata['tool_call_id'] = tool_call_id | |
| elif len(matching_tool_calls) > 1: | |
| # Multiple matches - use first one (best effort) | |
| tool_call_id = matching_tool_calls[0].get('tool_call_id') | |
| if tool_call_id: | |
| metadata['tool_call_id'] = tool_call_id | |
| # Generate tool_call_id if still missing (matching response_processor.py format) | |
| if not metadata.get('tool_call_id') and assistant_message_id and function_name: | |
| # Try to find index by matching function_name in assistant message tool_calls | |
| if assistant_messages: | |
| assistant_msg = next((m for m in assistant_messages if m.get('message_id') == assistant_message_id), None) | |
| if assistant_msg: | |
| assistant_metadata = safe_json_parse(assistant_msg.get('metadata', '{}'), {}) | |
| tool_calls = assistant_metadata.get('tool_calls') or [] | |
| if not isinstance(tool_calls, list): | |
| tool_calls = [] | |
| # Find index of matching tool call | |
| for idx, tc in enumerate(tool_calls): | |
| if tc.get('function_name') == function_name or (xml_tag_name and tc.get('xml_tag_name') == xml_tag_name): | |
| # Use XML format: xml_tool_index{idx}_{assistant_message_id} | |
| if return_format == "xml": | |
| tool_call_id = f"xml_tool_index{idx}_{assistant_message_id}" | |
| else: | |
| # Native format: use UUID v5 | |
| namespace = uuid.UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8') | |
| name = f"{assistant_message_id}:{function_name}:{idx}" | |
| tool_call_id = str(uuid.uuid5(namespace, name)) | |
| metadata['tool_call_id'] = tool_call_id | |
| break | |
| else: | |
| # No match found, use index 0 | |
| if return_format == "xml": | |
| tool_call_id = f"xml_tool_index0_{assistant_message_id}" | |
| else: | |
| namespace = uuid.UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8') | |
| name = f"{assistant_message_id}:{function_name}:0" | |
| tool_call_id = str(uuid.uuid5(namespace, name)) | |
| metadata['tool_call_id'] = tool_call_id | |
| else: | |
| # No assistant message found, use index 0 | |
| if return_format == "xml": | |
| tool_call_id = f"xml_tool_index0_{assistant_message_id}" | |
| else: | |
| namespace = uuid.UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8') | |
| name = f"{assistant_message_id}:{function_name}:0" | |
| tool_call_id = str(uuid.uuid5(namespace, name)) | |
| metadata['tool_call_id'] = tool_call_id | |
| # Store result in metadata (matching response_processor.py structure EXACTLY) | |
| # result should ONLY contain success, output, error - NO function_name or tool_call_id | |
| if result_data: | |
| # Ensure result_data only has the three allowed fields | |
| metadata['result'] = { | |
| "success": result_data.get('success', True), | |
| "output": result_data.get('output', ''), | |
| "error": result_data.get('error') | |
| } | |
| metadata['return_format'] = return_format | |
| # CLEAN UP: Remove old fields that shouldn't be in the new structure | |
| # Keep only: result, tool_call_id, function_name, return_format, assistant_message_id | |
| cleaned_metadata = { | |
| 'result': metadata.get('result'), | |
| 'tool_call_id': metadata.get('tool_call_id'), | |
| 'function_name': metadata.get('function_name'), | |
| 'return_format': metadata.get('return_format'), | |
| 'assistant_message_id': metadata.get('assistant_message_id') | |
| } | |
| # Remove None values | |
| cleaned_metadata = {k: v for k, v in cleaned_metadata.items() if v is not None} | |
| return { | |
| **message, | |
| 'metadata': cleaned_metadata | |
| } | |
| def migrate_message(message: Dict[str, Any], assistant_messages: Optional[List[Dict[str, Any]]] = None) -> Optional[Dict[str, Any]]: | |
| """ | |
| Migrate a single message to new format if needed. | |
| Returns: | |
| Migrated message dict, or None if no migration needed | |
| """ | |
| if not needs_migration(message): | |
| return None | |
| msg_type = message.get('type') | |
| if msg_type == 'assistant': | |
| return migrate_assistant_message(message) | |
| elif msg_type == 'tool': | |
| return migrate_tool_message(message, assistant_messages) | |
| return None | |
| async def migrate_thread_messages(client, thread_id: str, save: bool = False) -> Dict[str, int]: | |
| """ | |
| Migrate all messages in a thread. | |
| Args: | |
| client: Database client | |
| thread_id: Thread ID to migrate | |
| save: If True, save migrated messages back to database | |
| Returns: | |
| Dict with migration stats: {'migrated': count, 'skipped': count, 'errors': count} | |
| """ | |
| stats = {'migrated': 0, 'skipped': 0, 'errors': 0} | |
| try: | |
| # Fetch all messages for thread | |
| all_messages = [] | |
| batch_size = 1000 | |
| offset = 0 | |
| while True: | |
| result = await client.table('messages').select('*').eq('thread_id', thread_id).order('created_at').range(offset, offset + batch_size - 1).execute() | |
| if not result.data: | |
| break | |
| all_messages.extend(result.data) | |
| if len(result.data) < batch_size: | |
| break | |
| offset += batch_size | |
| # Separate assistant and tool messages | |
| assistant_messages = [m for m in all_messages if m.get('type') == 'assistant'] | |
| tool_messages = [m for m in all_messages if m.get('type') == 'tool'] | |
| other_messages = [m for m in all_messages if m.get('type') not in ['assistant', 'tool']] | |
| # Migrate assistant messages first | |
| for msg in assistant_messages: | |
| try: | |
| migrated = migrate_message(msg) | |
| if migrated: | |
| if save: | |
| await client.table('messages').update({ | |
| 'metadata': migrated['metadata'] | |
| }).eq('message_id', msg['message_id']).execute() | |
| stats['migrated'] += 1 | |
| else: | |
| stats['skipped'] += 1 | |
| except Exception as e: | |
| logger.error(f"Error migrating assistant message {msg.get('message_id')}: {e}") | |
| stats['errors'] += 1 | |
| # Migrate tool messages (with access to assistant messages for matching) | |
| for msg in tool_messages: | |
| try: | |
| migrated = migrate_message(msg, assistant_messages) | |
| if migrated: | |
| if save: | |
| await client.table('messages').update({ | |
| 'metadata': migrated['metadata'] | |
| }).eq('message_id', msg['message_id']).execute() | |
| stats['migrated'] += 1 | |
| else: | |
| stats['skipped'] += 1 | |
| except Exception as e: | |
| logger.error(f"Error migrating tool message {msg.get('message_id')}: {e}") | |
| stats['errors'] += 1 | |
| logger.info(f"Migration complete for thread {thread_id}: {stats}") | |
| return stats | |
| except Exception as e: | |
| logger.error(f"Error migrating thread {thread_id}: {e}") | |
| stats['errors'] += 1 | |
| return stats | |