handle XML-style tool calls for models without native function calling

app/main.py

@@ -8,10 +8,28 @@ from .config import DATASET_CONFIGS, load_prompt_template
 from openai import OpenAI
 from openai.types.chat import ChatCompletionMessageParam
 import json
+import re
 
 # Load environment variables
 load_dotenv()
 
+def parse_xml_tool_calls(content: str) -> list[dict]:
+    """Parse <longcat_tool_call> XML tags from model output into tool call dicts."""
+    calls = []
+    pattern = r'<longcat_tool_call>\s*(\w+)\s*(.*?)</longcat_tool_call>'
+    for match in re.finditer(pattern, content, re.DOTALL):
+        name = match.group(1)
+        body = match.group(2).strip()
+        args = {}
+        arg_keys = re.findall(r'<longcat_arg_key>(\w+)</longcat_arg_key>', body)
+        arg_values = re.findall(r'<longcat_arg_value>(.*?)</longcat_arg_value>', body, re.DOTALL)
+        for i, key in enumerate(arg_keys):
+            if i < len(arg_values):
+                args[key] = arg_values[i].strip()
+        if args or True:
+            calls.append({"name": name, **args})
+    return calls
+
 # Lazy imports to avoid blocking startup
 # from .pipeline import RAGPipeline  # Will import when needed
 # import umap  # Will import when needed for visualization
@@ -161,54 +179,67 @@ async def chat_with_ai(request: ChatRequest):
         
         message = response.choices[0].message
         finish_reason = response.choices[0].finish_reason
+        content = message.content or ""
+        
+        # Check for native tool calls or XML-style tool calls in text
+        xml_tool_calls = parse_xml_tool_calls(content)
+        has_native_tool_calls = finish_reason == "tool_calls" and hasattr(message, 'tool_calls') and message.tool_calls
         
-        # Handle tool calls
-        if finish_reason == "tool_calls" and hasattr(message, 'tool_calls') and message.tool_calls:
+        if has_native_tool_calls or xml_tool_calls:
             tool_results = []
             
-            # Execute tool calls
-            for tool_call in message.tool_calls:
-                if tool_call.function and tool_call.function.name == "rag_qa":
-                    # Parse arguments
-                    args = json.loads(tool_call.function.arguments or "{}")
-                    question = args.get("question")
-                    dataset = args.get("dataset", request.dataset)
-                    
-                    # Call the rag_qa function
-                    result = rag_qa(question, dataset)
-                    tool_results.append({
-                        "tool_call_id": tool_call.id,
-                        "result": result
-                    })
-            
-            # Add tool results to conversation and get final response
-            assistant_message: ChatCompletionMessageParam = {
-                "role": "assistant",
-                "content": message.content or "",
-                "tool_calls": [
-                    {
-                        "id": tc.id,
-                        "type": tc.type,
-                        "function": {
-                            "name": tc.function.name,
-                            "arguments": tc.function.arguments
+            if has_native_tool_calls:
+                for tool_call in message.tool_calls:
+                    if tool_call.function and tool_call.function.name == "rag_qa":
+                        args = json.loads(tool_call.function.arguments or "{}")
+                        question = args.get("question")
+                        dataset = args.get("dataset", request.dataset)
+                        result = rag_qa(question, dataset)
+                        tool_results.append({"result": result, "native": True})
+                
+                assistant_message: ChatCompletionMessageParam = {
+                    "role": "assistant",
+                    "content": content,
+                    "tool_calls": [
+                        {
+                            "id": tc.id,
+                            "type": tc.type,
+                            "function": {
+                                "name": tc.function.name,
+                                "arguments": tc.function.arguments
+                            }
                         }
-                    }
-                    for tc in message.tool_calls
-                    if tc.function
-                ]
-            }
-            messages.append(assistant_message)
-            
-            for tool_result in tool_results:
-                tool_message: ChatCompletionMessageParam = {
-                    "role": "tool",
-                    "tool_call_id": tool_result["tool_call_id"],
-                    "content": tool_result["result"]
+                        for tc in message.tool_calls
+                        if tc.function
+                    ]
                 }
-                messages.append(tool_message)
+                messages.append(assistant_message)
+                
+                for tr in tool_results:
+                    messages.append({
+                        "role": "tool",
+                        "tool_call_id": "call_1",
+                        "content": tr["result"]
+                    })
+            
+            if xml_tool_calls:
+                clean_content = re.sub(r'<longcat_tool_call>.*?</longcat_tool_call>', '', content, flags=re.DOTALL).strip()
+                for tc in xml_tool_calls:
+                    if tc["name"] == "rag_qa":
+                        result = rag_qa(tc.get("question", ""), tc.get("dataset", request.dataset))
+                        tool_results.append({"result": result, "native": False})
+                
+                messages.append({
+                    "role": "assistant",
+                    "content": clean_content or "Let me look that up..."
+                })
+                
+                rag_summary = "\n\n".join([tr["result"] for tr in tool_results])
+                messages.append({
+                    "role": "user",
+                    "content": f"Here is the retrieved information:\n{rag_summary}\n\nNow provide a helpful answer based on this information."
+                })
             
-            # Get final response
             final_response = openrouter_client.chat.completions.create(
                 model=MODEL_NAME,
                 messages=messages
@@ -216,18 +247,17 @@ async def chat_with_ai(request: ChatRequest):
             
             return {
                 "response": final_response.choices[0].message.content,
-                "tool_calls": [
+                "tool_calls": xml_tool_calls or [
                     {
                         "name": tc.function.name,
                         "arguments": tc.function.arguments
                     }
                     for tc in message.tool_calls
-                ]
+                ] if has_native_tool_calls else None
             }
         else:
-            # Direct response without tool calls
             return {
-                "response": message.content,
+                "response": content,
                 "tool_calls": None
             }
             

prompts/system-instruction.txt

@@ -12,6 +12,10 @@ When you use rag_qa tool, you MUST use retrieved information to answer about the
 - 🧠 Synthesize RAG results into natural responses about him
 - 📝 Format responses with markdown for readability
 
+## 🛠 Tool Calling Format
+If you don't support native function calling, use this XML format to call the rag_qa tool:
+<longcat_tool_call>rag_qa <longcat_arg_key>question</longcat_arg_key> <longcat_arg_value>your question here</longcat_arg_value> <longcat_arg_key>dataset</longcat_arg_key> <longcat_arg_value>developer-portfolio</longcat_arg_value> </longcat_tool_call>
+
 ## ✅ Examples:
 ❌ **Wrong:** "I am a Tech Lead at FleetEnable"
 ✅ **Right:** "He is currently working as a Tech Lead at FleetEnable"