Update llm_client.py

llm_client.py

@@ -1,4 +1,4 @@
-# llm_client.py – Primary: OpenRouter, Fallback: Groq, with severe truncation
+# llm_client.py – Primary: OpenRouter, Fallback: Groq, with aggressive truncation
 
 import json
 import re
@@ -23,13 +23,13 @@ class LLMClient:
         cls,
         messages: List[Dict[str, str]],
         temperature: float = 0.1,
-        max_tokens: int = 2048,          # reduced
+        max_tokens: int = 2048,
         response_format: Optional[str] = None,
         task_type: str = "default",
         track_metrics: bool = True
     ) -> Tuple[str, Dict[str, Any]]:
-        # First, truncate messages aggressively
-        truncated_messages = cls._truncate_messages(messages, max_tokens=2000)
+        # Aggressively truncate to fit token limits
+        truncated = cls._truncate_messages(messages, max_tokens=2500)
         
         metadata = {
             "model": "openrouter",
@@ -39,20 +39,19 @@ class LLMClient:
             "success": False,
         }
 
-        # Try OpenRouter first (higher limits)
+        # Try OpenRouter (higher limits)
         try:
-            result = await cls._call_openrouter(truncated_messages, temperature, max_tokens)
+            result = await cls._call_openrouter(truncated, temperature, max_tokens)
             metadata["success"] = True
             metadata["provider"] = "openrouter"
             return result, metadata
         except Exception as e:
             logger.warning(f"OpenRouter failed: {e}")
 
-        # Fallback to Groq with smaller model and even smaller context
+        # Fallback to Groq with even smaller context
         try:
-            # Further truncate for Groq's TPM
-            groq_messages = cls._truncate_messages(messages, max_tokens=1500)
-            result = await cls._call_groq(groq_messages, temperature, min(max_tokens, 1024), response_format)
+            groq_msgs = cls._truncate_messages(messages, max_tokens=1500)
+            result = await cls._call_groq(groq_msgs, temperature, min(max_tokens, 1024), response_format)
             metadata["success"] = True
             metadata["provider"] = "groq"
             metadata["fallback"] = True
@@ -68,41 +67,31 @@ class LLMClient:
         return result, metadata
 
     @classmethod
-    def _truncate_messages(cls, messages, max_tokens=2000):
-        """Aggressively truncate messages to stay under token limit."""
+    def _truncate_messages(cls, messages, max_tokens=2500):
+        """Truncate messages to stay under token limit."""
         new_messages = []
         for m in messages:
-            if m["role"] == "system":
-                # Shorten system prompt to essential instructions
-                content = m["content"]
-                # Keep only first 300 chars of system prompt
-                if len(content) > 300:
-                    content = content[:300] + "... (truncated)"
-                new_messages.append({"role": "system", "content": content})
-            elif m["role"] == "user":
-                # User messages: keep only last one, truncate to 500 chars
-                # But we keep all user messages but truncate each
-                content = m["content"]
+            role = m["role"]
+            content = m["content"]
+            if role == "system":
+                # Keep only first 500 chars of system prompt
                 if len(content) > 500:
-                    content = content[:500] + "\n... (truncated)"
-                new_messages.append({"role": "user", "content": content})
+                    content = content[:500] + "... (truncated)"
+                new_messages.append({"role": role, "content": content})
+            elif role == "user":
+                # Truncate user message to 800 chars
+                if len(content) > 800:
+                    content = content[:800] + "\n... (truncated)"
+                new_messages.append({"role": role, "content": content})
             else:
                 new_messages.append(m)
-        # Ensure total length is roughly within max_tokens (chars/4)
-        total_chars = sum(len(m.get("content", "")) for m in new_messages)
+        # If still too long, remove older user messages (keep only last one)
+        total_chars = sum(len(m["content"]) for m in new_messages)
         if total_chars / 4 > max_tokens:
-            # Remove some user messages (keep only last one)
             user_msgs = [m for m in new_messages if m["role"] == "user"]
             if len(user_msgs) > 1:
                 # Keep only the last user message
                 new_messages = [m for m in new_messages if m["role"] != "user"] + [user_msgs[-1]]
-                # Re-check
-                total_chars = sum(len(m.get("content", "")) for m in new_messages)
-                if total_chars / 4 > max_tokens:
-                    # Further truncate the last user message
-                    last_user = new_messages[-1]
-                    if len(last_user["content"]) > 300:
-                        last_user["content"] = last_user["content"][:300] + "\n... (truncated)"
         return new_messages
 
     @classmethod