Update app/rag/routes.py

app/rag/routes.py

@@ -4,6 +4,7 @@ from fastapi.responses import StreamingResponse
 from motor.motor_asyncio import AsyncIOMotorDatabase
 from bson import ObjectId
 from groq import AsyncGroq
+import groq
 import json
 import logging
 from datetime import datetime
@@ -34,6 +35,32 @@ WEB_SEARCH_TOOL = {
     },
 }
 
+# Single source of truth for completion token budgets — both call sites
+# below read from here, so there's no way for them to drift out of sync
+# with each other again.
+MODEL_MAX_TOKENS = {
+    "llama-3.1-8b-instant": 1024,
+    "meta-llama/llama-4-maverick-17b-128e-instruct": 1024,
+    "moonshotai/kimi-k2-instruct": 1024,
+    "qwen/qwen3-32b": 1024,
+    "groq/compound": 1024,
+}
+DEFAULT_MAX_TOKENS = 500
+
+# Trim conversation history sent to Groq so old turns don't keep eating
+# into the same shared per-minute token budget as the current message.
+MAX_HISTORY_MESSAGES = 8
+
+
+def _raise_for_groq_error(e: "groq.APIStatusError"):
+    logger.error(f"Groq API error ({e.status_code}): {e.message}")
+    if e.status_code == 413 or "rate_limit" in str(e.body).lower():
+        raise HTTPException(
+            status_code=429,
+            detail="Message too long or too many requests for the selected model. Try a shorter message, a shorter conversation, or switch models.",
+        )
+    raise HTTPException(status_code=502, detail="The AI provider returned an error. Please try again.")
+
 
 @router.post("/conversations", status_code=status.HTTP_201_CREATED)
 async def create_conversation(
@@ -46,9 +73,6 @@ async def create_conversation(
     return {"conversation_id": conv.id}
 
 
-# NEW — this was the missing endpoint causing conversations to
-# "disappear" on refresh. They were never gone from Mongo; the
-# frontend just had nothing to call to list them back out.
 @router.get("/conversations")
 async def list_conversations(
     current_user: UserPublic = Depends(get_current_user),
@@ -84,7 +108,6 @@ async def get_conversation(
     return conv
 
 
-# NEW — matches the delete button added to the dashboard
 @router.delete("/conversations/{conv_id}", status_code=status.HTTP_204_NO_CONTENT)
 async def delete_conversation(
     conv_id: str,
@@ -138,21 +161,30 @@ async def send_message(
 
     client = AsyncGroq(api_key=settings.groq_api_key)
     tools = [WEB_SEARCH_TOOL] if enable_web_search else None
+    completion_tokens = MODEL_MAX_TOKENS.get(model, DEFAULT_MAX_TOKENS)
 
-    chat_history = [system_msg] + [m.dict() for m in messages]
+    # Only send the most recent N messages + system prompt — full history
+    # otherwise keeps consuming the same shared per-minute token budget
+    # as the current turn, no matter how small max_tokens is set.
+    trimmed_messages = messages[-MAX_HISTORY_MESSAGES:]
+    chat_history = [system_msg] + [m.dict() for m in trimmed_messages]
 
     max_tool_loops = 3
     for _ in range(max_tool_loops):
-        completion = await client.chat.completions.create(
-            model=model,
-            messages=chat_history,
-            temperature=1,
-            max_tokens=700,
-            top_p=1,
-            stream=False,
-            stop=None,
-            tools=tools,
-        )
+        try:
+            completion = await client.chat.completions.create(
+                model=model,
+                messages=chat_history,
+                temperature=1,
+                max_tokens=completion_tokens,
+                top_p=1,
+                stream=False,
+                stop=None,
+                tools=tools,
+            )
+        except groq.APIStatusError as e:
+            _raise_for_groq_error(e)
+
         choice = completion.choices[0].message
         if not choice.tool_calls:
             break
@@ -172,15 +204,18 @@ async def send_message(
         logger.warning("Max tool loops reached")
         raise HTTPException(status_code=500, detail="Too many tool calls")
 
-    completion = await client.chat.completions.create(
-        model=model,
-        messages=chat_history,
-        temperature=1,
-        max_tokens=8192,
-        top_p=1,
-        stream=True,
-        stop=None,
-    )
+    try:
+        completion = await client.chat.completions.create(
+            model=model,
+            messages=chat_history,
+            temperature=1,
+            max_tokens=completion_tokens,
+            top_p=1,
+            stream=True,
+            stop=None,
+        )
+    except groq.APIStatusError as e:
+        _raise_for_groq_error(e)
 
     async def generate():
         response_content = ""