Thread lock

app.py

@@ -4,6 +4,8 @@ from pydantic import BaseModel
 import requests
 import os
 import logging
+import threading
+import time
 
 # Configure logging
 logging.basicConfig(level=logging.INFO)
@@ -25,6 +27,44 @@ OPENROUTER_API_KEY = os.environ.get("OPENROUTER_API_KEY")
 if not OPENROUTER_API_KEY:
     raise RuntimeError("OPENROUTER_API_KEY environment variable not set!")
 
+# ---------------------------------------------------------------------------
+# Global rate limiter for OpenRouter calls
+#
+# Free-tier OpenRouter models throttle/reject requests that arrive too close
+# together. Since /classify and /explain can be hit concurrently by the
+# frontend (each running in its own thread pool worker), we serialize ALL
+# outbound OpenRouter calls behind a single lock and enforce a minimum gap
+# between them. Whichever request arrives second simply waits its turn.
+# ---------------------------------------------------------------------------
+MIN_INTERVAL_SECONDS = float(os.environ.get("OPENROUTER_MIN_INTERVAL", "2.0"))
+
+_or_lock = threading.Lock()
+_last_call_time = 0.0
+
+def call_openrouter(payload: dict, headers: dict, timeout: int = 20):
+    """Thread-safe, rate-limited call to OpenRouter's chat completions endpoint.
+
+    Ensures at least MIN_INTERVAL_SECONDS has elapsed since the previous
+    OpenRouter call (across ALL endpoints) before firing this one.
+    """
+    global _last_call_time
+
+    url = "https://openrouter.ai/api/v1/chat/completions"
+
+    with _or_lock:
+        now = time.monotonic()
+        elapsed = now - _last_call_time
+        wait_for = MIN_INTERVAL_SECONDS - elapsed
+        if wait_for > 0:
+            logger.info(f"Rate limiting: waiting {wait_for:.2f}s before next OpenRouter call")
+            time.sleep(wait_for)
+
+        response = requests.post(url, headers=headers, json=payload, timeout=timeout)
+        _last_call_time = time.monotonic()
+
+    return response
+
+
 # Request schema
 class ExplainRequest(BaseModel):
     message: str
@@ -74,7 +114,6 @@ def explain(req: ExplainRequest):
         "Respond using the same language as the message."
     )
 
-    url = "https://openrouter.ai/api/v1/chat/completions"
     headers = {
         "Authorization": f"Bearer {OPENROUTER_API_KEY}",
         "Content-Type": "application/json"
@@ -89,17 +128,17 @@ def explain(req: ExplainRequest):
 
     try:
         logger.info(f"Calling OpenRouter /explain with model: {req.model_id}")
-        response = requests.post(url, headers=headers, json=payload, timeout=20)
+        response = call_openrouter(payload, headers)
         response.raise_for_status()
         result = response.json()
-        
+
         # Check for OpenRouter error in response body (even with 200 status)
         if "error" in result:
             error_detail = result.get("error", {}).get("message", str(result.get("error")))
             logger.error(f"OpenRouter returned error in /explain: {error_detail}")
             logger.error(f"Full response: {result}")
             raise HTTPException(status_code=500, detail=f"OpenRouter error: {error_detail}")
-        
+
         reply = result.get("choices", [{}])[0].get("message", {}).get("content", "").strip()
         if not reply:
             logger.error(f"OpenRouter returned empty response in /explain. Full result: {result}")
@@ -109,6 +148,8 @@ def explain(req: ExplainRequest):
     except requests.RequestException as e:
         logger.error(f"OpenRouter network error in /explain: {e}")
         raise HTTPException(status_code=500, detail=f"Error contacting OpenRouter: {e}")
+    except HTTPException:
+        raise
     except Exception as e:
         logger.error(f"Unexpected error in /explain: {e}")
         raise HTTPException(status_code=500, detail=f"Unexpected error: {e}")
@@ -129,7 +170,6 @@ def classify(req: ExplainRequest):
         'No other text.'
     )
 
-    url = "https://openrouter.ai/api/v1/chat/completions"
     headers = {
         "Authorization": f"Bearer {OPENROUTER_API_KEY}",
         "Content-Type": "application/json"
@@ -146,19 +186,19 @@ def classify(req: ExplainRequest):
 
     try:
         logger.info(f"Calling OpenRouter /classify with model: {model_id}")
-        response = requests.post(url, headers=headers, json=payload, timeout=20)
+        response = call_openrouter(payload, headers)
         response.raise_for_status()
         result = response.json()
-        
+
         # Check for OpenRouter error in response body (even with 200 status)
         if "error" in result:
             error_detail = result.get("error", {}).get("message", str(result.get("error")))
             logger.error(f"OpenRouter returned error in /classify: {error_detail}")
             logger.error(f"Full response: {result}")
             raise HTTPException(status_code=500, detail=f"OpenRouter error: {error_detail}")
-        
+
         reply = result.get("choices", [{}])[0].get("message", {}).get("content", "").strip()
-        
+
         if not reply:
             logger.warning(f"Empty content in response. Finish reason: {result.get('choices', [{}])[0].get('finish_reason')}")
             # If content is empty, try to extract from reasoning or log for debugging
@@ -167,12 +207,14 @@ def classify(req: ExplainRequest):
                 logger.warning(f"Model has reasoning but no content. This may indicate truncation.")
             logger.error(f"OpenRouter returned empty response in /classify. Full result: {result}")
             raise HTTPException(status_code=500, detail="No response from OpenRouter")
-        
+
         logger.info(f"Classification successful with model {model_id}: {reply}")
         return {"reply": reply, "status": response.status_code, "model": model_id}
     except requests.RequestException as e:
         logger.error(f"OpenRouter network error in /classify: {e}")
         raise HTTPException(status_code=500, detail=f"Error contacting OpenRouter: {e}")
+    except HTTPException:
+        raise
     except Exception as e:
         logger.error(f"Unexpected error in /classify: {e}")
         raise HTTPException(status_code=500, detail=f"Unexpected error: {e}")