Update app/rag/rag_processor.py

app/rag/rag_processor.py

@@ -1,5 +1,7 @@
 import os
 import re
+import time
+import asyncio
 import logging
 from typing import List, Tuple, Optional
 import faiss
@@ -12,6 +14,7 @@ import io
 import openpyxl
 import pandas as pd
 from duckduckgo_search import DDGS
+from duckduckgo_search.exceptions import RatelimitException, DuckDuckGoSearchException
 from fastapi import UploadFile
 
 logger = logging.getLogger(__name__)
@@ -19,6 +22,7 @@ logger = logging.getLogger(__name__)
 _EMBED_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
 _embedder: Optional[SentenceTransformer] = None
 
+
 def _get_embedder() -> SentenceTransformer:
     global _embedder
     if _embedder is None:
@@ -26,7 +30,7 @@ def _get_embedder() -> SentenceTransformer:
         _embedder = SentenceTransformer(_EMBED_MODEL_NAME)
     return _embedder
 
-# Enhanced File Extraction
+
 def extract_text(file: UploadFile) -> str:
     ext = os.path.splitext(file.filename)[1].lower()
     content = file.file.read()
@@ -63,25 +67,27 @@ def extract_text(file: UploadFile) -> str:
         except Exception as e:
             logger.error(f"CSV extract failed: {e}")
             return ""
-    elif ext in [".jpg", ".jpeg", ".png", ".gif"]:  # OCR for images
+    elif ext in [".jpg", ".jpeg", ".png", ".gif"]:
         try:
             img = Image.open(file_bytes)
             return pytesseract.image_to_string(img)
         except Exception as e:
             logger.error(f"Image OCR failed: {e}")
             return ""
-    else:  # Fallback text
+    else:
         try:
             return content.decode("utf-8", errors="ignore")
         except Exception as e:
             logger.error(f"Text extract failed: {e}")
             return ""
 
+
 def clean_text(text: str) -> str:
     t = re.sub(r"[ \t]+", " ", text)
     t = re.sub(r"\n{3,}", "\n\n", t)
     return t.strip()
 
+
 def chunk_text(text: str, max_tokens: int = 400, overlap: int = 50) -> List[str]:
     text = clean_text(text)
     if not text:
@@ -98,12 +104,14 @@ def chunk_text(text: str, max_tokens: int = 400, overlap: int = 50) -> List[str]
         start = max(0, end - overlap)
     return chunks
 
+
 class RagIndex:
     def __init__(self, index: faiss.IndexFlatIP, dim: int, chunks: List[str]):
         self.index = index
         self.dim = dim
         self.chunks = chunks
 
+
 def build_faiss_index(chunks: List[str]) -> RagIndex:
     emb = _get_embedder()
     vectors = emb.encode(chunks, convert_to_numpy=True, normalize_embeddings=True)
@@ -112,6 +120,7 @@ def build_faiss_index(chunks: List[str]) -> RagIndex:
     index.add(vectors)
     return RagIndex(index=index, dim=dim, chunks=chunks)
 
+
 def search(index: RagIndex, query: str, top_k: int = 6) -> List[Tuple[str, float]]:
     emb = _get_embedder()
     q = emb.encode([query], convert_to_numpy=True, normalize_embeddings=True)
@@ -123,13 +132,14 @@ def search(index: RagIndex, query: str, top_k: int = 6) -> List[Tuple[str, float
         hits.append((index.chunks[idx], float(score)))
     return hits
 
+
 def build_context_from_files(files: List[UploadFile], prompt: str, top_k: int = 6) -> str:
     all_text = []
     for file in files:
         txt = extract_text(file)
         if txt:
             all_text.append(txt)
-        file.file.seek(0)  # Reset
+        file.file.seek(0)
     big_text = "\n\n".join(all_text)
     chunks = chunk_text(big_text, max_tokens=450, overlap=80)
     if not chunks:
@@ -139,13 +149,45 @@ def build_context_from_files(files: List[UploadFile], prompt: str, top_k: int =
     context_sections = [f"[DOC#{i} score={score:.3f}]\n{chunk}" for i, (chunk, score) in enumerate(hits, 1)]
     return "\n\n".join(context_sections)
 
-# Web search tool
-def web_search(query: str) -> str:
-    try:
-        with DDGS() as ddgs:
-            results = [r for r in ddgs.text(query, max_results=5)]
-        sections = [f"[WEB#{i}] Title: {r['title']}\nSnippet: {r['body']}\nURL: {r['href']}" for i, r in enumerate(results, 1)]
-        return "\n\n".join(sections) if sections else "No results found."
-    except Exception as e:
-        logger.error(f"Web search failed: {e}")
-        return "Web search error."
+
+def _web_search_sync(query: str, max_results: int = 5) -> str:
+    """
+    DDGS is an unofficial DuckDuckGo scraper with no SLA — it rate-limits
+    datacenter/cloud IPs (like Hugging Face Spaces) intermittently and
+    without warning. Retrying with backoff smooths over most transient
+    hits; it can't fix a sustained block, which is a hosting-environment
+    limitation, not a code bug.
+    """
+    last_error = None
+    for attempt in range(3):
+        try:
+            with DDGS() as ddgs:
+                results = list(ddgs.text(query, max_results=max_results, backend="lite"))
+            if not results:
+                return "No results found."
+            sections = [
+                f"[WEB#{i}] Title: {r['title']}\nSnippet: {r['body']}\nURL: {r['href']}"
+                for i, r in enumerate(results, 1)
+            ]
+            return "\n\n".join(sections)
+        except RatelimitException as e:
+            last_error = e
+            logger.warning(f"Web search rate-limited, attempt {attempt + 1}/3: {e}")
+            time.sleep(1.5 * (attempt + 1))
+        except DuckDuckGoSearchException as e:
+            last_error = e
+            logger.error(f"Web search failed: {e}")
+            break
+        except Exception as e:
+            last_error = e
+            logger.error(f"Web search unexpected error: {e}")
+            break
+
+    logger.error(f"Web search exhausted retries: {last_error}")
+    return f"[Web search unavailable: {type(last_error).__name__ if last_error else 'unknown error'}]"
+
+
+async def web_search(query: str) -> str:
+    # DDGS is synchronous/blocking — offload it so it can't freeze
+    # the event loop (and therefore every other user's request) while waiting.
+    return await asyncio.to_thread(_web_search_sync, query)