Update app.py

app.py

@@ -15,6 +15,7 @@ import sys
 import concurrent.futures
 from concurrent.futures import ThreadPoolExecutor
 import threading
+from collections import deque
 
 # Import OpenAI library
 import openai
@@ -87,6 +88,44 @@ openai.api_base = "https://api.groq.com/openai/v1"
 api_lock = threading.Lock()
 last_api_call_time = 0
 
+# Rate Limiter Configuration
+RPM_LIMIT = 30  # Requests per minute
+TPM_LIMIT = 40000  # Tokens per minute
+
+# Implementing a Token Bucket Rate Limiter
+class TokenBucket:
+    def __init__(self, rate, capacity):
+        self.rate = rate  # tokens per second
+        self.capacity = capacity
+        self.tokens = capacity
+        self.timestamp = time.time()
+        self.lock = threading.Lock()
+
+    def consume(self, tokens=1):
+        with self.lock:
+            now = time.time()
+            elapsed = now - self.timestamp
+            # Refill tokens
+            refill = elapsed * self.rate
+            self.tokens = min(self.capacity, self.tokens + refill)
+            self.timestamp = now
+            if self.tokens >= tokens:
+                self.tokens -= tokens
+                return True
+            else:
+                return False
+
+    def wait_for_token(self, tokens=1):
+        while not self.consume(tokens):
+            time.sleep(0.1)
+
+# Initialize rate limiters
+rpm_rate = RPM_LIMIT / 60  # tokens per second
+tpm_rate = TPM_LIMIT / 60  # tokens per second
+
+rpm_bucket = TokenBucket(rate=rpm_rate, capacity=RPM_LIMIT)
+tpm_bucket = TokenBucket(rate=tpm_rate, capacity=TPM_LIMIT)
+
 def extract_main_content(soup):
     """
     Extract the main content from a webpage while filtering out boilerplate content.
@@ -169,16 +208,11 @@ def generate_summary_and_assign_category(bookmark):
 
     while retry_count < max_retries:
         try:
-            # Rate Limiting Logic
-            with api_lock:
-                global last_api_call_time
-                current_time = time.time()
-                elapsed = current_time - last_api_call_time
-                if elapsed < 2:
-                    sleep_duration = 2 - elapsed
-                    logger.info(f"Sleeping for {sleep_duration:.2f} seconds to respect rate limits.")
-                    time.sleep(sleep_duration)
-                last_api_call_time = time.time()
+            # Rate Limiting
+            rpm_bucket.wait_for_token()
+            # Estimate tokens: prompt + max_tokens
+            # Here, we assume max_tokens=150
+            tpm_bucket.wait_for_token(tokens=150)
 
             html_content = bookmark.get('html_content', '')
             soup = BeautifulSoup(html_content, 'html.parser')
@@ -447,7 +481,7 @@ def display_bookmarks():
             # For dead links, use 'summary' if available
             summary = bookmark.get('summary', '')
             if not summary:
-                # Optionally, you can skip setting summary or provide a default message
+                # Provide a default message or leave it empty
                 summary = 'No summary available.'
         elif bookmark.get('slow_link'):
             status = "⏳ Slow Response"
@@ -526,7 +560,7 @@ def process_uploaded_file(file, state_bookmarks):
 
     # Process bookmarks concurrently with LLM calls
     logger.info("Processing bookmarks with LLM concurrently")
-    with ThreadPoolExecutor(max_workers=1) as executor:
+    with ThreadPoolExecutor(max_workers=5) as executor:
         executor.map(generate_summary_and_assign_category, bookmarks)
 
     try:
@@ -654,15 +688,9 @@ def chatbot_response(user_query, chat_history):
     try:
         chat_history.append({"role": "user", "content": user_query})
 
-        with api_lock:
-            global last_api_call_time
-            current_time = time.time()
-            elapsed = current_time - last_api_call_time
-            if elapsed < 2:
-                sleep_duration = 2 - elapsed
-                logger.info(f"Sleeping for {sleep_duration:.2f} seconds to respect rate limits.")
-                time.sleep(sleep_duration)
-            last_api_call_time = time.time()
+        # Rate Limiting
+        rpm_bucket.wait_for_token()
+        tpm_bucket.wait_for_token(tokens=300)  # Assuming max_tokens=300
 
         query_vector = embedding_model.encode([user_query]).astype('float32')
         k = 5
@@ -670,6 +698,7 @@ def chatbot_response(user_query, chat_history):
         ids = ids.flatten()
 
         id_to_bookmark = {bookmark['id']: bookmark for bookmark in bookmarks}
+        # Filter out bookmarks without summaries
         matching_bookmarks = [id_to_bookmark.get(id) for id in ids if id in id_to_bookmark and id_to_bookmark.get(id).get('summary')]
 
         if not matching_bookmarks: