Spaces:
Sleeping
Sleeping
REFINE: implement request gating to manage concurrent access and improve rate limiting
Browse files- services/query_service.py +33 -25
services/query_service.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
import logging
|
| 2 |
-
|
|
|
|
| 3 |
from core.cache import credentials_cache, instant_reply_cache, rate_limiter
|
| 4 |
from repositories.comment import insert_comment
|
| 5 |
from repositories.social_account import (
|
|
@@ -11,7 +12,7 @@ from rag.query_router import classify_query_route
|
|
| 11 |
from rag.retriever import retrieve
|
| 12 |
from services.product_search_service import build_inventory_context
|
| 13 |
|
| 14 |
-
|
| 15 |
logger = logging.getLogger(__name__)
|
| 16 |
|
| 17 |
|
|
@@ -135,27 +136,34 @@ def answer_query_request(
|
|
| 135 |
if not query.strip():
|
| 136 |
raise ValueError("Query cannot be empty")
|
| 137 |
|
| 138 |
-
if not
|
| 139 |
-
logger.warning(
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
enriched_query = query
|
| 143 |
-
if post_caption:
|
| 144 |
-
enriched_query = f"Post context: {post_caption}\nUser question: {query}"
|
| 145 |
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import logging
|
| 2 |
+
import threading
|
| 3 |
+
from core.config import query_executor
|
| 4 |
from core.cache import credentials_cache, instant_reply_cache, rate_limiter
|
| 5 |
from repositories.comment import insert_comment
|
| 6 |
from repositories.social_account import (
|
|
|
|
| 12 |
from rag.retriever import retrieve
|
| 13 |
from services.product_search_service import build_inventory_context
|
| 14 |
|
| 15 |
+
REQUEST_GATE = threading.Semaphore(20)
|
| 16 |
logger = logging.getLogger(__name__)
|
| 17 |
|
| 18 |
|
|
|
|
| 136 |
if not query.strip():
|
| 137 |
raise ValueError("Query cannot be empty")
|
| 138 |
|
| 139 |
+
if not REQUEST_GATE.acquire(blocking=False):
|
| 140 |
+
logger.warning("Request gate full — returning fallback reply | identifier=%s", channel_identifier)
|
| 141 |
+
return "We're getting a lot of messages right now — we'll get back to you shortly."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
|
| 143 |
+
try:
|
| 144 |
+
if not rate_limiter.is_allowed(channel_identifier):
|
| 145 |
+
logger.warning(f"Rate limit hit for identifier={channel_identifier}")
|
| 146 |
+
raise RateLimitExceeded("Too many requests — please slow down")
|
| 147 |
+
|
| 148 |
+
enriched_query = query
|
| 149 |
+
if post_caption:
|
| 150 |
+
enriched_query = f"Post context: {post_caption}\nUser question: {query}"
|
| 151 |
+
|
| 152 |
+
caption_part = post_caption.strip().lower() if post_caption else ""
|
| 153 |
+
cache_key = f"comment:{channel_identifier}:{query.strip().lower()}:{caption_part}"
|
| 154 |
+
cached_reply = instant_reply_cache.get(cache_key)
|
| 155 |
+
if cached_reply:
|
| 156 |
+
logger.info(f"Cache hit for query='{query[:60]}'")
|
| 157 |
+
return cached_reply
|
| 158 |
+
|
| 159 |
+
reply = answer(
|
| 160 |
+
query=enriched_query,
|
| 161 |
+
channel=channel,
|
| 162 |
+
channel_identifier=channel_identifier,
|
| 163 |
+
external_id=external_id,
|
| 164 |
+
commenter_handle=commenter_handle,
|
| 165 |
+
)
|
| 166 |
+
instant_reply_cache.set(cache_key, reply)
|
| 167 |
+
return reply
|
| 168 |
+
finally:
|
| 169 |
+
REQUEST_GATE.release()
|