Qurrah commited on
Commit
da587f2
·
1 Parent(s): 3ecbd36

REFINE: implement request gating to manage concurrent access and improve rate limiting

Browse files
Files changed (1) hide show
  1. services/query_service.py +33 -25
services/query_service.py CHANGED
@@ -1,5 +1,6 @@
1
  import logging
2
-
 
3
  from core.cache import credentials_cache, instant_reply_cache, rate_limiter
4
  from repositories.comment import insert_comment
5
  from repositories.social_account import (
@@ -11,7 +12,7 @@ from rag.query_router import classify_query_route
11
  from rag.retriever import retrieve
12
  from services.product_search_service import build_inventory_context
13
 
14
-
15
  logger = logging.getLogger(__name__)
16
 
17
 
@@ -135,27 +136,34 @@ def answer_query_request(
135
  if not query.strip():
136
  raise ValueError("Query cannot be empty")
137
 
138
- if not rate_limiter.is_allowed(channel_identifier):
139
- logger.warning(f"Rate limit hit for identifier={channel_identifier}")
140
- raise RateLimitExceeded("Too many requests — please slow down")
141
-
142
- enriched_query = query
143
- if post_caption:
144
- enriched_query = f"Post context: {post_caption}\nUser question: {query}"
145
 
146
- caption_part = post_caption.strip().lower() if post_caption else ""
147
- cache_key = f"comment:{channel_identifier}:{query.strip().lower()}:{caption_part}"
148
- cached_reply = instant_reply_cache.get(cache_key)
149
- if cached_reply:
150
- logger.info(f"Cache hit for query='{query[:60]}'")
151
- return cached_reply
152
-
153
- reply = answer(
154
- query=enriched_query,
155
- channel=channel,
156
- channel_identifier=channel_identifier,
157
- external_id=external_id,
158
- commenter_handle=commenter_handle,
159
- )
160
- instant_reply_cache.set(cache_key, reply)
161
- return reply
 
 
 
 
 
 
 
 
 
 
 
 
1
  import logging
2
+ import threading
3
+ from core.config import query_executor
4
  from core.cache import credentials_cache, instant_reply_cache, rate_limiter
5
  from repositories.comment import insert_comment
6
  from repositories.social_account import (
 
12
  from rag.retriever import retrieve
13
  from services.product_search_service import build_inventory_context
14
 
15
+ REQUEST_GATE = threading.Semaphore(20)
16
  logger = logging.getLogger(__name__)
17
 
18
 
 
136
  if not query.strip():
137
  raise ValueError("Query cannot be empty")
138
 
139
+ if not REQUEST_GATE.acquire(blocking=False):
140
+ logger.warning("Request gate full — returning fallback reply | identifier=%s", channel_identifier)
141
+ return "We're getting a lot of messages right now — we'll get back to you shortly."
 
 
 
 
142
 
143
+ try:
144
+ if not rate_limiter.is_allowed(channel_identifier):
145
+ logger.warning(f"Rate limit hit for identifier={channel_identifier}")
146
+ raise RateLimitExceeded("Too many requests — please slow down")
147
+
148
+ enriched_query = query
149
+ if post_caption:
150
+ enriched_query = f"Post context: {post_caption}\nUser question: {query}"
151
+
152
+ caption_part = post_caption.strip().lower() if post_caption else ""
153
+ cache_key = f"comment:{channel_identifier}:{query.strip().lower()}:{caption_part}"
154
+ cached_reply = instant_reply_cache.get(cache_key)
155
+ if cached_reply:
156
+ logger.info(f"Cache hit for query='{query[:60]}'")
157
+ return cached_reply
158
+
159
+ reply = answer(
160
+ query=enriched_query,
161
+ channel=channel,
162
+ channel_identifier=channel_identifier,
163
+ external_id=external_id,
164
+ commenter_handle=commenter_handle,
165
+ )
166
+ instant_reply_cache.set(cache_key, reply)
167
+ return reply
168
+ finally:
169
+ REQUEST_GATE.release()