suhailshafi007 commited on
Commit
31eb696
·
1 Parent(s): e690935

feat(rag): multi-tier user search RAG configuration and endpoints

Browse files
.gitignore CHANGED
@@ -59,4 +59,4 @@ data/cookies.txt
59
  .wrangler/
60
  *.db
61
  data/*.db
62
- !data/tafseers.db
 
59
  .wrangler/
60
  *.db
61
  data/*.db
62
+ data/tafseers.db
app/hadith.py CHANGED
@@ -166,46 +166,23 @@ def compute_onnx_rerank_scores(pairs: List[List[str]]) -> np.ndarray:
166
  return _local_compute_rerank(pairs)
167
 
168
  # ---------------------------------------------------------------------
169
- # 🚀 DYNAMIC NEURAL SEARCH RUNTIME CONFIGURATION (30s In-Memory TTL)
170
  # ---------------------------------------------------------------------
171
- _cached_neural_config = None
172
- _cached_neural_config_time = 0.0
 
 
 
 
 
173
 
174
- async def get_active_neural_config() -> dict:
175
- """
176
- Retrieves dynamic neural search settings (Cross-Encoder toggle, model selection, gates)
177
- with a 30-second in-memory TTL cache to eliminate database load during high concurrency.
178
- """
179
- global _cached_neural_config, _cached_neural_config_time
180
- now = time.time()
181
- if _cached_neural_config is not None and (now - _cached_neural_config_time) < 30.0:
182
- return _cached_neural_config
183
-
184
- try:
185
- from app.database import db
186
- doc = await db["system_settings"].find_one({"_id": "main"})
187
- if doc and "neural_search" in doc and isinstance(doc["neural_search"], dict):
188
- _cached_neural_config = doc["neural_search"]
189
- _cached_neural_config_time = now
190
- return _cached_neural_config
191
- except Exception as e:
192
- logger.warning(f"[NEURAL CONFIG] Could not load dynamic settings: {e}")
193
-
194
- _cached_neural_config = {
195
- "cross_encoder_enabled": True,
196
- "reranker_provider": "openrouter",
197
- "reranker_model": "cohere/rerank-v3.5",
198
- "minimum_vector_similarity": SEARCH_CONFIG["minimum_vector_similarity"],
199
- "minimum_rerank_logit": SEARCH_CONFIG["minimum_rerank_logit"],
200
- }
201
- _cached_neural_config_time = now
202
- return _cached_neural_config
203
 
204
  def invalidate_neural_config_cache():
205
- """Invalidates the in-memory neural config cache so updates apply immediately."""
206
- global _cached_neural_config, _cached_neural_config_time
207
- _cached_neural_config = None
208
- _cached_neural_config_time = 0.0
209
 
210
  # ---------------------------------------------------------------------
211
  # 🚀 Quran Audio URL Generator (Dynamic Playlists)
@@ -350,34 +327,71 @@ async def stream_direct_message(message_text: str, user_id: str, session_id: str
350
  bg_tasks.add_task(vectorize_and_save_memory, user_id, session_id, query, message_text, source, grade, narrator, True, intent, language, top_k)
351
  return StreamingResponse(generator(), media_type="application/x-ndjson")
352
 
353
- async def stream_ai_output(sys_prompt, history, query, source, grade, narrator, raw_context, user_id, session_id, bg_tasks: BackgroundTasks, is_strict: bool, intent: str, ui_language: str, top_k: int, audio_urls: list):
354
 
355
  enforced_query = f"{query}\n\n[SYSTEM COMMAND: You MUST translate and format your entire response STRICTLY in {ui_language}. Do NOT reply in the language I used above.]"
356
 
357
  messages = [{"role": "system", "content": sys_prompt}] + (history[-12:] if history else []) + [{"role": "user", "content": enforced_query}]
358
 
359
- primary_provider = "deepseek"
360
- fallback_provider = "xai"
361
- if FORCE_XAI == 1:
362
- primary_provider = "xai"
363
- fallback_provider = None
364
- elif FORCE_DEEPSEEK == 1:
365
- primary_provider = "deepseek"
366
- fallback_provider = None
 
 
 
 
367
 
368
  configs = {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
369
  "deepseek": {
370
  "url": "https://api.deepseek.com/chat/completions",
371
  "headers": {"Authorization": f"Bearer {DEEPSEEK_API_KEY}"},
372
- "json": {"model": "deepseek-chat", "messages": messages, "temperature": 0.1, "stream": True}
373
  },
374
  "xai": {
375
  "url": "https://api.x.ai/v1/chat/completions",
376
  "headers": {"Authorization": f"Bearer {XAI_API_KEY}"},
377
- "json": {"model": "grok-2-latest", "messages": messages, "temperature": 0.1, "stream": True}
378
  }
379
  }
380
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
381
  async def generator():
382
  full_text = ""
383
  success = False
@@ -580,14 +594,18 @@ async def ask_handler(payload: AskRequest, request: Request, background_tasks: B
580
  payload.session_id = f"web-{user['uid']}"
581
  payload.session_id = str(payload.session_id)[:128]
582
 
583
- # 🚀 ENFORCE PRO / DAILY FREE QUOTA VIA REDIS
584
- is_pro = bool(user.get("is_pro", False)) or (isinstance(user.get("subscription"), dict) and user.get("subscription", {}).get("is_pro", False))
585
- if not is_pro:
586
- allowed, remaining = await redis_manager.check_and_increment_daily_ai_limit(user["uid"], limit=5)
 
 
 
587
  if not allowed:
 
588
  raise HTTPException(
589
  status_code=429,
590
- detail="Daily free AI query limit (5 queries) reached. Upgrade to Sahih Islam Pro for unlimited queries.",
591
  )
592
 
593
  valid_history = [msg for msg in payload.history if msg.get("role") in ["user", "assistant"] and str(msg.get("content", "")).strip()]
@@ -836,6 +854,10 @@ async def ask_handler(payload: AskRequest, request: Request, background_tasks: B
836
  print(f" • Search scope : quran={search_quran} hadith={search_hadith} "
837
  f"hadith_books={hadith_books_selected or 'ALL'}")
838
 
 
 
 
 
839
  # 1. Execute Hadith Search
840
  if search_hadith:
841
  hadith_stage = {
@@ -843,8 +865,8 @@ async def ask_handler(payload: AskRequest, request: Request, background_tasks: B
843
  "index": "vector_index",
844
  "path": "embedding_1024",
845
  "queryVector": query_vec,
846
- "numCandidates": SEARCH_CONFIG["vector_search_scope"],
847
- "limit": SEARCH_CONFIG["max_vector_results"]
848
  }
849
  }
850
  # Restrict to the chosen hadith collections; if none chosen,
@@ -855,7 +877,7 @@ async def ask_handler(payload: AskRequest, request: Request, background_tasks: B
855
  search_tasks.append(hadith_collection.aggregate([
856
  hadith_stage,
857
  {"$addFields": {"_vscore": {"$meta": "vectorSearchScore"}}},
858
- ]).to_list(length=SEARCH_CONFIG["max_vector_results"]))
859
  else:
860
  search_tasks.append(asyncio.sleep(0, result=[]))
861
 
@@ -866,14 +888,14 @@ async def ask_handler(payload: AskRequest, request: Request, background_tasks: B
866
  "index": "quran1024_index",
867
  "path": "embedding_1024",
868
  "queryVector": query_vec,
869
- "numCandidates": SEARCH_CONFIG["vector_search_scope"],
870
- "limit": SEARCH_CONFIG["max_vector_results"]
871
  }
872
  }
873
  search_tasks.append(quran_collection.database["quran_verses"].aggregate([
874
  quran_stage,
875
  {"$addFields": {"_vscore": {"$meta": "vectorSearchScore"}}},
876
- ]).to_list(length=SEARCH_CONFIG["max_vector_results"]))
877
  else:
878
  search_tasks.append(asyncio.sleep(0, result=[]))
879
 
@@ -889,16 +911,16 @@ async def ask_handler(payload: AskRequest, request: Request, background_tasks: B
889
  unique_docs = raw_quran + raw_hadiths
890
 
891
  if unique_docs:
892
- neural_cfg = await get_active_neural_config()
893
- cross_enabled = neural_cfg.get("cross_encoder_enabled", True)
894
- min_sim = float(neural_cfg.get("minimum_vector_similarity", SEARCH_CONFIG["minimum_vector_similarity"]))
895
- min_rerank = float(neural_cfg.get("minimum_rerank_logit", SEARCH_CONFIG["minimum_rerank_logit"]))
896
- active_provider = str(neural_cfg.get("reranker_provider", "openrouter")).strip().lower()
897
- active_model = str(neural_cfg.get("reranker_model", "cohere/rerank-v3.5")).strip()
898
 
899
  scores = None
900
  if cross_enabled:
901
- print(f"📊 [STEP 4: CROSS-ENCODER RERANKING ({len(unique_docs)} candidates) Provider={active_provider}, Model={active_model}]")
902
  try:
903
  pairs = [[search_term, str(d.get('text_english', ''))[:1000]] for d in unique_docs]
904
  scores = await async_get_rerank_scores(pairs, provider=active_provider, model=active_model)
@@ -908,7 +930,7 @@ async def ask_handler(payload: AskRequest, request: Request, background_tasks: B
908
  cross_enabled = False
909
 
910
  if not cross_enabled:
911
- print(f"⚡ [STEP 4: BI-ENCODER VECTOR ONLY (Cross-Encoder Bypassed / Off, {len(unique_docs)} candidates)]")
912
  ranked_indices = sorted(
913
  range(len(unique_docs)),
914
  key=lambda i: float(unique_docs[i].get('_vscore', 0.0) or 0.0),
@@ -917,7 +939,7 @@ async def ask_handler(payload: AskRequest, request: Request, background_tasks: B
917
 
918
  seen_texts = []
919
  for i in ranked_indices:
920
- if len(final_hadiths) >= payload.top_k:
921
  break
922
 
923
  doc = unique_docs[i]
@@ -1113,7 +1135,7 @@ async def ask_handler(payload: AskRequest, request: Request, background_tasks: B
1113
  print(f"⏱️ Total pipeline processing latency prior to generation stream: {time.perf_counter() - t_start:.4f}s")
1114
  print("═"*60 + "\n")
1115
 
1116
- return await stream_ai_output(sys_prompt, payload.history, payload.query, combined_refs, combined_grades, combined_narrators, full_context, payload.user_id, payload.session_id, background_tasks, payload.strict_mode, intent, payload.language, payload.top_k, combined_audio_urls)
1117
 
1118
 
1119
  # =========================================================
 
166
  return _local_compute_rerank(pairs)
167
 
168
  # ---------------------------------------------------------------------
169
+ # 🚀 MULTI-TIER USER SEARCH RAG RUNTIME CONFIGURATION
170
  # ---------------------------------------------------------------------
171
+ from app.rag_config import (
172
+ resolve_user_tier,
173
+ get_tier_rag_config,
174
+ get_all_tier_configs,
175
+ invalidate_tier_rag_cache,
176
+ DEFAULT_TIER_CONFIGS,
177
+ )
178
 
179
+ async def get_active_neural_config(tier: str = "free") -> dict:
180
+ """Backward-compatible helper: returns active tier RAG configuration."""
181
+ return await get_tier_rag_config(tier)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
182
 
183
  def invalidate_neural_config_cache():
184
+ """Invalidates the in-memory tier RAG cache so updates apply immediately."""
185
+ invalidate_tier_rag_cache()
 
 
186
 
187
  # ---------------------------------------------------------------------
188
  # 🚀 Quran Audio URL Generator (Dynamic Playlists)
 
327
  bg_tasks.add_task(vectorize_and_save_memory, user_id, session_id, query, message_text, source, grade, narrator, True, intent, language, top_k)
328
  return StreamingResponse(generator(), media_type="application/x-ndjson")
329
 
330
+ async def stream_ai_output(sys_prompt, history, query, source, grade, narrator, raw_context, user_id, session_id, bg_tasks: BackgroundTasks, is_strict: bool, intent: str, ui_language: str, top_k: int, audio_urls: list, tier_cfg: Optional[dict] = None):
331
 
332
  enforced_query = f"{query}\n\n[SYSTEM COMMAND: You MUST translate and format your entire response STRICTLY in {ui_language}. Do NOT reply in the language I used above.]"
333
 
334
  messages = [{"role": "system", "content": sys_prompt}] + (history[-12:] if history else []) + [{"role": "user", "content": enforced_query}]
335
 
336
+ tier_cfg = tier_cfg or {}
337
+ llm_provider = str(tier_cfg.get("llm_provider", "openrouter")).strip().lower()
338
+ llm_model = str(tier_cfg.get("llm_model", "deepseek/deepseek-chat")).strip()
339
+ max_tokens = int(tier_cfg.get("max_llm_tokens", 800))
340
+
341
+ openrouter_key = os.getenv("OPENROUTER_API_KEY", "").strip()
342
+ if not openrouter_key:
343
+ try:
344
+ from app.inference_client import OPENROUTER_API_KEY
345
+ openrouter_key = OPENROUTER_API_KEY
346
+ except Exception:
347
+ openrouter_key = ""
348
 
349
  configs = {
350
+ "openrouter": {
351
+ "url": "https://openrouter.ai/api/v1/chat/completions",
352
+ "headers": {
353
+ "Authorization": f"Bearer {openrouter_key}",
354
+ "HTTP-Referer": "https://sahihislam.com",
355
+ "X-Title": "Sahih Islam AI RAG Engine",
356
+ "Content-Type": "application/json"
357
+ },
358
+ "json": {
359
+ "model": llm_model,
360
+ "messages": messages,
361
+ "temperature": 0.1,
362
+ "max_tokens": max_tokens,
363
+ "stream": True
364
+ }
365
+ },
366
  "deepseek": {
367
  "url": "https://api.deepseek.com/chat/completions",
368
  "headers": {"Authorization": f"Bearer {DEEPSEEK_API_KEY}"},
369
+ "json": {"model": "deepseek-chat", "messages": messages, "temperature": 0.1, "max_tokens": max_tokens, "stream": True}
370
  },
371
  "xai": {
372
  "url": "https://api.x.ai/v1/chat/completions",
373
  "headers": {"Authorization": f"Bearer {XAI_API_KEY}"},
374
+ "json": {"model": "grok-2-latest", "messages": messages, "temperature": 0.1, "max_tokens": max_tokens, "stream": True}
375
  }
376
  }
377
 
378
+ # Resilient provider routing
379
+ if llm_provider == "openrouter" and openrouter_key:
380
+ primary_provider = "openrouter"
381
+ fallback_provider = "deepseek" if DEEPSEEK_API_KEY else ("xai" if XAI_API_KEY else None)
382
+ elif llm_provider == "xai" and XAI_API_KEY:
383
+ primary_provider = "xai"
384
+ fallback_provider = "openrouter" if openrouter_key else "deepseek"
385
+ elif llm_provider == "deepseek" and DEEPSEEK_API_KEY:
386
+ primary_provider = "deepseek"
387
+ fallback_provider = "openrouter" if openrouter_key else "xai"
388
+ elif openrouter_key:
389
+ primary_provider = "openrouter"
390
+ fallback_provider = "deepseek" if DEEPSEEK_API_KEY else "xai"
391
+ else:
392
+ primary_provider = "deepseek"
393
+ fallback_provider = "xai"
394
+
395
  async def generator():
396
  full_text = ""
397
  success = False
 
594
  payload.session_id = f"web-{user['uid']}"
595
  payload.session_id = str(payload.session_id)[:128]
596
 
597
+ # 🚀 ENFORCE PRO / DAILY FREE QUOTA VIA REDIS (DYNAMIC TIER PROFILE)
598
+ user_tier = resolve_user_tier(user)
599
+ tier_cfg = await get_tier_rag_config(user_tier)
600
+
601
+ daily_limit = int(tier_cfg.get("daily_query_limit", 10))
602
+ if daily_limit > 0:
603
+ allowed, remaining = await redis_manager.check_and_increment_daily_ai_limit(user["uid"], limit=daily_limit)
604
  if not allowed:
605
+ tier_name = tier_cfg.get("name", f"{user_tier.upper()} Tier")
606
  raise HTTPException(
607
  status_code=429,
608
+ detail=f"Daily AI search limit ({daily_limit} queries) reached for {tier_name}. Upgrade to Sahih Islam Pro for higher limits.",
609
  )
610
 
611
  valid_history = [msg for msg in payload.history if msg.get("role") in ["user", "assistant"] and str(msg.get("content", "")).strip()]
 
854
  print(f" • Search scope : quran={search_quran} hadith={search_hadith} "
855
  f"hadith_books={hadith_books_selected or 'ALL'}")
856
 
857
+ # Dynamic candidate scope and limits per user tier
858
+ scope = int(tier_cfg.get("num_candidates", SEARCH_CONFIG["vector_search_scope"]))
859
+ limit_docs = max(30, int(tier_cfg.get("top_k", 5)) * 10)
860
+
861
  # 1. Execute Hadith Search
862
  if search_hadith:
863
  hadith_stage = {
 
865
  "index": "vector_index",
866
  "path": "embedding_1024",
867
  "queryVector": query_vec,
868
+ "numCandidates": scope,
869
+ "limit": limit_docs
870
  }
871
  }
872
  # Restrict to the chosen hadith collections; if none chosen,
 
877
  search_tasks.append(hadith_collection.aggregate([
878
  hadith_stage,
879
  {"$addFields": {"_vscore": {"$meta": "vectorSearchScore"}}},
880
+ ]).to_list(length=limit_docs))
881
  else:
882
  search_tasks.append(asyncio.sleep(0, result=[]))
883
 
 
888
  "index": "quran1024_index",
889
  "path": "embedding_1024",
890
  "queryVector": query_vec,
891
+ "numCandidates": scope,
892
+ "limit": limit_docs
893
  }
894
  }
895
  search_tasks.append(quran_collection.database["quran_verses"].aggregate([
896
  quran_stage,
897
  {"$addFields": {"_vscore": {"$meta": "vectorSearchScore"}}},
898
+ ]).to_list(length=limit_docs))
899
  else:
900
  search_tasks.append(asyncio.sleep(0, result=[]))
901
 
 
911
  unique_docs = raw_quran + raw_hadiths
912
 
913
  if unique_docs:
914
+ cross_enabled = bool(tier_cfg.get("cross_encoder_enabled", True))
915
+ min_sim = float(tier_cfg.get("minimum_vector_similarity", SEARCH_CONFIG["minimum_vector_similarity"]))
916
+ min_rerank = float(tier_cfg.get("minimum_rerank_logit", SEARCH_CONFIG["minimum_rerank_logit"]))
917
+ active_provider = str(tier_cfg.get("reranker_provider", "openrouter")).strip().lower()
918
+ active_model = str(tier_cfg.get("reranker_model", "cohere/rerank-v3.5")).strip()
919
+ effective_top_k = min(payload.top_k, int(tier_cfg.get("top_k", 5)))
920
 
921
  scores = None
922
  if cross_enabled:
923
+ print(f"📊 [STEP 4: CROSS-ENCODER RERANKING ({len(unique_docs)} candidates) Tier={user_tier.upper()}, Provider={active_provider}, Model={active_model}]")
924
  try:
925
  pairs = [[search_term, str(d.get('text_english', ''))[:1000]] for d in unique_docs]
926
  scores = await async_get_rerank_scores(pairs, provider=active_provider, model=active_model)
 
930
  cross_enabled = False
931
 
932
  if not cross_enabled:
933
+ print(f"⚡ [STEP 4: BI-ENCODER VECTOR ONLY (Cross-Encoder Bypassed / Off, Tier={user_tier.upper()}, {len(unique_docs)} candidates)]")
934
  ranked_indices = sorted(
935
  range(len(unique_docs)),
936
  key=lambda i: float(unique_docs[i].get('_vscore', 0.0) or 0.0),
 
939
 
940
  seen_texts = []
941
  for i in ranked_indices:
942
+ if len(final_hadiths) >= effective_top_k:
943
  break
944
 
945
  doc = unique_docs[i]
 
1135
  print(f"⏱️ Total pipeline processing latency prior to generation stream: {time.perf_counter() - t_start:.4f}s")
1136
  print("═"*60 + "\n")
1137
 
1138
+ return await stream_ai_output(sys_prompt, payload.history, payload.query, combined_refs, combined_grades, combined_narrators, full_context, payload.user_id, payload.session_id, background_tasks, payload.strict_mode, intent, payload.language, payload.top_k, combined_audio_urls, tier_cfg=tier_cfg)
1139
 
1140
 
1141
  # =========================================================
app/rag_config.py ADDED
@@ -0,0 +1,336 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ RAG Configuration & Tier Management Module
3
+ =========================================
4
+ Centralized Single Source of Truth for:
5
+ - User Tier Resolution (Free, Pro, VIP)
6
+ - Per-Tier Cross-Encoder & Bi-Encoder Settings
7
+ - Per-Tier OpenRouter LLM Model Selection
8
+ - Per-Tier Rate Limits, Candidate Scopes, and Relevance Gates
9
+ - High-Performance 30-Second In-Memory TTL Cache
10
+ """
11
+
12
+ import os
13
+ import time
14
+ import logging
15
+ from typing import Dict, Any, List, Optional
16
+
17
+ logger = logging.getLogger("sahihislam.rag_config")
18
+
19
+ # =========================================================
20
+ # 1. CATALOGS: RERANKERS & LLM MODELS
21
+ # =========================================================
22
+
23
+ AVAILABLE_RERANKERS: List[Dict[str, str]] = [
24
+ {
25
+ "id": "openrouter:cohere/rerank-v3.5",
26
+ "provider": "openrouter",
27
+ "model": "cohere/rerank-v3.5",
28
+ "name": "Cohere Rerank v3.5 (OpenRouter)",
29
+ "description": "Enterprise-grade semantic reranking (~10 paise/search). Highest precision.",
30
+ "badge": "Recommended",
31
+ },
32
+ {
33
+ "id": "openrouter:voyageai/rerank-2.5-lite",
34
+ "provider": "openrouter",
35
+ "model": "voyageai/rerank-2.5-lite",
36
+ "name": "Voyage AI Rerank 2.5 Lite (OpenRouter)",
37
+ "description": "Ultra-fast, lower cost cross-encoder on OpenRouter.",
38
+ "badge": "Fast",
39
+ },
40
+ {
41
+ "id": "jina:jina-reranker-v2-base-multilingual",
42
+ "provider": "jina",
43
+ "model": "jina-reranker-v2-base-multilingual",
44
+ "name": "Jina AI Multilingual v2",
45
+ "description": "1M free tokens/mo. Exceptional Arabic, Urdu, and English capability.",
46
+ "badge": "Budget Friendly",
47
+ },
48
+ {
49
+ "id": "cohere:rerank-v3.5",
50
+ "provider": "cohere",
51
+ "model": "rerank-v3.5",
52
+ "name": "Cohere Direct API",
53
+ "description": "Direct Cohere API endpoint using COHERE_API_KEY.",
54
+ "badge": "Direct",
55
+ },
56
+ {
57
+ "id": "local:cross-encoder/ms-marco-MiniLM-L-6-v2",
58
+ "provider": "local",
59
+ "model": "cross-encoder/ms-marco-MiniLM-L-6-v2",
60
+ "name": "Local C++ ONNX Cross-Encoder",
61
+ "description": "Zero API costs. Runs locally on container CPU/RAM.",
62
+ "badge": "Offline/Free",
63
+ },
64
+ ]
65
+
66
+ AVAILABLE_LLM_MODELS: List[Dict[str, str]] = [
67
+ {
68
+ "id": "google/gemini-2.0-flash-001",
69
+ "provider": "openrouter",
70
+ "name": "Gemini 2.0 Flash (OpenRouter)",
71
+ "description": "Sub-second latency, near-zero cost, excellent multilingual formatting.",
72
+ "badge": "Best for Free",
73
+ },
74
+ {
75
+ "id": "deepseek/deepseek-chat",
76
+ "provider": "openrouter",
77
+ "name": "DeepSeek V3 / Chat (OpenRouter)",
78
+ "description": "Top-tier scholarly reasoning, authentic Arabic understanding, low cost.",
79
+ "badge": "Best for Pro",
80
+ },
81
+ {
82
+ "id": "anthropic/claude-3.5-sonnet",
83
+ "provider": "openrouter",
84
+ "name": "Claude 3.5 Sonnet (OpenRouter)",
85
+ "description": "Gold standard nuance, authoritative synthesis, zero-hallucination compliance.",
86
+ "badge": "Best for VIP",
87
+ },
88
+ {
89
+ "id": "meta-llama/llama-3.3-70b-instruct",
90
+ "provider": "openrouter",
91
+ "name": "Llama 3.3 70B Instruct (OpenRouter)",
92
+ "description": "Leading open-weights model with strong analytical performance.",
93
+ "badge": "Open Weights",
94
+ },
95
+ {
96
+ "id": "openai/gpt-4o-mini",
97
+ "provider": "openrouter",
98
+ "name": "GPT-4o Mini (OpenRouter)",
99
+ "description": "Ultra-reliable, high speed, and standard consistency.",
100
+ "badge": "Balanced",
101
+ },
102
+ {
103
+ "id": "mistralai/mistral-large-2411",
104
+ "provider": "openrouter",
105
+ "name": "Mistral Large 2411 (OpenRouter)",
106
+ "description": "Enterprise multilingual reasoning with European privacy standards.",
107
+ "badge": "Multilingual",
108
+ },
109
+ {
110
+ "id": "deepseek/deepseek-r1",
111
+ "provider": "openrouter",
112
+ "name": "DeepSeek R1 (OpenRouter)",
113
+ "description": "Deep reasoning and step-by-step chain of thought.",
114
+ "badge": "Reasoning",
115
+ },
116
+ {
117
+ "id": "deepseek-direct",
118
+ "provider": "deepseek",
119
+ "name": "DeepSeek Direct API",
120
+ "description": "Direct connection to api.deepseek.com using DEEPSEEK_API_KEY.",
121
+ "badge": "Direct",
122
+ },
123
+ {
124
+ "id": "xai-direct",
125
+ "provider": "xai",
126
+ "name": "xAI Grok Direct API",
127
+ "description": "Direct connection to api.x.ai using XAI_API_KEY.",
128
+ "badge": "Direct",
129
+ },
130
+ ]
131
+
132
+ # =========================================================
133
+ # 2. DEFAULT CONFIGURATIONS PER TIER
134
+ # =========================================================
135
+
136
+ DEFAULT_TIER_CONFIGS: Dict[str, Dict[str, Any]] = {
137
+ "free": {
138
+ "tier_id": "free",
139
+ "name": "Free Tier",
140
+ "badge": "Standard",
141
+ # Cross-Encoder Switch: Disabled to save 99% cost while maintaining high speed
142
+ "cross_encoder_enabled": False,
143
+ "reranker_provider": "openrouter",
144
+ "reranker_model": "cohere/rerank-v3.5",
145
+ # LLM Synthesis Model: High-speed, economical model
146
+ "llm_provider": "openrouter",
147
+ "llm_model": "google/gemini-2.0-flash-001",
148
+ # Search & Rate Limits
149
+ "daily_query_limit": 10,
150
+ "top_k": 3,
151
+ "num_candidates": 100,
152
+ "minimum_vector_similarity": 0.52,
153
+ "minimum_rerank_logit": -1.75,
154
+ # Feature Entitlements
155
+ "context_expansion_enabled": False,
156
+ "audio_generation_enabled": False,
157
+ "max_llm_tokens": 400,
158
+ },
159
+ "pro": {
160
+ "tier_id": "pro",
161
+ "name": "Pro Tier",
162
+ "badge": "Pro Member",
163
+ # Cross-Encoder Switch: Enabled for enterprise precision
164
+ "cross_encoder_enabled": True,
165
+ "reranker_provider": "openrouter",
166
+ "reranker_model": "cohere/rerank-v3.5",
167
+ # LLM Synthesis Model: DeepSeek V3 for scholarly depth
168
+ "llm_provider": "openrouter",
169
+ "llm_model": "deepseek/deepseek-chat",
170
+ # Search & Rate Limits
171
+ "daily_query_limit": 100,
172
+ "top_k": 5,
173
+ "num_candidates": 250,
174
+ "minimum_vector_similarity": 0.58,
175
+ "minimum_rerank_logit": -1.75,
176
+ # Feature Entitlements
177
+ "context_expansion_enabled": True,
178
+ "audio_generation_enabled": True,
179
+ "max_llm_tokens": 800,
180
+ },
181
+ "vip": {
182
+ "tier_id": "vip",
183
+ "name": "VIP / Scholar Tier",
184
+ "badge": "VIP / Scholar",
185
+ # Cross-Encoder Switch: Enabled with strictest precision
186
+ "cross_encoder_enabled": True,
187
+ "reranker_provider": "openrouter",
188
+ "reranker_model": "cohere/rerank-v3.5",
189
+ # LLM Synthesis Model: Claude 3.5 Sonnet for supreme Islamic nuance
190
+ "llm_provider": "openrouter",
191
+ "llm_model": "anthropic/claude-3.5-sonnet",
192
+ # Search & Rate Limits (0 = Unlimited)
193
+ "daily_query_limit": 0,
194
+ "top_k": 10,
195
+ "num_candidates": 500,
196
+ "minimum_vector_similarity": 0.60,
197
+ "minimum_rerank_logit": -1.50,
198
+ # Feature Entitlements
199
+ "context_expansion_enabled": True,
200
+ "audio_generation_enabled": True,
201
+ "max_llm_tokens": 1500,
202
+ },
203
+ }
204
+
205
+ # =========================================================
206
+ # 3. HIGH-SPEED IN-MEMORY TTL CACHE (30s)
207
+ # =========================================================
208
+
209
+ _cached_all_tiers: Optional[Dict[str, Dict[str, Any]]] = None
210
+ _cached_timestamp: float = 0.0
211
+ _CACHE_TTL_SECONDS: float = 30.0
212
+
213
+
214
+ def resolve_user_tier(user: Optional[Dict[str, Any]]) -> str:
215
+ """
216
+ Resolves the user's tier ('vip', 'pro', or 'free') authoritatively.
217
+ Checks user flags, role, and subscription sub-document.
218
+ """
219
+ if not user:
220
+ return "free"
221
+
222
+ # 1. VIP / SuperAdmin privileges
223
+ if (
224
+ user.get("role") in ["SUPERADMIN", "VIP"]
225
+ or user.get("is_vip") is True
226
+ or user.get("superadmin") is True
227
+ ):
228
+ return "vip"
229
+
230
+ sub = user.get("subscription")
231
+ if isinstance(sub, dict):
232
+ if sub.get("tier") in ["VIP", "ENTERPRISE"] or sub.get("is_vip") is True:
233
+ return "vip"
234
+ if sub.get("is_pro") is True or sub.get("tier") == "PRO":
235
+ return "pro"
236
+
237
+ # 2. Pro membership
238
+ if user.get("is_pro") is True or user.get("tier") == "PRO":
239
+ return "pro"
240
+
241
+ return "free"
242
+
243
+
244
+ async def get_all_tier_configs() -> Dict[str, Dict[str, Any]]:
245
+ """
246
+ Fetches the complete multi-tier configuration dictionary with 30s TTL caching.
247
+ Ensures default values exist for all tiers ('free', 'pro', 'vip').
248
+ """
249
+ global _cached_all_tiers, _cached_timestamp
250
+ now = time.time()
251
+
252
+ if _cached_all_tiers is not None and (now - _cached_timestamp) < _CACHE_TTL_SECONDS:
253
+ return _cached_all_tiers
254
+
255
+ try:
256
+ from app.database import db
257
+
258
+ doc = await db["system_settings"].find_one({"_id": "main"})
259
+ stored_tiers = (doc or {}).get("user_search_tiers")
260
+
261
+ merged = {}
262
+ for tier_key, default_cfg in DEFAULT_TIER_CONFIGS.items():
263
+ tier_data = stored_tiers.get(tier_key, {}) if isinstance(stored_tiers, dict) else {}
264
+ merged[tier_key] = {**default_cfg, **tier_data}
265
+
266
+ _cached_all_tiers = merged
267
+ _cached_timestamp = now
268
+ return _cached_all_tiers
269
+ except Exception as e:
270
+ logger.warning(f"[RAG CONFIG] Failed to read user_search_tiers from DB: {e}")
271
+ return DEFAULT_TIER_CONFIGS
272
+
273
+
274
+ async def get_tier_rag_config(tier_name: str = "free") -> Dict[str, Any]:
275
+ """
276
+ Fetches the active RAG search configuration for a specific user tier.
277
+ Falls back gracefully if the tier name is unrecognized.
278
+ """
279
+ all_tiers = await get_all_tier_configs()
280
+ normalized = tier_name.strip().lower()
281
+ if normalized in all_tiers:
282
+ return all_tiers[normalized]
283
+ return all_tiers.get("free", DEFAULT_TIER_CONFIGS["free"])
284
+
285
+
286
+ async def update_all_tier_configs(new_configs: Dict[str, Any]) -> Dict[str, Any]:
287
+ """
288
+ Updates the multi-tier RAG search configuration in MongoDB and
289
+ instantly invalidates the in-memory cache across workers.
290
+ """
291
+ global _cached_all_tiers, _cached_timestamp
292
+
293
+ from app.database import db
294
+
295
+ sanitized_tiers: Dict[str, Dict[str, Any]] = {}
296
+
297
+ for tier_key in ["free", "pro", "vip"]:
298
+ incoming = new_configs.get(tier_key, {})
299
+ default = DEFAULT_TIER_CONFIGS[tier_key]
300
+
301
+ sanitized_tiers[tier_key] = {
302
+ "tier_id": tier_key,
303
+ "name": str(incoming.get("name", default["name"])),
304
+ "badge": str(incoming.get("badge", default["badge"])),
305
+ "cross_encoder_enabled": bool(incoming.get("cross_encoder_enabled", default["cross_encoder_enabled"])),
306
+ "reranker_provider": str(incoming.get("reranker_provider", default["reranker_provider"])).strip().lower(),
307
+ "reranker_model": str(incoming.get("reranker_model", default["reranker_model"])).strip(),
308
+ "llm_provider": str(incoming.get("llm_provider", default["llm_provider"])).strip().lower(),
309
+ "llm_model": str(incoming.get("llm_model", default["llm_model"])).strip(),
310
+ "daily_query_limit": int(incoming.get("daily_query_limit", default["daily_query_limit"])),
311
+ "top_k": int(incoming.get("top_k", default["top_k"])),
312
+ "num_candidates": int(incoming.get("num_candidates", default["num_candidates"])),
313
+ "minimum_vector_similarity": float(incoming.get("minimum_vector_similarity", default["minimum_vector_similarity"])),
314
+ "minimum_rerank_logit": float(incoming.get("minimum_rerank_logit", default["minimum_rerank_logit"])),
315
+ "context_expansion_enabled": bool(incoming.get("context_expansion_enabled", default["context_expansion_enabled"])),
316
+ "audio_generation_enabled": bool(incoming.get("audio_generation_enabled", default["audio_generation_enabled"])),
317
+ "max_llm_tokens": int(incoming.get("max_llm_tokens", default["max_llm_tokens"])),
318
+ }
319
+
320
+ await db["system_settings"].update_one(
321
+ {"_id": "main"},
322
+ {"$set": {"user_search_tiers": sanitized_tiers}},
323
+ upsert=True,
324
+ )
325
+
326
+ _cached_all_tiers = sanitized_tiers
327
+ _cached_timestamp = time.time()
328
+ logger.info("[RAG CONFIG] User search tiers updated successfully in DB.")
329
+ return sanitized_tiers
330
+
331
+
332
+ def invalidate_tier_rag_cache() -> None:
333
+ """Invalidates the in-memory tier configuration cache."""
334
+ global _cached_all_tiers, _cached_timestamp
335
+ _cached_all_tiers = None
336
+ _cached_timestamp = 0.0
app/superadmin_routes.py CHANGED
@@ -2516,6 +2516,88 @@ async def update_neural_search_settings(
2516
  raise HTTPException(status_code=500, detail="Failed to update neural search settings")
2517
 
2518
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2519
  @router.get('/db/stats')
2520
  async def get_database_stats(admin: dict = Depends(require_superadmin)):
2521
  """Get database collection statistics."""
 
2516
  raise HTTPException(status_code=500, detail="Failed to update neural search settings")
2517
 
2518
 
2519
+ # =========================================================
2520
+ # MULTI-TIER USER SEARCH RAG CONFIGURATION
2521
+ # =========================================================
2522
+ @router.get('/search-tiers')
2523
+ async def get_search_tiers(admin: dict = Depends(require_superadmin)):
2524
+ """Get active search & RAG settings across all user tiers."""
2525
+ try:
2526
+ from app.rag_config import (
2527
+ get_all_tier_configs,
2528
+ AVAILABLE_RERANKERS,
2529
+ AVAILABLE_LLM_MODELS,
2530
+ DEFAULT_TIER_CONFIGS,
2531
+ )
2532
+ tiers = await get_all_tier_configs()
2533
+ return {
2534
+ "status": "ok",
2535
+ "tiers": tiers,
2536
+ "defaults": DEFAULT_TIER_CONFIGS,
2537
+ "available_rerankers": AVAILABLE_RERANKERS,
2538
+ "available_llm_models": AVAILABLE_LLM_MODELS,
2539
+ }
2540
+ except Exception as e:
2541
+ logger.error(f"Get search tiers error: {str(e)}")
2542
+ raise HTTPException(status_code=500, detail="Failed to fetch search tier configurations")
2543
+
2544
+
2545
+ @router.put('/search-tiers')
2546
+ async def update_search_tiers(
2547
+ payload: dict,
2548
+ admin: dict = Depends(require_superadmin)
2549
+ ):
2550
+ """Update search & RAG settings across all user tiers."""
2551
+ try:
2552
+ from app.rag_config import update_all_tier_configs
2553
+ tiers_payload = payload.get("tiers", payload)
2554
+ updated_tiers = await update_all_tier_configs(tiers_payload)
2555
+
2556
+ await create_audit_log(
2557
+ action="update",
2558
+ entity_type="system",
2559
+ admin_uid=admin["uid"],
2560
+ entity_id="user_search_tiers",
2561
+ details={"tiers_updated": list(updated_tiers.keys())}
2562
+ )
2563
+
2564
+ logger.info("superadmin: admin=%s updated search tier configs", admin["uid"])
2565
+ return {
2566
+ "status": "ok",
2567
+ "message": "Search tier configurations updated successfully",
2568
+ "tiers": updated_tiers
2569
+ }
2570
+ except Exception as e:
2571
+ logger.error(f"Update search tiers error: {str(e)}")
2572
+ raise HTTPException(status_code=500, detail="Failed to update search tier configurations")
2573
+
2574
+
2575
+ @router.post('/search-tiers/reset')
2576
+ async def reset_search_tiers(admin: dict = Depends(require_superadmin)):
2577
+ """Reset all search tiers to factory recommended optimal defaults."""
2578
+ try:
2579
+ from app.rag_config import update_all_tier_configs, DEFAULT_TIER_CONFIGS
2580
+ reset_tiers = await update_all_tier_configs(DEFAULT_TIER_CONFIGS)
2581
+
2582
+ await create_audit_log(
2583
+ action="reset",
2584
+ entity_type="system",
2585
+ admin_uid=admin["uid"],
2586
+ entity_id="user_search_tiers_reset",
2587
+ details={"reset_to": "factory_defaults"}
2588
+ )
2589
+
2590
+ logger.info("superadmin: admin=%s reset search tier configs to defaults", admin["uid"])
2591
+ return {
2592
+ "status": "ok",
2593
+ "message": "Search tier configurations restored to optimal defaults",
2594
+ "tiers": reset_tiers
2595
+ }
2596
+ except Exception as e:
2597
+ logger.error(f"Reset search tiers error: {str(e)}")
2598
+ raise HTTPException(status_code=500, detail="Failed to reset search tier configurations")
2599
+
2600
+
2601
  @router.get('/db/stats')
2602
  async def get_database_stats(admin: dict = Depends(require_superadmin)):
2603
  """Get database collection statistics."""
scripts/upload_translation_audio_to_r2.py CHANGED
@@ -1,5 +1,5 @@
1
  """
2
- Upload Quran Translation Audio files to Cloudflare R2 bucket
3
  Target: sahihislam/R2/Qtranslation_mp3/
4
  Structure:
5
  - R2/Qtranslation_mp3/english_sahih/001/ to 114/
@@ -10,62 +10,82 @@ import sys
10
  import time
11
  from pathlib import Path
12
  from concurrent.futures import ThreadPoolExecutor, as_completed
13
- import requests
 
 
14
 
15
  ROOT_DIR = Path(__file__).parent.parent.parent
16
  AUDIO_DIR = ROOT_DIR / "BACKEND" / "data" / "translation_audio"
17
  ACCOUNT_ID = "d7fc285dc6825480f6d8c887f93e7084"
18
  BUCKET_NAME = "sahihislam"
19
  R2_BASE_FOLDER = "R2/Qtranslation_mp3"
 
20
 
21
 
22
- def get_auth_token() -> str:
23
- """Retrieve authenticated Cloudflare token from wrangler config or .env"""
24
- wrangler_toml = Path(r"C:\Users\suhai\AppData\Roaming\xdg.config\.wrangler\config\default.toml")
25
- if wrangler_toml.exists():
26
- try:
27
- import toml
28
- with open(wrangler_toml, "r", encoding="utf-8") as f:
29
- data = toml.load(f)
30
- token = data.get("oauth_token")
31
- if token:
32
- return token
33
- except Exception as e:
34
- print(f"Warning: could not read token from wrangler default.toml: {e}")
35
-
36
- # Fallback to .env
37
  from dotenv import load_dotenv
38
  load_dotenv(str(ROOT_DIR / "BACKEND" / ".env"))
39
- token = os.getenv("CLOUDFLARE_API_TOKEN")
40
- if token:
41
- return token
42
-
43
- raise RuntimeError("No valid Cloudflare authentication token found.")
44
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
 
46
- def upload_single_file(file_path: Path, r2_key: str, token: str, retries: int = 3) -> tuple:
47
- """Upload a single MP3 file to R2"""
48
- url = f"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/r2/buckets/{BUCKET_NAME}/objects/{r2_key}"
49
- headers = {
50
- "Authorization": f"Bearer {token}",
51
- "Content-Type": "audio/mpeg"
52
- }
53
 
54
- file_bytes = file_path.read_bytes()
55
- size_kb = round(len(file_bytes) / 1024, 2)
 
56
 
57
  for attempt in range(1, retries + 1):
58
  try:
59
- res = requests.put(url, headers=headers, data=file_bytes, timeout=60)
60
- if res.status_code == 200:
61
- return (True, r2_key, size_kb, None)
62
- else:
63
- if attempt == retries:
64
- return (False, r2_key, size_kb, f"HTTP {res.status_code}: {res.text[:200]}")
 
 
 
 
 
 
 
65
  except Exception as e:
66
  if attempt == retries:
67
- return (False, r2_key, size_kb, str(e)[:200])
68
- time.sleep(1)
69
 
70
  return (False, r2_key, size_kb, "Max retries exceeded")
71
 
@@ -97,38 +117,29 @@ def collect_audio_files() -> list:
97
  return files_to_upload
98
 
99
 
100
- def verify_r2_contents(token: str, prefix: str) -> list:
101
- """List all objects with given prefix to verify completeness"""
102
- url = f"https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/r2/buckets/{BUCKET_NAME}/objects"
103
- headers = {"Authorization": f"Bearer {token}"}
104
-
105
  all_objects = []
106
- cursor = None
107
 
108
- while True:
109
- params = {"prefix": prefix, "per_page": 1000}
110
- if cursor:
111
- params["cursor"] = cursor
112
-
113
- res = requests.get(url, headers=headers, params=params, timeout=30)
114
- if res.status_code != 200:
115
- print(f"Failed to list R2 objects: {res.text}")
116
- break
117
-
118
- data = res.json()
119
- results = data.get("result", [])
120
- all_objects.extend(results)
121
 
122
- cursor = data.get("cursor")
123
- if not cursor or len(results) == 0:
124
- break
125
-
 
 
 
 
 
126
  return all_objects
127
 
128
 
129
  def main():
130
  print("=" * 75)
131
- print("🚀 Cloudflare R2 Upload: Quran Translation Audio")
132
  print(f"📦 Bucket: {BUCKET_NAME}")
133
  print(f"📁 Target Folder: {R2_BASE_FOLDER}/")
134
  print(f"📂 Local Source: {AUDIO_DIR}")
@@ -164,9 +175,9 @@ def main():
164
  if response.lower() != 'y':
165
  sys.exit(0)
166
 
167
- token = get_auth_token()
168
- print("\n🔑 Authenticated successfully with Cloudflare.")
169
- print("\nStarting concurrent upload (workers=10)...\n")
170
 
171
  start_time = time.time()
172
  success_count = 0
@@ -176,9 +187,9 @@ def main():
176
  english_uploaded = 0
177
  urdu_uploaded = 0
178
 
179
- with ThreadPoolExecutor(max_workers=10) as executor:
180
  future_to_file = {
181
- executor.submit(upload_single_file, file_path, r2_key, token): (file_path, r2_key, lang)
182
  for file_path, r2_key, lang in files_to_upload
183
  }
184
 
@@ -206,7 +217,8 @@ def main():
206
  else:
207
  failed_count += 1
208
  failed_details.append((r2_key, err))
209
- print(f"[{completed:5d}/{total_files}] ❌ {r2_key}: {err}")
 
210
 
211
  elapsed = round(time.time() - start_time, 2)
212
  elapsed_min = elapsed / 60
@@ -230,14 +242,14 @@ def main():
230
 
231
  # Verify English
232
  print("\n📝 Verifying English audio...")
233
- english_objects = verify_r2_contents(token, f"{R2_BASE_FOLDER}/english_sahih/")
234
- english_mp3s = [obj for obj in english_objects if obj.get("key", "").endswith(".mp3")]
235
  print(f" ✅ Found {len(english_mp3s)} English MP3 files in R2")
236
 
237
  # Verify Urdu
238
  print("\n📝 Verifying Urdu audio...")
239
- urdu_objects = verify_r2_contents(token, f"{R2_BASE_FOLDER}/urdu_jalandhry/")
240
- urdu_mp3s = [obj for obj in urdu_objects if obj.get("key", "").endswith(".mp3")]
241
  print(f" ✅ Found {len(urdu_mp3s)} Urdu MP3 files in R2")
242
 
243
  total_verified = len(english_mp3s) + len(urdu_mp3s)
 
1
  """
2
+ Upload Quran Translation Audio files to Cloudflare R2 bucket using boto3 S3-compatible API
3
  Target: sahihislam/R2/Qtranslation_mp3/
4
  Structure:
5
  - R2/Qtranslation_mp3/english_sahih/001/ to 114/
 
10
  import time
11
  from pathlib import Path
12
  from concurrent.futures import ThreadPoolExecutor, as_completed
13
+ import boto3
14
+ from botocore.config import Config
15
+ from botocore.exceptions import ClientError
16
 
17
  ROOT_DIR = Path(__file__).parent.parent.parent
18
  AUDIO_DIR = ROOT_DIR / "BACKEND" / "data" / "translation_audio"
19
  ACCOUNT_ID = "d7fc285dc6825480f6d8c887f93e7084"
20
  BUCKET_NAME = "sahihislam"
21
  R2_BASE_FOLDER = "R2/Qtranslation_mp3"
22
+ R2_ENDPOINT = f"https://{ACCOUNT_ID}.r2.cloudflarestorage.com"
23
 
24
 
25
+ def get_r2_client():
26
+ """Get boto3 S3 client configured for Cloudflare R2"""
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  from dotenv import load_dotenv
28
  load_dotenv(str(ROOT_DIR / "BACKEND" / ".env"))
29
+
30
+ access_key = os.getenv("R2_ACCESS_KEY_ID")
31
+ secret_key = os.getenv("R2_SECRET_ACCESS_KEY")
32
+
33
+ # Try parsing cloudflare_S3 URL format if keys not found
34
+ if not access_key or not secret_key:
35
+ cloudflare_s3 = os.getenv("cloudflare_S3")
36
+ if cloudflare_s3 and cloudflare_s3.startswith("https://"):
37
+ # Parse format: https://ACCESS_KEY:SECRET_KEY@ACCOUNT_ID.r2.cloudflarestorage.com/BUCKET
38
+ try:
39
+ # Remove https://
40
+ url_part = cloudflare_s3.replace("https://", "")
41
+ # Split credentials and endpoint
42
+ if "@" in url_part:
43
+ creds, endpoint = url_part.split("@", 1)
44
+ if ":" in creds:
45
+ access_key, secret_key = creds.split(":", 1)
46
+ print(f"✓ Parsed cloudflare_S3 credentials")
47
+ except Exception as e:
48
+ print(f"Warning: Failed to parse cloudflare_S3: {e}")
49
+
50
+ if not access_key or not secret_key:
51
+ raise RuntimeError("R2_ACCESS_KEY_ID and R2_SECRET_ACCESS_KEY must be set in .env file, or provide cloudflare_S3 URL")
52
+
53
+ return boto3.client(
54
+ 's3',
55
+ endpoint_url=R2_ENDPOINT,
56
+ aws_access_key_id=access_key,
57
+ aws_secret_access_key=secret_key,
58
+ config=Config(
59
+ signature_version='s3v4',
60
+ max_pool_connections=50 # Increase connection pool for concurrent uploads
61
+ ),
62
+ region_name='auto'
63
+ )
64
 
 
 
 
 
 
 
 
65
 
66
+ def upload_single_file(file_path: Path, r2_key: str, s3_client, retries: int = 3) -> tuple:
67
+ """Upload a single MP3 file to R2 using boto3"""
68
+ size_kb = round(file_path.stat().st_size / 1024, 2)
69
 
70
  for attempt in range(1, retries + 1):
71
  try:
72
+ with open(file_path, 'rb') as f:
73
+ s3_client.put_object(
74
+ Bucket=BUCKET_NAME,
75
+ Key=r2_key,
76
+ Body=f,
77
+ ContentType='audio/mpeg'
78
+ )
79
+ return (True, r2_key, size_kb, None)
80
+ except ClientError as e:
81
+ error_code = e.response.get('Error', {}).get('Code', 'Unknown')
82
+ if attempt == retries:
83
+ return (False, r2_key, size_kb, f"S3 Error {error_code}: {str(e)[:150]}")
84
+ time.sleep(min(2 ** attempt, 10)) # Exponential backoff
85
  except Exception as e:
86
  if attempt == retries:
87
+ return (False, r2_key, size_kb, str(e)[:150])
88
+ time.sleep(min(2 ** attempt, 10))
89
 
90
  return (False, r2_key, size_kb, "Max retries exceeded")
91
 
 
117
  return files_to_upload
118
 
119
 
120
+ def verify_r2_contents(s3_client, prefix: str) -> list:
121
+ """List all objects with given prefix to verify completeness using boto3"""
 
 
 
122
  all_objects = []
 
123
 
124
+ try:
125
+ paginator = s3_client.get_paginator('list_objects_v2')
126
+ pages = paginator.paginate(Bucket=BUCKET_NAME, Prefix=prefix)
 
 
 
 
 
 
 
 
 
 
127
 
128
+ for page in pages:
129
+ if 'Contents' in page:
130
+ all_objects.extend([
131
+ {'key': obj['Key'], 'size': obj['Size']}
132
+ for obj in page['Contents']
133
+ ])
134
+ except Exception as e:
135
+ print(f"Failed to list R2 objects: {e}")
136
+
137
  return all_objects
138
 
139
 
140
  def main():
141
  print("=" * 75)
142
+ print("🚀 Cloudflare R2 Upload: Quran Translation Audio (boto3 S3)")
143
  print(f"📦 Bucket: {BUCKET_NAME}")
144
  print(f"📁 Target Folder: {R2_BASE_FOLDER}/")
145
  print(f"📂 Local Source: {AUDIO_DIR}")
 
175
  if response.lower() != 'y':
176
  sys.exit(0)
177
 
178
+ s3_client = get_r2_client()
179
+ print("\n🔑 Connected to Cloudflare R2 via boto3 S3")
180
+ print("\nStarting concurrent upload (workers=20)...\n")
181
 
182
  start_time = time.time()
183
  success_count = 0
 
187
  english_uploaded = 0
188
  urdu_uploaded = 0
189
 
190
+ with ThreadPoolExecutor(max_workers=20) as executor:
191
  future_to_file = {
192
+ executor.submit(upload_single_file, file_path, r2_key, s3_client): (file_path, r2_key, lang)
193
  for file_path, r2_key, lang in files_to_upload
194
  }
195
 
 
217
  else:
218
  failed_count += 1
219
  failed_details.append((r2_key, err))
220
+ if failed_count <= 20: # Only print first 20 errors
221
+ print(f"[{completed:5d}/{total_files}] ❌ {r2_key}: {err}")
222
 
223
  elapsed = round(time.time() - start_time, 2)
224
  elapsed_min = elapsed / 60
 
242
 
243
  # Verify English
244
  print("\n📝 Verifying English audio...")
245
+ english_objects = verify_r2_contents(s3_client, f"{R2_BASE_FOLDER}/english_sahih/")
246
+ english_mp3s = [obj for obj in english_objects if obj.get('key', "").endswith(".mp3")]
247
  print(f" ✅ Found {len(english_mp3s)} English MP3 files in R2")
248
 
249
  # Verify Urdu
250
  print("\n📝 Verifying Urdu audio...")
251
+ urdu_objects = verify_r2_contents(s3_client, f"{R2_BASE_FOLDER}/urdu_jalandhry/")
252
+ urdu_mp3s = [obj for obj in urdu_objects if obj.get('key', "").endswith(".mp3")]
253
  print(f" ✅ Found {len(urdu_mp3s)} Urdu MP3 files in R2")
254
 
255
  total_verified = len(english_mp3s) + len(urdu_mp3s)
scripts/verify_search_tiers_api.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Verification Script: Multi-Tier Search & RAG Engine Backend
3
+ ===========================================================
4
+ Validates:
5
+ 1. rag_config.py: catalogs, defaults, tier resolution
6
+ 2. Cache TTL mechanism and cache invalidation
7
+ 3. MongoDB system_settings read/write
8
+ 4. Dynamic per-tier parameter validation
9
+ """
10
+
11
+ import asyncio
12
+ import os
13
+ import sys
14
+
15
+ # Add BACKEND to Python path
16
+ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
17
+
18
+ from app.rag_config import (
19
+ resolve_user_tier,
20
+ get_all_tier_configs,
21
+ get_tier_rag_config,
22
+ update_all_tier_configs,
23
+ invalidate_tier_rag_cache,
24
+ DEFAULT_TIER_CONFIGS,
25
+ AVAILABLE_RERANKERS,
26
+ AVAILABLE_LLM_MODELS,
27
+ )
28
+
29
+
30
+ async def main():
31
+ print("=" * 60)
32
+ print("🔬 TESTING MULTI-TIER SEARCH RAG BACKEND ENGINE")
33
+ print("=" * 60)
34
+
35
+ # 1. Tier Resolution Tests
36
+ print("\n1. Testing Tier Resolution:")
37
+ anonymous_user = None
38
+ free_user = {"uid": "usr1", "role": "USER", "is_vip": False, "is_pro": False}
39
+ pro_user = {"uid": "usr2", "role": "USER", "is_pro": True}
40
+ vip_user = {"uid": "usr3", "role": "VIP", "is_vip": True}
41
+ superadmin_user = {"uid": "usr4", "role": "SUPERADMIN", "superadmin": True}
42
+
43
+ assert resolve_user_tier(anonymous_user) == "free", "Anonymous should resolve to free"
44
+ assert resolve_user_tier(free_user) == "free", "Standard user should resolve to free"
45
+ assert resolve_user_tier(pro_user) == "pro", "Pro user should resolve to pro"
46
+ assert resolve_user_tier(vip_user) == "vip", "VIP user should resolve to vip"
47
+ assert resolve_user_tier(superadmin_user) == "vip", "SuperAdmin should resolve to vip"
48
+ print(" ✅ resolve_user_tier() successfully mapped all 5 personas!")
49
+
50
+ # 2. Catalogs & Defaults
51
+ print("\n2. Testing Reranker & LLM Catalogs:")
52
+ print(f" • Available Rerankers: {len(AVAILABLE_RERANKERS)} options")
53
+ for r in AVAILABLE_RERANKERS:
54
+ print(f" - [{r['id']}] {r['name']} ({r.get('badge')})")
55
+
56
+ print(f" • Available LLMs: {len(AVAILABLE_LLM_MODELS)} options")
57
+ for m in AVAILABLE_LLM_MODELS:
58
+ print(f" - [{m['id']}] {m['name']} ({m.get('badge')})")
59
+
60
+ # 3. Read All Tiers
61
+ print("\n3. Testing Tier Config Read & Defaults:")
62
+ all_tiers = await get_all_tier_configs()
63
+ for tier in ["free", "pro", "vip"]:
64
+ cfg = all_tiers[tier]
65
+ print(f"\n [{tier.upper()}] Tier Configuration:")
66
+ print(f" • Cross-Encoder Enabled : {cfg.get('cross_encoder_enabled')}")
67
+ print(f" • Reranker Model : {cfg.get('reranker_model')} ({cfg.get('reranker_provider')})")
68
+ print(f" • LLM Model : {cfg.get('llm_model')}")
69
+ print(f" • Daily Query Limit : {cfg.get('daily_query_limit')}")
70
+ print(f" • Top-K Context Docs : {cfg.get('top_k')}")
71
+ print(f" • Num Candidates Scope : {cfg.get('num_candidates')}")
72
+ print(f" • Min Vector Similarity : {cfg.get('minimum_vector_similarity')}")
73
+ print(f" • Context Expansion : {cfg.get('context_expansion_enabled')}")
74
+ print(f" • Audio Generation : {cfg.get('audio_generation_enabled')}")
75
+
76
+ # 4. Test Update & Invalidation
77
+ print("\n4. Testing Configuration Mutation & Invalidation:")
78
+ test_update = {
79
+ "free": {
80
+ "cross_encoder_enabled": False,
81
+ "llm_model": "google/gemini-2.0-flash-001",
82
+ "daily_query_limit": 15,
83
+ "top_k": 4,
84
+ },
85
+ "pro": {
86
+ "cross_encoder_enabled": True,
87
+ "llm_model": "deepseek/deepseek-chat",
88
+ "daily_query_limit": 120,
89
+ "top_k": 6,
90
+ },
91
+ "vip": {
92
+ "cross_encoder_enabled": True,
93
+ "llm_model": "anthropic/claude-3.5-sonnet",
94
+ "daily_query_limit": 0,
95
+ "top_k": 12,
96
+ },
97
+ }
98
+
99
+ updated = await update_all_tier_configs(test_update)
100
+ assert updated["free"]["daily_query_limit"] == 15, "Free daily limit should be 15"
101
+ assert updated["free"]["cross_encoder_enabled"] is False, "Free cross-encoder should be False"
102
+ assert updated["pro"]["daily_query_limit"] == 120, "Pro daily limit should be 120"
103
+ assert updated["vip"]["top_k"] == 12, "VIP top_k should be 12"
104
+ print(" ✅ update_all_tier_configs() successfully sanitized and saved update!")
105
+
106
+ # 5. Restore Defaults
107
+ print("\n5. Testing Factory Reset:")
108
+ reset_tiers = await update_all_tier_configs(DEFAULT_TIER_CONFIGS)
109
+ assert reset_tiers["free"]["daily_query_limit"] == 10
110
+ assert reset_tiers["free"]["cross_encoder_enabled"] is False
111
+ assert reset_tiers["pro"]["daily_query_limit"] == 100
112
+ assert reset_tiers["vip"]["daily_query_limit"] == 0
113
+ print(" ✅ Factory defaults restored cleanly!")
114
+
115
+ print("\n" + "=" * 60)
116
+ print("🎉 ALL MULTI-TIER SEARCH RAG BACKEND TESTS PASSED 100%!")
117
+ print("=" * 60)
118
+
119
+
120
+ if __name__ == "__main__":
121
+ asyncio.run(main())