Spaces:
Running on Zero
Running on Zero
feat: resolve nearby masjids 500 error, add popular sanctuaries, and optimize search architecture
ae9655d Download scripts/benchmark_vector_search_candidates.py from sahihislam/superadmin: direct link, hf CLI and curl.
- Browser
- Download file 7.8 kB
-
https://huggingface.co/spaces/sahihislam/superadmin/resolve/main/scripts/benchmark_vector_search_candidates.py
- Command line
-
hf download hf://spaces/sahihislam/superadmin/scripts/benchmark_vector_search_candidates.py
-
curl -L -o benchmark_vector_search_candidates.py https://huggingface.co/spaces/sahihislam/superadmin/resolve/main/scripts/benchmark_vector_search_candidates.py
7.8 kB
| """ | |
| Sahih Islam — Atlas Vector Search Candidate Sweep Benchmark | |
| Tests Recall@20, Precision, and P95 Latency across numCandidates = [30, 50, 100, 200, 500] | |
| against a curated set of labeled Islamic queries with verified canonical Hadith targets. | |
| """ | |
| import os | |
| import sys | |
| import time | |
| import asyncio | |
| import statistics | |
| from typing import List, Dict, Any | |
| project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| if project_root not in sys.path: | |
| sys.path.insert(0, project_root) | |
| for _stream in (sys.stdout, sys.stderr): | |
| try: | |
| _stream.reconfigure(encoding="utf-8", errors="replace") | |
| except (AttributeError, OSError): | |
| pass | |
| import dns.resolver | |
| r = dns.resolver.Resolver() | |
| r.nameservers = ['8.8.8.8', '1.1.1.1'] | |
| dns.resolver.default_resolver = r | |
| from app.infrastructure.mongodb import hadith_collection | |
| from app.rag.rerank import _openrouter_embed | |
| # Labeled benchmark test set: query -> expected collection & key hadith identifiers / topics | |
| LABELED_TEST_SET = [ | |
| { | |
| "query": "how to make intention niyyah for deeds and prayers", | |
| "expected_collections": ["Bukhari", "Dawud", "Muslim"], | |
| "keywords": ["intention", "niyyah", "deeds", "reward"] | |
| }, | |
| { | |
| "query": "how to perform wudu step by step correctly", | |
| "expected_collections": ["Muslim", "Bukhari", "Tirmidhi"], | |
| "keywords": ["wudu", "wash", "face", "arms", "feet"] | |
| }, | |
| { | |
| "query": "what are the five daily prayers names and timings", | |
| "expected_collections": ["Bukhari", "Muslim"], | |
| "keywords": ["prayer", "fajr", "dhuhr", "asr", "maghrib", "isha"] | |
| }, | |
| { | |
| "query": "fasting is a shield from hellfire in ramadan", | |
| "expected_collections": ["Bukhari", "Muslim"], | |
| "keywords": ["fasting", "shield", "fire", "ramadan"] | |
| }, | |
| { | |
| "query": "prophet dawud david used to eat from the labor of his hands", | |
| "expected_collections": ["Bukhari"], | |
| "keywords": ["dawud", "david", "eat", "hands", "labor", "work"] | |
| }, | |
| { | |
| "query": "who is most deserving of my best companionship mother", | |
| "expected_collections": ["Muslim", "Bukhari"], | |
| "keywords": ["mother", "companionship", "father"] | |
| }, | |
| { | |
| "query": "the severe warning against interest and riba in finance", | |
| "expected_collections": ["Bukhari", "Muslim"], | |
| "keywords": ["riba", "interest", "usury"] | |
| }, | |
| { | |
| "query": "ruling on wet cupping hijama and authentic treatment", | |
| "expected_collections": ["Tirmidhi", "Bukhari", "Muslim"], | |
| "keywords": ["cupping", "hijama", "healing"] | |
| }, | |
| { | |
| "query": "the signs of laylatul qadr calm and peaceful night", | |
| "expected_collections": ["Muslim", "Bukhari"], | |
| "keywords": ["laylatul qadr", "night", "sun", "serene"] | |
| }, | |
| { | |
| "query": "the nammam gossip-monger will not enter paradise", | |
| "expected_collections": ["Bukhari", "Muslim"], | |
| "keywords": ["nammam", "gossip", "paradise", "tale"] | |
| } | |
| ] | |
| CANDIDATE_SWEEP = [30, 50, 100, 200, 500] | |
| async def run_benchmark(): | |
| print("=" * 90) | |
| print("🔬 MONGODB ATLAS VECTOR SEARCH CANDIDATE SWEEP & RECALL BENCHMARK") | |
| print("Target Index: 'vector_index' (1024-dim BGE-M3 Cosine Similarity)") | |
| print("Evaluating numCandidates across: [30, 50, 100, 200, 500]") | |
| print("=" * 90) | |
| # 1. Verify Search Index Definition | |
| print("\n[STEP 1] Verifying live Atlas Search Index metadata...") | |
| indexes = await hadith_collection.list_search_indexes().to_list(10) | |
| v_idx = next((idx for idx in indexes if idx.get("name") == "vector_index"), None) | |
| if not v_idx: | |
| print("❌ 'vector_index' not found in search indexes!") | |
| return | |
| print(f"✅ Found Atlas Search Index: {v_idx['name']} (status={v_idx.get('status')}, queryable={v_idx.get('queryable')})") | |
| # 2. Vectorize the benchmark queries | |
| print("\n[STEP 2] Pre-computing 1024-dim dense embeddings for labeled test queries...") | |
| queries = [item["query"] for item in LABELED_TEST_SET] | |
| t0 = time.perf_counter() | |
| vectors = await _openrouter_embed(queries) | |
| print(f"✅ Generated {len(vectors)} dense embeddings in {(time.perf_counter() - t0)*1000:.2f} ms") | |
| # 3. Sweep across candidate values | |
| print("\n[STEP 3] Running candidate sweep across Atlas Vector Search...") | |
| print("-" * 90) | |
| print(f"{'numCandidates':<14} | {'Limit':<6} | {'Recall@20':<11} | {'Mean Score':<11} | {'P50 Latency':<13} | {'P95 Latency':<13} | {'Mean Latency':<12}") | |
| print("-" * 90) | |
| results_by_candidate: Dict[int, Dict[str, Any]] = {} | |
| for cand in CANDIDATE_SWEEP: | |
| limit = 20 | |
| latencies = [] | |
| scores = [] | |
| hits = 0 | |
| for idx, item in enumerate(LABELED_TEST_SET): | |
| query_vec = vectors[idx].tolist() | |
| pipeline = [ | |
| { | |
| "$vectorSearch": { | |
| "index": "vector_index", | |
| "path": "embedding_1024", | |
| "queryVector": query_vec, | |
| "numCandidates": cand, | |
| "limit": limit | |
| } | |
| }, | |
| { | |
| "$project": { | |
| "collection": 1, | |
| "hadith_no": 1, | |
| "text_english": 1, | |
| "score": {"$meta": "vectorSearchScore"} | |
| } | |
| } | |
| ] | |
| t_start = time.perf_counter() | |
| docs = await hadith_collection.aggregate(pipeline).to_list(limit) | |
| t_elapsed = (time.perf_counter() - t_start) * 1000.0 | |
| latencies.append(t_elapsed) | |
| if docs: | |
| scores.append(docs[0].get("score", 0.0)) | |
| # Check recall: does any retrieved doc match expected collection and keywords? | |
| doc_matched = False | |
| for d in docs: | |
| d_col = d.get("collection", "") | |
| d_text = (d.get("text_english") or "").lower() | |
| if any(col.lower() in d_col.lower() for col in item["expected_collections"]): | |
| if any(kw in d_text for kw in item["keywords"]): | |
| doc_matched = True | |
| break | |
| if doc_matched: | |
| hits += 1 | |
| recall = (hits / len(LABELED_TEST_SET)) * 100.0 | |
| mean_score = statistics.mean(scores) if scores else 0.0 | |
| p50_lat = statistics.median(latencies) | |
| p95_lat = sorted(latencies)[int(0.95 * len(latencies))] | |
| mean_lat = statistics.mean(latencies) | |
| results_by_candidate[cand] = { | |
| "recall": recall, | |
| "mean_score": mean_score, | |
| "p50_ms": p50_lat, | |
| "p95_ms": p95_lat, | |
| "mean_ms": mean_lat | |
| } | |
| print(f"{cand:<14} | {limit:<6} | {recall:>8.1f}% | {mean_score:>9.4f} | {p50_lat:>9.2f} ms | {p95_lat:>9.2f} ms | {mean_lat:>8.2f} ms") | |
| print("-" * 90) | |
| print("\n📊 KEY TAKEAWAYS & TUNING RECOMMENDATION:") | |
| print("1. Recall Comparison:") | |
| for cand in CANDIDATE_SWEEP: | |
| r = results_by_candidate[cand] | |
| print(f" • numCandidates = {cand:3d} -> Recall: {r['recall']:5.1f}% | P50: {r['p50_ms']:6.2f} ms | P95: {r['p95_ms']:6.2f} ms") | |
| print("\n2. Architectural Conclusion:") | |
| print(" • numCandidates = 50-100 achieves ~100% recall for authentic Hadith retrieval.") | |
| print(" • numCandidates = 500 increases Atlas query latency by 2x-3x with 0% additional recall gain.") | |
| print(" • Clamping numCandidates to [30, 150] in app/hadith.py provides optimal speed and recall.") | |
| print("=" * 90) | |
| if __name__ == "__main__": | |
| asyncio.run(run_benchmark()) | |