Update app.py

app.py

@@ -19,31 +19,18 @@ from typing import List, Dict, Any, Tuple, Set
 from fastapi import FastAPI, Query
 from fastapi.middleware.cors import CORSMiddleware
 from huggingface_hub import snapshot_download
-import torch
-import torch.nn.functional as F
-import faiss
-
-# Hugging Face imports (Only for embeddings now)
-from transformers import AutoTokenizer, AutoModel
 
 # --- CONFIGURATION & MULTIPROCESSING ---
 TOTAL_CORES = multiprocessing.cpu_count()
-torch.set_num_threads(TOTAL_CORES)
-faiss.omp_set_num_threads(TOTAL_CORES)
-print(f"🖥️ Total Cores Assigned for High-Speed Search: {TOTAL_CORES}")
+print(f"🖥️ Total Cores Assigned for High-Speed Lexical Search: {TOTAL_CORES}")
 
-HF_TOKEN = os.getenv("HF_TOKEN")
+# Apka HF Token (Environment variable se ya direct yaha daal sakte hain)
+HF_TOKEN = os.getenv("HF_TOKEN", "YOUR_HF_TOKEN_HERE")
 
 # Ultra-fast Cache for handling multiple users seamlessly
 QUERY_CACHE = {}
 MAX_CACHE_SIZE = 50000
 
-# --- DENSE RETRIEVAL GLOBALS (FAISS + E5) ---
-embed_tokenizer = None
-embed_model = None
-faiss_index = None
-faiss_metadata = []
-
 # --- 1. DETERMINISTIC PREPROCESSING & CONSTANTS ---
 STOP_WORDS = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "is", "are", "by", "this", "that", "it", "what", "who", "where", "when", "why", "how"}
 ADULT_KEYWORDS = {
@@ -65,16 +52,25 @@ EXPANSION_DICT = {
     "js": ["javascript"]
 }
 
-# Fine-tuned Weights for best Dense + Lexical Fusion
+# 🔥 Tuned Weights for Best Ranking WITHOUT AI Dense Models
 WEIGHTS = {
-    "W_BM25": 1.0, "W_TITLE": 4.0, "W_HEADINGS": 3.0, "W_DESCRIPTION": 2.0, "W_BODY": 1.0,
-    "W_URL": 1.5, "W_DOMAIN": 1.5, "W_PHRASE": 2.5, "W_EXACT": 3.0, "W_FRESHNESS": 1.0,
-    "W_QUALITY": 1.0, "W_POPULARITY": 0.5, "W_PROXIMITY": 1.5, 
-    "W_ROOT_DOMAIN": 8.0, "W_DENSE": 35.0 # Increased priority for Semantic Matching
+    "W_BM25": 15.0,        # Increased priority
+    "W_TITLE": 10.0, 
+    "W_HEADINGS": 5.0, 
+    "W_DESCRIPTION": 3.0, 
+    "W_BODY": 1.5,
+    "W_URL": 2.5, 
+    "W_DOMAIN": 3.5, 
+    "W_PHRASE": 8.0,       # Exact phrase matching heavily rewarded
+    "W_EXACT": 10.0,       # Exact queries boosted
+    "W_FRESHNESS": 2.0,
+    "W_QUALITY": 3.0, 
+    "W_PROXIMITY": 6.0, 
+    "W_ROOT_DOMAIN": 8.0
 }
 
-MAX_RESULTS_PER_DOMAIN = 2 # Diversity constraint
-STRICT_RESULT_LIMIT = 5    # ✨ AS REQUESTED: Only return Top 5 URLs to boost speed
+MAX_RESULTS_PER_DOMAIN = 2 
+STRICT_RESULT_LIMIT = 5    
 
 def normalize_text(text: str) -> str:
     if not text: return ""
@@ -89,11 +85,6 @@ def tokenize(text: str, remove_stopwords: bool = True) -> List[str]:
         tokens = [t for t in tokens if t not in STOP_WORDS]
     return tokens
 
-# --- E5 POOLING FUNCTION ---
-def average_pool(last_hidden_states, attention_mask):
-    last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)
-    return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
-
 # --- 2. QUERY INTENT CLASSIFIER ---
 def detect_query_intent(query_tokens: List[str]) -> Dict[str, Any]:
     intents = []
@@ -110,7 +101,7 @@ def detect_query_intent(query_tokens: List[str]) -> Dict[str, Any]:
     if len(query_tokens) <= 2 and not intents: intents.append("NAVIGATIONAL")
     return {"intent": intents[0] if intents else "GENERAL", "signals": intents}
 
-# --- 3. CORE DETERMINISTIC + DENSE HYBRID INDEX ---
+# --- 3. CORE OFFLINE HYBRID INDEX ---
 class HybridSearchIndex:
     def __init__(self):
         self.docs = []
@@ -130,7 +121,6 @@ class HybridSearchIndex:
 
     def add_document(self, doc_json: dict):
         doc_id = len(self.docs)
-        
         dataset_doc_id = doc_json.get("doc_id", "")
         if dataset_doc_id:
             self.dataset_id_to_internal[dataset_doc_id] = doc_id
@@ -151,6 +141,14 @@ class HybridSearchIndex:
         parsed_url = urllib.parse.urlparse(url)
         domain_str = sys.intern(parsed_url.netloc.lower().replace("www.", ""))
         
+        # 🔥 Extracting Image URL
+        image_url = None
+        open_graph = metadata.get("open_graph") or {}
+        if isinstance(open_graph, dict):
+            img = open_graph.get("image")
+            if img and isinstance(img, str) and img.startswith("http"):
+                image_url = img
+
         clean_doc = {
             "url": url,
             "_fast_domain": domain_str,
@@ -159,7 +157,8 @@ class HybridSearchIndex:
             "_fast_desc": desc_str[:250],
             "_fast_snippet": snippet_str[:300],
             "_fast_content": main_text,
-            "_fast_headings": (h1_str + " " + h2_str)[:200]
+            "_fast_headings": (h1_str + " " + h2_str)[:200],
+            "_fast_image": image_url # Hidden field for images output
         }
         
         title_tokens = tokenize(clean_doc["_fast_title"])
@@ -196,7 +195,7 @@ class HybridSearchIndex:
         self.high_freq_vocab = [word for word, count in Counter(self.df).most_common(15000)]
         self.suggestion_counter.clear()
         gc.collect()
-        print(f"🚀 Core Search Engine Indexing Complete! Total Docs: {self.total_docs}")
+        print(f"🚀 Offline Search Engine Indexing Complete! Total Docs: {self.total_docs}")
 
     def idf(self, term: str) -> float:
         df = self.df.get(term, 0)
@@ -301,9 +300,9 @@ class HybridSearchIndex:
 
         return scores
 
-    def search(self, query: str, debug: bool = False) -> Tuple[List[dict], Dict[str, Any], str]:
+    def search(self, query: str, debug: bool = False) -> Tuple[List[dict], List[str], Dict[str, Any], str]:
         if self.total_docs == 0:
-            return [], {"intent": "GENERAL", "signals": []}, None
+            return [], [], {"intent": "GENERAL", "signals": []}, None
 
         query_norm = normalize_text(query)
         raw_tokens = query_norm.split()
@@ -340,34 +339,9 @@ class HybridSearchIndex:
         intent = intent_info["intent"]
         user_wants_adult = any(aw in query_norm for aw in ADULT_KEYWORDS)
         
-        # 1. Lexical Retrieval (Fast BM25 Base Candidates)
+        # 1. Pure Fast Lexical Retrieval
         candidates_ids = self._get_candidates(sorted_tokens)
         
-        # 2. Dense Semantic Retrieval (FAISS - Top Priority)
-        dense_scores_map = {}
-        if faiss_index is not None and embed_model is not None and embed_tokenizer is not None:
-            try:
-                embed_inputs = embed_tokenizer(f"query: {query_norm}", return_tensors="pt", padding=True, truncation=True, max_length=512)
-                
-                # Use inference_mode for maximum PyTorch speed
-                with torch.inference_mode():
-                    embed_outputs = embed_model(**embed_inputs)
-                    xq = average_pool(embed_outputs.last_hidden_state, embed_inputs["attention_mask"])
-                    xq = F.normalize(xq, p=2, dim=1).numpy()
-                
-                # Fetch Top 20 Contextual Matches via FAISS
-                D, I = faiss_index.search(xq, k=20)
-                
-                for score, idx in zip(D[0], I[0]):
-                    if 0 <= idx < len(faiss_metadata):
-                        ds_id = faiss_metadata[idx].get("doc_id", "")
-                        if ds_id in self.dataset_id_to_internal:
-                            internal_id = self.dataset_id_to_internal[ds_id]
-                            dense_scores_map[internal_id] = float(score)
-                            candidates_ids.add(internal_id) # Merge into Hybrid Pool
-            except Exception as e:
-                print(f"⚠️ FAISS Search Warning: {e}")
-
         scored_results = []
         seen_domains = defaultdict(int)
         seen_urls = set()
@@ -386,12 +360,6 @@ class HybridSearchIndex:
             w_domain = WEIGHTS["W_DOMAIN"] + (4.0 if intent == "NAVIGATIONAL" else 0.0)
             w_title = WEIGHTS["W_TITLE"] + (2.0 if intent == "DEFINITION" else 0.0)
             w_phrase = WEIGHTS["W_PHRASE"] + (1.5 if intent in ("DEFINITION", "COMPARISON") else 0.0)
-            
-            dense_score = dense_scores_map.get(doc_id, 0.0)
-            features["dense_score"] = dense_score
-            
-            # Massive Dense Boost (Threshold >= 0.75 gets heavily rewarded)
-            dense_boost = max(0.0, dense_score - 0.75) * WEIGHTS["W_DENSE"]
 
             final_score = (
                 WEIGHTS["W_BM25"] * norm_bm25 + 
@@ -405,29 +373,9 @@ class HybridSearchIndex:
                 WEIGHTS["W_PROXIMITY"] * features["proximity"] + 
                 WEIGHTS["W_ROOT_DOMAIN"] * features["root_domain"] +
                 WEIGHTS["W_HEADINGS"] * features["headings"] +
-                WEIGHTS["W_BODY"] * features["body"] +
-                dense_boost 
+                WEIGHTS["W_BODY"] * features["body"]
             )
             
-            is_semantic_match = dense_score >= 0.81 
-            
-            if len(sorted_tokens) > 0:
-                top_term = sorted_tokens[0]
-                doc_title = tokenize(doc.get("_fast_title", ""), False)
-                doc_desc = tokenize(doc.get("_fast_desc", ""), False)
-                doc_content = doc.get("_fast_content", "").lower()
-                doc_headings = tokenize(doc.get("_fast_headings", ""), False)
-                
-                if top_term not in doc_title and top_term not in doc_desc and top_term not in doc.get("_fast_domain", "") and top_term not in doc_content and top_term not in doc_headings:
-                    if is_semantic_match: final_score *= 0.7 
-                    else: final_score *= 0.1 
-                    
-            if len(sorted_tokens) > 1:
-                second_term = sorted_tokens[1]
-                if second_term not in doc_title and second_term not in doc_desc and second_term not in doc.get("_fast_domain", "") and second_term not in doc_content and second_term not in doc_headings:
-                    if is_semantic_match: final_score *= 0.85 
-                    else: final_score *= 0.4 
-            
             if final_score > 0.1:
                 doc_out = {
                     "doc_id": str(doc_id),
@@ -437,61 +385,62 @@ class HybridSearchIndex:
                     "url": doc.get("url", ""),
                     "domain": doc.get("_fast_domain", ""),
                     "match_score": round(final_score, 4),
-                    "_internal_content": doc.get("_fast_content", "")
+                    "_internal_content": doc.get("_fast_content", ""),
+                    "_fast_image": doc.get("_fast_image", None) # Safe keeping for later
                 }
                 if debug:
                     doc_out["_debug_features"] = features
-                    doc_out["_debug_intent"] = intent_info
                 
                 scored_results.append((final_score, doc_out, base_url, doc["_fast_domain"]))
 
         scored_results.sort(key=lambda x: x[0], reverse=True)
         
+        # 🔥 Extract Top Images globally first (before applying strict domain filters)
+        global_images = []
+        for score, doc_info, b_url, domain in scored_results[:20]: # Check top 20 best results for images
+            img_url = doc_info.get("_fast_image")
+            if img_url and img_url not in global_images:
+                global_images.append(img_url)
+                if len(global_images) >= 10: break # Collect Max 10 relevant images
+        
         diversified_results = []
         for score, doc_info, b_url, domain in scored_results:
             if seen_domains[domain] >= MAX_RESULTS_PER_DOMAIN: continue
             seen_domains[domain] += 1
             seen_urls.add(b_url)
+            
+            doc_info.pop("_fast_image", None) # Remove it from URL output 
             diversified_results.append(doc_info)
-            if len(diversified_results) >= STRICT_RESULT_LIMIT: break # 🔥 Only 5 URLs
+            if len(diversified_results) >= STRICT_RESULT_LIMIT: break # 🔥 Top 5 URLs limit
                 
         for idx, item in enumerate(diversified_results): item["rank"] = idx + 1
-        return diversified_results, intent_info, corrected_query_str
+        return diversified_results, global_images, intent_info, corrected_query_str
 
 search_index = HybridSearchIndex()
 
-# --- DATA LOADER (FAISS & LEXICAL) ---
-def load_dense_retriever():
-    global embed_tokenizer, embed_model, faiss_index, faiss_metadata
-    print("🧠 Loading Lightning Fast E5-Small-v2 Embedder (CPU Optimized)...")
+# --- DATA LOADER (OFFLINE + SYNC) ---
+def sync_and_load_datasets():
+    print("📥 Syncing Required Datasets securely...")
     try:
-        embed_tokenizer = AutoTokenizer.from_pretrained("intfloat/e5-small-v2")
-        # Load directly in eval mode for speed
-        embed_model = AutoModel.from_pretrained("intfloat/e5-small-v2").eval()
-        print("✅ E5 Embedder Ready!")
-    except Exception as e:
-        print(f"❌ E5 Loading Failed: {e}")
-
-    print("📥 Downloading FAISS Index via HF_TOKEN...")
-    try:
-        dense_path = snapshot_download(repo_id="Nebulixlabs/embedded-data", repo_type="dataset", token=HF_TOKEN)
-        index_file = os.path.join(dense_path, "dataset_embeddings.index")
-        meta_file = os.path.join(dense_path, "metadata.json")
+        # Sync Main Lexical dataset to memory
+        dataset_path = snapshot_download(
+            repo_id="Nebulixlabs/web-search-dataset", 
+            repo_type="dataset", 
+            allow_patterns="data/*.jsonl", 
+            token=HF_TOKEN
+        )
         
-        if os.path.exists(index_file) and os.path.exists(meta_file):
-            faiss_index = faiss.read_index(index_file)
-            with open(meta_file, "r", encoding="utf-8") as f:
-                faiss_metadata = json.load(f)
-            print(f"✅ FAISS High-Speed Index Loaded! ({faiss_index.ntotal} vectors)")
-    except Exception as e:
-        print(f"❌ FAISS Loading Failed: {e}")
-
-def load_and_index_dataset():
-    print("📦 Streaming 'web-search-dataset' ...")
-    try:
-        dataset_path = snapshot_download(repo_id="Nebulixlabs/web-search-dataset", repo_type="dataset", allow_patterns="data/*.jsonl", token=HF_TOKEN)
+        # Simply downloading the Private Vector repo as per your request for offline storage
+        # (Vectors won't be queried live since you strictly requested NO AI MODEL for search engine)
+        snapshot_download(
+            repo_id="Nebulixlabs/web-search-faiss-index", 
+            repo_type="dataset", 
+            token=HF_TOKEN
+        )
+        print("✅ Private Vector & Public Datasets Synced Offline!")
     except Exception as e:
         dataset_path = "."
+        print(f"⚠️ Offline Fallback mode (No internet or token issue): {e}")
         
     data_folder = os.path.join(dataset_path, "data")
     if not os.path.exists(data_folder): data_folder = "data" 
@@ -515,11 +464,10 @@ def load_and_index_dataset():
 # --- FASTAPI APP ENDPOINTS ---
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    asyncio.create_task(asyncio.to_thread(load_dense_retriever))
-    asyncio.create_task(asyncio.to_thread(load_and_index_dataset))
+    asyncio.create_task(asyncio.to_thread(sync_and_load_datasets))
     yield
 
-app = FastAPI(title="Nebulix Ultra-Fast Hybrid Engine", lifespan=lifespan)
+app = FastAPI(title="Nebulix Ultra-Fast Lexical Engine", lifespan=lifespan)
 app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
 
 @app.get("/suggest")
@@ -533,16 +481,14 @@ async def pure_search(q: str = Query(...), content: bool = Query(False), debug:
     req_id = str(uuid.uuid4())
     cache_key = f"{normalize_text(q)}_{content}_{debug}"
 
-    # ⚡ Cache Hit: Instant response for concurrent/repeated queries (0.001s)
+    # ⚡ Cache Hit: Instant response
     if cache_key in QUERY_CACHE:
         cached_response = QUERY_CACHE[cache_key]
         cached_response["request_id"] = req_id 
         return cached_response
 
-    # ⚡ asyncio.to_thread prevents Event Loop blocking so multiple users can search simultaneously
-    results, intent_info, corrected_query = await asyncio.to_thread(search_index.search, q, debug)
+    results, global_images, intent_info, corrected_query = await asyncio.to_thread(search_index.search, q, debug)
     
-    # ⚡ Strictly Top 5 limits for ultra-fast network transfer
     top_5 = results[:STRICT_RESULT_LIMIT]
     
     for r in top_5:
@@ -553,11 +499,11 @@ async def pure_search(q: str = Query(...), content: bool = Query(False), debug:
         "request_id": req_id, 
         "query": q, 
         "corrected_query": corrected_query,
+        "images": global_images,      # 🖼️ Images output globally list ki tarah yaha set hai!
         "results_count": len(top_5), 
         "results": top_5
     }
 
-    # Save to Cache
     if len(QUERY_CACHE) > MAX_CACHE_SIZE: QUERY_CACHE.clear()
     QUERY_CACHE[cache_key] = final_response