Update app.py

app.py

@@ -12,7 +12,7 @@ import gc
 import sys
 import array
 import time
-import numpy as np # NEW: For ultra-fast vector math
+import numpy as np
 from datetime import datetime
 from collections import Counter, defaultdict
 from contextlib import asynccontextmanager
@@ -20,9 +20,9 @@ from typing import List, Dict, Any, Tuple, Set
 
 from fastapi import FastAPI, Query, Form
 from fastapi.middleware.cors import CORSMiddleware
-from huggingface_hub import snapshot_download, HfApi # NEW: HfApi for uploading
+from huggingface_hub import snapshot_download, HfApi
 import torch
-from openai import OpenAI # NEW: For embeddings
+from openai import OpenAI
 
 # Hugging Face imports
 from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, BitsAndBytesConfig
@@ -59,7 +59,7 @@ WEIGHTS = {
     "W_BM25": 1.0, "W_TITLE": 4.0, "W_HEADINGS": 3.0, "W_DESCRIPTION": 2.0, "W_BODY": 1.0,
     "W_URL": 1.5, "W_DOMAIN": 1.5, "W_PHRASE": 2.5, "W_EXACT": 3.0, "W_FRESHNESS": 1.0,
     "W_QUALITY": 1.0, "W_POPULARITY": 0.5, "W_PROXIMITY": 1.5, "W_ROOT_DOMAIN": 8.0,
-    "W_VECTOR": 5.0 # NEW: Weight for Semantic Search
+    "W_VECTOR": 5.0 # Weight for Semantic Search
 }
 
 MAX_RESULTS_PER_DOMAIN = 3
@@ -74,7 +74,6 @@ def tokenize(text: str, remove_stopwords: bool = True) -> List[str]:
     if remove_stopwords: tokens = [t for t in tokens if t not in STOP_WORDS]
     return tokens
 
-# --- NEW: OPENROUTER EMBEDDING FUNCTION ---
 def get_embeddings(texts: List[str]) -> List[List[float]]:
     """Batched API call to OpenRouter to get vectors."""
     if not OPENROUTER_API_KEY: return []
@@ -102,9 +101,8 @@ class HybridSearchIndex:
         
         self.inverted_index = defaultdict(lambda: array.array('I')) 
         
-        # NEW: Vector Search Data
         self.raw_embeddings = []
-        self.doc_vectors = None # NumPy matrix for ultra-fast math
+        self.doc_vectors = None
         
         self.suggestion_counter = Counter()
         self.suggestions = []
@@ -132,12 +130,11 @@ class HybridSearchIndex:
             "has_vector": False
         }
         
-        # NEW: Store embeddings if they exist in DB
         if "embedding" in doc_json and doc_json["embedding"]:
             self.raw_embeddings.append(doc_json["embedding"])
             clean_doc["has_vector"] = True
         else:
-            self.raw_embeddings.append([0.0]*1024) # Placeholder for non-embedded (Model dims depend on nemotron, assumed 1024)
+            self.raw_embeddings.append([0.0]*1024) 
 
         all_tokens = tokenize(clean_doc["_fast_title"]) + tokenize(clean_doc["_fast_desc"]) + tokenize(clean_doc["_fast_content"])[:250]
         clean_doc["_token_len"] = len(all_tokens)
@@ -154,18 +151,16 @@ class HybridSearchIndex:
         self.total_docs = len(self.docs)
         self.avg_doc_len = self.total_len / max(1, self.total_docs)
         
-        # NEW: Compile vectors into Ultra-Fast NumPy Matrix
         print("🧠 Compiling Vector Matrix for Fast Semantic Search...")
         try:
             vectors = np.array(self.raw_embeddings, dtype=np.float32)
-            # L2 Normalize for Cosine Similarity via Dot Product (Fastest method)
             norms = np.linalg.norm(vectors, axis=1, keepdims=True)
             self.doc_vectors = np.where(norms > 0, vectors / norms, 0)
         except Exception as e:
             print("⚠️ Vector compile failed:", e)
             self.doc_vectors = None
             
-        self.raw_embeddings = [] # Free RAM
+        self.raw_embeddings = [] 
         gc.collect()
         print(f"🚀 Indexing Complete! {self.total_docs} records ready.")
 
@@ -173,7 +168,6 @@ class HybridSearchIndex:
         query_norm = normalize_text(query)
         search_tokens = tokenize(query_norm)
         
-        # 1. NEW: Get Query Vector
         query_vector = None
         vector_scores = None
         if self.doc_vectors is not None and OPENROUTER_API_KEY:
@@ -183,15 +177,12 @@ class HybridSearchIndex:
                 q_norm_val = np.linalg.norm(q_vec)
                 if q_norm_val > 0:
                     q_vec = q_vec / q_norm_val
-                    # ULTRA-FAST Matrix Multiplication for all docs at once
                     vector_scores = np.dot(self.doc_vectors, q_vec)
 
-        # 2. Get Lexical Candidates
         candidates_ids = set()
         for term in search_tokens:
             if term in self.inverted_index: candidates_ids.update(self.inverted_index[term])
             
-        # If Vector Search worked, add Top 50 semantic matches to candidates
         if vector_scores is not None:
             top_vector_indices = np.argsort(vector_scores)[-50:][::-1]
             candidates_ids.update(top_vector_indices)
@@ -202,18 +193,15 @@ class HybridSearchIndex:
         for doc_id in candidates_ids:
             doc = self.docs[doc_id]
             
-            # --- Lexical Scoring (BM25) Simplified for brevity ---
             lexical_score = 0.0
             doc_text = (doc["_fast_title"] + " " + doc["_fast_desc"]).lower()
             for t in search_tokens:
                 if t in doc_text: lexical_score += 2.0
                 
-            # --- Vector Scoring ---
             v_score = 0.0
             if vector_scores is not None and doc["has_vector"]:
                 v_score = float(vector_scores[doc_id])
                 
-            # HYBRID FORMULA
             final_score = lexical_score + (WEIGHTS["W_VECTOR"] * v_score)
             
             if final_score > 0.1:
@@ -238,7 +226,6 @@ class HybridSearchIndex:
 
 search_index = HybridSearchIndex()
 
-# --- NEW: AUTO-EMBEDDING & HUGGINGFACE SYNC ---
 def load_and_index_dataset():
     print("📦 Loading dataset and checking for Vectors...")
     dataset_path = snapshot_download(
@@ -250,6 +237,8 @@ def load_and_index_dataset():
     new_data_folder = "./updated_data"
     os.makedirs(new_data_folder, exist_ok=True)
     
+    docs_embedded = 0
+    
     for file in os.listdir(data_folder):
         if not file.endswith(".jsonl"): continue
         file_path = os.path.join(data_folder, file)
@@ -261,53 +250,66 @@ def load_and_index_dataset():
                 if not line.strip(): continue
                 doc = json.loads(line)
                 
-                # Check if embedding is missing
                 if "embedding" not in doc and OPENROUTER_API_KEY:
                     batch_docs.append(doc)
-                    # Process in small batches to avoid OpenRouter rate limits
                     if len(batch_docs) >= 20: 
-                        texts = [f"{d.get('metadata',{}).get('title','')} {d.get('content',{}).get('snippet','')}" for d in batch_docs]
+                        texts = []
+                        for d in batch_docs:
+                            # FIX 1: Handle empty text so we don't get 400 Bad Request error
+                            raw_text = f"{d.get('metadata',{}).get('title','')} {d.get('content',{}).get('snippet','')}".strip()
+                            if len(raw_text) < 2: raw_text = "empty document"
+                            texts.append(raw_text)
+                            
                         embeddings = get_embeddings(texts)
                         
                         for i, d in enumerate(batch_docs):
-                            if i < len(embeddings): d["embedding"] = embeddings[i]
+                            if embeddings and i < len(embeddings): 
+                                d["embedding"] = embeddings[i]
                             out_f.write(json.dumps(d) + "\n")
                             search_index.add_document(d)
                             
                         batch_docs = []
                         needs_upload = True
-                        time.sleep(1) # Rate limit safety
+                        docs_embedded += 20
+                        print(f"🔄 Embedded {docs_embedded} new docs... (Waiting 4s to prevent Rate Limits ⏳)")
+                        # FIX 2: Wait 4 seconds (instead of 1s) to stay under Free Tier 20 limit!
+                        time.sleep(4) 
                 else:
                     out_f.write(line)
                     search_index.add_document(doc)
             
-            # Leftover batch
             if batch_docs:
-                texts = [f"{d.get('metadata',{}).get('title','')} {d.get('content',{}).get('snippet','')}" for d in batch_docs]
+                texts = []
+                for d in batch_docs:
+                    raw_text = f"{d.get('metadata',{}).get('title','')} {d.get('content',{}).get('snippet','')}".strip()
+                    if len(raw_text) < 2: raw_text = "empty document"
+                    texts.append(raw_text)
+                    
                 embeddings = get_embeddings(texts)
                 for i, d in enumerate(batch_docs):
-                    if i < len(embeddings): d["embedding"] = embeddings[i]
+                    if embeddings and i < len(embeddings): d["embedding"] = embeddings[i]
                     out_f.write(json.dumps(d) + "\n")
                     search_index.add_document(d)
                 needs_upload = True
 
     search_index.finalize_build()
 
-    # --- UPLOAD BACK TO HUGGING FACE ---
     if needs_upload and HF_TOKEN:
         print("☁️ New Embeddings generated! Uploading updated dataset to Hugging Face...")
-        api = HfApi(token=HF_TOKEN)
-        for file in os.listdir(new_data_folder):
-            api.upload_file(
-                path_or_fileobj=os.path.join(new_data_folder, file),
-                path_in_repo=f"data/{file}",
-                repo_id=HF_DATASET_REPO,
-                repo_type="dataset"
-            )
-        print("✅ Dataset successfully updated on Hugging Face!")
-
+        try:
+            api = HfApi(token=HF_TOKEN)
+            for file in os.listdir(new_data_folder):
+                api.upload_file(
+                    path_or_fileobj=os.path.join(new_data_folder, file),
+                    path_in_repo=f"data/{file}",
+                    repo_id=HF_DATASET_REPO,
+                    repo_type="dataset"
+                )
+            print("✅ Dataset successfully updated on Hugging Face!")
+        except Exception as e:
+            print(f"⚠️ Dataset upload failed: {e}")
 
-# --- AI GENERATION MODULE (Unchanged) ---
+# --- AI GENERATION MODULE ---
 def load_ai_models():
     global ai_generator
     print("🤖 Loading SmolLM2...")