Update app.py

app.py

@@ -1,7 +1,8 @@
 # ==============================================================================
-# NEBULIX SEARCH ENGINE v6 FINAL (PRODUCTION READY)
+# NEBULIX SEARCH ENGINE v6 FINAL
 # Target: Hugging Face Spaces | 2 vCPU • 16 GB RAM • True Agentic RAG • AI Streaming
 # strictly PRE-TRAINED MODELS ONLY (NO TRAINING)
+# REMOVED: Sentence-Transformers & Datasets library (No more train split errors)
 # ==============================================================================
 import json
 import uuid
@@ -10,7 +11,6 @@ import os
 import uvicorn
 import re
 import time
-import pickle
 import sqlite3
 import unicodedata
 import threading
@@ -24,9 +24,10 @@ from fastapi.middleware.cors import CORSMiddleware
 from rank_bm25 import BM25Okapi
 from cachetools import TTLCache, LRUCache
 import hashlib
+from huggingface_hub import HfApi, hf_hub_download
 
 # ==============================================================================
-# 25. CPU ALLOCATION (60% Search, 40% AI on 2 vCPUs)
+# 25. CPU ALLOCATION 
 # ==============================================================================
 os.environ["OMP_NUM_THREADS"] = "1"
 os.environ["OPENBLAS_NUM_THREADS"] = "1"
@@ -34,22 +35,19 @@ os.environ["MKL_NUM_THREADS"] = "1"
 
 # --- ML & Data Imports ---
 try:
-    from datasets import load_dataset
-    from sentence_transformers import SentenceTransformer
     import torch
     from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, TextIteratorStreamer
     
     # 🔥 STRICT NO-TRAINING LOCK 🔥
     torch.set_num_threads(1)
     torch.set_grad_enabled(False) 
-    
     ML_AVAILABLE = True
 except ImportError:
     ML_AVAILABLE = False
     print("⚠️ ML packages missing. Check requirements.txt")
 
 # ==============================================================================
-# 22. SQLITE ANALYTICS SETUP
+# SQLITE ANALYTICS SETUP
 # ==============================================================================
 DB_PATH = "analytics.db"
 def init_db():
@@ -76,11 +74,9 @@ url_to_data = {}
 bm25_index = None
 trie_root = {}
 
-SEARCH_CACHE = TTLCache(maxsize=10000, ttl=1800) # 30 Min TTL
+SEARCH_CACHE = TTLCache(maxsize=10000, ttl=1800)
 AI_ANSWER_CACHE = LRUCache(maxsize=2000)
 
-doc_embeddings = None
-text_embedder = None
 tokenizer = None
 pipe = None
 AI_SEMAPHORE = asyncio.Semaphore(1)
@@ -89,7 +85,7 @@ CLICK_TRACKER = {}
 STOP_WORDS = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "is"}
 
 # ==============================================================================
-# MATH & SPECIAL TAGS
+# UTILS & TEXT PROCESSING
 # ==============================================================================
 def update_dynamic_fields(doc):
     human = doc.get("human_opens", 0)
@@ -112,7 +108,6 @@ def update_dynamic_fields(doc):
     
     if ai >= 25: tags.add("Opened by AI")
     if total_interactive > 10 and ai_percent >= 80: tags.add("AI Favorite")
-    
     doc["special_tags"] = list(tags)
 
 def normalize_text(text: str) -> str:
@@ -131,22 +126,11 @@ def add_to_trie(word):
         node = node[char]
     node['_end'] = True
 
-def cosine_similarity(vec_a, mat_b):
-    if len(mat_b) == 0: return np.array([])
-    return np.dot(mat_b, vec_a) / (np.linalg.norm(mat_b, axis=1) * np.linalg.norm(vec_a) + 1e-10)
-
-def detect_language(text):
-    return "English"
-
 def parse_json_field(val):
-    """Safely converts stringified JSONs to dicts if dataset schema changed."""
-    if isinstance(val, dict):
-        return val
+    if isinstance(val, dict): return val
     if isinstance(val, str):
-        try:
-            return json.loads(val)
-        except:
-            return {}
+        try: return json.loads(val)
+        except: return {}
     return {}
 
 def build_searchable_text(doc):
@@ -159,8 +143,6 @@ def build_searchable_text(doc):
 
 def build_runtime_record(doc):
     r = doc.copy()
-    
-    # Safe JSON parsing to prevent crashes from schema changes
     r["meta_tags"] = parse_json_field(doc.get("meta_tags"))
     r["og_tags"] = parse_json_field(doc.get("og_tags"))
     r["specific_data"] = parse_json_field(doc.get("specific_data"))
@@ -173,56 +155,40 @@ def build_runtime_record(doc):
     
     r["human_opens"] = r.get("human_opens", 0)
     r["ai_opens"] = r.get("ai_opens", 0)
-    r["crawler_opens"] = r.get("crawler_opens", 0)
-    r["bot_opens"] = r.get("bot_opens", 0)
     r["trust_score"] = r.get("trust_score", 50.0)
     r["quality_score"] = r.get("quality_score", 50.0)
     r["learning_score"] = r.get("learning_score", 50.0)
     r["search_boost"] = r.get("search_boost", 0.0)
     r["spam_score"] = r.get("spam_score", 0.0)
-    r["domain_quality"] = r.get("domain_quality", 50.0)
     r["last_opened"] = r.get("last_opened", datetime.utcnow().isoformat())
-    r["language"] = r.get("language", detect_language(r.get("title", "")))
+    r["language"] = r.get("language", "English")
     r["country"] = r.get("country", "Global")
     
     update_dynamic_fields(r)
     return r
 
-def init_system():
-    global bm25_index, doc_embeddings, text_embedder, tokenizer, pipe, raw_data, url_to_data
-    init_db()
-    print("🚀 [1/4] Fetching Dataset & Building Indexes...")
-    
-    CACHE_FILE = "cached_dataset.jsonl"
+# ==============================================================================
+# DATASET FETCHING (Direct Raw Method, Bypassing "Train Split" Schema Errors)
+# ==============================================================================
+def load_dataset_direct():
+    global bm25_index, raw_data, url_to_data
+    hf_token = os.getenv("HF_TOKEN")
+    repo_id = "Nebulixlabs/Ultimate-Search-Engine-Dataset"
     
-    if not os.path.exists(CACHE_FILE):
-        hf_token = os.getenv("HF_TOKEN")
-        print("☁️ Downloading dataset from Hugging Face...")
+    try:
+        print("☁️ Fetching raw data files directly from Hugging Face Hub (No datasets library)...")
+        api = HfApi()
+        files_in_repo = api.list_repo_files(repo_id, repo_type="dataset", token=hf_token)
+        jsonl_files = [f for f in files_in_repo if f.endswith(".jsonl") or f.endswith(".json")]
         
-        try:
-            # Attempt 1: Standard load
-            dataset = load_dataset("Nebulixlabs/Ultimate-Search-Engine-Dataset", split="train", token=hf_token)
-            dataset.to_json(CACHE_FILE)
-        except Exception as e:
-            print(f"⚠️ Standard load failed due to schema mismatch ({e}). Trying fallback raw JSON loader...")
-            try:
-                # Attempt 2: Force raw JSON inference (bypasses broken schemas)
-                dataset = load_dataset("json", data_files="hf://datasets/Nebulixlabs/Ultimate-Search-Engine-Dataset/**/*.json*", split="train", token=hf_token)
-                dataset.to_json(CACHE_FILE)
-            except Exception as e2:
-                print(f"⚠️ Raw JSON load failed. Trying Parquet loader... ({e2})")
-                # Attempt 3: Parquet fallback
-                dataset = load_dataset("parquet", data_files="hf://datasets/Nebulixlabs/Ultimate-Search-Engine-Dataset/**/*.parquet", split="train", token=hf_token)
-                dataset.to_json(CACHE_FILE)
-
-        print("✅ Dataset successfully cached locally.")
-
-    if os.path.exists(CACHE_FILE):
         corpus_tokens = []
-        texts_to_embed = []
-        with open(CACHE_FILE, "r", encoding="utf-8") as f:
-            for line in f:
-                if line.strip():
+        for file_name in jsonl_files:
+            print(f"📥 Downloading {file_name}...")
+            local_path = hf_hub_download(repo_id, file_name, repo_type="dataset", token=hf_token)
+            
+            with open(local_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    if not line.strip(): continue
                     try:
                         doc = build_runtime_record(json.loads(line))
                         raw_data.append(doc)
@@ -230,63 +196,56 @@ def init_system():
                         
                         tokens = tokenize(doc["searchable_text"])
                         corpus_tokens.append(tokens)
-                        texts_to_embed.append(doc["searchable_text"])
                         for t in tokens: add_to_trie(t)
-                    except Exception as parse_e:
-                        # Ignore highly corrupted single lines
-                        pass
-
+                    except Exception as e:
+                        pass # Ignore broken lines
+        
+        print(f"✅ Loaded {len(raw_data)} documents safely. Building Search Index...")
         bm25_index = BM25Okapi(corpus_tokens)
+        print("✅ Fast BM25 Search Index Ready.")
         
-        if ML_AVAILABLE:
-            print("🧠 [2/4] Loading BAAI/bge-small-en-v1.5 (Pre-trained Inference Only)...")
-            text_embedder = SentenceTransformer('BAAI/bge-small-en-v1.5', device='cpu')
+    except Exception as e:
+        print(f"❌ Failed to load dataset: {e}")
 
-            if os.path.exists("embeddings.bin"):
-                print("💾 [3/4] Loading Precomputed Embeddings...")
-                with open("embeddings.bin", "rb") as fb: doc_embeddings = pickle.load(fb)
-            else:
-                print("⚙️ [3/4] Converting Text to Vector Embeddings (One-Time Inference)...")
-                if texts_to_embed:
-                    doc_embeddings = text_embedder.encode(texts_to_embed, convert_to_numpy=True)
-                    with open("embeddings.bin", "wb") as fb: pickle.dump(doc_embeddings, fb)
-                    with open("metadata.json", "w") as fm: json.dump({"count": len(doc_embeddings)}, fm)
-            
-            # =============== PRE-TRAINED TRANSFORMERS BASED SMOLLM2 ===============
-            print("🔥 [4/4] Loading Pre-trained SmolLM2 (360M Params)...")
-            try:
-                model_id = "HuggingFaceTB/SmolLM2-360M-Instruct"
-                tokenizer = AutoTokenizer.from_pretrained(model_id)
-                model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32)
-                model.eval() # Puts the model in strict inference mode
-                pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, device=-1)
-                print("✅ Model successfully loaded in INFERENCE mode.")
-            except Exception as e:
-                print(f"⚠️ LLM loading failed: {e}. Mocked mode active.")
-    print("✅ Nebulix v6 Ready. (AI Training Disabled)")
+def init_system():
+    global tokenizer, pipe
+    init_db()
+    print("🚀 [1/3] Initializing System...")
+    
+    # 1. Load Data
+    load_dataset_direct()
+    
+    # 2. Load GenAI Model
+    if ML_AVAILABLE:
+        print("🔥 [2/3] Loading Pre-trained SmolLM2 (360M Params)...")
+        try:
+            model_id = "HuggingFaceTB/SmolLM2-360M-Instruct"
+            tokenizer = AutoTokenizer.from_pretrained(model_id)
+            model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32)
+            model.eval() # STRICT INFERENCE MODE
+            pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, device=-1)
+            print("✅ Model successfully loaded in INFERENCE mode.")
+        except Exception as e:
+            print(f"⚠️ LLM loading failed: {e}. Mocked mode active.")
+    
+    print("✅ Nebulix v6 Ready. (Vector Embeddings Removed, 100% BM25 Mode)")
 
 # ==============================================================================
-# RANKING FORMULA
+# RANKING FORMULA (PURE BM25 + SIGNALS)
 # ==============================================================================
 def get_ranked_results(query: str):
     q_tokens = tokenize(query)
     if not q_tokens: return []
     
     bm25_scores = bm25_index.get_scores(q_tokens) if bm25_index else np.zeros(len(raw_data))
-    semantic_scores = np.zeros(len(raw_data))
-    
-    if text_embedder and doc_embeddings is not None and len(doc_embeddings) > 0:
-        q_vec = text_embedder.encode([query])[0]
-        semantic_scores = cosine_similarity(q_vec, doc_embeddings)
 
     scored_results = []
-    
     for i, doc in enumerate(raw_data):
-        score = (bm25_scores[i] * 0.35) + \
-                (semantic_scores[i] * 0.45) + \
+        # Only Keyword Match & Signals (No semantic similarity)
+        score = (bm25_scores[i] * 0.80) + \
                 (doc["trust_score"] * 0.05) + \
                 (doc["quality_score"] * 0.05) + \
-                (doc["learning_score"] * 0.03) + \
+                (doc["learning_score"] * 0.05) + \
                 (doc["popularity"] * 0.05) + \
                 doc["search_boost"] - \
                 doc["spam_score"]
@@ -321,16 +280,20 @@ def get_best_chunks(query, full_text):
         i += (512 - 64)
         
     if not chunks: return []
-    if text_embedder:
-        q_emb = text_embedder.encode([query], convert_to_numpy=True)[0]
-        c_embs = text_embedder.encode(chunks, convert_to_numpy=True)
-        scores = cosine_similarity(q_emb, c_embs)
-        best_idx = np.argsort(scores)[::-1][:5]
-        return [chunks[idx] for idx in best_idx]
-    return chunks[:5]
+    
+    # Simple Keyword Overlap Ranking (Since vector model is removed)
+    q_terms = set(query.lower().split())
+    scored_chunks = []
+    for c in chunks:
+        c_lower = c.lower()
+        score = sum(1 for q in q_terms if q in c_lower)
+        scored_chunks.append((score, c))
+        
+    scored_chunks.sort(key=lambda x: x[0], reverse=True)
+    return [c for score, c in scored_chunks[:5]]
 
 # ==============================================================================
-# TRUE AGENTIC RAG PLANNER (Max 10 Hops)
+# TRUE AGENTIC RAG PLANNER
 # ==============================================================================
 async def agent_planner_call(prompt):
     if not pipe: return "ACTION: ANSWER"
@@ -529,7 +492,7 @@ async def api_suggestions(q: str = Query(...)):
 async def api_stats():
     mem = psutil.virtual_memory()
     return {
-        "engine": "Nebulix v6 HF Production (SmolLM2 360M Inference-Only)",
+        "engine": "Nebulix v6 HF Production (No Vectors, BM25 Only, No Dataset split='train' error)",
         "documents": len(raw_data),
         "cpu_usage_percent": psutil.cpu_percent(),
         "ram_usage_percent": mem.percent,