Update app.py

app.py

@@ -1,604 +1,295 @@
-# ==============================================================================
-# NEBULIX SEARCH ENGINE v6 FINAL (ADVANCED RANKING & SAFE SEARCH)
-# Target: Hugging Face Spaces | 2 vCPU • 16 GB RAM • True Agentic RAG
-# Model: Qwen2.5-0.5B-Instruct (Inference Only)
-# Ranking: Advanced Algorithmic (Domain Trust, Proxy Penalty) + BM25 + Semantic
-# Includes ALL API Endpoints (Popular, Trending, Suggestions, Clicks, etc.)
-# ==============================================================================
 import json
 import uuid
 import asyncio
 import os
 import uvicorn
 import re
-import time
-import pickle
-import sqlite3
-import unicodedata
-import threading
-import numpy as np
-import psutil
+import multiprocessing
+import httpx
 import urllib.parse
-from datetime import datetime
+import xml.etree.ElementTree as ET
+from collections import deque
 from contextlib import asynccontextmanager
-from fastapi import FastAPI, Query, Request, Form, BackgroundTasks
-from fastapi.responses import JSONResponse, StreamingResponse
+from datetime import datetime
+from fastapi import FastAPI, Query, BackgroundTasks, Request
 from fastapi.middleware.cors import CORSMiddleware
-from rank_bm25 import BM25Okapi
-from cachetools import TTLCache, LRUCache
-import hashlib
-from huggingface_hub import HfApi, hf_hub_download
-
-# ==============================================================================
-# CPU & PYTORCH OPTIMIZATIONS (NO TRAINING)
-# ==============================================================================
-os.environ["OMP_NUM_THREADS"] = "1"
-os.environ["OPENBLAS_NUM_THREADS"] = "1"
-os.environ["MKL_NUM_THREADS"] = "1"
-
-try:
-    import torch
-    from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, TextIteratorStreamer
-    from sentence_transformers import SentenceTransformer
-    
-    # 🔥 STRICT NO-TRAINING LOCK 🔥
-    torch.set_num_threads(1)
-    torch.set_grad_enabled(False) # Strict Inference Mode
-    ML_AVAILABLE = True
-except ImportError:
-    ML_AVAILABLE = False
-    print("⚠️ ML packages missing. Check requirements.txt")
-
-# ==============================================================================
-# SQLITE ANALYTICS SETUP
-# ==============================================================================
-DB_PATH = "analytics.db"
-def init_db():
-    conn = sqlite3.connect(DB_PATH)
-    conn.execute('CREATE TABLE IF NOT EXISTS clicks (url TEXT, type TEXT, ts DATETIME)')
-    conn.execute('CREATE TABLE IF NOT EXISTS logs (query TEXT, latency REAL, ts DATETIME)')
-    conn.execute('CREATE TABLE IF NOT EXISTS planner_logs (query TEXT, hops INTEGER, decision_log TEXT, ts DATETIME)')
-    conn.commit()
-    conn.close()
-
-def log_to_db(query, args=()):
+from huggingface_hub import hf_hub_download
+
+# Elasticsearch Import
+from elasticsearch import AsyncElasticsearch, helpers
+
+# AI Models Imports
+import torch
+from transformers import pipeline
+
+# --- 1. SETUP & CONFIGURATION ---
+TOTAL_CORES = multiprocessing.cpu_count()
+AI_CORES = max(1, int(TOTAL_CORES * 0.8))
+torch.set_num_threads(AI_CORES) 
+
+print(f"🖥️ Allocating {AI_CORES} cores to AI.")
+
+# Initialize Elasticsearch Client
+es = AsyncElasticsearch(["http://localhost:9200"])
+INDEX_NAME = "nebulix_search_index"
+
+MAX_CONCURRENT_AI = max(1, AI_CORES // 2) 
+ai_semaphore = asyncio.Semaphore(MAX_CONCURRENT_AI)
+ai_generator = None
+
+ADULT_KEYWORDS = {"porn", "sex", "nude", "nsfw", "xxx", "adult", "erotic"}
+HF_TOKEN = os.getenv("HF_TOKEN")
+
+# --- 2. EXTERNAL SMART APIs ---
+async def get_currency_conversion(amount, base, target):
     try:
-        conn = sqlite3.connect(DB_PATH)
-        conn.execute(query, args)
-        conn.commit()
-        conn.close()
-    except: pass
-
-# ==============================================================================
-# GLOBAL DATA & CACHES
-# ==============================================================================
-raw_data = []
-url_to_data = {}
-bm25_index = None
-trie_root = {}
-doc_embeddings = None
-text_embedder = None
-
-SEARCH_CACHE = TTLCache(maxsize=10000, ttl=1800) # 30 Min TTL
-AI_ANSWER_CACHE = LRUCache(maxsize=2000)
-
-tokenizer = None
-pipe = None
-AI_SEMAPHORE = asyncio.Semaphore(1)
-
-CLICK_TRACKER = {}
-STOP_WORDS = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "is"}
-
-ADULT_KEYWORDS = {"porn", "sex", "xxx", "nsfw", "nude", "xvideos", "pornhub", "adult", "erotic"}
-
-# ==============================================================================
-# UTILS & SAFE SEARCH LOGIC
-# ==============================================================================
-def is_adult_content(text: str) -> bool:
-    text_lower = text.lower()
-    return any(word in text_lower for word in ADULT_KEYWORDS)
-
-def update_dynamic_fields(doc):
-    human = doc.get("human_opens", 0)
-    ai = doc.get("ai_opens", 0)
-    crawl = doc.get("crawler_opens", 0)
-    bot = doc.get("bot_opens", 0)
-    
-    doc["total_opens"] = human + ai + crawl + bot
-    doc["popularity"] = human + (ai * 2) + (crawl * 0.1) + (bot * 0)
-    
-    total_interactive = human + ai
-    ai_percent = (ai / total_interactive * 100) if total_interactive > 0 else 0
-    doc["ai_open_percent"] = ai_percent
-    
-    tags = set()
-    if doc.get("trust_score", 50) >= 80: tags.add("Trusted")
-    if doc.get("quality_score", 50) >= 80: tags.add("High Quality")
-    if doc.get("domain_quality", 50) >= 90: tags.add("Verified")
-    if doc.get("total_opens", 0) >= 500: tags.add("Popular")
-    
-    if ai >= 25: tags.add("Opened by AI")
-    if total_interactive > 10 and ai_percent >= 80: tags.add("AI Favorite")
-    
-    doc["special_tags"] = list(tags)
-
-def normalize_text(text: str) -> str:
-    if not text: return ""
-    text = unicodedata.normalize('NFKD', str(text)).encode('ASCII', 'ignore').decode('utf-8')
-    return re.sub(r'<[^>]+>', '', text.lower())
-
-def tokenize(text: str) -> list:
-    words = re.findall(r'\b[a-z0-9]+\b', normalize_text(text))
-    return [w for w in words if w not in STOP_WORDS]
-
-def add_to_trie(word):
-    node = trie_root
-    for char in word:
-        if char not in node: node[char] = {}
-        node = node[char]
-    node['_end'] = True
-
-def cosine_similarity(vec_a, mat_b):
-    if len(mat_b) == 0: return np.array([])
-    return np.dot(mat_b, vec_a) / (np.linalg.norm(mat_b, axis=1) * np.linalg.norm(vec_a) + 1e-10)
-
-def parse_json_field(val):
-    if isinstance(val, dict): return val
-    if isinstance(val, str):
-        try: return json.loads(val)
-        except: return {}
-    return {}
-
-def build_searchable_text(doc):
-    parts = [doc.get("title", ""), doc.get("description", ""), doc.get("tag", "")]
-    for k, v in doc.get("meta_tags", {}).items():
-        if isinstance(v, str): parts.append(v)
-    for k, v in doc.get("og_tags", {}).items():
-        if isinstance(v, str): parts.append(v)
-    return " ".join(p for p in parts if p).strip()
-
-def build_runtime_record(doc):
-    r = doc.copy()
-    r["meta_tags"] = parse_json_field(doc.get("meta_tags"))
-    r["og_tags"] = parse_json_field(doc.get("og_tags"))
-    r["specific_data"] = parse_json_field(doc.get("specific_data"))
-    
-    r["images"] = doc.get("images") or []
-    r["videos"] = doc.get("videos") or []
-    r["documents"] = doc.get("documents") or []
-    r["tag"] = doc.get("tag", "general")
-    r["searchable_text"] = build_searchable_text(r)
-    
-    # Safe Search Tagging
-    r["is_adult"] = is_adult_content(r["searchable_text"] + " " + r.get("url", ""))
-    
-    r["human_opens"] = r.get("human_opens", 0)
-    r["ai_opens"] = r.get("ai_opens", 0)
-    r["crawler_opens"] = r.get("crawler_opens", 0)
-    r["bot_opens"] = r.get("bot_opens", 0)
-    r["trust_score"] = r.get("trust_score", 50.0)
-    r["quality_score"] = r.get("quality_score", 50.0)
-    r["learning_score"] = r.get("learning_score", 50.0)
-    r["search_boost"] = r.get("search_boost", 0.0)
-    r["spam_score"] = r.get("spam_score", 0.0)
-    r["domain_quality"] = r.get("domain_quality", 50.0)
-    r["last_opened"] = r.get("last_opened", datetime.utcnow().isoformat())
-    r["language"] = r.get("language", "English")
-    r["country"] = r.get("country", "Global")
-    
-    update_dynamic_fields(r)
-    return r
-
-# ==============================================================================
-# DIRECT DATASET LOADING (No Train Split Errors)
-# ==============================================================================
-def load_dataset_direct():
-    global bm25_index, raw_data, url_to_data
-    hf_token = os.getenv("HF_TOKEN")
-    repo_id = "Nebulixlabs/Ultimate-Search-Engine-Dataset"
-    
+        url = f"https://cdn.jsdelivr.net/npm/@fawazahmed0/currency-api@latest/v1/currencies/{base}.json"
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(url, timeout=5.0)
+            if resp.status_code == 200:
+                rate = resp.json()[base][target]
+                return {"type": "currency", "content": f"💱 {amount} {base.upper()} = {float(amount) * rate:.2f} {target.upper()}"}
+    except Exception: pass
+    return None
+
+async def get_stock_crypto_price(query):
     try:
-        print("☁️ Fetching raw data files directly from Hugging Face Hub...")
-        api = HfApi()
-        files_in_repo = api.list_repo_files(repo_id, repo_type="dataset", token=hf_token)
-        jsonl_files = [f for f in files_in_repo if f.endswith(".jsonl") or f.endswith(".json")]
-        
-        corpus_tokens = []
-        texts_to_embed = []
-        for file_name in jsonl_files:
-            print(f"📥 Downloading {file_name}...")
-            local_path = hf_hub_download(repo_id, file_name, repo_type="dataset", token=hf_token)
-            
-            with open(local_path, "r", encoding="utf-8") as f:
-                for line in f:
-                    if not line.strip(): continue
-                    try:
-                        doc = build_runtime_record(json.loads(line))
-                        raw_data.append(doc)
-                        url_to_data[doc["url"]] = doc
-                        
-                        tokens = tokenize(doc["searchable_text"])
-                        corpus_tokens.append(tokens)
-                        texts_to_embed.append(doc["searchable_text"])
-                        for t in tokens: add_to_trie(t)
-                    except Exception: pass
-        
-        bm25_index = BM25Okapi(corpus_tokens)
-        return texts_to_embed
-    except Exception as e:
-        print(f"❌ Failed to load dataset: {e}")
-        return []
-
-def init_system():
-    global tokenizer, pipe, text_embedder, doc_embeddings
-    init_db()
-    print("🚀 [1/3] Initializing System & Downloading Data...")
+        search_url = f"https://query2.finance.yahoo.com/v1/finance/search?q={urllib.parse.quote(query)}"
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(search_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=5.0)
+            if resp.status_code == 200 and resp.json()['quotes']:
+                quote = resp.json()['quotes'][0]
+                symbol = quote.get('symbol')
+                price_url = f"https://query1.finance.yahoo.com/v8/finance/chart/{symbol}"
+                price_resp = await client.get(price_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=5.0)
+                if price_resp.status_code == 200:
+                    data = price_resp.json()['chart']['result'][0]['meta']
+                    return {"type": "finance", "content": f"📈 {quote.get('shortname')} ({symbol}) Price: {data['regularMarketPrice']} {data['currency']}"}
+    except Exception: pass
+    return None
+
+async def perform_translation(text: str, target_lang: str, source_lang: str = "auto"):
+    try:
+        url = "https://translate.googleapis.com/translate_a/single"
+        params = {"client": "gtx", "sl": source_lang, "tl": target_lang, "dt": "t", "q": text}
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(url, params=params, timeout=5.0)
+            if resp.status_code == 200:
+                translated_text = "".join([s[0] for s in resp.json()[0]])
+                return {"source_lang": source_lang, "target_lang": target_lang, "original": text, "translated": translated_text}
+    except Exception: pass
+    return None
+
+async def analyze_query_for_apis(query: str):
+    q_lower = query.lower().strip()
     
-    texts_to_embed = load_dataset_direct()
+    translate_match = re.search(r'^translate\s+(.*?)(?:\s+to\s+([a-zA-Z]+))?$', q_lower)
+    if translate_match:
+        return {"type": "translation", "content": f"🗣️ Translation: {(await perform_translation(translate_match.group(1).strip(), translate_match.group(2) or 'en'))['translated']}"}
+
+    curr_match = re.search(r'convert\s+([0-9.]+)\s+([a-z]{3})\s+(?:to|in)\s+([a-z]{3})', q_lower)
+    if curr_match:
+        return await get_currency_conversion(*curr_match.groups())
+
+    if any(k in q_lower for k in ["price of", "stock price", "crypto price"]):
+        return await get_stock_crypto_price(q_lower.replace("price of", "").replace("stock price", "").replace("crypto price", "").strip())
+
+    if "time" in q_lower and "now" in q_lower:
+        return {"type": "time", "content": f"⏰ Current Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"}
+
+    return None
+
+# --- 3. ELASTICSEARCH DATASET LOAD & INDEX ---
+async def load_and_index_dataset():
+    print("📥 Downloading Private Dataset from Hugging Face...")
+    dataset_path = "dataset.jsonl"
     
-    if ML_AVAILABLE:
-        print("🧠 [2/3] Loading Fast Semantic Embedder (all-MiniLM-L6-v2)...")
-        text_embedder = SentenceTransformer('all-MiniLM-L6-v2', device='cpu')
-        
-        # In-memory fast semantic indexing
-        if texts_to_embed:
-            doc_embeddings = text_embedder.encode(texts_to_embed, convert_to_numpy=True)
-            
-        print("🔥 [3/3] Loading Qwen2.5-0.5B-Instruct for Agentic RAG...")
+    if HF_TOKEN:
         try:
-            model_id = "Qwen/Qwen2.5-0.5B-Instruct"
-            tokenizer = AutoTokenizer.from_pretrained(model_id)
-            model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32)
-            model.eval()
-            pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, device=-1)
-            print("✅ Qwen Model successfully loaded.")
+            dataset_path = await asyncio.to_thread(
+                hf_hub_download,
+                repo_id="Nebulixlabs/Ultimate-Search-Engine-Dataset",
+                filename="dataset.jsonl",
+                repo_type="dataset",
+                token=HF_TOKEN 
+            )
+            print("✅ Private Dataset Downloaded Successfully!")
         except Exception as e:
-            print(f"⚠️ LLM loading failed: {e}")
-
-# ==============================================================================
-# ALGORITHMIC RANKING (DOMAIN TRUST + PROXY PENALTY + BM25 + SEMANTIC)
-# ==============================================================================
-def get_domain(url):
-    try: return urllib.parse.urlparse(url).netloc.lower()
-    except: return ""
-
-def calculate_domain_trust(url: str, query_terms: list):
-    domain = get_domain(url)
-    if not domain: return 0
-    
-    trust_score = 0
-    domain_parts = domain.split('.')
-    
-    # 1. Exact Brand/Domain Match (Massive Boost for official domains)
-    for qt in query_terms:
-        if qt in domain_parts and len(qt) > 2:
-            trust_score += 150  
-            
-    # 2. Proxy / Fake Mirror Penalty
-    proxy_indicators = ["proxy", "mirror", "unblock", "alt", "fake", "ip"]
-    if any(p in domain for p in proxy_indicators):
-        trust_score -= 200  
-        
-    # 3. High Authority TLDs
-    if domain.endswith(".gov") or domain.endswith(".edu"): trust_score += 30
-    elif domain.endswith(".org") or domain.endswith(".com"): trust_score += 10
-    
-    # 4. Spam Penalty for long subdomains
-    if len(domain_parts) > 3: trust_score -= 20 
-        
-    return trust_score
+            print(f"⚠️ Hugging Face Download failed: {e}")
+
+    # Check if index exists, if not create with SEO boosting map
+    if not await es.indices.exists(index=INDEX_NAME):
+        print("⚡ Creating Elasticsearch Index...")
+        await es.indices.create(index=INDEX_NAME, body={
+            "mappings": {
+                "properties": {
+                    "title": {"type": "text", "analyzer": "standard"},
+                    "description": {"type": "text", "analyzer": "standard"},
+                    "tag": {"type": "keyword"}, # Keyword is exact match
+                    "url": {"type": "keyword"}
+                }
+            }
+        })
+    else:
+        print("⚡ Elasticsearch Index already exists. Ready.")
 
-def get_ranked_results(query: str):
-    # Safe Search Check
-    is_safe_query = not is_adult_content(query)
-    
-    q_tokens = tokenize(query)
-    if not q_tokens: return []
-    query_terms_lower = [q.lower() for q in query.split()]
-    
-    bm25_scores = bm25_index.get_scores(q_tokens) if bm25_index else np.zeros(len(raw_data))
-    semantic_scores = np.zeros(len(raw_data))
+    print("🔄 Loading Data into Elasticsearch...")
+    actions = []
     
-    if text_embedder and doc_embeddings is not None and len(doc_embeddings) > 0:
-        q_vec = text_embedder.encode([query])[0]
-        semantic_scores = cosine_similarity(q_vec, doc_embeddings)
+    if os.path.exists(dataset_path):
+        with open(dataset_path, "r", encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    try:
+                        doc = json.loads(line)
+                        action = {
+                            "_op_type": "index",
+                            "_index": INDEX_NAME,
+                            "_source": doc # Pushing ENTIRE raw dictionary
+                        }
+                        actions.append(action)
+                    except: pass
+                    
+        if actions:
+            await helpers.async_bulk(es, actions)
+            print(f"🚀 Indexed {len(actions)} records into Elasticsearch!")
+    else:
+        print("❌ Dataset file not found!")
+
+def load_ai_models():
+    global ai_generator
+    print("🤖 Loading RAG AI Model...")
+    try:
+        ai_generator = pipeline("text-generation", model="Qwen/Qwen2.5-0.5B-Instruct", device=-1, dtype=torch.float32)
+        print("✅ AI Model Loaded Successfully!")
+    except Exception as e:
+        print("❌ AI Model Failed:", e)
 
-    scored_results = []
-    
-    for i, doc in enumerate(raw_data):
-        # Adult filter logic
-        if is_safe_query and doc.get("is_adult"):
-            continue
-            
-        domain_trust = calculate_domain_trust(doc["url"], query_terms_lower)
-        
-        # Advanced Hybrid Formula
-        score = (bm25_scores[i] * 2.0) + \
-                (semantic_scores[i] * 15.0) + \
-                domain_trust + \
-                (doc.get("trust_score", 50) * 0.1) + \
-                (doc.get("popularity", 0) * 0.05) + \
-                doc.get("search_boost", 0) - doc.get("spam_score", 0)
-                
-        if score > -50: # Ignore completely heavily penalized spam
-            scored_results.append((score, doc))
-
-    scored_results.sort(key=lambda x: x[0], reverse=True)
-    
-    formatted = []
-    for s, d in scored_results[:100]:
-        formatted.append({
-            "url": d["url"], "title": d["title"], "description": d["description"],
-            "domain": get_domain(d["url"]), "favicon": d.get("favicon", ""),
-            "language": d.get("language", "English"), "country": d.get("country", "Global"),
-            "images": d.get("images", []), "videos": d.get("videos", []), 
-            "tag": d.get("tag", "general"),
-            "is_adult": d.get("is_adult", False),
-            "last_opened": d["last_opened"], "trust_score": round(d.get("trust_score", 50), 2),
-            "quality_score": round(d.get("quality_score", 50), 2), 
-            "learning_score": round(d.get("learning_score", 50), 2),
-            "human_opens": d["human_opens"], "ai_opens": d["ai_opens"],
-            "total_opens": d["total_opens"], "special_tags": d.get("special_tags", []),
-            "match_score": round(s, 2), 
-            "searchable_text": d.get("searchable_text", "")
-        })
-    return formatted
+# --- 4. ELASTICSEARCH SEARCH LOGIC (SEO & SMART FILTER) ---
+async def perform_core_search(query: str):
+    if not await es.indices.exists(index=INDEX_NAME):
+        return [], "Search index not ready.", 0
 
-def get_best_chunks(query, full_text):
-    if not full_text: return []
-    words = full_text.split()
-    chunks = [" ".join(words[i:i + 256]) for i in range(0, len(words), 200)]
+    user_wants_adult = any(word in query.lower() for word in ADULT_KEYWORDS)
     
-    if text_embedder and chunks:
-        q_emb = text_embedder.encode([query], convert_to_numpy=True)[0]
-        c_embs = text_embedder.encode(chunks, convert_to_numpy=True)
-        scores = cosine_similarity(q_emb, c_embs)
-        best_idx = np.argsort(scores)[::-1][:3]
-        return [chunks[idx] for idx in best_idx]
-    return chunks[:3]
-
-# ==============================================================================
-# TRUE AGENTIC RAG PLANNER (QWEN)
-# ==============================================================================
-async def agent_planner_call(prompt):
-    if not pipe: return "ACTION: ANSWER"
-    res = await asyncio.to_thread(
-        pipe, prompt, max_new_tokens=15, temperature=0.1, do_sample=True, return_full_text=False
-    )
-    generated = res[0]["generated_text"].strip()
-    return generated.split('\n')[0]
-
-async def execute_agentic_rag(query, candidates, debug=False):
-    context_memory = ""
-    sources_used = []
-    visited_idx = set()
-    debug_log = []
-    
-    metadata_list = "\n".join([f"ID {i} | URL: {d['url']} | Title: {d['title']}" for i, d in enumerate(candidates[:10])])
+    # 🎯 Google-Grade SEO Query Construction
+    es_query = {
+        "query": {
+            "bool": {
+                "must": [
+                    {
+                        "multi_match": {
+                            "query": query,
+                            # Field Boosting for SEO: Title gets 4x points, Desc 2x, Meta 1x
+                            "fields": ["title^4", "description^2", "meta_tags.description^1", "url"],
+                            "fuzziness": "AUTO" # Auto typo correction (e.g. githb -> github)
+                        }
+                    }
+                ],
+                "must_not": []
+            }
+        },
+        "size": 100 # Can fetch up to 100 dynamically based on match
+    }
+
+    # 🛡️ Smart Tag Filter (Hide adult content if user didn't ask for it)
+    if not user_wants_adult:
+        es_query["query"]["bool"]["must_not"].extend([
+            {"match": {"tag": "adult"}},
+            {"match": {"tag": "nsfw"}}
+        ])
+
+    response = await es.search(index=INDEX_NAME, body=es_query)
+    hits = response["hits"]["hits"]
     
-    async with AI_SEMAPHORE:
-        for hop in range(5):
-            sys_prompt = f"Query: {query}\nURLs:\n{metadata_list}\nMemory:{context_memory}\nAction (Type OPEN(id) or ANSWER): "
-            
-            decision = await agent_planner_call(sys_prompt)
-            debug_log.append(f"Hop {hop+1}: {decision}")
-            
-            open_match = re.search(r'OPEN\((\d+)\)', decision, re.IGNORECASE)
-            if open_match:
-                idx = int(open_match.group(1))
-                if idx < len(candidates) and idx not in visited_idx:
-                    visited_idx.add(idx)
-                    doc = candidates[idx]
-                    best_chunks = get_best_chunks(query, doc.get("searchable_text", ""))
-                    context_memory += f"\n[From {doc['domain']}]:\n" + "\n".join(best_chunks) + "\n"
-                    
-                    if doc["url"] in url_to_data:
-                        url_to_data[doc["url"]]["ai_opens"] += 1
-                        url_to_data[doc["url"]]["learning_score"] += 1.0
-                        update_dynamic_fields(url_to_data[doc["url"]])
-                        
-                    sources_used.append({
-                        "url": doc["url"], "title": doc["title"], "domain": doc["domain"],
-                        "special_tags": doc["special_tags"], "favicon": doc["favicon"]
-                    })
-                else: break
-            else: break
-            
-        log_to_db("INSERT INTO planner_logs (query, hops, decision_log, ts) VALUES (?, ?, ?, ?)", 
-                 (query, len(visited_idx), " | ".join(debug_log), datetime.utcnow()))
-                 
-        final_prompt = f"You are a helpful AI assistant. Use ONLY the provided Memory to answer the query.\nQuery: {query}\nMemory:\n{context_memory}\n\nAnswer:\n"
+    best_score = hits[0]["_score"] if hits else 0
+    prediction = "High confidence" if best_score > 10 else "Medium confidence" if best_score > 3 else "Low confidence"
+
+    final_results = []
+    for hit in hits:
+        # RETURNING THE ENTIRE DOCUMENT (Favicon, images, specific_data, etc.)
+        doc = hit["_source"]
+        doc["match_score"] = hit["_score"] # Add ES score for frontend
+        final_results.append(doc)
         
-    return final_prompt, sources_used, debug_log
+    return final_results, prediction, best_score
+
+# --- AI GENERATION HELPER ---
+def _generate_text(messages, max_tokens: int):
+    result = ai_generator(messages, max_new_tokens=max_tokens, do_sample=True, temperature=0.3, top_p=0.85)
+    return result[0]['generated_text'][-1]['content']
+
+async def run_ai_task(messages, max_tokens: int):
+    if not ai_generator: return "⚠️ AI Model is starting up."
+    async with ai_semaphore:
+        return await asyncio.to_thread(_generate_text, messages, max_tokens)
 
-# ==============================================================================
-# FASTAPI APP & ALL ENDPOINTS
-# ==============================================================================
+# --- 5. FASTAPI APP & ENDPOINTS ---
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    asyncio.create_task(asyncio.to_thread(init_system))
+    # Run setup tasks
+    await load_and_index_dataset()
+    asyncio.create_task(asyncio.to_thread(load_ai_models))
     yield
+    await es.close()
 
-app = FastAPI(title="Nebulix v6 Qwen SafeSearch", lifespan=lifespan)
+app = FastAPI(title="Nebulix Search Engine (Powered by Elasticsearch)", lifespan=lifespan)
 app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
 
-@app.middleware("http")
-async def analytics_middleware(request: Request, call_next):
-    start_time = time.perf_counter()
-    response = await call_next(request)
-    latency = (time.perf_counter() - start_time) * 1000
-    if "/search" in request.url.path or "/aimode" in request.url.path:
-        q = request.query_params.get("q", "")
-        log_to_db("INSERT INTO logs (query, latency, ts) VALUES (?, ?, ?)", (q, latency, datetime.utcnow()))
-    return response
+@app.get("/translate")
+async def translate_api(q: str = Query(...), target: str = Query("en"), source: str = Query("auto")):
+    result = await perform_translation(q, target, source)
+    return {"status": "success", "data": result} if result else {"status": "error"}
 
 @app.get("/search")
-async def api_search(q: str = Query(...)):
-    cache_key = q.lower()
-    if cache_key in SEARCH_CACHE: return SEARCH_CACHE[cache_key]
+async def pure_search(q: str = Query(...)):
+    req_id = str(uuid.uuid4())
+    smart_data = await analyze_query_for_apis(q)
+    results, prediction, _ = await perform_core_search(q)
     
-    results = get_ranked_results(q)
-    for r in results: r.pop("searchable_text", None)
-    
-    resp = {"query": q, "results": results}
-    SEARCH_CACHE[cache_key] = resp
-    return resp
+    return {
+        "request_id": req_id,
+        "query": q,
+        "smart_widget": smart_data,
+        "results_count": len(results),
+        "results": results  # Now returns ENTIRE JSON structure including Favicon, Videos, Tag, etc.
+    }
 
 @app.get("/aimode")
-async def api_aimode(q: str = Query(...), debug: bool = False):
-    q_hash = hashlib.md5(q.encode()).hexdigest()
-    if q_hash in AI_ANSWER_CACHE: return AI_ANSWER_CACHE[q_hash]
-    
-    results = get_ranked_results(q)
-    final_prompt, sources, debug_log = await execute_agentic_rag(q, results, debug)
+async def ai_mode_short(q: str = Query(...)):
+    req_id = str(uuid.uuid4())
+    smart_data = await analyze_query_for_apis(q)
+    results, _, _ = await perform_core_search(q)
     
-    if not pipe: return {"Answer": "Mocked", "Sources": sources}
-    
-    async with AI_SEMAPHORE:
-        res = await asyncio.to_thread(
-            pipe, final_prompt, max_new_tokens=1024, temperature=0.2, do_sample=True, return_full_text=False
-        )
-        answer = res[0]["generated_text"].strip()
+    if not results and not smart_data: 
+        return {"request_id": req_id, "ai_response": "I couldn't find any results.", "results": []}
         
-    resp = {
-        "Answer": answer[:150] + "...",
-        "Summary": answer[:300],
-        "Detailed Explanation": answer,
-        "Sources": sources
-    }
-    if debug: resp["Debug_Planner"] = debug_log
-    AI_ANSWER_CACHE[q_hash] = resp
-    return resp
+    context = (f"Live Info: {smart_data['content']}. " if smart_data else "") + " ".join([f"{r.get('title', '')} - {r.get('description', '')}" for r in results[:2]])
+    messages = [
+        {"role": "system", "content": "Provide a brief 2-3 sentence summary based only on the provided context."},
+        {"role": "user", "content": f"Query: {q}\nContext: {context}"}
+    ]
+    
+    ai_reply = await run_ai_task(messages, max_tokens=150)
+    return {"request_id": req_id, "ai_response": ai_reply, "smart_widget": smart_data, "results": results}
 
 @app.get("/aimodechat")
-async def api_aimodechat(q: str = Query(...)):
-    results = get_ranked_results(q)
-    final_prompt, sources, _ = await execute_agentic_rag(q, results)
+async def ai_mode_chat(q: str = Query(...)):
+    req_id = str(uuid.uuid4())
+    smart_data = await analyze_query_for_apis(q)
+    results, _, _ = await perform_core_search(q)
     
-    async def true_event_stream():
-        if pipe and tokenizer:
-            async with AI_SEMAPHORE:
-                streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
-                def generate():
-                    try: pipe(final_prompt, max_new_tokens=1024, temperature=0.3, streamer=streamer, return_full_text=False)
-                    except: pass
-                
-                thread = threading.Thread(target=generate)
-                thread.start()
-                
-                while True:
-                    try:
-                        text = await asyncio.to_thread(next, streamer)
-                        if text: yield f"data: {text}\n\n"
-                    except StopIteration: break
-        else:
-            yield "data: [Qwen Model Missing]\n\n"
-        yield f"data: [SOURCES_JSON] {json.dumps(sources)}\n\n"
-        
-    return StreamingResponse(true_event_stream(), media_type="text/event-stream")
-
-@app.get("/popular")
-async def api_popular():
-    docs = sorted(raw_data, key=lambda x: x.get("popularity", 0), reverse=True)[:10]
-    return {"popular": [{"url": d["url"], "title": d["title"], "popularity": d["popularity"]} for d in docs]}
-
-@app.get("/ai-popular")
-async def api_ai_popular():
-    docs = sorted(raw_data, key=lambda x: x.get("ai_opens", 0), reverse=True)[:10]
-    return {"ai_popular": [{"url": d["url"], "title": d["title"], "ai_opens": d["ai_opens"], "special_tags": d["special_tags"]} for d in docs]}
-
-@app.get("/trending")
-async def api_trending():
-    docs = sorted(raw_data, key=lambda x: x.get("total_opens", 0), reverse=True)[:10]
-    return {"trending": [{"url": d["url"], "title": d["title"]} for d in docs]}
-
-@app.get("/suggestions")
-async def api_suggestions(q: str = Query(...)):
-    base_q = q.lower().split()[-1]
-    words = []
-    def dfs(node, prefix):
-        if len(words) >= 5: return
-        if node.get('_end'): words.append(prefix)
-        for char, child in node.items():
-            if char != '_end': dfs(child, prefix + char)
-            
-    node = trie_root
-    for c in base_q:
-        if c in node: node = node[c]
-        else: break
-    if node and node != trie_root: dfs(node, base_q)
-    return {"suggestions": [q.replace(base_q, w) for w in words]}
-
-@app.get("/stats")
-async def api_stats():
-    mem = psutil.virtual_memory()
-    return {
-        "engine": "Nebulix v6.5 HF Spaces Edition",
-        "llm_model": "Qwen2.5-0.5B-Instruct",
-        "ranking": "Hybrid (Algorithmic Domain Trust + BM25 + Semantic)",
-        "documents": len(raw_data),
-        "cpu_usage_percent": psutil.cpu_percent(),
-        "ram_usage_percent": mem.percent,
-        "search_cache_items": len(SEARCH_CACHE),
-        "ai_cache_items": len(AI_ANSWER_CACHE)
-    }
-
-@app.get("/health")
-async def api_health(): return {"status": "ok", "version": "6.5"}
-
-@app.get("/url-info")
-async def api_url_info(url: str = Query(...)):
-    if url in url_to_data: 
-        safe_doc = url_to_data[url].copy()
-        safe_doc.pop("searchable_text", None)
-        return safe_doc
-    return {"error": "Not Found"}
-
-@app.post("/click")
-async def api_click(req: Request):
-    data = await req.json()
-    url = data.get("url")
-    ip = req.client.host
-    track_id = f"{ip}_{url}"
-    if track_id in CLICK_TRACKER and time.time() - CLICK_TRACKER[track_id] < 1800:
-        return {"status": "ignored_duplicate"}
+    context_parts = [f"LIVE SYSTEM DATA: {smart_data['content']}"] if smart_data else []
+    for i, r in enumerate(results[:5]): 
+        context_parts.append(f"Source [{i+1}]: {r.get('title')} - {r.get('description')}")
         
-    CLICK_TRACKER[track_id] = time.time()
-    if url in url_to_data:
-        d = url_to_data[url]
-        d["human_opens"] += 1
-        d["last_opened"] = datetime.utcnow().isoformat()
-        update_dynamic_fields(d)
-        log_to_db("INSERT INTO clicks (url, type, ts) VALUES (?, ?, ?)", (url, "human", datetime.utcnow()))
-        return {"status": "success", "human_opens": d["human_opens"]}
-    return {"error": "Not found"}
-
-@app.post("/ai-open")
-async def api_ai_open(req: Request):
-    data = await req.json()
-    url = data.get("url")
-    if url in url_to_data:
-        d = url_to_data[url]
-        d["ai_opens"] += 1
-        d["learning_score"] += 1.0
-        d["last_opened"] = datetime.utcnow().isoformat()
-        update_dynamic_fields(d)
-        log_to_db("INSERT INTO clicks (url, type, ts) VALUES (?, ?, ?)", (url, "ai", datetime.utcnow()))
-        return {"status": "success"}
-    return {"error": "Not found"}
-
-@app.post("/feedback")
-async def api_feedback(req: Request):
-    data = await req.json()
-    url = data.get("url")
-    if url in url_to_data:
-        url_to_data[url]["trust_score"] += 2.0 if data.get("helpful") else -1.0
-        update_dynamic_fields(url_to_data[url])
-    return {"status": "feedback_registered"}
-
-@app.post("/search-log")
-async def api_search_log(req: Request):
-    return {"status": "logged"}
+    messages = [
+        {"role": "system", "content": "You are a Smart AI Search Assistant. Use live data and search sources to answer accurately."},
+        {"role": "user", "content": f"User Query: {q}\n\nContext & Sources:\n{chr(10).join(context_parts)}"}
+    ]
+    
+    ai_reply = await run_ai_task(messages, max_tokens=400) 
+    return {"request_id": req_id, "ai_response": ai_reply, "smart_widget": smart_data, "results": results}
 
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=7860, log_level="warning")
+    uvicorn.run(app, host="0.0.0.0", port=7860)