Update app.py

app.py

@@ -1,295 +1,551 @@
+# ==============================================================================
+# NEBULIX SEARCH ENGINE v6 FINAL (PRODUCTION READY)
+# Target: 2 vCPU • 16 GB RAM • True Agentic RAG • AI Streaming • SQLite Analytics
+# Includes Hugging Face Dataset Integration & SmolLM2 (Million Params) integration
+# ==============================================================================
 import json
 import uuid
 import asyncio
 import os
 import uvicorn
 import re
+import time
+import pickle
+import sqlite3
+import unicodedata
 import multiprocessing
-import httpx
-import urllib.parse
-import xml.etree.ElementTree as ET
-from collections import deque
-from contextlib import asynccontextmanager
+import numpy as np
+import psutil
 from datetime import datetime
-from fastapi import FastAPI, Query, BackgroundTasks, Request
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, Query, Request, Form, BackgroundTasks
+from fastapi.responses import JSONResponse, StreamingResponse
 from fastapi.middleware.cors import CORSMiddleware
-from huggingface_hub import hf_hub_download
-
-# Elasticsearch Import
-from elasticsearch import AsyncElasticsearch, helpers
-
-# AI Models Imports
-import torch
-from transformers import pipeline
-
-# --- 1. SETUP & CONFIGURATION ---
-TOTAL_CORES = multiprocessing.cpu_count()
-AI_CORES = max(1, int(TOTAL_CORES * 0.8))
-torch.set_num_threads(AI_CORES) 
-
-print(f"🖥️ Allocating {AI_CORES} cores to AI.")
-
-# Initialize Elasticsearch Client
-es = AsyncElasticsearch(["http://localhost:9200"])
-INDEX_NAME = "nebulix_search_index"
-
-MAX_CONCURRENT_AI = max(1, AI_CORES // 2) 
-ai_semaphore = asyncio.Semaphore(MAX_CONCURRENT_AI)
-ai_generator = None
-
-ADULT_KEYWORDS = {"porn", "sex", "nude", "nsfw", "xxx", "adult", "erotic"}
-HF_TOKEN = os.getenv("HF_TOKEN")
-
-# --- 2. EXTERNAL SMART APIs ---
-async def get_currency_conversion(amount, base, target):
-    try:
-        url = f"https://cdn.jsdelivr.net/npm/@fawazahmed0/currency-api@latest/v1/currencies/{base}.json"
-        async with httpx.AsyncClient() as client:
-            resp = await client.get(url, timeout=5.0)
-            if resp.status_code == 200:
-                rate = resp.json()[base][target]
-                return {"type": "currency", "content": f"💱 {amount} {base.upper()} = {float(amount) * rate:.2f} {target.upper()}"}
-    except Exception: pass
-    return None
-
-async def get_stock_crypto_price(query):
+from rank_bm25 import BM25Okapi
+from cachetools import TTLCache, LRUCache
+import hashlib
+
+# --- ML & Data Imports ---
+try:
+    from datasets import load_dataset
+    from llama_cpp import Llama
+    from sentence_transformers import SentenceTransformer
+    ML_AVAILABLE = True
+except ImportError:
+    ML_AVAILABLE = False
+    print("⚠️ ML packages missing. Run: pip install llama-cpp-python sentence-transformers psutil numpy datasets")
+
+# ==============================================================================
+# 25. CPU ALLOCATION (60% Search, 40% AI on 2 vCPUs)
+# ==============================================================================
+os.environ["OMP_NUM_THREADS"] = "1"
+os.environ["OPENBLAS_NUM_THREADS"] = "1"
+os.environ["MKL_NUM_THREADS"] = "1"
+
+# ==============================================================================
+# 22. SQLITE ANALYTICS SETUP
+# ==============================================================================
+DB_PATH = "analytics.db"
+def init_db():
+    conn = sqlite3.connect(DB_PATH)
+    conn.execute('CREATE TABLE IF NOT EXISTS clicks (url TEXT, type TEXT, ts DATETIME)')
+    conn.execute('CREATE TABLE IF NOT EXISTS logs (query TEXT, latency REAL, ts DATETIME)')
+    conn.execute('CREATE TABLE IF NOT EXISTS planner_logs (query TEXT, hops INTEGER, decision_log TEXT, ts DATETIME)')
+    conn.commit()
+    conn.close()
+
+def log_to_db(query, args=()):
     try:
-        search_url = f"https://query2.finance.yahoo.com/v1/finance/search?q={urllib.parse.quote(query)}"
-        async with httpx.AsyncClient() as client:
-            resp = await client.get(search_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=5.0)
-            if resp.status_code == 200 and resp.json()['quotes']:
-                quote = resp.json()['quotes'][0]
-                symbol = quote.get('symbol')
-                price_url = f"https://query1.finance.yahoo.com/v8/finance/chart/{symbol}"
-                price_resp = await client.get(price_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=5.0)
-                if price_resp.status_code == 200:
-                    data = price_resp.json()['chart']['result'][0]['meta']
-                    return {"type": "finance", "content": f"📈 {quote.get('shortname')} ({symbol}) Price: {data['regularMarketPrice']} {data['currency']}"}
-    except Exception: pass
-    return None
-
-async def perform_translation(text: str, target_lang: str, source_lang: str = "auto"):
-    try:
-        url = "https://translate.googleapis.com/translate_a/single"
-        params = {"client": "gtx", "sl": source_lang, "tl": target_lang, "dt": "t", "q": text}
-        async with httpx.AsyncClient() as client:
-            resp = await client.get(url, params=params, timeout=5.0)
-            if resp.status_code == 200:
-                translated_text = "".join([s[0] for s in resp.json()[0]])
-                return {"source_lang": source_lang, "target_lang": target_lang, "original": text, "translated": translated_text}
-    except Exception: pass
-    return None
-
-async def analyze_query_for_apis(query: str):
-    q_lower = query.lower().strip()
+        conn = sqlite3.connect(DB_PATH)
+        conn.execute(query, args)
+        conn.commit()
+        conn.close()
+    except Exception as e: pass
+
+# ==============================================================================
+# GLOBAL DATA & CACHES
+# ==============================================================================
+raw_data = []
+url_to_data = {}
+bm25_index = None
+trie_root = {}
+
+SEARCH_CACHE = TTLCache(maxsize=10000, ttl=1800) # 30 Min TTL
+AI_ANSWER_CACHE = LRUCache(maxsize=2000)
+
+doc_embeddings = None
+text_embedder = None
+llm = None
+AI_SEMAPHORE = asyncio.Semaphore(1)
+
+CLICK_TRACKER = {}
+STOP_WORDS = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "is"}
+
+# ==============================================================================
+# MATH & SPECIAL TAGS
+# ==============================================================================
+def update_dynamic_fields(doc):
+    human = doc.get("human_opens", 0)
+    ai = doc.get("ai_opens", 0)
+    crawl = doc.get("crawler_opens", 0)
+    bot = doc.get("bot_opens", 0)
     
-    translate_match = re.search(r'^translate\s+(.*?)(?:\s+to\s+([a-zA-Z]+))?$', q_lower)
-    if translate_match:
-        return {"type": "translation", "content": f"🗣️ Translation: {(await perform_translation(translate_match.group(1).strip(), translate_match.group(2) or 'en'))['translated']}"}
-
-    curr_match = re.search(r'convert\s+([0-9.]+)\s+([a-z]{3})\s+(?:to|in)\s+([a-z]{3})', q_lower)
-    if curr_match:
-        return await get_currency_conversion(*curr_match.groups())
-
-    if any(k in q_lower for k in ["price of", "stock price", "crypto price"]):
-        return await get_stock_crypto_price(q_lower.replace("price of", "").replace("stock price", "").replace("crypto price", "").strip())
-
-    if "time" in q_lower and "now" in q_lower:
-        return {"type": "time", "content": f"⏰ Current Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"}
-
-    return None
+    doc["total_opens"] = human + ai + crawl + bot
+    doc["popularity"] = human + (ai * 2) + (crawl * 0.1) + (bot * 0)
+    
+    total_interactive = human + ai
+    ai_percent = (ai / total_interactive * 100) if total_interactive > 0 else 0
+    doc["ai_open_percent"] = ai_percent
+    
+    tags = set()
+    if doc.get("trust_score", 50) >= 80: tags.add("Trusted")
+    if doc.get("quality_score", 50) >= 80: tags.add("High Quality")
+    if doc.get("domain_quality", 50) >= 90: tags.add("Verified")
+    if doc.get("total_opens", 0) >= 500: tags.add("Popular")
+    
+    if ai >= 25: tags.add("Opened by AI")
+    if total_interactive > 10 and ai_percent >= 80: tags.add("AI Favorite")
+    
+    doc["special_tags"] = list(tags)
+
+# ==============================================================================
+# TOKENIZER & UTILS
+# ==============================================================================
+def normalize_text(text: str) -> str:
+    if not text: return ""
+    text = unicodedata.normalize('NFKD', str(text)).encode('ASCII', 'ignore').decode('utf-8')
+    return re.sub(r'<[^>]+>', '', text.lower())
+
+def tokenize(text: str) -> list:
+    words = re.findall(r'\b[a-z0-9]+\b', normalize_text(text))
+    return [w for w in words if w not in STOP_WORDS]
+
+def add_to_trie(word):
+    node = trie_root
+    for char in word:
+        if char not in node: node[char] = {}
+        node = node[char]
+    node['_end'] = True
+
+def cosine_similarity(vec_a, mat_b):
+    if len(mat_b) == 0: return np.array([])
+    return np.dot(mat_b, vec_a) / (np.linalg.norm(mat_b, axis=1) * np.linalg.norm(vec_a) + 1e-10)
+
+def detect_language(text):
+    return "English"
+
+# ==============================================================================
+# DATASET LOADING & NEW SCHEMA MAPPING
+# ==============================================================================
+def build_searchable_text(doc):
+    """Combines title, description, tag, and string values from meta/og tags into one text."""
+    parts = [doc.get("title", ""), doc.get("description", ""), doc.get("tag", "")]
+    
+    for k, v in doc.get("meta_tags", {}).items():
+        if isinstance(v, str): parts.append(v)
+            
+    for k, v in doc.get("og_tags", {}).items():
+        if isinstance(v, str): parts.append(v)
+        
+    return " ".join(p for p in parts if p).strip()
 
-# --- 3. ELASTICSEARCH DATASET LOAD & INDEX ---
-async def load_and_index_dataset():
-    print("📥 Downloading Private Dataset from Hugging Face...")
-    dataset_path = "dataset.jsonl"
+def build_runtime_record(doc):
+    r = doc.copy()
     
-    if HF_TOKEN:
-        try:
-            dataset_path = await asyncio.to_thread(
-                hf_hub_download,
-                repo_id="Nebulixlabs/Ultimate-Search-Engine-Dataset",
-                filename="dataset.jsonl",
-                repo_type="dataset",
-                token=HF_TOKEN 
-            )
-            print("✅ Private Dataset Downloaded Successfully!")
-        except Exception as e:
-            print(f"⚠️ Hugging Face Download failed: {e}")
-
-    # Check if index exists, if not create with SEO boosting map
-    if not await es.indices.exists(index=INDEX_NAME):
-        print("⚡ Creating Elasticsearch Index...")
-        await es.indices.create(index=INDEX_NAME, body={
-            "mappings": {
-                "properties": {
-                    "title": {"type": "text", "analyzer": "standard"},
-                    "description": {"type": "text", "analyzer": "standard"},
-                    "tag": {"type": "keyword"}, # Keyword is exact match
-                    "url": {"type": "keyword"}
-                }
-            }
-        })
-    else:
-        print("⚡ Elasticsearch Index already exists. Ready.")
+    r["meta_tags"] = doc.get("meta_tags", {})
+    r["og_tags"] = doc.get("og_tags", {})
+    r["images"] = doc.get("images", [])
+    r["videos"] = doc.get("videos", [])
+    r["documents"] = doc.get("documents", [])
+    r["tag"] = doc.get("tag", "general")
+    r["specific_data"] = doc.get("specific_data", {})
+    r["searchable_text"] = build_searchable_text(r)
+    
+    r["human_opens"] = r.get("human_opens", 0)
+    r["ai_opens"] = r.get("ai_opens", 0)
+    r["crawler_opens"] = r.get("crawler_opens", 0)
+    r["bot_opens"] = r.get("bot_opens", 0)
+    r["trust_score"] = r.get("trust_score", 50.0)
+    r["quality_score"] = r.get("quality_score", 50.0)
+    r["learning_score"] = r.get("learning_score", 50.0)
+    r["search_boost"] = r.get("search_boost", 0.0)
+    r["spam_score"] = r.get("spam_score", 0.0)
+    r["domain_quality"] = r.get("domain_quality", 50.0)
+    r["last_opened"] = r.get("last_opened", datetime.utcnow().isoformat())
+    r["language"] = r.get("language", detect_language(r.get("title", "")))
+    r["country"] = r.get("country", "Global")
+    
+    update_dynamic_fields(r)
+    return r
 
-    print("🔄 Loading Data into Elasticsearch...")
-    actions = []
+def init_system():
+    global bm25_index, doc_embeddings, text_embedder, llm, raw_data, url_to_data
+    init_db()
+    print("🚀 [1/4] Fetching Dataset & Building Indexes...")
     
-    if os.path.exists(dataset_path):
-        with open(dataset_path, "r", encoding="utf-8") as f:
+    CACHE_FILE = "cached_dataset.jsonl"
+    
+    if not os.path.exists(CACHE_FILE):
+        hf_token = os.getenv("HF_TOKEN")
+        if not hf_token:
+            raise ValueError("❌ HF_TOKEN environment variable is missing. Cannot fetch dataset.")
+        print("☁️ Downloading dataset from Hugging Face...")
+        dataset = load_dataset("Nebulixlabs/Ultimate-Search-Engine-Dataset", split="train", token=hf_token)
+        dataset.to_json(CACHE_FILE)
+        print("✅ Dataset successfully cached locally.")
+
+    if os.path.exists(CACHE_FILE):
+        corpus_tokens = []
+        texts_to_embed = []
+        with open(CACHE_FILE, "r", encoding="utf-8") as f:
             for line in f:
                 if line.strip():
-                    try:
-                        doc = json.loads(line)
-                        action = {
-                            "_op_type": "index",
-                            "_index": INDEX_NAME,
-                            "_source": doc # Pushing ENTIRE raw dictionary
-                        }
-                        actions.append(action)
-                    except: pass
+                    doc = build_runtime_record(json.loads(line))
+                    raw_data.append(doc)
+                    url_to_data[doc["url"]] = doc
                     
-        if actions:
-            await helpers.async_bulk(es, actions)
-            print(f"🚀 Indexed {len(actions)} records into Elasticsearch!")
-    else:
-        print("❌ Dataset file not found!")
-
-def load_ai_models():
-    global ai_generator
-    print("🤖 Loading RAG AI Model...")
-    try:
-        ai_generator = pipeline("text-generation", model="Qwen/Qwen2.5-0.5B-Instruct", device=-1, dtype=torch.float32)
-        print("✅ AI Model Loaded Successfully!")
-    except Exception as e:
-        print("❌ AI Model Failed:", e)
-
-# --- 4. ELASTICSEARCH SEARCH LOGIC (SEO & SMART FILTER) ---
-async def perform_core_search(query: str):
-    if not await es.indices.exists(index=INDEX_NAME):
-        return [], "Search index not ready.", 0
+                    tokens = tokenize(doc["searchable_text"])
+                    corpus_tokens.append(tokens)
+                    texts_to_embed.append(doc["searchable_text"])
+                    for t in tokens: add_to_trie(t)
 
-    user_wants_adult = any(word in query.lower() for word in ADULT_KEYWORDS)
+        bm25_index = BM25Okapi(corpus_tokens)
+        
+        if ML_AVAILABLE:
+            print("🧠 [2/4] Loading BAAI/bge-small-en-v1.5...")
+            text_embedder = SentenceTransformer('BAAI/bge-small-en-v1.5', device='cpu')
+
+            if os.path.exists("embeddings.bin"):
+                print("💾 [3/4] Loading Precomputed Embeddings...")
+                with open("embeddings.bin", "rb") as fb: doc_embeddings = pickle.load(fb)
+            else:
+                print("⚙️ [3/4] Generating Embeddings (One-Time)...")
+                doc_embeddings = text_embedder.encode(texts_to_embed, convert_to_numpy=True)
+                with open("embeddings.bin", "wb") as fb: pickle.dump(doc_embeddings, fb)
+                with open("metadata.json", "w") as fm: json.dump({"count": len(doc_embeddings)}, fm)
+            
+            # =============== CHANGED TO SMOLLM2 (360 Million Params) ===============
+            print("🔥 [4/4] Loading Llama.cpp SmolLM2-360M GGUF (Lightweight CPU Model)...")
+            try:
+                llm = Llama(
+                    model_path="./models/smollm2-360m-instruct-q8_0.gguf", # Update filename if downloaded differently
+                    n_threads=1, 
+                    n_ctx=3000, 
+                    verbose=False
+                )
+            except:
+                print("⚠️ LLM not found locally in ./models/, mocked mode active.")
+    print("✅ Nebulix v6 Ready.")
+
+# ==============================================================================
+# RANKING FORMULA
+# ==============================================================================
+def get_ranked_results(query: str):
+    q_tokens = tokenize(query)
+    if not q_tokens: return []
     
-    # 🎯 Google-Grade SEO Query Construction
-    es_query = {
-        "query": {
-            "bool": {
-                "must": [
-                    {
-                        "multi_match": {
-                            "query": query,
-                            # Field Boosting for SEO: Title gets 4x points, Desc 2x, Meta 1x
-                            "fields": ["title^4", "description^2", "meta_tags.description^1", "url"],
-                            "fuzziness": "AUTO" # Auto typo correction (e.g. githb -> github)
-                        }
-                    }
-                ],
-                "must_not": []
-            }
-        },
-        "size": 100 # Can fetch up to 100 dynamically based on match
-    }
-
-    # 🛡️ Smart Tag Filter (Hide adult content if user didn't ask for it)
-    if not user_wants_adult:
-        es_query["query"]["bool"]["must_not"].extend([
-            {"match": {"tag": "adult"}},
-            {"match": {"tag": "nsfw"}}
-        ])
+    bm25_scores = bm25_index.get_scores(q_tokens) if bm25_index else np.zeros(len(raw_data))
+    semantic_scores = np.zeros(len(raw_data))
+    
+    if text_embedder and doc_embeddings is not None:
+        q_vec = text_embedder.encode([query])[0]
+        semantic_scores = cosine_similarity(q_vec, doc_embeddings)
 
-    response = await es.search(index=INDEX_NAME, body=es_query)
-    hits = response["hits"]["hits"]
+    scored_results = []
     
-    best_score = hits[0]["_score"] if hits else 0
-    prediction = "High confidence" if best_score > 10 else "Medium confidence" if best_score > 3 else "Low confidence"
-
-    final_results = []
-    for hit in hits:
-        # RETURNING THE ENTIRE DOCUMENT (Favicon, images, specific_data, etc.)
-        doc = hit["_source"]
-        doc["match_score"] = hit["_score"] # Add ES score for frontend
-        final_results.append(doc)
+    for i, doc in enumerate(raw_data):
+        score = (bm25_scores[i] * 0.35) + \
+                (semantic_scores[i] * 0.45) + \
+                (doc["trust_score"] * 0.05) + \
+                (doc["quality_score"] * 0.05) + \
+                (doc["learning_score"] * 0.03) + \
+                (doc["popularity"] * 0.05) + \
+                doc["search_boost"] - \
+                doc["spam_score"]
+                
+        scored_results.append((score, doc))
+
+    scored_results.sort(key=lambda x: x[0], reverse=True)
+    
+    formatted = []
+    for s, d in scored_results[:100]:
+        formatted.append({
+            "url": d["url"], "title": d["title"], "description": d["description"],
+            "favicon": d.get("favicon", ""), "language": d["language"], "country": d["country"],
+            "images": d.get("images", []), "videos": d.get("videos", []), 
+            "tag": d.get("tag", "general"),
+            "last_opened": d["last_opened"], "trust_score": round(d["trust_score"], 2),
+            "quality_score": round(d["quality_score"], 2), "learning_score": round(d["learning_score"], 2),
+            "human_opens": d["human_opens"], "ai_opens": d["ai_opens"],
+            "total_opens": d["total_opens"], "special_tags": d["special_tags"],
+            "match_score": round(s, 2), 
+            "searchable_text": d.get("searchable_text", "")
+        })
+    return formatted
+
+# ==============================================================================
+# CHUNKING
+# ==============================================================================
+def get_best_chunks(query, full_text):
+    if not full_text: return []
+    words = full_text.split()
+    chunks = []
+    i = 0
+    while i < len(words):
+        chunks.append(" ".join(words[i:i + 512]))
+        i += (512 - 64)
         
-    return final_results, prediction, best_score
-
-# --- AI GENERATION HELPER ---
-def _generate_text(messages, max_tokens: int):
-    result = ai_generator(messages, max_new_tokens=max_tokens, do_sample=True, temperature=0.3, top_p=0.85)
-    return result[0]['generated_text'][-1]['content']
-
-async def run_ai_task(messages, max_tokens: int):
-    if not ai_generator: return "⚠️ AI Model is starting up."
-    async with ai_semaphore:
-        return await asyncio.to_thread(_generate_text, messages, max_tokens)
+    if not chunks: return []
+    if text_embedder:
+        q_emb = text_embedder.encode([query], convert_to_numpy=True)[0]
+        c_embs = text_embedder.encode(chunks, convert_to_numpy=True)
+        scores = cosine_similarity(q_emb, c_embs)
+        best_idx = np.argsort(scores)[::-1][:5]
+        return [chunks[idx] for idx in best_idx]
+    return chunks[:5]
+
+# ==============================================================================
+# TRUE AGENTIC RAG PLANNER (Max 10 Hops)
+# ==============================================================================
+async def agent_planner_call(prompt):
+    if not llm: return "ACTION: ANSWER"
+    res = await asyncio.to_thread(llm, prompt, max_tokens=20, temperature=0.1, stop=["\n"])
+    return res["choices"][0]["text"].strip()
+
+async def execute_agentic_rag(query, candidates, debug=False):
+    context_memory = ""
+    sources_used = []
+    visited_idx = set()
+    debug_log = []
+    
+    metadata_list = "\n".join([f"ID {i} | URL: {d['url']} | Title: {d['title']}" for i, d in enumerate(candidates[:20])])
+    
+    async with AI_SEMAPHORE:
+        for hop in range(10):
+            # Prompt simplified slightly for million-parameter models to ensure strict output
+            sys_prompt = f"""System: You are a Planner. Output ONLY the action command.
+Query: {query}
+URLs:
+{metadata_list}
+Memory:
+{context_memory if context_memory else 'Empty'}
+To read a URL, type: OPEN(id)
+To answer, type: ANSWER
+Action: """
+            
+            decision = await agent_planner_call(sys_prompt)
+            debug_log.append(f"Hop {hop+1}: {decision}")
+            
+            open_match = re.search(r'OPEN\((\d+)\)', decision, re.IGNORECASE)
+            if open_match:
+                idx = int(open_match.group(1))
+                if idx < len(candidates) and idx not in visited_idx:
+                    visited_idx.add(idx)
+                    doc = candidates[idx]
+                    
+                    target_text = doc.get("searchable_text", "") or doc.get("description", "")
+                    best_chunks = get_best_chunks(query, target_text)
+                    context_memory += f"\n[From {doc['url']}]:\n" + "\n".join(best_chunks) + "\n"
+                    
+                    if doc["url"] in url_to_data:
+                        url_to_data[doc["url"]]["ai_opens"] += 1
+                        url_to_data[doc["url"]]["learning_score"] += 1.0
+                        update_dynamic_fields(url_to_data[doc["url"]])
+                    
+                    sources_used.append({
+                        "url": doc["url"], "title": doc["title"], "language": doc["language"],
+                        "country": doc["country"], "last_opened": doc["last_opened"],
+                        "human_opens": doc["human_opens"], "ai_opens": doc["ai_opens"],
+                        "trust_score": doc["trust_score"], "quality_score": doc["quality_score"],
+                        "learning_score": doc["learning_score"], "special_tags": doc["special_tags"],
+                        "favicon": doc["favicon"]
+                    })
+                else:
+                    context_memory += "\n(System: ID already read. Choose another or ANSWER.)"
+            else:
+                break
+        
+        log_to_db("INSERT INTO planner_logs (query, hops, decision_log, ts) VALUES (?, ?, ?, ?)", 
+                 (query, len(visited_idx), " | ".join(debug_log), datetime.utcnow()))
+        
+        final_prompt = f"System: Use ONLY the provided Memory to answer the query.\nQuery: {query}\nMemory:\n{context_memory}\n\nProvide Answer:\n"
+        
+    return final_prompt, sources_used, debug_log
 
-# --- 5. FASTAPI APP & ENDPOINTS ---
+# ==============================================================================
+# FASTAPI APP & ENDPOINTS
+# ==============================================================================
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    # Run setup tasks
-    await load_and_index_dataset()
-    asyncio.create_task(asyncio.to_thread(load_ai_models))
+    asyncio.create_task(asyncio.to_thread(init_system))
     yield
-    await es.close()
 
-app = FastAPI(title="Nebulix Search Engine (Powered by Elasticsearch)", lifespan=lifespan)
+app = FastAPI(title="Nebulix v6 Core", lifespan=lifespan)
 app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
 
-@app.get("/translate")
-async def translate_api(q: str = Query(...), target: str = Query("en"), source: str = Query("auto")):
-    result = await perform_translation(q, target, source)
-    return {"status": "success", "data": result} if result else {"status": "error"}
+@app.middleware("http")
+async def analytics_middleware(request: Request, call_next):
+    start_time = time.perf_counter()
+    response = await call_next(request)
+    latency = (time.perf_counter() - start_time) * 1000
+    if "/search" in request.url.path or "/aimode" in request.url.path:
+        q = request.query_params.get("q", "")
+        log_to_db("INSERT INTO logs (query, latency, ts) VALUES (?, ?, ?)", (q, latency, datetime.utcnow()))
+    return response
 
 @app.get("/search")
-async def pure_search(q: str = Query(...)):
-    req_id = str(uuid.uuid4())
-    smart_data = await analyze_query_for_apis(q)
-    results, prediction, _ = await perform_core_search(q)
+async def api_search(q: str = Query(...)):
+    cache_key = q.lower()
+    if cache_key in SEARCH_CACHE: return SEARCH_CACHE[cache_key]
     
-    return {
-        "request_id": req_id,
-        "query": q,
-        "smart_widget": smart_data,
-        "results_count": len(results),
-        "results": results  # Now returns ENTIRE JSON structure including Favicon, Videos, Tag, etc.
-    }
+    results = get_ranked_results(q)
+    for r in results: r.pop("searchable_text", None)
+    
+    resp = {"query": q, "results": results}
+    SEARCH_CACHE[cache_key] = resp
+    return resp
 
 @app.get("/aimode")
-async def ai_mode_short(q: str = Query(...)):
-    req_id = str(uuid.uuid4())
-    smart_data = await analyze_query_for_apis(q)
-    results, _, _ = await perform_core_search(q)
+async def api_aimode(q: str = Query(...), debug: bool = False):
+    q_hash = hashlib.md5(q.encode()).hexdigest()
+    if q_hash in AI_ANSWER_CACHE: return AI_ANSWER_CACHE[q_hash]
     
-    if not results and not smart_data: 
-        return {"request_id": req_id, "ai_response": "I couldn't find any results.", "results": []}
-        
-    context = (f"Live Info: {smart_data['content']}. " if smart_data else "") + " ".join([f"{r.get('title', '')} - {r.get('description', '')}" for r in results[:2]])
-    messages = [
-        {"role": "system", "content": "Provide a brief 2-3 sentence summary based only on the provided context."},
-        {"role": "user", "content": f"Query: {q}\nContext: {context}"}
-    ]
+    results = get_ranked_results(q)
+    final_prompt, sources, debug_log = await execute_agentic_rag(q, results, debug)
+    
+    if not llm: return {"Answer": "Mocked", "Sources": sources}
     
-    ai_reply = await run_ai_task(messages, max_tokens=150)
-    return {"request_id": req_id, "ai_response": ai_reply, "smart_widget": smart_data, "results": results}
+    async with AI_SEMAPHORE:
+        res = await asyncio.to_thread(llm, final_prompt, max_tokens=1024, temperature=0.2)
+        answer = res["choices"][0]["text"].strip()
+        
+    resp = {
+        "Answer": answer[:150] + "...",
+        "Summary": answer[:300],
+        "Detailed Explanation": answer,
+        "Sources": sources
+    }
+    if debug: resp["Debug_Planner"] = debug_log
+    AI_ANSWER_CACHE[q_hash] = resp
+    return resp
 
 @app.get("/aimodechat")
-async def ai_mode_chat(q: str = Query(...)):
-    req_id = str(uuid.uuid4())
-    smart_data = await analyze_query_for_apis(q)
-    results, _, _ = await perform_core_search(q)
+async def api_aimodechat(q: str = Query(...)):
+    results = get_ranked_results(q)
+    final_prompt, sources, _ = await execute_agentic_rag(q, results)
     
-    context_parts = [f"LIVE SYSTEM DATA: {smart_data['content']}"] if smart_data else []
-    for i, r in enumerate(results[:5]): 
-        context_parts.append(f"Source [{i+1}]: {r.get('title')} - {r.get('description')}")
+    async def true_event_stream():
+        if llm:
+            async with AI_SEMAPHORE:
+                for chunk in llm(final_prompt, max_tokens=1024, stream=True, temperature=0.2):
+                    text = chunk["choices"][0]["text"]
+                    if text: yield f"data: {text}\n\n"
+        else:
+            yield "data: [Mocked stream due to missing LLM model]\n\n"
+        yield f"data: [SOURCES_JSON] {json.dumps(sources)}\n\n"
         
-    messages = [
-        {"role": "system", "content": "You are a Smart AI Search Assistant. Use live data and search sources to answer accurately."},
-        {"role": "user", "content": f"User Query: {q}\n\nContext & Sources:\n{chr(10).join(context_parts)}"}
-    ]
-    
-    ai_reply = await run_ai_task(messages, max_tokens=400) 
-    return {"request_id": req_id, "ai_response": ai_reply, "smart_widget": smart_data, "results": results}
+    return StreamingResponse(true_event_stream(), media_type="text/event-stream")
+
+@app.get("/popular")
+async def api_popular():
+    docs = sorted(raw_data, key=lambda x: x.get("popularity", 0), reverse=True)[:10]
+    return {"popular": [{"url": d["url"], "title": d["title"], "popularity": d["popularity"]} for d in docs]}
+
+@app.get("/ai-popular")
+async def api_ai_popular():
+    docs = sorted(raw_data, key=lambda x: x.get("ai_opens", 0), reverse=True)[:10]
+    return {"ai_popular": [{"url": d["url"], "title": d["title"], "ai_opens": d["ai_opens"], "special_tags": d["special_tags"]} for d in docs]}
+
+@app.get("/trending")
+async def api_trending():
+    docs = sorted(raw_data, key=lambda x: x.get("total_opens", 0), reverse=True)[:10]
+    return {"trending": [{"url": d["url"], "title": d["title"]} for d in docs]}
+
+@app.get("/suggestions")
+async def api_suggestions(q: str = Query(...)):
+    base_q = q.lower().split()[-1]
+    words = []
+    def dfs(node, prefix):
+        if len(words) >= 5: return
+        if node.get('_end'): words.append(prefix)
+        for char, child in node.items():
+            if char != '_end': dfs(child, prefix + char)
+            
+    node = trie_root
+    for c in base_q:
+        if c in node: node = node[c]
+        else: break
+    if node and node != trie_root: dfs(node, base_q)
+    return {"suggestions": [q.replace(base_q, w) for w in words]}
+
+@app.get("/stats")
+async def api_stats():
+    mem = psutil.virtual_memory()
+    return {
+        "engine": "Nebulix v6 HF Production (SmolLM2 360M)",
+        "documents": len(raw_data),
+        "cpu_usage_percent": psutil.cpu_percent(),
+        "ram_usage_percent": mem.percent,
+        "search_cache_items": len(SEARCH_CACHE),
+        "ai_cache_items": len(AI_ANSWER_CACHE)
+    }
+
+@app.get("/health")
+async def api_health(): return {"status": "ok", "version": "6.1"}
+
+@app.get("/url-info")
+async def api_url_info(url: str = Query(...)):
+    if url in url_to_data: 
+        safe_doc = url_to_data[url].copy()
+        safe_doc.pop("searchable_text", None)
+        return safe_doc
+    return {"error": "Not Found"}
+
+@app.post("/click")
+async def api_click(req: Request):
+    data = await req.json()
+    url = data.get("url")
+    ip = req.client.host
+    track_id = f"{ip}_{url}"
+    if track_id in CLICK_TRACKER and time.time() - CLICK_TRACKER[track_id] < 1800:
+        return {"status": "ignored_duplicate"}
+        
+    CLICK_TRACKER[track_id] = time.time()
+    if url in url_to_data:
+        d = url_to_data[url]
+        d["human_opens"] += 1
+        d["last_opened"] = datetime.utcnow().isoformat()
+        update_dynamic_fields(d)
+        log_to_db("INSERT INTO clicks (url, type, ts) VALUES (?, ?, ?)", (url, "human", datetime.utcnow()))
+        return {"status": "success", "human_opens": d["human_opens"]}
+    return {"error": "Not found"}
+
+@app.post("/ai-open")
+async def api_ai_open(req: Request):
+    data = await req.json()
+    url = data.get("url")
+    if url in url_to_data:
+        d = url_to_data[url]
+        d["ai_opens"] += 1
+        d["learning_score"] += 1.0
+        d["last_opened"] = datetime.utcnow().isoformat()
+        update_dynamic_fields(d)
+        log_to_db("INSERT INTO clicks (url, type, ts) VALUES (?, ?, ?)", (url, "ai", datetime.utcnow()))
+        return {"status": "success"}
+    return {"error": "Not found"}
+
+@app.post("/feedback")
+async def api_feedback(req: Request):
+    data = await req.json()
+    url = data.get("url")
+    if url in url_to_data:
+        url_to_data[url]["trust_score"] += 2.0 if data.get("helpful") else -1.0
+        update_dynamic_fields(url_to_data[url])
+    return {"status": "feedback_registered"}
+
+@app.post("/search-log")
+async def api_search_log(req: Request):
+    return {"status": "logged"}
 
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=7860)
+    uvicorn.run(app, host="0.0.0.0", port=7860, log_level="warning")