Update app.py

app.py

@@ -1,8 +1,3 @@
-# ==============================================================================
-# NEBULIX SEARCH ENGINE v6 FINAL (PRODUCTION READY)
-# Target: 2 vCPU • 16 GB RAM • True Agentic RAG • AI Streaming • SQLite Analytics
-# Includes Hugging Face Dataset Integration & SmolLM2 (Million Params) integration
-# ==============================================================================
 import json
 import uuid
 import asyncio
@@ -10,542 +5,337 @@ import os
 import uvicorn
 import re
 import time
-import pickle
-import sqlite3
-import unicodedata
 import multiprocessing
-import numpy as np
-import psutil
+import urllib.parse
 from datetime import datetime
 from contextlib import asynccontextmanager
-from fastapi import FastAPI, Query, Request, Form, BackgroundTasks
-from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi import FastAPI, Query, Request, UploadFile, File, Form
+from fastapi.responses import JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
+from huggingface_hub import hf_hub_download
 from rank_bm25 import BM25Okapi
-from cachetools import TTLCache, LRUCache
-import hashlib
-
-# --- ML & Data Imports ---
-try:
-    from datasets import load_dataset
-    from llama_cpp import Llama
-    from sentence_transformers import SentenceTransformer
-    ML_AVAILABLE = True
-except ImportError:
-    ML_AVAILABLE = False
-    print("⚠️ ML packages missing. Run: pip install llama-cpp-python sentence-transformers psutil numpy datasets")
-
-# ==============================================================================
-# 25. CPU ALLOCATION (60% Search, 40% AI on 2 vCPUs)
-# ==============================================================================
-os.environ["OMP_NUM_THREADS"] = "1"
-os.environ["OPENBLAS_NUM_THREADS"] = "1"
-os.environ["MKL_NUM_THREADS"] = "1"
-
-# ==============================================================================
-# 22. SQLITE ANALYTICS SETUP
-# ==============================================================================
-DB_PATH = "analytics.db"
-def init_db():
-    conn = sqlite3.connect(DB_PATH)
-    conn.execute('CREATE TABLE IF NOT EXISTS clicks (url TEXT, type TEXT, ts DATETIME)')
-    conn.execute('CREATE TABLE IF NOT EXISTS logs (query TEXT, latency REAL, ts DATETIME)')
-    conn.execute('CREATE TABLE IF NOT EXISTS planner_logs (query TEXT, hops INTEGER, decision_log TEXT, ts DATETIME)')
-    conn.commit()
-    conn.close()
-
-def log_to_db(query, args=()):
-    try:
-        conn = sqlite3.connect(DB_PATH)
-        conn.execute(query, args)
-        conn.commit()
-        conn.close()
-    except Exception as e: pass
-
-# ==============================================================================
-# GLOBAL DATA & CACHES
-# ==============================================================================
+import torch
+from PIL import Image
+import io
+DIRECT IMPORTS TO FIX THE PIPELINE BUG
+from transformers import pipeline, BlipProcessor, BlipForConditionalGeneration
+--- 1. SETUP & CPU ALLOCATION (40% for AI) ---
+TOTAL_CORES = multiprocessing.cpu_count()
+AI_CORES = max(1, int(TOTAL_CORES * 0.4))
+torch.set_num_threads(AI_CORES)
+print(f"🖥️ Total Cores: {TOTAL_CORES} | Allocating {AI_CORES} cores strictly to AI.")
 raw_data = []
-url_to_data = {}
 bm25_index = None
-trie_root = {}
-
-SEARCH_CACHE = TTLCache(maxsize=10000, ttl=1800) # 30 Min TTL
-AI_ANSWER_CACHE = LRUCache(maxsize=2000)
-
-doc_embeddings = None
-text_embedder = None
-llm = None
-AI_SEMAPHORE = asyncio.Semaphore(1)
-
-CLICK_TRACKER = {}
-STOP_WORDS = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "is"}
-
-# ==============================================================================
-# MATH & SPECIAL TAGS
-# ==============================================================================
-def update_dynamic_fields(doc):
-    human = doc.get("human_opens", 0)
-    ai = doc.get("ai_opens", 0)
-    crawl = doc.get("crawler_opens", 0)
-    bot = doc.get("bot_opens", 0)
-    
-    doc["total_opens"] = human + ai + crawl + bot
-    doc["popularity"] = human + (ai * 2) + (crawl * 0.1) + (bot * 0)
-    
-    total_interactive = human + ai
-    ai_percent = (ai / total_interactive * 100) if total_interactive > 0 else 0
-    doc["ai_open_percent"] = ai_percent
-    
-    tags = set()
-    if doc.get("trust_score", 50) >= 80: tags.add("Trusted")
-    if doc.get("quality_score", 50) >= 80: tags.add("High Quality")
-    if doc.get("domain_quality", 50) >= 90: tags.add("Verified")
-    if doc.get("total_opens", 0) >= 500: tags.add("Popular")
+MAX_CONCURRENT_AI = max(1, AI_CORES)
+ai_semaphore = asyncio.Semaphore(MAX_CONCURRENT_AI)
+ai_generator = None
+vision_processor = None
+vision_model = None
+--- AI CACHING SYSTEM ---
+AI_RESPONSE_CACHE = {}
+MAX_CACHE_SIZE = 5000
+ADULT_KEYWORDS = {"porn", "sex", "nude", "nsfw", "xxx", "adult", "erotic", "onlyfans"}
+STOP_WORDS = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "is", "are"}
+HF_TOKEN = os.getenv("HF_TOKEN")
+BANNED_IPS = set()
+IP_TRACKER = {}
+--- 2. ADVANCED NLP & IN-MEMORY INDEXING ---
+def tokenize(text):
+if not text: return []
+tokens = re.findall(r'[a-z0-9]+', str(text).lower())
+return [t for t in tokens if t not in STOP_WORDS]
+def extract_nested_data(obj):
+if isinstance(obj, dict):
+return " ".join(extract_nested_data(v) for v in obj.values())
+elif isinstance(obj, list):
+return " ".join(extract_nested_data(v) for v in obj)
+return str(obj)
+def load_and_index_dataset():
+global bm25_index, raw_data
+print("📥 Loading Dataset & Building Deep Index...")
+code
+Code
+dataset_path = "dataset.jsonl"
+if HF_TOKEN:
+    try:
+        dataset_path = hf_hub_download(
+            repo_id="Nebulixlabs/Ultimate-Search-Engine-Dataset",
+            filename="dataset.jsonl",
+            repo_type="dataset",
+            token=HF_TOKEN 
+        )
+    except Exception as e:
+        print(f"⚠️ Hugging Face Download failed: {e}")
+
+tokenized_corpus = []
+
+if os.path.exists(dataset_path):
+    with open(dataset_path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.strip():
+                try:
+                    doc = json.loads(line)
+                    raw_data.append(doc)
+                    
+                    title_tokens = tokenize(doc.get("title", ""))
+                    desc_tokens = tokenize(doc.get("description", ""))
+                    meta_tokens = tokenize(doc.get("meta_tags", {}).get("description", ""))
+                    url_tokens = tokenize(doc.get("url", ""))
+                    specific_tokens = tokenize(extract_nested_data(doc.get("specific_data", {})))
+                    
+                    boosted_tokens = (title_tokens * 5) + (desc_tokens * 3) + meta_tokens + url_tokens + (specific_tokens * 2)
+                    tokenized_corpus.append(boosted_tokens)
+                except: pass
+                
+    if tokenized_corpus:
+        print("⚡ Building High-Speed BM25 Okapi Index...")
+        bm25_index = BM25Okapi(tokenized_corpus)
+        print(f"🚀 Indexing Complete! {len(raw_data)} records ready.")
+else:
+    print("❌ Dataset file not found!")
+def load_ai_models():
+global ai_generator, vision_processor, vision_model
+print("🤖 Loading Lightweight AI Models (Text & Vision)...")
+try:
+# 1. Text AI
+ai_generator = pipeline("text-generation", model="Qwen/Qwen2.5-0.5B-Instruct", device=-1, dtype=torch.float32)
+code
+Code
+# 2. Vision AI
+    print("👁️ Loading Vision AI...")
+    vision_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
+    vision_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
     
-    if ai >= 25: tags.add("Opened by AI")
-    if total_interactive > 10 and ai_percent >= 80: tags.add("AI Favorite")
+    print("✅ All AI Models Loaded! System ready.")
+except Exception as e:
+    print(f"❌ Model Loading Failed: {e}")
+--- 3. CORE SEARCH ALGORITHM ---
+async def perform_core_search(raw_query: str):
+if not raw_data or not bm25_index:
+return [], "Search index not ready.", 0
+code
+Code
+site_filter = None
+tag_filter = None
+
+site_match = re.search(r'site:([^\s]+)', raw_query, re.IGNORECASE)
+if site_match:
+    site_filter = site_match.group(1).lower()
+    raw_query = raw_query.replace(site_match.group(0), "")
     
-    doc["special_tags"] = list(tags)
-
-# ==============================================================================
-# TOKENIZER & UTILS
-# ==============================================================================
-def normalize_text(text: str) -> str:
-    if not text: return ""
-    text = unicodedata.normalize('NFKD', str(text)).encode('ASCII', 'ignore').decode('utf-8')
-    return re.sub(r'<[^>]+>', '', text.lower())
+tag_match = re.search(r'tag:([^\s]+)', raw_query, re.IGNORECASE)
+if tag_match:
+    tag_filter = tag_match.group(1).lower()
+    raw_query = raw_query.replace(tag_match.group(0), "")
 
-def tokenize(text: str) -> list:
-    words = re.findall(r'\b[a-z0-9]+\b', normalize_text(text))
-    return [w for w in words if w not in STOP_WORDS]
+query = raw_query.strip()
+query_tokens = tokenize(query)
 
-def add_to_trie(word):
-    node = trie_root
-    for char in word:
-        if char not in node: node[char] = {}
-        node = node[char]
-    node['_end'] = True
+if not query_tokens and not site_filter and not tag_filter: 
+    return [], "Invalid Query", 0
 
-def cosine_similarity(vec_a, mat_b):
-    if len(mat_b) == 0: return np.array([])
-    return np.dot(mat_b, vec_a) / (np.linalg.norm(mat_b, axis=1) * np.linalg.norm(vec_a) + 1e-10)
+user_wants_adult = any(word in ADULT_KEYWORDS for word in query_tokens)
+scores = bm25_index.get_scores(query_tokens) if query_tokens else [1.0] * len(raw_data)
 
-def detect_language(text):
-    return "English"
+scored_results = []
+seen_urls = set() 
 
-# ==============================================================================
-# DATASET LOADING & NEW SCHEMA MAPPING
-# ==============================================================================
-def build_searchable_text(doc):
-    """Combines title, description, tag, and string values from meta/og tags into one text."""
-    parts = [doc.get("title", ""), doc.get("description", ""), doc.get("tag", "")]
-    
-    for k, v in doc.get("meta_tags", {}).items():
-        if isinstance(v, str): parts.append(v)
-            
-    for k, v in doc.get("og_tags", {}).items():
-        if isinstance(v, str): parts.append(v)
+for index, score in enumerate(scores):
+    if score > 0.5 or not query_tokens:
+        doc = raw_data[index]
+        doc_url = doc.get("url", "")
         
-    return " ".join(p for p in parts if p).strip()
-
-def build_runtime_record(doc):
-    r = doc.copy()
-    
-    r["meta_tags"] = doc.get("meta_tags", {})
-    r["og_tags"] = doc.get("og_tags", {})
-    r["images"] = doc.get("images", [])
-    r["videos"] = doc.get("videos", [])
-    r["documents"] = doc.get("documents", [])
-    r["tag"] = doc.get("tag", "general")
-    r["specific_data"] = doc.get("specific_data", {})
-    r["searchable_text"] = build_searchable_text(r)
-    
-    r["human_opens"] = r.get("human_opens", 0)
-    r["ai_opens"] = r.get("ai_opens", 0)
-    r["crawler_opens"] = r.get("crawler_opens", 0)
-    r["bot_opens"] = r.get("bot_opens", 0)
-    r["trust_score"] = r.get("trust_score", 50.0)
-    r["quality_score"] = r.get("quality_score", 50.0)
-    r["learning_score"] = r.get("learning_score", 50.0)
-    r["search_boost"] = r.get("search_boost", 0.0)
-    r["spam_score"] = r.get("spam_score", 0.0)
-    r["domain_quality"] = r.get("domain_quality", 50.0)
-    r["last_opened"] = r.get("last_opened", datetime.utcnow().isoformat())
-    r["language"] = r.get("language", detect_language(r.get("title", "")))
-    r["country"] = r.get("country", "Global")
-    
-    update_dynamic_fields(r)
-    return r
-
-def init_system():
-    global bm25_index, doc_embeddings, text_embedder, llm, raw_data, url_to_data
-    init_db()
-    print("🚀 [1/4] Fetching Dataset & Building Indexes...")
-    
-    CACHE_FILE = "cached_dataset.jsonl"
-    
-    if not os.path.exists(CACHE_FILE):
-        hf_token = os.getenv("HF_TOKEN")
-        if not hf_token:
-            raise ValueError("❌ HF_TOKEN environment variable is missing. Cannot fetch dataset.")
-        print("☁️ Downloading dataset from Hugging Face...")
-        dataset = load_dataset("Nebulixlabs/Ultimate-Search-Engine-Dataset", split="train", token=hf_token)
-        dataset.to_json(CACHE_FILE)
-        print("✅ Dataset successfully cached locally.")
-
-    if os.path.exists(CACHE_FILE):
-        corpus_tokens = []
-        texts_to_embed = []
-        with open(CACHE_FILE, "r", encoding="utf-8") as f:
-            for line in f:
-                if line.strip():
-                    doc = build_runtime_record(json.loads(line))
-                    raw_data.append(doc)
-                    url_to_data[doc["url"]] = doc
-                    
-                    tokens = tokenize(doc["searchable_text"])
-                    corpus_tokens.append(tokens)
-                    texts_to_embed.append(doc["searchable_text"])
-                    for t in tokens: add_to_trie(t)
-
-        bm25_index = BM25Okapi(corpus_tokens)
+        if doc_url in seen_urls: continue
+        parsed_url = urllib.parse.urlparse(doc_url)
         
-        if ML_AVAILABLE:
-            print("🧠 [2/4] Loading BAAI/bge-small-en-v1.5...")
-            text_embedder = SentenceTransformer('BAAI/bge-small-en-v1.5', device='cpu')
-
-            if os.path.exists("embeddings.bin"):
-                print("💾 [3/4] Loading Precomputed Embeddings...")
-                with open("embeddings.bin", "rb") as fb: doc_embeddings = pickle.load(fb)
-            else:
-                print("⚙️ [3/4] Generating Embeddings (One-Time)...")
-                doc_embeddings = text_embedder.encode(texts_to_embed, convert_to_numpy=True)
-                with open("embeddings.bin", "wb") as fb: pickle.dump(doc_embeddings, fb)
-                with open("metadata.json", "w") as fm: json.dump({"count": len(doc_embeddings)}, fm)
-            
-            # =============== CHANGED TO SMOLLM2 (360 Million Params) ===============
-            print("🔥 [4/4] Loading Llama.cpp SmolLM2-360M GGUF (Lightweight CPU Model)...")
-            try:
-                llm = Llama(
-                    model_path="./models/smollm2-360m-instruct-q8_0.gguf", # Update filename if downloaded differently
-                    n_threads=1, 
-                    n_ctx=3000, 
-                    verbose=False
-                )
-            except:
-                print("⚠️ LLM not found locally in ./models/, mocked mode active.")
-    print("✅ Nebulix v6 Ready.")
-
-# ==============================================================================
-# RANKING FORMULA
-# ==============================================================================
-def get_ranked_results(query: str):
-    q_tokens = tokenize(query)
-    if not q_tokens: return []
-    
-    bm25_scores = bm25_index.get_scores(q_tokens) if bm25_index else np.zeros(len(raw_data))
-    semantic_scores = np.zeros(len(raw_data))
-    
-    if text_embedder and doc_embeddings is not None:
-        q_vec = text_embedder.encode([query])[0]
-        semantic_scores = cosine_similarity(q_vec, doc_embeddings)
-
-    scored_results = []
-    
-    for i, doc in enumerate(raw_data):
-        score = (bm25_scores[i] * 0.35) + \
-                (semantic_scores[i] * 0.45) + \
-                (doc["trust_score"] * 0.05) + \
-                (doc["quality_score"] * 0.05) + \
-                (doc["learning_score"] * 0.03) + \
-                (doc["popularity"] * 0.05) + \
-                doc["search_boost"] - \
-                doc["spam_score"]
-                
-        scored_results.append((score, doc))
-
-    scored_results.sort(key=lambda x: x[0], reverse=True)
-    
-    formatted = []
-    for s, d in scored_results[:100]:
-        formatted.append({
-            "url": d["url"], "title": d["title"], "description": d["description"],
-            "favicon": d.get("favicon", ""), "language": d["language"], "country": d["country"],
-            "images": d.get("images", []), "videos": d.get("videos", []), 
-            "tag": d.get("tag", "general"),
-            "last_opened": d["last_opened"], "trust_score": round(d["trust_score"], 2),
-            "quality_score": round(d["quality_score"], 2), "learning_score": round(d["learning_score"], 2),
-            "human_opens": d["human_opens"], "ai_opens": d["ai_opens"],
-            "total_opens": d["total_opens"], "special_tags": d["special_tags"],
-            "match_score": round(s, 2), 
-            "searchable_text": d.get("searchable_text", "")
-        })
-    return formatted
-
-# ==============================================================================
-# CHUNKING
-# ==============================================================================
-def get_best_chunks(query, full_text):
-    if not full_text: return []
-    words = full_text.split()
-    chunks = []
-    i = 0
-    while i < len(words):
-        chunks.append(" ".join(words[i:i + 512]))
-        i += (512 - 64)
+        if site_filter and site_filter not in parsed_url.netloc.lower(): continue
+        doc_tag = str(doc.get("tag", "general")).lower()
+        if tag_filter and tag_filter != doc_tag: continue
         
-    if not chunks: return []
-    if text_embedder:
-        q_emb = text_embedder.encode([query], convert_to_numpy=True)[0]
-        c_embs = text_embedder.encode(chunks, convert_to_numpy=True)
-        scores = cosine_similarity(q_emb, c_embs)
-        best_idx = np.argsort(scores)[::-1][:5]
-        return [chunks[idx] for idx in best_idx]
-    return chunks[:5]
-
-# ==============================================================================
-# TRUE AGENTIC RAG PLANNER (Max 10 Hops)
-# ==============================================================================
-async def agent_planner_call(prompt):
-    if not llm: return "ACTION: ANSWER"
-    res = await asyncio.to_thread(llm, prompt, max_tokens=20, temperature=0.1, stop=["\n"])
-    return res["choices"][0]["text"].strip()
-
-async def execute_agentic_rag(query, candidates, debug=False):
-    context_memory = ""
-    sources_used = []
-    visited_idx = set()
-    debug_log = []
-    
-    metadata_list = "\n".join([f"ID {i} | URL: {d['url']} | Title: {d['title']}" for i, d in enumerate(candidates[:20])])
-    
-    async with AI_SEMAPHORE:
-        for hop in range(10):
-            # Prompt simplified slightly for million-parameter models to ensure strict output
-            sys_prompt = f"""System: You are a Planner. Output ONLY the action command.
-Query: {query}
-URLs:
-{metadata_list}
-Memory:
-{context_memory if context_memory else 'Empty'}
-To read a URL, type: OPEN(id)
-To answer, type: ANSWER
-Action: """
-            
-            decision = await agent_planner_call(sys_prompt)
-            debug_log.append(f"Hop {hop+1}: {decision}")
-            
-            open_match = re.search(r'OPEN\((\d+)\)', decision, re.IGNORECASE)
-            if open_match:
-                idx = int(open_match.group(1))
-                if idx < len(candidates) and idx not in visited_idx:
-                    visited_idx.add(idx)
-                    doc = candidates[idx]
-                    
-                    target_text = doc.get("searchable_text", "") or doc.get("description", "")
-                    best_chunks = get_best_chunks(query, target_text)
-                    context_memory += f"\n[From {doc['url']}]:\n" + "\n".join(best_chunks) + "\n"
-                    
-                    if doc["url"] in url_to_data:
-                        url_to_data[doc["url"]]["ai_opens"] += 1
-                        url_to_data[doc["url"]]["learning_score"] += 1.0
-                        update_dynamic_fields(url_to_data[doc["url"]])
-                    
-                    sources_used.append({
-                        "url": doc["url"], "title": doc["title"], "language": doc["language"],
-                        "country": doc["country"], "last_opened": doc["last_opened"],
-                        "human_opens": doc["human_opens"], "ai_opens": doc["ai_opens"],
-                        "trust_score": doc["trust_score"], "quality_score": doc["quality_score"],
-                        "learning_score": doc["learning_score"], "special_tags": doc["special_tags"],
-                        "favicon": doc["favicon"]
-                    })
-                else:
-                    context_memory += "\n(System: ID already read. Choose another or ANSWER.)"
-            else:
-                break
+        is_adult = doc_tag in ["adult", "nsfw"] or any(w in str(doc).lower() for w in ADULT_KEYWORDS)
+        if not user_wants_adult and is_adult: continue 
         
-        log_to_db("INSERT INTO planner_logs (query, hops, decision_log, ts) VALUES (?, ?, ?, ?)", 
-                 (query, len(visited_idx), " | ".join(debug_log), datetime.utcnow()))
+        final_score = score
+        exact_phrase = query.lower()
+        doc_title = str(doc.get("title", "")).lower()
         
-        final_prompt = f"System: Use ONLY the provided Memory to answer the query.\nQuery: {query}\nMemory:\n{context_memory}\n\nProvide Answer:\n"
+        if exact_phrase == doc_title: final_score *= 5.0
+        elif exact_phrase in doc_title: final_score *= 3.0
+        if exact_phrase in str(doc.get("description", "")).lower(): final_score *= 1.8
+            
+        path = parsed_url.path.strip("/")
+        if not path: final_score *= 1.5 
+        else: final_score *= (0.90 ** path.count("/"))
         
-    return final_prompt, sources_used, debug_log
-
-# ==============================================================================
-# FASTAPI APP & ENDPOINTS
-# ==============================================================================
+        seen_urls.add(doc_url)
+        scored_results.append((final_score, doc))
+
+scored_results.sort(key=lambda x: x[0], reverse=True)
+best_score = scored_results[0][0] if scored_results else 0
+prediction = "High confidence" if best_score > 15 else "Medium confidence" if best_score > 5 else "Low confidence"
+
+# Yaha par hum wahi structure bana rahe hain jo frontend ko chahiye
+final_results = [{"match_score": round(s, 2), **doc} for s, doc in scored_results[:100]]
+return final_results, prediction, best_score
+--- AI GENERATION & CACHE HELPER ---
+def _generate_text(messages, max_tokens: int):
+result = ai_generator(messages, max_new_tokens=max_tokens, do_sample=True, temperature=0.3, top_p=0.85)
+return result[0]['generated_text'][-1]['content']
+def _generate_image_review(image: Image):
+inputs = vision_processor(image, return_tensors="pt")
+with torch.no_grad():
+out = vision_model.generate(**inputs, max_new_tokens=50)
+return vision_processor.decode(out[0], skip_special_tokens=True).capitalize()
+async def run_ai_task(messages, max_tokens: int, query_key: str):
+if not ai_generator: return "⚠️ AI Model is starting up."
+if query_key in AI_RESPONSE_CACHE: return AI_RESPONSE_CACHE[query_key]
+code
+Code
+async with ai_semaphore:
+    response = await asyncio.to_thread(_generate_text, messages, max_tokens)
+    if len(AI_RESPONSE_CACHE) > MAX_CACHE_SIZE: AI_RESPONSE_CACHE.clear()
+    AI_RESPONSE_CACHE[query_key] = response
+    return response
+--- 4. FASTAPI APP & ENDPOINTS ---
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    asyncio.create_task(asyncio.to_thread(init_system))
-    yield
-
-app = FastAPI(title="Nebulix v6 Core", lifespan=lifespan)
-app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
-
+asyncio.create_task(asyncio.to_thread(load_and_index_dataset))
+asyncio.create_task(asyncio.to_thread(load_ai_models))
+yield
+app = FastAPI(title="Nebulix Advanced SEO & Visual API Engine", lifespan=lifespan)
+app.add_middleware(CORSMiddleware, allow_origins=[""], allow_credentials=True, allow_methods=[""], allow_headers=["*"])
 @app.middleware("http")
-async def analytics_middleware(request: Request, call_next):
-    start_time = time.perf_counter()
-    response = await call_next(request)
-    latency = (time.perf_counter() - start_time) * 1000
-    if "/search" in request.url.path or "/aimode" in request.url.path:
-        q = request.query_params.get("q", "")
-        log_to_db("INSERT INTO logs (query, latency, ts) VALUES (?, ?, ?)", (q, latency, datetime.utcnow()))
-    return response
-
+async def rate_limit_middleware(request: Request, call_next):
+client_ip = request.client.host
+if client_ip in BANNED_IPS:
+return JSONResponse(status_code=403, content={"error": "BANNED: Your IP is permanently banned."})
+code
+Code
+current_time = int(time.time())
+if client_ip not in IP_TRACKER:
+    IP_TRACKER[client_ip] = {"time": current_time, "count": 1}
+else:
+    if IP_TRACKER[client_ip]["time"] == current_time: IP_TRACKER[client_ip]["count"] += 1
+    else: IP_TRACKER[client_ip] = {"time": current_time, "count": 1}
+        
+if IP_TRACKER[client_ip]["count"] >= 6:
+    BANNED_IPS.add(client_ip)
+    return JSONResponse(status_code=403, content={"error": "IP BANNED: Exceeded limit."})
+
+return await call_next(request)
+@app.get("/")
+async def health_check():
+return {"status": "ok", "message": "Nebulix API Engine is running smoothly."}
 @app.get("/search")
-async def api_search(q: str = Query(...)):
-    cache_key = q.lower()
-    if cache_key in SEARCH_CACHE: return SEARCH_CACHE[cache_key]
-    
-    results = get_ranked_results(q)
-    for r in results: r.pop("searchable_text", None)
-    
-    resp = {"query": q, "results": results}
-    SEARCH_CACHE[cache_key] = resp
-    return resp
-
+async def pure_search(q: str = Query(...)):
+req_id = str(uuid.uuid4())
+results, prediction, _ = await perform_core_search(q)
+return {"request_id": req_id, "query": q, "results_count": len(results), "results": results}
 @app.get("/aimode")
-async def api_aimode(q: str = Query(...), debug: bool = False):
-    q_hash = hashlib.md5(q.encode()).hexdigest()
-    if q_hash in AI_ANSWER_CACHE: return AI_ANSWER_CACHE[q_hash]
-    
-    results = get_ranked_results(q)
-    final_prompt, sources, debug_log = await execute_agentic_rag(q, results, debug)
-    
-    if not llm: return {"Answer": "Mocked", "Sources": sources}
+async def ai_mode_short(q: str = Query(...)):
+req_id = str(uuid.uuid4())
+results, _, _ = await perform_core_search(q)
+if not results: return {"request_id": req_id, "ai_response": "I couldn't find any results.", "results": []}
+code
+Code
+context = " ".join([f"{r.get('title', '')} - {r.get('description', '')}" for r in results[:3]])
+messages = [
+    {"role": "system", "content": "Provide a brief 2-3 sentence summary based only on the provided context."},
+    {"role": "user", "content": f"Query: {q}\nContext: {context}"}
+]
+
+ai_reply = await run_ai_task(messages, 150, query_key=f"short_{q}")
+return {"request_id": req_id, "ai_response": ai_reply, "results": results}
+1. PURANA CHAT API (Current UI ke liye)
+@app.get("/aimodechat")
+async def ai_mode_chat(q: str = Query(...)):
+req_id = str(uuid.uuid4())
+results, _, _ = await perform_core_search(q)
+code
+Code
+context_parts = [f"Source [{i+1}]: {r.get('title')} - {r.get('description')}" for i, r in enumerate(results[:5])]
+messages = [
+    {"role": "system", "content": "You are a Smart AI Search Assistant. Use search sources to answer accurately."},
+    {"role": "user", "content": f"User Query: {q}\n\nContext:\n{chr(10).join(context_parts)}"}
+]
+
+ai_reply = await run_ai_task(messages, 400, query_key=f"chat_{q}") 
+return {"request_id": req_id, "ai_response": ai_reply, "results": results}
+2. NAYA CHAT API (Future UI - Image + Text Chat ke liye)
+@app.post("/aimodechat")
+async def ai_mode_chat_vision(q: str = Form(...), file: UploadFile = File(None)):
+req_id = str(uuid.uuid4())
+code
+Code
+vision_context = ""
+# Agar user ne chat me image upload ki hai, toh AI image ko decode karega
+if file and vision_model:
+    try:
+        image_bytes = await file.read()
+        img = Image.open(io.BytesIO(image_bytes)).convert("RGB")
+        async with ai_semaphore:
+            ai_review = await asyncio.to_thread(_generate_image_review, img)
+        vision_context = f"\n[User uploaded an image showing: {ai_review}]\n"
+    except Exception:
+        vision_context = "\n[User uploaded an image but it could not be analyzed.]\n"
+
+# Search query combining text and vision
+search_query = f"{q} {vision_context}".strip()
+results, _, _ = await perform_core_search(search_query)
+
+context_parts = [f"Source [{i+1}]: {r.get('title')} - {r.get('description')}" for i, r in enumerate(results[:5])]
+
+messages = [
+    {"role": "system", "content": "You are a Smart AI Search Assistant capable of understanding text and visual context. Use the provided search sources and image details to answer accurately."},
+    {"role": "user", "content": f"User Query: {q}{vision_context}\n\nContext:\n{chr(10).join(context_parts)}"}
+]
+
+# Generate Chat response
+cache_key = f"chat_{q}_{len(vision_context)}"
+ai_reply = await run_ai_task(messages, 400, query_key=cache_key) 
+
+return {"request_id": req_id, "ai_response": ai_reply, "results": results}
+IMAGE SEARCH (Fixed: Same JSON structure as normal text search)
+@app.post("/image-search")
+async def reverse_image_search(file: UploadFile = File(...)):
+if not vision_model:
+return {"error": "Vision AI Model is still loading."}
+code
+Code
+req_id = str(uuid.uuid4())
+
+try:
+    image_bytes = await file.read()
+    img = Image.open(io.BytesIO(image_bytes)).convert("RGB")
     
-    async with AI_SEMAPHORE:
-        res = await asyncio.to_thread(llm, final_prompt, max_tokens=1024, temperature=0.2)
-        answer = res["choices"][0]["text"].strip()
+    # 1. Vision AI process karega
+    async with ai_semaphore:
+        ai_review = await asyncio.to_thread(_generate_image_review, img)
         
-    resp = {
-        "Answer": answer[:150] + "...",
-        "Summary": answer[:300],
-        "Detailed Explanation": answer,
-        "Sources": sources
-    }
-    if debug: resp["Debug_Planner"] = debug_log
-    AI_ANSWER_CACHE[q_hash] = resp
-    return resp
-
-@app.get("/aimodechat")
-async def api_aimodechat(q: str = Query(...)):
-    results = get_ranked_results(q)
-    final_prompt, sources, _ = await execute_agentic_rag(q, results)
+    # 2. Text Search hoga vision ke basis pe
+    raw_results, _, _ = await perform_core_search(ai_review)
+    visual_results = []
     
-    async def true_event_stream():
-        if llm:
-            async with AI_SEMAPHORE:
-                for chunk in llm(final_prompt, max_tokens=1024, stream=True, temperature=0.2):
-                    text = chunk["choices"][0]["text"]
-                    if text: yield f"data: {text}\n\n"
-        else:
-            yield "data: [Mocked stream due to missing LLM model]\n\n"
-        yield f"data: [SOURCES_JSON] {json.dumps(sources)}\n\n"
+    # 3. DIRECTLY append the original FULL document
+    for doc in raw_results:
+        is_adult = str(doc.get("tag", "general")).lower() in ["adult", "nsfw"] or any(w in str(doc).lower() for w in ADULT_KEYWORDS)
+        if is_adult: continue 
         
-    return StreamingResponse(true_event_stream(), media_type="text/event-stream")
-
-@app.get("/popular")
-async def api_popular():
-    docs = sorted(raw_data, key=lambda x: x.get("popularity", 0), reverse=True)[:10]
-    return {"popular": [{"url": d["url"], "title": d["title"], "popularity": d["popularity"]} for d in docs]}
-
-@app.get("/ai-popular")
-async def api_ai_popular():
-    docs = sorted(raw_data, key=lambda x: x.get("ai_opens", 0), reverse=True)[:10]
-    return {"ai_popular": [{"url": d["url"], "title": d["title"], "ai_opens": d["ai_opens"], "special_tags": d["special_tags"]} for d in docs]}
-
-@app.get("/trending")
-async def api_trending():
-    docs = sorted(raw_data, key=lambda x: x.get("total_opens", 0), reverse=True)[:10]
-    return {"trending": [{"url": d["url"], "title": d["title"]} for d in docs]}
-
-@app.get("/suggestions")
-async def api_suggestions(q: str = Query(...)):
-    base_q = q.lower().split()[-1]
-    words = []
-    def dfs(node, prefix):
-        if len(words) >= 5: return
-        if node.get('_end'): words.append(prefix)
-        for char, child in node.items():
-            if char != '_end': dfs(child, prefix + char)
+        # Pura document direct bhej rahe hain bina modify kiye 
+        # jisse exact text search wala JSON format barkarar rahe
+        visual_results.append(doc)
+        
+        if len(visual_results) >= 40: break
             
-    node = trie_root
-    for c in base_q:
-        if c in node: node = node[c]
-        else: break
-    if node and node != trie_root: dfs(node, base_q)
-    return {"suggestions": [q.replace(base_q, w) for w in words]}
-
-@app.get("/stats")
-async def api_stats():
-    mem = psutil.virtual_memory()
+    # 4. Same structure return karenge
     return {
-        "engine": "Nebulix v6 HF Production (SmolLM2 360M)",
-        "documents": len(raw_data),
-        "cpu_usage_percent": psutil.cpu_percent(),
-        "ram_usage_percent": mem.percent,
-        "search_cache_items": len(SEARCH_CACHE),
-        "ai_cache_items": len(AI_ANSWER_CACHE)
+        "request_id": req_id,
+        "query": f"Image Search Analysis: {ai_review}", # Taki UI me ye "query" ki tarah display ho
+        "results_count": len(visual_results),
+        "results": visual_results
     }
-
-@app.get("/health")
-async def api_health(): return {"status": "ok", "version": "6.1"}
-
-@app.get("/url-info")
-async def api_url_info(url: str = Query(...)):
-    if url in url_to_data: 
-        safe_doc = url_to_data[url].copy()
-        safe_doc.pop("searchable_text", None)
-        return safe_doc
-    return {"error": "Not Found"}
-
-@app.post("/click")
-async def api_click(req: Request):
-    data = await req.json()
-    url = data.get("url")
-    ip = req.client.host
-    track_id = f"{ip}_{url}"
-    if track_id in CLICK_TRACKER and time.time() - CLICK_TRACKER[track_id] < 1800:
-        return {"status": "ignored_duplicate"}
-        
-    CLICK_TRACKER[track_id] = time.time()
-    if url in url_to_data:
-        d = url_to_data[url]
-        d["human_opens"] += 1
-        d["last_opened"] = datetime.utcnow().isoformat()
-        update_dynamic_fields(d)
-        log_to_db("INSERT INTO clicks (url, type, ts) VALUES (?, ?, ?)", (url, "human", datetime.utcnow()))
-        return {"status": "success", "human_opens": d["human_opens"]}
-    return {"error": "Not found"}
-
-@app.post("/ai-open")
-async def api_ai_open(req: Request):
-    data = await req.json()
-    url = data.get("url")
-    if url in url_to_data:
-        d = url_to_data[url]
-        d["ai_opens"] += 1
-        d["learning_score"] += 1.0
-        d["last_opened"] = datetime.utcnow().isoformat()
-        update_dynamic_fields(d)
-        log_to_db("INSERT INTO clicks (url, type, ts) VALUES (?, ?, ?)", (url, "ai", datetime.utcnow()))
-        return {"status": "success"}
-    return {"error": "Not found"}
-
-@app.post("/feedback")
-async def api_feedback(req: Request):
-    data = await req.json()
-    url = data.get("url")
-    if url in url_to_data:
-        url_to_data[url]["trust_score"] += 2.0 if data.get("helpful") else -1.0
-        update_dynamic_fields(url_to_data[url])
-    return {"status": "feedback_registered"}
-
-@app.post("/search-log")
-async def api_search_log(req: Request):
-    return {"status": "logged"}
-
-if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=7860, log_level="warning")
+    
+except Exception as e:
+    return {"error": f"Image processing failed: {str(e)}"}
+if name == "main":
+uvicorn.run(app, host="0.0.0.0", port=7860)