Update app.py

app.py

@@ -3,6 +3,7 @@
 # Target: Hugging Face Spaces | 2 vCPU • 16 GB RAM • True Agentic RAG
 # Model: Qwen2.5-0.5B-Instruct (Inference Only)
 # Ranking: Advanced Algorithmic (Domain Trust, Proxy Penalty) + BM25 + Semantic
+# Includes ALL API Endpoints (Popular, Trending, Suggestions, Clicks, etc.)
 # ==============================================================================
 import json
 import uuid
@@ -11,6 +12,7 @@ import os
 import uvicorn
 import re
 import time
+import pickle
 import sqlite3
 import unicodedata
 import threading
@@ -39,6 +41,7 @@ try:
     from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, TextIteratorStreamer
     from sentence_transformers import SentenceTransformer
     
+    # 🔥 STRICT NO-TRAINING LOCK 🔥
     torch.set_num_threads(1)
     torch.set_grad_enabled(False) # Strict Inference Mode
     ML_AVAILABLE = True
@@ -47,13 +50,14 @@ except ImportError:
     print("⚠️ ML packages missing. Check requirements.txt")
 
 # ==============================================================================
-# GLOBAL SETTINGS & DATABASES
+# SQLITE ANALYTICS SETUP
 # ==============================================================================
 DB_PATH = "analytics.db"
 def init_db():
     conn = sqlite3.connect(DB_PATH)
     conn.execute('CREATE TABLE IF NOT EXISTS clicks (url TEXT, type TEXT, ts DATETIME)')
     conn.execute('CREATE TABLE IF NOT EXISTS logs (query TEXT, latency REAL, ts DATETIME)')
+    conn.execute('CREATE TABLE IF NOT EXISTS planner_logs (query TEXT, hops INTEGER, decision_log TEXT, ts DATETIME)')
     conn.commit()
     conn.close()
 
@@ -65,6 +69,9 @@ def log_to_db(query, args=()):
         conn.close()
     except: pass
 
+# ==============================================================================
+# GLOBAL DATA & CACHES
+# ==============================================================================
 raw_data = []
 url_to_data = {}
 bm25_index = None
@@ -72,7 +79,7 @@ trie_root = {}
 doc_embeddings = None
 text_embedder = None
 
-SEARCH_CACHE = TTLCache(maxsize=10000, ttl=1800)
+SEARCH_CACHE = TTLCache(maxsize=10000, ttl=1800) # 30 Min TTL
 AI_ANSWER_CACHE = LRUCache(maxsize=2000)
 
 tokenizer = None
@@ -91,6 +98,30 @@ def is_adult_content(text: str) -> bool:
     text_lower = text.lower()
     return any(word in text_lower for word in ADULT_KEYWORDS)
 
+def update_dynamic_fields(doc):
+    human = doc.get("human_opens", 0)
+    ai = doc.get("ai_opens", 0)
+    crawl = doc.get("crawler_opens", 0)
+    bot = doc.get("bot_opens", 0)
+    
+    doc["total_opens"] = human + ai + crawl + bot
+    doc["popularity"] = human + (ai * 2) + (crawl * 0.1) + (bot * 0)
+    
+    total_interactive = human + ai
+    ai_percent = (ai / total_interactive * 100) if total_interactive > 0 else 0
+    doc["ai_open_percent"] = ai_percent
+    
+    tags = set()
+    if doc.get("trust_score", 50) >= 80: tags.add("Trusted")
+    if doc.get("quality_score", 50) >= 80: tags.add("High Quality")
+    if doc.get("domain_quality", 50) >= 90: tags.add("Verified")
+    if doc.get("total_opens", 0) >= 500: tags.add("Popular")
+    
+    if ai >= 25: tags.add("Opened by AI")
+    if total_interactive > 10 and ai_percent >= 80: tags.add("AI Favorite")
+    
+    doc["special_tags"] = list(tags)
+
 def normalize_text(text: str) -> str:
     if not text: return ""
     text = unicodedata.normalize('NFKD', str(text)).encode('ASCII', 'ignore').decode('utf-8')
@@ -122,28 +153,44 @@ def build_searchable_text(doc):
     parts = [doc.get("title", ""), doc.get("description", ""), doc.get("tag", "")]
     for k, v in doc.get("meta_tags", {}).items():
         if isinstance(v, str): parts.append(v)
+    for k, v in doc.get("og_tags", {}).items():
+        if isinstance(v, str): parts.append(v)
     return " ".join(p for p in parts if p).strip()
 
 def build_runtime_record(doc):
     r = doc.copy()
     r["meta_tags"] = parse_json_field(doc.get("meta_tags"))
+    r["og_tags"] = parse_json_field(doc.get("og_tags"))
+    r["specific_data"] = parse_json_field(doc.get("specific_data"))
+    
+    r["images"] = doc.get("images") or []
+    r["videos"] = doc.get("videos") or []
+    r["documents"] = doc.get("documents") or []
+    r["tag"] = doc.get("tag", "general")
     r["searchable_text"] = build_searchable_text(r)
     
     # Safe Search Tagging
     r["is_adult"] = is_adult_content(r["searchable_text"] + " " + r.get("url", ""))
     
+    r["human_opens"] = r.get("human_opens", 0)
+    r["ai_opens"] = r.get("ai_opens", 0)
+    r["crawler_opens"] = r.get("crawler_opens", 0)
+    r["bot_opens"] = r.get("bot_opens", 0)
     r["trust_score"] = r.get("trust_score", 50.0)
-    r["popularity"] = r.get("popularity", 0.0)
-    
-    tags = set()
-    if r["trust_score"] >= 80: tags.add("Trusted")
-    if r.get("domain_quality", 50) >= 90: tags.add("Verified")
-    r["special_tags"] = list(tags)
+    r["quality_score"] = r.get("quality_score", 50.0)
+    r["learning_score"] = r.get("learning_score", 50.0)
+    r["search_boost"] = r.get("search_boost", 0.0)
+    r["spam_score"] = r.get("spam_score", 0.0)
+    r["domain_quality"] = r.get("domain_quality", 50.0)
+    r["last_opened"] = r.get("last_opened", datetime.utcnow().isoformat())
+    r["language"] = r.get("language", "English")
+    r["country"] = r.get("country", "Global")
     
+    update_dynamic_fields(r)
     return r
 
 # ==============================================================================
-# DIRECT DATASET LOADING
+# DIRECT DATASET LOADING (No Train Split Errors)
 # ==============================================================================
 def load_dataset_direct():
     global bm25_index, raw_data, url_to_data
@@ -185,18 +232,19 @@ def load_dataset_direct():
 def init_system():
     global tokenizer, pipe, text_embedder, doc_embeddings
     init_db()
+    print("🚀 [1/3] Initializing System & Downloading Data...")
     
     texts_to_embed = load_dataset_direct()
     
     if ML_AVAILABLE:
-        print("🧠 [1/2] Loading Fast Semantic Embedder (all-MiniLM-L6-v2)...")
+        print("🧠 [2/3] Loading Fast Semantic Embedder (all-MiniLM-L6-v2)...")
         text_embedder = SentenceTransformer('all-MiniLM-L6-v2', device='cpu')
         
         # In-memory fast semantic indexing
         if texts_to_embed:
             doc_embeddings = text_embedder.encode(texts_to_embed, convert_to_numpy=True)
             
-        print("🔥 [2/2] Loading Qwen2.5-0.5B-Instruct for Agentic RAG...")
+        print("🔥 [3/3] Loading Qwen2.5-0.5B-Instruct for Agentic RAG...")
         try:
             model_id = "Qwen/Qwen2.5-0.5B-Instruct"
             tokenizer = AutoTokenizer.from_pretrained(model_id)
@@ -221,28 +269,27 @@ def calculate_domain_trust(url: str, query_terms: list):
     trust_score = 0
     domain_parts = domain.split('.')
     
-    # 1. Exact Brand/Domain Match (Massive Boost)
-    # Ex: if query is "github", and domain is "github.com"
+    # 1. Exact Brand/Domain Match (Massive Boost for official domains)
     for qt in query_terms:
         if qt in domain_parts and len(qt) > 2:
-            trust_score += 150  # Very high boost for official domains
+            trust_score += 150  
             
     # 2. Proxy / Fake Mirror Penalty
     proxy_indicators = ["proxy", "mirror", "unblock", "alt", "fake", "ip"]
     if any(p in domain for p in proxy_indicators):
-        trust_score -= 200  # Kill proxies from top ranking
+        trust_score -= 200  
         
     # 3. High Authority TLDs
     if domain.endswith(".gov") or domain.endswith(".edu"): trust_score += 30
     elif domain.endswith(".org") or domain.endswith(".com"): trust_score += 10
     
-    # 4. Domain Length Penalty (Long subdomains are usually spam)
+    # 4. Spam Penalty for long subdomains
     if len(domain_parts) > 3: trust_score -= 20 
         
     return trust_score
 
 def get_ranked_results(query: str):
-    # 1. Adult Query Check
+    # Safe Search Check
     is_safe_query = not is_adult_content(query)
     
     q_tokens = tokenize(query)
@@ -259,32 +306,39 @@ def get_ranked_results(query: str):
     scored_results = []
     
     for i, doc in enumerate(raw_data):
-        # Apply SafeSearch Filter
+        # Adult filter logic
         if is_safe_query and doc.get("is_adult"):
-            continue # Skip adult content if query is clean
+            continue
             
         domain_trust = calculate_domain_trust(doc["url"], query_terms_lower)
         
-        # Hybrid Formula
+        # Advanced Hybrid Formula
         score = (bm25_scores[i] * 2.0) + \
                 (semantic_scores[i] * 15.0) + \
                 domain_trust + \
                 (doc.get("trust_score", 50) * 0.1) + \
-                (doc.get("popularity", 0) * 0.05)
+                (doc.get("popularity", 0) * 0.05) + \
+                doc.get("search_boost", 0) - doc.get("spam_score", 0)
                 
-        if score > 0:
+        if score > -50: # Ignore completely heavily penalized spam
             scored_results.append((score, doc))
 
     scored_results.sort(key=lambda x: x[0], reverse=True)
     
     formatted = []
-    for s, d in scored_results[:50]:
+    for s, d in scored_results[:100]:
         formatted.append({
             "url": d["url"], "title": d["title"], "description": d["description"],
-            "domain": get_domain(d["url"]),
+            "domain": get_domain(d["url"]), "favicon": d.get("favicon", ""),
+            "language": d.get("language", "English"), "country": d.get("country", "Global"),
+            "images": d.get("images", []), "videos": d.get("videos", []), 
+            "tag": d.get("tag", "general"),
             "is_adult": d.get("is_adult", False),
-            "trust_score": round(d.get("trust_score", 50), 2),
-            "special_tags": d.get("special_tags", []),
+            "last_opened": d["last_opened"], "trust_score": round(d.get("trust_score", 50), 2),
+            "quality_score": round(d.get("quality_score", 50), 2), 
+            "learning_score": round(d.get("learning_score", 50), 2),
+            "human_opens": d["human_opens"], "ai_opens": d["ai_opens"],
+            "total_opens": d["total_opens"], "special_tags": d.get("special_tags", []),
             "match_score": round(s, 2), 
             "searchable_text": d.get("searchable_text", "")
         })
@@ -304,7 +358,7 @@ def get_best_chunks(query, full_text):
     return chunks[:3]
 
 # ==============================================================================
-# AGENTIC RAG (QWEN)
+# TRUE AGENTIC RAG PLANNER (QWEN)
 # ==============================================================================
 async def agent_planner_call(prompt):
     if not pipe: return "ACTION: ANSWER"
@@ -320,7 +374,7 @@ async def execute_agentic_rag(query, candidates, debug=False):
     visited_idx = set()
     debug_log = []
     
-    metadata_list = "\n".join([f"ID {i} | URL: {d['url']} | Title: {d['title']}" for i, d in enumerate(candidates[:15])])
+    metadata_list = "\n".join([f"ID {i} | URL: {d['url']} | Title: {d['title']}" for i, d in enumerate(candidates[:10])])
     
     async with AI_SEMAPHORE:
         for hop in range(5):
@@ -337,25 +391,47 @@ async def execute_agentic_rag(query, candidates, debug=False):
                     doc = candidates[idx]
                     best_chunks = get_best_chunks(query, doc.get("searchable_text", ""))
                     context_memory += f"\n[From {doc['domain']}]:\n" + "\n".join(best_chunks) + "\n"
-                    sources_used.append({"url": doc["url"], "title": doc["title"], "domain": doc["domain"]})
+                    
+                    if doc["url"] in url_to_data:
+                        url_to_data[doc["url"]]["ai_opens"] += 1
+                        url_to_data[doc["url"]]["learning_score"] += 1.0
+                        update_dynamic_fields(url_to_data[doc["url"]])
+                        
+                    sources_used.append({
+                        "url": doc["url"], "title": doc["title"], "domain": doc["domain"],
+                        "special_tags": doc["special_tags"], "favicon": doc["favicon"]
+                    })
                 else: break
             else: break
             
+        log_to_db("INSERT INTO planner_logs (query, hops, decision_log, ts) VALUES (?, ?, ?, ?)", 
+                 (query, len(visited_idx), " | ".join(debug_log), datetime.utcnow()))
+                 
         final_prompt = f"You are a helpful AI assistant. Use ONLY the provided Memory to answer the query.\nQuery: {query}\nMemory:\n{context_memory}\n\nAnswer:\n"
         
     return final_prompt, sources_used, debug_log
 
 # ==============================================================================
-# FASTAPI ENDPOINTS
+# FASTAPI APP & ALL ENDPOINTS
 # ==============================================================================
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     asyncio.create_task(asyncio.to_thread(init_system))
     yield
 
-app = FastAPI(title="Nebulix v6 Qwen", lifespan=lifespan)
+app = FastAPI(title="Nebulix v6 Qwen SafeSearch", lifespan=lifespan)
 app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
 
+@app.middleware("http")
+async def analytics_middleware(request: Request, call_next):
+    start_time = time.perf_counter()
+    response = await call_next(request)
+    latency = (time.perf_counter() - start_time) * 1000
+    if "/search" in request.url.path or "/aimode" in request.url.path:
+        q = request.query_params.get("q", "")
+        log_to_db("INSERT INTO logs (query, latency, ts) VALUES (?, ?, ?)", (q, latency, datetime.utcnow()))
+    return response
+
 @app.get("/search")
 async def api_search(q: str = Query(...)):
     cache_key = q.lower()
@@ -368,6 +444,32 @@ async def api_search(q: str = Query(...)):
     SEARCH_CACHE[cache_key] = resp
     return resp
 
+@app.get("/aimode")
+async def api_aimode(q: str = Query(...), debug: bool = False):
+    q_hash = hashlib.md5(q.encode()).hexdigest()
+    if q_hash in AI_ANSWER_CACHE: return AI_ANSWER_CACHE[q_hash]
+    
+    results = get_ranked_results(q)
+    final_prompt, sources, debug_log = await execute_agentic_rag(q, results, debug)
+    
+    if not pipe: return {"Answer": "Mocked", "Sources": sources}
+    
+    async with AI_SEMAPHORE:
+        res = await asyncio.to_thread(
+            pipe, final_prompt, max_new_tokens=1024, temperature=0.2, do_sample=True, return_full_text=False
+        )
+        answer = res[0]["generated_text"].strip()
+        
+    resp = {
+        "Answer": answer[:150] + "...",
+        "Summary": answer[:300],
+        "Detailed Explanation": answer,
+        "Sources": sources
+    }
+    if debug: resp["Debug_Planner"] = debug_log
+    AI_ANSWER_CACHE[q_hash] = resp
+    return resp
+
 @app.get("/aimodechat")
 async def api_aimodechat(q: str = Query(...)):
     results = get_ranked_results(q)
@@ -395,8 +497,108 @@ async def api_aimodechat(q: str = Query(...)):
         
     return StreamingResponse(true_event_stream(), media_type="text/event-stream")
 
+@app.get("/popular")
+async def api_popular():
+    docs = sorted(raw_data, key=lambda x: x.get("popularity", 0), reverse=True)[:10]
+    return {"popular": [{"url": d["url"], "title": d["title"], "popularity": d["popularity"]} for d in docs]}
+
+@app.get("/ai-popular")
+async def api_ai_popular():
+    docs = sorted(raw_data, key=lambda x: x.get("ai_opens", 0), reverse=True)[:10]
+    return {"ai_popular": [{"url": d["url"], "title": d["title"], "ai_opens": d["ai_opens"], "special_tags": d["special_tags"]} for d in docs]}
+
+@app.get("/trending")
+async def api_trending():
+    docs = sorted(raw_data, key=lambda x: x.get("total_opens", 0), reverse=True)[:10]
+    return {"trending": [{"url": d["url"], "title": d["title"]} for d in docs]}
+
+@app.get("/suggestions")
+async def api_suggestions(q: str = Query(...)):
+    base_q = q.lower().split()[-1]
+    words = []
+    def dfs(node, prefix):
+        if len(words) >= 5: return
+        if node.get('_end'): words.append(prefix)
+        for char, child in node.items():
+            if char != '_end': dfs(child, prefix + char)
+            
+    node = trie_root
+    for c in base_q:
+        if c in node: node = node[c]
+        else: break
+    if node and node != trie_root: dfs(node, base_q)
+    return {"suggestions": [q.replace(base_q, w) for w in words]}
+
+@app.get("/stats")
+async def api_stats():
+    mem = psutil.virtual_memory()
+    return {
+        "engine": "Nebulix v6.5 HF Spaces Edition",
+        "llm_model": "Qwen2.5-0.5B-Instruct",
+        "ranking": "Hybrid (Algorithmic Domain Trust + BM25 + Semantic)",
+        "documents": len(raw_data),
+        "cpu_usage_percent": psutil.cpu_percent(),
+        "ram_usage_percent": mem.percent,
+        "search_cache_items": len(SEARCH_CACHE),
+        "ai_cache_items": len(AI_ANSWER_CACHE)
+    }
+
 @app.get("/health")
-async def api_health(): return {"status": "ok", "model": "Qwen2.5-0.5B"}
+async def api_health(): return {"status": "ok", "version": "6.5"}
+
+@app.get("/url-info")
+async def api_url_info(url: str = Query(...)):
+    if url in url_to_data: 
+        safe_doc = url_to_data[url].copy()
+        safe_doc.pop("searchable_text", None)
+        return safe_doc
+    return {"error": "Not Found"}
+
+@app.post("/click")
+async def api_click(req: Request):
+    data = await req.json()
+    url = data.get("url")
+    ip = req.client.host
+    track_id = f"{ip}_{url}"
+    if track_id in CLICK_TRACKER and time.time() - CLICK_TRACKER[track_id] < 1800:
+        return {"status": "ignored_duplicate"}
+        
+    CLICK_TRACKER[track_id] = time.time()
+    if url in url_to_data:
+        d = url_to_data[url]
+        d["human_opens"] += 1
+        d["last_opened"] = datetime.utcnow().isoformat()
+        update_dynamic_fields(d)
+        log_to_db("INSERT INTO clicks (url, type, ts) VALUES (?, ?, ?)", (url, "human", datetime.utcnow()))
+        return {"status": "success", "human_opens": d["human_opens"]}
+    return {"error": "Not found"}
+
+@app.post("/ai-open")
+async def api_ai_open(req: Request):
+    data = await req.json()
+    url = data.get("url")
+    if url in url_to_data:
+        d = url_to_data[url]
+        d["ai_opens"] += 1
+        d["learning_score"] += 1.0
+        d["last_opened"] = datetime.utcnow().isoformat()
+        update_dynamic_fields(d)
+        log_to_db("INSERT INTO clicks (url, type, ts) VALUES (?, ?, ?)", (url, "ai", datetime.utcnow()))
+        return {"status": "success"}
+    return {"error": "Not found"}
+
+@app.post("/feedback")
+async def api_feedback(req: Request):
+    data = await req.json()
+    url = data.get("url")
+    if url in url_to_data:
+        url_to_data[url]["trust_score"] += 2.0 if data.get("helpful") else -1.0
+        update_dynamic_fields(url_to_data[url])
+    return {"status": "feedback_registered"}
+
+@app.post("/search-log")
+async def api_search_log(req: Request):
+    return {"status": "logged"}
 
 if __name__ == "__main__":
     uvicorn.run(app, host="0.0.0.0", port=7860, log_level="warning")