Update app.py

app.py

@@ -1,215 +1,181 @@
-import gradio as gr
 import json
-import hashlib
 import uuid
-import re
-from datetime import datetime
-from rapidfuzz import fuzz
-from transformers import pipeline
+import asyncio
 import os
+from datetime import datetime, timedelta
+from urllib.parse import urlparse
+from fastapi import FastAPI, Query, BackgroundTasks
+from fastapi.responses import JSONResponse
+from rapidfuzz import process, fuzz
+from cryptography.fernet import Fernet
 
-# ==========================================
-# 1. DATA LOADING (IN-MEMORY DATABASE)
-# ==========================================
-DATA_FILE = "indexed_search_db.jsonl"
-search_db = []
-
-print("Loading Database...")
-if os.path.exists(DATA_FILE):
-    with open(DATA_FILE, "r", encoding="utf-8") as f:
-        for line in f:
-            if line.strip():
-                search_db.append(json.loads(line))
-    print(f"Loaded {len(search_db)} records from database.")
-else:
-    print(f"Warning: {DATA_FILE} not found! Please upload your data file to the space.")
-
-# ==========================================
-# 2. AI MODEL SETUP (Zero-Shot Intent)
-# ==========================================
-print("Loading AI Model for Smart Decision Making...")
-intent_classifier = pipeline(
-    "zero-shot-classification", 
-    model="typeform/distilbert-base-uncased-mnli" 
-)
-INTENT_LABELS = ["self-harm", "adult", "technology", "news", "entertainment", "sports", "information"]
-
-# ==========================================
-# 3. HELPER FUNCTIONS
-# ==========================================
-def is_url(text):
-    # Check agar user ne direct URL daala hai
-    pattern = re.compile(r'^(https?://|www\.)\S+$', re.IGNORECASE)
-    return bool(pattern.match(text.strip()))
-
-def generate_user_id():
-    return f"user_{uuid.uuid4().hex[:10]}"
-
-# ==========================================
-# 4. CORE BATCH SEARCH ENGINE (AI + FUZZY)
-# ==========================================
-def batch_search_engine(queries, limits, user_ids):
-    # Returns 2 lists: one for JSON outputs, one for HTML (Redirections)
-    final_json_responses = []
-    final_html_responses = []
+# --- 1. SETUP, ENCRYPTION & DIRECTORY ---
+# Note: Production me Fernet key ko environment variable se lena chahiye.
+SECRET_KEY = Fernet.generate_key()
+cipher_suite = Fernet(SECRET_KEY)
+
+# Logs ke liye alag folder banayenge taki clean rahe
+LOG_DIR = "logs"
+os.makedirs(LOG_DIR, exist_ok=True)
+
+app = FastAPI(title="Fast Search API")
+
+# --- 2. LOAD DATA FROM dataset.jsonl ---
+raw_data = []
+search_corpus = []
+
+def load_dataset():
+    global raw_data, search_corpus
+    try:
+        with open("dataset.jsonl", "r", encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    item = json.loads(line)
+                    raw_data.append(item)
+                    
+                    # Domain, title aur description nikal kar corpus me dalna
+                    domain = urlparse(item.get("url", "")).netloc
+                    text_to_search = f"{domain} {item.get('title', '')} {item.get('description', '')}"
+                    search_corpus.append(text_to_search.lower())
+        print(f"✅ Successfully loaded {len(raw_data)} records from dataset.jsonl")
+    except FileNotFoundError:
+        print("❌ ERROR: dataset.jsonl file not found in the directory!")
+
+# Load dataset when app starts
+load_dataset()
+
+
+# --- 3. DATE-BASED LOGGING & ENCRYPTION ---
+async def log_query_encrypted(req_id: str, query: str):
+    encrypted_q = cipher_suite.encrypt(query.encode()).decode()
+    now = datetime.now()
+    
+    # Date ke hisaab se file ka naam banega (e.g., search_logs_2026-07-22.jsonl)
+    date_str = now.strftime("%Y-%m-%d")
+    log_file = os.path.join(LOG_DIR, f"search_logs_{date_str}.jsonl")
     
-    # Check limits
-    processed_limits = []
-    for l in limits:
+    log_entry = {
+        "id": req_id,
+        "query": encrypted_q,
+        "timestamp": now.isoformat()
+    }
+    
+    # File me append karna (Agar file nahi hai toh khud ban jayegi)
+    with open(log_file, "a", encoding="utf-8") as f:
+        f.write(json.dumps(log_entry) + "\n")
+
+
+# --- 4. OPTIMIZED 1 MONTH LOG CLEANUP ---
+async def cleanup_old_logs():
+    while True:
         try:
-            l_int = int(l)
-            if l_int <= 0 or l_int > 30:
-                processed_limits.append(30)
-            else:
-                processed_limits.append(l_int)
-        except:
-            processed_limits.append(30)
-            
-    # Process batch with AI (Skip direct URLs for AI processing to save time)
-    ai_queries = [q if not is_url(q) else "bypass" for q in queries]
-    ai_results = intent_classifier(ai_queries, INTENT_LABELS)
-    if isinstance(ai_results, dict):
-        ai_results = [ai_results]
-
-    for i in range(len(queries)):
-        query = str(queries[i]).strip()
-        limit = processed_limits[i]
-        user_id = user_ids[i]
-        
-        # --- DIRECT URL REDIRECT LOGIC ---
-        if is_url(query):
-            url_to_open = query if query.startswith("http") else "http://" + query
-            # Redirect via HTML meta tag
-            redirect_html = f'<meta http-equiv="refresh" content="0; url={url_to_open}">'
-            final_html_responses.append(redirect_html)
-            
-            # Add small info in JSON
-            final_json_responses.append({
-                "status": "redirecting",
-                "message": f"Direct URL detected. Redirecting to {url_to_open}",
-                "url": url_to_open
-            })
-            continue
-            
-        # --- PRIVACY LOGGING ---
-        query_hash = hashlib.sha256(query.encode()).hexdigest()[:16]
-        top_intent = ai_results[i]['labels'][0]
-        
-        log_entry = {
-            "user_id": user_id,
-            "query_hash": query_hash,
-            "timestamp": datetime.utcnow().isoformat(),
-            "ai_intent": top_intent
-        }
-        print(f"SECURE LOG: {log_entry}") 
-        
-        # --- SELF HARM CHECK ---
-        if top_intent == "self-harm":
-            final_html_responses.append("<div></div>") # No HTML redirect
-            final_json_responses.append({
-                "status": "blocked",
-                "ai_intent": "self-harm",
-                "message": "⚠️ Safety Alert: We do not provide search results for self-harm. Please seek help.",
-                "results": []
-            })
-            continue
-
-        # --- AI + FUZZY MATCH LOGIC ---
-        results_with_scores = []
-        
-        for item in search_db:
-            text_to_search = item.get('search_index', '')
-            
-            # 1. Fuzzy Token Set Ratio (Keywords aage-peeche ho toh bhi chalega)
-            fuzzy_score = fuzz.token_set_ratio(query.lower(), text_to_search.lower())
-            
-            # 2. AI Decision Boosting (Konsa dena hai)
-            boost = 0
-            domain = item.get('domain', '').lower()
-            
-            # AI samajh raha hai ki user ko exact kis domain ka content chahiye
-            if top_intent == "adult" and domain in ["xhname", "avideos", "redwap", "dlsite"]:
-                boost = 25
-            elif top_intent == "technology" and domain in ["github", "scmp", "workspace", "bitdefender"]:
-                boost = 25
-            elif top_intent == "news" and domain in ["news", "nymag", "abc"]:
-                boost = 25
-            elif top_intent == "sports" and domain in ["polymarket"]:
-                boost = 25
+            if os.path.exists(LOG_DIR):
+                thirty_days_ago = datetime.now() - timedelta(days=30)
                 
-            final_score = fuzzy_score + boost
-            
-            # Minimum 30 score hona chahiye text match ka tabhi result aayega
-            if final_score > 30:
-                results_with_scores.append({
-                    "score": final_score,
-                    "data": item
-                })
-                
-        # Highest score (AI decides + Fuzzy match) will be on TOP
-        results_with_scores.sort(key=lambda x: x["score"], reverse=True)
-        
-        # Limit the results (Max 30)
-        final_data = [res["data"] for res in results_with_scores[:limit]]
+                # Folder ki saari files check karenge
+                for filename in os.listdir(LOG_DIR):
+                    if filename.startswith("search_logs_") and filename.endswith(".jsonl"):
+                        # File ke naam se date nikalenge
+                        date_part = filename.replace("search_logs_", "").replace(".jsonl", "")
+                        try:
+                            file_date = datetime.strptime(date_part, "%Y-%m-%d")
+                            # Agar file 30 din se purani hai, toh puri file delete kar do
+                            if file_date < thirty_days_ago:
+                                os.remove(os.path.join(LOG_DIR, filename))
+                                print(f"🗑️ Deleted 1-month old log file: {filename}")
+                        except ValueError:
+                            pass
+        except Exception as e:
+            print("Cleanup Error:", e)
         
-        final_html_responses.append("<div></div>") # Empty HTML for normal searches
-        
-        # Using Python Dictionary directly to avoid UI bug `{`
-        final_json_responses.append({
-            "status": "success",
-            "ai_predicted_intent": top_intent,
-            "total_found": len(final_data),
-            "limit_applied": limit,
-            "results": final_data
-        })
-
-    # Return lists because batch=True requires list of outputs for each component
-    return final_json_responses, final_html_responses
-
-# ==========================================
-# 5. GRADIO UI & QUEUE SYSTEM
-# ==========================================
-with gr.Blocks(theme=gr.themes.Soft()) as demo:
-    gr.Markdown("# 🚀 Ultimate AI Search Engine")
-    gr.Markdown("Type your search query or paste a direct **URL**. Powered by AI Intent & Fuzzy Text Search.")
+        # Har 24 ghante me ek baar check karega
+        await asyncio.sleep(24 * 60 * 60)
+
+@app.on_event("startup")
+async def startup_event():
+    # Background me cleanup task on startup
+    asyncio.create_task(cleanup_old_logs())
+
+
+# --- 5. FAST AI / NLP LOGIC ---
+def check_self_harm(query: str) -> bool:
+    harm_keywords = ["suicide", "kill myself", "die", "harm myself", "end my life", "suiside"]
+    return any(word in query.lower() for word in harm_keywords)
+
+def check_adult_content(query: str) -> bool:
+    adult_keywords = ["porn", "sex", "xxx", "nude", "adult"]
+    return any(word in query.lower() for word in adult_keywords)
+
+def ai_fallback_search(query: str):
+    return {
+        "ai_fallback_triggered": True,
+        "message": f"Humne apne local data me '{query}' dhunda par kuch nahi mila. AI ne sab jagah ghuma aur ye result laya hai.",
+        "results": [
+            {
+                "url": f"https://www.google.com/search?q={query.replace(' ', '+')}",
+                "title": f"AI Web Search: {query}",
+                "description": "Local DB me result na milne par ye AI generated fallback result hai.",
+                "images": [],
+                "videos": [],
+                "docs": []
+            }
+        ]
+    }
+
+
+# --- 6. SEARCH API ENDPOINT ---
+@app.get("/search")
+async def search(q: str = Query(..., description="Search query"), background_tasks: BackgroundTasks = BackgroundTasks()):
+    req_id = str(uuid.uuid4())
     
-    session_user_id = gr.State(generate_user_id)
+    # Background Logging (FastAPI wait nahi karega iske liye)
+    background_tasks.add_task(log_query_encrypted, req_id, q)
     
-    with gr.Row():
-        query_input = gr.Textbox(label="Search Box", placeholder="Search anything or paste a URL (e.g. https://google.com)...", scale=4)
-        limit_input = gr.Number(label="Limit (Max 30)", value=30, precision=0, scale=1)
-        
-    search_btn = gr.Button("Search 🔍", variant="primary")
+    # Self-Harm Check
+    if check_self_harm(q):
+        return JSONResponse(
+            status_code=400,
+            content={
+                "request_id": req_id,
+                "error": "Safety Alert",
+                "message": "Please do not harm yourself. Help is available. You are valuable."
+            }
+        )
+    
+    is_adult = check_adult_content(q)
     
-    # Fixed UI Issue: Using gr.JSON instead of gr.Code
-    output_json = gr.JSON(label="Search Results")
+    final_results = []
     
-    # Hidden HTML component for URL redirection
-    redirect_html = gr.HTML(label="", visible=False)
+    # RapidFuzz search algorithm - searches in domain, title, and description
+    if search_corpus:
+        results = process.extract(
+            q.lower(), 
+            search_corpus, 
+            limit=30, 
+            scorer=fuzz.WRatio
+        )
+        
+        for match_str, score, index in results:
+            if score > 40:
+                result_item = raw_data[index].copy() # Full data return hoga including images
+                result_item["match_score"] = round(score, 2)
+                final_results.append(result_item)
+            
+    # Fallback to AI if no results found
+    if len(final_results) == 0:
+        response_data = ai_fallback_search(q)
+    else:
+        response_data = {
+            "total_results": len(final_results),
+            "results": final_results
+        }
+        
+    response_data["request_id"] = req_id
+
+    # Adult Content Header & JSON Modifier
+    if is_adult:
+        response_data["safety_message"] = "you are safe you can do anything"
+        headers = {"X-Safety-Message": "you are safe you can do anything"}
+        return JSONResponse(content=response_data, headers=headers)
     
-    # Link Button & Enter Key (Batch Mode enabled)
-    search_btn.click(
-        fn=batch_search_engine,
-        inputs=[query_input, limit_input, session_user_id],
-        outputs=[output_json, redirect_html],
-        batch=True,          
-        max_batch_size=10,   
-        api_name="search"    
-    )
-
-    query_input.submit(
-        fn=batch_search_engine,
-        inputs=[query_input, limit_input, session_user_id],
-        outputs=[output_json, redirect_html],
-        batch=True,
-        max_batch_size=10
-    )
-
-# Queue System: First come first serve with concurrency
-demo.queue(
-    default_concurrency_limit=5, 
-    max_size=200                 
-)
-
-if __name__ == "__main__":
-    demo.launch()
+    return response_data