Nebulixlabs commited on
Commit
0d4757d
·
verified ·
1 Parent(s): b72fd22

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +207 -204
app.py CHANGED
@@ -7,58 +7,80 @@ import re
7
  import time
8
  import multiprocessing
9
  import urllib.parse
10
- from datetime import datetime
11
  from contextlib import asynccontextmanager
12
- from fastapi import FastAPI, Query, Request, UploadFile, File, Form
13
  from fastapi.responses import JSONResponse
14
  from fastapi.middleware.cors import CORSMiddleware
15
  from huggingface_hub import hf_hub_download
16
  from rank_bm25 import BM25Okapi
17
  import torch
18
- from PIL import Image
19
- import io
 
20
 
21
- # DIRECT IMPORTS TO FIX THE PIPELINE BUG
22
- from transformers import pipeline, BlipProcessor, BlipForConditionalGeneration
23
 
24
- # --- 1. SETUP & CPU ALLOCATION (40% for AI) ---
25
  TOTAL_CORES = multiprocessing.cpu_count()
26
  AI_CORES = max(1, int(TOTAL_CORES * 0.4))
27
  torch.set_num_threads(AI_CORES)
28
  print(f"🖥️ Total Cores: {TOTAL_CORES} | Allocating {AI_CORES} cores strictly to AI.")
29
 
30
  raw_data = []
 
31
  bm25_index = None
32
  MAX_CONCURRENT_AI = max(1, AI_CORES)
33
  ai_semaphore = asyncio.Semaphore(MAX_CONCURRENT_AI)
34
  ai_generator = None
35
- vision_processor = None
36
- vision_model = None
 
37
 
38
  # --- AI CACHING SYSTEM ---
39
  AI_RESPONSE_CACHE = {}
40
  MAX_CACHE_SIZE = 5000
41
- ADULT_KEYWORDS = {"porn", "sex", "nude", "nsfw", "xxx", "adult", "erotic", "onlyfans"}
42
  STOP_WORDS = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "is", "are"}
43
  HF_TOKEN = os.getenv("HF_TOKEN")
44
  BANNED_IPS = set()
45
  IP_TRACKER = {}
46
 
47
- # --- 2. ADVANCED NLP & IN-MEMORY INDEXING ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  def tokenize(text):
49
  if not text: return []
50
  tokens = re.findall(r'[a-z0-9]+', str(text).lower())
51
  return [t for t in tokens if t not in STOP_WORDS]
52
 
53
- def extract_nested_data(obj):
54
- if isinstance(obj, dict):
55
- return " ".join(extract_nested_data(v) for v in obj.values())
56
- elif isinstance(obj, list):
57
- return " ".join(extract_nested_data(v) for v in obj)
58
- return str(obj)
 
59
 
60
  def load_and_index_dataset():
61
- global bm25_index, raw_data
62
  print("📥 Loading Dataset & Building Deep Index...")
63
  dataset_path = "dataset.jsonl"
64
 
@@ -81,98 +103,140 @@ def load_and_index_dataset():
81
  if line.strip():
82
  try:
83
  doc = json.loads(line)
 
 
 
 
 
 
 
 
 
84
  raw_data.append(doc)
85
 
86
- title_tokens = tokenize(doc.get("title", ""))
 
 
 
 
87
  desc_tokens = tokenize(doc.get("description", ""))
88
- meta_tokens = tokenize(doc.get("meta_tags", {}).get("description", ""))
89
- url_tokens = tokenize(doc.get("url", ""))
90
- specific_tokens = tokenize(extract_nested_data(doc.get("specific_data", {})))
91
 
92
- boosted_tokens = (title_tokens * 5) + (desc_tokens * 3) + meta_tokens + url_tokens + (specific_tokens * 2)
93
  tokenized_corpus.append(boosted_tokens)
94
  except:
95
  pass
96
 
97
  if tokenized_corpus:
98
  print("⚡ Building High-Speed BM25 Okapi Index...")
 
99
  bm25_index = BM25Okapi(tokenized_corpus)
100
  print(f"🚀 Indexing Complete! {len(raw_data)} records ready.")
101
  else:
102
  print("❌ Dataset file not found!")
103
 
104
  def load_ai_models():
105
- global ai_generator, vision_processor, vision_model
106
- print("🤖 Loading Lightweight AI Models (Text & Vision)...")
107
  try:
108
- # 1. Text AI
109
- ai_generator = pipeline("text-generation", model="Qwen/Qwen2.5-0.5B-Instruct", device=-1, dtype=torch.float32)
110
-
111
- # 2. Vision AI
112
- print("👁️ Loading Vision AI...")
113
- vision_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
114
- vision_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
115
-
116
- print("✅ All AI Models Loaded! System ready.")
117
  except Exception as e:
118
  print(f"❌ Model Loading Failed: {e}")
119
 
120
- # --- 3. CORE SEARCH ALGORITHM ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121
  async def perform_core_search(raw_query: str):
122
  if not raw_data or not bm25_index:
123
  return [], "Search index not ready.", 0
124
 
125
- site_filter = None
126
- tag_filter = None
127
-
128
- site_match = re.search(r'site:([^\s]+)', raw_query, re.IGNORECASE)
129
- if site_match:
130
- site_filter = site_match.group(1).lower()
131
- raw_query = raw_query.replace(site_match.group(0), "")
132
-
133
- tag_match = re.search(r'tag:([^\s]+)', raw_query, re.IGNORECASE)
134
- if tag_match:
135
- tag_filter = tag_match.group(1).lower()
136
- raw_query = raw_query.replace(tag_match.group(0), "")
137
-
138
- query = raw_query.strip()
139
  query_tokens = tokenize(query)
140
 
141
- if not query_tokens and not site_filter and not tag_filter:
142
- return [], "Invalid Query", 0
143
-
144
- user_wants_adult = any(word in ADULT_KEYWORDS for word in query_tokens)
145
  scores = bm25_index.get_scores(query_tokens) if query_tokens else [1.0] * len(raw_data)
146
-
147
  scored_results = []
148
  seen_urls = set()
149
 
150
  for index, score in enumerate(scores):
151
- if score > 0.5 or not query_tokens:
152
  doc = raw_data[index]
153
  doc_url = doc.get("url", "")
154
 
155
  if doc_url in seen_urls: continue
156
  parsed_url = urllib.parse.urlparse(doc_url)
 
157
 
158
- if site_filter and site_filter not in parsed_url.netloc.lower(): continue
159
- doc_tag = str(doc.get("tag", "general")).lower()
160
- if tag_filter and tag_filter != doc_tag: continue
161
-
162
- is_adult = doc_tag in ["adult", "nsfw"] or any(w in str(doc).lower() for w in ADULT_KEYWORDS)
163
- if not user_wants_adult and is_adult: continue
164
 
 
 
 
 
165
  final_score = score
166
- exact_phrase = query.lower()
167
- doc_title = str(doc.get("title", "")).lower()
168
 
169
- if exact_phrase == doc_title: final_score *= 5.0
170
- elif exact_phrase in doc_title: final_score *= 3.0
171
- if exact_phrase in str(doc.get("description", "")).lower(): final_score *= 1.8
 
 
172
 
173
- path = parsed_url.path.strip("/")
174
- if not path: final_score *= 1.5
175
- else: final_score *= (0.90 ** path.count("/"))
 
 
 
176
 
177
  seen_urls.add(doc_url)
178
  scored_results.append((final_score, doc))
@@ -184,16 +248,11 @@ async def perform_core_search(raw_query: str):
184
  final_results = [{"match_score": round(s, 2), **doc} for s, doc in scored_results[:100]]
185
  return final_results, prediction, best_score
186
 
187
- # --- AI GENERATION & CACHE HELPER ---
188
  def _generate_text(messages, max_tokens: int):
189
- result = ai_generator(messages, max_new_tokens=max_tokens, do_sample=True, temperature=0.3, top_p=0.85)
190
- return result[0]['generated_text'][-1]['content']
191
-
192
- def _generate_image_review(image: Image):
193
- inputs = vision_processor(image, return_tensors="pt")
194
- with torch.no_grad():
195
- out = vision_model.generate(**inputs, max_new_tokens=50)
196
- return vision_processor.decode(out[0], skip_special_tokens=True).capitalize()
197
 
198
  async def run_ai_task(messages, max_tokens: int, query_key: str):
199
  if not ai_generator: return "⚠️ AI Model is starting up."
@@ -205,16 +264,15 @@ async def run_ai_task(messages, max_tokens: int, query_key: str):
205
  AI_RESPONSE_CACHE[query_key] = response
206
  return response
207
 
208
- # --- 4. FASTAPI APP & ENDPOINTS ---
209
  @asynccontextmanager
210
  async def lifespan(app: FastAPI):
211
  asyncio.create_task(asyncio.to_thread(load_and_index_dataset))
212
  asyncio.create_task(asyncio.to_thread(load_ai_models))
213
  yield
214
 
215
- app = FastAPI(title="Nebulix Advanced SEO & Visual API Engine", lifespan=lifespan)
216
 
217
- # CORS FIX: allow_origins aur allow_methods mein empty sting [""] ki jagah ["*"] chahiye
218
  app.add_middleware(
219
  CORSMiddleware,
220
  allow_origins=["*"],
@@ -223,31 +281,25 @@ app.add_middleware(
223
  allow_headers=["*"]
224
  )
225
 
226
- @app.middleware("http")
227
- async def rate_limit_middleware(request: Request, call_next):
228
- client_ip = request.client.host if request.client else "127.0.0.1"
229
-
230
- if client_ip in BANNED_IPS:
231
- return JSONResponse(status_code=403, content={"error": "BANNED: Your IP is permanently banned."})
232
-
233
- current_time = int(time.time())
234
- if client_ip not in IP_TRACKER:
235
- IP_TRACKER[client_ip] = {"time": current_time, "count": 1}
236
- else:
237
- if IP_TRACKER[client_ip]["time"] == current_time:
238
- IP_TRACKER[client_ip]["count"] += 1
239
- else:
240
- IP_TRACKER[client_ip] = {"time": current_time, "count": 1}
241
-
242
- if IP_TRACKER[client_ip]["count"] >= 6:
243
- BANNED_IPS.add(client_ip)
244
- return JSONResponse(status_code=403, content={"error": "IP BANNED: Exceeded limit."})
245
-
246
- return await call_next(request)
247
-
248
- @app.get("/")
249
- async def health_check():
250
- return {"status": "ok", "message": "Nebulix API Engine is running smoothly."}
251
 
252
  @app.get("/search")
253
  async def pure_search(q: str = Query(...)):
@@ -255,113 +307,64 @@ async def pure_search(q: str = Query(...)):
255
  results, prediction, _ = await perform_core_search(q)
256
  return {"request_id": req_id, "query": q, "results_count": len(results), "results": results}
257
 
258
- @app.get("/aimode")
259
- async def ai_mode_short(q: str = Query(...)):
260
- req_id = str(uuid.uuid4())
261
- results, _, _ = await perform_core_search(q)
262
- if not results:
263
- return {"request_id": req_id, "ai_response": "I couldn't find any results.", "results": []}
264
-
265
- context = " ".join([f"{r.get('title', '')} - {r.get('description', '')}" for r in results[:3]])
266
- messages = [
267
- {"role": "system", "content": "Provide a brief 2-3 sentence summary based only on the provided context."},
268
- {"role": "user", "content": f"Query: {q}\nContext: {context}"}
269
- ]
270
-
271
- ai_reply = await run_ai_task(messages, 150, query_key=f"short_{q}")
272
- return {"request_id": req_id, "ai_response": ai_reply, "results": results}
273
-
274
- # 1. PURANA CHAT API (Current UI ke liye)
275
- @app.get("/aimodechat")
276
- async def ai_mode_chat(q: str = Query(...)):
277
- req_id = str(uuid.uuid4())
278
- results, _, _ = await perform_core_search(q)
279
-
280
- context_parts = [f"Source [{i+1}]: {r.get('title')} - {r.get('description')}" for i, r in enumerate(results[:5])]
281
- messages = [
282
- {"role": "system", "content": "You are a Smart AI Search Assistant. Use search sources to answer accurately."},
283
- {"role": "user", "content": f"User Query: {q}\n\nContext:\n{chr(10).join(context_parts)}"}
284
- ]
285
-
286
- ai_reply = await run_ai_task(messages, 400, query_key=f"chat_{q}")
287
- return {"request_id": req_id, "ai_response": ai_reply, "results": results}
288
-
289
- # 2. NAYA CHAT API (Future UI - Image + Text Chat ke liye)
290
  @app.post("/aimodechat")
291
- async def ai_mode_chat_vision(q: str = Form(...), file: UploadFile = File(None)):
292
  req_id = str(uuid.uuid4())
293
- vision_context = ""
294
 
295
- # Agar user ne chat me image upload ki hai, toh AI image ko decode karega
296
- if file and vision_model:
297
- try:
298
- image_bytes = await file.read()
299
- img = Image.open(io.BytesIO(image_bytes)).convert("RGB")
300
- async with ai_semaphore:
301
- ai_review = await asyncio.to_thread(_generate_image_review, img)
302
- vision_context = f"\n[User uploaded an image showing: {ai_review}]\n"
303
- except Exception:
304
- vision_context = "\n[User uploaded an image but it could not be analyzed.]\n"
305
 
306
- # Search query combining text and vision
307
- search_query = f"{q} {vision_context}".strip()
308
- results, _, _ = await perform_core_search(search_query)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
309
 
310
- context_parts = [f"Source [{i+1}]: {r.get('title')} - {r.get('description')}" for i, r in enumerate(results[:5])]
 
 
311
 
312
  messages = [
313
- {"role": "system", "content": "You are a Smart AI Search Assistant capable of understanding text and visual context. Use the provided search sources and image details to answer accurately."},
314
- {"role": "user", "content": f"User Query: {q}{vision_context}\n\nContext:\n{chr(10).join(context_parts)}"}
315
  ]
316
 
317
- # Generate Chat response
318
- cache_key = f"chat_{q}_{len(vision_context)}"
319
- ai_reply = await run_ai_task(messages, 400, query_key=cache_key)
320
-
321
- return {"request_id": req_id, "ai_response": ai_reply, "results": results}
322
 
323
- # IMAGE SEARCH (Fixed: Same JSON structure as normal text search)
324
- @app.post("/image-search")
325
- async def reverse_image_search(file: UploadFile = File(...)):
326
- if not vision_model:
327
- return {"error": "Vision AI Model is still loading."}
328
-
329
- req_id = str(uuid.uuid4())
330
-
331
- try:
332
- image_bytes = await file.read()
333
- img = Image.open(io.BytesIO(image_bytes)).convert("RGB")
334
-
335
- # 1. Vision AI process karega
336
- async with ai_semaphore:
337
- ai_review = await asyncio.to_thread(_generate_image_review, img)
338
-
339
- # 2. Text Search hoga vision ke basis pe
340
- raw_results, _, _ = await perform_core_search(ai_review)
341
- visual_results = []
342
-
343
- # 3. DIRECTLY append the original FULL document
344
- for doc in raw_results:
345
- is_adult = str(doc.get("tag", "general")).lower() in ["adult", "nsfw"] or any(w in str(doc).lower() for w in ADULT_KEYWORDS)
346
- if is_adult: continue
347
-
348
- # Pura document direct bhej rahe hain bina modify kiye
349
- # jisse exact text search wala JSON format barkarar rahe
350
- visual_results.append(doc)
351
-
352
- if len(visual_results) >= 40: break
353
-
354
- # 4. Same structure return karenge
355
- return {
356
- "request_id": req_id,
357
- "query": f"Image Search Analysis: {ai_review}", # Taki UI me ye "query" ki tarah display ho
358
- "results_count": len(visual_results),
359
- "results": visual_results
360
- }
361
-
362
- except Exception as e:
363
- return {"error": f"Image processing failed: {str(e)}"}
364
 
365
- # DUNDAR NAME FIX: name == "main" ki jagah __name__ == "__main__"
366
  if __name__ == "__main__":
367
- uvicorn.run(app, host="0.0.0.0", port=7860)
 
7
  import time
8
  import multiprocessing
9
  import urllib.parse
 
10
  from contextlib import asynccontextmanager
11
+ from fastapi import FastAPI, Query, Request, Form
12
  from fastapi.responses import JSONResponse
13
  from fastapi.middleware.cors import CORSMiddleware
14
  from huggingface_hub import hf_hub_download
15
  from rank_bm25 import BM25Okapi
16
  import torch
17
+ import aiohttp
18
+ from bs4 import BeautifulSoup
19
+ from langdetect import detect
20
 
21
+ # DIRECT IMPORTS
22
+ from transformers import pipeline
23
 
24
+ # --- 1. SETUP & CPU ALLOCATION ---
25
  TOTAL_CORES = multiprocessing.cpu_count()
26
  AI_CORES = max(1, int(TOTAL_CORES * 0.4))
27
  torch.set_num_threads(AI_CORES)
28
  print(f"🖥️ Total Cores: {TOTAL_CORES} | Allocating {AI_CORES} cores strictly to AI.")
29
 
30
  raw_data = []
31
+ suggestion_list = []
32
  bm25_index = None
33
  MAX_CONCURRENT_AI = max(1, AI_CORES)
34
  ai_semaphore = asyncio.Semaphore(MAX_CONCURRENT_AI)
35
  ai_generator = None
36
+
37
+ # --- TRACKING & TRUST SYSTEM ---
38
+ URL_STATS = {} # Format: { "https://example.com": {"user_visits": 0, "ai_visits": 0, "is_trusted": False} }
39
 
40
  # --- AI CACHING SYSTEM ---
41
  AI_RESPONSE_CACHE = {}
42
  MAX_CACHE_SIZE = 5000
 
43
  STOP_WORDS = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "is", "are"}
44
  HF_TOKEN = os.getenv("HF_TOKEN")
45
  BANNED_IPS = set()
46
  IP_TRACKER = {}
47
 
48
+ # 🔥 MASSIVE DESI & MODERN ADULT KEYWORDS LIST
49
+ ADULT_KEYWORDS = {
50
+ "porn", "sex", "nude", "nsfw", "xxx", "adult", "erotic", "onlyfans",
51
+ "mms", "chudai", "bhabhi", "devar", "savita bhabhi", "xvideos", "pornhub",
52
+ "brazzers", "mia khalifa", "bf video", "blue film", "kamasutra", "nude",
53
+ "boobs", "tits", "ass", "booty", "naked", "desi porn", "hot desi", "xnxx",
54
+ "kacha badam", "ullu", "kooku", "mastram", "gandi baat", "nangi", "nanga",
55
+ "sex tape", "milf", "stepmom", "incest", "playboy", "scandal", "leaked"
56
+ }
57
+
58
+ # 🔥 MASSIVE LANGUAGE MAPPER (50+ Languages)
59
+ LANG_MAP = {
60
+ "en": "English", "hi": "Hindi", "mr": "Marathi", "bn": "Bengali", "ta": "Tamil",
61
+ "te": "Telugu", "gu": "Gujarati", "ur": "Urdu", "kn": "Kannada", "ml": "Malayalam",
62
+ "pa": "Punjabi", "or": "Odia", "ne": "Nepali", "fr": "French", "es": "Spanish",
63
+ "de": "German", "ru": "Russian", "ja": "Japanese", "zh-cn": "Chinese (Simplified)",
64
+ "zh-tw": "Chinese (Traditional)", "ar": "Arabic", "pt": "Portuguese", "it": "Italian",
65
+ "ko": "Korean", "id": "Indonesian", "tr": "Turkish", "nl": "Dutch", "vi": "Vietnamese"
66
+ }
67
+
68
+ # --- 2. ADVANCED NLP, INDEXING & LANGUAGE ---
69
  def tokenize(text):
70
  if not text: return []
71
  tokens = re.findall(r'[a-z0-9]+', str(text).lower())
72
  return [t for t in tokens if t not in STOP_WORDS]
73
 
74
+ def detect_language_name(text):
75
+ try:
76
+ if not text or len(text.strip()) < 10: return "Unknown"
77
+ code = detect(text)
78
+ return LANG_MAP.get(code, code.upper()) # Agar list me nahi mila toh code return karega
79
+ except:
80
+ return "Unknown"
81
 
82
  def load_and_index_dataset():
83
+ global bm25_index, raw_data, suggestion_list
84
  print("📥 Loading Dataset & Building Deep Index...")
85
  dataset_path = "dataset.jsonl"
86
 
 
103
  if line.strip():
104
  try:
105
  doc = json.loads(line)
106
+ url = doc.get("url", "")
107
+
108
+ if url not in URL_STATS:
109
+ URL_STATS[url] = {"user_visits": 0, "ai_visits": 0, "is_trusted": False}
110
+
111
+ # Extract language and append to document
112
+ combined_text = str(doc.get("title", "")) + " " + str(doc.get("description", ""))
113
+ doc["language"] = detect_language_name(combined_text)
114
+
115
  raw_data.append(doc)
116
 
117
+ title_str = doc.get("title", "")
118
+ if title_str and len(title_str) < 60:
119
+ suggestion_list.append(title_str.lower())
120
+
121
+ title_tokens = tokenize(title_str)
122
  desc_tokens = tokenize(doc.get("description", ""))
123
+ url_tokens = tokenize(url)
 
 
124
 
125
+ boosted_tokens = (title_tokens * 5) + (desc_tokens * 3) + url_tokens
126
  tokenized_corpus.append(boosted_tokens)
127
  except:
128
  pass
129
 
130
  if tokenized_corpus:
131
  print("⚡ Building High-Speed BM25 Okapi Index...")
132
+ suggestion_list = sorted(list(set(suggestion_list)))
133
  bm25_index = BM25Okapi(tokenized_corpus)
134
  print(f"🚀 Indexing Complete! {len(raw_data)} records ready.")
135
  else:
136
  print("❌ Dataset file not found!")
137
 
138
  def load_ai_models():
139
+ global ai_generator
140
+ print("🤖 Loading Modern Fast Text AI Model...")
141
  try:
142
+ # Chota, modern aur tezz model
143
+ ai_generator = pipeline("text-generation", model="Qwen/Qwen2.5-0.5B-Instruct", device=-1, torch_dtype=torch.float32)
144
+ print("✅ AI Model Loaded! System ready.")
 
 
 
 
 
 
145
  except Exception as e:
146
  print(f"❌ Model Loading Failed: {e}")
147
 
148
+ # --- 3. SMART SCRIPT RAG: EARLY STOPPING CHUNKING ---
149
+ def chunk_text(text, chunk_size=120):
150
+ words = text.split()
151
+ return [" ".join(words[i:i+chunk_size]) for i in range(0, len(words), chunk_size)]
152
+
153
+ async def scrape_and_find_best_chunk(url, query):
154
+ """Script website visit karegi, sirf Headers & Paragraphs legi, aur best chunk lekar ruk jayegi"""
155
+ try:
156
+ async with aiohttp.ClientSession() as session:
157
+ # Sirf 3 second ka timeout taki endpoint na atke
158
+ async with session.get(url, timeout=3) as resp:
159
+ if resp.status == 200:
160
+ html = await resp.text()
161
+ soup = BeautifulSoup(html, "html.parser")
162
+
163
+ # Sirf Paragraphs aur Headers nikal rahe hain kachra avoid karne ke liye
164
+ tags_to_extract = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']
165
+ text = " ".join([tag.get_text(strip=True) for tag in soup.find_all(tags_to_extract)])
166
+
167
+ if not text: return ""
168
+
169
+ chunks = chunk_text(text)
170
+ query_tokens = set(tokenize(query))
171
+
172
+ best_chunk = ""
173
+ max_score = 0
174
+ # Threshold: Agar query ke aade tokens mil gaye chunk mein toh wahi ruk jao
175
+ early_stop_threshold = max(1, len(query_tokens) // 2)
176
+
177
+ for chunk in chunks:
178
+ chunk_tokens = set(tokenize(chunk))
179
+ score = len(query_tokens.intersection(chunk_tokens))
180
+
181
+ if score > max_score:
182
+ max_score = score
183
+ best_chunk = chunk
184
+
185
+ # 🔥 EARLY STOPPING: Jo kaam ka chunk mila wahi liya, aage process nahi kiya
186
+ if max_score >= early_stop_threshold and max_score > 0:
187
+ break
188
+
189
+ return best_chunk
190
+ except Exception:
191
+ return "" # Scraping failed (timeout/block), ignore quietly
192
+ return ""
193
+
194
+ # --- 4. CORE SEARCH ALGORITHM & RANKING ---
195
  async def perform_core_search(raw_query: str):
196
  if not raw_data or not bm25_index:
197
  return [], "Search index not ready.", 0
198
 
199
+ query = raw_query.strip().lower()
 
 
 
 
 
 
 
 
 
 
 
 
 
200
  query_tokens = tokenize(query)
201
 
202
+ # 🚨 SMART ADULT FILTER (Using 'in' operator to catch hidden adult queries)
203
+ user_wants_adult = any(adult_word in query for adult_word in ADULT_KEYWORDS)
204
+
 
205
  scores = bm25_index.get_scores(query_tokens) if query_tokens else [1.0] * len(raw_data)
 
206
  scored_results = []
207
  seen_urls = set()
208
 
209
  for index, score in enumerate(scores):
210
+ if score > 0.1 or not query_tokens:
211
  doc = raw_data[index]
212
  doc_url = doc.get("url", "")
213
 
214
  if doc_url in seen_urls: continue
215
  parsed_url = urllib.parse.urlparse(doc_url)
216
+ domain = parsed_url.netloc.lower()
217
 
218
+ # Text combine to check adult words in document
219
+ doc_text = (str(doc.get("title", "")) + " " + str(doc.get("description", "")) + " " + doc_url).lower()
220
+ is_adult_doc = doc.get("tag", "general").lower() in ["adult", "nsfw"] or any(aw in doc_text for aw in ADULT_KEYWORDS)
 
 
 
221
 
222
+ # Agar user ki query normal hai, toh adult result filter ho jayega
223
+ if not user_wants_adult and is_adult_doc:
224
+ continue
225
+
226
  final_score = score
 
 
227
 
228
+ # 🔥 DOMAIN RANKING BOOST
229
+ if query.replace(" ", "") in domain.replace(" ", ""):
230
+ final_score *= 10.0 # Huge boost for direct domain match
231
+ elif query in doc.get("title", "").lower():
232
+ final_score *= 3.0
233
 
234
+ # 🔥 TRUST RANKING BOOST
235
+ stats = URL_STATS.get(doc_url, {})
236
+ if stats.get("is_trusted", False):
237
+ final_score *= 1.5
238
+
239
+ final_score += (stats.get("user_visits", 0) * 0.05)
240
 
241
  seen_urls.add(doc_url)
242
  scored_results.append((final_score, doc))
 
248
  final_results = [{"match_score": round(s, 2), **doc} for s, doc in scored_results[:100]]
249
  return final_results, prediction, best_score
250
 
251
+ # --- AI GENERATION HELPER ---
252
  def _generate_text(messages, max_tokens: int):
253
+ prompt = "\n".join([f"{m['role'].capitalize()}: {m['content']}" for m in messages]) + "\nAssistant:"
254
+ result = ai_generator(prompt, max_new_tokens=max_tokens, do_sample=True, temperature=0.3, top_p=0.85)
255
+ return result[0]['generated_text'].split("Assistant:")[-1].strip()
 
 
 
 
 
256
 
257
  async def run_ai_task(messages, max_tokens: int, query_key: str):
258
  if not ai_generator: return "⚠️ AI Model is starting up."
 
264
  AI_RESPONSE_CACHE[query_key] = response
265
  return response
266
 
267
+ # --- 5. FASTAPI APP & ENDPOINTS ---
268
  @asynccontextmanager
269
  async def lifespan(app: FastAPI):
270
  asyncio.create_task(asyncio.to_thread(load_and_index_dataset))
271
  asyncio.create_task(asyncio.to_thread(load_ai_models))
272
  yield
273
 
274
+ app = FastAPI(title="Nebulix Advanced Text AI & RAG Engine", lifespan=lifespan)
275
 
 
276
  app.add_middleware(
277
  CORSMiddleware,
278
  allow_origins=["*"],
 
281
  allow_headers=["*"]
282
  )
283
 
284
+ # 🔥 1. AUTO-SUGGESTIONS ENDPOINT
285
+ @app.get("/suggest")
286
+ async def get_suggestions(q: str = Query(..., min_length=1)):
287
+ q = q.lower()
288
+ matches = [s for s in suggestion_list if s.startswith(q)][:7]
289
+ return {"query": q, "suggestions": matches}
290
+
291
+ # 🔥 2. URL TRACKING & TRUST ENDPOINTS
292
+ @app.post("/track-click")
293
+ async def track_url_click(url: str = Query(...)):
294
+ if url not in URL_STATS:
295
+ URL_STATS[url] = {"user_visits": 0, "ai_visits": 0, "is_trusted": False}
296
+ URL_STATS[url]["user_visits"] += 1
297
+ return {"status": "success", "url": url, "visits": URL_STATS[url]["user_visits"]}
298
+
299
+ @app.get("/url-stats")
300
+ async def get_url_stats(url: str = Query(...)):
301
+ stats = URL_STATS.get(url, {"user_visits": 0, "ai_visits": 0, "is_trusted": False})
302
+ return {"url": url, "stats": stats}
 
 
 
 
 
 
303
 
304
  @app.get("/search")
305
  async def pure_search(q: str = Query(...)):
 
307
  results, prediction, _ = await perform_core_search(q)
308
  return {"request_id": req_id, "query": q, "results_count": len(results), "results": results}
309
 
310
+ # 🔥 3. SMART RAG CHAT API (Only Text & Scraped Chunks)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
311
  @app.post("/aimodechat")
312
+ async def ai_mode_chat(q: str = Form(...)):
313
  req_id = str(uuid.uuid4())
 
314
 
315
+ # AI Logic to decide RAG need
316
+ needs_rag = any(w in q.lower() for w in ["what", "who", "where", "how", "why", "tell me", "explain", "latest"])
317
+
318
+ results, _, _ = await perform_core_search(q)
319
+ rag_context = ""
320
+ used_sources = []
 
 
 
 
321
 
322
+ if needs_rag and results:
323
+ top_urls = results[:10]
324
+
325
+ # Script asynchronously scrapes multiple URLs at once
326
+ tasks = [scrape_and_find_best_chunk(doc["url"], q) for doc in top_urls]
327
+ scraped_chunks = await asyncio.gather(*tasks)
328
+
329
+ context_parts = []
330
+ for i, chunk in enumerate(scraped_chunks):
331
+ doc = top_urls[i]
332
+ url = doc["url"]
333
+
334
+ # Trust Tracking: Mark AI visits
335
+ if url not in URL_STATS: URL_STATS[url] = {"user_visits": 0, "ai_visits": 0, "is_trusted": False}
336
+ URL_STATS[url]["ai_visits"] += 1
337
+ if URL_STATS[url]["ai_visits"] >= 100:
338
+ URL_STATS[url]["is_trusted"] = True
339
+
340
+ # Sirf kaam ka chunk (Early stopped chunk) lenge ya fall back pe description
341
+ if chunk:
342
+ context_parts.append(f"Source [{i+1}] ({url}): {chunk}")
343
+ used_sources.append(doc)
344
+ else:
345
+ context_parts.append(f"Source [{i+1}] ({url}): {doc.get('description', '')}")
346
+ used_sources.append(doc)
347
+
348
+ rag_context = "\n".join(context_parts)
349
 
350
+ system_prompt = "You are a Smart AI Search Assistant. Use the provided search chunks to answer. Always mention the Source URL if you use information from it."
351
+ if not needs_rag:
352
+ system_prompt = "You are a helpful AI Assistant. Answer the user's query conversationally."
353
 
354
  messages = [
355
+ {"role": "system", "content": system_prompt},
356
+ {"role": "user", "content": f"Query: {q}\n\nLive Context Data:\n{rag_context}" if needs_rag else f"Query: {q}"}
357
  ]
358
 
359
+ ai_reply = await run_ai_task(messages, 400, query_key=f"chat_{q}_{needs_rag}")
 
 
 
 
360
 
361
+ return {
362
+ "request_id": req_id,
363
+ "ai_response": ai_reply,
364
+ "rag_used": needs_rag,
365
+ "sources": used_sources,
366
+ "results": results[:15]
367
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
368
 
 
369
  if __name__ == "__main__":
370
+ uvicorn.run("main:app", host="0.0.0.0", port=7860, reload=True)