Nebulixlabs commited on
Commit
1738073
·
verified ·
1 Parent(s): 6db84f4

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +53 -50
app.py CHANGED
@@ -2,6 +2,8 @@ import json
2
  import uuid
3
  import asyncio
4
  import os
 
 
5
  from datetime import datetime, timedelta
6
  from urllib.parse import urlparse
7
  from fastapi import FastAPI, Query, BackgroundTasks
@@ -9,22 +11,45 @@ from fastapi.responses import JSONResponse
9
  from rapidfuzz import process, fuzz
10
  from cryptography.fernet import Fernet
11
 
12
- # --- 1. SETUP, ENCRYPTION & DIRECTORY ---
13
- # Note: Production me Fernet key ko environment variable se lena chahiye.
14
  SECRET_KEY = Fernet.generate_key()
15
  cipher_suite = Fernet(SECRET_KEY)
16
 
17
- # Logs ke liye alag folder banayenge taki clean rahe
18
  LOG_DIR = "logs"
19
  os.makedirs(LOG_DIR, exist_ok=True)
20
 
21
- app = FastAPI(title="Fast Search API")
22
-
23
- # --- 2. LOAD DATA FROM dataset.jsonl ---
24
  raw_data = []
25
  search_corpus = []
26
 
27
- def load_dataset():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  global raw_data, search_corpus
29
  try:
30
  with open("dataset.jsonl", "r", encoding="utf-8") as f:
@@ -33,7 +58,6 @@ def load_dataset():
33
  item = json.loads(line)
34
  raw_data.append(item)
35
 
36
- # Domain, title aur description nikal kar corpus me dalna
37
  domain = urlparse(item.get("url", "")).netloc
38
  text_to_search = f"{domain} {item.get('title', '')} {item.get('description', '')}"
39
  search_corpus.append(text_to_search.lower())
@@ -41,16 +65,23 @@ def load_dataset():
41
  except FileNotFoundError:
42
  print("❌ ERROR: dataset.jsonl file not found in the directory!")
43
 
44
- # Load dataset when app starts
45
- load_dataset()
 
 
 
 
 
 
 
 
46
 
47
 
48
- # --- 3. DATE-BASED LOGGING & ENCRYPTION ---
49
  async def log_query_encrypted(req_id: str, query: str):
50
  encrypted_q = cipher_suite.encrypt(query.encode()).decode()
51
  now = datetime.now()
52
 
53
- # Date ke hisaab se file ka naam banega (e.g., search_logs_2026-07-22.jsonl)
54
  date_str = now.strftime("%Y-%m-%d")
55
  log_file = os.path.join(LOG_DIR, f"search_logs_{date_str}.jsonl")
56
 
@@ -60,43 +91,10 @@ async def log_query_encrypted(req_id: str, query: str):
60
  "timestamp": now.isoformat()
61
  }
62
 
63
- # File me append karna (Agar file nahi hai toh khud ban jayegi)
64
  with open(log_file, "a", encoding="utf-8") as f:
65
  f.write(json.dumps(log_entry) + "\n")
66
 
67
 
68
- # --- 4. OPTIMIZED 1 MONTH LOG CLEANUP ---
69
- async def cleanup_old_logs():
70
- while True:
71
- try:
72
- if os.path.exists(LOG_DIR):
73
- thirty_days_ago = datetime.now() - timedelta(days=30)
74
-
75
- # Folder ki saari files check karenge
76
- for filename in os.listdir(LOG_DIR):
77
- if filename.startswith("search_logs_") and filename.endswith(".jsonl"):
78
- # File ke naam se date nikalenge
79
- date_part = filename.replace("search_logs_", "").replace(".jsonl", "")
80
- try:
81
- file_date = datetime.strptime(date_part, "%Y-%m-%d")
82
- # Agar file 30 din se purani hai, toh puri file delete kar do
83
- if file_date < thirty_days_ago:
84
- os.remove(os.path.join(LOG_DIR, filename))
85
- print(f"🗑️ Deleted 1-month old log file: {filename}")
86
- except ValueError:
87
- pass
88
- except Exception as e:
89
- print("Cleanup Error:", e)
90
-
91
- # Har 24 ghante me ek baar check karega
92
- await asyncio.sleep(24 * 60 * 60)
93
-
94
- @app.on_event("startup")
95
- async def startup_event():
96
- # Background me cleanup task on startup
97
- asyncio.create_task(cleanup_old_logs())
98
-
99
-
100
  # --- 5. FAST AI / NLP LOGIC ---
101
  def check_self_harm(query: str) -> bool:
102
  harm_keywords = ["suicide", "kill myself", "die", "harm myself", "end my life", "suiside"]
@@ -128,7 +126,7 @@ def ai_fallback_search(query: str):
128
  async def search(q: str = Query(..., description="Search query"), background_tasks: BackgroundTasks = BackgroundTasks()):
129
  req_id = str(uuid.uuid4())
130
 
131
- # Background Logging (FastAPI wait nahi karega iske liye)
132
  background_tasks.add_task(log_query_encrypted, req_id, q)
133
 
134
  # Self-Harm Check
@@ -146,7 +144,7 @@ async def search(q: str = Query(..., description="Search query"), background_tas
146
 
147
  final_results = []
148
 
149
- # RapidFuzz search algorithm - searches in domain, title, and description
150
  if search_corpus:
151
  results = process.extract(
152
  q.lower(),
@@ -157,7 +155,7 @@ async def search(q: str = Query(..., description="Search query"), background_tas
157
 
158
  for match_str, score, index in results:
159
  if score > 40:
160
- result_item = raw_data[index].copy() # Full data return hoga including images
161
  result_item["match_score"] = round(score, 2)
162
  final_results.append(result_item)
163
 
@@ -172,10 +170,15 @@ async def search(q: str = Query(..., description="Search query"), background_tas
172
 
173
  response_data["request_id"] = req_id
174
 
175
- # Adult Content Header & JSON Modifier
176
  if is_adult:
177
  response_data["safety_message"] = "you are safe you can do anything"
178
  headers = {"X-Safety-Message": "you are safe you can do anything"}
179
  return JSONResponse(content=response_data, headers=headers)
180
 
181
- return response_data
 
 
 
 
 
 
2
  import uuid
3
  import asyncio
4
  import os
5
+ import uvicorn
6
+ from contextlib import asynccontextmanager
7
  from datetime import datetime, timedelta
8
  from urllib.parse import urlparse
9
  from fastapi import FastAPI, Query, BackgroundTasks
 
11
  from rapidfuzz import process, fuzz
12
  from cryptography.fernet import Fernet
13
 
14
+ # --- 1. SETUP & ENCRYPTION ---
 
15
  SECRET_KEY = Fernet.generate_key()
16
  cipher_suite = Fernet(SECRET_KEY)
17
 
18
+ # Logs directory setup
19
  LOG_DIR = "logs"
20
  os.makedirs(LOG_DIR, exist_ok=True)
21
 
22
+ # Global variables for dataset
 
 
23
  raw_data = []
24
  search_corpus = []
25
 
26
+ # --- 2. OPTIMIZED CLEANUP TASK ---
27
+ async def cleanup_old_logs():
28
+ while True:
29
+ try:
30
+ if os.path.exists(LOG_DIR):
31
+ thirty_days_ago = datetime.now() - timedelta(days=30)
32
+
33
+ for filename in os.listdir(LOG_DIR):
34
+ if filename.startswith("search_logs_") and filename.endswith(".jsonl"):
35
+ date_part = filename.replace("search_logs_", "").replace(".jsonl", "")
36
+ try:
37
+ file_date = datetime.strptime(date_part, "%Y-%m-%d")
38
+ if file_date < thirty_days_ago:
39
+ os.remove(os.path.join(LOG_DIR, filename))
40
+ print(f"🗑️ Deleted 1-month old log file: {filename}")
41
+ except ValueError:
42
+ pass
43
+ except Exception as e:
44
+ print("Cleanup Error:", e)
45
+
46
+ # Har 24 ghante me ek baar check
47
+ await asyncio.sleep(24 * 60 * 60)
48
+
49
+ # --- 3. LIFESPAN (Replaces @app.on_event("startup")) ---
50
+ @asynccontextmanager
51
+ async def lifespan(app: FastAPI):
52
+ # a. Dataset Loading during startup
53
  global raw_data, search_corpus
54
  try:
55
  with open("dataset.jsonl", "r", encoding="utf-8") as f:
 
58
  item = json.loads(line)
59
  raw_data.append(item)
60
 
 
61
  domain = urlparse(item.get("url", "")).netloc
62
  text_to_search = f"{domain} {item.get('title', '')} {item.get('description', '')}"
63
  search_corpus.append(text_to_search.lower())
 
65
  except FileNotFoundError:
66
  print("❌ ERROR: dataset.jsonl file not found in the directory!")
67
 
68
+ # b. Start background cleanup task
69
+ cleanup_task = asyncio.create_task(cleanup_old_logs())
70
+
71
+ yield # App jab tak chalegi, yaha yield rahega
72
+
73
+ # c. App shutdown hone par task cancel karna
74
+ cleanup_task.cancel()
75
+
76
+ # Pass lifespan to FastAPI
77
+ app = FastAPI(title="Fast Search API", lifespan=lifespan)
78
 
79
 
80
+ # --- 4. DATE-BASED LOGGING & ENCRYPTION ---
81
  async def log_query_encrypted(req_id: str, query: str):
82
  encrypted_q = cipher_suite.encrypt(query.encode()).decode()
83
  now = datetime.now()
84
 
 
85
  date_str = now.strftime("%Y-%m-%d")
86
  log_file = os.path.join(LOG_DIR, f"search_logs_{date_str}.jsonl")
87
 
 
91
  "timestamp": now.isoformat()
92
  }
93
 
 
94
  with open(log_file, "a", encoding="utf-8") as f:
95
  f.write(json.dumps(log_entry) + "\n")
96
 
97
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  # --- 5. FAST AI / NLP LOGIC ---
99
  def check_self_harm(query: str) -> bool:
100
  harm_keywords = ["suicide", "kill myself", "die", "harm myself", "end my life", "suiside"]
 
126
  async def search(q: str = Query(..., description="Search query"), background_tasks: BackgroundTasks = BackgroundTasks()):
127
  req_id = str(uuid.uuid4())
128
 
129
+ # Background Logging
130
  background_tasks.add_task(log_query_encrypted, req_id, q)
131
 
132
  # Self-Harm Check
 
144
 
145
  final_results = []
146
 
147
+ # RapidFuzz search algorithm
148
  if search_corpus:
149
  results = process.extract(
150
  q.lower(),
 
155
 
156
  for match_str, score, index in results:
157
  if score > 40:
158
+ result_item = raw_data[index].copy()
159
  result_item["match_score"] = round(score, 2)
160
  final_results.append(result_item)
161
 
 
170
 
171
  response_data["request_id"] = req_id
172
 
173
+ # Adult Content Safety Modification
174
  if is_adult:
175
  response_data["safety_message"] = "you are safe you can do anything"
176
  headers = {"X-Safety-Message": "you are safe you can do anything"}
177
  return JSONResponse(content=response_data, headers=headers)
178
 
179
+ return response_data
180
+
181
+ # --- 7. START SERVER FOR GRADIO SPACE ---
182
+ # Ye block Gradio Space ko bata dega ki port 7860 par API continuously chalani hai.
183
+ if __name__ == "__main__":
184
+ uvicorn.run(app, host="0.0.0.0", port=7860)