import sys import asyncio import json import logging import threading from pathlib import Path from typing import List, Dict, Any from fastapi import FastAPI, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles # Ensure project root is in python path root_path = Path(__file__).resolve().parent.parent.parent if str(root_path) not in sys.path: sys.path.insert(0, str(root_path)) # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger("sherlock_web_backend") # Import site loader from sherlock_project.sites import SitesInformation # Load the workers from the desktop application modules from qt_app.workers import ( run_sherlock_worker, run_maigret_worker, run_nexfil_worker, run_uscan_username_worker, run_uscan_email_worker, run_socialscan_worker, run_h8mail_worker, run_emailharvester_worker, run_phoneinfoga_worker, run_recon_ng_worker, run_blackbird_worker, run_holehe_worker, ) app = FastAPI(title="Sherlock Web OSINT Suite") # CORS middleware to allow the Bun/React dev server to communicate with this backend app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # --------------------------------------------------------------------------- # Data Loading # --------------------------------------------------------------------------- sherlock_data = {} maigret_data = {} nexfil_url_count = 0 def load_data_sources(): global sherlock_data, maigret_data, nexfil_url_count logger.info("Loading resource databases...") # 1. Sherlock base = root_path / "sherlock_project" / "resources" / "data.json" if base.exists(): sites = SitesInformation(str(base)) sherlock_data = {s.name: s.information for s in sites} logger.info(f"Loaded {len(sherlock_data)} Sherlock sites.") else: logger.warning("Sherlock data.json not found!") # 2. Maigret try: from maigret import MaigretDatabase db_path = Path(__import__("maigret", fromlist=["__file__"]).__file__).parent / "resources" / "data.json" db = MaigretDatabase().load_from_path(str(db_path)) maigret_data = db.ranked_sites_dict() logger.info(f"Loaded {len(maigret_data)} Maigret sites.") except Exception as e: logger.error(f"Failed to load Maigret database: {e}") maigret_data = {} # 3. NExfil try: import nexfil pkg = Path(nexfil.__file__).parent with open(pkg / "url_store.json") as f: nexfil_url_count = len(json.loads(f.read())) logger.info(f"Loaded {nexfil_url_count} NExfil target URLs.") except Exception as e: logger.error(f"Failed to load NExfil url store: {e}") nexfil_url_count = 0 # Load on startup load_data_sources() # --------------------------------------------------------------------------- # Signal Mock for Worker Bridge # --------------------------------------------------------------------------- class SignalBridge: def __init__(self, name: str, websocket: WebSocket, loop: asyncio.AbstractEventLoop): self.name = name self.websocket = websocket self.loop = loop def emit(self, *args): # We need to construct a message payload and push it to the WebSocket queue. # This is safe to invoke from sub-threads using asyncio.run_coroutine_threadsafe. payload = { "event": self.name, "data": list(args) } async def send(): try: await self.websocket.send_json(payload) except Exception: pass asyncio.run_coroutine_threadsafe(send(), self.loop) class WorkerSignalsBridge: """Replicates the PySide6 QObject signals interface needed by workers.py""" def __init__(self, websocket: WebSocket, loop: asyncio.AbstractEventLoop): self.started = SignalBridge("started", websocket, loop) self.result = SignalBridge("result", websocket, loop) self.progress = SignalBridge("progress", websocket, loop) self.finished = SignalBridge("finished", websocket, loop) self.error = SignalBridge("error", websocket, loop) # --------------------------------------------------------------------------- # API Routes # --------------------------------------------------------------------------- @app.get("/api/info") def get_info(): """Returns basic configuration and loaded targets info for display.""" return { "sherlock_sites_count": len(sherlock_data), "maigret_sites_count": len(maigret_data), "nexfil_url_count": nexfil_url_count, "available_engines": [ {"id": "sherlock", "name": "Sherlock", "type": "username", "color": "#10b981", "desc": f"Scans {len(sherlock_data)} social networks"}, {"id": "maigret", "name": "Maigret", "type": "username", "color": "#8b5cf6", "desc": f"Scans {len(maigret_data)} sites with high-depth analysis"}, {"id": "nexfil", "name": "NExfil", "type": "username", "color": "#f59e0b", "desc": f"Ultra-fast parallel query on {nexfil_url_count} urls"}, {"id": "user-scanner", "name": "UserScanner", "type": "username/email", "color": "#06b6d4", "desc": "Subdomain and page response checker"}, {"id": "socialscan", "name": "SocialScan", "type": "username/email", "color": "#ec4899", "desc": "Accurate HTTP query on popular platforms"}, {"id": "blackbird", "name": "Blackbird", "type": "username", "color": "#ec4899", "desc": "Fast correlation tool for user accounts"}, {"id": "h8mail", "name": "h8mail", "type": "email", "color": "#ef4444", "desc": "Breach lookup and email credentials scanner"}, {"id": "email_harvester", "name": "EmailHarvester", "type": "email", "color": "#06b6d4", "desc": "Search engine parser for sub-domain emails"}, {"id": "holehe", "name": "Holehe", "type": "email", "color": "#f59e0b", "desc": "Checked registered mail on 120+ sites"}, {"id": "phoneinfoga", "name": "PhoneInfoga", "type": "phone", "color": "#10b981", "desc": "Information gathering on phone numbers"}, {"id": "recon_ng", "name": "Recon-ng", "type": "domain", "color": "#8b5cf6", "desc": "Keyless domain sub-reconnaissance"}, ] } @app.websocket("/ws/search") async def websocket_search(websocket: WebSocket): await websocket.accept() logger.info("WebSocket connection established.") loop = asyncio.get_running_loop() signals = WorkerSignalsBridge(websocket, loop) threads: List[threading.Thread] = [] try: while True: data = await websocket.receive_text() params = json.loads(data) action = params.get("action") if action == "start": username = params.get("username", "").strip() email = params.get("email", "").strip() phone = params.get("phone", "").strip() domain = params.get("domain", "").strip() enabled_engines = params.get("engines", []) logger.info(f"Starting OSINT Scan - U:{username} E:{email} P:{phone} D:{domain} Engines:{enabled_engines}") def launch_worker(target_fn, *args): t = threading.Thread(target=target_fn, args=(signals, *args), daemon=True) t.start() threads.append(t) # 1. Usernames if username: if "sherlock" in enabled_engines: launch_worker(run_sherlock_worker, username, sherlock_data) if "maigret" in enabled_engines: launch_worker(run_maigret_worker, username, maigret_data) if "nexfil" in enabled_engines: launch_worker(run_nexfil_worker, username) if "user-scanner" in enabled_engines: launch_worker(run_uscan_username_worker, username) if "socialscan" in enabled_engines: launch_worker(run_socialscan_worker, username, False) if "blackbird" in enabled_engines: launch_worker(run_blackbird_worker, username) # 2. Emails if email: if "user-scanner" in enabled_engines: launch_worker(run_uscan_email_worker, email) if "socialscan" in enabled_engines: launch_worker(run_socialscan_worker, email, True) if "h8mail" in enabled_engines: launch_worker(run_h8mail_worker, email) if "email_harvester" in enabled_engines: launch_worker(run_emailharvester_worker, email) if "holehe" in enabled_engines: launch_worker(run_holehe_worker, email) # 3. Phones if phone: if "phoneinfoga" in enabled_engines: launch_worker(run_phoneinfoga_worker, phone) # 4. Domains if domain: if "recon_ng" in enabled_engines: launch_worker(run_recon_ng_worker, domain) elif action == "stop": logger.info("Search cancellation requested by client.") # Note: Python threads running OSINT libraries can't easily be killed abruptly. # However, sending a cancel notification and resetting states will cause the client to disconnect/reset. # In standard usage, closing the websocket safely halts UI state. await websocket.send_json({"event": "cancelled", "data": []}) except WebSocketDisconnect: logger.info("WebSocket disconnected.") except Exception as e: logger.error(f"Error in websocket handler: {e}") finally: # Threads are daemon threads and will safely detach and die when process exits or runs out of tasks. pass # --------------------------------------------------------------------------- # Static Files Serving (production compilation fallback) # --------------------------------------------------------------------------- frontend_dist = root_path / "web_app" / "frontend" / "dist" if frontend_dist.exists(): app.mount("/", StaticFiles(directory=str(frontend_dist), html=True), name="static") else: @app.get("/") def read_root(): return { "status": "online", "message": "Sherlock Web OSINT Suite backend running. To view the UI, build the frontend or run the Bun development server." }