Spaces:
Build error
Build error
Download web_app/backend/server.py from Gianzfx/SherlockOSINT: direct link, hf CLI and curl.
- Browser
- Download file 10.9 kB
-
https://huggingface.co/spaces/Gianzfx/SherlockOSINT/resolve/main/web_app/backend/server.py
- Command line
-
hf download hf://spaces/Gianzfx/SherlockOSINT/web_app/backend/server.py
-
curl -L -o server.py https://huggingface.co/spaces/Gianzfx/SherlockOSINT/resolve/main/web_app/backend/server.py
10.9 kB
| import sys | |
| import asyncio | |
| import json | |
| import logging | |
| import threading | |
| from pathlib import Path | |
| from typing import List, Dict, Any | |
| from fastapi import FastAPI, WebSocket, WebSocketDisconnect | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from fastapi.staticfiles import StaticFiles | |
| # Ensure project root is in python path | |
| root_path = Path(__file__).resolve().parent.parent.parent | |
| if str(root_path) not in sys.path: | |
| sys.path.insert(0, str(root_path)) | |
| # Configure logging | |
| logging.basicConfig(level=logging.INFO) | |
| logger = logging.getLogger("sherlock_web_backend") | |
| # Import site loader | |
| from sherlock_project.sites import SitesInformation | |
| # Load the workers from the desktop application modules | |
| from qt_app.workers import ( | |
| run_sherlock_worker, | |
| run_maigret_worker, | |
| run_nexfil_worker, | |
| run_uscan_username_worker, | |
| run_uscan_email_worker, | |
| run_socialscan_worker, | |
| run_h8mail_worker, | |
| run_emailharvester_worker, | |
| run_phoneinfoga_worker, | |
| run_recon_ng_worker, | |
| run_blackbird_worker, | |
| run_holehe_worker, | |
| ) | |
| app = FastAPI(title="Sherlock Web OSINT Suite") | |
| # CORS middleware to allow the Bun/React dev server to communicate with this backend | |
| app.add_middleware( | |
| CORSMiddleware, | |
| allow_origins=["*"], | |
| allow_credentials=True, | |
| allow_methods=["*"], | |
| allow_headers=["*"], | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Data Loading | |
| # --------------------------------------------------------------------------- | |
| sherlock_data = {} | |
| maigret_data = {} | |
| nexfil_url_count = 0 | |
| def load_data_sources(): | |
| global sherlock_data, maigret_data, nexfil_url_count | |
| logger.info("Loading resource databases...") | |
| # 1. Sherlock | |
| base = root_path / "sherlock_project" / "resources" / "data.json" | |
| if base.exists(): | |
| sites = SitesInformation(str(base)) | |
| sherlock_data = {s.name: s.information for s in sites} | |
| logger.info(f"Loaded {len(sherlock_data)} Sherlock sites.") | |
| else: | |
| logger.warning("Sherlock data.json not found!") | |
| # 2. Maigret | |
| try: | |
| from maigret import MaigretDatabase | |
| db_path = Path(__import__("maigret", fromlist=["__file__"]).__file__).parent / "resources" / "data.json" | |
| db = MaigretDatabase().load_from_path(str(db_path)) | |
| maigret_data = db.ranked_sites_dict() | |
| logger.info(f"Loaded {len(maigret_data)} Maigret sites.") | |
| except Exception as e: | |
| logger.error(f"Failed to load Maigret database: {e}") | |
| maigret_data = {} | |
| # 3. NExfil | |
| try: | |
| import nexfil | |
| pkg = Path(nexfil.__file__).parent | |
| with open(pkg / "url_store.json") as f: | |
| nexfil_url_count = len(json.loads(f.read())) | |
| logger.info(f"Loaded {nexfil_url_count} NExfil target URLs.") | |
| except Exception as e: | |
| logger.error(f"Failed to load NExfil url store: {e}") | |
| nexfil_url_count = 0 | |
| # Load on startup | |
| load_data_sources() | |
| # --------------------------------------------------------------------------- | |
| # Signal Mock for Worker Bridge | |
| # --------------------------------------------------------------------------- | |
| class SignalBridge: | |
| def __init__(self, name: str, websocket: WebSocket, loop: asyncio.AbstractEventLoop): | |
| self.name = name | |
| self.websocket = websocket | |
| self.loop = loop | |
| def emit(self, *args): | |
| # We need to construct a message payload and push it to the WebSocket queue. | |
| # This is safe to invoke from sub-threads using asyncio.run_coroutine_threadsafe. | |
| payload = { | |
| "event": self.name, | |
| "data": list(args) | |
| } | |
| async def send(): | |
| try: | |
| await self.websocket.send_json(payload) | |
| except Exception: | |
| pass | |
| asyncio.run_coroutine_threadsafe(send(), self.loop) | |
| class WorkerSignalsBridge: | |
| """Replicates the PySide6 QObject signals interface needed by workers.py""" | |
| def __init__(self, websocket: WebSocket, loop: asyncio.AbstractEventLoop): | |
| self.started = SignalBridge("started", websocket, loop) | |
| self.result = SignalBridge("result", websocket, loop) | |
| self.progress = SignalBridge("progress", websocket, loop) | |
| self.finished = SignalBridge("finished", websocket, loop) | |
| self.error = SignalBridge("error", websocket, loop) | |
| # --------------------------------------------------------------------------- | |
| # API Routes | |
| # --------------------------------------------------------------------------- | |
| def get_info(): | |
| """Returns basic configuration and loaded targets info for display.""" | |
| return { | |
| "sherlock_sites_count": len(sherlock_data), | |
| "maigret_sites_count": len(maigret_data), | |
| "nexfil_url_count": nexfil_url_count, | |
| "available_engines": [ | |
| {"id": "sherlock", "name": "Sherlock", "type": "username", "color": "#10b981", "desc": f"Scans {len(sherlock_data)} social networks"}, | |
| {"id": "maigret", "name": "Maigret", "type": "username", "color": "#8b5cf6", "desc": f"Scans {len(maigret_data)} sites with high-depth analysis"}, | |
| {"id": "nexfil", "name": "NExfil", "type": "username", "color": "#f59e0b", "desc": f"Ultra-fast parallel query on {nexfil_url_count} urls"}, | |
| {"id": "user-scanner", "name": "UserScanner", "type": "username/email", "color": "#06b6d4", "desc": "Subdomain and page response checker"}, | |
| {"id": "socialscan", "name": "SocialScan", "type": "username/email", "color": "#ec4899", "desc": "Accurate HTTP query on popular platforms"}, | |
| {"id": "blackbird", "name": "Blackbird", "type": "username", "color": "#ec4899", "desc": "Fast correlation tool for user accounts"}, | |
| {"id": "h8mail", "name": "h8mail", "type": "email", "color": "#ef4444", "desc": "Breach lookup and email credentials scanner"}, | |
| {"id": "email_harvester", "name": "EmailHarvester", "type": "email", "color": "#06b6d4", "desc": "Search engine parser for sub-domain emails"}, | |
| {"id": "holehe", "name": "Holehe", "type": "email", "color": "#f59e0b", "desc": "Checked registered mail on 120+ sites"}, | |
| {"id": "phoneinfoga", "name": "PhoneInfoga", "type": "phone", "color": "#10b981", "desc": "Information gathering on phone numbers"}, | |
| {"id": "recon_ng", "name": "Recon-ng", "type": "domain", "color": "#8b5cf6", "desc": "Keyless domain sub-reconnaissance"}, | |
| ] | |
| } | |
| async def websocket_search(websocket: WebSocket): | |
| await websocket.accept() | |
| logger.info("WebSocket connection established.") | |
| loop = asyncio.get_running_loop() | |
| signals = WorkerSignalsBridge(websocket, loop) | |
| threads: List[threading.Thread] = [] | |
| try: | |
| while True: | |
| data = await websocket.receive_text() | |
| params = json.loads(data) | |
| action = params.get("action") | |
| if action == "start": | |
| username = params.get("username", "").strip() | |
| email = params.get("email", "").strip() | |
| phone = params.get("phone", "").strip() | |
| domain = params.get("domain", "").strip() | |
| enabled_engines = params.get("engines", []) | |
| logger.info(f"Starting OSINT Scan - U:{username} E:{email} P:{phone} D:{domain} Engines:{enabled_engines}") | |
| def launch_worker(target_fn, *args): | |
| t = threading.Thread(target=target_fn, args=(signals, *args), daemon=True) | |
| t.start() | |
| threads.append(t) | |
| # 1. Usernames | |
| if username: | |
| if "sherlock" in enabled_engines: | |
| launch_worker(run_sherlock_worker, username, sherlock_data) | |
| if "maigret" in enabled_engines: | |
| launch_worker(run_maigret_worker, username, maigret_data) | |
| if "nexfil" in enabled_engines: | |
| launch_worker(run_nexfil_worker, username) | |
| if "user-scanner" in enabled_engines: | |
| launch_worker(run_uscan_username_worker, username) | |
| if "socialscan" in enabled_engines: | |
| launch_worker(run_socialscan_worker, username, False) | |
| if "blackbird" in enabled_engines: | |
| launch_worker(run_blackbird_worker, username) | |
| # 2. Emails | |
| if email: | |
| if "user-scanner" in enabled_engines: | |
| launch_worker(run_uscan_email_worker, email) | |
| if "socialscan" in enabled_engines: | |
| launch_worker(run_socialscan_worker, email, True) | |
| if "h8mail" in enabled_engines: | |
| launch_worker(run_h8mail_worker, email) | |
| if "email_harvester" in enabled_engines: | |
| launch_worker(run_emailharvester_worker, email) | |
| if "holehe" in enabled_engines: | |
| launch_worker(run_holehe_worker, email) | |
| # 3. Phones | |
| if phone: | |
| if "phoneinfoga" in enabled_engines: | |
| launch_worker(run_phoneinfoga_worker, phone) | |
| # 4. Domains | |
| if domain: | |
| if "recon_ng" in enabled_engines: | |
| launch_worker(run_recon_ng_worker, domain) | |
| elif action == "stop": | |
| logger.info("Search cancellation requested by client.") | |
| # Note: Python threads running OSINT libraries can't easily be killed abruptly. | |
| # However, sending a cancel notification and resetting states will cause the client to disconnect/reset. | |
| # In standard usage, closing the websocket safely halts UI state. | |
| await websocket.send_json({"event": "cancelled", "data": []}) | |
| except WebSocketDisconnect: | |
| logger.info("WebSocket disconnected.") | |
| except Exception as e: | |
| logger.error(f"Error in websocket handler: {e}") | |
| finally: | |
| # Threads are daemon threads and will safely detach and die when process exits or runs out of tasks. | |
| pass | |
| # --------------------------------------------------------------------------- | |
| # Static Files Serving (production compilation fallback) | |
| # --------------------------------------------------------------------------- | |
| frontend_dist = root_path / "web_app" / "frontend" / "dist" | |
| if frontend_dist.exists(): | |
| app.mount("/", StaticFiles(directory=str(frontend_dist), html=True), name="static") | |
| else: | |
| def read_root(): | |
| return { | |
| "status": "online", | |
| "message": "Sherlock Web OSINT Suite backend running. To view the UI, build the frontend or run the Bun development server." | |
| } | |