Gianzfx
deploy: restore frontend src and readme metadata
01ec222
Raw History Blame Contribute Delete
10.9 kB
import sys
import asyncio
import json
import logging
import threading
from pathlib import Path
from typing import List, Dict, Any
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
# Ensure project root is in python path
root_path = Path(__file__).resolve().parent.parent.parent
if str(root_path) not in sys.path:
sys.path.insert(0, str(root_path))
# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("sherlock_web_backend")
# Import site loader
from sherlock_project.sites import SitesInformation
# Load the workers from the desktop application modules
from qt_app.workers import (
run_sherlock_worker,
run_maigret_worker,
run_nexfil_worker,
run_uscan_username_worker,
run_uscan_email_worker,
run_socialscan_worker,
run_h8mail_worker,
run_emailharvester_worker,
run_phoneinfoga_worker,
run_recon_ng_worker,
run_blackbird_worker,
run_holehe_worker,
)
app = FastAPI(title="Sherlock Web OSINT Suite")
# CORS middleware to allow the Bun/React dev server to communicate with this backend
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# ---------------------------------------------------------------------------
# Data Loading
# ---------------------------------------------------------------------------
sherlock_data = {}
maigret_data = {}
nexfil_url_count = 0
def load_data_sources():
global sherlock_data, maigret_data, nexfil_url_count
logger.info("Loading resource databases...")
# 1. Sherlock
base = root_path / "sherlock_project" / "resources" / "data.json"
if base.exists():
sites = SitesInformation(str(base))
sherlock_data = {s.name: s.information for s in sites}
logger.info(f"Loaded {len(sherlock_data)} Sherlock sites.")
else:
logger.warning("Sherlock data.json not found!")
# 2. Maigret
try:
from maigret import MaigretDatabase
db_path = Path(__import__("maigret", fromlist=["__file__"]).__file__).parent / "resources" / "data.json"
db = MaigretDatabase().load_from_path(str(db_path))
maigret_data = db.ranked_sites_dict()
logger.info(f"Loaded {len(maigret_data)} Maigret sites.")
except Exception as e:
logger.error(f"Failed to load Maigret database: {e}")
maigret_data = {}
# 3. NExfil
try:
import nexfil
pkg = Path(nexfil.__file__).parent
with open(pkg / "url_store.json") as f:
nexfil_url_count = len(json.loads(f.read()))
logger.info(f"Loaded {nexfil_url_count} NExfil target URLs.")
except Exception as e:
logger.error(f"Failed to load NExfil url store: {e}")
nexfil_url_count = 0
# Load on startup
load_data_sources()
# ---------------------------------------------------------------------------
# Signal Mock for Worker Bridge
# ---------------------------------------------------------------------------
class SignalBridge:
def __init__(self, name: str, websocket: WebSocket, loop: asyncio.AbstractEventLoop):
self.name = name
self.websocket = websocket
self.loop = loop
def emit(self, *args):
# We need to construct a message payload and push it to the WebSocket queue.
# This is safe to invoke from sub-threads using asyncio.run_coroutine_threadsafe.
payload = {
"event": self.name,
"data": list(args)
}
async def send():
try:
await self.websocket.send_json(payload)
except Exception:
pass
asyncio.run_coroutine_threadsafe(send(), self.loop)
class WorkerSignalsBridge:
"""Replicates the PySide6 QObject signals interface needed by workers.py"""
def __init__(self, websocket: WebSocket, loop: asyncio.AbstractEventLoop):
self.started = SignalBridge("started", websocket, loop)
self.result = SignalBridge("result", websocket, loop)
self.progress = SignalBridge("progress", websocket, loop)
self.finished = SignalBridge("finished", websocket, loop)
self.error = SignalBridge("error", websocket, loop)
# ---------------------------------------------------------------------------
# API Routes
# ---------------------------------------------------------------------------
@app.get("/api/info")
def get_info():
"""Returns basic configuration and loaded targets info for display."""
return {
"sherlock_sites_count": len(sherlock_data),
"maigret_sites_count": len(maigret_data),
"nexfil_url_count": nexfil_url_count,
"available_engines": [
{"id": "sherlock", "name": "Sherlock", "type": "username", "color": "#10b981", "desc": f"Scans {len(sherlock_data)} social networks"},
{"id": "maigret", "name": "Maigret", "type": "username", "color": "#8b5cf6", "desc": f"Scans {len(maigret_data)} sites with high-depth analysis"},
{"id": "nexfil", "name": "NExfil", "type": "username", "color": "#f59e0b", "desc": f"Ultra-fast parallel query on {nexfil_url_count} urls"},
{"id": "user-scanner", "name": "UserScanner", "type": "username/email", "color": "#06b6d4", "desc": "Subdomain and page response checker"},
{"id": "socialscan", "name": "SocialScan", "type": "username/email", "color": "#ec4899", "desc": "Accurate HTTP query on popular platforms"},
{"id": "blackbird", "name": "Blackbird", "type": "username", "color": "#ec4899", "desc": "Fast correlation tool for user accounts"},
{"id": "h8mail", "name": "h8mail", "type": "email", "color": "#ef4444", "desc": "Breach lookup and email credentials scanner"},
{"id": "email_harvester", "name": "EmailHarvester", "type": "email", "color": "#06b6d4", "desc": "Search engine parser for sub-domain emails"},
{"id": "holehe", "name": "Holehe", "type": "email", "color": "#f59e0b", "desc": "Checked registered mail on 120+ sites"},
{"id": "phoneinfoga", "name": "PhoneInfoga", "type": "phone", "color": "#10b981", "desc": "Information gathering on phone numbers"},
{"id": "recon_ng", "name": "Recon-ng", "type": "domain", "color": "#8b5cf6", "desc": "Keyless domain sub-reconnaissance"},
]
}
@app.websocket("/ws/search")
async def websocket_search(websocket: WebSocket):
await websocket.accept()
logger.info("WebSocket connection established.")
loop = asyncio.get_running_loop()
signals = WorkerSignalsBridge(websocket, loop)
threads: List[threading.Thread] = []
try:
while True:
data = await websocket.receive_text()
params = json.loads(data)
action = params.get("action")
if action == "start":
username = params.get("username", "").strip()
email = params.get("email", "").strip()
phone = params.get("phone", "").strip()
domain = params.get("domain", "").strip()
enabled_engines = params.get("engines", [])
logger.info(f"Starting OSINT Scan - U:{username} E:{email} P:{phone} D:{domain} Engines:{enabled_engines}")
def launch_worker(target_fn, *args):
t = threading.Thread(target=target_fn, args=(signals, *args), daemon=True)
t.start()
threads.append(t)
# 1. Usernames
if username:
if "sherlock" in enabled_engines:
launch_worker(run_sherlock_worker, username, sherlock_data)
if "maigret" in enabled_engines:
launch_worker(run_maigret_worker, username, maigret_data)
if "nexfil" in enabled_engines:
launch_worker(run_nexfil_worker, username)
if "user-scanner" in enabled_engines:
launch_worker(run_uscan_username_worker, username)
if "socialscan" in enabled_engines:
launch_worker(run_socialscan_worker, username, False)
if "blackbird" in enabled_engines:
launch_worker(run_blackbird_worker, username)
# 2. Emails
if email:
if "user-scanner" in enabled_engines:
launch_worker(run_uscan_email_worker, email)
if "socialscan" in enabled_engines:
launch_worker(run_socialscan_worker, email, True)
if "h8mail" in enabled_engines:
launch_worker(run_h8mail_worker, email)
if "email_harvester" in enabled_engines:
launch_worker(run_emailharvester_worker, email)
if "holehe" in enabled_engines:
launch_worker(run_holehe_worker, email)
# 3. Phones
if phone:
if "phoneinfoga" in enabled_engines:
launch_worker(run_phoneinfoga_worker, phone)
# 4. Domains
if domain:
if "recon_ng" in enabled_engines:
launch_worker(run_recon_ng_worker, domain)
elif action == "stop":
logger.info("Search cancellation requested by client.")
# Note: Python threads running OSINT libraries can't easily be killed abruptly.
# However, sending a cancel notification and resetting states will cause the client to disconnect/reset.
# In standard usage, closing the websocket safely halts UI state.
await websocket.send_json({"event": "cancelled", "data": []})
except WebSocketDisconnect:
logger.info("WebSocket disconnected.")
except Exception as e:
logger.error(f"Error in websocket handler: {e}")
finally:
# Threads are daemon threads and will safely detach and die when process exits or runs out of tasks.
pass
# ---------------------------------------------------------------------------
# Static Files Serving (production compilation fallback)
# ---------------------------------------------------------------------------
frontend_dist = root_path / "web_app" / "frontend" / "dist"
if frontend_dist.exists():
app.mount("/", StaticFiles(directory=str(frontend_dist), html=True), name="static")
else:
@app.get("/")
def read_root():
return {
"status": "online",
"message": "Sherlock Web OSINT Suite backend running. To view the UI, build the frontend or run the Bun development server."
}