searx_runtime.py

"""User-space SearXNG bootstrap for the Respite search backend.

SearXNG is not on PyPI (the `searxng` package there is the abandoned
pre-NG searx). Instead we fetch the upstream source tarball into a local
cache, install it into a userspace virtualenv on first boot (no root),
and launch the built-in development server bound to 127.0.0.1:8888.

The instance is configured by searxng_settings.yml in the repo root
(SEARXNG_SETTINGS_PATH points there), with the DDG + Google engines
enabled and JSON output on.

Env:
  SEARXNG_VERSION  source tag/branch to fetch (default: master)
  SEARXNG_PORT     localhost port (default 8888)
  SEARXNG_OFF      set to "1" to skip the bootstrap entirely
"""

import os
import shutil
import subprocess
import tarfile
import threading
import time
import urllib.request

SEARXNG_VERSION = os.environ.get("SEARXNG_VERSION", "master")
SEARXNG_PORT = int(os.environ.get("SEARXNG_PORT", "8888"))
SEARXNG_URL = f"https://github.com/searxng/searxng/archive/refs/heads/{SEARXNG_VERSION}.tar.gz"

ROOT = os.path.dirname(os.path.abspath(__file__))
RUNTIME_DIR = os.path.join(ROOT, ".searxng-runtime")
SRC_DIR = os.path.join(RUNTIME_DIR, f"searxng-{SEARXNG_VERSION}")
VENV_DIR = os.path.join(RUNTIME_DIR, "venv")
VENV_PIP = os.path.join(VENV_DIR, "bin", "pip")
SETTINGS_FILE = os.path.join(ROOT, "searxng_settings.yml")

_proc = None
_lock = threading.Lock()
_ready = threading.Event()
_boot_error = ""


def _log(msg):
    print(f"[searxng] {msg}", flush=True)


def _fetch_source() -> bool:
    if os.path.isdir(SRC_DIR) and os.path.exists(os.path.join(SRC_DIR, "searx", "webapp.py")):
        return True
    os.makedirs(RUNTIME_DIR, exist_ok=True)
    tgz = os.path.join(RUNTIME_DIR, "searxng.tar.gz")
    _log(f"downloading {SEARXNG_URL}")
    urllib.request.urlretrieve(SEARXNG_URL, tgz)
    _log("extracting")
    with tarfile.open(tgz, "r:gz") as tf:
        tf.extractall(RUNTIME_DIR)
    os.remove(tgz)
    return os.path.isdir(SRC_DIR)


def _venv_python() -> str | None:
    """Create the venv and install searxng + deps (idempotent)."""
    py = os.path.join(VENV_DIR, "bin", "python")
    if os.path.exists(py):
        # A stale venv from a different interpreter breaks imports; recreate.
        r = subprocess.run([py, "-c", "import tomllib"], capture_output=True)
        if r.returncode != 0:
            _log("stale venv (python<3.11) — recreating")
            shutil.rmtree(VENV_DIR, ignore_errors=True)
        else:
            return py
    # Prefer the newest interpreter available (SearXNG master needs 3.11+;
    # the container base image ships 3.10).
    base = None
    for cand in ("python3.13", "python3.12", "python3.11", "python3"):
        if shutil.which(cand):
            base = cand
            break
    _log(f"creating venv with {base}")
    if not base:
        return None
    # --without-pip: many slim images ship the interpreter without the
    # venv/ensurepip pieces; get-pip.py bootstraps pip inside afterwards.
    r = subprocess.run([base, "-m", "venv", "--without-pip", VENV_DIR], capture_output=True)
    if r.returncode != 0:
        r = subprocess.run([base, "-m", "venv", VENV_DIR], capture_output=True)
    if r.returncode != 0:
        _log(f"venv creation failed: {r.stderr[-500:]}")
        return None
    vpy = os.path.join(VENV_DIR, "bin", "python")
    if not os.path.exists(os.path.join(VENV_DIR, "bin", "pip")):
        _log("bootstrapping pip via get-pip.py")
        gp = os.path.join(RUNTIME_DIR, "get-pip.py")
        urllib.request.urlretrieve("https://bootstrap.pypa.io/get-pip.py", gp)
        if subprocess.run([vpy, gp], capture_output=True).returncode != 0:
            _log("get-pip failed")
            return None
    _log("installing searxng deps (this takes a few minutes on first boot)")
    r1 = subprocess.run(
        [vpy, "-m", "pip", "install", "--no-input", "-r", os.path.join(SRC_DIR, "requirements.txt")],
        capture_output=True, text=True, timeout=900,
    )
    if r1.returncode != 0:
        _log(f"deps install failed: {r1.stderr[-2000:]}")
        return None
    r2 = subprocess.run(
        [vpy, "-m", "pip", "install", "--no-input", "--no-deps", "-e", SRC_DIR],
        capture_output=True, text=True, timeout=300,
    )
    if r2.returncode != 0:
        _log(f"searxng install failed: {r2.stderr[-2000:]}")
        return None
    _log("searxng installed")
    return py


def _wait_ready(proc: subprocess.Popen, timeout: float = 120.0) -> bool:
    import socket
    deadline = time.time() + timeout
    while time.time() < deadline:
        if proc.poll() is not None:
            _log(f"searxng exited early with code {proc.returncode}")
            return False
        try:
            with socket.create_connection(("127.0.0.1", SEARXNG_PORT), timeout=1):
                return True
        except OSError:
            time.sleep(0.5)
    return False


def start_searxng() -> bool:
    """Launch SearXNG bound to localhost. Safe to call once at startup."""
    global _proc
    if os.environ.get("SEARXNG_OFF") == "1":
        _log("disabled via SEARXNG_OFF")
        return False
    with _lock:
        if _proc and _proc.poll() is None:
            return True
        global _boot_error
        try:
            if not _fetch_source():
                _boot_error = "source fetch failed"
                _log(_boot_error)
                return False
            py = _venv_python()
            if not py:
                _boot_error = "venv/pip install failed (see logs)"
                return False
        except Exception as e:
            _boot_error = f"bootstrap failed: {e}"
            _log(_boot_error)
            return False
        env = os.environ.copy()
        env["SEARXNG_SETTINGS_PATH"] = SETTINGS_FILE
        env["PYTHONPATH"] = SRC_DIR
        _log(f"starting searxng on 127.0.0.1:{SEARXNG_PORT}")
        _proc = subprocess.Popen(
            [py, "-m", "searx.webapp"],
            cwd=SRC_DIR,
            env=env,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            text=True,
            bufsize=1,
        )
        threading.Thread(target=_pump_logs, daemon=True).start()
        if _wait_ready(_proc):
            _log("searxng is READY")
            _ready.set()
            return True
        _log("searxng failed to become ready")
        return False


def _pump_logs():
    assert _proc and _proc.stdout
    for line in _proc.stdout:
        _log(line.rstrip())


def searxng_ready() -> bool:
    return _ready.is_set()


def boot_error() -> str:
    return _boot_error