Spaces:
Paused
Paused
Peng Ding commited on
Commit ·
a4ba91f
1
Parent(s): 39e5cf3
feat: remove cloakbrowser pip dependency, auto-download binary
Browse files- Drop cloakbrowser from dependencies (no more playwright transitive dep)
- LocalWorker auto-detects binary: env var → ~/.cloakbrowser/ → download
- Binary downloaded directly from GitHub Releases via pure Python
- CLOAKBROWSER_MIRROR env var for China (e.g. https://ghfast.top)
- scripts/download-cloakbrowser.py for Dockerfile and manual use
- Stealth args generated inline (random fingerprint)
- Dockerfile full target: downloads binary directly, no pip cloakbrowser
Image sizes: full 1.07GB (was 1.43GB), gateway 336MB (was 498MB)
- Dockerfile +4 -3
- pyproject.toml +0 -1
- scripts/download-cloakbrowser.py +90 -0
- src/veilrender/browser.py +114 -3
Dockerfile
CHANGED
|
@@ -27,9 +27,10 @@ RUN useradd -m -u 1000 user || true
|
|
| 27 |
WORKDIR /app
|
| 28 |
COPY pyproject.toml .
|
| 29 |
COPY src/ src/
|
|
|
|
| 30 |
RUN pip install --no-cache-dir .
|
| 31 |
|
| 32 |
-
## ── gateway (no browser, ~
|
| 33 |
## docker build --target gateway -t oaklight/veilrender:gateway .
|
| 34 |
FROM base AS gateway
|
| 35 |
|
|
@@ -47,12 +48,12 @@ RUN mkdir /gbm-libs && \
|
|
| 47 |
cp -a /usr/lib/x86_64-linux-gnu/${lib}.so* /gbm-libs/ 2>/dev/null || true; \
|
| 48 |
done
|
| 49 |
|
| 50 |
-
## ── full (CloakBrowser
|
| 51 |
## docker build -t oaklight/veilrender:latest .
|
| 52 |
FROM base AS full
|
| 53 |
|
| 54 |
COPY --from=gbm-donor /gbm-libs/* /usr/lib/x86_64-linux-gnu/
|
| 55 |
USER 1000
|
| 56 |
-
RUN python -
|
| 57 |
EXPOSE 7860
|
| 58 |
CMD ["python", "-m", "veilrender"]
|
|
|
|
| 27 |
WORKDIR /app
|
| 28 |
COPY pyproject.toml .
|
| 29 |
COPY src/ src/
|
| 30 |
+
COPY scripts/ scripts/
|
| 31 |
RUN pip install --no-cache-dir .
|
| 32 |
|
| 33 |
+
## ── gateway (no browser, ~350MB) ─────────────────────────
|
| 34 |
## docker build --target gateway -t oaklight/veilrender:gateway .
|
| 35 |
FROM base AS gateway
|
| 36 |
|
|
|
|
| 48 |
cp -a /usr/lib/x86_64-linux-gnu/${lib}.so* /gbm-libs/ 2>/dev/null || true; \
|
| 49 |
done
|
| 50 |
|
| 51 |
+
## ── full (CloakBrowser binary downloaded directly) ───────
|
| 52 |
## docker build -t oaklight/veilrender:latest .
|
| 53 |
FROM base AS full
|
| 54 |
|
| 55 |
COPY --from=gbm-donor /gbm-libs/* /usr/lib/x86_64-linux-gnu/
|
| 56 |
USER 1000
|
| 57 |
+
RUN python scripts/download-cloakbrowser.py
|
| 58 |
EXPOSE 7860
|
| 59 |
CMD ["python", "-m", "veilrender"]
|
pyproject.toml
CHANGED
|
@@ -17,7 +17,6 @@ classifiers = [
|
|
| 17 |
]
|
| 18 |
|
| 19 |
dependencies = [
|
| 20 |
-
"cloakbrowser>=0.3.0",
|
| 21 |
"patchright>=1.0.0",
|
| 22 |
"minio>=7.2.0",
|
| 23 |
]
|
|
|
|
| 17 |
]
|
| 18 |
|
| 19 |
dependencies = [
|
|
|
|
| 20 |
"patchright>=1.0.0",
|
| 21 |
"minio>=7.2.0",
|
| 22 |
]
|
scripts/download-cloakbrowser.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Download CloakBrowser binary from cloakbrowser.dev.
|
| 2 |
+
|
| 3 |
+
Resolves the Chromium version from the PyPI wheel metadata (no pip
|
| 4 |
+
install needed), then downloads and extracts the binary tarball.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
import platform
|
| 10 |
+
import re
|
| 11 |
+
import sys
|
| 12 |
+
import tarfile
|
| 13 |
+
import urllib.request
|
| 14 |
+
import zipfile
|
| 15 |
+
from io import BytesIO
|
| 16 |
+
|
| 17 |
+
PYPI_JSON = "https://pypi.org/pypi/cloakbrowser/json"
|
| 18 |
+
GITHUB_BASE = "https://github.com/CloakHQ/cloakbrowser/releases/download"
|
| 19 |
+
MIRROR = os.environ.get("CLOAKBROWSER_MIRROR", "")
|
| 20 |
+
CACHE_DIR = os.environ.get(
|
| 21 |
+
"CLOAKBROWSER_CACHE_DIR", os.path.expanduser("~/.cloakbrowser")
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _get_chromium_version(pip_version: str | None = None) -> str:
|
| 26 |
+
"""Get CloakBrowser's Chromium version from PyPI wheel metadata."""
|
| 27 |
+
pypi = json.loads(
|
| 28 |
+
urllib.request.urlopen(PYPI_JSON, timeout=30).read()
|
| 29 |
+
)
|
| 30 |
+
whl_url = next(
|
| 31 |
+
u["url"] for u in pypi["urls"] if u["filename"].endswith(".whl")
|
| 32 |
+
)
|
| 33 |
+
whl_data = urllib.request.urlopen(whl_url, timeout=60).read()
|
| 34 |
+
with zipfile.ZipFile(BytesIO(whl_data)) as zf:
|
| 35 |
+
for name in zf.namelist():
|
| 36 |
+
if name.endswith("config.py"):
|
| 37 |
+
content = zf.read(name).decode()
|
| 38 |
+
m = re.search(r'CHROMIUM_VERSION\s*=\s*"([^"]+)"', content)
|
| 39 |
+
if m:
|
| 40 |
+
return m.group(1)
|
| 41 |
+
raise RuntimeError("Could not find CHROMIUM_VERSION in cloakbrowser wheel")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _detect_platform() -> str:
|
| 45 |
+
machine = platform.machine().lower()
|
| 46 |
+
system = platform.system().lower()
|
| 47 |
+
if system == "linux":
|
| 48 |
+
arch = "x64" if machine in ("x86_64", "amd64") else "arm64"
|
| 49 |
+
return f"linux-{arch}"
|
| 50 |
+
if system == "darwin":
|
| 51 |
+
arch = "arm64" if machine == "arm64" else "x64"
|
| 52 |
+
return f"mac-{arch}"
|
| 53 |
+
raise RuntimeError(f"Unsupported platform: {system}-{machine}")
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def main() -> None:
|
| 57 |
+
version = sys.argv[1] if len(sys.argv) > 1 else None
|
| 58 |
+
if not version:
|
| 59 |
+
print("Resolving CloakBrowser Chromium version from PyPI...")
|
| 60 |
+
version = _get_chromium_version()
|
| 61 |
+
|
| 62 |
+
plat = _detect_platform()
|
| 63 |
+
dest = os.path.join(CACHE_DIR, f"chromium-{version}")
|
| 64 |
+
binary = os.path.join(dest, "chrome")
|
| 65 |
+
|
| 66 |
+
if os.path.isfile(binary):
|
| 67 |
+
print(f"Already installed: {binary}")
|
| 68 |
+
return
|
| 69 |
+
|
| 70 |
+
gh_url = f"{GITHUB_BASE}/chromium-v{version}/cloakbrowser-{plat}.tar.gz"
|
| 71 |
+
url = f"{MIRROR}/{gh_url}" if MIRROR else gh_url
|
| 72 |
+
os.makedirs(dest, exist_ok=True)
|
| 73 |
+
tmp = os.path.join(CACHE_DIR, "download.tar.gz")
|
| 74 |
+
|
| 75 |
+
print(f"Downloading CloakBrowser {version} ({plat})...")
|
| 76 |
+
print(f" URL: {url}")
|
| 77 |
+
urllib.request.urlretrieve(url, tmp)
|
| 78 |
+
|
| 79 |
+
print("Extracting...")
|
| 80 |
+
with tarfile.open(tmp, "r:gz") as tf:
|
| 81 |
+
tf.extractall(dest)
|
| 82 |
+
os.unlink(tmp)
|
| 83 |
+
|
| 84 |
+
if os.path.isfile(binary):
|
| 85 |
+
os.chmod(binary, 0o755)
|
| 86 |
+
print(f"Installed: {binary}")
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
if __name__ == "__main__":
|
| 90 |
+
main()
|
src/veilrender/browser.py
CHANGED
|
@@ -7,15 +7,17 @@ workers when ``VEILRENDER_WORKERS`` is configured.
|
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
import asyncio
|
|
|
|
| 10 |
import json
|
| 11 |
import logging
|
|
|
|
|
|
|
| 12 |
import subprocess
|
| 13 |
import urllib.request
|
| 14 |
from collections.abc import AsyncIterator, Callable
|
| 15 |
from contextlib import asynccontextmanager
|
| 16 |
from typing import Any
|
| 17 |
|
| 18 |
-
from cloakbrowser import ensure_binary, get_default_stealth_args
|
| 19 |
from patchright.async_api import Browser, BrowserContext, Page, async_playwright
|
| 20 |
|
| 21 |
from veilrender.config import settings
|
|
@@ -25,6 +27,115 @@ logger = logging.getLogger(__name__)
|
|
| 25 |
|
| 26 |
CDP_PORT = 9222
|
| 27 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
def _fetch_json(url: str, timeout: int = 2) -> Any:
|
| 30 |
"""Fetch and parse JSON from a URL (blocking)."""
|
|
@@ -169,8 +280,8 @@ class LocalWorker(_BaseWorker):
|
|
| 169 |
self._lock = asyncio.Lock()
|
| 170 |
|
| 171 |
async def start(self) -> None:
|
| 172 |
-
executable_path =
|
| 173 |
-
stealth_args =
|
| 174 |
|
| 175 |
chrome_args = [
|
| 176 |
executable_path,
|
|
|
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
import asyncio
|
| 10 |
+
import glob
|
| 11 |
import json
|
| 12 |
import logging
|
| 13 |
+
import os
|
| 14 |
+
import random
|
| 15 |
import subprocess
|
| 16 |
import urllib.request
|
| 17 |
from collections.abc import AsyncIterator, Callable
|
| 18 |
from contextlib import asynccontextmanager
|
| 19 |
from typing import Any
|
| 20 |
|
|
|
|
| 21 |
from patchright.async_api import Browser, BrowserContext, Page, async_playwright
|
| 22 |
|
| 23 |
from veilrender.config import settings
|
|
|
|
| 27 |
|
| 28 |
CDP_PORT = 9222
|
| 29 |
|
| 30 |
+
_CLOAKBROWSER_DEFAULT_DIR = os.path.expanduser("~/.cloakbrowser")
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
_CLOAKBROWSER_PYPI_URL = "https://pypi.org/pypi/cloakbrowser/json"
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _resolve_chromium_version() -> str:
|
| 37 |
+
"""Resolve CloakBrowser's Chromium version from the PyPI wheel."""
|
| 38 |
+
import re
|
| 39 |
+
import zipfile
|
| 40 |
+
from io import BytesIO
|
| 41 |
+
|
| 42 |
+
pypi = _fetch_json(_CLOAKBROWSER_PYPI_URL, timeout=30)
|
| 43 |
+
whl_url = next(u["url"] for u in pypi["urls"] if u["filename"].endswith(".whl"))
|
| 44 |
+
whl_data = urllib.request.urlopen(whl_url, timeout=60).read()
|
| 45 |
+
with zipfile.ZipFile(BytesIO(whl_data)) as zf:
|
| 46 |
+
for name in zf.namelist():
|
| 47 |
+
if name.endswith("config.py"):
|
| 48 |
+
content = zf.read(name).decode()
|
| 49 |
+
m = re.search(r'CHROMIUM_VERSION\s*=\s*"([^"]+)"', content)
|
| 50 |
+
if m:
|
| 51 |
+
return m.group(1)
|
| 52 |
+
raise RuntimeError("Could not resolve CloakBrowser Chromium version")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _detect_platform() -> str:
|
| 56 |
+
"""Detect platform string for CloakBrowser download URL."""
|
| 57 |
+
import platform as _platform
|
| 58 |
+
|
| 59 |
+
machine = _platform.machine().lower()
|
| 60 |
+
system = _platform.system().lower()
|
| 61 |
+
if system == "linux":
|
| 62 |
+
arch = "x64" if machine in ("x86_64", "amd64") else "arm64"
|
| 63 |
+
return f"linux-{arch}"
|
| 64 |
+
if system == "darwin":
|
| 65 |
+
arch = "arm64" if machine == "arm64" else "x64"
|
| 66 |
+
return f"mac-{arch}"
|
| 67 |
+
raise RuntimeError(f"Unsupported platform: {system}-{machine}")
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _download_browser_binary() -> str:
|
| 71 |
+
"""Download CloakBrowser binary from cloakbrowser.dev (or mirror)."""
|
| 72 |
+
import tarfile
|
| 73 |
+
|
| 74 |
+
version = _resolve_chromium_version()
|
| 75 |
+
plat = _detect_platform()
|
| 76 |
+
gh_url = f"https://github.com/CloakHQ/cloakbrowser/releases/download/chromium-v{version}/cloakbrowser-{plat}.tar.gz"
|
| 77 |
+
mirror = os.environ.get("CLOAKBROWSER_MIRROR", "")
|
| 78 |
+
url = f"{mirror}/{gh_url}" if mirror else gh_url
|
| 79 |
+
dest_dir = os.path.join(_CLOAKBROWSER_DEFAULT_DIR, f"chromium-{version}")
|
| 80 |
+
binary = os.path.join(dest_dir, "chrome")
|
| 81 |
+
|
| 82 |
+
if os.path.isfile(binary):
|
| 83 |
+
logger.info("Binary already exists at %s", binary)
|
| 84 |
+
return binary
|
| 85 |
+
|
| 86 |
+
logger.info("Downloading CloakBrowser %s (%s)...", version, plat)
|
| 87 |
+
os.makedirs(dest_dir, exist_ok=True)
|
| 88 |
+
tmp = os.path.join(dest_dir, ".download.tar.gz")
|
| 89 |
+
try:
|
| 90 |
+
urllib.request.urlretrieve(url, tmp)
|
| 91 |
+
with tarfile.open(tmp, "r:gz") as tf:
|
| 92 |
+
tf.extractall(dest_dir)
|
| 93 |
+
finally:
|
| 94 |
+
if os.path.exists(tmp):
|
| 95 |
+
os.unlink(tmp)
|
| 96 |
+
|
| 97 |
+
if os.path.isfile(binary):
|
| 98 |
+
os.chmod(binary, 0o755)
|
| 99 |
+
logger.info("Downloaded CloakBrowser %s to %s", version, binary)
|
| 100 |
+
return binary
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _find_browser_binary() -> str:
|
| 104 |
+
"""Locate a CloakBrowser (or compatible) Chromium binary.
|
| 105 |
+
|
| 106 |
+
Search order:
|
| 107 |
+
1. CLOAKBROWSER_BINARY env var
|
| 108 |
+
2. ~/.cloakbrowser/*/chrome (pre-downloaded or Docker-installed)
|
| 109 |
+
3. Auto-download from cloakbrowser.dev
|
| 110 |
+
"""
|
| 111 |
+
env_path = os.environ.get("CLOAKBROWSER_BINARY")
|
| 112 |
+
if env_path:
|
| 113 |
+
if os.path.isfile(env_path) and os.access(env_path, os.X_OK):
|
| 114 |
+
logger.info("Using browser binary from CLOAKBROWSER_BINARY: %s", env_path)
|
| 115 |
+
return env_path
|
| 116 |
+
raise FileNotFoundError(
|
| 117 |
+
f"CLOAKBROWSER_BINARY={env_path} not found or not executable"
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
candidates = sorted(
|
| 121 |
+
glob.glob(f"{_CLOAKBROWSER_DEFAULT_DIR}/*/chrome"), reverse=True
|
| 122 |
+
)
|
| 123 |
+
if candidates:
|
| 124 |
+
logger.info("Found browser binary: %s", candidates[0])
|
| 125 |
+
return candidates[0]
|
| 126 |
+
|
| 127 |
+
return _download_browser_binary()
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _get_stealth_args() -> list[str]:
|
| 131 |
+
"""Get stealth launch arguments for CloakBrowser."""
|
| 132 |
+
fingerprint = random.randint(10000, 99999)
|
| 133 |
+
return [
|
| 134 |
+
"--no-sandbox",
|
| 135 |
+
f"--fingerprint={fingerprint}",
|
| 136 |
+
"--fingerprint-platform=linux",
|
| 137 |
+
]
|
| 138 |
+
|
| 139 |
|
| 140 |
def _fetch_json(url: str, timeout: int = 2) -> Any:
|
| 141 |
"""Fetch and parse JSON from a URL (blocking)."""
|
|
|
|
| 280 |
self._lock = asyncio.Lock()
|
| 281 |
|
| 282 |
async def start(self) -> None:
|
| 283 |
+
executable_path = _find_browser_binary()
|
| 284 |
+
stealth_args = _get_stealth_args()
|
| 285 |
|
| 286 |
chrome_args = [
|
| 287 |
executable_path,
|