Peng Ding commited on
Commit
a4ba91f
·
1 Parent(s): 39e5cf3

feat: remove cloakbrowser pip dependency, auto-download binary

Browse files

- Drop cloakbrowser from dependencies (no more playwright transitive dep)
- LocalWorker auto-detects binary: env var → ~/.cloakbrowser/ → download
- Binary downloaded directly from GitHub Releases via pure Python
- CLOAKBROWSER_MIRROR env var for China (e.g. https://ghfast.top)
- scripts/download-cloakbrowser.py for Dockerfile and manual use
- Stealth args generated inline (random fingerprint)
- Dockerfile full target: downloads binary directly, no pip cloakbrowser

Image sizes: full 1.07GB (was 1.43GB), gateway 336MB (was 498MB)

Dockerfile CHANGED
@@ -27,9 +27,10 @@ RUN useradd -m -u 1000 user || true
27
  WORKDIR /app
28
  COPY pyproject.toml .
29
  COPY src/ src/
 
30
  RUN pip install --no-cache-dir .
31
 
32
- ## ── gateway (no browser, ~500MB) ─────────────────────────
33
  ## docker build --target gateway -t oaklight/veilrender:gateway .
34
  FROM base AS gateway
35
 
@@ -47,12 +48,12 @@ RUN mkdir /gbm-libs && \
47
  cp -a /usr/lib/x86_64-linux-gnu/${lib}.so* /gbm-libs/ 2>/dev/null || true; \
48
  done
49
 
50
- ## ── full (CloakBrowser embedded, ~1.2GB) ─────────────────
51
  ## docker build -t oaklight/veilrender:latest .
52
  FROM base AS full
53
 
54
  COPY --from=gbm-donor /gbm-libs/* /usr/lib/x86_64-linux-gnu/
55
  USER 1000
56
- RUN python -c "from cloakbrowser import ensure_binary; ensure_binary()"
57
  EXPOSE 7860
58
  CMD ["python", "-m", "veilrender"]
 
27
  WORKDIR /app
28
  COPY pyproject.toml .
29
  COPY src/ src/
30
+ COPY scripts/ scripts/
31
  RUN pip install --no-cache-dir .
32
 
33
+ ## ── gateway (no browser, ~350MB) ─────────────────────────
34
  ## docker build --target gateway -t oaklight/veilrender:gateway .
35
  FROM base AS gateway
36
 
 
48
  cp -a /usr/lib/x86_64-linux-gnu/${lib}.so* /gbm-libs/ 2>/dev/null || true; \
49
  done
50
 
51
+ ## ── full (CloakBrowser binary downloaded directly) ───────
52
  ## docker build -t oaklight/veilrender:latest .
53
  FROM base AS full
54
 
55
  COPY --from=gbm-donor /gbm-libs/* /usr/lib/x86_64-linux-gnu/
56
  USER 1000
57
+ RUN python scripts/download-cloakbrowser.py
58
  EXPOSE 7860
59
  CMD ["python", "-m", "veilrender"]
pyproject.toml CHANGED
@@ -17,7 +17,6 @@ classifiers = [
17
  ]
18
 
19
  dependencies = [
20
- "cloakbrowser>=0.3.0",
21
  "patchright>=1.0.0",
22
  "minio>=7.2.0",
23
  ]
 
17
  ]
18
 
19
  dependencies = [
 
20
  "patchright>=1.0.0",
21
  "minio>=7.2.0",
22
  ]
scripts/download-cloakbrowser.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Download CloakBrowser binary from cloakbrowser.dev.
2
+
3
+ Resolves the Chromium version from the PyPI wheel metadata (no pip
4
+ install needed), then downloads and extracts the binary tarball.
5
+ """
6
+
7
+ import json
8
+ import os
9
+ import platform
10
+ import re
11
+ import sys
12
+ import tarfile
13
+ import urllib.request
14
+ import zipfile
15
+ from io import BytesIO
16
+
17
+ PYPI_JSON = "https://pypi.org/pypi/cloakbrowser/json"
18
+ GITHUB_BASE = "https://github.com/CloakHQ/cloakbrowser/releases/download"
19
+ MIRROR = os.environ.get("CLOAKBROWSER_MIRROR", "")
20
+ CACHE_DIR = os.environ.get(
21
+ "CLOAKBROWSER_CACHE_DIR", os.path.expanduser("~/.cloakbrowser")
22
+ )
23
+
24
+
25
+ def _get_chromium_version(pip_version: str | None = None) -> str:
26
+ """Get CloakBrowser's Chromium version from PyPI wheel metadata."""
27
+ pypi = json.loads(
28
+ urllib.request.urlopen(PYPI_JSON, timeout=30).read()
29
+ )
30
+ whl_url = next(
31
+ u["url"] for u in pypi["urls"] if u["filename"].endswith(".whl")
32
+ )
33
+ whl_data = urllib.request.urlopen(whl_url, timeout=60).read()
34
+ with zipfile.ZipFile(BytesIO(whl_data)) as zf:
35
+ for name in zf.namelist():
36
+ if name.endswith("config.py"):
37
+ content = zf.read(name).decode()
38
+ m = re.search(r'CHROMIUM_VERSION\s*=\s*"([^"]+)"', content)
39
+ if m:
40
+ return m.group(1)
41
+ raise RuntimeError("Could not find CHROMIUM_VERSION in cloakbrowser wheel")
42
+
43
+
44
+ def _detect_platform() -> str:
45
+ machine = platform.machine().lower()
46
+ system = platform.system().lower()
47
+ if system == "linux":
48
+ arch = "x64" if machine in ("x86_64", "amd64") else "arm64"
49
+ return f"linux-{arch}"
50
+ if system == "darwin":
51
+ arch = "arm64" if machine == "arm64" else "x64"
52
+ return f"mac-{arch}"
53
+ raise RuntimeError(f"Unsupported platform: {system}-{machine}")
54
+
55
+
56
+ def main() -> None:
57
+ version = sys.argv[1] if len(sys.argv) > 1 else None
58
+ if not version:
59
+ print("Resolving CloakBrowser Chromium version from PyPI...")
60
+ version = _get_chromium_version()
61
+
62
+ plat = _detect_platform()
63
+ dest = os.path.join(CACHE_DIR, f"chromium-{version}")
64
+ binary = os.path.join(dest, "chrome")
65
+
66
+ if os.path.isfile(binary):
67
+ print(f"Already installed: {binary}")
68
+ return
69
+
70
+ gh_url = f"{GITHUB_BASE}/chromium-v{version}/cloakbrowser-{plat}.tar.gz"
71
+ url = f"{MIRROR}/{gh_url}" if MIRROR else gh_url
72
+ os.makedirs(dest, exist_ok=True)
73
+ tmp = os.path.join(CACHE_DIR, "download.tar.gz")
74
+
75
+ print(f"Downloading CloakBrowser {version} ({plat})...")
76
+ print(f" URL: {url}")
77
+ urllib.request.urlretrieve(url, tmp)
78
+
79
+ print("Extracting...")
80
+ with tarfile.open(tmp, "r:gz") as tf:
81
+ tf.extractall(dest)
82
+ os.unlink(tmp)
83
+
84
+ if os.path.isfile(binary):
85
+ os.chmod(binary, 0o755)
86
+ print(f"Installed: {binary}")
87
+
88
+
89
+ if __name__ == "__main__":
90
+ main()
src/veilrender/browser.py CHANGED
@@ -7,15 +7,17 @@ workers when ``VEILRENDER_WORKERS`` is configured.
7
  from __future__ import annotations
8
 
9
  import asyncio
 
10
  import json
11
  import logging
 
 
12
  import subprocess
13
  import urllib.request
14
  from collections.abc import AsyncIterator, Callable
15
  from contextlib import asynccontextmanager
16
  from typing import Any
17
 
18
- from cloakbrowser import ensure_binary, get_default_stealth_args
19
  from patchright.async_api import Browser, BrowserContext, Page, async_playwright
20
 
21
  from veilrender.config import settings
@@ -25,6 +27,115 @@ logger = logging.getLogger(__name__)
25
 
26
  CDP_PORT = 9222
27
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
 
29
  def _fetch_json(url: str, timeout: int = 2) -> Any:
30
  """Fetch and parse JSON from a URL (blocking)."""
@@ -169,8 +280,8 @@ class LocalWorker(_BaseWorker):
169
  self._lock = asyncio.Lock()
170
 
171
  async def start(self) -> None:
172
- executable_path = ensure_binary()
173
- stealth_args = get_default_stealth_args()
174
 
175
  chrome_args = [
176
  executable_path,
 
7
  from __future__ import annotations
8
 
9
  import asyncio
10
+ import glob
11
  import json
12
  import logging
13
+ import os
14
+ import random
15
  import subprocess
16
  import urllib.request
17
  from collections.abc import AsyncIterator, Callable
18
  from contextlib import asynccontextmanager
19
  from typing import Any
20
 
 
21
  from patchright.async_api import Browser, BrowserContext, Page, async_playwright
22
 
23
  from veilrender.config import settings
 
27
 
28
  CDP_PORT = 9222
29
 
30
+ _CLOAKBROWSER_DEFAULT_DIR = os.path.expanduser("~/.cloakbrowser")
31
+
32
+
33
+ _CLOAKBROWSER_PYPI_URL = "https://pypi.org/pypi/cloakbrowser/json"
34
+
35
+
36
+ def _resolve_chromium_version() -> str:
37
+ """Resolve CloakBrowser's Chromium version from the PyPI wheel."""
38
+ import re
39
+ import zipfile
40
+ from io import BytesIO
41
+
42
+ pypi = _fetch_json(_CLOAKBROWSER_PYPI_URL, timeout=30)
43
+ whl_url = next(u["url"] for u in pypi["urls"] if u["filename"].endswith(".whl"))
44
+ whl_data = urllib.request.urlopen(whl_url, timeout=60).read()
45
+ with zipfile.ZipFile(BytesIO(whl_data)) as zf:
46
+ for name in zf.namelist():
47
+ if name.endswith("config.py"):
48
+ content = zf.read(name).decode()
49
+ m = re.search(r'CHROMIUM_VERSION\s*=\s*"([^"]+)"', content)
50
+ if m:
51
+ return m.group(1)
52
+ raise RuntimeError("Could not resolve CloakBrowser Chromium version")
53
+
54
+
55
+ def _detect_platform() -> str:
56
+ """Detect platform string for CloakBrowser download URL."""
57
+ import platform as _platform
58
+
59
+ machine = _platform.machine().lower()
60
+ system = _platform.system().lower()
61
+ if system == "linux":
62
+ arch = "x64" if machine in ("x86_64", "amd64") else "arm64"
63
+ return f"linux-{arch}"
64
+ if system == "darwin":
65
+ arch = "arm64" if machine == "arm64" else "x64"
66
+ return f"mac-{arch}"
67
+ raise RuntimeError(f"Unsupported platform: {system}-{machine}")
68
+
69
+
70
+ def _download_browser_binary() -> str:
71
+ """Download CloakBrowser binary from cloakbrowser.dev (or mirror)."""
72
+ import tarfile
73
+
74
+ version = _resolve_chromium_version()
75
+ plat = _detect_platform()
76
+ gh_url = f"https://github.com/CloakHQ/cloakbrowser/releases/download/chromium-v{version}/cloakbrowser-{plat}.tar.gz"
77
+ mirror = os.environ.get("CLOAKBROWSER_MIRROR", "")
78
+ url = f"{mirror}/{gh_url}" if mirror else gh_url
79
+ dest_dir = os.path.join(_CLOAKBROWSER_DEFAULT_DIR, f"chromium-{version}")
80
+ binary = os.path.join(dest_dir, "chrome")
81
+
82
+ if os.path.isfile(binary):
83
+ logger.info("Binary already exists at %s", binary)
84
+ return binary
85
+
86
+ logger.info("Downloading CloakBrowser %s (%s)...", version, plat)
87
+ os.makedirs(dest_dir, exist_ok=True)
88
+ tmp = os.path.join(dest_dir, ".download.tar.gz")
89
+ try:
90
+ urllib.request.urlretrieve(url, tmp)
91
+ with tarfile.open(tmp, "r:gz") as tf:
92
+ tf.extractall(dest_dir)
93
+ finally:
94
+ if os.path.exists(tmp):
95
+ os.unlink(tmp)
96
+
97
+ if os.path.isfile(binary):
98
+ os.chmod(binary, 0o755)
99
+ logger.info("Downloaded CloakBrowser %s to %s", version, binary)
100
+ return binary
101
+
102
+
103
+ def _find_browser_binary() -> str:
104
+ """Locate a CloakBrowser (or compatible) Chromium binary.
105
+
106
+ Search order:
107
+ 1. CLOAKBROWSER_BINARY env var
108
+ 2. ~/.cloakbrowser/*/chrome (pre-downloaded or Docker-installed)
109
+ 3. Auto-download from cloakbrowser.dev
110
+ """
111
+ env_path = os.environ.get("CLOAKBROWSER_BINARY")
112
+ if env_path:
113
+ if os.path.isfile(env_path) and os.access(env_path, os.X_OK):
114
+ logger.info("Using browser binary from CLOAKBROWSER_BINARY: %s", env_path)
115
+ return env_path
116
+ raise FileNotFoundError(
117
+ f"CLOAKBROWSER_BINARY={env_path} not found or not executable"
118
+ )
119
+
120
+ candidates = sorted(
121
+ glob.glob(f"{_CLOAKBROWSER_DEFAULT_DIR}/*/chrome"), reverse=True
122
+ )
123
+ if candidates:
124
+ logger.info("Found browser binary: %s", candidates[0])
125
+ return candidates[0]
126
+
127
+ return _download_browser_binary()
128
+
129
+
130
+ def _get_stealth_args() -> list[str]:
131
+ """Get stealth launch arguments for CloakBrowser."""
132
+ fingerprint = random.randint(10000, 99999)
133
+ return [
134
+ "--no-sandbox",
135
+ f"--fingerprint={fingerprint}",
136
+ "--fingerprint-platform=linux",
137
+ ]
138
+
139
 
140
  def _fetch_json(url: str, timeout: int = 2) -> Any:
141
  """Fetch and parse JSON from a URL (blocking)."""
 
280
  self._lock = asyncio.Lock()
281
 
282
  async def start(self) -> None:
283
+ executable_path = _find_browser_binary()
284
+ stealth_args = _get_stealth_args()
285
 
286
  chrome_args = [
287
  executable_path,