Spaces:
Paused
Paused
Peng Ding commited on
Commit ·
fabae1f
1
Parent(s): 9178807
feat: add PlaywrightWorker for Firefox/Camoufox support (#26)
Browse filesSupport mixed Chromium+Firefox worker pools via protocol-aware config:
VEILRENDER_WORKERS=cdp://host1:9222,playwright://host2:1234/ws
- PlaywrightWorker: connects via playwright.firefox.connect() for
Camoufox and other Playwright-compatible browsers
- Protocol prefix parsing in config: cdp:// (default) or playwright://
- Page count via Playwright API (browser.contexts) instead of CDP HTTP
- CDP proxy naturally excludes Firefox workers (get_cdp_url returns None)
- deploy/Dockerfile.camoufox: Camoufox server image
- deploy/compose-pool-mixed.yaml: CloakBrowser + Camoufox example
Closes #26
- deploy/Dockerfile.camoufox +31 -0
- deploy/compose-pool-mixed.yaml +53 -0
- src/veilrender/browser.py +78 -8
- src/veilrender/config.py +26 -2
deploy/Dockerfile.camoufox
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Camoufox server — stealth Firefox browser with Playwright endpoint
|
| 2 |
+
#
|
| 3 |
+
# Exposes a Playwright WebSocket endpoint for VeilRender gateway.
|
| 4 |
+
# Usage: docker build -f deploy/Dockerfile.camoufox -t veilrender-camoufox .
|
| 5 |
+
# docker run -p 1234:1234 veilrender-camoufox
|
| 6 |
+
#
|
| 7 |
+
# Connect from VeilRender:
|
| 8 |
+
# VEILRENDER_WORKERS=playwright://camoufox:1234/ws
|
| 9 |
+
|
| 10 |
+
FROM python:3.12-slim
|
| 11 |
+
|
| 12 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 13 |
+
libgtk-3-0 \
|
| 14 |
+
libdbus-glib-1-2 \
|
| 15 |
+
libxt6 \
|
| 16 |
+
libasound2t64 \
|
| 17 |
+
libx11-xcb1 \
|
| 18 |
+
libxcomposite1 \
|
| 19 |
+
libxdamage1 \
|
| 20 |
+
libxrandr2 \
|
| 21 |
+
fonts-liberation \
|
| 22 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 23 |
+
|
| 24 |
+
RUN pip install --no-cache-dir "camoufox[geoip]"
|
| 25 |
+
RUN python -m camoufox fetch
|
| 26 |
+
|
| 27 |
+
RUN useradd -m -u 1000 user || true
|
| 28 |
+
USER 1000
|
| 29 |
+
|
| 30 |
+
EXPOSE 1234
|
| 31 |
+
CMD ["python", "-m", "camoufox", "server", "--headless", "--port", "1234", "--ws-path", "ws"]
|
deploy/compose-pool-mixed.yaml
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Mixed pool: VeilRender gateway + CloakBrowser (Chromium) + Camoufox (Firefox)
|
| 2 |
+
#
|
| 3 |
+
# Usage:
|
| 4 |
+
# docker compose -f deploy/compose-pool-mixed.yaml up -d
|
| 5 |
+
#
|
| 6 |
+
# This gives dual-engine stealth coverage:
|
| 7 |
+
# - CloakBrowser: stealth Chromium (CDP) via Patchright
|
| 8 |
+
# - Camoufox: stealth Firefox (Playwright protocol)
|
| 9 |
+
|
| 10 |
+
services:
|
| 11 |
+
veilrender:
|
| 12 |
+
image: oaklight/veilrender:gateway
|
| 13 |
+
container_name: veilrender
|
| 14 |
+
restart: unless-stopped
|
| 15 |
+
ports:
|
| 16 |
+
- "7860:7860"
|
| 17 |
+
environment:
|
| 18 |
+
- VEILRENDER_API_TOKEN=${VEILRENDER_API_TOKEN:-changeme}
|
| 19 |
+
- VEILRENDER_RESOURCE_FILTER=true
|
| 20 |
+
- VEILRENDER_WORKERS=cdp://chromium-worker:9222,playwright://camoufox-worker:1234/ws
|
| 21 |
+
- VEILRENDER_WORKER_MAX_CONCURRENT=${VEILRENDER_WORKER_MAX_CONCURRENT:-5}
|
| 22 |
+
- VEILRENDER_WORKER_HEALTH_INTERVAL=${VEILRENDER_WORKER_HEALTH_INTERVAL:-10}
|
| 23 |
+
volumes:
|
| 24 |
+
- veilrender-data:/data
|
| 25 |
+
depends_on:
|
| 26 |
+
- chromium-worker
|
| 27 |
+
- camoufox-worker
|
| 28 |
+
deploy:
|
| 29 |
+
resources:
|
| 30 |
+
limits:
|
| 31 |
+
memory: 512M
|
| 32 |
+
|
| 33 |
+
chromium-worker:
|
| 34 |
+
image: cloakhq/cloakbrowser
|
| 35 |
+
restart: unless-stopped
|
| 36 |
+
command: ["cloakserve"]
|
| 37 |
+
deploy:
|
| 38 |
+
resources:
|
| 39 |
+
limits:
|
| 40 |
+
memory: 1G
|
| 41 |
+
|
| 42 |
+
camoufox-worker:
|
| 43 |
+
build:
|
| 44 |
+
context: .
|
| 45 |
+
dockerfile: deploy/Dockerfile.camoufox
|
| 46 |
+
restart: unless-stopped
|
| 47 |
+
deploy:
|
| 48 |
+
resources:
|
| 49 |
+
limits:
|
| 50 |
+
memory: 1G
|
| 51 |
+
|
| 52 |
+
volumes:
|
| 53 |
+
veilrender-data:
|
src/veilrender/browser.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
-
"""
|
| 2 |
|
| 3 |
-
Supports a single local browser (default) or a pool of remote
|
| 4 |
-
|
| 5 |
"""
|
| 6 |
|
| 7 |
from __future__ import annotations
|
|
@@ -459,6 +459,70 @@ class RemoteWorker(_BaseWorker):
|
|
| 459 |
return self._browser is not None and self._browser.is_connected()
|
| 460 |
|
| 461 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 462 |
# ---------------------------------------------------------------------------
|
| 463 |
# BrowserManager — pool coordinator
|
| 464 |
# ---------------------------------------------------------------------------
|
|
@@ -469,15 +533,21 @@ class BrowserManager:
|
|
| 469 |
|
| 470 |
When ``VEILRENDER_WORKERS`` is not set, spawns a single local
|
| 471 |
Chromium process (identical to pre-pool behavior). When set,
|
| 472 |
-
connects to remote
|
| 473 |
"""
|
| 474 |
|
| 475 |
def __init__(self) -> None:
|
| 476 |
if settings.workers:
|
| 477 |
-
self._workers: list[_BaseWorker] = [
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 481 |
self._is_local = False
|
| 482 |
else:
|
| 483 |
self._workers = [LocalWorker(CDP_PORT, settings.max_concurrent)]
|
|
|
|
| 1 |
+
"""Browser lifecycle management.
|
| 2 |
|
| 3 |
+
Supports a single local browser (default) or a pool of remote workers
|
| 4 |
+
via CDP (Chromium) or Playwright protocol (Firefox/Camoufox).
|
| 5 |
"""
|
| 6 |
|
| 7 |
from __future__ import annotations
|
|
|
|
| 459 |
return self._browser is not None and self._browser.is_connected()
|
| 460 |
|
| 461 |
|
| 462 |
+
# ---------------------------------------------------------------------------
|
| 463 |
+
# Playwright-protocol worker — for Firefox/Camoufox
|
| 464 |
+
# ---------------------------------------------------------------------------
|
| 465 |
+
|
| 466 |
+
|
| 467 |
+
class PlaywrightWorker(_BaseWorker):
|
| 468 |
+
"""Browser worker connected via Playwright protocol (Firefox/Camoufox)."""
|
| 469 |
+
|
| 470 |
+
def __init__(self, endpoint: str, max_concurrent: int) -> None:
|
| 471 |
+
super().__init__(max_concurrent)
|
| 472 |
+
self.endpoint = endpoint
|
| 473 |
+
self._playwright = None
|
| 474 |
+
self._browser: Browser | None = None
|
| 475 |
+
self._lock = asyncio.Lock()
|
| 476 |
+
|
| 477 |
+
async def start(self) -> None:
|
| 478 |
+
self._playwright = await async_playwright().start()
|
| 479 |
+
self._browser = await self._playwright.firefox.connect(self.endpoint)
|
| 480 |
+
ctx = await self._browser.new_context()
|
| 481 |
+
page = await ctx.new_page()
|
| 482 |
+
await page.close()
|
| 483 |
+
await ctx.close()
|
| 484 |
+
self.healthy = True
|
| 485 |
+
logger.info(
|
| 486 |
+
"Connected to Playwright/Firefox browser at %s (verified)", self.endpoint
|
| 487 |
+
)
|
| 488 |
+
|
| 489 |
+
async def stop(self) -> None:
|
| 490 |
+
self.healthy = False
|
| 491 |
+
if self._browser:
|
| 492 |
+
try:
|
| 493 |
+
await self._browser.close()
|
| 494 |
+
except Exception:
|
| 495 |
+
pass
|
| 496 |
+
self._browser = None
|
| 497 |
+
if self._playwright:
|
| 498 |
+
await self._playwright.stop()
|
| 499 |
+
self._playwright = None
|
| 500 |
+
|
| 501 |
+
async def ensure_ready(self) -> Browser:
|
| 502 |
+
async with self._lock:
|
| 503 |
+
if self._browser is None or not self._browser.is_connected():
|
| 504 |
+
logger.warning(
|
| 505 |
+
"Playwright browser %s disconnected, reconnecting...",
|
| 506 |
+
self.endpoint,
|
| 507 |
+
)
|
| 508 |
+
await self.stop()
|
| 509 |
+
await self.start()
|
| 510 |
+
assert self._browser is not None
|
| 511 |
+
return self._browser
|
| 512 |
+
|
| 513 |
+
async def get_cdp_url(self) -> str | None:
|
| 514 |
+
return None
|
| 515 |
+
|
| 516 |
+
async def browser_page_count(self) -> int:
|
| 517 |
+
if self._browser and self._browser.is_connected():
|
| 518 |
+
return sum(len(c.pages) for c in self._browser.contexts)
|
| 519 |
+
return -1
|
| 520 |
+
|
| 521 |
+
@property
|
| 522 |
+
def is_alive(self) -> bool:
|
| 523 |
+
return self._browser is not None and self._browser.is_connected()
|
| 524 |
+
|
| 525 |
+
|
| 526 |
# ---------------------------------------------------------------------------
|
| 527 |
# BrowserManager — pool coordinator
|
| 528 |
# ---------------------------------------------------------------------------
|
|
|
|
| 533 |
|
| 534 |
When ``VEILRENDER_WORKERS`` is not set, spawns a single local
|
| 535 |
Chromium process (identical to pre-pool behavior). When set,
|
| 536 |
+
connects to remote workers via CDP or Playwright protocol.
|
| 537 |
"""
|
| 538 |
|
| 539 |
def __init__(self) -> None:
|
| 540 |
if settings.workers:
|
| 541 |
+
self._workers: list[_BaseWorker] = []
|
| 542 |
+
for protocol, endpoint in settings.workers:
|
| 543 |
+
if protocol == "playwright":
|
| 544 |
+
self._workers.append(
|
| 545 |
+
PlaywrightWorker(endpoint, settings.worker_max_concurrent)
|
| 546 |
+
)
|
| 547 |
+
else:
|
| 548 |
+
self._workers.append(
|
| 549 |
+
RemoteWorker(endpoint, settings.worker_max_concurrent)
|
| 550 |
+
)
|
| 551 |
self._is_local = False
|
| 552 |
else:
|
| 553 |
self._workers = [LocalWorker(CDP_PORT, settings.max_concurrent)]
|
src/veilrender/config.py
CHANGED
|
@@ -5,6 +5,24 @@ from __future__ import annotations
|
|
| 5 |
import os
|
| 6 |
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
class Settings:
|
| 9 |
"""Configuration from ``VEILRENDER_*`` environment variables."""
|
| 10 |
|
|
@@ -22,9 +40,15 @@ class Settings:
|
|
| 22 |
self.max_concurrent: int = int(os.environ.get("VEILRENDER_MAX_CONCURRENT", "5"))
|
| 23 |
|
| 24 |
# Remote browser worker pool
|
|
|
|
|
|
|
| 25 |
_workers_raw = os.environ.get("VEILRENDER_WORKERS", "")
|
| 26 |
-
self.workers: list[str] = (
|
| 27 |
-
[
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
if _workers_raw
|
| 29 |
else []
|
| 30 |
)
|
|
|
|
| 5 |
import os
|
| 6 |
|
| 7 |
|
| 8 |
+
def _parse_worker_entry(entry: str) -> tuple[str, str]:
|
| 9 |
+
"""Parse a worker entry into (protocol, endpoint).
|
| 10 |
+
|
| 11 |
+
Supported formats:
|
| 12 |
+
- ``cdp://host:9222`` → ("cdp", "http://host:9222")
|
| 13 |
+
- ``playwright://host:1234/path`` → ("playwright", "ws://host:1234/path")
|
| 14 |
+
- ``http://host:9222`` → ("cdp", "http://host:9222")
|
| 15 |
+
- ``host:9222`` → ("cdp", "http://host:9222")
|
| 16 |
+
"""
|
| 17 |
+
if entry.startswith("playwright://"):
|
| 18 |
+
return ("playwright", "ws://" + entry[len("playwright://") :])
|
| 19 |
+
if entry.startswith("cdp://"):
|
| 20 |
+
return ("cdp", "http://" + entry[len("cdp://") :])
|
| 21 |
+
if entry.startswith(("http://", "https://", "ws://", "wss://")):
|
| 22 |
+
return ("cdp", entry)
|
| 23 |
+
return ("cdp", "http://" + entry)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
class Settings:
|
| 27 |
"""Configuration from ``VEILRENDER_*`` environment variables."""
|
| 28 |
|
|
|
|
| 40 |
self.max_concurrent: int = int(os.environ.get("VEILRENDER_MAX_CONCURRENT", "5"))
|
| 41 |
|
| 42 |
# Remote browser worker pool
|
| 43 |
+
# Format: cdp://host:9222,playwright://host:1234/ws-path
|
| 44 |
+
# No prefix or http:// defaults to cdp://
|
| 45 |
_workers_raw = os.environ.get("VEILRENDER_WORKERS", "")
|
| 46 |
+
self.workers: list[tuple[str, str]] = (
|
| 47 |
+
[
|
| 48 |
+
_parse_worker_entry(w.strip())
|
| 49 |
+
for w in _workers_raw.split(",")
|
| 50 |
+
if w.strip()
|
| 51 |
+
]
|
| 52 |
if _workers_raw
|
| 53 |
else []
|
| 54 |
)
|