Peng Ding commited on
Commit
fabae1f
·
1 Parent(s): 9178807

feat: add PlaywrightWorker for Firefox/Camoufox support (#26)

Browse files

Support mixed Chromium+Firefox worker pools via protocol-aware config:
VEILRENDER_WORKERS=cdp://host1:9222,playwright://host2:1234/ws

- PlaywrightWorker: connects via playwright.firefox.connect() for
Camoufox and other Playwright-compatible browsers
- Protocol prefix parsing in config: cdp:// (default) or playwright://
- Page count via Playwright API (browser.contexts) instead of CDP HTTP
- CDP proxy naturally excludes Firefox workers (get_cdp_url returns None)
- deploy/Dockerfile.camoufox: Camoufox server image
- deploy/compose-pool-mixed.yaml: CloakBrowser + Camoufox example

Closes #26

deploy/Dockerfile.camoufox ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Camoufox server — stealth Firefox browser with Playwright endpoint
2
+ #
3
+ # Exposes a Playwright WebSocket endpoint for VeilRender gateway.
4
+ # Usage: docker build -f deploy/Dockerfile.camoufox -t veilrender-camoufox .
5
+ # docker run -p 1234:1234 veilrender-camoufox
6
+ #
7
+ # Connect from VeilRender:
8
+ # VEILRENDER_WORKERS=playwright://camoufox:1234/ws
9
+
10
+ FROM python:3.12-slim
11
+
12
+ RUN apt-get update && apt-get install -y --no-install-recommends \
13
+ libgtk-3-0 \
14
+ libdbus-glib-1-2 \
15
+ libxt6 \
16
+ libasound2t64 \
17
+ libx11-xcb1 \
18
+ libxcomposite1 \
19
+ libxdamage1 \
20
+ libxrandr2 \
21
+ fonts-liberation \
22
+ && rm -rf /var/lib/apt/lists/*
23
+
24
+ RUN pip install --no-cache-dir "camoufox[geoip]"
25
+ RUN python -m camoufox fetch
26
+
27
+ RUN useradd -m -u 1000 user || true
28
+ USER 1000
29
+
30
+ EXPOSE 1234
31
+ CMD ["python", "-m", "camoufox", "server", "--headless", "--port", "1234", "--ws-path", "ws"]
deploy/compose-pool-mixed.yaml ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Mixed pool: VeilRender gateway + CloakBrowser (Chromium) + Camoufox (Firefox)
2
+ #
3
+ # Usage:
4
+ # docker compose -f deploy/compose-pool-mixed.yaml up -d
5
+ #
6
+ # This gives dual-engine stealth coverage:
7
+ # - CloakBrowser: stealth Chromium (CDP) via Patchright
8
+ # - Camoufox: stealth Firefox (Playwright protocol)
9
+
10
+ services:
11
+ veilrender:
12
+ image: oaklight/veilrender:gateway
13
+ container_name: veilrender
14
+ restart: unless-stopped
15
+ ports:
16
+ - "7860:7860"
17
+ environment:
18
+ - VEILRENDER_API_TOKEN=${VEILRENDER_API_TOKEN:-changeme}
19
+ - VEILRENDER_RESOURCE_FILTER=true
20
+ - VEILRENDER_WORKERS=cdp://chromium-worker:9222,playwright://camoufox-worker:1234/ws
21
+ - VEILRENDER_WORKER_MAX_CONCURRENT=${VEILRENDER_WORKER_MAX_CONCURRENT:-5}
22
+ - VEILRENDER_WORKER_HEALTH_INTERVAL=${VEILRENDER_WORKER_HEALTH_INTERVAL:-10}
23
+ volumes:
24
+ - veilrender-data:/data
25
+ depends_on:
26
+ - chromium-worker
27
+ - camoufox-worker
28
+ deploy:
29
+ resources:
30
+ limits:
31
+ memory: 512M
32
+
33
+ chromium-worker:
34
+ image: cloakhq/cloakbrowser
35
+ restart: unless-stopped
36
+ command: ["cloakserve"]
37
+ deploy:
38
+ resources:
39
+ limits:
40
+ memory: 1G
41
+
42
+ camoufox-worker:
43
+ build:
44
+ context: .
45
+ dockerfile: deploy/Dockerfile.camoufox
46
+ restart: unless-stopped
47
+ deploy:
48
+ resources:
49
+ limits:
50
+ memory: 1G
51
+
52
+ volumes:
53
+ veilrender-data:
src/veilrender/browser.py CHANGED
@@ -1,7 +1,7 @@
1
- """Playwright browser lifecycle management.
2
 
3
- Supports a single local browser (default) or a pool of remote CDP
4
- workers when ``VEILRENDER_WORKERS`` is configured.
5
  """
6
 
7
  from __future__ import annotations
@@ -459,6 +459,70 @@ class RemoteWorker(_BaseWorker):
459
  return self._browser is not None and self._browser.is_connected()
460
 
461
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
462
  # ---------------------------------------------------------------------------
463
  # BrowserManager — pool coordinator
464
  # ---------------------------------------------------------------------------
@@ -469,15 +533,21 @@ class BrowserManager:
469
 
470
  When ``VEILRENDER_WORKERS`` is not set, spawns a single local
471
  Chromium process (identical to pre-pool behavior). When set,
472
- connects to remote CDP endpoints and distributes load.
473
  """
474
 
475
  def __init__(self) -> None:
476
  if settings.workers:
477
- self._workers: list[_BaseWorker] = [
478
- RemoteWorker(url, settings.worker_max_concurrent)
479
- for url in settings.workers
480
- ]
 
 
 
 
 
 
481
  self._is_local = False
482
  else:
483
  self._workers = [LocalWorker(CDP_PORT, settings.max_concurrent)]
 
1
+ """Browser lifecycle management.
2
 
3
+ Supports a single local browser (default) or a pool of remote workers
4
+ via CDP (Chromium) or Playwright protocol (Firefox/Camoufox).
5
  """
6
 
7
  from __future__ import annotations
 
459
  return self._browser is not None and self._browser.is_connected()
460
 
461
 
462
+ # ---------------------------------------------------------------------------
463
+ # Playwright-protocol worker — for Firefox/Camoufox
464
+ # ---------------------------------------------------------------------------
465
+
466
+
467
+ class PlaywrightWorker(_BaseWorker):
468
+ """Browser worker connected via Playwright protocol (Firefox/Camoufox)."""
469
+
470
+ def __init__(self, endpoint: str, max_concurrent: int) -> None:
471
+ super().__init__(max_concurrent)
472
+ self.endpoint = endpoint
473
+ self._playwright = None
474
+ self._browser: Browser | None = None
475
+ self._lock = asyncio.Lock()
476
+
477
+ async def start(self) -> None:
478
+ self._playwright = await async_playwright().start()
479
+ self._browser = await self._playwright.firefox.connect(self.endpoint)
480
+ ctx = await self._browser.new_context()
481
+ page = await ctx.new_page()
482
+ await page.close()
483
+ await ctx.close()
484
+ self.healthy = True
485
+ logger.info(
486
+ "Connected to Playwright/Firefox browser at %s (verified)", self.endpoint
487
+ )
488
+
489
+ async def stop(self) -> None:
490
+ self.healthy = False
491
+ if self._browser:
492
+ try:
493
+ await self._browser.close()
494
+ except Exception:
495
+ pass
496
+ self._browser = None
497
+ if self._playwright:
498
+ await self._playwright.stop()
499
+ self._playwright = None
500
+
501
+ async def ensure_ready(self) -> Browser:
502
+ async with self._lock:
503
+ if self._browser is None or not self._browser.is_connected():
504
+ logger.warning(
505
+ "Playwright browser %s disconnected, reconnecting...",
506
+ self.endpoint,
507
+ )
508
+ await self.stop()
509
+ await self.start()
510
+ assert self._browser is not None
511
+ return self._browser
512
+
513
+ async def get_cdp_url(self) -> str | None:
514
+ return None
515
+
516
+ async def browser_page_count(self) -> int:
517
+ if self._browser and self._browser.is_connected():
518
+ return sum(len(c.pages) for c in self._browser.contexts)
519
+ return -1
520
+
521
+ @property
522
+ def is_alive(self) -> bool:
523
+ return self._browser is not None and self._browser.is_connected()
524
+
525
+
526
  # ---------------------------------------------------------------------------
527
  # BrowserManager — pool coordinator
528
  # ---------------------------------------------------------------------------
 
533
 
534
  When ``VEILRENDER_WORKERS`` is not set, spawns a single local
535
  Chromium process (identical to pre-pool behavior). When set,
536
+ connects to remote workers via CDP or Playwright protocol.
537
  """
538
 
539
  def __init__(self) -> None:
540
  if settings.workers:
541
+ self._workers: list[_BaseWorker] = []
542
+ for protocol, endpoint in settings.workers:
543
+ if protocol == "playwright":
544
+ self._workers.append(
545
+ PlaywrightWorker(endpoint, settings.worker_max_concurrent)
546
+ )
547
+ else:
548
+ self._workers.append(
549
+ RemoteWorker(endpoint, settings.worker_max_concurrent)
550
+ )
551
  self._is_local = False
552
  else:
553
  self._workers = [LocalWorker(CDP_PORT, settings.max_concurrent)]
src/veilrender/config.py CHANGED
@@ -5,6 +5,24 @@ from __future__ import annotations
5
  import os
6
 
7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  class Settings:
9
  """Configuration from ``VEILRENDER_*`` environment variables."""
10
 
@@ -22,9 +40,15 @@ class Settings:
22
  self.max_concurrent: int = int(os.environ.get("VEILRENDER_MAX_CONCURRENT", "5"))
23
 
24
  # Remote browser worker pool
 
 
25
  _workers_raw = os.environ.get("VEILRENDER_WORKERS", "")
26
- self.workers: list[str] = (
27
- [w.strip() for w in _workers_raw.split(",") if w.strip()]
 
 
 
 
28
  if _workers_raw
29
  else []
30
  )
 
5
  import os
6
 
7
 
8
+ def _parse_worker_entry(entry: str) -> tuple[str, str]:
9
+ """Parse a worker entry into (protocol, endpoint).
10
+
11
+ Supported formats:
12
+ - ``cdp://host:9222`` → ("cdp", "http://host:9222")
13
+ - ``playwright://host:1234/path`` → ("playwright", "ws://host:1234/path")
14
+ - ``http://host:9222`` → ("cdp", "http://host:9222")
15
+ - ``host:9222`` → ("cdp", "http://host:9222")
16
+ """
17
+ if entry.startswith("playwright://"):
18
+ return ("playwright", "ws://" + entry[len("playwright://") :])
19
+ if entry.startswith("cdp://"):
20
+ return ("cdp", "http://" + entry[len("cdp://") :])
21
+ if entry.startswith(("http://", "https://", "ws://", "wss://")):
22
+ return ("cdp", entry)
23
+ return ("cdp", "http://" + entry)
24
+
25
+
26
  class Settings:
27
  """Configuration from ``VEILRENDER_*`` environment variables."""
28
 
 
40
  self.max_concurrent: int = int(os.environ.get("VEILRENDER_MAX_CONCURRENT", "5"))
41
 
42
  # Remote browser worker pool
43
+ # Format: cdp://host:9222,playwright://host:1234/ws-path
44
+ # No prefix or http:// defaults to cdp://
45
  _workers_raw = os.environ.get("VEILRENDER_WORKERS", "")
46
+ self.workers: list[tuple[str, str]] = (
47
+ [
48
+ _parse_worker_entry(w.strip())
49
+ for w in _workers_raw.split(",")
50
+ if w.strip()
51
+ ]
52
  if _workers_raw
53
  else []
54
  )