Peng Ding commited on
Commit
9aca48e
·
1 Parent(s): 36f221a

feat: add remote browser worker pool support (#24)

Browse files

Extract LocalWorker/RemoteWorker from BrowserManager to support
horizontal scaling of browser instances.

When VEILRENDER_WORKERS is not set, behavior is identical to before
(single local Chromium). When set to comma-separated CDP endpoints,
requests are distributed across remote workers via least-connections
routing.

- LocalWorker: spawns local Chromium process (existing behavior)
- RemoteWorker: connects to external CDP endpoint
- BrowserManager: pool coordinator with least-connections routing
- Health loop: periodic checks, automatic reconnection for remote workers
- CDP proxy: ?worker=N query param for targeted routing
- Per-worker Prometheus gauges (healthy, active_pages)
- Dashboard capacity derived from total pool capacity
- New config: VEILRENDER_WORKERS, VEILRENDER_WORKER_MAX_CONCURRENT,
VEILRENDER_WORKER_HEALTH_INTERVAL

Closes #24

src/veilrender/app.py CHANGED
@@ -102,13 +102,24 @@ def main() -> None:
102
 
103
  # Check if this is a WebSocket upgrade to /cdp
104
  if is_websocket_upgrade(method, headers, path):
 
 
 
 
 
 
 
 
 
 
 
105
  await handle_cdp_upgrade(
106
  reader,
107
  writer,
108
  headers,
109
  path,
110
  query_string,
111
- browser_manager.get_cdp_url,
112
  )
113
  return
114
 
 
102
 
103
  # Check if this is a WebSocket upgrade to /cdp
104
  if is_websocket_upgrade(method, headers, path):
105
+ worker_idx: int | None = None
106
+ for part in query_string.split("&"):
107
+ if part.startswith("worker="):
108
+ try:
109
+ worker_idx = int(part.split("=", 1)[1])
110
+ except ValueError:
111
+ pass
112
+
113
+ async def _get_cdp_url() -> str | None:
114
+ return await browser_manager.get_cdp_url(worker_idx)
115
+
116
  await handle_cdp_upgrade(
117
  reader,
118
  writer,
119
  headers,
120
  path,
121
  query_string,
122
+ _get_cdp_url,
123
  )
124
  return
125
 
src/veilrender/browser.py CHANGED
@@ -1,12 +1,18 @@
1
- """Playwright browser lifecycle management."""
 
 
 
 
2
 
3
  from __future__ import annotations
4
 
5
  import asyncio
 
6
  import logging
7
  import subprocess
 
 
8
  from contextlib import asynccontextmanager
9
- from collections.abc import AsyncIterator
10
 
11
  from cloakbrowser import ensure_binary, get_default_stealth_args
12
  from playwright.async_api import Browser, BrowserContext, Page, async_playwright
@@ -19,39 +25,98 @@ logger = logging.getLogger(__name__)
19
  CDP_PORT = 9222
20
 
21
 
22
- class BrowserManager:
23
- """Manages a shared Playwright browser instance.
 
24
 
25
- Launches Chromium directly (not via Playwright's launch()) so that
26
- ``--remote-debugging-port`` actually takes effect. Playwright then
27
- connects over CDP, and the CDP port is also available for external
28
- clients via the WebSocket proxy.
29
- """
30
 
31
- def __init__(self) -> None:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  self._playwright = None
33
  self._browser: Browser | None = None
34
  self._chrome_proc: subprocess.Popen | None = None # type: ignore[type-arg]
35
- self._semaphore = asyncio.Semaphore(settings.max_concurrent)
36
  self._lock = asyncio.Lock()
37
 
38
- # Load blocklist once at init if resource filtering is enabled
39
- if settings.resource_filter:
40
- self._blocklist = load_blocklist(settings.blocked_domains_extra)
41
- self._route_handler = make_route_handler(self._blocklist)
42
- else:
43
- self._blocklist = frozenset()
44
- self._route_handler = None
45
-
46
  async def start(self) -> None:
47
- """Launch Chromium with CDP and connect Playwright to it."""
48
  executable_path = ensure_binary()
49
  stealth_args = get_default_stealth_args()
50
 
51
  chrome_args = [
52
  executable_path,
53
  "--headless",
54
- f"--remote-debugging-port={CDP_PORT}",
55
  "--no-first-run",
56
  "--no-default-browser-check",
57
  "--disable-setuid-sandbox",
@@ -67,12 +132,9 @@ class BrowserManager:
67
  stderr=subprocess.PIPE,
68
  )
69
 
70
- # Wait for CDP to be ready
71
- cdp_url = f"http://127.0.0.1:{CDP_PORT}"
72
- for attempt in range(30):
73
  try:
74
- import urllib.request
75
-
76
  urllib.request.urlopen(f"{cdp_url}/json/version", timeout=1)
77
  break
78
  except Exception:
@@ -87,35 +149,17 @@ class BrowserManager:
87
  f"Chromium CDP not ready after 15s. stderr: {stderr[:500]}"
88
  )
89
 
90
- # Connect Playwright over CDP
91
  self._playwright = await async_playwright().start()
92
- self._browser = await self._playwright.chromium.connect_over_cdp(
93
- f"http://127.0.0.1:{CDP_PORT}"
94
- )
95
  logger.info(
96
- "Browser started (CloakBrowser %s, CDP on :%d)", executable_path, CDP_PORT
 
 
97
  )
98
 
99
- async def get_cdp_url(self) -> str | None:
100
- """Return the internal CDP WebSocket URL, or None if unavailable."""
101
- await self._ensure_browser()
102
- try:
103
- import urllib.request
104
- import json
105
-
106
- resp = urllib.request.urlopen(
107
- f"http://127.0.0.1:{CDP_PORT}/json/version", timeout=2
108
- )
109
- data = json.loads(resp.read())
110
- ws_url = data.get("webSocketDebuggerUrl")
111
- if ws_url:
112
- return ws_url
113
- except Exception:
114
- logger.debug("Failed to get CDP WebSocket URL", exc_info=True)
115
- return f"ws://127.0.0.1:{CDP_PORT}"
116
-
117
  async def stop(self) -> None:
118
- """Close the browser, Playwright, and Chromium process."""
119
  if self._browser:
120
  try:
121
  await self._browser.close()
@@ -132,31 +176,219 @@ class BrowserManager:
132
  except subprocess.TimeoutExpired:
133
  self._chrome_proc.kill()
134
  self._chrome_proc = None
135
- logger.info("Browser stopped")
136
 
137
- async def _ensure_browser(self) -> Browser:
138
- """Restart browser if it crashed."""
139
  async with self._lock:
140
  chrome_dead = (
141
  self._chrome_proc is None or self._chrome_proc.poll() is not None
142
  )
143
  browser_dead = self._browser is None or not self._browser.is_connected()
144
  if chrome_dead or browser_dead:
145
- logger.warning("Browser not connected, restarting...")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
146
  await self.stop()
147
  await self.start()
148
  assert self._browser is not None
149
  return self._browser
150
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
151
  @property
152
  def active_pages(self) -> int:
153
- """Number of browser pages currently in use."""
154
- return settings.max_concurrent - self._semaphore._value
155
 
156
  @property
157
  def is_browser_alive(self) -> bool:
158
- """Whether the browser process is connected."""
159
- return self._browser is not None and self._browser.is_connected()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
160
 
161
  @asynccontextmanager
162
  async def get_page(
@@ -165,29 +397,13 @@ class BrowserManager:
165
  viewport_width: int | None = None,
166
  viewport_height: int | None = None,
167
  ) -> AsyncIterator[tuple[BrowserContext, Page]]:
168
- """Create an isolated browser context and page.
169
-
170
- Yields:
171
- A (context, page) tuple. Both are closed automatically.
172
- """
173
- async with self._semaphore:
174
- browser = await self._ensure_browser()
175
- context: BrowserContext | None = None
176
- try:
177
- context = await browser.new_context(
178
- viewport={
179
- "width": viewport_width or settings.viewport_width,
180
- "height": viewport_height or settings.viewport_height,
181
- },
182
- user_agent=None, # use Playwright default
183
- )
184
- page = await context.new_page()
185
- if self._route_handler:
186
- await page.route("**/*", self._route_handler)
187
- yield context, page
188
- finally:
189
- if context:
190
- await context.close()
191
 
192
 
193
  browser_manager = BrowserManager()
 
1
+ """Playwright browser lifecycle management.
2
+
3
+ Supports a single local browser (default) or a pool of remote CDP
4
+ workers when ``VEILRENDER_WORKERS`` is configured.
5
+ """
6
 
7
  from __future__ import annotations
8
 
9
  import asyncio
10
+ import json
11
  import logging
12
  import subprocess
13
+ import urllib.request
14
+ from collections.abc import AsyncIterator, Callable
15
  from contextlib import asynccontextmanager
 
16
 
17
  from cloakbrowser import ensure_binary, get_default_stealth_args
18
  from playwright.async_api import Browser, BrowserContext, Page, async_playwright
 
25
  CDP_PORT = 9222
26
 
27
 
28
+ # ---------------------------------------------------------------------------
29
+ # Worker base interface
30
+ # ---------------------------------------------------------------------------
31
 
 
 
 
 
 
32
 
33
+ class _BaseWorker:
34
+ """Common interface for local and remote browser workers."""
35
+
36
+ endpoint: str = "unknown"
37
+
38
+ def __init__(self, max_concurrent: int) -> None:
39
+ self.max_concurrent = max_concurrent
40
+ self._semaphore = asyncio.Semaphore(max_concurrent)
41
+ self.healthy = False
42
+
43
+ async def start(self) -> None:
44
+ raise NotImplementedError
45
+
46
+ async def stop(self) -> None:
47
+ raise NotImplementedError
48
+
49
+ async def ensure_ready(self) -> Browser:
50
+ raise NotImplementedError
51
+
52
+ async def get_cdp_url(self) -> str | None:
53
+ raise NotImplementedError
54
+
55
+ @property
56
+ def is_alive(self) -> bool:
57
+ raise NotImplementedError
58
+
59
+ @property
60
+ def active(self) -> int:
61
+ return self.max_concurrent - self._semaphore._value
62
+
63
+ @property
64
+ def available(self) -> int:
65
+ return self._semaphore._value
66
+
67
+ @asynccontextmanager
68
+ async def get_page(
69
+ self,
70
+ *,
71
+ viewport_width: int | None = None,
72
+ viewport_height: int | None = None,
73
+ route_handler: Callable | None = None,
74
+ ) -> AsyncIterator[tuple[BrowserContext, Page]]:
75
+ async with self._semaphore:
76
+ browser = await self.ensure_ready()
77
+ context: BrowserContext | None = None
78
+ try:
79
+ context = await browser.new_context(
80
+ viewport={
81
+ "width": viewport_width or settings.viewport_width,
82
+ "height": viewport_height or settings.viewport_height,
83
+ },
84
+ user_agent=None,
85
+ )
86
+ page = await context.new_page()
87
+ if route_handler:
88
+ await page.route("**/*", route_handler)
89
+ yield context, page
90
+ finally:
91
+ if context:
92
+ await context.close()
93
+
94
+
95
+ # ---------------------------------------------------------------------------
96
+ # Local worker — spawns a Chromium process on this machine
97
+ # ---------------------------------------------------------------------------
98
+
99
+
100
+ class LocalWorker(_BaseWorker):
101
+ """Browser worker backed by a local Chromium process."""
102
+
103
+ def __init__(self, cdp_port: int, max_concurrent: int) -> None:
104
+ super().__init__(max_concurrent)
105
+ self.cdp_port = cdp_port
106
+ self.endpoint = "local"
107
  self._playwright = None
108
  self._browser: Browser | None = None
109
  self._chrome_proc: subprocess.Popen | None = None # type: ignore[type-arg]
 
110
  self._lock = asyncio.Lock()
111
 
 
 
 
 
 
 
 
 
112
  async def start(self) -> None:
 
113
  executable_path = ensure_binary()
114
  stealth_args = get_default_stealth_args()
115
 
116
  chrome_args = [
117
  executable_path,
118
  "--headless",
119
+ f"--remote-debugging-port={self.cdp_port}",
120
  "--no-first-run",
121
  "--no-default-browser-check",
122
  "--disable-setuid-sandbox",
 
132
  stderr=subprocess.PIPE,
133
  )
134
 
135
+ cdp_url = f"http://127.0.0.1:{self.cdp_port}"
136
+ for _ in range(30):
 
137
  try:
 
 
138
  urllib.request.urlopen(f"{cdp_url}/json/version", timeout=1)
139
  break
140
  except Exception:
 
149
  f"Chromium CDP not ready after 15s. stderr: {stderr[:500]}"
150
  )
151
 
 
152
  self._playwright = await async_playwright().start()
153
+ self._browser = await self._playwright.chromium.connect_over_cdp(cdp_url)
154
+ self.healthy = True
 
155
  logger.info(
156
+ "Local browser started (CloakBrowser %s, CDP on :%d)",
157
+ executable_path,
158
+ self.cdp_port,
159
  )
160
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
161
  async def stop(self) -> None:
162
+ self.healthy = False
163
  if self._browser:
164
  try:
165
  await self._browser.close()
 
176
  except subprocess.TimeoutExpired:
177
  self._chrome_proc.kill()
178
  self._chrome_proc = None
 
179
 
180
+ async def ensure_ready(self) -> Browser:
 
181
  async with self._lock:
182
  chrome_dead = (
183
  self._chrome_proc is None or self._chrome_proc.poll() is not None
184
  )
185
  browser_dead = self._browser is None or not self._browser.is_connected()
186
  if chrome_dead or browser_dead:
187
+ logger.warning("Local browser not connected, restarting...")
188
+ await self.stop()
189
+ await self.start()
190
+ assert self._browser is not None
191
+ return self._browser
192
+
193
+ async def get_cdp_url(self) -> str | None:
194
+ await self.ensure_ready()
195
+ try:
196
+ resp = urllib.request.urlopen(
197
+ f"http://127.0.0.1:{self.cdp_port}/json/version", timeout=2
198
+ )
199
+ data = json.loads(resp.read())
200
+ ws_url = data.get("webSocketDebuggerUrl")
201
+ if ws_url:
202
+ return ws_url
203
+ except Exception:
204
+ logger.debug("Failed to get CDP WebSocket URL", exc_info=True)
205
+ return f"ws://127.0.0.1:{self.cdp_port}"
206
+
207
+ @property
208
+ def is_alive(self) -> bool:
209
+ return self._browser is not None and self._browser.is_connected()
210
+
211
+
212
+ # ---------------------------------------------------------------------------
213
+ # Remote worker — connects to an external CDP endpoint
214
+ # ---------------------------------------------------------------------------
215
+
216
+
217
+ class RemoteWorker(_BaseWorker):
218
+ """Browser worker backed by a remote CDP endpoint."""
219
+
220
+ def __init__(self, endpoint: str, max_concurrent: int) -> None:
221
+ super().__init__(max_concurrent)
222
+ self.endpoint = endpoint
223
+ self._playwright = None
224
+ self._browser: Browser | None = None
225
+ self._lock = asyncio.Lock()
226
+
227
+ async def start(self) -> None:
228
+ self._playwright = await async_playwright().start()
229
+ self._browser = await self._playwright.chromium.connect_over_cdp(self.endpoint)
230
+ self.healthy = True
231
+ logger.info("Connected to remote browser at %s", self.endpoint)
232
+
233
+ async def stop(self) -> None:
234
+ self.healthy = False
235
+ if self._browser:
236
+ try:
237
+ await self._browser.close()
238
+ except Exception:
239
+ pass
240
+ self._browser = None
241
+ if self._playwright:
242
+ await self._playwright.stop()
243
+ self._playwright = None
244
+
245
+ async def ensure_ready(self) -> Browser:
246
+ async with self._lock:
247
+ if self._browser is None or not self._browser.is_connected():
248
+ logger.warning(
249
+ "Remote browser %s disconnected, reconnecting...", self.endpoint
250
+ )
251
  await self.stop()
252
  await self.start()
253
  assert self._browser is not None
254
  return self._browser
255
 
256
+ async def get_cdp_url(self) -> str | None:
257
+ await self.ensure_ready()
258
+ http_endpoint = self.endpoint.replace("ws://", "http://", 1)
259
+ try:
260
+ resp = urllib.request.urlopen(f"{http_endpoint}/json/version", timeout=2)
261
+ data = json.loads(resp.read())
262
+ ws_url = data.get("webSocketDebuggerUrl")
263
+ if ws_url:
264
+ return ws_url
265
+ except Exception:
266
+ logger.debug("Failed to get CDP URL for %s", self.endpoint, exc_info=True)
267
+ return self.endpoint if self.endpoint.startswith("ws://") else None
268
+
269
+ @property
270
+ def is_alive(self) -> bool:
271
+ return self._browser is not None and self._browser.is_connected()
272
+
273
+
274
+ # ---------------------------------------------------------------------------
275
+ # BrowserManager — pool coordinator
276
+ # ---------------------------------------------------------------------------
277
+
278
+
279
+ class BrowserManager:
280
+ """Manages browser workers — local or remote pool.
281
+
282
+ When ``VEILRENDER_WORKERS`` is not set, spawns a single local
283
+ Chromium process (identical to pre-pool behavior). When set,
284
+ connects to remote CDP endpoints and distributes load.
285
+ """
286
+
287
+ def __init__(self) -> None:
288
+ if settings.workers:
289
+ self._workers: list[_BaseWorker] = [
290
+ RemoteWorker(url, settings.worker_max_concurrent)
291
+ for url in settings.workers
292
+ ]
293
+ self._is_local = False
294
+ else:
295
+ self._workers = [LocalWorker(CDP_PORT, settings.max_concurrent)]
296
+ self._is_local = True
297
+
298
+ self._health_task: asyncio.Task | None = None # type: ignore[type-arg]
299
+
300
+ if settings.resource_filter:
301
+ self._blocklist = load_blocklist(settings.blocked_domains_extra)
302
+ self._route_handler = make_route_handler(self._blocklist)
303
+ else:
304
+ self._blocklist = frozenset()
305
+ self._route_handler = None
306
+
307
+ async def start(self) -> None:
308
+ for w in self._workers:
309
+ try:
310
+ await w.start()
311
+ except Exception:
312
+ logger.error("Failed to start worker %s", w.endpoint, exc_info=True)
313
+ w.healthy = False
314
+ if not self._is_local:
315
+ self._health_task = asyncio.create_task(self._health_loop())
316
+
317
+ async def stop(self) -> None:
318
+ if self._health_task:
319
+ self._health_task.cancel()
320
+ self._health_task = None
321
+ for w in self._workers:
322
+ await w.stop()
323
+ logger.info("Browser stopped")
324
+
325
+ async def _health_loop(self) -> None:
326
+ interval = settings.worker_health_interval
327
+ while True:
328
+ await asyncio.sleep(interval)
329
+ for w in self._workers:
330
+ try:
331
+ alive = w.is_alive
332
+ if not alive and w.healthy:
333
+ logger.warning("Worker %s went unhealthy", w.endpoint)
334
+ w.healthy = False
335
+ elif not alive and not w.healthy:
336
+ logger.info("Attempting reconnect to %s", w.endpoint)
337
+ try:
338
+ await w.stop()
339
+ await w.start()
340
+ except Exception:
341
+ logger.debug("Reconnect failed for %s", w.endpoint)
342
+ elif alive and not w.healthy:
343
+ w.healthy = True
344
+ logger.info("Worker %s recovered", w.endpoint)
345
+ except Exception:
346
+ logger.debug("Health check error for %s", w.endpoint, exc_info=True)
347
+
348
+ def _pick_worker(self) -> _BaseWorker:
349
+ healthy = [w for w in self._workers if w.healthy]
350
+ if not healthy:
351
+ raise RuntimeError("No healthy browser workers available")
352
+ return max(healthy, key=lambda w: w.available)
353
+
354
  @property
355
  def active_pages(self) -> int:
356
+ return sum(w.active for w in self._workers)
 
357
 
358
  @property
359
  def is_browser_alive(self) -> bool:
360
+ return any(w.is_alive for w in self._workers)
361
+
362
+ @property
363
+ def total_capacity(self) -> int:
364
+ return sum(w.max_concurrent for w in self._workers if w.healthy)
365
+
366
+ def worker_stats(self) -> list[dict]:
367
+ result = []
368
+ for i, w in enumerate(self._workers):
369
+ result.append(
370
+ {
371
+ "index": i,
372
+ "endpoint": w.endpoint,
373
+ "healthy": w.healthy,
374
+ "active": w.active,
375
+ "max_concurrent": w.max_concurrent,
376
+ "is_alive": w.is_alive,
377
+ }
378
+ )
379
+ return result
380
+
381
+ async def get_cdp_url(self, worker_index: int | None = None) -> str | None:
382
+ if worker_index is not None and 0 <= worker_index < len(self._workers):
383
+ w = self._workers[worker_index]
384
+ if w.healthy:
385
+ return await w.get_cdp_url()
386
+ return None
387
+ try:
388
+ w = self._pick_worker()
389
+ return await w.get_cdp_url()
390
+ except RuntimeError:
391
+ return None
392
 
393
  @asynccontextmanager
394
  async def get_page(
 
397
  viewport_width: int | None = None,
398
  viewport_height: int | None = None,
399
  ) -> AsyncIterator[tuple[BrowserContext, Page]]:
400
+ worker = self._pick_worker()
401
+ async with worker.get_page(
402
+ viewport_width=viewport_width,
403
+ viewport_height=viewport_height,
404
+ route_handler=self._route_handler,
405
+ ) as result:
406
+ yield result
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
407
 
408
 
409
  browser_manager = BrowserManager()
src/veilrender/config.py CHANGED
@@ -20,6 +20,21 @@ class Settings:
20
  os.environ.get("VEILRENDER_VIEWPORT_HEIGHT", "720")
21
  )
22
  self.max_concurrent: int = int(os.environ.get("VEILRENDER_MAX_CONCURRENT", "5"))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  self.resource_filter: bool = (
24
  os.environ.get("VEILRENDER_RESOURCE_FILTER", "true").lower() == "true"
25
  )
 
20
  os.environ.get("VEILRENDER_VIEWPORT_HEIGHT", "720")
21
  )
22
  self.max_concurrent: int = int(os.environ.get("VEILRENDER_MAX_CONCURRENT", "5"))
23
+
24
+ # Remote browser worker pool
25
+ _workers_raw = os.environ.get("VEILRENDER_WORKERS", "")
26
+ self.workers: list[str] = (
27
+ [w.strip() for w in _workers_raw.split(",") if w.strip()]
28
+ if _workers_raw
29
+ else []
30
+ )
31
+ self.worker_max_concurrent: int = int(
32
+ os.environ.get("VEILRENDER_WORKER_MAX_CONCURRENT", str(self.max_concurrent))
33
+ )
34
+ self.worker_health_interval: int = int(
35
+ os.environ.get("VEILRENDER_WORKER_HEALTH_INTERVAL", "10")
36
+ )
37
+
38
  self.resource_filter: bool = (
39
  os.environ.get("VEILRENDER_RESOURCE_FILTER", "true").lower() == "true"
40
  )
src/veilrender/routes/dashboard.py CHANGED
@@ -8,7 +8,6 @@ import math
8
  from veilrender import stats
9
  from veilrender._vendor.httpserver import App, Request, Response
10
  from veilrender.browser import browser_manager
11
- from veilrender.config import settings
12
 
13
  _BADGE_STYLE = "style=flat&amp;labelColor=1a1d1d&amp;color=2d3d32"
14
 
@@ -32,7 +31,7 @@ _RING_CIRCUMFERENCE = 2 * math.pi * _RING_RADIUS
32
  def _stats_json() -> dict:
33
  """Collect all dashboard data as a dict."""
34
  active = browser_manager.active_pages
35
- max_conc = settings.max_concurrent
36
  return {
37
  "browser_alive": browser_manager.is_browser_alive,
38
  "uptime": stats.format_uptime(),
@@ -46,6 +45,7 @@ def _stats_json() -> dict:
46
  "/render": _ep_dict(stats.render),
47
  "/screenshot": _ep_dict(stats.screenshot),
48
  },
 
49
  }
50
 
51
 
 
8
  from veilrender import stats
9
  from veilrender._vendor.httpserver import App, Request, Response
10
  from veilrender.browser import browser_manager
 
11
 
12
  _BADGE_STYLE = "style=flat&amp;labelColor=1a1d1d&amp;color=2d3d32"
13
 
 
31
  def _stats_json() -> dict:
32
  """Collect all dashboard data as a dict."""
33
  active = browser_manager.active_pages
34
+ max_conc = browser_manager.total_capacity
35
  return {
36
  "browser_alive": browser_manager.is_browser_alive,
37
  "uptime": stats.format_uptime(),
 
45
  "/render": _ep_dict(stats.render),
46
  "/screenshot": _ep_dict(stats.screenshot),
47
  },
48
+ "workers": browser_manager.worker_stats(),
49
  }
50
 
51
 
src/veilrender/routes/metrics.py CHANGED
@@ -5,7 +5,6 @@ from __future__ import annotations
5
  from veilrender import stats
6
  from veilrender._vendor.httpserver import App, Request, Response
7
  from veilrender.browser import browser_manager
8
- from veilrender.config import settings
9
 
10
  _CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
11
  _ENDPOINTS = (("/render", stats.render), ("/screenshot", stats.screenshot))
@@ -36,9 +35,34 @@ def _build_metrics() -> str:
36
  parts.append(
37
  "# HELP veilrender_max_concurrent Maximum concurrent page slots.\n"
38
  "# TYPE veilrender_max_concurrent gauge\n"
39
- f"veilrender_max_concurrent {settings.max_concurrent}\n"
40
  )
41
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  # Counters — requests
43
  lines = [
44
  "# HELP veilrender_requests_total Total requests by endpoint and status.\n",
 
5
  from veilrender import stats
6
  from veilrender._vendor.httpserver import App, Request, Response
7
  from veilrender.browser import browser_manager
 
8
 
9
  _CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
10
  _ENDPOINTS = (("/render", stats.render), ("/screenshot", stats.screenshot))
 
35
  parts.append(
36
  "# HELP veilrender_max_concurrent Maximum concurrent page slots.\n"
37
  "# TYPE veilrender_max_concurrent gauge\n"
38
+ f"veilrender_max_concurrent {browser_manager.total_capacity}\n"
39
  )
40
 
41
+ # Per-worker gauges
42
+ workers = browser_manager.worker_stats()
43
+ if len(workers) > 1 or (workers and workers[0]["endpoint"] != "local"):
44
+ lines = [
45
+ "# HELP veilrender_worker_healthy Whether a worker is healthy.\n",
46
+ "# TYPE veilrender_worker_healthy gauge\n",
47
+ ]
48
+ for ws in workers:
49
+ lines.append(
50
+ f'veilrender_worker_healthy{{worker="{ws["index"]}",endpoint="{ws["endpoint"]}"}} '
51
+ f"{1 if ws['healthy'] else 0}\n"
52
+ )
53
+ parts.append("".join(lines))
54
+
55
+ lines = [
56
+ "# HELP veilrender_worker_active_pages Active pages per worker.\n",
57
+ "# TYPE veilrender_worker_active_pages gauge\n",
58
+ ]
59
+ for ws in workers:
60
+ lines.append(
61
+ f'veilrender_worker_active_pages{{worker="{ws["index"]}",endpoint="{ws["endpoint"]}"}} '
62
+ f"{ws['active']}\n"
63
+ )
64
+ parts.append("".join(lines))
65
+
66
  # Counters — requests
67
  lines = [
68
  "# HELP veilrender_requests_total Total requests by endpoint and status.\n",