File size: 4,954 Bytes
176d459 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | """Quick benchmark: subprocess eval vs supervisor-WS eval.
Runs both paths against the same live Chrome and prints a comparison table.
Not a pytest — a script you run manually for the PR description.
Usage:
.venv/bin/python scripts/benchmark_browser_eval.py [--iterations N]
"""
from __future__ import annotations
import argparse
import shutil
import statistics
import subprocess
import sys
import tempfile
import time
import urllib.request
import json
def _find_chrome() -> str:
for c in ("google-chrome", "chromium", "chromium-browser"):
p = shutil.which(c)
if p:
return p
print("No Chrome binary found.", file=sys.stderr)
sys.exit(1)
def _start_chrome(port: int):
profile = tempfile.mkdtemp(prefix="hermes-bench-eval-")
proc = subprocess.Popen(
[
_find_chrome(),
f"--remote-debugging-port={port}",
f"--user-data-dir={profile}",
"--no-first-run",
"--no-default-browser-check",
"--headless=new",
"--disable-gpu",
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/version", timeout=1) as r:
info = json.loads(r.read().decode())
return proc, profile, info["webSocketDebuggerUrl"]
except Exception:
time.sleep(0.25)
proc.terminate()
raise RuntimeError("Chrome didn't expose CDP")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--iterations", type=int, default=50)
parser.add_argument("--port", type=int, default=9333)
args = parser.parse_args()
proc, profile, cdp_url = _start_chrome(args.port)
try:
from tools.browser_supervisor import SUPERVISOR_REGISTRY
# Warm up: start the supervisor, navigate to a page.
supervisor = SUPERVISOR_REGISTRY.get_or_start(
task_id="bench-eval", cdp_url=cdp_url
)
# Give it a moment to attach.
time.sleep(1.0)
# Sanity check: one eval over WS should succeed.
sanity = supervisor.evaluate_runtime("1 + 1")
if not sanity.get("ok") or sanity.get("result") != 2:
print(f"sanity check failed: {sanity}", file=sys.stderr)
sys.exit(2)
# ── Bench 1: supervisor WS path ──────────────────────────────────
ws_times: list[float] = []
for _ in range(args.iterations):
t0 = time.monotonic()
out = supervisor.evaluate_runtime("1 + 1")
t1 = time.monotonic()
assert out.get("ok"), out
ws_times.append((t1 - t0) * 1000)
# ── Bench 2: agent-browser subprocess path ────────────────────────
# Skip if agent-browser isn't installed — the WS bench still tells
# us what we need.
if shutil.which("agent-browser") is None and shutil.which("npx") is None:
print("agent-browser CLI not found — skipping subprocess bench.")
sub_times = []
else:
from tools.browser_tool import _run_browser_command, _last_session_key
task_id = _last_session_key("bench-eval")
sub_times = []
for _ in range(args.iterations):
t0 = time.monotonic()
_run_browser_command(task_id, "eval", ["1 + 1"])
t1 = time.monotonic()
sub_times.append((t1 - t0) * 1000)
def fmt(name: str, ts: list[float]) -> str:
if not ts:
return f" {name:<40} (skipped)"
mean = statistics.mean(ts)
median = statistics.median(ts)
mn, mx = min(ts), max(ts)
return (
f" {name:<40} mean={mean:>7.2f}ms median={median:>7.2f}ms "
f"min={mn:>7.2f}ms max={mx:>7.2f}ms"
)
print()
print(f"browser_eval benchmark — {args.iterations} iterations of `1 + 1`")
print("-" * 90)
print(fmt("supervisor WS (Runtime.evaluate)", ws_times))
print(fmt("agent-browser subprocess (eval)", sub_times))
if ws_times and sub_times:
speedup = statistics.mean(sub_times) / statistics.mean(ws_times)
print()
print(f"Speedup: {speedup:.1f}x (mean)")
finally:
SUPERVISOR_REGISTRY.stop_all()
proc.terminate()
try:
proc.wait(timeout=3)
except Exception:
proc.kill()
shutil.rmtree(profile, ignore_errors=True)
if __name__ == "__main__":
main()
|