Spaces:
Running
Running
File size: 3,329 Bytes
0a2e8cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | """Prefill-only benchmark of a running server: unique prompts (so prefix caching cannot hide the cost), single-stream p50 and
throughput at a concurrency. `openjev bench --endpoint http://localhost:3000 --tokens 1400 --conc 32`."""
from __future__ import annotations
import asyncio
import json
import random
import statistics
import time
import httpx
def _request(i: int, tokens: int, rng: random.Random) -> dict:
words = [
"the",
"page",
"shows",
"a",
"form",
"with",
"fields",
"and",
"buttons",
"the",
"user",
"wants",
"to",
"book",
"a",
"flight",
"from",
"SFO",
"to",
"JFK",
"on",
"Oct",
"3",
"previous",
"actions",
"none",
"candidates",
]
body = " ".join(rng.choice(words) + str(rng.randint(0, 99)) for _ in range(max(8, tokens // 2)))
return {
"model": "bench",
"state": {"page": {"title": f"State {i}", "text": body}},
"questions": {
"operation": {
"type": "choice",
"instructions": {"goal": "find flights", "rules": "Which candidate should be acted on next?"},
"criteria": {"From": "input", "To": "input", "Search": "button", "Hotels": "link"},
}
},
}
async def run(endpoint: str, token: str = "", n: int = 120, conc: int = 32, tokens: int = 1400) -> dict:
rng = random.Random(0)
headers = {"Authorization": f"Bearer {token}"} if token else {}
async with httpx.AsyncClient(timeout=180.0, headers=headers, limits=httpx.Limits(max_connections=conc + 4)) as client:
async def one(i):
t = time.perf_counter()
r = await client.post(endpoint.rstrip("/") + "/v1/systemone", json=_request(i, tokens, rng))
r.raise_for_status()
return time.perf_counter() - t, r.json()["usage"]["input_tokens"]
for i in range(3):
await one(-1 - i)
single = [(await one(i))[0] for i in range(10)]
sem = asyncio.Semaphore(conc)
async def bounded(i):
async with sem:
return await one(i)
t0 = time.perf_counter()
res = await asyncio.gather(*(bounded(i) for i in range(n)))
wall = time.perf_counter() - t0
lat = sorted(r[0] for r in res)
return {
"n": n,
"conc": conc,
"prompt_tokens_mean": round(statistics.mean(r[1] for r in res)),
"single_p50_ms": round(statistics.median(single) * 1000),
"loaded_p50_ms": round(statistics.median(lat) * 1000),
"loaded_p90_ms": round(lat[int(0.9 * n)] * 1000),
"req_per_s": round(n / wall, 2),
"tok_per_s": round(sum(r[1] for r in res) / wall),
}
def main(argv=None):
import argparse
ap = argparse.ArgumentParser(prog="openjev bench")
ap.add_argument("--endpoint", default="http://localhost:3000")
ap.add_argument("--token", default="")
ap.add_argument("--n", type=int, default=120)
ap.add_argument("--conc", type=int, default=32)
ap.add_argument("--tokens", type=int, default=1400)
a = ap.parse_args(argv)
print(json.dumps(asyncio.run(run(a.endpoint, a.token, a.n, a.conc, a.tokens))))
|