Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /BerriAI_litellm /scripts /benchmark_mock.py
| #!/usr/bin/env python3 | |
| """Quick benchmark for network_mock proxy overhead measurement.""" | |
| import argparse | |
| import asyncio | |
| import time | |
| import statistics | |
| import aiohttp | |
| REQUEST_BODY = { | |
| "model": "db-openai-endpoint", | |
| "messages": [{"role": "user", "content": "hi"}], | |
| "max_tokens": 100, | |
| "user": "new_user", | |
| } | |
| HEADERS = { | |
| "Authorization": "Bearer sk-1234", | |
| "Content-Type": "application/json", | |
| } | |
| async def send_request(session, url, semaphore): | |
| async with semaphore: | |
| start = time.perf_counter() | |
| try: | |
| async with session.post(url, json=REQUEST_BODY, headers=HEADERS) as resp: | |
| await resp.read() | |
| elapsed = time.perf_counter() - start | |
| return elapsed if resp.status == 200 else None | |
| except Exception: | |
| return None | |
| async def run_benchmark(url, n_requests, max_concurrent): | |
| semaphore = asyncio.Semaphore(max_concurrent) | |
| connector_limit = min(max_concurrent * 2, 200) | |
| connector = aiohttp.TCPConnector( | |
| limit=connector_limit, | |
| limit_per_host=max_concurrent, | |
| force_close=False, | |
| enable_cleanup_closed=True, | |
| ) | |
| async with aiohttp.ClientSession(connector=connector) as session: | |
| # warmup | |
| await asyncio.gather( | |
| *[send_request(session, url, semaphore) for _ in range(min(50, n_requests))] | |
| ) | |
| # timed run | |
| wall_start = time.perf_counter() | |
| results = await asyncio.gather( | |
| *[send_request(session, url, semaphore) for _ in range(n_requests)] | |
| ) | |
| wall_elapsed = time.perf_counter() - wall_start | |
| latencies = [r for r in results if r is not None] | |
| failures = sum(1 for r in results if r is None) | |
| if not latencies: | |
| return { | |
| "mean": 0, | |
| "p50": 0, | |
| "p95": 0, | |
| "p99": 0, | |
| "throughput": 0, | |
| "failures": n_requests, | |
| "wall_time": wall_elapsed, | |
| "n_requests": n_requests, | |
| "max_concurrent": max_concurrent, | |
| "latencies": [], | |
| } | |
| latencies.sort() | |
| n = len(latencies) | |
| mean = statistics.mean(latencies) * 1000 | |
| p50 = latencies[n // 2] * 1000 | |
| p95 = latencies[int(n * 0.95)] * 1000 | |
| p99 = latencies[int(n * 0.99)] * 1000 | |
| throughput = n_requests / wall_elapsed | |
| return { | |
| "mean": mean, | |
| "p50": p50, | |
| "p95": p95, | |
| "p99": p99, | |
| "throughput": throughput, | |
| "failures": failures, | |
| "wall_time": wall_elapsed, | |
| "n_requests": n_requests, | |
| "max_concurrent": max_concurrent, | |
| "latencies": latencies, | |
| } | |
| def print_run_results(run_num, total_runs, result): | |
| label = f" Run {run_num}/{total_runs}" if total_runs > 1 else " Results" | |
| print(f"\n{'='*60}") | |
| print(label) | |
| print(f"{'='*60}") | |
| print(f" Requests: {result['n_requests']} (failures: {result['failures']})") | |
| print(f" Concurrency: {result['max_concurrent']}") | |
| print(f" Wall time: {result['wall_time']:.2f}s") | |
| print(f" Throughput: {result['throughput']:.0f} req/s") | |
| print(f" Mean: {result['mean']:.2f} ms") | |
| print(f" P50: {result['p50']:.2f} ms") | |
| print(f" P95: {result['p95']:.2f} ms") | |
| print(f" P99: {result['p99']:.2f} ms") | |
| def print_aggregate(results): | |
| all_latencies = [] | |
| for r in results: | |
| all_latencies.extend(r["latencies"]) | |
| all_latencies.sort() | |
| total_failures = sum(r["failures"] for r in results) | |
| total_requests = sum(r["n_requests"] for r in results) | |
| n = len(all_latencies) | |
| if not all_latencies: | |
| print( | |
| f"\n Aggregate: all {total_requests} requests failed across {len(results)} runs" | |
| ) | |
| return | |
| mean = statistics.mean(all_latencies) * 1000 | |
| p50 = all_latencies[n // 2] * 1000 | |
| p95 = all_latencies[int(n * 0.95)] * 1000 | |
| p99 = all_latencies[int(n * 0.99)] * 1000 | |
| avg_throughput = statistics.mean(r["throughput"] for r in results) | |
| print(f"\n{'='*60}") | |
| print(f" Aggregate ({len(results)} runs, {total_requests} total requests)") | |
| print(f"{'='*60}") | |
| print(f" Failures: {total_failures}") | |
| print(f" Throughput: {avg_throughput:.0f} req/s (avg across runs)") | |
| print(f" Mean: {mean:.2f} ms") | |
| print(f" P50: {p50:.2f} ms") | |
| print(f" P95: {p95:.2f} ms") | |
| print(f" P99: {p99:.2f} ms") | |
| # Run-to-run variance | |
| run_means = [r["mean"] for r in results] | |
| run_throughputs = [r["throughput"] for r in results] | |
| if len(run_means) > 1: | |
| cov_latency = statistics.stdev(run_means) / statistics.mean(run_means) * 100 | |
| cov_throughput = ( | |
| statistics.stdev(run_throughputs) / statistics.mean(run_throughputs) * 100 | |
| ) | |
| print(f"\n Run-to-run variance:") | |
| print(f" Latency CoV: {cov_latency:.1f}%") | |
| print(f" Throughput CoV: {cov_throughput:.1f}%") | |
| async def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--url", default="http://localhost:4000/chat/completions") | |
| parser.add_argument("--requests", type=int, default=2000) | |
| parser.add_argument("--max-concurrent", type=int, default=200) | |
| parser.add_argument("--runs", type=int, default=1) | |
| args = parser.parse_args() | |
| print(f"Benchmarking {args.url}") | |
| print( | |
| f" {args.requests} requests, {args.max_concurrent} concurrency, {args.runs} run(s)" | |
| ) | |
| results = [] | |
| for run_num in range(1, args.runs + 1): | |
| result = await run_benchmark(args.url, args.requests, args.max_concurrent) | |
| results.append(result) | |
| print_run_results(run_num, args.runs, result) | |
| if args.runs > 1: | |
| print_aggregate(results) | |
| if __name__ == "__main__": | |
| asyncio.run(main()) | |
Xet Storage Details
- Size:
- 5.82 kB
- Xet hash:
- b740d22f78ebf4958af4905e0275a2b39c620fd1b2dbde431376bd75857f8fbf
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.