Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /nijaru_zenith /benchmarks /run_benchmarks.py
| #!/usr/bin/env python | |
| """Run performance benchmarks for all frameworks.""" | |
| import asyncio | |
| import json | |
| import subprocess | |
| import time | |
| from collections import defaultdict | |
| from datetime import datetime | |
| from pathlib import Path | |
| import httpx | |
| import psutil | |
| # Configuration | |
| FRAMEWORKS = { | |
| "zenith": {"file": "zenith_minimal.py", "port": 8100}, | |
| "fastapi": {"file": "fastapi_app.py", "port": 8001}, | |
| "flask": {"file": "flask_app.py", "port": 8002}, | |
| } | |
| TESTS = [ | |
| { | |
| "name": "hello_world", | |
| "method": "GET", | |
| "path": "/", | |
| "description": "Simple JSON response", | |
| }, | |
| { | |
| "name": "get_user", | |
| "method": "GET", | |
| "path": "/users/1", | |
| "description": "Single database query", | |
| }, | |
| { | |
| "name": "list_users", | |
| "method": "GET", | |
| "path": "/users?limit=100", | |
| "description": "Bulk database query", | |
| }, | |
| { | |
| "name": "create_user", | |
| "method": "POST", | |
| "path": "/users", | |
| "body": {"name": "Test User", "email": "test@example.com"}, | |
| "description": "Create with validation", | |
| }, | |
| { | |
| "name": "validate", | |
| "method": "POST", | |
| "path": "/validate", | |
| "body": {"name": "Test", "email": "test@example.com"}, | |
| "description": "Pydantic validation only", | |
| }, | |
| ] | |
| class BenchmarkRunner: | |
| def __init__(self): | |
| self.results = defaultdict(dict) | |
| self.processes = {} | |
| async def start_server(self, framework: str, config: dict): | |
| """Start a framework server.""" | |
| print(f"Starting {framework} server...") | |
| venv_python = Path(__file__).parent.parent / ".venv" / "bin" / "python" | |
| cmd = [str(venv_python), config["file"]] | |
| process = subprocess.Popen( | |
| cmd, | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.PIPE, | |
| cwd=Path(__file__).parent, | |
| ) | |
| self.processes[framework] = process | |
| # Wait for server to be ready | |
| url = f"http://127.0.0.1:{config['port']}/" | |
| async with httpx.AsyncClient() as client: | |
| for _ in range(30): # 30 second timeout | |
| try: | |
| response = await client.get(url) | |
| if response.status_code == 200: | |
| print(f" {framework} server ready") | |
| return | |
| except (httpx.ConnectError, httpx.RequestError): | |
| pass | |
| await asyncio.sleep(1) | |
| raise RuntimeError(f"Failed to start {framework} server") | |
| def stop_server(self, framework: str): | |
| """Stop a framework server.""" | |
| if framework in self.processes: | |
| process = self.processes[framework] | |
| process.terminate() | |
| try: | |
| process.wait(timeout=5) | |
| except subprocess.TimeoutExpired: | |
| process.kill() | |
| del self.processes[framework] | |
| async def run_warmup(self, port: int, test: dict): | |
| """Warm up the server before benchmarking.""" | |
| url = f"http://127.0.0.1:{port}{test['path']}" | |
| async with httpx.AsyncClient() as client: | |
| for _ in range(100): | |
| try: | |
| if test["method"] == "GET": | |
| await client.get(url) | |
| else: | |
| await client.post(url, json=test.get("body", {})) | |
| except (httpx.RequestError, httpx.HTTPStatusError): | |
| pass | |
| async def run_benchmark( | |
| self, framework: str, port: int, test: dict, duration: int = 10 | |
| ): | |
| """Run a single benchmark test.""" | |
| print(f" Running {test['name']}...") | |
| # Warm up | |
| await self.run_warmup(port, test) | |
| # Prepare request | |
| url = f"http://127.0.0.1:{port}{test['path']}" | |
| # Track metrics | |
| start_time = time.time() | |
| request_count = 0 | |
| latencies = [] | |
| errors = 0 | |
| # Get initial memory | |
| process = self.processes[framework] | |
| proc = psutil.Process(process.pid) | |
| initial_memory = proc.memory_info().rss / 1024 / 1024 # MB | |
| # Run benchmark | |
| async with httpx.AsyncClient() as client: | |
| tasks = [] | |
| async def make_request(): | |
| nonlocal request_count, errors | |
| try: | |
| start = time.perf_counter() | |
| if test["method"] == "GET": | |
| response = await client.get(url) | |
| else: | |
| response = await client.post(url, json=test.get("body", {})) | |
| if response.status_code < 400: | |
| request_count += 1 | |
| latencies.append((time.perf_counter() - start) * 1000) # ms | |
| else: | |
| errors += 1 | |
| except Exception: | |
| errors += 1 | |
| # Run for duration seconds with concurrent requests | |
| end_time = start_time + duration | |
| while time.time() < end_time: | |
| # Maintain 100 concurrent requests | |
| while len(tasks) < 100: | |
| tasks.append(asyncio.create_task(make_request())) | |
| # Wait for some to complete | |
| _done, tasks = await asyncio.wait( | |
| tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED | |
| ) | |
| tasks = list(tasks) | |
| # Wait for remaining tasks | |
| if tasks: | |
| await asyncio.gather(*tasks, return_exceptions=True) | |
| # Calculate results | |
| elapsed = time.time() - start_time | |
| final_memory = proc.memory_info().rss / 1024 / 1024 # MB | |
| if latencies: | |
| latencies.sort() | |
| p50 = latencies[int(len(latencies) * 0.50)] | |
| p95 = latencies[int(len(latencies) * 0.95)] | |
| p99 = latencies[int(len(latencies) * 0.99)] | |
| else: | |
| p50 = p95 = p99 = 0 | |
| return { | |
| "requests_per_second": request_count / elapsed, | |
| "total_requests": request_count, | |
| "errors": errors, | |
| "latency_p50": p50, | |
| "latency_p95": p95, | |
| "latency_p99": p99, | |
| "memory_initial_mb": initial_memory, | |
| "memory_final_mb": final_memory, | |
| "memory_delta_mb": final_memory - initial_memory, | |
| } | |
| async def run_all_benchmarks(self): | |
| """Run all benchmarks for all frameworks.""" | |
| print("=" * 60) | |
| print("Running Performance Benchmarks") | |
| print("=" * 60) | |
| for framework, config in FRAMEWORKS.items(): | |
| print(f"\nBenchmarking {framework.upper()}") | |
| print("-" * 40) | |
| # Start server | |
| await self.start_server(framework, config) | |
| # Run tests | |
| for test in TESTS: | |
| result = await self.run_benchmark( | |
| framework, | |
| config["port"], | |
| test, | |
| duration=10, # 10 seconds per test | |
| ) | |
| self.results[framework][test["name"]] = result | |
| # Print immediate results | |
| print(f" {test['name']}:") | |
| print(f" RPS: {result['requests_per_second']:.1f}") | |
| print(f" Latency P50: {result['latency_p50']:.2f}ms") | |
| print(f" Latency P95: {result['latency_p95']:.2f}ms") | |
| print(f" Memory: {result['memory_delta_mb']:.1f}MB") | |
| # Stop server | |
| self.stop_server(framework) | |
| return self.results | |
| def print_summary(self): | |
| """Print benchmark summary.""" | |
| print("\n" + "=" * 60) | |
| print("BENCHMARK SUMMARY") | |
| print("=" * 60) | |
| for test in TESTS: | |
| print(f"\n{test['description']} ({test['name']})") | |
| print("-" * 40) | |
| print( | |
| f"{'Framework':<12} {'RPS':>10} {'P50 (ms)':>10} {'P95 (ms)':>10} {'Memory Δ':>10}" | |
| ) | |
| print("-" * 40) | |
| for framework in FRAMEWORKS: | |
| if test["name"] in self.results[framework]: | |
| r = self.results[framework][test["name"]] | |
| print( | |
| f"{framework:<12} {r['requests_per_second']:>10.1f} " | |
| f"{r['latency_p50']:>10.2f} {r['latency_p95']:>10.2f} " | |
| f"{r['memory_delta_mb']:>9.1f}MB" | |
| ) | |
| # Overall winner analysis | |
| print("\n" + "=" * 60) | |
| print("PERFORMANCE RANKINGS") | |
| print("=" * 60) | |
| scores = defaultdict(int) | |
| for test in TESTS: | |
| # Rank by RPS | |
| rps_scores = sorted( | |
| [ | |
| (f, self.results[f][test["name"]]["requests_per_second"]) | |
| for f in FRAMEWORKS | |
| ], | |
| key=lambda x: x[1], | |
| reverse=True, | |
| ) | |
| for i, (framework, _) in enumerate(rps_scores): | |
| scores[framework] += 3 - i # 3 points for 1st, 2 for 2nd, 1 for 3rd | |
| print("\nOverall Performance Score:") | |
| for framework, score in sorted( | |
| scores.items(), key=lambda x: x[1], reverse=True | |
| ): | |
| print(f" {framework}: {score} points") | |
| def save_results(self): | |
| """Save results to JSON file.""" | |
| output = { | |
| "timestamp": datetime.now().isoformat(), | |
| "system": { | |
| "cpu_count": psutil.cpu_count(), | |
| "memory_gb": psutil.virtual_memory().total / (1024**3), | |
| "python_version": subprocess.run( | |
| ["python", "--version"], capture_output=True, text=True | |
| ).stdout.strip(), | |
| }, | |
| "results": self.results, | |
| } | |
| with Path("benchmark_results.json").open("w") as f: | |
| json.dump(output, f, indent=2) | |
| print("\nResults saved to benchmark_results.json") | |
| async def main(): | |
| runner = BenchmarkRunner() | |
| try: | |
| await runner.run_all_benchmarks() | |
| runner.print_summary() | |
| runner.save_results() | |
| finally: | |
| # Clean up any remaining processes | |
| for framework in list(runner.processes.keys()): | |
| runner.stop_server(framework) | |
| if __name__ == "__main__": | |
| asyncio.run(main()) | |
Xet Storage Details
- Size:
- 10.4 kB
- Xet hash:
- d4194e5522365917842e08c86843407d378d18dd094e0fb37b20f768ed28243e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.