Spaces:
Running
Running
| { | |
| "status": "ok", | |
| "meta": { | |
| "device": "cuda", | |
| "dtype": "torch.float16", | |
| "hidden": 1024, | |
| "batch": 16, | |
| "seq_len": 128, | |
| "warmup": 30, | |
| "requests_per_worker": 120, | |
| "concurrency_levels": [ | |
| 4, | |
| 8 | |
| ], | |
| "modes": [ | |
| "eager", | |
| "compiled_aten", | |
| "arena" | |
| ], | |
| "compile_mode": "default", | |
| "inductor_gemm_backends": "ATEN", | |
| "allow_tf32": false, | |
| "use_inplace_residual": false | |
| }, | |
| "results": [ | |
| { | |
| "mode": "eager", | |
| "concurrency": 4, | |
| "requests_per_worker": 120, | |
| "total_requests": 480, | |
| "wall_time_sec": 0.2665, | |
| "throughput_requests_per_sec": 1801.24, | |
| "latency_ms": { | |
| "p50": 1.8897, | |
| "p95": 3.2932, | |
| "p99": 3.3079, | |
| "mean": 2.206, | |
| "min": 1.6086, | |
| "max": 4.319 | |
| }, | |
| "memory_pre": { | |
| "allocated_bytes": 67647488, | |
| "reserved_bytes": 270532608, | |
| "peak_allocated_bytes": 67647488, | |
| "allocation_events": 1334, | |
| "segment_alloc_events": 19 | |
| }, | |
| "memory_post": { | |
| "allocated_bytes": 101726208, | |
| "reserved_bytes": 270532608, | |
| "peak_allocated_bytes": 202390016, | |
| "allocation_events": 6618, | |
| "segment_alloc_events": 19 | |
| }, | |
| "allocation_event_delta": 5284, | |
| "segment_alloc_event_delta": 0, | |
| "memory_stable": false, | |
| "compile_applied": false | |
| }, | |
| { | |
| "mode": "eager", | |
| "concurrency": 8, | |
| "requests_per_worker": 120, | |
| "total_requests": 960, | |
| "wall_time_sec": 0.506, | |
| "throughput_requests_per_sec": 1897.15, | |
| "latency_ms": { | |
| "p50": 4.1548, | |
| "p95": 4.562, | |
| "p99": 4.8759, | |
| "mean": 4.1445, | |
| "min": 3.2656, | |
| "max": 5.8043 | |
| }, | |
| "memory_pre": { | |
| "allocated_bytes": 186660864, | |
| "reserved_bytes": 765460480, | |
| "peak_allocated_bytes": 186660864, | |
| "allocation_events": 9280, | |
| "segment_alloc_events": 53 | |
| }, | |
| "memory_post": { | |
| "allocated_bytes": 254818304, | |
| "reserved_bytes": 765460480, | |
| "peak_allocated_bytes": 489699840, | |
| "allocation_events": 19848, | |
| "segment_alloc_events": 53 | |
| }, | |
| "allocation_event_delta": 10568, | |
| "segment_alloc_event_delta": 0, | |
| "memory_stable": false, | |
| "compile_applied": false | |
| }, | |
| { | |
| "mode": "compiled_aten", | |
| "concurrency": 4, | |
| "requests_per_worker": 120, | |
| "total_requests": 480, | |
| "wall_time_sec": 0.2899, | |
| "throughput_requests_per_sec": 1655.52, | |
| "latency_ms": { | |
| "p50": 2.1664, | |
| "p95": 2.6192, | |
| "p99": 3.5204, | |
| "mean": 2.2723, | |
| "min": 0.6953, | |
| "max": 30.5198 | |
| }, | |
| "memory_pre": { | |
| "allocated_bytes": 272119808, | |
| "reserved_bytes": 941621248, | |
| "peak_allocated_bytes": 272119808, | |
| "allocation_events": 20852, | |
| "segment_alloc_events": 66 | |
| }, | |
| "memory_post": { | |
| "allocated_bytes": 306198528, | |
| "reserved_bytes": 1004535808, | |
| "peak_allocated_bytes": 390084608, | |
| "allocation_events": 23736, | |
| "segment_alloc_events": 69 | |
| }, | |
| "allocation_event_delta": 2884, | |
| "segment_alloc_event_delta": 3, | |
| "memory_stable": false, | |
| "compile_applied": true | |
| }, | |
| { | |
| "mode": "compiled_aten", | |
| "concurrency": 8, | |
| "requests_per_worker": 120, | |
| "total_requests": 960, | |
| "wall_time_sec": 0.5575, | |
| "throughput_requests_per_sec": 1722.03, | |
| "latency_ms": { | |
| "p50": 4.17, | |
| "p95": 7.0699, | |
| "p99": 9.742, | |
| "mean": 4.3512, | |
| "min": 0.8245, | |
| "max": 13.0942 | |
| }, | |
| "memory_pre": { | |
| "allocated_bytes": 391133184, | |
| "reserved_bytes": 1323302912, | |
| "peak_allocated_bytes": 391133184, | |
| "allocation_events": 25198, | |
| "segment_alloc_events": 93 | |
| }, | |
| "memory_post": { | |
| "allocated_bytes": 459290624, | |
| "reserved_bytes": 1491075072, | |
| "peak_allocated_bytes": 622868480, | |
| "allocation_events": 30966, | |
| "segment_alloc_events": 101 | |
| }, | |
| "allocation_event_delta": 5768, | |
| "segment_alloc_event_delta": 8, | |
| "memory_stable": false, | |
| "compile_applied": true | |
| }, | |
| { | |
| "mode": "arena", | |
| "concurrency": 4, | |
| "requests_per_worker": 120, | |
| "total_requests": 480, | |
| "wall_time_sec": 0.2513, | |
| "throughput_requests_per_sec": 1910.3, | |
| "latency_ms": { | |
| "p50": 1.6249, | |
| "p95": 3.0671, | |
| "p99": 3.0711, | |
| "mean": 2.0835, | |
| "min": 0.6909, | |
| "max": 3.0763 | |
| }, | |
| "memory_pre": { | |
| "allocated_bytes": 451034112, | |
| "reserved_bytes": 754974720, | |
| "peak_allocated_bytes": 451034112, | |
| "allocation_events": 31023, | |
| "segment_alloc_events": 115 | |
| }, | |
| "memory_post": { | |
| "allocated_bytes": 451034112, | |
| "reserved_bytes": 754974720, | |
| "peak_allocated_bytes": 451034112, | |
| "allocation_events": 31023, | |
| "segment_alloc_events": 115 | |
| }, | |
| "allocation_event_delta": 0, | |
| "segment_alloc_event_delta": 0, | |
| "memory_stable": true, | |
| "compile_applied": false | |
| }, | |
| { | |
| "mode": "arena", | |
| "concurrency": 8, | |
| "requests_per_worker": 120, | |
| "total_requests": 960, | |
| "wall_time_sec": 0.5052, | |
| "throughput_requests_per_sec": 1900.27, | |
| "latency_ms": { | |
| "p50": 4.2197, | |
| "p95": 4.4899, | |
| "p99": 4.9838, | |
| "mean": 4.1886, | |
| "min": 2.3138, | |
| "max": 5.7986 | |
| }, | |
| "memory_pre": { | |
| "allocated_bytes": 467811328, | |
| "reserved_bytes": 1040187392, | |
| "peak_allocated_bytes": 467811328, | |
| "allocation_events": 31125, | |
| "segment_alloc_events": 140 | |
| }, | |
| "memory_post": { | |
| "allocated_bytes": 467811328, | |
| "reserved_bytes": 1040187392, | |
| "peak_allocated_bytes": 467811328, | |
| "allocation_events": 31125, | |
| "segment_alloc_events": 140 | |
| }, | |
| "allocation_event_delta": 0, | |
| "segment_alloc_event_delta": 0, | |
| "memory_stable": true, | |
| "compile_applied": false | |
| } | |
| ] | |
| } | |