| { | |
| "gpu": "NVIDIA GeForce RTX 4090", | |
| "torch": "2.14.1+cu130", | |
| "dtype": "fp32 weights + bf16 autocast", | |
| "batch_size": 1, | |
| "warmup": 100, | |
| "measured": 1000, | |
| "inputs": "test_idd, seed 0 (labels unused)", | |
| "software": { | |
| "laya": "0.3.28", | |
| "transformers": "4.57.6", | |
| "cuda": "13.0" | |
| }, | |
| "questions_per_step": "operation + target (all candidates), one forward", | |
| "max_len": 3072, | |
| "head_max_len": 1536, | |
| "models": { | |
| "laya_android": { | |
| "forward_ms": { | |
| "n": 1000, | |
| "median": 39.74, | |
| "p90": 41.95, | |
| "p95": 43.51, | |
| "mean": 40.26 | |
| }, | |
| "e2e_ms": { | |
| "n": 1000, | |
| "median": 44.46, | |
| "p90": 48.63, | |
| "p95": 51.13, | |
| "mean": 45.34 | |
| }, | |
| "forward_ms_by_candidates": { | |
| "1-5": { | |
| "n": 102, | |
| "median": 39.66, | |
| "p90": 41.49, | |
| "p95": 42.07, | |
| "mean": 39.85 | |
| }, | |
| "6-10": { | |
| "n": 171, | |
| "median": 39.49, | |
| "p90": 41.12, | |
| "p95": 42.08, | |
| "mean": 39.72 | |
| }, | |
| "11-20": { | |
| "n": 406, | |
| "median": 39.62, | |
| "p90": 40.93, | |
| "p95": 41.55, | |
| "mean": 39.71 | |
| }, | |
| "21-50": { | |
| "n": 295, | |
| "median": 40.27, | |
| "p90": 42.85, | |
| "p95": 43.91, | |
| "mean": 40.47 | |
| }, | |
| ">50": { | |
| "n": 21, | |
| "median": 48.97, | |
| "p90": 69.71, | |
| "p95": 78.59, | |
| "mean": 54.25 | |
| } | |
| }, | |
| "forward_ms_by_input_tokens": { | |
| "<=512": { | |
| "n": 649, | |
| "median": 39.6, | |
| "p90": 41.14, | |
| "p95": 41.81, | |
| "mean": 39.78 | |
| }, | |
| "513-1024": { | |
| "n": 292, | |
| "median": 39.85, | |
| "p90": 41.68, | |
| "p95": 42.49, | |
| "mean": 39.92 | |
| }, | |
| "1025-1536": { | |
| "n": 49, | |
| "median": 42.88, | |
| "p90": 46.74, | |
| "p95": 48.44, | |
| "mean": 43.75 | |
| }, | |
| ">1536": { | |
| "n": 10, | |
| "median": 53.1, | |
| "p90": 83.47, | |
| "p95": 105.45, | |
| "mean": 64.05 | |
| } | |
| }, | |
| "peak_vram_allocated_gb_bs1": 1.66, | |
| "peak_vram_reserved_gb_bs1": 1.91, | |
| "batch_sweep": { | |
| "1": { | |
| "batch_ms": { | |
| "n": 1000, | |
| "median": 39.06, | |
| "p90": 41.12, | |
| "p95": 42.26, | |
| "mean": 39.48 | |
| }, | |
| "throughput_steps_per_s": 25.3, | |
| "peak_vram_allocated_gb": 1.66, | |
| "peak_vram_reserved_gb": 1.91 | |
| }, | |
| "4": { | |
| "batch_ms": { | |
| "n": 250, | |
| "median": 39.99, | |
| "p90": 51.89, | |
| "p95": 67.4, | |
| "mean": 44.25 | |
| }, | |
| "throughput_steps_per_s": 90.4, | |
| "peak_vram_allocated_gb": 2.3, | |
| "peak_vram_reserved_gb": 3.45 | |
| }, | |
| "8": { | |
| "batch_ms": { | |
| "n": 125, | |
| "median": 71.35, | |
| "p90": 134.79, | |
| "p95": 158.07, | |
| "mean": 81.02 | |
| }, | |
| "throughput_steps_per_s": 98.7, | |
| "peak_vram_allocated_gb": 3.16, | |
| "peak_vram_reserved_gb": 6.3 | |
| }, | |
| "16": { | |
| "batch_ms": { | |
| "n": 63, | |
| "median": 177.91, | |
| "p90": 293.32, | |
| "p95": 371.29, | |
| "mean": 198.02 | |
| }, | |
| "throughput_steps_per_s": 80.2, | |
| "peak_vram_allocated_gb": 4.89, | |
| "peak_vram_reserved_gb": 11.99 | |
| }, | |
| "32": { | |
| "batch_ms": { | |
| "n": 32, | |
| "median": 431.11, | |
| "p90": 778.52, | |
| "p95": 827.76, | |
| "mean": 535.89 | |
| }, | |
| "throughput_steps_per_s": 58.3, | |
| "peak_vram_allocated_gb": 8.34, | |
| "peak_vram_reserved_gb": 23.35 | |
| } | |
| }, | |
| "params_m": 321.9 | |
| } | |
| } | |
| } |