| { |
| "created_at": "2026-07-21T02:07:16.839792+00:00", |
| "device": "cuda:6", |
| "graph_enabled": false, |
| "measurement": "inference-only KV cache; BF16 numerical equivalence is checked against full-sequence logits; prefill and autoregressive decode are reported separately", |
| "program": "STRATA-HEADQUOTIENT-Q25-CACHED-DECODE-PROFILE", |
| "rows": [ |
| { |
| "batch_size": 1, |
| "dense": { |
| "batch_size": 1, |
| "decode_seconds": 0.23088740557432175, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 138.59569308426106, |
| "first_cached_token_ms": 7.441435630122821, |
| "peak_allocated_bytes": 2767192064, |
| "persistent_kv_bytes": 204472320, |
| "prefill_seconds": 0.02532536784807841, |
| "prefill_tokens_per_second": 80867.53220271169, |
| "prefix_length": 2048 |
| }, |
| "prefix_length": 2048, |
| "q25": { |
| "batch_size": 1, |
| "decode_seconds": 0.21997203739980856, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 145.47303547422547, |
| "first_cached_token_ms": 7.406104976932208, |
| "peak_allocated_bytes": 2728734208, |
| "persistent_kv_bytes": 178520064, |
| "prefill_seconds": 0.02850375821193059, |
| "prefill_tokens_per_second": 71850.1744497252, |
| "prefix_length": 2048 |
| }, |
| "ratios": { |
| "decode_throughput": 1.0496216169270371, |
| "persistent_kv": 0.8730769230769231, |
| "prefill_throughput": 0.8884922352968238 |
| } |
| }, |
| { |
| "batch_size": 4, |
| "dense": { |
| "batch_size": 4, |
| "decode_seconds": 0.23466505793233713, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 545.458284790347, |
| "first_cached_token_ms": 7.920100043217341, |
| "peak_allocated_bytes": 4261779456, |
| "persistent_kv_bytes": 817889280, |
| "prefill_seconds": 0.09082256692151229, |
| "prefill_tokens_per_second": 90197.84705137681, |
| "prefix_length": 2048 |
| }, |
| "prefix_length": 2048, |
| "q25": { |
| "batch_size": 4, |
| "decode_seconds": 0.2278061956167221, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 561.881118524786, |
| "first_cached_token_ms": 7.969736432035764, |
| "peak_allocated_bytes": 4070479872, |
| "persistent_kv_bytes": 714080256, |
| "prefill_seconds": 0.0923770306011041, |
| "prefill_tokens_per_second": 88680.05332812775, |
| "prefix_length": 2048 |
| }, |
| "ratios": { |
| "decode_throughput": 1.0301083221070724, |
| "persistent_kv": 0.8730769230769231, |
| "prefill_throughput": 0.9831726169430127 |
| } |
| }, |
| { |
| "batch_size": 8, |
| "dense": { |
| "batch_size": 8, |
| "decode_seconds": 0.25229039539893466, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 1014.7037091729136, |
| "first_cached_token_ms": 8.601184313495954, |
| "peak_allocated_bytes": 6273153024, |
| "persistent_kv_bytes": 1635778560, |
| "prefill_seconds": 0.19029431355496249, |
| "prefill_tokens_per_second": 86098.2112072825, |
| "prefix_length": 2048 |
| }, |
| "prefix_length": 2048, |
| "q25": { |
| "batch_size": 8, |
| "decode_seconds": 0.23515919471780458, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 1088.6242415790068, |
| "first_cached_token_ms": 7.94680913289388, |
| "peak_allocated_bytes": 5870467072, |
| "persistent_kv_bytes": 1428160512, |
| "prefill_seconds": 0.19251583764950433, |
| "prefill_tokens_per_second": 85104.6864509341, |
| "prefix_length": 2048 |
| }, |
| "ratios": { |
| "decode_throughput": 1.072849376362629, |
| "persistent_kv": 0.8730769230769231, |
| "prefill_throughput": 0.9884605644830823 |
| } |
| }, |
| { |
| "batch_size": 16, |
| "dense": { |
| "batch_size": 16, |
| "decode_seconds": 0.3312743529677391, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 1545.5467512447628, |
| "first_cached_token_ms": 11.202741414308548, |
| "peak_allocated_bytes": 10296591872, |
| "persistent_kv_bytes": 3271557120, |
| "prefill_seconds": 0.40776471979916096, |
| "prefill_tokens_per_second": 80360.06650143603, |
| "prefix_length": 2048 |
| }, |
| "prefix_length": 2048, |
| "q25": { |
| "batch_size": 16, |
| "decode_seconds": 0.29237379630406696, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 1751.1829256665776, |
| "first_cached_token_ms": 9.928669159611067, |
| "peak_allocated_bytes": 9478374912, |
| "persistent_kv_bytes": 2856321024, |
| "prefill_seconds": 0.41604801764090854, |
| "prefill_tokens_per_second": 78760.13972089658, |
| "prefix_length": 2048 |
| }, |
| "ratios": { |
| "decode_throughput": 1.1330507629459918, |
| "persistent_kv": 0.8730769230769231, |
| "prefill_throughput": 0.9800905244334155 |
| } |
| }, |
| { |
| "batch_size": 1, |
| "dense": { |
| "batch_size": 1, |
| "decode_seconds": 0.23166167673965296, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 138.13247167317354, |
| "first_cached_token_ms": 7.798002411921819, |
| "peak_allocated_bytes": 3247719936, |
| "persistent_kv_bytes": 405798912, |
| "prefill_seconds": 0.04814621185262998, |
| "prefill_tokens_per_second": 85074.19052068697, |
| "prefix_length": 4096 |
| }, |
| "prefix_length": 4096, |
| "q25": { |
| "batch_size": 1, |
| "decode_seconds": 0.22054913143316904, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 145.0923873155069, |
| "first_cached_token_ms": 7.15858240922292, |
| "peak_allocated_bytes": 3113641472, |
| "persistent_kv_bytes": 329515008, |
| "prefill_seconds": 0.05360218447943529, |
| "prefill_tokens_per_second": 76414.79614644154, |
| "prefix_length": 4096 |
| }, |
| "ratios": { |
| "decode_throughput": 1.0503858039896714, |
| "persistent_kv": 0.812015503875969, |
| "prefill_throughput": 0.8982136142436785 |
| } |
| }, |
| { |
| "batch_size": 4, |
| "dense": { |
| "batch_size": 4, |
| "decode_seconds": 0.25190934414664906, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 508.1193015431964, |
| "first_cached_token_ms": 8.424771949648857, |
| "peak_allocated_bytes": 6247872512, |
| "persistent_kv_bytes": 1623195648, |
| "prefill_seconds": 0.20185994543135166, |
| "prefill_tokens_per_second": 81165.18591635038, |
| "prefix_length": 4096 |
| }, |
| "prefix_length": 4096, |
| "q25": { |
| "batch_size": 4, |
| "decode_seconds": 0.231443390250206, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 553.0510068212504, |
| "first_cached_token_ms": 7.94626772403717, |
| "peak_allocated_bytes": 5648562176, |
| "persistent_kv_bytes": 1318060032, |
| "prefill_seconds": 0.20270894095301628, |
| "prefill_tokens_per_second": 80825.2459066296, |
| "prefix_length": 4096 |
| }, |
| "ratios": { |
| "decode_throughput": 1.0884274719373839, |
| "persistent_kv": 0.812015503875969, |
| "prefill_throughput": 0.9958117509880267 |
| } |
| }, |
| { |
| "batch_size": 8, |
| "dense": { |
| "batch_size": 8, |
| "decode_seconds": 0.321361793205142, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 796.6099437234028, |
| "first_cached_token_ms": 10.821449880798658, |
| "peak_allocated_bytes": 10243749888, |
| "persistent_kv_bytes": 3246391296, |
| "prefill_seconds": 0.4328818339854479, |
| "prefill_tokens_per_second": 75697.33222184962, |
| "prefix_length": 4096 |
| }, |
| "prefix_length": 4096, |
| "q25": { |
| "batch_size": 8, |
| "decode_seconds": 0.27276255128284294, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 938.5452614224124, |
| "first_cached_token_ms": 9.249638145168623, |
| "peak_allocated_bytes": 9035757568, |
| "persistent_kv_bytes": 2636120064, |
| "prefill_seconds": 0.43729556041459244, |
| "prefill_tokens_per_second": 74933.30133270327, |
| "prefix_length": 4096 |
| }, |
| "ratios": { |
| "decode_throughput": 1.1781741727144346, |
| "persistent_kv": 0.812015503875969, |
| "prefill_throughput": 0.9899067659754881 |
| } |
| }, |
| { |
| "batch_size": 16, |
| "dense": { |
| "batch_size": 16, |
| "decode_seconds": 0.4728281206140916, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 1082.8459173177632, |
| "first_cached_token_ms": 15.76905200878779, |
| "peak_allocated_bytes": 18239866368, |
| "persistent_kv_bytes": 6492782592, |
| "prefill_seconds": 0.8857398132483164, |
| "prefill_tokens_per_second": 73990.12556481644, |
| "prefix_length": 4096 |
| }, |
| "prefix_length": 4096, |
| "q25": { |
| "batch_size": 16, |
| "decode_seconds": 0.3972581879546245, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 1288.8343538899733, |
| "first_cached_token_ms": 13.64798347155253, |
| "peak_allocated_bytes": 15803827712, |
| "persistent_kv_bytes": 5272240128, |
| "prefill_seconds": 0.8932362391302983, |
| "prefill_tokens_per_second": 73369.1683443221, |
| "prefix_length": 4096 |
| }, |
| "ratios": { |
| "decode_throughput": 1.1902287604153772, |
| "persistent_kv": 0.812015503875969, |
| "prefill_throughput": 0.9916075663373436 |
| } |
| }, |
| { |
| "batch_size": 1, |
| "dense": { |
| "batch_size": 1, |
| "decode_seconds": 0.22510927046338716, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 142.1531860244051, |
| "first_cached_token_ms": 7.497106368343036, |
| "peak_allocated_bytes": 4234716672, |
| "persistent_kv_bytes": 805306368, |
| "prefill_seconds": 0.10965904593467712, |
| "prefill_tokens_per_second": 74412.46575189826, |
| "prefix_length": 8160 |
| }, |
| "prefix_length": 8160, |
| "q25": { |
| "batch_size": 1, |
| "decode_seconds": 0.224536320194602, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 142.5159189046392, |
| "first_cached_token_ms": 7.5872062395016355, |
| "peak_allocated_bytes": 3902450176, |
| "persistent_kv_bytes": 629145600, |
| "prefill_seconds": 0.11838383475939433, |
| "prefill_tokens_per_second": 68928.32975536353, |
| "prefix_length": 8160 |
| }, |
| "ratios": { |
| "decode_throughput": 1.0025517041888306, |
| "persistent_kv": 0.78125, |
| "prefill_throughput": 0.9263008429955861 |
| } |
| }, |
| { |
| "batch_size": 4, |
| "dense": { |
| "batch_size": 4, |
| "decode_seconds": 0.31844382857282955, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 401.95472015789375, |
| "first_cached_token_ms": 10.655300070842108, |
| "peak_allocated_bytes": 10188224512, |
| "persistent_kv_bytes": 3221225472, |
| "prefill_seconds": 0.47874748582641286, |
| "prefill_tokens_per_second": 68177.9037307254, |
| "prefix_length": 8160 |
| }, |
| "prefix_length": 8160, |
| "q25": { |
| "batch_size": 4, |
| "decode_seconds": 0.27023141148189705, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 473.6681028236971, |
| "first_cached_token_ms": 8.966594313581785, |
| "peak_allocated_bytes": 8791914496, |
| "persistent_kv_bytes": 2516582400, |
| "prefill_seconds": 0.4703146008153756, |
| "prefill_tokens_per_second": 69400.35445085618, |
| "prefix_length": 8160 |
| }, |
| "ratios": { |
| "decode_throughput": 1.1784115948125533, |
| "persistent_kv": 0.78125, |
| "prefill_throughput": 1.0179303066424417 |
| } |
| }, |
| { |
| "batch_size": 8, |
| "dense": { |
| "batch_size": 8, |
| "decode_seconds": 0.45806841490169364, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 558.8684826805627, |
| "first_cached_token_ms": 15.288629879554113, |
| "peak_allocated_bytes": 18128516096, |
| "persistent_kv_bytes": 6442450944, |
| "prefill_seconds": 0.9819728036721548, |
| "prefill_tokens_per_second": 66478.41952025653, |
| "prefix_length": 8160 |
| }, |
| "prefix_length": 8160, |
| "q25": { |
| "batch_size": 8, |
| "decode_seconds": 0.3758538272231817, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 681.1158526476502, |
| "first_cached_token_ms": 12.771941100557646, |
| "peak_allocated_bytes": 15313089536, |
| "persistent_kv_bytes": 5033164800, |
| "prefill_seconds": 0.9673846984903017, |
| "prefill_tokens_per_second": 67480.9102334116, |
| "prefix_length": 8160 |
| }, |
| "ratios": { |
| "decode_throughput": 1.2187408554168933, |
| "persistent_kv": 0.78125, |
| "prefill_throughput": 1.0150799420381769 |
| } |
| }, |
| { |
| "batch_size": 16, |
| "dense": { |
| "batch_size": 16, |
| "decode_seconds": 0.7506915175666412, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 682.037811829342, |
| "first_cached_token_ms": 25.01120666662852, |
| "peak_allocated_bytes": 34005301248, |
| "persistent_kv_bytes": 12884901888, |
| "prefill_seconds": 2.1065259805570045, |
| "prefill_tokens_per_second": 61978.82257567861, |
| "prefix_length": 8160 |
| }, |
| "prefix_length": 8160, |
| "q25": { |
| "batch_size": 16, |
| "decode_seconds": 0.606967706233263, |
| "decode_steps": 32, |
| "decode_tokens_per_second": 843.5374645833858, |
| "first_cached_token_ms": 20.8244522412618, |
| "peak_allocated_bytes": 28370909184, |
| "persistent_kv_bytes": 10066329600, |
| "prefill_seconds": 2.119566404571136, |
| "prefill_tokens_per_second": 61597.50396044655, |
| "prefix_length": 8160 |
| }, |
| "ratios": { |
| "decode_throughput": 1.2367898816648804, |
| "persistent_kv": 0.78125, |
| "prefill_throughput": 0.9938475982700953 |
| } |
| } |
| ], |
| "torch_version": "2.5.1+cu124", |
| "verification": { |
| "dense": { |
| "maximum_absolute_logit_difference": 0.0625, |
| "maximum_absolute_probability_difference": 0.0015385262668132782, |
| "mean_absolute_logit_difference": 0.009482888504862785, |
| "numerically_equivalent": true, |
| "top_token_exact": true |
| }, |
| "q25": { |
| "maximum_absolute_logit_difference": 0.078125, |
| "maximum_absolute_probability_difference": 0.002120431512594223, |
| "mean_absolute_logit_difference": 0.011962523683905602, |
| "numerically_equivalent": true, |
| "top_token_exact": true |
| } |
| } |
| } |
|
|
