| { |
| "scope": { |
| "prompts": 18, |
| "repeats": 2, |
| "temperature": 0, |
| "max_new_tokens": 160, |
| "max_concurrent_requests": 1, |
| "context_length": 2048, |
| "mtp_steps": 1, |
| "draft_tokens": 2, |
| "cuda_graphs": false, |
| "overlap": false, |
| "radix_cache": false, |
| "deterministic_inference": true, |
| "tool_test": "emitted JSON only; no external tool service", |
| "code_test": "bounded Python function execution cases", |
| "not_covered": [ |
| "sampled decoding", |
| "concurrency", |
| "prefix cache", |
| "CUDA graphs", |
| "MTP2", |
| "multimodal MTP", |
| "other engines", |
| "other quantizations", |
| "FP8 with this head", |
| "GGUF Q4/Q8 with this head" |
| ], |
| "authored_prompts": 10, |
| "occamy_heldout_prefixes": 8, |
| "heldout_prefix_length": 128 |
| }, |
| "runtime": { |
| "sglang": "0.5.13.post1", |
| "source_revision": "85fd90072d1a9f2432842b03588f63b745e524e4", |
| "torch": "2.11.0+cu130", |
| "hook": "canonical_attention:install" |
| }, |
| "results": { |
| "BF16": { |
| "baseline": { |
| "requests": 36, |
| "quality_pass": 12, |
| "quality_total": 16, |
| "stopped": 22, |
| "length_limited": 14, |
| "acceptance": null, |
| "accepted": 0, |
| "proposed": 0, |
| "median_e2e_seconds": 4.305030163610354, |
| "repeat_token_parity": 18, |
| "exact_tokens": 36, |
| "exact_logprobs": 36 |
| }, |
| "original-head": { |
| "requests": 36, |
| "quality_pass": 12, |
| "quality_total": 16, |
| "stopped": 22, |
| "length_limited": 14, |
| "acceptance": 0.8767942583732058, |
| "accepted": 1466, |
| "proposed": 1672, |
| "median_e2e_seconds": 6.00265346840024, |
| "repeat_token_parity": 18, |
| "full_token_matches_baseline": 36, |
| "median_matched_output_speed_ratio": 0.7472627789451953, |
| "exact_tokens": 36, |
| "exact_logprobs": 36 |
| }, |
| "trained-head": { |
| "requests": 36, |
| "quality_pass": 12, |
| "quality_total": 16, |
| "stopped": 22, |
| "length_limited": 14, |
| "acceptance": 0.8812949640287769, |
| "accepted": 1470, |
| "proposed": 1668, |
| "median_e2e_seconds": 4.158929664408788, |
| "repeat_token_parity": 18, |
| "full_token_matches_baseline": 36, |
| "median_matched_output_speed_ratio": 1.1084637954727476, |
| "exact_tokens": 36, |
| "exact_logprobs": 36 |
| } |
| }, |
| "NVFP4": { |
| "baseline": { |
| "requests": 36, |
| "quality_pass": 12, |
| "quality_total": 16, |
| "stopped": 20, |
| "length_limited": 16, |
| "acceptance": null, |
| "accepted": 0, |
| "proposed": 0, |
| "median_e2e_seconds": 3.7523877734784037, |
| "repeat_token_parity": 18, |
| "exact_tokens": 36, |
| "exact_logprobs": 36 |
| }, |
| "trained-head": { |
| "requests": 36, |
| "quality_pass": 12, |
| "quality_total": 16, |
| "stopped": 20, |
| "length_limited": 16, |
| "acceptance": 0.857630979498861, |
| "accepted": 1506, |
| "proposed": 1756, |
| "median_e2e_seconds": 4.066564766457304, |
| "repeat_token_parity": 18, |
| "full_token_matches_baseline": 36, |
| "median_matched_output_speed_ratio": 1.0389619272561033, |
| "exact_tokens": 36, |
| "exact_logprobs": 36 |
| } |
| } |
| }, |
| "head_sha256": "56baea0ef88de178c6c858d81adee9e532983798dce81202f21a7483e59001df", |
| "acceptance_by_subset": { |
| "bf16": { |
| "strict": { |
| "tokens": 36, |
| "logprobs": 36, |
| "total": 36 |
| }, |
| "variants": { |
| "original-head": { |
| "authored": { |
| "accepted": 338, |
| "proposed": 380, |
| "rate": 0.8894736842105263 |
| }, |
| "occamy_heldout": { |
| "accepted": 1128, |
| "proposed": 1292, |
| "rate": 0.8730650154798761 |
| } |
| }, |
| "trained-head": { |
| "authored": { |
| "accepted": 340, |
| "proposed": 378, |
| "rate": 0.8994708994708994 |
| }, |
| "occamy_heldout": { |
| "accepted": 1130, |
| "proposed": 1290, |
| "rate": 0.875968992248062 |
| } |
| } |
| } |
| }, |
| "nvfp4": { |
| "strict": { |
| "tokens": 36, |
| "logprobs": 36, |
| "total": 36 |
| }, |
| "variants": { |
| "trained-head": { |
| "authored": { |
| "accepted": 328, |
| "proposed": 384, |
| "rate": 0.8541666666666666 |
| }, |
| "occamy_heldout": { |
| "accepted": 1178, |
| "proposed": 1372, |
| "rate": 0.858600583090379 |
| } |
| } |
| } |
| } |
| }, |
| "timing_caveat": "Diagnostic only. Concurrent activity occurred; BF16 original-head timing included additional requests from an invalid parallel attempt. No controlled training-induced speed improvement established. Invalid NVFP4 attempt excluded in full; replacement NVFP4 run used a separate port and checked served model identity." |
| } |
|
|
