diff --git a/.gitattributes b/.gitattributes index 964754ccda35dfb0a411bfe219fa3f9a79213ca6..6c93463c466ac496d8d2b8cf10cd6da55bc32f17 100644 --- a/.gitattributes +++ b/.gitattributes @@ -46,3 +46,5 @@ tokenizer.json filter=lfs diff=lfs merge=lfs -text examples/odyssey/odyssey-preview.gif filter=lfs diff=lfs merge=lfs -text figures/performance-summary.png filter=lfs diff=lfs merge=lfs -text figures/quality-summary.png filter=lfs diff=lfs merge=lfs -text +figures/runtime-v2-performance.png filter=lfs diff=lfs merge=lfs -text +figures/runtime-v2-quality.png filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 5175954efd8aa87f2b3b993d3922ceccc1145065..9d8f35da2bd3219a9dfddbb654c206eea21fe007 100644 --- a/README.md +++ b/README.md @@ -33,21 +33,73 @@ so MTP speculative decoding works, validated on an NVIDIA DGX Spark (GB10, SM121 - **Weights and complete model files:** [https://huggingface.co/gitcommit90/SenseNova-Loop-9B-Preview-NVFP4](https://huggingface.co/gitcommit90/SenseNova-Loop-9B-Preview-NVFP4) - **Runtime, quantization recipe, evaluation harness and charts:** [https://github.com/gitcommit90/sensenova-loop-9b-nvfp4-dgx-spark](https://github.com/gitcommit90/sensenova-loop-9b-nvfp4-dgx-spark) -This is the **October 30 functional-release baseline**, not the later speed-optimization work. +**Updated October 31: runtime v2 is included below, alongside the preserved October 30 functional-release baseline.** Weights live on Hugging Face; the GitHub companion does not duplicate the 9.38 GB checkpoint. -## Measured results at a glance +## Runtime v2 — final optimization results (October 31) -[![Measured single-user speed, TTFT and aggregate concurrency throughput](figures/performance-summary.png)](figures/performance-summary.pdf) +**Same 9.38 GB NVFP4 weights; new runtime kernels and a smaller MTP draft vocabulary.** +No retraining, recalibration, reduced loop passes, or changed KV/state precision. MTP is supported in all profiles. +The target still verifies over the full vocabulary. Only greedy decoding was evaluated. -[![Small-suite quality preservation and paired uncertainty](figures/quality-summary.png)](figures/quality-summary.pdf) +[![Final runtime v2 single-user decode, concurrency and TTFT](figures/runtime-v2-performance.png)](figures/runtime-v2-performance.pdf) -Figures are regenerated directly from the checked-in evaluation JSONs with -[`figures/plot_benchmarks.py`](figures/plot_benchmarks.py), following the publication-figure conventions of -[figures4papers](https://github.com/ChenLiu-1996/figures4papers). -PNG previews, vector PDF/SVG exports, and plotting dependencies are included. -**Aggregate throughput is not per-user decode speed.** Quality scores are small-subset regression measurements, -not full-benchmark scores or a comparison against Qwen. +| Final matched measurement | Frozen NVFP4 + MTP | v2 exact + MTP | v2 fast + MTP | +| --- | ---: | ---: | ---: | +| Single-request decode, ~1K input (tok/s, median) | 12.4 | **27.3** | **28.9** | +| Single-request code decode, ~90 input (tok/s, median) | 16.0 | **33.4** | **35.9** | +| Single-request document decode, ~4K input (tok/s, median) | 14.3 | **30.6** | **33.3** | +| Eight-request aggregate, ~1K input (tok/s) | 64.3 | **117.2** | **119.4** | +| Eight-request code aggregate (tok/s) | 97.4 | **172.7** | **196.7** | +| Single-request TTFT, ~1K input (seconds, median) | 0.74 | 0.51 | **0.32** | + +Exactly 256 output tokens; unique prompts; no prefix-cache hits; zero errors in these final performance runs. +**Aggregate throughput is combined across requests, not each user's speed.** These are the final matched runs, +not the higher preliminary screening peaks. Full sample counts, token lengths and raw measurements: +[`runtime-v2/RESULTS.md`](runtime-v2/RESULTS.md) and [`runtime-v2/runs/`](runtime-v2/runs/). + +### Which profile should I use? + +- **`MHC_MODE=exact` — recommended conservative profile:** fused Sinkhorn with reference operation order; + cuBLAS routing projection and reference merge retained. Reference routing math matched bitwise in differential + tests; the 12-case final feature check matched the frozen output. This is not a universal promise of identical + generations under every batch/hardware configuration. +- **`MHC_MODE=fast` — faster, numerically different:** also fuses routing projection/merge. FP32 is preserved, + but summation order changes and greedy text can diverge. Choose this explicitly if that trade-off suits you. + The supplied v2 launcher's inherited default is `fast`; commands below explicitly select `exact`. +- **`runtime/` — frozen v1:** preserved for reproduction and rollback. Both v2 profiles use the same pinned image. + +### Quality: report the losses and uncertainty, not just the speedup + +[![Runtime v2 quality results and paired uncertainty](figures/runtime-v2-quality.png)](figures/runtime-v2-quality.pdf) + +On the same 300-item suite, frozen v1 scored **251/300 (83.7%)**, v2 exact **249/300 (83.0%)**, +and v2 fast **247/300 (82.3%)**. Paired-bootstrap 95% delta intervals vs frozen: +exact **−0.67 pp [−3.00, +1.67]**; fast **−1.33 pp [−4.33, +1.67]**. +These small, noisy subsets do **not** establish losslessness or full-benchmark equivalence. + +**Fast's first MMLU-Pro run fell from 70.0% to 58.3% (−11.7 pp)** and truncations rose from 14 to 20 across the suite. +Those exceed the original first-run per-task and ref+3 truncation tripwires. Fast MMLU-Pro reruns scored 71.7% and +65.0%; frozen reruns scored 70.0% and 65.0%. All runs are retained. Variability and the token budget complicate +interpretation; they do not erase the first-run regression. See [`RESULTS.md`](runtime-v2/RESULTS.md). + +The improvements come from **fused FP32 mHC/Sinkhorn kernels** and a **65,536-token MTP draft shortlist** selected +from training-split corpora. The draft is restricted; the verifying target is not. The exact kernel preserves all +20 configured normalization iterations. No learned model weights changed. + +Source, numerical tests, optimized launcher, token-map builder, and experiment ledger are in +[`runtime-v2/`](runtime-v2/). Numerical tests and final feature/quality/performance results are included. +Figures follow [figures4papers](https://github.com/ChenLiu-1996/figures4papers), with +[`figures/plot_runtime_v2.py`](figures/plot_runtime_v2.py), PNG previews and vector PDF/SVG exports. + +## Original quantization baseline (October 30; retained) + +[![Original BF16 versus NVFP4 performance baseline](figures/performance-summary.png)](figures/performance-summary.pdf) + +[![Original quantization quality comparison](figures/quality-summary.png)](figures/quality-summary.pdf) + +These older figures compare BF16 to the **original v1** NVFP4 runtime, not runtime v2. +Their source script remains [`figures/plot_benchmarks.py`](figures/plot_benchmarks.py). ## Provenance | Item | Value | @@ -65,6 +117,7 @@ not full-benchmark scores or a comparison against Qwen. - `mtp.*` (MTP head) is included in `model.safetensors` as BF16 (upstream shipped it as a separate FP32 `mtp.safetensors`). - Tokenizer, processor configs, chat template, generation config and `modeling_sensenova_loop.py` are byte-identical to the pinned upstream revision (see `SHA256SUMS`). +- `runtime-v2/` adds fused routing kernels and a 64K draft token map, with no checkpoint weight changes. - `runtime/` contains SGLang source files from the open SGLang PR [#47281](https://github.com/sgl-project/sglang/pull/47281) (Apache-2.0, sgl-project) plus two small patches of ours (`runtime/patches/`), needed to serve this checkpoint. @@ -125,7 +178,7 @@ Tool calls: 100 % well-formed for both. 0 request errors. Truncations at the max With MTP enabled the NVFP4 model scored GSM8K 96.3 % (n=80), HumanEval 80.0 % (n=40), tools 90 % (n=20); MTP is lossless in principle at temperature 0, small differences come from batch-dependent numerics. -## Performance (DGX Spark GB10, SGLang, same flags for all rows) +## Original v1 performance (DGX Spark GB10, SGLang, same flags for all rows) Prompt 1,051 tokens, exactly 256 output tokens (`ignore_eos`), greedy, unique prompts (no prefix-cache hits), `--mem-fraction-static 0.45`, CUDA graphs on. No MTP rows: 16 requests per level; MTP rows: 8 requests per level. Aggregate output tokens/s (per-request decode tok/s p50 in parentheses): @@ -171,13 +224,13 @@ python3 -m venv .venv-download .venv-download/bin/hf download gitcommit90/SenseNova-Loop-9B-Preview-NVFP4 --local-dir models/SenseNova-Loop-9B-Preview-NVFP4 # Verify the downloaded release before serving: (cd models/SenseNova-Loop-9B-Preview-NVFP4 && sha256sum -c SHA256SUMS) -bash runtime/serve_sglang.sh models/SenseNova-Loop-9B-Preview-NVFP4 +MHC_MODE=exact bash runtime-v2/serve_sglang.sh models/SenseNova-Loop-9B-Preview-NVFP4 ``` **From a complete Hugging Face snapshot** containing `model.safetensors`: ```bash # from the snapshot root -bash runtime/serve_sglang.sh . # NVFP4 W4A4 + MTP (NEXTN, 3 steps), 127.0.0.1:30080 +MHC_MODE=exact bash runtime-v2/serve_sglang.sh . # conservative optimized W4A4 + MTP, 127.0.0.1:30080 curl -s http://127.0.0.1:30080/v1/models python3 runtime/probe.py http://127.0.0.1:30080/v1 sensenova-loop-9b-nvfp4 [image.jpg] # text / thinking / tool / image smoke test # options: --no-mtp ; W4A16: SGLANG_DISABLE_SILU_FP4_QUANT_FUSION=1 runtime/serve_sglang.sh . --fp4-gemm-backend marlin @@ -185,14 +238,24 @@ python3 runtime/probe.py http://127.0.0.1:30080/v1 sensenova-loop-9b-nvfp4 [imag OpenAI-compatible API; thinking is toggled with `chat_template_kwargs: {"enable_thinking": false|true}`; tools use the `qwen3_coder` parser (the model's XML `` format). +### Profile selection and rollback +```bash +MHC_MODE=fast bash runtime-v2/serve_sglang.sh /path/to/model # fastest; see numerical/quality caveats +MHC_MODE=exact bash runtime-v2/serve_sglang.sh /path/to/model # conservative optimized +bash runtime/serve_sglang.sh /path/to/model # frozen v1 rollback +``` +The launcher replaces only its named model container. No extra quantization step is required. +Benchmark reproduction is documented in [`runtime-v2/bench/README.md`](runtime-v2/bench/README.md). + ## Limitations - Upstream is a *Preview* model; all upstream limitations and intended-use terms apply. - The serving path depends on an **unmerged** SGLang PR plus our patches, pinned to one nightly image; it was tested only on GB10. Pipeline/tensor parallelism and other GPUs were not tested. - Evaluation is small (300 items, thinking off, contexts ≤ ~3 K tokens). Long context (> 32 K), video input, thinking-mode quality at scale and non-English quality were not evaluated. Only greedy decoding was benchmarked. -- The mHC Sinkhorn routing runs as many small FP32 PyTorch kernels and dominates a large share of GPU time in both BF16 and - NVFP4; a fused kernel would likely improve speed further (not attempted). +- The frozen v1 mHC path uses many small FP32 kernels. Runtime v2 fuses those operations; the remaining profile is more GEMM/output-head bound. +- Runtime v2 fast is not bitwise equivalent to v1 and did not pass every original first-run quality tripwire. Exact is the conservative option. +- The draft vocabulary shortlist was built from training corpora; no broad sampling-equivalence or full-context quality study was performed. - In this SGLang build, `--fp4-gemm-backend marlin` crashes unless `SGLANG_DISABLE_SILU_FP4_QUANT_FUSION=1` is set (upstream bug: the fused SiLU+FP4-quant path is enabled regardless of backend). - Quantization may change behaviour on inputs unlike the calibration mixture. @@ -200,7 +263,8 @@ OpenAI-compatible API; thinking is toggled with `chat_template_kwargs: {"enable_ ## Files - `model.safetensors`, `config.json`, `hf_quant_config.json` — NVFP4 checkpoint (+ BF16 MTP head, FP32 mHC). - Upstream (unchanged): `tokenizer*.json`, `chat_template.jinja`, `*processor_config.json`, `generation_config.json`, `modeling_sensenova_loop.py`. -- `runtime/` — tested launcher, SGLang overlay and patches, probe script, compatibility notes. +- `runtime-v2/` — optimized profiles, Triton source, draft token map, final results and differential tests. +- `runtime/` — original tested launcher, SGLang overlay and patches, probe script, compatibility notes. - `quantization/` — quantization/verification scripts, stats, structural verification report, environment lock, calibration manifest. - `evaluation/` — quality gates, eval summaries, paired comparison, perf JSONs, kernel trace summary, reference checks. - `SHA256SUMS` — checksums of every file. diff --git a/SHA256SUMS b/SHA256SUMS index b0d71ce99428758ca4cf0047164af5dcc0d235c3..14a7e1b19f162008b9fb32238adb004fdfd8d30c 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,6 +1,6 @@ 8951a99119cdcd06c699d91ccc5db0a40b595961041a5ec2e6103b88112757e8 .gitattributes c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4 LICENSE -60e1c4899e3a776a61e0223abb3ed02317be0fae58d3beb24d86dfbc981fd408 README.md +f7e05b28104f305f9fe6bf5cd25bc17e67c72cd8a7b0b29ea511f5f7f3a966bc README.md a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715 chat_template.jinja 5bec5643138a09f2917a0b7471a673baecfbf85f59af49cc048f4318f32e7a3d config.json ccdb9203010fc0534d37b0b475d85f088f5e62568d3cbd76229af4d8e50ffb28 evaluation/QUALITY_GATES.md @@ -33,15 +33,22 @@ ec88f909edcbf225936bbeac88ede9312520593eb53db976d5079d9dd6c45ed1 evaluation/sum c6d75330180b2d1bd5f2dda574afeee8844e18e44c46ba0f3a1128eb052d2326 evaluation/summary_sglang_nvfp4_v1.json aff7bcc52ad3e83e6394a8d31e733d8b5725d213836beb98824c397e726c66c6 evaluation/summary_sglang_nvfp4_v1_marlin_w4a16_gsm8k.json 944d3ce06cffbfb7a8e10c1af186aaa62a196e6b96cd8c3a1e6800e5d998fa70 evaluation/summary_sglang_nvfp4_v1_mtp.json -e6df880d4632abe34376bbf9d349ff694129e21067f4e58a757919715804a8b9 figures/README.md +cc0be931111ecb00e1ceef1d24adfa6cd1e0af04167ce6d3a999a0cc08883d89 figures/README.md 47eecb7dab94c2cbe44756977f2e4e81380a059f1661060c0ac5e0909d9b1c24 figures/performance-summary.pdf 7012b65a258263617bcd3a9d57e93c4cb24836d9da35a69d723a0f6ef6416d68 figures/performance-summary.png 975894bd93a506565d81573ecfc75b9e6202406c2f21940750ec1ad7c9635e8c figures/performance-summary.svg 058f67de2a71ee67cf48f4976b69d353cfaea0607bb2d91a20e33774c35c2df7 figures/plot_benchmarks.py +277d66c8e57b9fe00c599a526d378e798d55be661d1a29da9ee6fb4eaaa691e0 figures/plot_runtime_v2.py ceca3cd14f612774aad2f9f076a1d77177bd6f6090c41f8a3ca985318ada3793 figures/quality-summary.pdf 761cb0de5f120ec2934b7b900b0d4961464ba24686b4175095289e9d245b0a04 figures/quality-summary.png 0315f8e622b4b9cb9ca36ed3718403ba64ea4f8f3df582853fd3e4c0b4fe15c5 figures/quality-summary.svg 472fcc1fa6325424ea016e5fb386728d1bf661e21582f7e2e829387053eb9a0e figures/requirements.txt +a2e2fe69b92b12eaedab329f96b8434fb5f42cbfdbefb7a16742be9dd62e0906 figures/runtime-v2-performance.pdf +6b0145854cf885771b77dd327e3d5cb0a742b04c9c744bf24006ba98f4569313 figures/runtime-v2-performance.png +4de7bc0cd7d6f953bf1c864b0a66dbda1ef425d18c5f98d2ca14396df8e61f98 figures/runtime-v2-performance.svg +8c26bbb48e9646822b64e2624b94cb4b4502c2d0987e74b6eff0b2cec17f4a09 figures/runtime-v2-quality.pdf +c66a6044c2160795ef18de6906060864ed793adfe08a27a8fce20e5f59306695 figures/runtime-v2-quality.png +d6422b140d441f9028b53b6f6035cb3296a4c55f96b1ecbfc249e0644136e6ab figures/runtime-v2-quality.svg f288bb4281580138395c6acc7b3adb62f45c21530bc0708cb2cf9edf093f6a4f generation_config.json b197af747a1f072419a8e7e2db3252d9aeea84d21f65db37f748430ab7f83bfd hf_quant_config.json 1e420ea69111b7bb3c7c529cdbbb4ff4e6ebb531c0cd2be5c634cd88cd40eaca model.safetensors @@ -57,6 +64,145 @@ dc54da6f2e37136c3172779d86ae0134a353942f341eaa8934d3000a01f34682 quantization/b 4d71bc0045ee80c45f5eef89e5e721e064e2505020b327942501b017daf0fdf8 quantization/quantization_stats.json 723419a376e5f2339f4f0d4f9376b16ea2d5359e54659ccb85c2bb2d5c3028fd quantization/requirements-quant.lock be632c83213d44b4a18f367c59fcb55f5d0d738b8c67f64ccc61df3c6a9fb598 quantization/verify.json +32dae3052f331ee34d628ef535709b301259a45df7c7522c4d35dcf49873f00b runtime-v2/.gitignore +4aa348318e825643f1edb8b4516862de1dca0cc6c56960b10619676a03b6a5ac runtime-v2/LEDGER.md +f43e21163bea7f9f6e9adcd73c8ac9914fc4b4988f0ef917b2071792c94879f4 runtime-v2/README.md +5373b331a9ad0ca78e3bf362f4d3974808e8f820366781c76318f8abfa23c9d0 runtime-v2/RESULTS.md +8d58f6918095d93a67b83b267ee51e48e5895b2d54e30ccdd5d2b7c7e60128ac runtime-v2/bench/README.md +4b5700057f8bbabf7a42e40fcf361710fb0a1976a9353abbdc1d79bc38ef564e runtime-v2/bench/fp4_gemm_bench.py +95a4ed870e9bb68e143fa459a55eb84963617ba5f0cda023a35f6e08082a17f3 runtime-v2/bench/prof_drive.py +b95119c8ad457bf182319d4a8e10cc67d278efa9c5458a0caa2df4d941372e44 runtime-v2/bench/report_tables.py +27e569ec5be6157c31e81ff6d09c62871542e82b1a397c95961a8a347e60de65 runtime-v2/bench/screen.py +d548ad5c9741c59e51e32a78a39c305594a62a12bf13b06b06195b6f5d1f5c2f runtime-v2/bench/trace_graphs.py +8117abdb9b399c44a57ee0a5ad32831df0d83fc68880e60e24cb9ca65fee74fa runtime-v2/bench/trace_summary.py +5bb587d19dce89cbce317a812a17e30891150afca578f60aa94f1a77b4070c46 runtime-v2/bench/trace_summary_lib.py +2f1d8b4d676321dce894f03f97fa17d99842a73054605d9e7f70c3f8426bc806 runtime-v2/bench/wait_ready.sh +e00d2fafe780fcf4c04ab52b040e43ec7143c7bd6e070637563f2724b59888d1 runtime-v2/overlay/python/sglang/srt/configs/__init__.py +bdc7fa2055b799ce07441a5d499fafa554b7b797d4c4a24fd251b846b51260fd runtime-v2/overlay/python/sglang/srt/configs/model_config.py +eb6df8f37ac06e6f439dd24f687e5bf1078599ade28acaa20d8799bf77c7e057 runtime-v2/overlay/python/sglang/srt/configs/sensenova_loop.py +b3b4959f2336d1afdf4f0353a0a0eecfc38495226912022dd862489cbb532c98 runtime-v2/overlay/python/sglang/srt/layers/rotary_embedding/mrope_rope_index.py +68bc3cfe9749f90ecc817ccede7fdbd9167684e902637dbec0b738a9d7ea7e25 runtime-v2/overlay/python/sglang/srt/layers/sensenova_mhc_fused.py +87ccee5659fb8d95f7f54a038d7d0f71ba8afab04684c72c6bd1556e31d119ce runtime-v2/overlay/python/sglang/srt/models/sensenova_loop.py +7fc07bd18b974aa8f54013f2bb39536a65d32ad0cc1a1df2d239ead0dc136754 runtime-v2/overlay/python/sglang/srt/multimodal/processors/qwen_vl.py +68e1076237ed67adfa06530139d79df0783a2519620443a58cce94c5143d56c4 runtime-v2/overlay/python/sglang/srt/utils/hf_transformers/common.py +d93e6342f9487567a7847443eb47b41701b4510474cd5664c8dc51b37eb6f6fd runtime-v2/patches/model_config_mtp_draft.patch +029fa25bb4ea5a16c9fce8de412e7ea6481fc540508c63bdd629026b9a56f926 runtime-v2/patches/sensenova_loop_quant.patch +cfd86c1b46c3ac7592ed50ec5641d36e1a69025791bd986907b3c6b54c31a4b6 runtime-v2/patches/v2_mhc_fused_router.patch +aeabae0b23eb5eea12d4d5c9fc57b115bbf702d90019a61cb1bdd6b912cde07e runtime-v2/runs/chain1.log +8eac091fab019b0d871b8c20c4ef240f758ef512fefa7bdb2e5f3dd6267df276 runtime-v2/runs/chain2.log +76984e7fc55c0065fb8c9d1646b4f61de0c24b9700fd21348fce82f1fffa5a22 runtime-v2/runs/chain3.log +f82fb0102b5a9cea9aa39c720d35781128a7d4330d26e808a101ecbaef4c617a runtime-v2/runs/final_chain.log +3cb5e44353508026e19a34c728a0cfd62b234a1de02bb2e501bd904b856be246 runtime-v2/runs/final_frozen_v1/eval.log +ae09e6590b834fe3d2eb4d0efbe1710f745fb3a2dfb894111035a81c1a0273bf runtime-v2/runs/final_frozen_v1/eval/compare_vs_ref.json +c7b22c9374b33ddbf580521917bb226e1535a28464d682690d4c0b4b74d0af65 runtime-v2/runs/final_frozen_v1/eval/raw_gsm8k.jsonl +6c24a6a63e53f694029de6fd48bf4b0a31e8b3a8d4978ff0c1c2b1274627402f runtime-v2/runs/final_frozen_v1/eval/raw_humaneval.jsonl +d6102c5faacfc6d6460a076c4a4e73c09f4c3fc4b1f69c26694fa473c63a200c runtime-v2/runs/final_frozen_v1/eval/raw_ifeval.jsonl +723bc26360e96ea005eaf282c6c400104ad26a3df22a13328fbdc35b07164549 runtime-v2/runs/final_frozen_v1/eval/raw_mmlu_pro.jsonl +37b7b5b550d8f3ced41d5fc0c33b3268b8112442412bc7e9216d58ccb0c2797f runtime-v2/runs/final_frozen_v1/eval/raw_tools.jsonl +fe1c19237fa5ab99b626d44830afb7dc7c8bc1ecacb1d5b6ac09729f3e6be0cd runtime-v2/runs/final_frozen_v1/eval/raw_vqa.jsonl +233917924301edfed636e906a80fbc7180b0c8f1465fd1df32b2acd6db9159e3 runtime-v2/runs/final_frozen_v1/eval/summary.json +138edee8b67c2f8cdc9fbe34b48067ec99dc9b2b20729611e6a856af6f74af71 runtime-v2/runs/final_frozen_v1/eval_compare.txt +69bf0e6859cd641d10341d0575012918944b49c59b9dada302324cb6b6df1476 runtime-v2/runs/final_frozen_v1/gpu_after.txt +03ff9c79c916f05ec992381c0f635426e32cbe907d41f91d506d4cb1ffdf666c runtime-v2/runs/final_frozen_v1/gpu_before.txt +a3e9d1074d6b2b2d6af2eff5f1c35761e46067243a45c9d318562273dfea5417 runtime-v2/runs/final_frozen_v1/greedy.json +5a29fa8c0bafdc6dcb66c6172008d892cb8144ec5066a2e79d3512b870164c37 runtime-v2/runs/final_frozen_v1/greedy_diff.txt +3d3193ebc85d9703ceca209b0e9b2b8bdf8729dfd40bfb4ca36e1d6b05738ee6 runtime-v2/runs/final_frozen_v1/mem.txt +d835c86c49b359ee814f608bc268ad1c843426bd4722d9b048ac3a7f7ec45579 runtime-v2/runs/final_frozen_v1/perf_matched.json +e292a5bcb39f3793fbf94d7ca6bcaf80f7bd3427e8522fc9ef463646563dbacb runtime-v2/runs/final_frozen_v1/perf_matched.log +c9501741babdc2cbfc877be168b64621cfa1cc65bc4606ed93079b7d9bdbe71c runtime-v2/runs/final_frozen_v1/screen_code.json +6f6459cd295bbe224275e829e2379017f8e7a028a3f03e6de3b67b5ee14ac00d runtime-v2/runs/final_frozen_v1/screen_doc.json +4e2ead04c10b476e955991de1d1527dac816545fea5bac42042cdb12d0453ff4 runtime-v2/runs/final_frozen_v1/startup.txt +1ab6b379ed646e189a284b44408989a79637a2c6f478aa7c03e47dc6b1418f8f runtime-v2/runs/final_v2_exact/eval.log +1be03124a62cea4fe99ea8dae682947570adf8e004061bec0a603bd04935ac42 runtime-v2/runs/final_v2_exact/eval/compare_vs_ref.json +7193b8d845182d28f4f3f71cad94e3db146c65315384ac85b29e39c094e00ce7 runtime-v2/runs/final_v2_exact/eval/raw_gsm8k.jsonl +0017ed83f0e36cd6ec678eb1158efae0bbf850715be242e11ba41fa2fc6a3b5d runtime-v2/runs/final_v2_exact/eval/raw_humaneval.jsonl +b2b5ea9fed29ff5317ee06aa4dee888cb2c60a64d33f188f49b0126797643ed5 runtime-v2/runs/final_v2_exact/eval/raw_ifeval.jsonl +107687c7b517761160a2d8b35ccad29a10ac5667bdbbb3aff60fd20aeda19a92 runtime-v2/runs/final_v2_exact/eval/raw_mmlu_pro.jsonl +975732afa2c61c16dc6e8877c72414adb4f97b2aa778c184ac32f9e33e56b2c1 runtime-v2/runs/final_v2_exact/eval/raw_tools.jsonl +84a9e2908ce2dacc1b863e189ef1223f60f25256f46d3dc8ea0a20886c63106a runtime-v2/runs/final_v2_exact/eval/raw_vqa.jsonl +a50e1b698802fcdc0a1bac9e68a2e73a5fd57c53cdbe1e7bf4767cd776e284fe runtime-v2/runs/final_v2_exact/eval/summary.json +c0e7cb0b03610f45de18bd1ff73e7f9f5c766b1d687229857c69230f3e9e6596 runtime-v2/runs/final_v2_exact/eval_compare.txt +ac7f0ee9c9c9d76b3f78530de8078f14f12b180bfb4bb87a8cb906101e5fc4de runtime-v2/runs/final_v2_exact/gpu_after.txt +33d8f8e097ac88d774be66b5375867309baf2cc1e71b7ddfa083499e04669270 runtime-v2/runs/final_v2_exact/gpu_before.txt +5587c74db231fdb0f009e1fc521673c9464c8d16aa60df08d5729544315786fa runtime-v2/runs/final_v2_exact/greedy.json +5c6c788f0353dd10fb95bbd2cac62af3a774514b3d94acc6f0177f06e17f13cf runtime-v2/runs/final_v2_exact/greedy_diff.txt +4fb42769c273405b3cca964c55857838ec5ae5f505be5aa6608f9873d40abc1a runtime-v2/runs/final_v2_exact/mem.txt +0765ac69ab713c826222911c7a3b348ce6d63622b0990a3d858d6792bb01803a runtime-v2/runs/final_v2_exact/perf_matched.json +8d920e9d0edeeb83fc6f0500dff75a12225bc6d76c3ce59d4d4ceb593cb390b7 runtime-v2/runs/final_v2_exact/perf_matched.log +83ec9f38a72972d41fcd91d40dd7feb13c2dd7a3925f92653b03b93ab750e0c2 runtime-v2/runs/final_v2_exact/screen_code.json +c4e0990219587fe88a78488d08e32fe76eeb4d4d77535626f644e97387595128 runtime-v2/runs/final_v2_exact/screen_doc.json +8ff918ad3e2c406fdc71be2997f998bbe8b71dd9330df1fbe8df4571f590e117 runtime-v2/runs/final_v2_exact/startup.txt +92cbd830f90bd66caa59be28ec3c8a0494abc089100472392eca1636281e292b runtime-v2/runs/final_v2_fast/eval.log +dece7c45c38bcbdb721fabb340b72bc1410735df883146a440aff629bea1998d runtime-v2/runs/final_v2_fast/eval/compare_vs_ref.json +6b8f184b2b8a1af194a06edeaec8a46c5ec1c1deec246f5cf646fe7dcdef3d67 runtime-v2/runs/final_v2_fast/eval/raw_gsm8k.jsonl +40dcf3d22f9c3727e2c6112561cbe128e72180a909ad8e51d8bb8e75d5cba75b runtime-v2/runs/final_v2_fast/eval/raw_humaneval.jsonl +e383f09c1a104b9f0413240e11456b962d9b736ea91fff64eec3daff762b9616 runtime-v2/runs/final_v2_fast/eval/raw_ifeval.jsonl +ccfc6392d774720393c16ce659dfb50b58c110f4e5ea4cb745b3df1ee7bdb4f3 runtime-v2/runs/final_v2_fast/eval/raw_mmlu_pro.jsonl +73d5fc4a7f459500e61fb1f0a2d073c69d2f5e9befd2c7fa2b04d1594efe9cd1 runtime-v2/runs/final_v2_fast/eval/raw_tools.jsonl +771af56bd6e7f91dd9a20ba6d271f07846c0eccdac0d4ffe0cb82991c4d0a1ba runtime-v2/runs/final_v2_fast/eval/raw_vqa.jsonl +92df69d5c58818a00d2071d14dad8f0155789297b7582de15a583dd4a98c4a71 runtime-v2/runs/final_v2_fast/eval/summary.json +1bf1c5ca785fce32bf0a7248f91e070e7b4c9588369e863d2438bf4d7c788276 runtime-v2/runs/final_v2_fast/eval_compare.txt +e6c066419165ce26f5599e0efc5a41d4fc0b1f59bb862c15cf763a4f3ba3a0ea runtime-v2/runs/final_v2_fast/gpu_after.txt +f3da2a51469f7551f24bfe0833be0567b0374f5ce8caa0d9631fe0327d00221f runtime-v2/runs/final_v2_fast/gpu_before.txt +630d186e382f56a9d42a8f4cb3a5f2ff923f77a3876b58a72e99c44befc45631 runtime-v2/runs/final_v2_fast/greedy.json +37eda03a85270d0c0629ee54c4965e003c06536063f4100a44baff3c31da965e runtime-v2/runs/final_v2_fast/greedy_diff.txt +6eb0bb17cf5b6b0e8a652ef7fb6d31a11485b1074182ce6752a970494de9acf1 runtime-v2/runs/final_v2_fast/mem.txt +ec82b03916c6d8b88c47bb041136182d4139fd82b759b3190e680b0ec4bf5082 runtime-v2/runs/final_v2_fast/perf_matched.json +efc50e8834d005b2eb1b7fb1d0179987eea4f1381c12bdcc932fef96d01d5f09 runtime-v2/runs/final_v2_fast/perf_matched.log +3c5bf2f1dd670eb8dc3a48edd12e0a8685543541a146d25600dbd9c6f572c999 runtime-v2/runs/final_v2_fast/screen_code.json +8f591e3b6c57b106db8e9fb4df8a99a54fcf3184ac68aebfb9f953e9dfc7891a runtime-v2/runs/final_v2_fast/screen_doc.json +0e35cc376c50fd8efe4dedff41a05c94e2619c0cee896250fed3bd0506555e50 runtime-v2/runs/final_v2_fast/startup.txt +ea4b7427752378606219e1cf9357ba40aa67dd33235c12301d500c16a997a71b runtime-v2/runs/hiconc_full_mtp_code.json +0a9ba9984b6cbb131ab25fd6d8f8af8057141dbafcefe4d7f4b762943968a125 runtime-v2/runs/hiconc_full_mtp_synthetic.json +6b93bc4ddd725b554bd3a4967135a8c099a5ad977362b553a780a724d1fbf896 runtime-v2/runs/hiconc_full_nomtp_code.json +8a719835ac522fb33336a5e88a274165bd08ebb9fecf2c668fe699e8358139ec runtime-v2/runs/hiconc_full_nomtp_synthetic.json +66b3ffa838ab282f36ad13cae239e8b4f964ccfdbcc68e08ce706404bc8b01d0 runtime-v2/runs/screen_exact_nomtp_code.json +7c42a88aff1c10777774242f8361cc06382c366d23e3e8e9e300952037b2e2ee runtime-v2/runs/screen_exact_nomtp_synthetic.json +1fb4c028f4786cac1dbf522a38a02a32855539b4dae5bf91add73b9d56722f89 runtime-v2/runs/screen_exact_tm32k_code.json +d608a24eaec0b3124b9c0df90e8fb24a4cf62a3d04f931408f710ef954b0cf31 runtime-v2/runs/screen_exact_tm32k_synthetic.json +1937cc250984d9af1c34fc22b5ca39158d8701eaec2cd78d1cd3b573db8287c8 runtime-v2/runs/screen_exact_tm64k_code.json +d0aaaa27413ca37e111e647db68196948b3278af2f7153058851c168d691dbe3 runtime-v2/runs/screen_exact_tm64k_cutlass_code.json +6f6feb4492963164dbc7dad10f086b96d70cbf68adb83fd27662459de492c422 runtime-v2/runs/screen_exact_tm64k_cutlass_synthetic.json +064eeb61e42a384264749e2902bf0e7c255c6614c158f2c917e70f37f9dafe23 runtime-v2/runs/screen_exact_tm64k_marlin_code.json +c0073c475b7cbd4a6182cecd30cffe17a56ddb98918b559a3d7a5a26453ce437 runtime-v2/runs/screen_exact_tm64k_marlin_synthetic.json +0ad793cdb2cf70fd030c5968f0816772e86d0f0a780064b9b1632f2aff7850e3 runtime-v2/runs/screen_exact_tm64k_s2_code.json +14d4d07b9a95d40f609b487921077ccef2bf57b2f9d25f3c5e020c3ae9fd2676 runtime-v2/runs/screen_exact_tm64k_s2_synthetic.json +5d709710bb44b230b6ee08cf6a562ea45981949935df624b215ac17d4db5cd3b runtime-v2/runs/screen_exact_tm64k_s4_code.json +23c561017cbcff5f3a2730da4992f3f65910026ed310d1947f689e0292163c0b runtime-v2/runs/screen_exact_tm64k_s4_synthetic.json +e1b1fb5cc625e5aa44742c12982bef45e194ec3b59394f7ed110732fc470f2c9 runtime-v2/runs/screen_exact_tm64k_s5_code.json +cdc7bbf511211f8b688d6abcb8278ae7ee45b38d9246c16dae205cacf9b43fdf runtime-v2/runs/screen_exact_tm64k_s5_synthetic.json +da53ac443d172f43721562867a1ccacc8775bf785dbd787816c7d5fd918aaf21 runtime-v2/runs/screen_exact_tm64k_synthetic.json +42f790ca23bf3707695de96fdb2edc8ef31099433ed74073d847cc2c2c41db2c runtime-v2/runs/screen_exactfused_code.json +df8cc82b05501ccc78770b178ef3fcbb5a2c5e2b24d8aed72c0c732737a66a4e runtime-v2/runs/screen_exactfused_synthetic.json +68bff4c8a1d0c80151d734141d8e933c1e8ec64041cb00dca1357741fd8f6921 runtime-v2/runs/screen_frozen_code.json +1ef02dcabdc8052687a7f10093589d3489791af6fdd7db57a61bed59de2640db runtime-v2/runs/screen_frozen_synthetic.json +c30ea79feba85166f0907f328e89173db73d859ba309a008ec48e7f3ebf800e3 runtime-v2/runs/screen_full_tm64k_code.json +3759cbc1e65b4281836a19cfd9bd1ec2d885e6af3dd1d522b97257bfdb75acfb runtime-v2/runs/screen_full_tm64k_synthetic.json +fdebbccb4ccb10b29f43a3339671ac155f756afd6d27a8765c04ceb3f8320567 runtime-v2/runs/screen_fused_code.json +3744bf31104e186420a0103a2aab4578a066d6ae189f75a745038a1e64b7bd2a runtime-v2/runs/screen_fused_synthetic.json +b2a551e06fbe60688f584d30d3c0155a9c362e0d94f677bc381a09df9868f43b runtime-v2/runs/screen_tm32k_code.json +a9cb5f518f107ebae9e68aef34ef4859432f5d3df28e056b21010a7c2080182a runtime-v2/runs/screen_tm32k_synthetic.json +0e4fed3fa486fa7785aff17c0d1bfffa2ff32a6995a48e12a8f81fa197bc8168 runtime-v2/runs/test_mhc_fused.log +73341e398b509936823aeed0a988bf4979cadddc86dcf835967722741dbc68e0 runtime-v2/runs/var/frozen_mmlu_1.log +fb767a933a78863687942d4d229c8a83db618402a788da3451fec189f1d2ea9f runtime-v2/runs/var/frozen_mmlu_1/raw_mmlu_pro.jsonl +96cc5e3107540e10a06de960a373db2a36c7197823da58487ff35d54daca4695 runtime-v2/runs/var/frozen_mmlu_1/summary.json +882f35e5ea9cbdf01c9e384a51b0a73cbbd080093dfea7b09d5521bcaff6935f runtime-v2/runs/var/frozen_mmlu_2.log +cea910838afd0172f8c3996d936bda06d211ac390bf2955a5b67834dcb363b0c runtime-v2/runs/var/frozen_mmlu_2/raw_mmlu_pro.jsonl +3ad1cd1d31662f10549a4896e9f95ce5cced9df1aaabc584b932024fe4f180f4 runtime-v2/runs/var/frozen_mmlu_2/summary.json +6eeac27f8871fcdebccdfeb68b967dd426471b000d386b2d91fd9e8bd9157f0d runtime-v2/runs/var/v2_fast_mmlu_1.log +e32b230c458f1f34167320e18a82c9f67d71f9be5595baf8fe329dc493dff65d runtime-v2/runs/var/v2_fast_mmlu_1/raw_mmlu_pro.jsonl +a123b42dd5e412d7b6ba769ed28f6baabc5d5e6eb991feb6701e61b3b2756c54 runtime-v2/runs/var/v2_fast_mmlu_1/summary.json +ea40811ae5e7b86d215368593563b73043e9c81796a57a37df01801b5402e561 runtime-v2/runs/var/v2_fast_mmlu_2.log +b9f451c6035eaafde49e67513e321c7b97301c019fc0e59a1053c08e2c46e67d runtime-v2/runs/var/v2_fast_mmlu_2/raw_mmlu_pro.jsonl +fd36f9094ea1f847fe0577c05954c4400d93720dba30011eb03ea36a7bf455da runtime-v2/runs/var/v2_fast_mmlu_2/summary.json +333515ef56dddab835aaf72d7d3e41e69acd13feddd29b3eea1a796cdba2c8cf runtime-v2/serve_sglang.sh +608f51ea2fc61860cd74e414c506310abd5bc196b6e98c36a34b58d6c61d08b3 runtime-v2/tests/greedy_outputs.py +f70ffa501857fc0957c3ffbcf6a4143bea059dcf4a02cf7ba568adf5fe45e2ec runtime-v2/tests/test_mhc_fused.py +0e4fed3fa486fa7785aff17c0d1bfffa2ff32a6995a48e12a8f81fa197bc8168 runtime-v2/tests/test_mhc_fused.result.txt +68ea210548cadb0adcc84c9c312bb22b6a4713656ee302c3b0d08f4d18dc1aef runtime-v2/tokenmap/build_token_map.py +3f15c5ab3623692cd2ac59a3fa4bd10cf41bfa6f36049b3a182bb82be99af31e runtime-v2/tokenmap/hot_32k.pt +119fa796636a94aa5bc9b3cc31e0410d34d4a6f4e023b9ee8ed783320c916a57 runtime-v2/tokenmap/hot_64k.pt +e12571bb950e842d6ee21782ca5f15bc7b2375461800744db589c163bb5fec08 runtime-v2/tokenmap/token_map_stats.json 1b95e589cd65a97a97fae93cdeec07fcadbf0ff204be746fe20db32550a2c248 runtime/COMPAT_NOTES.md e00d2fafe780fcf4c04ab52b040e43ec7143c7bd6e070637563f2724b59888d1 runtime/overlay/python/sglang/srt/configs/__init__.py bdc7fa2055b799ce07441a5d499fafa554b7b797d4c4a24fd251b846b51260fd runtime/overlay/python/sglang/srt/configs/model_config.py diff --git a/figures/README.md b/figures/README.md index 4ab6bae75e7da787c29a4b54b183bbc616dddfff..81007633bf3d33ce620b5e40c09eb5a68d46012c 100644 --- a/figures/README.md +++ b/figures/README.md @@ -15,3 +15,9 @@ Quality panels show task scores and paired-bootstrap 95% intervals from the orig These intervals reflect this small item set, not run-to-run throughput variation or full-benchmark accuracy. The originals are 300-DPI PNG plus vector PDF and SVG. No new GPU benchmarks were run to make these figures. + +## Runtime v2 final figures + +`python figures/plot_runtime_v2.py` reads only `runtime-v2/runs/final_*/` JSONs. +Original v1 figures remain unchanged. New charts separate final matched runs from screening, and +show the first complete quality runs and paired intervals, including fast-profile regressions. diff --git a/figures/plot_runtime_v2.py b/figures/plot_runtime_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..bbea85c1a6e03f621a7c4907e7158ddc5a37a538 --- /dev/null +++ b/figures/plot_runtime_v2.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Final runtime-v2 figures, read only from archived JSON evidence (figures4papers style).""" +from pathlib import Path +import json +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import numpy as np +ROOT=Path(__file__).resolve().parent.parent;OUT=ROOT/'figures';DATA=ROOT/'runtime-v2/runs' +plt.rcParams.update({'font.family':'sans-serif','font.sans-serif':['DejaVu Sans','Arial','Helvetica'], + 'font.size':12,'axes.titlesize':14,'axes.labelsize':12,'axes.linewidth':1.8,'axes.spines.right':False, + 'axes.spines.top':False,'legend.frameon':False,'svg.fonttype':'none','pdf.fonttype':42}) +NAMES=['Frozen v1 + MTP','v2 exact + MTP','v2 fast + MTP'];DIRS=['final_frozen_v1','final_v2_exact','final_v2_fast'];COLORS=['#767676','#3775BA','#0F4D92'] +def load(d,f):return json.loads((DATA/d/f).read_text()) +def save(fig,name): + for ext in ['png','pdf','svg']:fig.savefig(OUT/f'{name}.{ext}',dpi=300,bbox_inches='tight',facecolor='white') + plt.close(fig) +fig=plt.figure(figsize=(13,8.4));gs=fig.add_gridspec(2,2,left=.085,right=.98,top=.83,bottom=.23,wspace=.30,hspace=.63) +a=fig.add_subplot(gs[0,:]);b=fig.add_subplot(gs[1,0]);c=fig.add_subplot(gs[1,1]) +fig.suptitle('Runtime v2 — faster serving, unchanged NVFP4 weights',y=.98,fontsize=19,fontweight='bold') +fig.text(.5,.925,'Final validation on one DGX Spark • W4A4 + MTP • October 31, 2026',ha='center',color='#4D4D4D') +x=np.arange(3);w=.24 +for i,(d,col,lab) in enumerate(zip(DIRS,COLORS,NAMES)): + vals=[load(d,'perf_matched.json')['levels'][0]['decode_tok_s_per_req']['p50']]+[load(d,f'screen_{k}.json')['levels'][0]['decode_tok_s_per_req_p50'] for k in ['code','doc']] + bars=a.bar(x+(i-1)*w,vals,w,color=col,edgecolor='black',linewidth=.8,label=lab) + for q,v in zip(bars,vals):a.text(q.get_x()+q.get_width()/2,v+.55,f'{v:.1f}',ha='center',fontsize=11,fontweight='bold') +a.set(title='A Single-request decode (not aggregate throughput)',ylabel='Median decode tokens/s',xticks=x,xticklabels=['Matched synthetic (~1K input)','Short code (~90 input)','Document (~4K input)'],ylim=(0,44));a.legend(loc='upper left',ncol=3,fontsize=10);a.grid(axis='y',alpha=.16);a.set_axisbelow(True) +for d,col,lab,m in zip(DIRS,COLORS,NAMES,['o','s','^']): + data=load(d,'perf_matched.json')['levels'];xs=[t['concurrency'] for t in data] + b.plot(xs,[t['aggregate_output_tok_s'] for t in data],color=col,marker=m,lw=2.5,ms=7,label=lab) + c.plot(xs,[t['ttft_s']['p50'] for t in data],color=col,marker=m,lw=2.5,ms=7) +b.set(title='B Matched-workload concurrency',xlabel='Concurrent requests',ylabel='Aggregate output tokens/s',xticks=[1,2,4,8],ylim=(0,145));b.grid(axis='y',alpha=.16) +c.set(title='C Matched-workload first token',xlabel='Concurrent requests',ylabel='Median TTFT (seconds)',xticks=[1,2,4,8],ylim=(0,6));c.grid(axis='y',alpha=.16) +fig.text(.5,.075,'Exactly 256 output tokens • greedy • no prefix-cache hits • 0 errors in final performance runs\nMatched: 8 requests/level; code/document: 4 at c=1 • final results, not earlier screening peaks\nExact preserves reference routing math; fast changes FP32 reduction order and may change generated text.\nSource: runtime-v2/runs/final_*/{perf_matched,screen_code,screen_doc}.json',ha='center',va='center',fontsize=10,color='#4D4D4D',linespacing=1.6) +save(fig,'runtime-v2-performance') +keys=['ifeval','gsm8k','mmlu_pro','humaneval','tools','vqa'];labels=['IFEval*','GSM8K','MMLU-Pro','HumanEval','Tools**','TextVQA'] +fig,axes=plt.subplots(1,2,figsize=(13.8,6.9),gridspec_kw={'width_ratios':[1.6,1]});fig.subplots_adjust(left=.07,right=.98,top=.77,bottom=.37,wspace=.29) +fig.suptitle('Runtime v2 quality — preserved evidence, explicit trade-offs',y=.98,fontsize=18,fontweight='bold') +fig.text(.5,.914,'Same 300-item suite • thinking off • greedy • evaluation concurrency 16',ha='center',color='#4D4D4D') +x=np.arange(6);w=.24 +for i,(d,col,lab) in enumerate(zip(DIRS,COLORS,NAMES)): + j=load(d,'eval/summary.json');vals=[100*j['tasks'][k]['accuracy'] for k in keys] + axes[0].bar(x+(i-1)*w,vals,w,color=col,edgecolor='black',linewidth=.7,label=lab) +axes[0].set(title='A First complete regression run',ylabel='Accuracy (%)',xticks=x,xticklabels=labels,ylim=(0,105));axes[0].tick_params(axis='x',labelrotation=25);fig.legend(handles=axes[0].get_legend_handles_labels()[0],labels=NAMES,loc='upper center',bbox_to_anchor=(.5,.865),ncol=3,fontsize=10);axes[0].grid(axis='y',alpha=.16);axes[0].set_axisbelow(True) +for i,(d,col) in enumerate(zip(DIRS[1:],COLORS[1:])): + q=load(d,'eval/compare_vs_ref.json')['overall_micro'];v=q['delta']*100;lo,hi=np.array(q['delta_boot95'])*100 + axes[1].errorbar(v,i,xerr=[[v-lo],[hi-v]],fmt='o',color=col,capsize=5,lw=2) + axes[1].text(-5.5,i+.22,f'{v:+.2f} pp [{lo:+.2f}, {hi:+.2f}]',fontsize=10) +axes[1].axvline(0,color='#767676',lw=1,ls='--');axes[1].set(title='B Pooled change vs frozen v1',xlabel='Percentage points (paired-bootstrap 95% CI)',yticks=[0,1],yticklabels=['v2 exact','v2 fast'],xlim=(-6,3),ylim=(1.55,-.55));axes[1].grid(axis='x',alpha=.16) +fig.text(.5,.16,'Frozen: 251/300 (83.7%) • exact: 249/300 (83.0%) • fast: 247/300 (82.3%)\nFast MMLU-Pro first run: 58.3% vs 70.0% frozen (−11.7 pp); fast reruns: 71.7%, 65.0%.\nFast truncations: 20 vs 14 frozen. Fast fails original first-run per-task/stability tripwires; not “lossless.”\n*Re-implemented IFEval checker. **20 hand-written tool cases. Small subsets, not full-benchmark results.\nSource: runtime-v2/runs/final_*/eval; all reruns retained in runtime-v2/runs/var and RESULTS.md.',ha='center',va='center',fontsize=10,color='#4D4D4D',linespacing=1.7) +save(fig,'runtime-v2-quality') +print('Generated final v2 performance/quality PNG, PDF, SVG from JSONs.') diff --git a/figures/runtime-v2-performance.pdf b/figures/runtime-v2-performance.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9d1d3f576cb703ab8d4fcb3096bdd91126846985 Binary files /dev/null and b/figures/runtime-v2-performance.pdf differ diff --git a/figures/runtime-v2-performance.png b/figures/runtime-v2-performance.png new file mode 100644 index 0000000000000000000000000000000000000000..ce3cc4161dbdb12ac1a9bbeca3f13fb4b7cbdf4e --- /dev/null +++ b/figures/runtime-v2-performance.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b0145854cf885771b77dd327e3d5cb0a742b04c9c744bf24006ba98f4569313 +size 507237 diff --git a/figures/runtime-v2-performance.svg b/figures/runtime-v2-performance.svg new file mode 100644 index 0000000000000000000000000000000000000000..12341b1ab6a54a3fe7b8215b7449a975d835330c --- /dev/null +++ b/figures/runtime-v2-performance.svg @@ -0,0 +1,691 @@ + + + + + + + + 2026-10-31T07:43:20.148372 + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + Matched synthetic (~1K input) + + + + + + + + + + Short code (~90 input) + + + + + + + + + + Document (~4K input) + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 10 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 30 + + + + + + + + + + + + + 40 + + + + Median decode tokens/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 12.4 + + + 16.0 + + + 14.3 + + + 27.3 + + + 33.4 + + + 30.6 + + + 28.9 + + + 35.9 + + + 33.3 + + + A Single-request decode (not aggregate throughput) + + + + + + + Frozen v1 + MTP + + + + + + v2 exact + MTP + + + + + + v2 fast + MTP + + + + + + + + + + + + + + + + 1 + + + + + + + + + + 2 + + + + + + + + + + 4 + + + + + + + + + + 8 + + + + Concurrent requests + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 50 + + + + + + + + + + + + + 100 + + + + Aggregate output tokens/s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + B Matched-workload concurrency + + + + + + + + + + + + + + + 1 + + + + + + + + + + 2 + + + + + + + + + + 4 + + + + + + + + + + 8 + + + + Concurrent requests + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 2 + + + + + + + + + + + + + 4 + + + + + + + + + + + + + 6 + + + + Median TTFT (seconds) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + C Matched-workload first token + + + + Runtime v2 — faster serving, unchanged NVFP4 weights + + + Final validation on one DGX Spark • W4A4 + MTP • October 31, 2026 + + + Exactly 256 output tokens • greedy • no prefix-cache hits • 0 errors in final performance runs + Matched: 8 requests/level; code/document: 4 at c=1 • final results, not earlier screening peaks + Exact preserves reference routing math; fast changes FP32 reduction order and may change generated text. + Source: runtime-v2/runs/final_*/{perf_matched,screen_code,screen_doc}.json + + + + + + + + + + + + + + diff --git a/figures/runtime-v2-quality.pdf b/figures/runtime-v2-quality.pdf new file mode 100644 index 0000000000000000000000000000000000000000..63fe166ee177ed9257f91c4469e24c599acab40e Binary files /dev/null and b/figures/runtime-v2-quality.pdf differ diff --git a/figures/runtime-v2-quality.png b/figures/runtime-v2-quality.png new file mode 100644 index 0000000000000000000000000000000000000000..874521c84610cdc67945e24bdcea2859ad2569a5 --- /dev/null +++ b/figures/runtime-v2-quality.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c66a6044c2160795ef18de6906060864ed793adfe08a27a8fce20e5f59306695 +size 396018 diff --git a/figures/runtime-v2-quality.svg b/figures/runtime-v2-quality.svg new file mode 100644 index 0000000000000000000000000000000000000000..f5cd71f7673d8a5aa0f1de811d7f7653ae2bac93 --- /dev/null +++ b/figures/runtime-v2-quality.svg @@ -0,0 +1,634 @@ + + + + + + + + 2026-10-31T07:43:21.592377 + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + IFEval* + + + + + + + + + + GSM8K + + + + + + + + + + MMLU-Pro + + + + + + + + + + HumanEval + + + + + + + + + + Tools** + + + + + + + + + + TextVQA + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 20 + + + + + + + + + + + + + 40 + + + + + + + + + + + + + 60 + + + + + + + + + + + + + 80 + + + + + + + + + + + + + 100 + + + + Accuracy (%) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A First complete regression run + + + + + + + + + + + + + + + + + + −6 + + + + + + + + + + + + + −4 + + + + + + + + + + + + + −2 + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 2 + + + + Percentage points (paired-bootstrap 95% CI) + + + + + + + + + + + v2 exact + + + + + + + + + + v2 fast + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -0.67 pp [-3.00, +1.67] + + + -1.33 pp [-4.33, +1.67] + + + B Pooled change vs frozen v1 + + + + Runtime v2 quality — preserved evidence, explicit trade-offs + + + Same 300-item suite • thinking off • greedy • evaluation concurrency 16 + + + Frozen: 251/300 (83.7%) • exact: 249/300 (83.0%) • fast: 247/300 (82.3%) + Fast MMLU-Pro first run: 58.3% vs 70.0% frozen (−11.7 pp); fast reruns: 71.7%, 65.0%. + Fast truncations: 20 vs 14 frozen. Fast fails original first-run per-task/stability tripwires; not “lossless.” + *Re-implemented IFEval checker. **20 hand-written tool cases. Small subsets, not full-benchmark results. + Source: runtime-v2/runs/final_*/eval; all reruns retained in runtime-v2/runs/var and RESULTS.md. + + + + + + + Frozen v1 + MTP + + + + + + v2 exact + MTP + + + + + + v2 fast + MTP + + + + + + + + + + + + diff --git a/runtime-v2/.gitignore b/runtime-v2/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..c18dd8d83ceed1806b50b0aaa46beb7e335fff13 --- /dev/null +++ b/runtime-v2/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/runtime-v2/LEDGER.md b/runtime-v2/LEDGER.md new file mode 100644 index 0000000000000000000000000000000000000000..222fe9e96dc4629cf8d7683eeb67ff14d8b45714 --- /dev/null +++ b/runtime-v2/LEDGER.md @@ -0,0 +1,19 @@ +# Experiment ledger — SenseNova-Loop-9B NVFP4 runtime optimization (GB10, 2026-10-30) +Screen = bench/screen_set.sh: synthetic(1062 in)/code(90 in) prompts, 256 out, c=1 (4 req) + c=4 (8 req), unique prompts, temp 0, ignore_eos. +Greedy check = tests/greedy_outputs.py 12 prompts (text/math/think/code/zh/json/long/tool/image) vs frozen first run on a fresh server. + +| # | Hypothesis | Change | Correctness | synth c1 / c4 | code c1 / c4 | Keep? | +|---|---|---|---|---|---|---| +| 0 | baseline | frozen release runtime, W4A4+MTP(3) | ref; same-server repeat 9/12 (prefix-cache rerun) | 11.6 / 40.8 | 15.4 / 58.4 | ref | +| P | profile c1 | — | — | cycle 157 ms: verify 117 (mHC ~60 incl gaps, FP4 35, lm_head 8.7, router cuBLAS 5.5), draft 27.5+13.7 (BF16 lm_head x3) | | | +| 1 | mHC is launch-bound (39k kernels/fwd) | fused Triton Sinkhorn (bitwise), cuBLAS linear + torch merge kept ("exact") | 11/12 identical (only random tool-call id) | 18.7 / 62.9 | 24.8 / 87.0 | KEEP | +| 1b | router linear+merge also fused | single-kernel rmsnorm+linear+Sinkhorn+branch, fused merge ("full") | 2/12 identical; FP32 sum-order diffs (closer to fp64 than cuBLAS) flip near-ties | 21.2 / 73.8 | 25.7 / 92.6 | option (not bitwise) | +| 2 | draft lm_head (2 GB BF16, 3x/cycle) dominates draft | FR-Spec hot-vocab draft head --speculative-token-map (train-split corpora), target verify stays full-vocab | 32k: 10/12; 64k: 11/12 | 32k 27.6/72.0; 64k 29.0/80.7 | 32k 32.9/105.8; 64k 32.2/106.7 | KEEP 64k | +| X | target-only vs MTP (same math) | --no-mtp, exact | 1/12 identical vs MTP: M=1 decode vs M=4 verify kernels differ -> inherent spec/target near-tie flips (also true for frozen) | 19.5 / 63.0 | 20.0 / 70.2 | info | +| P2 | profile exact+tm64k c1 | — | — | cycle ~69 ms: verify 56 (FP4 34.8, lm_head 8.7, router cuBLAS 5.8), draft 8.6 + extend 4.3 | | | +| 3 | more/fewer MTP steps pay after draft got cheap | steps 2/4/5 (draft tokens = steps+1), exact+tm64k | — | s2 26.4/79.0; s4 25.9/68.5 | s2 32.8/108.9; s4 30.8/104.1; s5 31.6/94.3 | REJECT (keep 3) | +| 4 | cutlass FP4 faster on down_proj (microbench -7% FP4) | --fp4-gemm-backend flashinfer_cutlass | 10/12 (different FP4 accumulation) | 26.3/80.8 | 33.2/108.6 | REJECT (noise-level) | +| 5 | fused router linear+merge worth it once mHC is not dominant | full fused (default env) + tm64k | 2/12 (FP32 sum order, see 1b) | 31.0/90.6 (+7%/+12% vs exact) | 37.5/117.5 (+16%/+10%) | FINALIST (needs 300-item) | +| 6 | Marlin W4A16 + MTP better single-user | marlin + SILU fusion off, exact+tm64k | 3/12 (W4A16 numerics) | 27.9/77.4, TTFT c4 2.2 s | 33.8/107.7 | REJECT (parity decode, worse TTFT) | +| 7 | no-MTP throughput profile for high concurrency | --no-mtp vs MTP, fast+tm64k, c=8/16 | — | c8 129.4 vs 130.2; c16 194.3 vs 171.4 | c8 150.6 vs 198.4; c16 258.0 vs 295.0 | REJECT (single MTP profile) | +| F | final validation | frozen_v1 vs v2_fast vs v2_exact (RESULTS.md) | 300-item: 251 / 247 / 249; exact == frozen 12/12; mmlu_pro reruns show frozen spans 0.65-0.70 | matched c1 12.0 -> 27.8 / 26.0; c8 64.3 -> 119.4 / 117.2 | code c1 15.6 -> 35.3; c8 97.4 -> 196.7 | SHIP v2 (default fast, exact switch) | diff --git a/runtime-v2/README.md b/runtime-v2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..49f122fc092d9badc1a5c0c1041df8533af415f1 --- /dev/null +++ b/runtime-v2/README.md @@ -0,0 +1,86 @@ +# SenseNova-Loop-9B-Preview-NVFP4 — runtime v2 (published companion update) + +Faster serving runtime for the **unchanged** published checkpoint +[`gitcommit90/SenseNova-Loop-9B-Preview-NVFP4`](https://huggingface.co/gitcommit90/SenseNova-Loop-9B-Preview-NVFP4) (weights, config and the v1 `runtime/` are untouched). +Same pinned SGLang image and PR #47281 overlay as v1; W4A4 NVFP4 (flashinfer cuDNN FP4 GEMMs) + MTP (NEXTN, 3 steps). + +## Use +```bash +runtime-v2/serve_sglang.sh /path/to/SenseNova-Loop-9B-Preview-NVFP4 # default: MHC_MODE=fast + 64k draft vocab + MTP +MHC_MODE=exact runtime-v2/serve_sglang.sh # bitwise-identical target math to v1 (greedy outputs = v1) +MHC_MODE=reference TOKEN_MAP=none runtime-v2/serve_sglang.sh # = v1 runtime behaviour and speed +runtime-v2/serve_sglang.sh --no-mtp # target only (MTP still recommended, see results) +``` +Env as in v1 (`PORT`, `HOST`, `MEMFRAC`, `CTX`, `MAXREQ`, `SERVED`, `NAME`) plus `MHC_MODE`, `TOKEN_MAP`, `SGL_CACHE`. +**Recommended conservative command:** `MHC_MODE=exact bash runtime-v2/serve_sglang.sh /path/to/model`. +The launcher’s existing default remains `fast`; choose explicitly. Exact describes target routing math, not a +guarantee of identical sampled sequences on every workload/batch/hardware. Fast may change greedy outputs. + +Startup to ready: ~75 s (v1: ~90 s). Rollback: stop the container and start `bash runtime/serve_sglang.sh /path/to/model` — nothing is shared +except the JIT cache dir (`~/.cache/sensenova-sglang`, safe to delete). + +## What changed (two things) +1. **Fused mHC router** (`overlay/python/sglang/srt/layers/sensenova_mhc_fused.py`, wired in `models/sensenova_loop.py`, + diff: `patches/v2_mhc_fused_router.patch`). The original router ran FP32 log-space Sinkhorn (20 iterations on the 2x2 stream + matrix) as ~300 tiny ATen kernels per call; 96 router calls per forward (24 looped layers x 2 visits x 2 routers) = ~39k kernels, + ~60 ms of a 157 ms decode cycle. Now Triton: + * Sinkhorn/sigmoid kernel: same operation order, 20 iterations from config, torch.logsumexp semantics (NaN-propagating max, + inf-masked shift), precise libdevice exp/log/div, no FTZ: **bitwise equal** to eager (all T 1..4096, extreme values, CUDA graphs). + * `MHC_MODE=fast` (default) additionally fuses the FP32 RMS-normalize + 8x8192 FP32 router linear + branch, and the merge. + Still FP32 everywhere and the same dtype boundaries, but the FP32 dot-product summation order differs from cuBLAS + (coefficient diff <= 6.6e-7; vs a float64 reference the fused result is *more* accurate: 2.4e-7 vs cuBLAS 6.6e-7). + Not bitwise, so greedy text can diverge at near-ties (see correctness). +7..16% over `exact`. + * Eager reference kept verbatim (`_forward_reference`/`_merge_reference`, `MHC_MODE=reference`). +2. **FR-Spec hot-vocab MTP draft head** (`--speculative-token-map tokenmap/hot_64k.pt`). The draft proposed tokens through the + full 2 GB BF16 lm_head 3-4x per cycle (~35 ms). The draft now scores only the 65,536 most frequent tokens (+ all 33 special + tokens) counted on TRAIN-split corpora (oasst2, glaive-fc-v2, mbpp, gsm8k train; 37 M tokens; 99.1% weighted coverage; + `tokenmap/build_token_map.py`). **The target still verifies over the full vocabulary**. Vocabulary restriction applies only to drafting. + Speculative decoding preserves the target distribution in exact arithmetic with correct verification; finite-precision + kernels/batching can change actual outputs. Only greedy decoding was validated here; sampling equivalence was not tested. No weights changed (the draft head is a row subset of the same lm_head). + +Nothing else: same image digest, same flags (cuDNN FP4, steps 3, mem 0.45, ctx 32768, max 16 running), no KV/state precision change, +no change to loop passes, layers, routing, Sinkhorn iterations, sampling, context or output budgets. + +## Rejected after measurement (see LEDGER.md) +MTP steps 2/4/5 (no gain), flashinfer_cutlass FP4 (noise-level), Marlin W4A16+MTP (decode parity, 2x worse TTFT), 32k token map +(64k equal speed, better fidelity), no-MTP throughput profile (MTP wins on realistic text up to c=16; no-MTP only wins on +random-word synthetic at c=16). + +## Results (full tables: RESULTS.md; screening history: LEDGER.md) +Per-user decode speed and aggregate throughput, W4A4 + MTP, v1 frozen -> v2 default (`fast`) [`exact`]: + +| workload | c | aggregate tok/s | per-request decode tok/s p50 | TTFT p50 | +|---|---:|---|---|---| +| matched (1051 in / 256 out) | 1 | 12.0 -> **27.8** [26.0] | 12.4 -> **28.9** [27.3] | 0.74 -> 0.32 s | +| matched | 2 | 23.4 -> **49.2** [45.5] | 12.2 -> 28.4 | 0.89 -> 0.41 s | +| matched | 4 | 42.0 -> **87.0** [72.6] | 12.1 -> 25.4 | 1.84 -> 0.95 s | +| matched | 8 | 64.3 -> **119.4** [117.2] | 12.1 -> 19.8 | 5.02 -> 2.90 s | +| code (90 in / 256 out) | 1 | 15.6 -> **35.3** [32.8] | 16.0 -> 35.9 | 355 -> 134 ms | +| code | 8 | 97.4 -> **196.7** [172.7] | 14.1 -> 26.3 | 744 -> 332 ms | +| document (~4k in / 256 out) | 1 | 12.8 -> **29.1** [25.6] | 14.3 -> 33.3 | 2.03 -> 1.11 s | +| document | 8 | 52.4 -> **92.3** [70.4] | 9.0 -> 16.1 | 5.49 -> 3.91 s | + +MTP acceptance unchanged within noise (code 2.43-2.51 -> 2.31-2.41; doc 2.16-2.26 -> 2.16-2.19). 0 errors; every request produced +exactly the requested 256 tokens. 300-item regression: frozen 251/300, v2_exact 249, v2_fast 247. These are small, noisy regression tests, not proof of equivalence. +The first fast MMLU-Pro run dropped 11.7 percentage points (70.0% → 58.3%); reruns varied substantially. +See RESULTS.md for every run. Fast also had 20 truncated outputs vs 14 frozen, above the original ref+3 tripwire. +**Fast does not pass every original per-task/stability gate on the first run; use exact for the conservative profile.** + +## Correctness evidence +* `tests/test_mhc_fused.py` (result: `tests/test_mhc_fused.result.txt`): Sinkhorn kernel bitwise vs eager for T in + {1..4096} (decode, MTP verify 4/16/32, prefill 1051/4096), logit scales 0.1..1e4, +-inf, NaN, 3e38, denormals, non-contiguous + inputs, iterations 1/5/20/50, and under CUDA-graph capture/replay; exact router path bitwise end-to-end on real layer 4/15/27 + router weights; fast path coefficient diff <= 6.6e-7 (closer to fp64 than cuBLAS), bf16 branch identical. +* Greedy feature check `tests/greedy_outputs.py`: exact == frozen 12/12; fast valid on all 12 (text, thinking, code, zh, JSON, long, + tool call, image). +* MTP verified throughout (acceptance logged per request); target-only vs MTP divergence characterised (inherent to v1 too). + +## Remaining bottleneck (profile, c=1, exact mode, ~63-69 ms per speculative cycle; GPU ~100% busy, no launch gaps left) +* FP4 W4A4 GEMMs ~35 ms (55%): 6.8 GB of FP4 weights per forward because the 24 looped layers are read twice; cuDNN/CUTLASS + already at 85-95% of the 273 GB/s DRAM roofline in isolation. Floor ~25 ms. +* Target BF16 lm_head 8.7 ms (2 GB, required for verification) — only reducible by quantizing lm_head (a weight change; needs approval). +* Draft (BF16 MTP layer + 64k head) ~13 ms; GDN/attention/norms ~6 ms; router linear ~5.8 ms in exact mode (~0.5 ms in fast). +* Honest ceiling: ~8.8 GB of weights per target forward => <=31 forwards/s at 100% bandwidth; with accept length 2.3-2.9 that is + ~70-90 tok/s theoretical, realistically ~40-45 with this stack. Measured: 28-36 tok/s single-user. Reaching 60 tok/s single-user + would need changes outside this brief: FP8/NVFP4 lm_head and/or draft weights (weight changes), a better-trained draft (higher + acceptance), or a fused-weight persistent decode kernel. Each is a separate approval/effort decision. diff --git a/runtime-v2/RESULTS.md b/runtime-v2/RESULTS.md new file mode 100644 index 0000000000000000000000000000000000000000..e445756a76cd33610928b3c6e099f8c01e0da79c --- /dev/null +++ b/runtime-v2/RESULTS.md @@ -0,0 +1,83 @@ +# Runtime v2 — final validation results (GB10, 2026-10-31, same session, clocks 2.4 GHz, 47-65 C) + +frozen_v1 = published release runtime (W4A4 + MTP, eager mHC). v2_fast = default launcher. v2_exact = MHC_MODE=exact. All W4A4 NVFP4 + MTP(3), identical flags/budgets, unique prompts (no prefix-cache hits), temp 0. + +### Matched workload (eval/perf_bench.py: ~1051 prompt tok, 256 out, unique prompts, 8 req/level, 2 warmup) + +| runtime | c | aggregate tok/s | per-req decode tok/s p50 | TTFT p50 / p90 (s) | ITL p50 / p90 (ms) | E2E p50 / p90 (s) | out tok min-max | errors | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| frozen_v1 | 1 | 12.0 | 12.4 | 0.74 / 0.74 | 81 / 87 | 21.3 / 22.8 | 256-256 | 0 | +| frozen_v1 | 2 | 23.4 | 12.2 | 0.89 / 1.27 | 82 / 89 | 21.9 / 23.5 | 256-256 | 0 | +| frozen_v1 | 4 | 42.0 | 12.1 | 1.84 / 2.78 | 82 / 92 | 22.9 / 24.4 | 256-256 | 0 | +| frozen_v1 | 8 | 64.3 | 12.1 | 5.02 / 5.11 | 82 / 94 | 26.0 / 29.1 | 256-256 | 0 | +| v2_fast | 1 | 27.8 | 28.9 | 0.32 / 0.34 | 35 / 39 | 9.2 / 10.2 | 256-256 | 0 | +| v2_fast | 2 | 49.2 | 28.4 | 0.41 / 0.57 | 35 / 42 | 9.5 / 11.1 | 256-256 | 0 | +| v2_fast | 4 | 87.0 | 25.4 | 0.95 / 1.23 | 39 / 44 | 10.9 / 12.2 | 256-256 | 0 | +| v2_fast | 8 | 119.4 | 19.8 | 2.90 / 2.91 | 51 / 55 | 15.8 / 17.0 | 256-256 | 0 | +| v2_exact | 1 | 26.0 | 27.3 | 0.51 / 0.51 | 37 / 39 | 9.8 / 10.5 | 256-256 | 0 | +| v2_exact | 2 | 45.5 | 25.7 | 0.58 / 0.98 | 39 / 47 | 10.5 / 12.6 | 256-256 | 0 | +| v2_exact | 4 | 72.6 | 21.7 | 1.15 / 1.81 | 46 / 50 | 12.7 / 14.0 | 256-256 | 0 | +| v2_exact | 8 | 117.2 | 21.2 | 4.35 / 4.36 | 47 / 51 | 16.4 / 17.4 | 256-256 | 0 | + +### code: short interactive coding/reasoning prompt (~90 tok in, 256 out) + +| runtime | c | n | aggregate tok/s | per-req decode p50 | TTFT p50 (ms) | ITL p50 / p95 (ms) | MTP accept len | errors | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| frozen_v1 | 1 | 4 | 15.6 | 16.0 | 355 | 63 / 66 | 2.51 | 0 | +| frozen_v1 | 4 | 8 | 56.3 | 15.0 | 615 | 67 / 70 | 2.45 | 0 | +| frozen_v1 | 8 | 16 | 97.4 | 14.1 | 744 | 71 / 81 | 2.43 | 0 | +| v2_fast | 1 | 4 | 35.3 | 35.9 | 134 | 28 / 29 | 2.31 | 0 | +| v2_fast | 4 | 8 | 116.7 | 30.9 | 264 | 32 / 34 | 2.33 | 0 | +| v2_fast | 8 | 16 | 196.7 | 26.3 | 332 | 38 / 41 | 2.41 | 0 | +| v2_exact | 1 | 4 | 32.8 | 33.4 | 148 | 30 / 31 | 2.37 | 0 | +| v2_exact | 4 | 8 | 106.5 | 29.0 | 282 | 35 / 38 | 2.41 | 0 | +| v2_exact | 8 | 16 | 172.7 | 23.4 | 398 | 43 / 46 | 2.37 | 0 | + +### doc: long document (~4k tok in, 256 out) + +| runtime | c | n | aggregate tok/s | per-req decode p50 | TTFT p50 (ms) | ITL p50 / p95 (ms) | MTP accept len | errors | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| frozen_v1 | 1 | 4 | 12.8 | 14.3 | 2029 | 70 / 74 | 2.26 | 0 | +| frozen_v1 | 4 | 8 | 36.9 | 12.1 | 5490 | 83 / 108 | 2.16 | 0 | +| frozen_v1 | 8 | 16 | 52.4 | 9.0 | 5491 | 111 / 156 | 2.18 | 0 | +| v2_fast | 1 | 4 | 29.1 | 33.3 | 1111 | 30 / 31 | 2.19 | 0 | +| v2_fast | 4 | 8 | 69.9 | 25.0 | 4012 | 40 / 51 | 2.16 | 0 | +| v2_fast | 8 | 16 | 92.3 | 16.1 | 3909 | 62 / 85 | 2.16 | 0 | +| v2_exact | 1 | 4 | 25.6 | 30.6 | 1791 | 33 / 33 | 2.26 | 0 | +| v2_exact | 4 | 8 | 56.1 | 20.3 | 4736 | 49 / 64 | 2.13 | 0 | +| v2_exact | 8 | 16 | 70.4 | 14.2 | 7989 | 71 / 121 | 2.16 | 0 | + +### 300-item regression (thinking off, temp 0, eval concurrency 16) + +| runtime | ifeval | gsm8k | mmlu_pro | humaneval | tools | vqa | pooled | truncated | errors | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| frozen_v1 | 0.767 | 0.950 | 0.700 | 0.825 | 0.900 | 0.900 | 251/300 = 0.837 | 14 | 0 | +| v2_fast | 0.817 | 0.963 | 0.583 | 0.775 | 0.900 | 0.925 | 247/300 = 0.823 | 20 | 0 | +| v2_exact | 0.750 | 0.975 | 0.683 | 0.800 | 0.900 | 0.875 | 249/300 = 0.830 | 9 | 0 | + +### MMLU-Pro run-to-run spread (identical settings, no changes between reps; 60 items, 1024-token budget) + +| runtime | runs (acc) | mean | truncated per run | +|---|---|---:|---| +| frozen_v1 | 0.700, 0.700, 0.650 | 0.683 | 9, 7, 10 | +| v2_fast | 0.583, 0.717, 0.650 | 0.650 | 14, 10, 9 | +| v2_exact | 0.683 | — | 6 | +| BF16 original (v1 eval) | 0.650 | — | 13 | + +The first v2_fast mmlu_pro result (8 losses / 1 gain vs frozen, McNemar p=0.039) did not reproduce: the same config scored 0.717 and +0.650 on reruns, and frozen itself spans 0.650-0.700. Losses are dominated by long CoT hitting the 1024-token budget; v2_fast's mean +length (~600-618 tok) is between frozen (~572-594) and BF16 (622). Pooled 300: frozen 251, v2_exact 249, v2_fast 247 (delta -1.3 pp, +bootstrap 95% CI [-4.3, +1.7]). + +### Greedy feature check (12 prompts: text x2, math, thinking, code x2, translate, zh, JSON, long, tool call, image) +* v2_exact vs frozen_v1 (same validation round): **12/12 identical** (tool-call function + arguments; ids are random). +* v2_fast vs frozen_v1: 3/12 identical; all 12 outputs valid (tool call correct, image description correct, thinking present). +* Reference variance: frozen vs frozen across rounds 10/12; frozen target-only vs frozen MTP 1/12 (decode M=1 vs verify M=4 kernels). + +### Startup / memory (same --mem-fraction-static 0.45) +| | frozen_v1 | v2_fast | +|---|---:|---:| +| launch -> ready | 92 s | 77 s | +| target-verify CUDA graph capture | 22.4 s, 6.16 GB | 4.4 s, 1.14 GB | +| KV capacity (tokens) | 402,460 | 393,742 (-2%: 0.5 GB sliced draft head) | +| 300-item eval wall time | 549 s | 332 s | diff --git a/runtime-v2/bench/README.md b/runtime-v2/bench/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d07faf207c03da15082a714e49a2bfd79f2b1855 --- /dev/null +++ b/runtime-v2/bench/README.md @@ -0,0 +1,36 @@ +# Reproduce runtime v2 measurements + +Run one GPU workload at a time. Historical `runs/` contains the original final JSONs, per-item evaluation outputs, +screening history and logs; paths inside archived JSONs may refer to the original machine. No trace files or +session-local GPU-lock/orchestration wrappers are required for serving. + +From the repository/snapshot root, after downloading model files as in the main README: + +```bash +python3 -m venv .venv-eval +.venv-eval/bin/pip install -r evaluation/harness/requirements.lock +MODEL=/absolute/path/to/SenseNova-Loop-9B-Preview-NVFP4 +# Select one runtime, wait for readiness, then measure. Stop it before switching to another. +bash runtime/serve_sglang.sh "$MODEL" # frozen v1 +# MHC_MODE=exact bash runtime-v2/serve_sglang.sh "$MODEL" # conservative optimized +# MHC_MODE=fast bash runtime-v2/serve_sglang.sh "$MODEL" # fastest, different summation order +curl -fsS http://127.0.0.1:30080/v1/models +mkdir -p results +.venv-eval/bin/python evaluation/harness/perf_bench.py --base-url http://127.0.0.1:30080/v1 \ + --model sensenova-loop-9b-nvfp4 --concurrency 1,2,4,8 --num-requests 8 --warmup 2 \ + --input-tokens 1024 --output-tokens 256 --tokenizer "$MODEL/tokenizer.json" --out results/matched.json +.venv-eval/bin/python runtime-v2/bench/screen.py --base-url http://127.0.0.1:30080 \ + --model sensenova-loop-9b-nvfp4 --concurrency 1,4,8 --output-tokens 256 --prompt-kind code \ + --tokenizer "$MODEL/tokenizer.json" --out results/code.json +# Repeat with --prompt-kind doc for ~4K input tokens. +# Build the pinned small evaluation set into evaluation/harness/data/ before running quality tests: +.venv-eval/bin/python evaluation/harness/build_eval_data.py +.venv-eval/bin/python evaluation/harness/run_eval.py --base-url http://127.0.0.1:30080/v1 \ + --model sensenova-loop-9b-nvfp4 --out results/quality --concurrency 16 +``` + +For the 12-case feature test, supply `--image` (the recorded image was TextVQA validation image 34730): +`python runtime-v2/tests/greedy_outputs.py results/features.json --image /path/to/image.jpg`. +Kernel differential tests run in the pinned SGLang container; see `tests/test_mhc_fused.py`. +All run counts and exact prompts/lengths are in saved JSONs. Code/doc final tests use 4/8/16 requests at c=1/4/8; +matched tests use 8 per level. Screening c=16 results are not final matched-sweep numbers. diff --git a/runtime-v2/bench/fp4_gemm_bench.py b/runtime-v2/bench/fp4_gemm_bench.py new file mode 100644 index 0000000000000000000000000000000000000000..587f9ff2c8311e067a581b0995490769b9ae18a0 --- /dev/null +++ b/runtime-v2/bench/fp4_gemm_bench.py @@ -0,0 +1,47 @@ +"""Microbench NVFP4 W4A4 GEMM backends at SenseNova-Loop decoder shapes (run inside the SGLang image).""" +import torch, sys +from flashinfer import fp4_quantize, mm_fp4 +torch.manual_seed(0); dev = "cuda" +SHAPES = {"gate_up": (4096, 24576), "down": (12288, 4096), "gdn_qkvz": (4096, 12288), "attn_qkv": (4096, 10240), "o_proj": (4096, 4096)} +MS = [int(m) for m in (sys.argv[1] if len(sys.argv) > 1 else "1,4,8,16,32,64").split(",")] +BACKENDS = ["cudnn", "cutlass", "b12x", "cake", "cutedsl_low_latency", "auto"] +def gs(t): return (6.0 * 448.0 / t.abs().max().float()).float() +def timeit(fn, n=50): + for _ in range(3): fn() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for _ in range(n): fn() + torch.cuda.synchronize(); s, e = torch.cuda.Event(True), torch.cuda.Event(True) + s.record(); g.replay(); e.record(); torch.cuda.synchronize(); return s.elapsed_time(e) / n * 1e3 +kda = None +try: + import sglang.kernels.kda_kernels.qwen3x_nvfp4_gemm as q + q._has_runtime = lambda d: True + from sglang.kernels.ops.gemm import try_qwen3x_nvfp4_gemm as kda +except Exception as ex: print("kda import fail", ex) +tot = {} +for name, (K, N) in SHAPES.items(): + w = torch.randn(N, K, device=dev, dtype=torch.bfloat16) * 0.02; wg = gs(w) + wq, ws = fp4_quantize(w, wg, 16, False, True) + gbytes = N * K * (0.5 + 1 / 16) / 1e9 + for M in MS: + x = torch.randn(M, K, device=dev, dtype=torch.bfloat16); xg = gs(x) + xq, xs = fp4_quantize(x, xg, 16, False, True); alpha = (1.0 / (xg * wg)).reshape(1) + ref = (x.float() @ w.float().t()) + row = [] + for be in BACKENDS + (["kda"] if kda else []): + try: + out = torch.empty(M, N, device=dev, dtype=torch.bfloat16) + if be == "kda": + f = lambda: kda(xq, wq, xs, ws, alpha, torch.bfloat16, N) + if f() is None: raise RuntimeError("unsupported shape") + else: + f = lambda: mm_fp4(xq, wq.t(), xs, ws.t(), alpha, torch.bfloat16, out, backend=be) + o = f(); o = out if o is None else o + err = ((o.float() - ref).norm() / ref.norm()).item() + us = timeit(f); row.append(f"{be}={us:6.1f}us({gbytes/us*1e6:4.0f}GB/s,e{err:.3f})") + tot.setdefault((be, M), 0); tot[(be, M)] += us + except Exception as ex: + row.append(f"{be}=FAIL[{str(ex)[:40]}]") + print(f"{name:9s} M={M:3d} " + " ".join(row), flush=True) +print("sum over 5 shapes (us):", {f"{b}@{m}": round(v, 1) for (b, m), v in sorted(tot.items(), key=lambda t: (t[0][1], t[1]))}) diff --git a/runtime-v2/bench/prof_drive.py b/runtime-v2/bench/prof_drive.py new file mode 100644 index 0000000000000000000000000000000000000000..5847fb42b2598dc189bb65628fc6f82175fcbd66 --- /dev/null +++ b/runtime-v2/bench/prof_drive.py @@ -0,0 +1,19 @@ +"""Drive a short torch-profiler capture: warmup, then start_profile(num_steps) + N concurrent requests.""" +import sys, time, json, threading, requests +sys.path.insert(0, "/home/tux/sensenova-loop-nvfp4/eval") +from perf_bench import build_prompt_text +base = "http://127.0.0.1:30080" +conc = int(sys.argv[1]); steps = int(sys.argv[2]); prefix = sys.argv[3] +text, _, _ = build_prompt_text(1024, "/home/tux/sensenova-loop-nvfp4/upstream/SenseNova-Loop-9B-Preview/tokenizer.json") +def req(i, n=64): + r = requests.post(base + "/v1/chat/completions", json={"model": "x", "messages": [{"role": "user", "content": f"[{time.time()}-{i}] " + text}], + "max_tokens": n, "min_tokens": n, "ignore_eos": True, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, timeout=600) + return r.json()["usage"] +print("warm", req(-1, 32)) +ths = [threading.Thread(target=req, args=(i, 600)) for i in range(conc)] +for t in ths: t.start() +time.sleep(2.0 + 0.5 * conc) # past prefill, into decode +r = requests.post(base + "/start_profile", json={"num_steps": steps, "activities": ["CPU", "GPU"], "profile_prefix": prefix}) +print("profile", r.status_code, r.text[:200]) +for t in ths: t.join() +print(json.dumps(requests.get(base + "/get_server_info").json().get("internal_states", [{}])[0].get("avg_spec_accept_length"))) diff --git a/runtime-v2/bench/report_tables.py b/runtime-v2/bench/report_tables.py new file mode 100644 index 0000000000000000000000000000000000000000..c586ffc6cb79bf55f0be50564f00221b03b5c043 --- /dev/null +++ b/runtime-v2/bench/report_tables.py @@ -0,0 +1,41 @@ +"""Before/after markdown tables from runs/final_