NVFP4 (ModelOpt) quantization of sensenova/SenseNova-Loop-9B-Preview @27abea77 with BF16 MTP head

.gitattributes

@@ -33,3 +33,14 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/loop-mhc-overview.jpg filter=lfs diff=lfs merge=lfs -text
+assets/loop9b-benchmark-results.png filter=lfs diff=lfs merge=lfs -text
+examples/chocolate/chocolate-horizontal.jpg filter=lfs diff=lfs merge=lfs -text
+examples/energy-analysis/energy-analysis.png filter=lfs diff=lfs merge=lfs -text
+examples/iceland/iceland-horizontal-v2.jpg filter=lfs diff=lfs merge=lfs -text
+examples/ink-journey/ink-journey-demo.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/ink-journey/ink-journey-preview.gif filter=lfs diff=lfs merge=lfs -text
+examples/moon-simulation/moon-simulation.gif filter=lfs diff=lfs merge=lfs -text
+examples/teacher-day/teacher-day-horizontal.jpg filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+examples/odyssey/odyssey-preview.gif filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,186 @@
+---
+license: apache-2.0
+base_model: sensenova/SenseNova-Loop-9B-Preview
+base_model_relation: quantized
+library_name: transformers
+pipeline_tag: image-text-to-text
+language:
+- en
+- zh
+tags:
+- nvfp4
+- modelopt
+- quantized
+- multimodal
+- looped-transformer
+- mhc
+- mtp
+- custom_code
+---
+
+# SenseNova-Loop-9B-Preview-NVFP4
+
+> **Unofficial community quantization.** The original model, architecture and weights are by **SenseNova**:
+> [sensenova/SenseNova-Loop-9B-Preview](https://huggingface.co/sensenova/SenseNova-Loop-9B-Preview)
+> (revision `27abea779b1e6634b136805b6519371739d551f7`), licensed Apache-2.0. We did not train this model.
+> This repository contains modified weights (post-training quantization only — no fine-tuning, QAT or distillation).
+
+A calibrated, packed **NVFP4** (ModelOpt) checkpoint of SenseNova-Loop-9B-Preview with the **MTP head kept BF16**
+so MTP speculative decoding works, validated on an NVIDIA DGX Spark (GB10, SM121) with **native W4A4 FP4 GEMMs**.
+
+## Provenance
+| Item | Value |
+| --- | --- |
+| Base model | `sensenova/SenseNova-Loop-9B-Preview` @ `27abea779b1e6634b136805b6519371739d551f7` (weights stored FP32 upstream; loaded as BF16 backbone + FP32 mHC, as the upstream card prescribes) |
+| Quantization tool | NVIDIA Model Optimizer `nvidia-modelopt==0.48.0rc1` (`NVFP4_DEFAULT_CFG`, `max` calibration), PyTorch 2.11.0+cu130, Transformers 5.18.0 |
+| Format | NVFP4: E2M1 weights, group size 16, FP8-E4M3 block scales, FP32 per-tensor global scale (`weight_scale_2`) and static activation global scale (`input_scale`); activations quantized to NVFP4 at runtime (W4A4); KV cache **not** quantized (BF16) |
+| Export | 2026-10-30, `export_hf_checkpoint`, single `model.safetensors` (9.38 GB) |
+| Hardware | NVIDIA DGX Spark: GB10 (SM121), 128 GB unified memory, driver 580.159.03, CUDA 13.0 |
+
+## Changes from the original (Apache-2.0 §4(b) notice)
+- Decoder linear-layer weights quantized to NVFP4 (see exclusions); `config.json` gains `quantization_config`;
+  `hf_quant_config.json` added. Upstream FP32-stored tensors that were not quantized are stored as BF16, except the
+  mHC routing parameters, which remain FP32.
+- `mtp.*` (MTP head) is included in `model.safetensors` as BF16 (upstream shipped it as a separate FP32 `mtp.safetensors`).
+- Tokenizer, processor configs, chat template, generation config and `modeling_sensenova_loop.py` are byte-identical
+  to the pinned upstream revision (see `SHA256SUMS`).
+- `runtime/` contains SGLang source files from the open SGLang PR
+  [#47281](https://github.com/sgl-project/sglang/pull/47281) (Apache-2.0, sgl-project) plus two small patches of ours
+  (`runtime/patches/`), needed to serve this checkpoint.
+
+## Architecture-specific handling
+- **Looped block (layers 4–27 run twice, shared weights):** calibration ran through the real two-pass forward; every
+  looped layer was observed on both visits for every sample, so each shared quantizer's amax is the max over both
+  visits. One physical copy of each layer is stored (32 layers, not 56); per-visit KV/recurrent state remains
+  independent in the runtime.
+- **mHC routers** (`mhc_attn`, `mhc_ffn`): FP32 parameters and FP32 routing math (Sinkhorn, 20 iterations), untouched.
+- No scale folding / smoothing / graph transforms were applied (plain max calibration), so nothing assumes a single visit.
+
+## Precision exclusions
+| Module group | Kept in | Reason |
+| --- | --- | --- |
+| Vision encoder + merger (`model.visual*`) | BF16 | fragile, small share of compute |
+| `lm_head`, `embed_tokens` | BF16 | output/embedding sensitivity |
+| mHC routing (`mhc_attn`, `mhc_ffn`, 96 tensors) | FP32 | required by the architecture |
+| Gated-DeltaNet `in_proj_a`, `in_proj_b`, `conv1d`, `A_log`, `dt_bias`, norms | BF16 / FP32 | recurrent-state gates, tiny |
+| All RMSNorms | BF16 | not linear |
+| MTP head (`mtp.*`, 15 tensors) | BF16 | draft head; keeps MTP acceptance |
+| **Quantized (NVFP4):** full-attention `q/k/v/o_proj`, GDN `in_proj_qkv`, `in_proj_z`, `out_proj`, MLP `gate/up/down_proj` in all 32 layers (200 linears) | NVFP4 | |
+
+Parameters quantized: **73.4 %** of the backbone loaded by Transformers (6.91 B of 9.41 B), 71.6 % including the MTP
+head. Checkpoint size: **9.38 GB** (upstream: 38.1 GB FP32 on disk ≈ 19.3 GB as BF16). Weight memory in SGLang:
+8.8 GB (vs 17.5 GiB BF16 weights in Transformers).
+
+## Calibration data
+256 chat-formatted examples (`calib_v1`, sha256 `9a4d2112ef71820a…`, see `quantization/calib_v1.jsonl.sha256`), rendered
+with the model's own chat template/processor: 98,458 tokens incl. 39 images. Calibration data is not redistributed.
+
+| Share | Source | Split / revision | Licence |
+| --- | --- | --- | --- |
+| 30 % chat (en/zh/es/ca) | OpenAssistant/oasst2 | train @ 179dd21f | Apache-2.0 |
+| 20 % code | google-research-datasets/mbpp | full/train @ 4bb6404f | CC-BY-4.0 |
+| 20 % math (half with `<think>` traces) | openai/gsm8k | main/train @ 740312ad | MIT |
+| 15 % tool-calling trajectories | glaiveai/glaive-function-calling-v2 | @ e7f4b645 | Apache-2.0 |
+| 15 % image QA | lmms-lab/textvqa | train shard 0 @ 9c0699cd | CC-BY-4.0 (images: OpenImages, CC-BY-2.0) |
+
+Disjoint from all evaluation items (different splits/datasets; 13-gram overlap check found only shared tool-schema boilerplate).
+
+## Quality vs BF16 reference (same SGLang build and flags, greedy, thinking off)
+Small fixed suite (300 items; data hashes in `evaluation/eval_data_MANIFEST.json`). Gates were declared before
+any NVFP4 result was seen (`evaluation/QUALITY_GATES.md`). Paired-bootstrap 95 % CI for the delta. These are
+**subsets**, not full-benchmark scores; differences inside the CI are "not detectable at this n", not "lossless".
+
+| Task | n | BF16 | NVFP4 | Δ (pp) [95 % CI] |
+| --- | --- | --- | --- | --- |
+| IFEval subset (prompt-level strict, re-implemented checker) | 60 | 78.3 | 75.0 | −3.3 [−13.3, +6.7] |
+| GSM8K test subset | 80 | 92.5 | 95.0 | +2.5 [−2.5, +7.5] |
+| MMLU-Pro test subset | 60 | 65.0 | 68.3 | +3.3 [−6.7, +13.3] |
+| HumanEval subset (pass@1) | 40 | 82.5 | 80.0 | −2.5 [−7.5, 0.0] |
+| Function calling (20 hand-written BFCL-style cases) | 20 | 95.0 | 95.0 | 0.0 [−15, +15] |
+| TextVQA val subset | 40 | 90.0 | 90.0 | 0.0 [−7.5, +7.5] |
+| **All (micro)** | **300** | **82.7** | **83.0** | **+0.3 [−3.0, +4.0]** |
+
+Tool calls: 100 % well-formed for both. 0 request errors. Truncations at the max-token limit: 14 (NVFP4) vs 19 (BF16).
+With MTP enabled the NVFP4 model scored GSM8K 96.3 % (n=80), HumanEval 80.0 % (n=40), tools 90 % (n=20); MTP is
+lossless in principle at temperature 0, small differences come from batch-dependent numerics.
+
+## Performance (DGX Spark GB10, SGLang, same flags for all rows)
+Prompt 1,051 tokens, exactly 256 output tokens (`ignore_eos`), greedy, unique prompts (no prefix-cache hits),
+`--mem-fraction-static 0.45`, CUDA graphs on. No MTP rows: 16 requests per level; MTP rows: 8 requests per level.
+Aggregate output tokens/s (per-request decode tok/s p50 in parentheses):
+
+| Concurrency | BF16 | NVFP4 W4A4 | BF16 + MTP | **NVFP4 W4A4 + MTP** | NVFP4 W4A16 (Marlin) |
+| --- | --- | --- | --- | --- | --- |
+| 1 | 5.75 (5.8) | 9.39 (9.6) | 7.51 (7.6) | **12.21 (12.6)** | 10.98 (11.4) |
+| 2 | 11.44 | 17.61 | 14.77 | **24.79** | – |
+| 4 | 21.86 | 33.48 | 26.81 | **42.51** | – |
+| 8 | 39.93 (5.5) | 64.79 (9.3) | 45.70 (7.0) | **64.43 (12.2)** | 58.17 (8.8) |
+
+TTFT p50 at c=1: 0.75 s BF16 → 0.58 s NVFP4. 0 errors / OOMs at any level. MTP mean accepted length (3 draft
+steps): **2.93 NVFP4 vs 2.95 BF16** on the eval tasks; on the synthetic perf prompts ≈2.0, which is why MTP gives
+no extra aggregate throughput at c=8 there. KV-cache capacity at the same memory budget: 407 k tokens (NVFP4) vs
+308 k (BF16).
+
+**Kernel evidence (native FP4):** a torch-profiler trace of NVFP4 serving shows the quantized linears executing
+`cutlass3x_sm120_bstensorop_…gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1…` (block-scaled FP4×FP4 tensor-core GEMM,
+dispatched via the FlashInfer cuDNN backend) with activations quantized by `quantize_with_block_size` — i.e. true
+W4A4, not dequantize-to-BF16 (`evaluation/kernel_summary.txt`). The W4A16 column uses the same checkpoint with
+Marlin weight-only kernels; it is faster for single-stream decode on GB10 but slower under concurrency.
+
+## Hardware & software requirements
+- NVIDIA Blackwell GPU with FP4 tensor cores. **Tested only on DGX Spark (GB10, SM121)**. Weights need ≈9 GB plus
+  KV/recurrent-state memory.
+- Serving: SGLang image `lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17`
+  (`sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47`) with the files in `runtime/overlay`
+  (SGLang PR #47281 @ `336c54ac1e721197806d94b6929f567446a59693` + our patches) mounted over the image.
+  As of 2026-10-30 we found no engine supporting `sensenova_loop` + NVFP4 out of the box (checked: SGLang main,
+  LightLLM main — model supported but no NVFP4, vLLM 0.27.1 — no `sensenova_loop`).
+- Plain Transformers (`trust_remote_code=True`) loads the original BF16 model but **cannot** execute this packed NVFP4
+  checkpoint or MTP; use the SGLang runtime above.
+
+## Usage
+```bash
+# from the repository root (Docker + NVIDIA container runtime required)
+runtime/serve_sglang.sh .                 # NVFP4 W4A4 + MTP (NEXTN, 3 steps), 127.0.0.1:30080
+curl -s http://127.0.0.1:30080/v1/models
+python3 runtime/probe.py http://127.0.0.1:30080/v1 sensenova-loop-9b-nvfp4 [image.jpg]   # text / thinking / tool / image smoke test
+# options: --no-mtp ; W4A16: SGLANG_DISABLE_SILU_FP4_QUANT_FUSION=1 runtime/serve_sglang.sh . --fp4-gemm-backend marlin
+```
+OpenAI-compatible API; thinking is toggled with `chat_template_kwargs: {"enable_thinking": false|true}`; tools use the
+`qwen3_coder` parser (the model's XML `<tool_call><function=…>` format).
+
+## Limitations
+- Upstream is a *Preview* model; all upstream limitations and intended-use terms apply.
+- The serving path depends on an **unmerged** SGLang PR plus our patches, pinned to one nightly image; it was tested
+  only on GB10. Pipeline/tensor parallelism and other GPUs were not tested.
+- Evaluation is small (300 items, thinking off, contexts ≤ ~3 K tokens). Long context (> 32 K), video input, thinking-mode
+  quality at scale and non-English quality were not evaluated. Only greedy decoding was benchmarked.
+- The mHC Sinkhorn routing runs as many small FP32 PyTorch kernels and dominates a large share of GPU time in both BF16 and
+  NVFP4; a fused kernel would likely improve speed further (not attempted).
+- In this SGLang build, `--fp4-gemm-backend marlin` crashes unless `SGLANG_DISABLE_SILU_FP4_QUANT_FUSION=1` is set
+  (upstream bug: the fused SiLU+FP4-quant path is enabled regardless of backend).
+- Quantization may change behaviour on inputs unlike the calibration mixture.
+
+## Files
+- `model.safetensors`, `config.json`, `hf_quant_config.json` — NVFP4 checkpoint (+ BF16 MTP head, FP32 mHC).
+- Upstream (unchanged): `tokenizer*.json`, `chat_template.jinja`, `*processor_config.json`, `generation_config.json`, `modeling_sensenova_loop.py`.
+- `runtime/` — tested launcher, SGLang overlay and patches, probe script, compatibility notes.
+- `quantization/` — quantization/verification scripts, stats, structural verification report, environment lock, calibration manifest.
+- `evaluation/` — quality gates, eval summaries, paired comparison, perf JSONs, kernel trace summary, reference checks.
+- `SHA256SUMS` — checksums of every file.
+
+## Licence
+Apache-2.0, inherited from the base model (`LICENSE`, verbatim from `OpenSenseNova/SenseNova-Loop` @ `a521b0af`; the
+upstream repository has no NOTICE file). SGLang files in `runtime/overlay` are Apache-2.0 (sgl-project).
+
+## Citation
+Please cite the original model:
+```bibtex
+@misc{sensenova2026loop,
+  title        = {{SenseNova-Loop-9B-Preview}},
+  author       = {{SenseNova}},
+  year         = {2026},
+  howpublished = {Hugging Face model repository},
+  url          = {https://huggingface.co/sensenova/SenseNova-Loop-9B-Preview}
+}
+```

SHA256SUMS

@@ -0,0 +1,65 @@
+a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715  ./chat_template.jinja
+5bec5643138a09f2917a0b7471a673baecfbf85f59af49cc048f4318f32e7a3d  ./config.json
+021b420e4757b37909200d1cc52bf308116461735a6b50c67df652267f54c587  ./evaluation/compare_vs_ref.json
+5d01579bee6c0863addc3236392b4d88c8d7a7d3cf66b62ae65db1430c0110a3  ./evaluation/eval_data_MANIFEST.json
+6df2bf0438b64d1353c1737e3659c86bce501dc188de9ce448492895e8b5ae63  ./evaluation/harness/build_eval_data.py
+e2818e8c8ff581d65ac8a4dbfc07b58eaea97546dfd63cc3f8f31faaf8ed0f22  ./evaluation/harness/common.py
+0423139c0573e09181f065dbdb273b123aa4a966a5ec57a3643b2fe4f8f43b92  ./evaluation/harness/compare.py
+535255ccff9b6032494d918331dce7d7bbd5a455771143afeaf7977458b52bf4  ./evaluation/harness/ifeval_lite.py
+c0e6d611823b672513730fae4f83d6e72cb0927202a0df1112176465d316731f  ./evaluation/harness/mock_server.py
+8909f46e7c6383cce45dbad9c64d78035316031b6f7d99be664ae8b7c3d3577a  ./evaluation/harness/perf_bench.py
+d2b24fe51830bf6ed99af01c66ed1b861a50eff88d130fac3f9369e6865837cd  ./evaluation/harness/requirements.lock
+49f85f32fb01f029182491d5dd803c26c37814e9ecfa283f37d705543242bc3d  ./evaluation/harness/run_eval.py
+fe25e6a9d5432922ff4509a9a80e7b996072ee2091353840a6d0fdf764d8daef  ./evaluation/harness/tasks.py
+8c28fad23d10bfa56897a6c0d306b0a474cb6bc2ac7a08355682bd697a2dcd3f  ./evaluation/harness/test_harness.py
+ec6899745f66eaf5eb92975341bd44b8c6d5ede2116ea6d0d56d8ba008ae7702  ./evaluation/harness/tool_cases.py
+90dbf7789104b8763f8980f564d8da2c6d033a2207707a386dba8756cd8d2063  ./evaluation/hf_bf16_checks.json
+a1a5de6483f0d8c2d759b9c60cb86da4794a4036c6101bba104cc1d5dba63dfb  ./evaluation/kernel_summary.txt
+ce49434ad0c1806bcf01a57957add6fb2ca67f1aa6a61394ae248a75d41b7c80  ./evaluation/nvfp4_v1_mtp_release_probe.json
+4cfff7659e057c02133576124dd0dd65b28a8d6869d1b460d68055dc01a2454a  ./evaluation/nvfp4_v1_probe.json
+a588317f6ba1071ab5f67a850e0ced977dfbf3f3dd0b34ea2d5ac54767947c17  ./evaluation/perf_sglang_bf16.json
+7f9aa0a53f019234a256057b0f759695cf0ac266ac033e3e02b84bf71ffcec41  ./evaluation/perf_sglang_bf16_mtp.json
+8a115863344c244470a3a18959e89c914aa852602e75cb365554d62adfd66c9f  ./evaluation/perf_sglang_nvfp4_v1.json
+d68a9f7a151718b65b6782ea48f4db0aa60f068ab8636eb51f2f2935a61b0058  ./evaluation/perf_sglang_nvfp4_v1_marlin_w4a16.json
+546730231cdf739cf84f0d92768c2a31378c165b3e48a1840117b9a2faa5035a  ./evaluation/perf_sglang_nvfp4_v1_mtp.json
+ccdb9203010fc0534d37b0b475d85f088f5e62568d3cbd76229af4d8e50ffb28  ./evaluation/QUALITY_GATES.md
+8f50277ef8a232dc4abab998fa80ffdcc46c13cbebea1d9328e93878af035157  ./evaluation/sglang_bf16_mtp_probe.json
+92c0708bb0d27772bedefc3d5f458b742c48325025af5a1269535de6dca2d914  ./evaluation/sglang_bf16_probe.json
+ec88f909edcbf225936bbeac88ede9312520593eb53db976d5079d9dd6c45ed1  ./evaluation/summary_sglang_bf16.json
+58cd6bbadfc9fc070a784e65810414a24919084e4570e1ff7680e8a00f14504b  ./evaluation/summary_sglang_bf16_mtp.json
+c6d75330180b2d1bd5f2dda574afeee8844e18e44c46ba0f3a1128eb052d2326  ./evaluation/summary_sglang_nvfp4_v1.json
+aff7bcc52ad3e83e6394a8d31e733d8b5725d213836beb98824c397e726c66c6  ./evaluation/summary_sglang_nvfp4_v1_marlin_w4a16_gsm8k.json
+944d3ce06cffbfb7a8e10c1af186aaa62a196e6b96cd8c3a1e6800e5d998fa70  ./evaluation/summary_sglang_nvfp4_v1_mtp.json
+f288bb4281580138395c6acc7b3adb62f45c21530bc0708cb2cf9edf093f6a4f  ./generation_config.json
+8951a99119cdcd06c699d91ccc5db0a40b595961041a5ec2e6103b88112757e8  ./.gitattributes
+b197af747a1f072419a8e7e2db3252d9aeea84d21f65db37f748430ab7f83bfd  ./hf_quant_config.json
+c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4  ./LICENSE
+26e11d3522aa730c0ba031340ce42632bc2834da8cc0b6804006bb207d979a5b  ./modeling_sensenova_loop.py
+1e420ea69111b7bb3c7c529cdbbb4ff4e6ebb531c0cd2be5c634cd88cd40eaca  ./model.safetensors
+3a159dfec9978a186a72ba085e0ad6a050f3968d8b364218d7bd13f5c89381f2  ./preprocessor_config.json
+d89ef49ce9cd37fbf510158e13c1ef063d9286411c1ec9049932dbe0487143b1  ./processor_config.json
+df49d802cffe959b3176eed9abee2277d3e4291cdaf2d9abf8430ee17aeba700  ./quantization/10_reference_checks.py
+67d6edacb11e519afb6d81adb4c559930b17ce9a8adeed0e7c8c2043d8383337  ./quantization/20_quantize.py
+27dcbf33dfb7e83cace92cc8221e20d2ead29d4090bf15c9348e181b6bb5ba47  ./quantization/30_verify_export.py
+dc54da6f2e37136c3172779d86ae0134a353942f341eaa8934d3000a01f34682  ./quantization/build_calib.py
+04c164fb57e8125eed5d624cefa806a61bc28cb929aca6406d2923ea43f11825  ./quantization/calib_v1.jsonl.sha256
+3a2015bbf02262d97e0de28f4fa2b3e73d43f58635a507a96788bf7b587236d8  ./quantization/calib_v1.meta.json
+4d71bc0045ee80c45f5eef89e5e721e064e2505020b327942501b017daf0fdf8  ./quantization/quantization_stats.json
+723419a376e5f2339f4f0d4f9376b16ea2d5359e54659ccb85c2bb2d5c3028fd  ./quantization/requirements-quant.lock
+be632c83213d44b4a18f367c59fcb55f5d0d738b8c67f64ccc61df3c6a9fb598  ./quantization/verify.json
+9d1df67a6eb124431baf4bfbd9eb9b30b00d8f9a8116dec05c847a39036ba8a3  ./README.md
+1b95e589cd65a97a97fae93cdeec07fcadbf0ff204be746fe20db32550a2c248  ./runtime/COMPAT_NOTES.md
+e00d2fafe780fcf4c04ab52b040e43ec7143c7bd6e070637563f2724b59888d1  ./runtime/overlay/python/sglang/srt/configs/__init__.py
+bdc7fa2055b799ce07441a5d499fafa554b7b797d4c4a24fd251b846b51260fd  ./runtime/overlay/python/sglang/srt/configs/model_config.py
+eb6df8f37ac06e6f439dd24f687e5bf1078599ade28acaa20d8799bf77c7e057  ./runtime/overlay/python/sglang/srt/configs/sensenova_loop.py
+b3b4959f2336d1afdf4f0353a0a0eecfc38495226912022dd862489cbb532c98  ./runtime/overlay/python/sglang/srt/layers/rotary_embedding/mrope_rope_index.py
+0883bffeff0519db0fb8035b3260eca57c48ebe00e42323fb75cb7f455c1ddad  ./runtime/overlay/python/sglang/srt/models/sensenova_loop.py
+7fc07bd18b974aa8f54013f2bb39536a65d32ad0cc1a1df2d239ead0dc136754  ./runtime/overlay/python/sglang/srt/multimodal/processors/qwen_vl.py
+68e1076237ed67adfa06530139d79df0783a2519620443a58cce94c5143d56c4  ./runtime/overlay/python/sglang/srt/utils/hf_transformers/common.py
+d93e6342f9487567a7847443eb47b41701b4510474cd5664c8dc51b37eb6f6fd  ./runtime/patches/model_config_mtp_draft.patch
+029fa25bb4ea5a16c9fce8de412e7ea6481fc540508c63bdd629026b9a56f926  ./runtime/patches/sensenova_loop_quant.patch
+7b9fd04a5e756a982cac292d314ef2ca84dca0517a4abb4f8a4c009745fd0267  ./runtime/probe.py
+be2e56aca0bdbc48bab616db27b37ab36a170b2a3a35534e8be25b291508e54a  ./runtime/serve_sglang.sh
+792fa3f0cb88b111e54ef3134c873531008c4df471d108da17903426e308aa7b  ./tokenizer_config.json
+06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523  ./tokenizer.json
+bd96830cce1312679aa1f2ea43ed17da20e8ebbcea2398cfa64e2d275a96fdd7  ./video_preprocessor_config.json

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,261 @@
+{
+    "architectures": [
+        "SenseNovaLoopForConditionalGeneration"
+    ],
+    "auto_map": {
+        "AutoConfig": "modeling_sensenova_loop.SenseNovaLoopConfig",
+        "AutoModel": "modeling_sensenova_loop.SenseNovaLoopModel",
+        "AutoModelForCausalLM": "modeling_sensenova_loop.SenseNovaLoopForConditionalGeneration",
+        "AutoModelForImageTextToText": "modeling_sensenova_loop.SenseNovaLoopForConditionalGeneration"
+    },
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": 248046,
+    "hidden_size": 4096,
+    "image_token_id": 248056,
+    "loop_settings": {
+        "counts": [
+            2
+        ],
+        "end": 28,
+        "eval_count": 2,
+        "start": 4
+    },
+    "mhc": {
+        "end": 28,
+        "fixed_identity": false,
+        "iterations": 20,
+        "pre_scale": 1.0,
+        "start": 4,
+        "streams": 2
+    },
+    "model_name": "SenseNova-Loop-9B-Preview",
+    "model_type": "sensenova_loop",
+    "pad_token_id": 248044,
+    "patch_conv3d": true,
+    "text_config": {
+        "attention_bias": false,
+        "attention_dropout": 0.0,
+        "attn_output_gate": true,
+        "bos_token_id": null,
+        "dtype": "bfloat16",
+        "eos_token_id": 248044,
+        "full_attention_interval": 4,
+        "head_dim": 256,
+        "hidden_act": "silu",
+        "hidden_size": 4096,
+        "initializer_range": 0.02,
+        "intermediate_size": 12288,
+        "layer_types": [
+            "linear_attention",
+            "linear_attention",
+            "linear_attention",
+            "full_attention",
+            "linear_attention",
+            "linear_attention",
+            "linear_attention",
+            "full_attention",
+            "linear_attention",
+            "linear_attention",
+            "linear_attention",
+            "full_attention",
+            "linear_attention",
+            "linear_attention",
+            "linear_attention",
+            "full_attention",
+            "linear_attention",
+            "linear_attention",
+            "linear_attention",
+            "full_attention",
+            "linear_attention",
+            "linear_attention",
+            "linear_attention",
+            "full_attention",
+            "linear_attention",
+            "linear_attention",
+            "linear_attention",
+            "full_attention",
+            "linear_attention",
+            "linear_attention",
+            "linear_attention",
+            "full_attention"
+        ],
+        "linear_conv_kernel_dim": 4,
+        "linear_key_head_dim": 128,
+        "linear_num_key_heads": 16,
+        "linear_num_value_heads": 32,
+        "linear_value_head_dim": 128,
+        "mamba_ssm_dtype": "float32",
+        "max_position_embeddings": 262144,
+        "mlp_only_layers": [],
+        "model_type": "qwen3_5_text",
+        "mtp_num_hidden_layers": 1,
+        "mtp_use_dedicated_embeddings": false,
+        "num_attention_heads": 16,
+        "num_hidden_layers": 32,
+        "num_key_value_heads": 4,
+        "pad_token_id": 248044,
+        "partial_rotary_factor": 0.25,
+        "rms_norm_eps": 1e-06,
+        "rope_parameters": {
+            "mrope_interleaved": true,
+            "mrope_section": [
+                11,
+                11,
+                10
+            ],
+            "partial_rotary_factor": 0.25,
+            "rope_theta": 10000000,
+            "rope_type": "default"
+        },
+        "tie_word_embeddings": false,
+        "use_cache": true,
+        "vocab_size": 248320
+    },
+    "tie_word_embeddings": false,
+    "transformers_version": "5.18.0",
+    "video_token_id": 248057,
+    "vision_config": {
+        "deepstack_visual_indexes": [],
+        "depth": 27,
+        "dtype": "bfloat16",
+        "hidden_act": "gelu_pytorch_tanh",
+        "hidden_size": 1152,
+        "in_channels": 3,
+        "initializer_range": 0.02,
+        "intermediate_size": 4304,
+        "model_type": "qwen3_5_vision",
+        "num_heads": 16,
+        "num_position_embeddings": 2304,
+        "out_hidden_size": 4096,
+        "patch_size": 16,
+        "rope_parameters": {
+            "rope_theta": 10000.0,
+            "rope_type": "axial"
+        },
+        "spatial_merge_size": 2,
+        "temporal_patch_size": 2
+    },
+    "vision_end_token_id": 248054,
+    "vision_start_token_id": 248053,
+    "quantization_config": {
+        "config_groups": {
+            "group_0": {
+                "input_activations": {
+                    "dynamic": false,
+                    "num_bits": 4,
+                    "type": "float",
+                    "group_size": 16
+                },
+                "weights": {
+                    "dynamic": false,
+                    "num_bits": 4,
+                    "type": "float",
+                    "group_size": 16
+                },
+                "targets": [
+                    "Linear"
+                ]
+            }
+        },
+        "ignore": [
+            "lm_head",
+            "model.language_model.embed_tokens",
+            "model.language_model.layers.0.linear_attn.conv1d",
+            "model.language_model.layers.0.linear_attn.in_proj_a",
+            "model.language_model.layers.0.linear_attn.in_proj_b",
+            "model.language_model.layers.1.linear_attn.conv1d",
+            "model.language_model.layers.1.linear_attn.in_proj_a",
+            "model.language_model.layers.1.linear_attn.in_proj_b",
+            "model.language_model.layers.10.linear_attn.conv1d",
+            "model.language_model.layers.10.linear_attn.in_proj_a",
+            "model.language_model.layers.10.linear_attn.in_proj_b",
+            "model.language_model.layers.12.linear_attn.conv1d",
+            "model.language_model.layers.12.linear_attn.in_proj_a",
+            "model.language_model.layers.12.linear_attn.in_proj_b",
+            "model.language_model.layers.13.linear_attn.conv1d",
+            "model.language_model.layers.13.linear_attn.in_proj_a",
+            "model.language_model.layers.13.linear_attn.in_proj_b",
+            "model.language_model.layers.14.linear_attn.conv1d",
+            "model.language_model.layers.14.linear_attn.in_proj_a",
+            "model.language_model.layers.14.linear_attn.in_proj_b",
+            "model.language_model.layers.16.linear_attn.conv1d",
+            "model.language_model.layers.16.linear_attn.in_proj_a",
+            "model.language_model.layers.16.linear_attn.in_proj_b",
+            "model.language_model.layers.17.linear_attn.conv1d",
+            "model.language_model.layers.17.linear_attn.in_proj_a",
+            "model.language_model.layers.17.linear_attn.in_proj_b",
+            "model.language_model.layers.18.linear_attn.conv1d",
+            "model.language_model.layers.18.linear_attn.in_proj_a",
+            "model.language_model.layers.18.linear_attn.in_proj_b",
+            "model.language_model.layers.2.linear_attn.conv1d",
+            "model.language_model.layers.2.linear_attn.in_proj_a",
+            "model.language_model.layers.2.linear_attn.in_proj_b",
+            "model.language_model.layers.20.linear_attn.conv1d",
+            "model.language_model.layers.20.linear_attn.in_proj_a",
+            "model.language_model.layers.20.linear_attn.in_proj_b",
+            "model.language_model.layers.21.linear_attn.conv1d",
+            "model.language_model.layers.21.linear_attn.in_proj_a",
+            "model.language_model.layers.21.linear_attn.in_proj_b",
+            "model.language_model.layers.22.linear_attn.conv1d",
+            "model.language_model.layers.22.linear_attn.in_proj_a",
+            "model.language_model.layers.22.linear_attn.in_proj_b",
+            "model.language_model.layers.24.linear_attn.conv1d",
+            "model.language_model.layers.24.linear_attn.in_proj_a",
+            "model.language_model.layers.24.linear_attn.in_proj_b",
+            "model.language_model.layers.25.linear_attn.conv1d",
+            "model.language_model.layers.25.linear_attn.in_proj_a",
+            "model.language_model.layers.25.linear_attn.in_proj_b",
+            "model.language_model.layers.26.linear_attn.conv1d",
+            "model.language_model.layers.26.linear_attn.in_proj_a",
+            "model.language_model.layers.26.linear_attn.in_proj_b",
+            "model.language_model.layers.28.linear_attn.conv1d",
+            "model.language_model.layers.28.linear_attn.in_proj_a",
+            "model.language_model.layers.28.linear_attn.in_proj_b",
+            "model.language_model.layers.29.linear_attn.conv1d",
+            "model.language_model.layers.29.linear_attn.in_proj_a",
+            "model.language_model.layers.29.linear_attn.in_proj_b",
+            "model.language_model.layers.30.linear_attn.conv1d",
+            "model.language_model.layers.30.linear_attn.in_proj_a",
+            "model.language_model.layers.30.linear_attn.in_proj_b",
+            "model.language_model.layers.4.linear_attn.conv1d",
+            "model.language_model.layers.4.linear_attn.in_proj_a",
+            "model.language_model.layers.4.linear_attn.in_proj_b",
+            "model.language_model.layers.5.linear_attn.conv1d",
+            "model.language_model.layers.5.linear_attn.in_proj_a",
+            "model.language_model.layers.5.linear_attn.in_proj_b",
+            "model.language_model.layers.6.linear_attn.conv1d",
+            "model.language_model.layers.6.linear_attn.in_proj_a",
+            "model.language_model.layers.6.linear_attn.in_proj_b",
+            "model.language_model.layers.8.linear_attn.conv1d",
+            "model.language_model.layers.8.linear_attn.in_proj_a",
+            "model.language_model.layers.8.linear_attn.in_proj_b",
+            "model.language_model.layers.9.linear_attn.conv1d",
+            "model.language_model.layers.9.linear_attn.in_proj_a",
+            "model.language_model.layers.9.linear_attn.in_proj_b",
+            "model.visual*",
+            "mtp.fc",
+            "mtp.layers.0.input_layernorm",
+            "mtp.layers.0.mlp.down_proj",
+            "mtp.layers.0.mlp.gate_proj",
+            "mtp.layers.0.mlp.up_proj",
+            "mtp.layers.0.post_attention_layernorm",
+            "mtp.layers.0.self_attn.k_norm",
+            "mtp.layers.0.self_attn.k_proj",
+            "mtp.layers.0.self_attn.o_proj",
+            "mtp.layers.0.self_attn.q_norm",
+            "mtp.layers.0.self_attn.q_proj",
+            "mtp.layers.0.self_attn.v_proj",
+            "mtp.norm",
+            "mtp.pre_fc_norm_embedding",
+            "mtp.pre_fc_norm_hidden"
+        ],
+        "quant_algo": "NVFP4",
+        "group_size": 16,
+        "producer": {
+            "name": "modelopt",
+            "version": "0.48.0rc1"
+        },
+        "quant_method": "modelopt"
+    }
+}

evaluation/QUALITY_GATES.md

@@ -0,0 +1,14 @@
+# Quality / performance gates (declared 2026-10-30 ~15:25 PDT, BEFORE any NVFP4 eval result was seen)
+
+Eval: eval/ (300 fixed items: IFEval 60, GSM8K 80, MMLU-Pro 60, HumanEval 40, tools 20, TextVQA 40),
+temperature 0, thinking off, same SGLang build/flags for reference (BF16 backbone, FP32 mHC) and candidate.
+
+Quality (candidate vs BF16 reference, paired):
+- G1 pooled accuracy (all 300 items, item-weighted): delta point estimate >= -3.0 pp AND paired-bootstrap 95% CI lower bound >= -7.0 pp.
+- G2 no single task point-estimate drop > 8 pp (n=20-80 per task, so this is a coarse tripwire, not a significance test).
+- G3 tool calling: 0 malformed tool calls beyond the reference's count; image task (TextVQA) drop <= 8 pp.
+- G4 stability: 0 request errors, all outputs finite/non-degenerate (no repetition collapse in truncation counts beyond ref + 3).
+- G5 MTP: mean accepted length (spec) with NVFP4 target >= 85% of the BF16-target value under identical spec settings.
+Performance (same engine, same flags, fixed 1024-in/256-out, concurrency 1/2/4/8):
+- P1 0 errors/OOM at c=1..8. P2 report decode tok/s, TTFT, aggregate throughput honestly; speedup is reported, not gated.
+If a recipe fails G1-G5, try the next recipe (max 3: decoder-all -> MLP-only -> decoder w/ first/last layers BF16).

evaluation/compare_vs_ref.json

@@ -0,0 +1,138 @@
+{
+  "ref": "eval/runs/sglang_bf16",
+  "cand": "eval/runs/sglang_nvfp4_v1",
+  "tasks": {
+    "gsm8k": {
+      "n": 80,
+      "ref_acc": 0.925,
+      "cand_acc": 0.95,
+      "delta": 0.025,
+      "delta_boot95": [
+        -0.025,
+        0.075
+      ],
+      "ref_only_correct": 1,
+      "cand_only_correct": 3,
+      "mcnemar_p": 0.625,
+      "ref_agreement": 0.9375,
+      "identical_output": 0.075,
+      "mean_tokens_ref": 191.5,
+      "mean_tokens_cand": 187.2,
+      "truncated_ref": 1,
+      "truncated_cand": 1,
+      "errors_cand": 0
+    },
+    "humaneval": {
+      "n": 40,
+      "ref_acc": 0.825,
+      "cand_acc": 0.8,
+      "delta": -0.025,
+      "delta_boot95": [
+        -0.075,
+        0.0
+      ],
+      "ref_only_correct": 1,
+      "cand_only_correct": 0,
+      "mcnemar_p": 1.0,
+      "ref_agreement": 0.425,
+      "identical_output": 0.375,
+      "mean_tokens_ref": 226.6,
+      "mean_tokens_cand": 198.0,
+      "truncated_ref": 0,
+      "truncated_cand": 0,
+      "errors_cand": 0
+    },
+    "ifeval": {
+      "n": 60,
+      "ref_acc": 0.7833333333333333,
+      "cand_acc": 0.75,
+      "delta": -0.03333333333333333,
+      "delta_boot95": [
+        -0.13333333333333333,
+        0.06666666666666667
+      ],
+      "ref_only_correct": 6,
+      "cand_only_correct": 4,
+      "mcnemar_p": 0.75390625,
+      "ref_agreement": 0.05,
+      "identical_output": 0.05,
+      "mean_tokens_ref": 318.9,
+      "mean_tokens_cand": 272.7,
+      "truncated_ref": 5,
+      "truncated_cand": 4,
+      "errors_cand": 0
+    },
+    "mmlu_pro": {
+      "n": 60,
+      "ref_acc": 0.65,
+      "cand_acc": 0.6833333333333333,
+      "delta": 0.03333333333333333,
+      "delta_boot95": [
+        -0.06666666666666667,
+        0.13333333333333333
+      ],
+      "ref_only_correct": 4,
+      "cand_only_correct": 6,
+      "mcnemar_p": 0.75390625,
+      "ref_agreement": 0.7666666666666667,
+      "identical_output": 0.0,
+      "mean_tokens_ref": 622.1,
+      "mean_tokens_cand": 588.5,
+      "truncated_ref": 13,
+      "truncated_cand": 9,
+      "errors_cand": 0
+    },
+    "tools": {
+      "n": 20,
+      "ref_acc": 0.95,
+      "cand_acc": 0.95,
+      "delta": 0.0,
+      "delta_boot95": [
+        -0.15,
+        0.15
+      ],
+      "ref_only_correct": 1,
+      "cand_only_correct": 1,
+      "mcnemar_p": 1.0,
+      "ref_agreement": 0.85,
+      "identical_output": 0.0,
+      "mean_tokens_ref": 64.2,
+      "mean_tokens_cand": 54.3,
+      "truncated_ref": 0,
+      "truncated_cand": 0,
+      "errors_cand": 0
+    },
+    "vqa": {
+      "n": 40,
+      "ref_acc": 0.9,
+      "cand_acc": 0.9,
+      "delta": 0.0,
+      "delta_boot95": [
+        -0.075,
+        0.075
+      ],
+      "ref_only_correct": 1,
+      "cand_only_correct": 1,
+      "mcnemar_p": 1.0,
+      "ref_agreement": 0.925,
+      "identical_output": 0.925,
+      "mean_tokens_ref": 4.1,
+      "mean_tokens_cand": 4.1,
+      "truncated_ref": 0,
+      "truncated_cand": 0,
+      "errors_cand": 0
+    }
+  },
+  "overall_micro": {
+    "n": 300,
+    "ref_acc": 0.8266666666666667,
+    "cand_acc": 0.83,
+    "delta": 0.0033333333333333335,
+    "delta_boot95": [
+      -0.03,
+      0.04
+    ],
+    "ref_agreement": 0.65,
+    "identical_output": 0.20333333333333334
+  }
+}

evaluation/eval_data_MANIFEST.json

@@ -0,0 +1,67 @@
+{
+  "eval_version": "sn-loop-eval-v1",
+  "seed": 20261030,
+  "tasks": {
+    "ifeval": {
+      "n": 60,
+      "file": "ifeval_input_data.jsonl",
+      "sha256": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+      "repo": "google/IFEval",
+      "rev": "966cd89545d6b6acfd7638bc708b98261ca58e84",
+      "split": "train (only split; = the IFEval eval set)",
+      "license": "Apache-2.0",
+      "eligible_pool": "495/541 prompts with only supported instruction ids",
+      "raw_file_sha256": "6a85310ca8ce15eff755aa08a3a4ff931c7e273e7515ebb3c492ea85fd8288f2"
+    },
+    "gsm8k": {
+      "n": 80,
+      "file": "main/test-00000-of-00001.parquet",
+      "sha256": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+      "repo": "openai/gsm8k",
+      "rev": "740312add88f781978c0658806c59bc2815b9866",
+      "split": "main/test",
+      "license": "MIT",
+      "raw_file_sha256": "ee7b8da9e381df27b9e3f7758a159ab2bdaa4dbaa910546cbbc47e0cb44e4f59"
+    },
+    "mmlu_pro": {
+      "n": 60,
+      "file": "data/test-00000-of-00001.parquet",
+      "sha256": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+      "repo": "TIGER-Lab/MMLU-Pro",
+      "rev": "b189ec765aa7ed75c8acfea42df31fdae71f97be",
+      "split": "test",
+      "license": "MIT",
+      "raw_file_sha256": "0e24a191921c2f453518a537a8b2117bd137e7714d4ef1565e9ba06c1ecb9ad8"
+    },
+    "humaneval": {
+      "n": 40,
+      "file": "openai_humaneval/test-00000-of-00001.parquet",
+      "sha256": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+      "repo": "openai/openai_humaneval",
+      "rev": "7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544",
+      "split": "test",
+      "license": "MIT",
+      "raw_file_sha256": "2f2871a15fbc95b6c683043359f4ed8e144c5a1c4f24f25f66bc51f598dfcfb6"
+    },
+    "tools": {
+      "n": 20,
+      "file": "eval/tool_cases.py",
+      "sha256": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+      "repo": "(hand-written, eval/tool_cases.py)",
+      "rev": "in-repo",
+      "split": "n/a",
+      "license": "Apache-2.0 (this project); BFCL-style categories, no BFCL text"
+    },
+    "vqa": {
+      "n": 40,
+      "file": "data/validation-00000-of-00003.parquet",
+      "sha256": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d",
+      "repo": "lmms-lab/textvqa",
+      "rev": "9c0699cd19768ac5ab97568f6b3cbac4c0062884",
+      "split": "validation (shard 0 of 3, 1667 of 5000 rows)",
+      "license": "TextVQA annotations CC-BY-4.0; images from OpenImages (CC-BY-2.0, per-image Flickr attribution)",
+      "eligible_pool": "1630/1667 questions with a >=3-annotator consensus answer",
+      "raw_file_sha256": "491649b5b8add2c5bde13a1e36707002fcd11898699255d65e2e568ac4fbddff"
+    }
+  }
+}

evaluation/harness/build_eval_data.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+"""Select the FIXED eval items from pinned dataset revisions and write them to eval/data/.
+
+Run once (needs HF access only if eval/data/raw is missing; see download list in DATASETS):
+    .venv-eval/bin/python eval/build_eval_data.py
+Outputs: eval/data/<task>.jsonl, eval/data/images/*.jpg, eval/data/MANIFEST.json (with sha256s).
+Re-running produces byte-identical files (fixed seeds, sorted inputs).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import random
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import pyarrow.parquet as pq  # noqa: E402
+
+import ifeval_lite  # noqa: E402
+from common import EVAL_VERSION, write_jsonl  # noqa: E402
+from tool_cases import CASES  # noqa: E402
+
+DATA = HERE / "data"
+RAW = DATA / "raw"
+SEED = 20261030
+N = {"ifeval": 60, "gsm8k": 80, "mmlu_pro": 60, "humaneval": 40, "tools": len(CASES), "vqa": 40}
+
+DATASETS = {
+    "ifeval": dict(repo="google/IFEval", rev="966cd89545d6b6acfd7638bc708b98261ca58e84",
+                   file="ifeval_input_data.jsonl", split="train (only split; = the IFEval eval set)",
+                   license="Apache-2.0"),
+    "gsm8k": dict(repo="openai/gsm8k", rev="740312add88f781978c0658806c59bc2815b9866",
+                  file="main/test-00000-of-00001.parquet", split="main/test", license="MIT"),
+    "mmlu_pro": dict(repo="TIGER-Lab/MMLU-Pro", rev="b189ec765aa7ed75c8acfea42df31fdae71f97be",
+                     file="data/test-00000-of-00001.parquet", split="test", license="MIT"),
+    "humaneval": dict(repo="openai/openai_humaneval", rev="7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544",
+                      file="openai_humaneval/test-00000-of-00001.parquet", split="test", license="MIT"),
+    "tools": dict(repo="(hand-written, eval/tool_cases.py)", rev="in-repo", file="eval/tool_cases.py",
+                  split="n/a", license="Apache-2.0 (this project); BFCL-style categories, no BFCL text"),
+    "vqa": dict(repo="lmms-lab/textvqa", rev="9c0699cd19768ac5ab97568f6b3cbac4c0062884",
+                file="data/validation-00000-of-00003.parquet",
+                split="validation (shard 0 of 3, 1667 of 5000 rows)",
+                license="TextVQA annotations CC-BY-4.0; images from OpenImages (CC-BY-2.0, per-image Flickr attribution)"),
+}
+
+
+def raw_path(task):
+    d = DATASETS[task]
+    return RAW / d["repo"].replace("/", "__") / d["file"]
+
+
+def sha256(p):
+    h = hashlib.sha256()
+    with open(p, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def pick(rows, n, task):
+    rng = random.Random(f"{SEED}-{task}")
+    idx = sorted(rng.sample(range(len(rows)), n))
+    return idx
+
+
+def build():
+    DATA.mkdir(exist_ok=True)
+    (DATA / "images").mkdir(exist_ok=True)
+    out = {}
+
+    # IFEval: restrict to prompts whose instructions are all supported by ifeval_lite
+    rows = [json.loads(l) for l in open(raw_path("ifeval"))]
+    rows.sort(key=lambda r: r["key"])
+    elig = [r for r in rows if all(i in ifeval_lite.SUPPORTED for i in r["instruction_id_list"])]
+    sel = [elig[i] for i in pick(elig, N["ifeval"], "ifeval")]
+    out["ifeval"] = [dict(id=f"ifeval_{r['key']}", key=r["key"], prompt=r["prompt"],
+                          instruction_id_list=r["instruction_id_list"], kwargs=r["kwargs"]) for r in sel]
+    DATASETS["ifeval"]["eligible_pool"] = f"{len(elig)}/{len(rows)} prompts with only supported instruction ids"
+
+    rows = pq.read_table(raw_path("gsm8k")).to_pylist()
+    idx = pick(rows, N["gsm8k"], "gsm8k")
+    out["gsm8k"] = [dict(id=f"gsm8k_test_{i}", source_index=i, question=rows[i]["question"],
+                         answer=rows[i]["answer"].split("####")[-1].strip().replace(",", ""),
+                         solution=rows[i]["answer"]) for i in idx]
+
+    rows = pq.read_table(raw_path("mmlu_pro"), columns=["question_id", "question", "options", "answer",
+                                                          "category", "src"]).to_pylist()
+    rows.sort(key=lambda r: r["question_id"])
+    idx = pick(rows, N["mmlu_pro"], "mmlu_pro")
+    out["mmlu_pro"] = [dict(id=f"mmlupro_{rows[i]['question_id']}", **rows[i]) for i in idx]
+
+    rows = pq.read_table(raw_path("humaneval")).to_pylist()
+    idx = pick(rows, N["humaneval"], "humaneval")
+    out["humaneval"] = [dict(id=rows[i]["task_id"].replace("/", "_"), **rows[i]) for i in idx]
+
+    out["tools"] = [dict(c) for c in CASES]
+
+    tbl = pq.read_table(raw_path("vqa"), columns=["question_id", "question", "answers", "image_id",
+                                                    "image", "flickr_original_url"])
+    qids = tbl.column("question_id").to_pylist()
+    order = sorted(range(len(qids)), key=lambda i: qids[i])
+    # only questions with a consensus answer (some normalized answer given by >=3 of 10 annotators)
+    from collections import Counter
+    from tasks import vqa_norm
+    answers = tbl.column("answers").to_pylist()
+    order = [i for i in order if Counter(vqa_norm(a) for a in answers[i]).most_common(1)[0][1] >= 3]
+    DATASETS["vqa"]["eligible_pool"] = f"{len(order)}/{len(qids)} questions with a >=3-annotator consensus answer"
+    idx = [order[i] for i in pick(order, N["vqa"], "vqa")]
+    items = []
+    for i in idx:
+        r = tbl.slice(i, 1).to_pylist()[0]
+        img = r["image"]["bytes"]
+        ext = "png" if img[:4] == b"\x89PNG" else "jpg"
+        rel = f"images/textvqa_val_{r['question_id']}.{ext}"
+        (DATA / rel).write_bytes(img)
+        items.append(dict(id=f"textvqa_val_{r['question_id']}", question_id=r["question_id"],
+                          image_id=r["image_id"], image=rel, image_sha256=hashlib.sha256(img).hexdigest(),
+                          question=r["question"], answers=r["answers"],
+                          image_source_url=r["flickr_original_url"]))
+    out["vqa"] = items
+
+    manifest = {"eval_version": EVAL_VERSION, "seed": SEED, "tasks": {}}
+    for task, rows in out.items():
+        assert len(rows) == N[task], (task, len(rows))
+        p = DATA / f"{task}.jsonl"
+        write_jsonl(p, rows)
+        manifest["tasks"][task] = {"n": len(rows), "file": p.name, "sha256": sha256(p), **DATASETS[task]}
+        if raw_path(task).exists() if task != "tools" else False:
+            manifest["tasks"][task]["raw_file_sha256"] = sha256(raw_path(task))
+    with open(DATA / "MANIFEST.json", "w") as f:
+        json.dump(manifest, f, indent=2)
+    print(json.dumps({t: v["n"] for t, v in manifest["tasks"].items()}), "total", sum(N.values()))
+
+
+if __name__ == "__main__":
+    build()

evaluation/harness/common.py

@@ -0,0 +1,177 @@
+"""Shared helpers for the SenseNova-Loop NVFP4 eval harness (stdlib + requests only)."""
+from __future__ import annotations
+
+import json
+import math
+import re
+import time
+from typing import Any
+
+EVAL_VERSION = "sn-loop-eval-v1"
+
+
+# ----------------------------------------------------------------------------- stats
+def wilson_ci(k: int, n: int, z: float = 1.959964) -> tuple[float, float]:
+    if n == 0:
+        return (float("nan"), float("nan"))
+    p = k / n
+    den = 1 + z * z / n
+    centre = (p + z * z / (2 * n)) / den
+    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / den
+    return (max(0.0, centre - half), min(1.0, centre + half))
+
+
+# ----------------------------------------------------------------------------- text utils
+THINK_RE = re.compile(r"<think>.*?</think>\s*", re.S)
+
+
+def strip_think(text: str | None) -> str:
+    """Remove any <think>...</think> block (and a dangling '...</think>' prefix)."""
+    if not text:
+        return ""
+    text = THINK_RE.sub("", text)
+    if "</think>" in text:
+        text = text.split("</think>")[-1]
+    return text.strip()
+
+
+# ----------------------------------------------------------------------------- tool-call parsing
+# Chat template (Qwen3.5-style XML) emits:
+# <tool_call>\n<function=NAME>\n<parameter=ARG>\nVALUE\n</parameter>\n...</function>\n</tool_call>
+# String values are written raw; dict/list values as JSON (tojson); numbers/bools via |string
+# (so Python-ish 'True'/'False' are possible).
+TOOL_CALL_RE = re.compile(r"<tool_call>\s*<function=([^>\n]+)>(.*?)</function>\s*</tool_call>", re.S)
+PARAM_RE = re.compile(r"<parameter=([^>\n]+)>\n?(.*?)\n?</parameter>", re.S)
+
+
+def _coerce(value: str, schema: dict | None) -> Any:
+    typ = (schema or {}).get("type")
+    if typ == "string":
+        return value
+    v = value.strip()
+    if typ == "boolean" or v in ("True", "False"):
+        if v.lower() in ("true", "false"):
+            return v.lower() == "true"
+    try:
+        return json.loads(v)
+    except Exception:
+        pass
+    if typ in ("integer", "number"):
+        try:
+            return float(v) if "." in v else int(v)
+        except ValueError:
+            return v
+    return value
+
+
+def parse_xml_tool_calls(text: str, tools: list[dict] | None = None) -> tuple[list[dict], dict]:
+    """Fallback parser for raw-text tool calls. Returns (calls, format_report)."""
+    schemas: dict[str, dict] = {}
+    for t in tools or []:
+        fn = t.get("function", t)
+        schemas[fn.get("name")] = (fn.get("parameters") or {}).get("properties") or {}
+    calls = []
+    for m in TOOL_CALL_RE.finditer(text or ""):
+        name = m.group(1).strip()
+        args = {}
+        for pm in PARAM_RE.finditer(m.group(2)):
+            pname = pm.group(1).strip()
+            args[pname] = _coerce(pm.group(2), schemas.get(name, {}).get(pname))
+        calls.append({"name": name, "arguments": args})
+    n_open = (text or "").count("<tool_call>")
+    n_close = (text or "").count("</tool_call>")
+    tail = (text or "").split("</tool_call>")[-1].strip() if n_close else ""
+    report = {
+        "n_tool_call_tags": n_open,
+        "balanced_tags": n_open == n_close,
+        "all_blocks_parsed": n_open == len(calls),
+        "suffix_after_last_call": bool(tail),  # template says: NO suffix after the call
+        "well_formed": n_open > 0 and n_open == n_close == len(calls),
+    }
+    return calls, report
+
+
+def calls_from_message(msg: dict, tools: list[dict] | None) -> tuple[list[dict], str, dict]:
+    """Return (calls, source, format_report). source in {'server','fallback_xml','none'}."""
+    tcs = msg.get("tool_calls") or []
+    if tcs:
+        calls, ok = [], True
+        for tc in tcs:
+            fn = tc.get("function", tc)
+            raw = fn.get("arguments")
+            if isinstance(raw, str):
+                try:
+                    args = json.loads(raw) if raw.strip() else {}
+                except Exception:
+                    args, ok = {"__unparseable__": raw}, False
+            else:
+                args = raw or {}
+            calls.append({"name": fn.get("name"), "arguments": args})
+        return calls, "server", {"well_formed": ok, "args_valid_json": ok}
+    content = strip_think(msg.get("content") or "")
+    calls, rep = parse_xml_tool_calls(content, tools)
+    return calls, ("fallback_xml" if calls else "none"), rep
+
+
+# ----------------------------------------------------------------------------- HTTP client
+def chat(session, base_url: str, model: str, messages: list, *, max_tokens: int,
+         tools: list | None = None, enable_thinking: bool = False, timeout: float = 600,
+         retries: int = 2, extra: dict | None = None) -> dict:
+    """One deterministic chat completion. Returns a dict with response + timing (never raises)."""
+    body: dict[str, Any] = {
+        "model": model,
+        "messages": messages,
+        "temperature": 0.0,
+        "top_p": 1.0,
+        "max_tokens": max_tokens,
+        "seed": 0,
+        "chat_template_kwargs": {"enable_thinking": enable_thinking},
+    }
+    if tools:
+        body["tools"] = tools
+        body["tool_choice"] = "auto"
+    if extra:
+        body.update(extra)
+    url = base_url.rstrip("/") + "/chat/completions"
+    last_err = None
+    for attempt in range(retries + 1):
+        t0 = time.time()
+        try:
+            r = session.post(url, json=body, timeout=timeout)
+            dt = time.time() - t0
+            if r.status_code != 200:
+                last_err = f"HTTP {r.status_code}: {r.text[:500]}"
+                if r.status_code < 500:
+                    break
+                time.sleep(1 + attempt)
+                continue
+            j = r.json()
+            ch = j["choices"][0]
+            msg = ch.get("message") or {}
+            return {
+                "ok": True,
+                "content": msg.get("content") or "",
+                "reasoning_content": msg.get("reasoning_content") or msg.get("reasoning") or "",
+                "tool_calls": msg.get("tool_calls") or [],
+                "message": msg,
+                "finish_reason": ch.get("finish_reason"),
+                "usage": j.get("usage") or {},
+                "latency_s": round(dt, 3),
+                "error": None,
+            }
+        except Exception as e:  # noqa: BLE001
+            last_err = f"{type(e).__name__}: {e}"
+            time.sleep(1 + attempt)
+    return {"ok": False, "content": "", "reasoning_content": "", "tool_calls": [], "message": {},
+            "finish_reason": None, "usage": {}, "latency_s": None, "error": last_err}
+
+
+def read_jsonl(path) -> list[dict]:
+    with open(path) as f:
+        return [json.loads(l) for l in f if l.strip()]
+
+
+def write_jsonl(path, rows) -> None:
+    with open(path, "w") as f:
+        for r in rows:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")

evaluation/harness/compare.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+"""Paired comparison of two run_eval.py output dirs (reference vs candidate).
+
+  .venv-eval/bin/python eval/compare.py eval/runs/bf16 eval/runs/nvfp4 [--json out.json]
+
+Per task (on the intersection of item ids):
+  * ref / cand accuracy, delta, paired bootstrap 95% CI of the delta (10k resamples, seed 0),
+    discordant counts (ref-only correct / cand-only correct) and exact McNemar p-value;
+  * reference agreement (label-free): fraction of items whose extracted answer == reference's;
+  * identical raw outputs %, mean completion tokens ref/cand, truncation counts.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import random
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+from common import read_jsonl  # noqa: E402
+
+
+def load(d):
+    d = Path(d)
+    out = {}
+    for p in sorted(d.glob("raw_*.jsonl")):
+        task = p.stem[4:]
+        out[task] = {r["id"]: r for r in read_jsonl(p)}
+    return out
+
+
+def boot_ci(a, b, iters=10000, seed=0):
+    n = len(a)
+    if n == 0:
+        return (float("nan"), float("nan"))
+    rng = random.Random(seed)
+    diffs = [y - x for x, y in zip(a, b)]
+    stats = []
+    for _ in range(iters):
+        s = 0
+        for _ in range(n):
+            s += diffs[rng.randrange(n)]
+        stats.append(s / n)
+    stats.sort()
+    return (stats[int(0.025 * iters)], stats[int(0.975 * iters) - 1])
+
+
+def mcnemar_exact(b, c):
+    n = b + c
+    if n == 0:
+        return 1.0
+    k = min(b, c)
+    p = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
+    return min(1.0, 2 * p)
+
+
+def norm_out(r):
+    return json.dumps([r.get("content") or "", r.get("tool_calls") or []], sort_keys=True)
+
+
+def compare(ref_dir, cand_dir):
+    ref, cand = load(ref_dir), load(cand_dir)
+    res = {"ref": str(ref_dir), "cand": str(cand_dir), "tasks": {}}
+    tot = {"n": 0, "ref": 0, "cand": 0, "agree": 0, "ident": 0, "a": [], "b": []}
+    for task in ref:
+        if task not in cand:
+            continue
+        ids = sorted(set(ref[task]) & set(cand[task]))
+        a = [int(ref[task][i]["correct"]) for i in ids]
+        b = [int(cand[task][i]["correct"]) for i in ids]
+        n = len(ids)
+        agree = sum(1 for i in ids if json.dumps(ref[task][i].get("agree_key"), sort_keys=True)
+                    == json.dumps(cand[task][i].get("agree_key"), sort_keys=True))
+        ident = sum(1 for i in ids if norm_out(ref[task][i]) == norm_out(cand[task][i]))
+        ref_only = sum(1 for x, y in zip(a, b) if x and not y)
+        cand_only = sum(1 for x, y in zip(a, b) if y and not x)
+
+        def mtok(rs):
+            t = [r["completion_tokens"] for r in rs if r.get("completion_tokens") is not None]
+            return round(sum(t) / len(t), 1) if t else None
+        lo, hi = boot_ci(a, b)
+        res["tasks"][task] = {
+            "n": n, "ref_acc": sum(a) / n, "cand_acc": sum(b) / n, "delta": (sum(b) - sum(a)) / n,
+            "delta_boot95": [lo, hi], "ref_only_correct": ref_only, "cand_only_correct": cand_only,
+            "mcnemar_p": mcnemar_exact(ref_only, cand_only),
+            "ref_agreement": agree / n, "identical_output": ident / n,
+            "mean_tokens_ref": mtok([ref[task][i] for i in ids]),
+            "mean_tokens_cand": mtok([cand[task][i] for i in ids]),
+            "truncated_ref": sum(1 for i in ids if ref[task][i].get("finish_reason") == "length"),
+            "truncated_cand": sum(1 for i in ids if cand[task][i].get("finish_reason") == "length"),
+            "errors_cand": sum(1 for i in ids if cand[task][i].get("error")),
+        }
+        tot["n"] += n
+        tot["ref"] += sum(a)
+        tot["cand"] += sum(b)
+        tot["agree"] += agree
+        tot["ident"] += ident
+        tot["a"] += a
+        tot["b"] += b
+    if tot["n"]:
+        lo, hi = boot_ci(tot["a"], tot["b"])
+        res["overall_micro"] = {"n": tot["n"], "ref_acc": tot["ref"] / tot["n"], "cand_acc": tot["cand"] / tot["n"],
+                                "delta": (tot["cand"] - tot["ref"]) / tot["n"], "delta_boot95": [lo, hi],
+                                "ref_agreement": tot["agree"] / tot["n"], "identical_output": tot["ident"] / tot["n"]}
+    return res
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("ref_dir")
+    ap.add_argument("cand_dir")
+    ap.add_argument("--json", default=None)
+    args = ap.parse_args()
+    res = compare(args.ref_dir, args.cand_dir)
+    hdr = f"{'task':10s} {'n':>4s} {'ref':>6s} {'cand':>6s} {'delta':>7s} {'boot95':>17s} {'r-/c+':>6s} {'mcnemar':>7s} {'agree':>6s} {'ident':>6s} {'tok r/c':>13s}"
+    print(hdr)
+    rows = list(res["tasks"].items()) + ([("ALL(micro)", res["overall_micro"])] if "overall_micro" in res else [])
+    for t, s in rows:
+        print(f"{t:10s} {s['n']:4d} {s['ref_acc']:6.3f} {s['cand_acc']:6.3f} {s['delta']:+7.3f} "
+              f"[{s['delta_boot95'][0]:+.3f},{s['delta_boot95'][1]:+.3f}] "
+              f"{str(s.get('ref_only_correct', '')) + '/' + str(s.get('cand_only_correct', '')):>6s} "
+              f"{s.get('mcnemar_p', float('nan')):7.3f} {s['ref_agreement']:6.3f} {s['identical_output']:6.3f} "
+              f"{str(s.get('mean_tokens_ref'))+'/'+str(s.get('mean_tokens_cand')):>13s}")
+    out = args.json or str(Path(args.cand_dir) / "compare_vs_ref.json")
+    Path(out).write_text(json.dumps(res, indent=2))
+    print("wrote", out)
+
+
+if __name__ == "__main__":
+    main()

evaluation/harness/ifeval_lite.py

@@ -0,0 +1,236 @@
+"""Light-weight re-implementation of a SUBSET of the IFEval instruction checkers.
+
+Semantics follow google-research/google-research `instruction_following_eval/instructions.py`
+(Apache-2.0, (c) Google LLC), re-written from scratch with these documented deviations:
+  * word tokenisation uses regex \\w+ (orig: nltk RegexpTokenizer(r"\\w+") -> identical) and
+    capital-word counting uses whitespace/punct split instead of nltk.word_tokenize;
+  * `length_constraints:number_sentences` and a few rarely used ids are NOT supported; the eval
+    builder only selects prompts whose every instruction id is in SUPPORTED.
+  * language detection uses `langdetect` (Apache-2.0) with a fixed seed, like the original.
+Only "strict" scoring is implemented (prompt-level: all instructions pass).
+"""
+from __future__ import annotations
+
+import json
+import re
+import threading
+
+_LANG_LOCK = threading.Lock()  # langdetect seeds the *global* random module -> not thread-safe
+
+try:
+    import langdetect
+    langdetect.DetectorFactory.seed = 0
+except Exception:  # pragma: no cover
+    langdetect = None
+
+
+def _rel(count: int, relation: str, target: int) -> bool:
+    if relation == "less than":
+        return count < target
+    if relation == "at least":
+        return count >= target
+    raise ValueError(relation)
+
+
+def _lang(text: str) -> str | None:
+    if langdetect is None:
+        return None
+    try:
+        with _LANG_LOCK:
+            langdetect.DetectorFactory.seed = 0
+            return langdetect.detect(text)
+    except Exception:
+        return None
+
+
+def no_comma(r, **k):
+    return "," not in r
+
+
+def number_highlighted_sections(r, num_highlights, **k):
+    n = 0
+    for h in re.findall(r"\*[^\n\*]*\*", r):
+        if h.strip("*").strip():
+            n += 1
+    for h in re.findall(r"\*\*[^\n\*]*\*\*", r):
+        if h.removeprefix("**").removesuffix("**").strip():
+            n += 1
+    return n >= num_highlights
+
+
+def title(r, **k):
+    return any(t.lstrip("<").rstrip(">").strip() for t in re.findall(r"<<[^\n]+>>", r))
+
+
+def number_bullet_lists(r, num_bullets, **k):
+    a = re.findall(r"^\s*\*[^\*].*$", r, flags=re.M)
+    b = re.findall(r"^\s*-.*$", r, flags=re.M)
+    return len(a) + len(b) == num_bullets
+
+
+def json_format(r, **k):
+    v = r.strip()
+    v = re.sub(r"^```(?:json|Json|JSON)?\s*", "", v)
+    v = re.sub(r"\s*```$", "", v).strip()
+    try:
+        json.loads(v)
+        return True
+    except ValueError:
+        return False
+
+
+def multiple_sections(r, section_spliter, num_sections, **k):
+    pat = r"\s?" + re.escape(section_spliter.strip()) + r"\s?\d+\s?"
+    return len(re.split(pat, r)) - 1 >= num_sections
+
+
+def constrained_response(r, **k):
+    v = r.strip()
+    return any(o in v for o in ("My answer is yes.", "My answer is no.", "My answer is maybe."))
+
+
+def english_lowercase(r, **k):
+    return r == r.lower() and _lang(r) == "en"
+
+
+def english_capital(r, **k):
+    return r == r.upper() and _lang(r) == "en"
+
+
+def capital_word_frequency(r, capital_frequency, capital_relation, **k):
+    words = re.findall(r"[A-Za-z][A-Za-z'\-]*", r)
+    n = sum(1 for w in words if w.isupper())
+    return _rel(n, capital_relation, capital_frequency)
+
+
+def number_words(r, num_words, relation, **k):
+    return _rel(len(re.findall(r"\w+", r)), relation, num_words)
+
+
+def number_paragraphs(r, num_paragraphs, **k):
+    paras = re.split(r"\s?\*\*\*\s?", r)
+    n = len(paras)
+    for i, p in enumerate(paras):
+        if not p.strip():
+            if i == 0 or i == len(paras) - 1:
+                n -= 1
+            else:
+                return False
+    return n == num_paragraphs
+
+
+def nth_paragraph_first_word(r, num_paragraphs, nth_paragraph, first_word, **k):
+    paras = re.split(r"\n\n", r)
+    n = sum(1 for p in paras if p.strip())
+    paras_ne = [p for p in paras if p.strip()]
+    if nth_paragraph > len(paras_ne):
+        return False
+    p = paras_ne[nth_paragraph - 1].strip()
+    if not p:
+        return False
+    word = p.split()[0].strip()
+    word = word.lstrip("'\"").rstrip(".,?!'\"").lower()
+    # original strips leading/trailing punctuation char by char
+    word = re.sub(r"[^\w]+$", "", re.sub(r"^[^\w]+", "", word))
+    return n == num_paragraphs and word == first_word.lower()
+
+
+def existence(r, keywords, **k):
+    return all(re.search(kw, r, flags=re.I) for kw in keywords)
+
+
+def frequency(r, keyword, frequency, relation, **k):
+    return _rel(len(re.findall(keyword, r, flags=re.I)), relation, frequency)
+
+
+def forbidden_words(r, forbidden_words, **k):
+    return not any(re.search(r"\b" + w + r"\b", r, flags=re.I) for w in forbidden_words)
+
+
+def letter_frequency(r, letter, let_frequency, let_relation, **k):
+    return _rel(r.lower().count(letter.lower()), let_relation, let_frequency)
+
+
+def end_checker(r, end_phrase, **k):
+    return r.strip().strip('"').lower().endswith(end_phrase.strip().lower())
+
+
+def quotation(r, **k):
+    v = r.strip()
+    return len(v) > 1 and v[0] == '"' and v[-1] == '"'
+
+
+def postscript(r, postscript_marker, **k):
+    v = r.lower()
+    m = postscript_marker.strip()
+    if m == "P.P.S":
+        pat = r"\s*p\.\s?p\.\s?s.*$"
+    elif m == "P.S.":
+        pat = r"\s*p\.\s?s\..*$"
+    else:
+        pat = r"\s*" + re.escape(m.lower()) + r".*$"
+    return bool(re.findall(pat, v, flags=re.M))
+
+
+def number_placeholders(r, num_placeholders, **k):
+    return len(re.findall(r"\[.*?\]", r)) >= num_placeholders
+
+
+def repeat_prompt(r, prompt_to_repeat, **k):
+    return r.strip().lower().startswith(prompt_to_repeat.strip().lower())
+
+
+def two_responses(r, **k):
+    parts = r.split("******")
+    valid = []
+    for i, p in enumerate(parts):
+        if not p.strip():
+            if i != 0 and i != len(parts) - 1:
+                return False
+        else:
+            valid.append(p)
+    return len(valid) == 2 and valid[0].strip() != valid[1].strip()
+
+
+def response_language(r, language, **k):
+    return _lang(r) == language
+
+
+SUPPORTED = {
+    "punctuation:no_comma": no_comma,
+    "detectable_format:number_highlighted_sections": number_highlighted_sections,
+    "detectable_format:title": title,
+    "detectable_format:number_bullet_lists": number_bullet_lists,
+    "detectable_format:json_format": json_format,
+    "detectable_format:multiple_sections": multiple_sections,
+    "detectable_format:constrained_response": constrained_response,
+    "change_case:english_lowercase": english_lowercase,
+    "change_case:english_capital": english_capital,
+    "change_case:capital_word_frequency": capital_word_frequency,
+    "length_constraints:number_words": number_words,
+    "length_constraints:number_paragraphs": number_paragraphs,
+    "length_constraints:nth_paragraph_first_word": nth_paragraph_first_word,
+    "keywords:existence": existence,
+    "keywords:frequency": frequency,
+    "keywords:forbidden_words": forbidden_words,
+    "keywords:letter_frequency": letter_frequency,
+    "startend:end_checker": end_checker,
+    "startend:quotation": quotation,
+    "detectable_content:postscript": postscript,
+    "detectable_content:number_placeholders": number_placeholders,
+    "combination:repeat_prompt": repeat_prompt,
+    "combination:two_responses": two_responses,
+    "language:response_language": response_language,
+}
+
+
+def check(response: str, instruction_ids: list[str], kwargs_list: list[dict]) -> list[bool]:
+    out = []
+    for iid, kw in zip(instruction_ids, kwargs_list):
+        kw = {k: v for k, v in (kw or {}).items() if v is not None}
+        fn = SUPPORTED[iid]
+        try:
+            out.append(bool(response.strip()) and bool(fn(response, **kw)))
+        except Exception:
+            out.append(False)
+    return out

evaluation/harness/mock_server.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+"""Tiny OpenAI-compatible mock server that answers eval items with their GOLD answers.
+Used only to test the harness offline (scorers, raw-text tool-call fallback parser, compare, perf_bench).
+
+  .venv-eval/bin/python eval/mock_server.py --port 18999 [--tool-mode server|xml] [--corrupt-every K]
+
+--tool-mode xml returns tool calls as raw Qwen3.5-XML text in `content` (as a server without a
+tool parser would), exercising the fallback parser. --corrupt-every K returns a wrong answer for
+items whose id hash % K == 0 (to test compare.py deltas). Streaming is supported for perf_bench.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+import time
+from collections import Counter
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+from common import read_jsonl  # noqa: E402
+from tasks import TASKS, vqa_norm  # noqa: E402
+
+ARGS = None
+INDEX: dict[str, tuple[str, dict]] = {}
+
+
+def last_user_text(messages):
+    for m in reversed(messages):
+        if m["role"] == "user":
+            c = m["content"]
+            if isinstance(c, list):
+                return "".join(p.get("text", "") for p in c if p.get("type") == "text")
+            return c
+    return ""
+
+
+def build_index():
+    for task, (build, _) in TASKS.items():
+        for item in read_jsonl(HERE / "data" / f"{task}.jsonl"):
+            msgs, _ = build(item, HERE / "data")
+            INDEX[last_user_text(msgs)] = (task, item)
+
+
+def xml_call(c):
+    s = f"<tool_call>\n<function={c['name']}>\n"
+    for k, v in c["arguments"].items():
+        v = json.dumps(v) if isinstance(v, (dict, list)) else (str(v) if not isinstance(v, bool) else str(v))
+        s += f"<parameter={k}>\n{v}\n</parameter>\n"
+    return s + "</function>\n</tool_call>"
+
+
+def gold_response(task, item, corrupt):
+    """Return (content, tool_calls)."""
+    if task == "ifeval":
+        return "this is a mock response without commas.", None
+    if task == "gsm8k":
+        ans = str(int(item["answer"]) + 1) if corrupt else item["answer"]
+        body = item["solution"].split("####")[0].strip()
+        return f"{body}\nThe final answer is {ans}", None
+    if task == "mmlu_pro":
+        a = "ABCDEFGHIJ"[(("ABCDEFGHIJ".index(item["answer"]) + 1) % len(item["options"]))] if corrupt else item["answer"]
+        return f"Let's think. The answer is ({a}).", None
+    if task == "humaneval":
+        sol = "    return None\n" if corrupt else item["canonical_solution"]
+        return f"Here is the code:\n```python\n{item['prompt']}{sol}```", None
+    if task == "vqa":
+        return ("zzz" if corrupt else Counter(vqa_norm(a) for a in item["answers"]).most_common(1)[0][0]), None
+    if task == "tools":
+        if not item["gold_calls"] or corrupt:
+            return "I can answer that directly: mock.", None
+        calls = []
+        for g in item["gold_calls"]:
+            args = {k: v[0] for k, v in g["arguments"].items() if v and v[0] != "__optional__"}
+            calls.append({"name": g["name"], "arguments": args})
+        if ARGS.tool_mode == "xml":
+            return "\n".join(xml_call(c) for c in calls), None
+        return None, [{"id": f"call_{i}", "type": "function",
+                       "function": {"name": c["name"], "arguments": json.dumps(c["arguments"])}}
+                      for i, c in enumerate(calls)]
+    return "unknown", None
+
+
+class H(BaseHTTPRequestHandler):
+    protocol_version = "HTTP/1.1"
+
+    def log_message(self, *a):
+        pass
+
+    def _send(self, code, obj):
+        b = json.dumps(obj).encode()
+        self.send_response(code)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(b)))
+        self.end_headers()
+        self.wfile.write(b)
+
+    def do_GET(self):
+        if self.path.endswith("/models"):
+            return self._send(200, {"object": "list", "data": [{"id": "mock", "object": "model"}]})
+        self._send(404, {"error": "not found"})
+
+    def do_POST(self):
+        body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))))
+        msgs = body["messages"]
+        text = last_user_text(msgs)
+        prompt_tokens = sum(len(json.dumps(m["content"])) for m in msgs) // 4
+        if body.get("stream"):
+            return self._stream(body, prompt_tokens)
+        task_item = INDEX.get(text)
+        if task_item is None:
+            content, tcs = "mock: unknown prompt", None
+        else:
+            task, item = task_item
+            corrupt = ARGS.corrupt_every and int(hashlib.md5(item["id"].encode()).hexdigest(), 16) % ARGS.corrupt_every == 0
+            content, tcs = gold_response(task, item, corrupt)
+        msg = {"role": "assistant", "content": content}
+        if tcs:
+            msg["tool_calls"] = tcs
+        ct = max(1, len((content or "") + json.dumps(tcs or "")) // 4)
+        self._send(200, {"id": "mock", "object": "chat.completion", "model": body["model"],
+                         "choices": [{"index": 0, "message": msg, "finish_reason": "tool_calls" if tcs else "stop"}],
+                         "usage": {"prompt_tokens": prompt_tokens, "completion_tokens": ct,
+                                   "total_tokens": prompt_tokens + ct}})
+
+    def _stream(self, body, prompt_tokens):
+        n = int(body.get("max_tokens") or 16)
+        self.send_response(200)
+        self.send_header("Content-Type", "text/event-stream")
+        self.send_header("Transfer-Encoding", "chunked")
+        self.end_headers()
+
+        def chunk(obj):
+            data = f"data: {obj if isinstance(obj, str) else json.dumps(obj)}\n\n".encode()
+            self.wfile.write(f"{len(data):x}\r\n".encode() + data + b"\r\n")
+            self.wfile.flush()
+        time.sleep(ARGS.ttft)
+        for i in range(n):
+            chunk({"choices": [{"index": 0, "delta": {"content": "tok "}, "finish_reason": None}]})
+            time.sleep(ARGS.itl)
+        chunk({"choices": [{"index": 0, "delta": {}, "finish_reason": "length"}]})
+        if (body.get("stream_options") or {}).get("include_usage"):
+            chunk({"choices": [], "usage": {"prompt_tokens": prompt_tokens, "completion_tokens": n,
+                                            "total_tokens": prompt_tokens + n}})
+        chunk("[DONE]")
+        self.wfile.write(b"0\r\n\r\n")
+        self.wfile.flush()
+
+
+def main():
+    global ARGS
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--port", type=int, default=18999)
+    ap.add_argument("--tool-mode", choices=["server", "xml"], default="server")
+    ap.add_argument("--corrupt-every", type=int, default=0)
+    ap.add_argument("--ttft", type=float, default=0.02)
+    ap.add_argument("--itl", type=float, default=0.002)
+    ARGS = ap.parse_args()
+    build_index()
+    print(f"mock server on 127.0.0.1:{ARGS.port} ({len(INDEX)} items indexed)", flush=True)
+    srv = ThreadingHTTPServer(("127.0.0.1", ARGS.port), H)
+    srv.handle_error = lambda *a: None  # quiet on client disconnects
+    srv.daemon_threads = True
+    srv.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

evaluation/harness/perf_bench.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+"""Streaming throughput/latency benchmark for an OpenAI-compatible Chat Completions server.
+
+  .venv-eval/bin/python eval/perf_bench.py --base-url http://localhost:30000/v1 \
+      --model SenseNova-Loop-9B-Preview --concurrency 1,2,4,8 --num-requests 16 \
+      --input-tokens 1024 --output-tokens 256 --out eval/runs/perf_bf16.json
+
+Prompt: deterministic pseudo-text (fixed seed) cut to exactly --input-tokens tokens with the model's
+own tokenizer.json (the chat template adds a few more; actual prompt_tokens come from `usage`).
+Each request gets a unique numeric prefix so prefix caching cannot serve it (disable with
+--allow-prefix-cache). Requests use temperature 0, ignore_eos=true and min_tokens=max_tokens
+(vLLM/SGLang extensions; disable with --no-ignore-eos) and thinking off.
+Metrics per concurrency level: TTFT p50/p90/mean, per-request decode tok/s ((n_out-1)/(t_last-t_first)),
+inter-token latency, E2E latency p50/p90, aggregate output tok/s (sum out tokens / level wall time),
+request throughput, errors, actual generated lengths (from usage; falls back to chunk count).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import platform
+import random
+import statistics
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import requests
+
+HERE = Path(__file__).resolve().parent
+DEFAULT_TOK = HERE.parent / "upstream/SenseNova-Loop-9B-Preview/tokenizer.json"
+WORDS = ("the of and to in is was for on that with as by at from his her an were are which this be has "
+         "had it not or have one their they its first new after who two been more year also other some "
+         "into all time only most over three city river mountain village market winter summer story "
+         "engine signal harbor garden library window letter bridge forest island station machine "
+         "children ancient quiet bright careful distant golden narrow sudden gentle measured").split()
+
+
+def build_prompt_text(n_tokens: int, tokenizer_path: str | None):
+    rng = random.Random(1234)
+    sents = []
+    for _ in range(n_tokens):  # plenty of text
+        k = rng.randint(8, 18)
+        s = " ".join(rng.choice(WORDS) for _ in range(k))
+        sents.append(s[0].upper() + s[1:] + ".")
+    text = " ".join(sents)
+    if tokenizer_path and Path(tokenizer_path).exists():
+        from tokenizers import Tokenizer
+        tok = Tokenizer.from_file(str(tokenizer_path))
+        ids = tok.encode(text, add_special_tokens=False).ids[:n_tokens]
+        text = tok.decode(ids)
+        return text, len(tok.encode(text, add_special_tokens=False).ids), "tokenizer.json"
+    words = text.split()[: int(n_tokens * 0.75)]
+    return " ".join(words), None, "approx(0.75 words/token)"
+
+
+def pct(xs, q):
+    if not xs:
+        return None
+    xs = sorted(xs)
+    k = (len(xs) - 1) * q
+    f = int(k)
+    c = min(f + 1, len(xs) - 1)
+    return xs[f] + (xs[c] - xs[f]) * (k - f)
+
+
+def one_request(args, session, text, idx):
+    prefix = "" if args.allow_prefix_cache else f"[req {idx:06d}] "
+    body = {"model": args.model, "stream": True, "stream_options": {"include_usage": True},
+            "messages": [{"role": "user", "content": prefix + text + "\n\nContinue this text."}],
+            "max_tokens": args.output_tokens, "temperature": 0.0,
+            "chat_template_kwargs": {"enable_thinking": False}}
+    if not args.no_ignore_eos:
+        body.update({"ignore_eos": True, "min_tokens": args.output_tokens})
+    t0 = time.perf_counter()
+    t_first = t_last = None
+    n_chunks = 0
+    usage = None
+    finish = None
+    try:
+        with session.post(args.base_url.rstrip("/") + "/chat/completions", json=body, stream=True,
+                          timeout=args.timeout) as r:
+            if r.status_code != 200:
+                return {"ok": False, "error": f"HTTP {r.status_code}: {r.text[:300]}"}
+            for line in r.iter_lines():
+                if not line or not line.startswith(b"data:"):
+                    continue
+                data = line[5:].strip()
+                if data == b"[DONE]":
+                    break
+                j = json.loads(data)
+                if j.get("usage"):
+                    usage = j["usage"]
+                for ch in j.get("choices") or []:
+                    d = ch.get("delta") or {}
+                    if d.get("content") or d.get("reasoning_content") or d.get("reasoning"):
+                        now = time.perf_counter()
+                        if t_first is None:
+                            t_first = now
+                        t_last = now
+                        n_chunks += 1
+                    if ch.get("finish_reason"):
+                        finish = ch["finish_reason"]
+    except Exception as e:  # noqa: BLE001
+        return {"ok": False, "error": f"{type(e).__name__}: {e}"}
+    t_end = time.perf_counter()
+    if t_first is None:
+        return {"ok": False, "error": "no tokens received"}
+    n_out = (usage or {}).get("completion_tokens") or n_chunks
+    decode_t = (t_last - t_first) if t_last and t_last > t_first else None
+    return {"ok": True, "ttft_s": t_first - t0, "e2e_s": t_end - t0, "output_tokens": n_out,
+            "prompt_tokens": (usage or {}).get("prompt_tokens"), "n_chunks": n_chunks, "finish_reason": finish,
+            "decode_tok_s": (n_out - 1) / decode_t if decode_t and n_out > 1 else None,
+            "itl_s": decode_t / (n_out - 1) if decode_t and n_out > 1 else None,
+            "usage_present": usage is not None}
+
+
+def run_level(args, session, text, c, base_idx):
+    n = args.num_requests if args.num_requests else max(8, 4 * c)
+    t0 = time.perf_counter()
+    with ThreadPoolExecutor(c) as ex:
+        res = list(ex.map(lambda i: one_request(args, session, text, base_idx + i), range(n)))
+    wall = time.perf_counter() - t0
+    ok = [r for r in res if r["ok"]]
+
+    def stat(key):
+        xs = [r[key] for r in ok if r.get(key) is not None]
+        return {"p50": pct(xs, 0.5), "p90": pct(xs, 0.9), "mean": statistics.fmean(xs) if xs else None}
+    outs = [r["output_tokens"] for r in ok]
+    return {"concurrency": c, "n_requests": n, "n_ok": len(ok), "n_errors": len(res) - len(ok),
+            "errors": [r["error"] for r in res if not r["ok"]][:5], "wall_s": wall,
+            "ttft_s": stat("ttft_s"), "e2e_s": stat("e2e_s"), "decode_tok_s_per_req": stat("decode_tok_s"),
+            "itl_s": stat("itl_s"),
+            "output_tokens_mean": statistics.fmean(outs) if outs else None,
+            "output_tokens_min": min(outs) if outs else None, "output_tokens_max": max(outs) if outs else None,
+            "prompt_tokens_mean": statistics.fmean([r["prompt_tokens"] for r in ok if r.get("prompt_tokens")] or [0]),
+            "n_short_outputs": sum(1 for o in outs if o < args.output_tokens),
+            "aggregate_output_tok_s": sum(outs) / wall if wall else None,
+            "requests_per_s": len(ok) / wall if wall else None,
+            "usage_reported": all(r.get("usage_present") for r in ok)}
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base-url", required=True)
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--concurrency", default="1,2,4,8")
+    ap.add_argument("--num-requests", type=int, default=16, help="per level (0 => max(8, 4*c))")
+    ap.add_argument("--warmup", type=int, default=2)
+    ap.add_argument("--input-tokens", type=int, default=1024)
+    ap.add_argument("--output-tokens", type=int, default=256)
+    ap.add_argument("--tokenizer", default=str(DEFAULT_TOK))
+    ap.add_argument("--allow-prefix-cache", action="store_true")
+    ap.add_argument("--no-ignore-eos", action="store_true")
+    ap.add_argument("--timeout", type=float, default=600)
+    ap.add_argument("--label", default="")
+    ap.add_argument("--out", required=True)
+    args = ap.parse_args()
+
+    text, n_tok, how = build_prompt_text(args.input_tokens, args.tokenizer)
+    levels = [int(x) for x in args.concurrency.split(",")]
+    session = requests.Session()
+    session.mount("http://", requests.adapters.HTTPAdapter(pool_maxsize=max(levels) * 2 + 4))
+    for i in range(args.warmup):
+        w = one_request(args, session, text, 900000 + i)
+        if not w["ok"]:
+            print("warmup error:", w["error"], file=sys.stderr)
+    res = {"label": args.label, "model": args.model, "base_url": args.base_url, "host": platform.node(),
+           "time": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+           "config": {**vars(args), "prompt_text_tokens": n_tok, "prompt_build": how}, "levels": []}
+    base = 0
+    for c in levels:
+        lv = run_level(args, session, text, c, base)
+        base += 10000
+        res["levels"].append(lv)
+        f = lambda x: "n/a" if x is None else f"{x:.3f}"  # noqa: E731
+        print(f"c={c:3d} ok={lv['n_ok']}/{lv['n_requests']} TTFT p50={f(lv['ttft_s']['p50'])}s "
+              f"p90={f(lv['ttft_s']['p90'])}s decode/req p50={f(lv['decode_tok_s_per_req']['p50'])} tok/s "
+              f"E2E p50={f(lv['e2e_s']['p50'])}s agg={f(lv['aggregate_output_tok_s'])} tok/s "
+              f"out_len={lv['output_tokens_mean']}", flush=True)
+    Path(args.out).parent.mkdir(parents=True, exist_ok=True)
+    Path(args.out).write_text(json.dumps(res, indent=2))
+    print("wrote", args.out)
+
+
+if __name__ == "__main__":
+    main()

evaluation/harness/requirements.lock

@@ -0,0 +1,48 @@
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.5
+aiosignal==1.4.0
+anyio==4.15.1
+attrs==26.1.0
+certifi==2026.10.23
+charset-normalizer==3.5.2
+click==8.5.0
+cloudpickle==3.1.2
+datasets==5.1.0
+defusedxml==0.7.1
+dill==0.4.1
+filelock==4.2.5
+frozenlist==1.8.0
+fsspec==2026.7.0
+h11==0.16.0
+hf-xet==1.8.0
+httpcore2==2.16.0
+httpx2==2.16.0
+huggingface-hub==2.4.0
+idna==3.20
+immutabledict==4.3.1
+jinja2==3.1.6
+joblib==1.6.0
+langdetect==1.0.9
+markupsafe==3.0.4
+multidict==7.2.1
+multiprocess==0.70.19
+nltk==3.10.4
+numpy==2.5.4
+packaging==26.3
+pandas==3.1.0
+pillow==13.0.0
+propcache==0.5.4
+pyarrow==26.0.0
+python-dateutil==3.0.0
+pyyaml==6.0.3
+regex==2026.9.29
+requests==2.34.2
+six==1.17.0
+tokenizers==0.23.3
+tqdm==4.70.1
+truststore==0.10.5
+typing-extensions==4.16.0
+tzdata==2026.6
+urllib3==2.8.0
+xxhash==4.0.2
+yarl==1.25.1

evaluation/harness/run_eval.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+"""Run the fixed eval suite against an OpenAI-compatible Chat Completions endpoint.
+
+Example:
+  .venv-eval/bin/python eval/run_eval.py --base-url http://localhost:30000/v1 \
+      --model SenseNova-Loop-9B-Preview --out eval/runs/bf16 --concurrency 16
+Writes <out>/raw_<task>.jsonl (one line per item, full response + score) and <out>/summary.json.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import platform
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import requests  # noqa: E402
+
+from common import EVAL_VERSION, chat, read_jsonl, wilson_ci, write_jsonl  # noqa: E402
+from tasks import MAX_TOKENS, TASKS, agreement_key  # noqa: E402
+
+DATA = HERE / "data"
+
+
+def run_item(task, item, args, session):
+    build, score = TASKS[task]
+    messages, tools = build(item, DATA)
+    resp = chat(session, args.base_url, args.model, messages, max_tokens=MAX_TOKENS[task], tools=tools,
+                enable_thinking=args.enable_thinking, timeout=args.timeout)
+    try:
+        sc = score(item, resp) if resp["ok"] else {"pred": None, "correct": False}
+    except Exception as e:  # noqa: BLE001
+        sc = {"pred": None, "correct": False, "score_error": f"{type(e).__name__}: {e}"}
+    sc["correct"] = bool(sc["correct"])
+    return {
+        "id": item["id"], "task": task,
+        "prompt_sha256": hashlib.sha256(json.dumps(messages, sort_keys=True)[:200000].encode()).hexdigest()[:16],
+        "content": resp["content"], "reasoning_content": resp["reasoning_content"],
+        "tool_calls": resp["tool_calls"], "finish_reason": resp["finish_reason"],
+        "usage": resp["usage"], "completion_tokens": (resp["usage"] or {}).get("completion_tokens"),
+        "latency_s": resp["latency_s"], "error": resp["error"],
+        "agree_key": agreement_key(task, sc, resp), **sc,
+    }
+
+
+def summarize_task(rows):
+    n = len(rows)
+    k = sum(r["correct"] for r in rows)
+    lo, hi = wilson_ci(k, n)
+    toks = [r["completion_tokens"] for r in rows if r["completion_tokens"] is not None]
+    s = {"n": n, "correct": k, "accuracy": k / n if n else None, "wilson95": [lo, hi],
+         "n_errors": sum(1 for r in rows if r["error"]),
+         "n_truncated": sum(1 for r in rows if r["finish_reason"] == "length"),
+         "mean_completion_tokens": sum(toks) / len(toks) if toks else None,
+         "sum_completion_tokens": sum(toks)}
+    if rows and "vqa_soft_acc" in rows[0]:
+        s["vqa_soft_acc"] = sum(r.get("vqa_soft_acc", 0) for r in rows) / n
+    if rows and rows[0]["task"] == "tools":
+        s["names_correct"] = sum(1 for r in rows if r.get("names_correct")) / n
+        s["parse_source_counts"] = {}
+        for r in rows:
+            src = r.get("parse_source", "none")
+            s["parse_source_counts"][src] = s["parse_source_counts"].get(src, 0) + 1
+        fmt = [r.get("format", {}) for r in rows if r.get("parse_source") != "none"]
+        s["format_well_formed"] = (sum(1 for f in fmt if f.get("well_formed")) / len(fmt)) if fmt else None
+        s["raw_tool_call_tags_in_content"] = sum(1 for r in rows if "<tool_call>" in (r["content"] or ""))
+        by = {}
+        for r in rows:
+            c = r["id"].split("_")[1]
+            by.setdefault(c, [0, 0])
+            by[c][0] += r["correct"]
+            by[c][1] += 1
+        s["by_category"] = {c: f"{a}/{b}" for c, (a, b) in sorted(by.items())}
+    if rows and rows[0]["task"] == "ifeval":
+        flat = [x for r in rows for x in r.get("per_instruction", [])]
+        s["instruction_level_acc"] = sum(flat) / len(flat) if flat else None
+        s["n_instructions"] = len(flat)
+    return s
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base-url", required=True, help="e.g. http://localhost:30000/v1")
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--tasks", default=",".join(TASKS))
+    ap.add_argument("--concurrency", type=int, default=8)
+    ap.add_argument("--limit", type=int, default=None, help="debug: first N items per task")
+    ap.add_argument("--enable-thinking", action="store_true", help="default OFF")
+    ap.add_argument("--timeout", type=float, default=900)
+    ap.add_argument("--api-key", default="EMPTY")
+    args = ap.parse_args()
+
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    session = requests.Session()
+    session.headers["Authorization"] = f"Bearer {args.api_key}"
+    adapter = requests.adapters.HTTPAdapter(pool_maxsize=max(16, args.concurrency * 2))
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+
+    manifest = json.loads((DATA / "MANIFEST.json").read_text())
+    summary = {"eval_version": EVAL_VERSION, "model": args.model, "base_url": args.base_url,
+               "started": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "enable_thinking": args.enable_thinking,
+               "sampling": {"temperature": 0.0, "top_p": 1.0, "seed": 0, "max_tokens": MAX_TOKENS},
+               "concurrency": args.concurrency, "limit": args.limit, "host": platform.node(),
+               "data_sha256": {t: v["sha256"] for t, v in manifest["tasks"].items()}, "tasks": {}}
+    try:
+        summary["server_models"] = session.get(args.base_url.rstrip("/") + "/models", timeout=10).json()
+    except Exception as e:  # noqa: BLE001
+        summary["server_models"] = f"unavailable: {e}"
+
+    t_all = time.time()
+    for task in args.tasks.split(","):
+        items = read_jsonl(DATA / f"{task}.jsonl")
+        if args.limit:
+            items = items[: args.limit]
+        t0 = time.time()
+        with ThreadPoolExecutor(args.concurrency) as ex:
+            rows = list(ex.map(lambda it: run_item(task, it, args, session), items))
+        write_jsonl(out / f"raw_{task}.jsonl", rows)
+        s = summarize_task(rows)
+        s["wall_s"] = round(time.time() - t0, 1)
+        summary["tasks"][task] = s
+        print(f"{task:10s} n={s['n']:3d} acc={s['accuracy']:.3f} "
+              f"CI95=[{s['wilson95'][0]:.3f},{s['wilson95'][1]:.3f}] err={s['n_errors']} "
+              f"trunc={s['n_truncated']} mean_tok={s['mean_completion_tokens']} {s['wall_s']}s", flush=True)
+    summary["wall_s"] = round(time.time() - t_all, 1)
+    (out / "summary.json").write_text(json.dumps(summary, indent=2, default=str))
+    print("wrote", out / "summary.json")
+
+
+if __name__ == "__main__":
+    main()

evaluation/harness/tasks.py

@@ -0,0 +1,284 @@
+"""Task definitions: prompt construction, max_tokens, answer extraction and scoring.
+
+Every task exposes:
+  build_messages(item, data_dir) -> (messages, tools|None)
+  score(item, resp) -> dict(pred=..., correct=bool, **details)   (resp = common.chat() result)
+"""
+from __future__ import annotations
+
+import base64
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import ifeval_lite
+from common import calls_from_message, strip_think
+
+# Fixed per-task generation budgets (thinking OFF). Changing these => bump EVAL_VERSION.
+MAX_TOKENS = {"ifeval": 1024, "gsm8k": 1024, "mmlu_pro": 1024, "humaneval": 1024,
+              "tools": 512, "vqa": 64}
+
+# ============================================================================ IFEval
+def ifeval_messages(item, data_dir):
+    return [{"role": "user", "content": item["prompt"]}], None
+
+
+def ifeval_score(item, resp):
+    text = strip_think(resp["content"])
+    per = ifeval_lite.check(text, item["instruction_id_list"], item["kwargs"])
+    return {"pred": None, "correct": all(per), "per_instruction": per}
+
+
+# ============================================================================ GSM8K
+GSM_TMPL = ("Solve the following math problem step by step. At the end, write the final "
+            "answer on its own line in the form \"The final answer is <number>\".\n\n{q}")
+NUM_RE = r"-?\$?\s?\d[\d,]*(?:\.\d+)?"
+
+
+def _norm_num(s):
+    if s is None:
+        return None
+    s = s.replace(",", "").replace("$", "").replace(" ", "").rstrip(".")
+    try:
+        x = float(s)
+    except ValueError:
+        return None
+    return str(int(x)) if x == int(x) else repr(round(x, 6))
+
+
+def gsm_extract(text):
+    text = strip_think(text)
+    m = re.findall(r"final answer is[:\s]*\**\s*\\?\(?\\?(?:boxed\{)?\s*(" + NUM_RE + ")", text, re.I)
+    if m:
+        return _norm_num(m[-1])
+    m = re.findall(r"\\boxed\{\s*(" + NUM_RE + ")", text)
+    if m:
+        return _norm_num(m[-1])
+    m = re.findall(NUM_RE, text)
+    return _norm_num(m[-1]) if m else None
+
+
+def gsm8k_messages(item, data_dir):
+    return [{"role": "user", "content": GSM_TMPL.format(q=item["question"])}], None
+
+
+def gsm8k_score(item, resp):
+    pred = gsm_extract(resp["content"])
+    return {"pred": pred, "correct": pred is not None and pred == _norm_num(item["answer"])}
+
+
+# ============================================================================ MMLU-Pro
+LETTERS = "ABCDEFGHIJ"
+MC_TMPL = ("The following is a multiple choice question about {cat}. Think step by step briefly, "
+           "then finish with \"The answer is (X)\" where X is the letter of the correct option.\n\n"
+           "Question: {q}\n{opts}")
+
+
+def mmlu_pro_messages(item, data_dir):
+    opts = "\n".join(f"({LETTERS[i]}) {o}" for i, o in enumerate(item["options"]))
+    return [{"role": "user", "content": MC_TMPL.format(cat=item["category"], q=item["question"], opts=opts)}], None
+
+
+def mc_extract(text, n_opts):
+    text = strip_think(text)
+    valid = LETTERS[:n_opts]
+    m = re.findall(r"answer is:?\s*\**\(?([A-J])\)?", text, re.I)
+    m = [x.upper() for x in m if x.upper() in valid]
+    if m:
+        return m[-1]
+    m = re.findall(r"(?:^|\s)\(?([A-J])\)?[\.\s]*$", text.strip())
+    if m and m[-1] in valid:
+        return m[-1]
+    m = re.findall(r"\(([A-J])\)", text)
+    m = [x for x in m if x in valid]
+    return m[-1] if m else None
+
+
+def mmlu_pro_score(item, resp):
+    pred = mc_extract(resp["content"], len(item["options"]))
+    return {"pred": pred, "correct": pred == item["answer"]}
+
+
+# ============================================================================ HumanEval
+HE_TMPL = ("Complete the following Python function. Return the complete function definition "
+           "(including the signature and any needed imports) in a single ```python code block.\n\n"
+           "```python\n{p}```")
+
+
+def humaneval_messages(item, data_dir):
+    return [{"role": "user", "content": HE_TMPL.format(p=item["prompt"])}], None
+
+
+def extract_code(text, item):
+    text = strip_think(text)
+    blocks = re.findall(r"```(?:python|py|Python)?\s*\n(.*?)```", text, re.S)
+    code = None
+    for b in blocks:
+        if f"def {item['entry_point']}" in b:
+            code = b
+            break
+    if code is None and blocks:
+        code = blocks[0]
+    if code is None:
+        code = text
+    if f"def {item['entry_point']}" not in code:  # body-only completion
+        code = item["prompt"] + code
+    else:  # keep the prompt's imports/helper defs available
+        code = item["prompt"].split(f"def {item['entry_point']}")[0] + "\n" + code
+    return code
+
+
+def run_python_sandboxed(program: str, timeout: float = 10.0) -> tuple[bool, str]:
+    """Execute untrusted code in a fresh isolated interpreter in a temp dir with rlimits."""
+    with tempfile.TemporaryDirectory(prefix="he_") as td:
+        path = os.path.join(td, "prog.py")
+        with open(path, "w") as f:
+            f.write(program)
+
+        def limits():
+            import resource
+            resource.setrlimit(resource.RLIMIT_AS, (2 << 30, 2 << 30))
+            resource.setrlimit(resource.RLIMIT_CPU, (int(timeout) + 1, int(timeout) + 1))
+            resource.setrlimit(resource.RLIMIT_FSIZE, (10 << 20, 10 << 20))
+            os.setsid()
+        env = {"PATH": "/usr/bin:/bin", "HOME": td, "PYTHONHASHSEED": "0"}
+        try:
+            p = subprocess.run([sys.executable, "-I", "-B", path], cwd=td, env=env, capture_output=True,
+                               timeout=timeout, preexec_fn=limits, text=True)
+            return p.returncode == 0, (p.stderr or "")[-400:]
+        except subprocess.TimeoutExpired:
+            return False, "timeout"
+
+
+def humaneval_score(item, resp):
+    code = extract_code(resp["content"], item)
+    prog = code + "\n\n" + item["test"] + f"\n\ncheck({item['entry_point']})\n"
+    ok, err = run_python_sandboxed(prog)
+    return {"pred": code, "correct": ok, "exec_error": None if ok else err}
+
+
+# ============================================================================ Tool calling
+def tools_messages(item, data_dir):
+    msgs = []
+    if item.get("system"):
+        msgs.append({"role": "system", "content": item["system"]})
+    msgs.append({"role": "user", "content": item["user"]})
+    return msgs, item["tools"]
+
+
+def _val_match(got, allowed):
+    for a in allowed:
+        if isinstance(a, (int, float)) and not isinstance(a, bool):
+            try:
+                if abs(float(got) - float(a)) < 1e-6:
+                    return True
+            except (TypeError, ValueError):
+                pass
+        elif isinstance(a, str) and isinstance(got, str):
+            if got.strip().lower() == a.strip().lower():
+                return True
+        elif got == a:
+            return True
+    return False
+
+
+def _call_match(call, gold):
+    if call.get("name") != gold["name"]:
+        return False
+    args = call.get("arguments") or {}
+    if not isinstance(args, dict):
+        return False
+    spec = gold["arguments"]  # name -> list of allowed values; [] or containing "__optional__"
+    for k, allowed in spec.items():
+        optional = "__optional__" in allowed
+        allowed = [a for a in allowed if a != "__optional__"]
+        if k not in args:
+            if not optional:
+                return False
+            continue
+        if allowed and not _val_match(args[k], allowed):
+            return False
+    return all(k in spec for k in args)  # no hallucinated params
+
+
+def tools_score(item, resp):
+    calls, source, fmt = calls_from_message(resp["message"] or {"content": resp["content"]}, item["tools"])
+    gold = item["gold_calls"]
+    if not gold:  # irrelevance: must NOT call any tool
+        correct = len(calls) == 0
+    else:
+        remaining = list(calls)
+        correct = len(calls) == len(gold)
+        for g in gold:  # order-insensitive matching for parallel calls
+            hit = next((c for c in remaining if _call_match(c, g)), None)
+            if hit is None:
+                correct = False
+                break
+            remaining.remove(hit)
+    names_ok = sorted(c.get("name") or "" for c in calls) == sorted(g["name"] for g in gold)
+    return {"pred": calls, "correct": correct, "names_correct": names_ok, "parse_source": source,
+            "format": fmt}
+
+
+# ============================================================================ VQA (TextVQA)
+VQA_SUFFIX = "\nAnswer the question using a single word or phrase."
+_CONTRACT = {"dont": "don't", "doesnt": "doesn't", "isnt": "isn't", "cant": "can't", "wont": "won't",
+             "im": "i'm", "ive": "i've", "youre": "you're", "thats": "that's"}
+_NUMW = {"none": "0", "zero": "0", "one": "1", "two": "2", "three": "3", "four": "4", "five": "5",
+         "six": "6", "seven": "7", "eight": "8", "nine": "9", "ten": "10"}
+_ART = {"a", "an", "the"}
+
+
+def vqa_norm(s: str) -> str:
+    s = (s or "").strip().lower().replace("\n", " ").replace("\t", " ")
+    s = s.rstrip(".")
+    s = re.sub(r"(?<=\d),(?=\d)", "", s)
+    s = re.sub(r"[;/\[\]\"{}()=+\\_\-><@`,?!*#%^&$:~|]", " ", s)
+    s = re.sub(r"(?<!\d)\.(?!\d)", "", s)
+    words = [_NUMW.get(w, _CONTRACT.get(w, w)) for w in s.split()]
+    return " ".join(w for w in words if w not in _ART)
+
+
+def vqa_messages(item, data_dir):
+    img_path = Path(data_dir) / item["image"]
+    b64 = base64.b64encode(img_path.read_bytes()).decode()
+    return [{"role": "user", "content": [
+        {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}},
+        {"type": "text", "text": item["question"] + VQA_SUFFIX},
+    ]}], None
+
+
+def vqa_score(item, resp):
+    pred = vqa_norm(strip_think(resp["content"]).split("\n")[0])
+    golds = [vqa_norm(a) for a in item["answers"]]
+    matches = sum(1 for g in golds if g == pred)
+    soft = min(1.0, matches / 3.0)
+    # primary binary metric (relaxed): normalized exact match with >=2 of the 10 annotator answers
+    # (robust to a single noisy annotation); standard VQA soft accuracy min(1, m/3) also recorded.
+    return {"pred": pred, "correct": matches >= 2 and bool(pred), "vqa_soft_acc": soft,
+            "any_annotator_match": matches >= 1}
+
+
+TASKS = {
+    "ifeval": (ifeval_messages, ifeval_score),
+    "gsm8k": (gsm8k_messages, gsm8k_score),
+    "mmlu_pro": (mmlu_pro_messages, mmlu_pro_score),
+    "humaneval": (humaneval_messages, humaneval_score),
+    "tools": (tools_messages, tools_score),
+    "vqa": (vqa_messages, vqa_score),
+}
+
+
+def agreement_key(task: str, scored: dict, resp: dict):
+    """Label-free comparable answer used for reference agreement."""
+    if task == "ifeval":
+        return strip_think(resp.get("content", "")).strip()
+    if task == "humaneval":
+        return (scored.get("pred") or "").strip()
+    if task == "tools":
+        return json.dumps(scored.get("pred"), sort_keys=True, default=str)
+    return scored.get("pred")

evaluation/harness/test_harness.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+"""Offline tests for scorers + end-to-end harness against eval/mock_server.py (no GPU/model needed).
+
+  .venv-eval/bin/python eval/test_harness.py
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import ifeval_lite  # noqa: E402
+from common import parse_xml_tool_calls, read_jsonl, strip_think, wilson_ci  # noqa: E402
+from tasks import (extract_code, gsm_extract, humaneval_score, mc_extract, run_python_sandboxed,  # noqa: E402
+                   tools_score, vqa_norm, vqa_score)
+
+PY = sys.executable
+FAILS = []
+
+
+def check(cond, msg):
+    if not cond:
+        FAILS.append(msg)
+        print("FAIL:", msg)
+
+
+def unit():
+    lo, hi = wilson_ci(50, 100)
+    check(abs(lo - 0.4038) < 1e-3 and abs(hi - 0.5962) < 1e-3, f"wilson {lo},{hi}")
+    check(strip_think("<think>\nabc\n</think>\n\nHi") == "Hi", "strip_think")
+    # GSM8K
+    check(gsm_extract("so 3+4=7.\nThe final answer is 1,234.") == "1234", "gsm final")
+    check(gsm_extract("The final answer is $18.00") == "18", "gsm dollars")
+    check(gsm_extract("answer \\boxed{42}") == "42", "gsm boxed")
+    check(gsm_extract("it is -3.5 or so") == "-3.5", "gsm last num")
+    # MC
+    check(mc_extract("blah (B) ... The answer is (C).", 4) == "C", "mc 1")
+    check(mc_extract("The answer is J", 10) == "J", "mc 2")
+    check(mc_extract("The answer is (J)", 4) is None, "mc invalid letter")
+    # IFEval
+    C = ifeval_lite.check
+    check(C("hello world", ["punctuation:no_comma"], [{}]) == [True], "no_comma +")
+    check(C("hello, world", ["punctuation:no_comma"], [{}]) == [False], "no_comma -")
+    check(C("<<My Title>> text", ["detectable_format:title"], [{}]) == [True], "title")
+    check(C("* a\n* b\n- c", ["detectable_format:number_bullet_lists"], [{"num_bullets": 3}]) == [True], "bullets")
+    check(C('```json\n{"a": 1}\n```', ["detectable_format:json_format"], [{}]) == [True], "json")
+    check(C("p1\n***\np2", ["length_constraints:number_paragraphs"], [{"num_paragraphs": 2}]) == [True], "paras")
+    check(C("one two three", ["length_constraints:number_words"], [{"num_words": 5, "relation": "less than"}]) == [True], "words")
+    check(C("THIS IS ALL CAPS ENGLISH TEXT ABOUT THE WEATHER TODAY", ["change_case:english_capital"], [{}]) == [True], "caps")
+    check(C("text\nP.S. bye", ["detectable_content:postscript"], [{"postscript_marker": "P.S."}]) == [True], "ps")
+    check(C("a ****** b", ["combination:two_responses"], [{}]) == [True], "two resp")
+    check(C("Is it? My answer is yes.", ["detectable_format:constrained_response"], [{}]) == [True], "constrained")
+    check(C("Das ist ein deutscher Satz über das Wetter in Berlin heute.", ["language:response_language"],
+            [{"language": "de"}]) == [True], "lang")
+    check(C("We like apples", ["keywords:forbidden_words"], [{"forbidden_words": ["apples"]}]) == [False], "forbidden")
+    # Sandbox
+    ok, _ = run_python_sandboxed("assert 1+1==2\n")
+    check(ok, "sandbox pass")
+    ok, _ = run_python_sandboxed("assert 1+1==3\n")
+    check(not ok, "sandbox fail")
+    t = time.time()
+    ok, err = run_python_sandboxed("while True: pass\n", timeout=2)
+    check(not ok and time.time() - t < 6, f"sandbox timeout {err}")
+    ok, _ = run_python_sandboxed("import os; open('/tmp/../' + 'x','r')\n")  # expected to fail cleanly
+    he = read_jsonl(HERE / "data/humaneval.jsonl")[0]
+    r = humaneval_score(he, {"content": f"```python\n{he['prompt']}{he['canonical_solution']}```"})
+    check(r["correct"], "humaneval canonical passes")
+    r = humaneval_score(he, {"content": f"```python\n{he['canonical_solution']}```"})  # body-only
+    check(r["correct"], "humaneval body-only passes")
+    check("def " in extract_code("no code", he), "extract fallback")
+    # XML tool-call parser (format from chat_template.jinja)
+    tools = [{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {
+        "s": {"type": "string"}, "n": {"type": "integer"}, "b": {"type": "boolean"}, "o": {"type": "object"}}}}}]
+    txt = ("Sure.\n\n<tool_call>\n<function=f>\n<parameter=s>\n42\n</parameter>\n<parameter=n>\n7\n</parameter>\n"
+           "<parameter=b>\nTrue\n</parameter>\n<parameter=o>\n{\"k\": [1, 2]}\n</parameter>\n</function>\n</tool_call>")
+    calls, rep = parse_xml_tool_calls(txt, tools)
+    check(calls == [{"name": "f", "arguments": {"s": "42", "n": 7, "b": True, "o": {"k": [1, 2]}}}], f"xml {calls}")
+    check(rep["well_formed"] and not rep["suffix_after_last_call"], f"xml rep {rep}")
+    calls, rep = parse_xml_tool_calls("<tool_call>\n<function=f>\n<parameter=s>\nx", tools)
+    check(calls == [] and not rep["well_formed"], "xml truncated")
+    tc = read_jsonl(HERE / "data/tools.jsonl")
+    par = next(c for c in tc if c["id"] == "tool_parallel_03")
+    msg = {"content": None, "tool_calls": [
+        {"function": {"name": "set_timer", "arguments": '{"minutes": 10}'}},
+        {"function": {"name": "set_light_state", "arguments": '{"room": "Bedroom", "on": false}'}}]}
+    check(tools_score(par, {"message": msg, "content": ""})["correct"], "tools parallel order-insensitive")
+    msg["tool_calls"][0]["function"]["arguments"] = '{"minutes": 10, "bogus": 1}'
+    check(not tools_score(par, {"message": msg, "content": ""})["correct"], "tools hallucinated param")
+    irr = next(c for c in tc if c["category"] == "irrelevance")
+    check(tools_score(irr, {"message": {"content": "Jane Austen"}, "content": "Jane Austen"})["correct"], "irrelevance")
+    # VQA
+    check(vqa_norm("The Two Dogs.") == "2 dogs", f"vqa norm {vqa_norm('The Two Dogs.')}")
+    v = {"answers": ["dakota"] * 6 + ["x"] * 4}
+    check(vqa_score(v, {"content": "Dakota"})["correct"], "vqa")
+    check(not vqa_score({"answers": ["a1"] + ["z"] * 9}, {"content": "a1"})["correct"], "vqa single-annotator")
+
+
+def e2e():
+    with tempfile.TemporaryDirectory() as td:
+        procs = []
+        try:
+            for port, extra in ((18991, []), (18992, ["--tool-mode", "xml"]), (18993, ["--corrupt-every", "5"])):
+                procs.append(subprocess.Popen([PY, "-I", str(HERE / "mock_server.py"), "--port", str(port), *extra],
+                                              stdout=subprocess.DEVNULL))
+            time.sleep(2.5)
+            for port, name in ((18991, "gold"), (18992, "xml"), (18993, "corrupt")):
+                subprocess.run([PY, "-I", str(HERE / "run_eval.py"), "--base-url", f"http://127.0.0.1:{port}/v1",
+                                "--model", "mock", "--out", f"{td}/{name}", "--concurrency", "8"], check=True)
+            g = json.load(open(f"{td}/gold/summary.json"))["tasks"]
+            for t in ("gsm8k", "mmlu_pro", "humaneval", "tools", "vqa"):
+                check(g[t]["accuracy"] == 1.0, f"gold run {t} acc={g[t]['accuracy']}")
+            x = json.load(open(f"{td}/xml/summary.json"))["tasks"]["tools"]
+            check(x["accuracy"] == 1.0 and x["parse_source_counts"].get("fallback_xml") == 17,
+                  f"xml fallback {x}")
+            check(x["format_well_formed"] == 1.0, "xml well formed")
+            out = subprocess.run([PY, "-I", str(HERE / "compare.py"), f"{td}/gold", f"{td}/corrupt"],
+                                 check=True, capture_output=True, text=True).stdout
+            print(out)
+            cmp_ = json.load(open(f"{td}/corrupt/compare_vs_ref.json"))
+            o = cmp_["overall_micro"]
+            check(o["delta"] < 0 and o["delta_boot95"][1] < 0 and o["ref_agreement"] < 1, f"compare {o}")
+            check(cmp_["tasks"]["ifeval"]["ref_agreement"] == 1.0, "ifeval agreement identical")
+            pb = subprocess.run([PY, "-I", str(HERE / "perf_bench.py"), "--base-url", "http://127.0.0.1:18991/v1",
+                                 "--model", "mock", "--concurrency", "1,4", "--num-requests", "4",
+                                 "--input-tokens", "256", "--output-tokens", "32", "--out", f"{td}/perf.json"],
+                                capture_output=True, text=True)
+            check(pb.returncode == 0, f"perf_bench rc {pb.stderr[-800:]}")
+            if pb.returncode == 0:
+                p = json.load(open(f"{td}/perf.json"))
+                lv = p["levels"][-1]
+                check(lv["n_ok"] == 4 and lv["output_tokens_mean"] == 32, f"perf {lv}")
+        finally:
+            for p in procs:
+                p.terminate()
+
+
+if __name__ == "__main__":
+    unit()
+    if "--unit-only" not in sys.argv:
+        e2e()
+    print("ALL OK" if not FAILS else f"{len(FAILS)} FAILURES")
+    sys.exit(1 if FAILS else 0)

evaluation/harness/tool_cases.py

@@ -0,0 +1,116 @@
+"""20 hand-written function-calling cases (provenance: authored for this project, 2026-10-30,
+released under Apache-2.0 alongside the harness; style modelled on BFCL categories
+simple / multiple / parallel / irrelevance; no text copied from BFCL).
+
+gold_calls: list of {name, arguments: {param: [allowed values...]}}; "__optional__" in the list
+means the parameter may be omitted. Empty gold_calls => the model must NOT call a tool.
+"""
+
+
+def fn(name, desc, props, required):
+    return {"type": "function", "function": {"name": name, "description": desc, "parameters": {
+        "type": "object", "properties": props, "required": required}}}
+
+
+S = lambda d, **kw: {"type": "string", "description": d, **kw}  # noqa: E731
+I = lambda d: {"type": "integer", "description": d}  # noqa: E731
+N = lambda d: {"type": "number", "description": d}  # noqa: E731
+B = lambda d: {"type": "boolean", "description": d}  # noqa: E731
+
+WEATHER = fn("get_current_weather", "Get the current weather for a city.",
+             {"city": S("City name, e.g. 'Paris'"), "unit": S("Temperature unit", enum=["celsius", "fahrenheit"])},
+             ["city"])
+FX = fn("convert_currency", "Convert an amount between currencies using today's rate.",
+        {"amount": N("Amount to convert"), "from_currency": S("ISO 4217 code"), "to_currency": S("ISO 4217 code")},
+        ["amount", "from_currency", "to_currency"])
+EMAIL = fn("send_email", "Send an email.",
+           {"to": S("Recipient email address"), "subject": S("Subject line"), "body": S("Plain-text body")},
+           ["to", "subject", "body"])
+FLIGHTS = fn("search_flights", "Search one-way flights.",
+             {"origin": S("IATA airport code"), "destination": S("IATA airport code"),
+              "date": S("Departure date YYYY-MM-DD"), "passengers": I("Number of passengers")},
+             ["origin", "destination", "date"])
+TIMER = fn("set_timer", "Start a countdown timer.", {"minutes": I("Duration in minutes"), "label": S("Optional label")},
+           ["minutes"])
+STOCK = fn("get_stock_price", "Get the latest stock price for a ticker symbol.", {"ticker": S("Ticker, e.g. AAPL")},
+           ["ticker"])
+TRANSLATE = fn("translate_text", "Translate text to a target language.",
+               {"text": S("Text to translate"), "target_language": S("ISO 639-1 code, e.g. 'de'")},
+               ["text", "target_language"])
+CAL = fn("create_calendar_event", "Create a calendar event.",
+         {"title": S("Event title"), "date": S("YYYY-MM-DD"), "start_time": S("HH:MM 24h"),
+          "duration_minutes": I("Length in minutes")}, ["title", "date", "start_time"])
+SQL = fn("run_sql_query", "Run a read-only SQL query against the analytics warehouse.",
+         {"query": S("SQL SELECT statement")}, ["query"])
+FILES = fn("list_files", "List files in a directory.", {"path": S("Directory path"), "recursive": B("Recurse")},
+           ["path"])
+BMI = fn("calculate_bmi", "Calculate body-mass index.", {"weight_kg": N("Weight in kg"), "height_m": N("Height in metres")},
+         ["weight_kg", "height_m"])
+LIGHT = fn("set_light_state", "Turn a smart light on/off and set brightness.",
+           {"room": S("Room name"), "on": B("Power state"), "brightness": I("0-100")}, ["room", "on"])
+TICKET = fn("create_support_ticket", "Open a customer-support ticket.",
+            {"customer_id": S("Customer id"), "priority": S("Priority", enum=["low", "medium", "high"]),
+             "summary": S("Short summary")}, ["customer_id", "priority", "summary"])
+TZ = fn("get_time_in_timezone", "Get the current local time for an IANA timezone.", {"timezone": S("e.g. Asia/Tokyo")},
+        ["timezone"])
+
+O = "__optional__"
+CASES = [
+    # ---------------- simple (single tool available, single call)
+    dict(id="tool_simple_01", category="simple", tools=[WEATHER], user="What's the weather like in Lisbon right now? Use celsius.",
+         gold_calls=[{"name": "get_current_weather", "arguments": {"city": ["Lisbon", "Lisbon, Portugal"], "unit": ["celsius"]}}]),
+    dict(id="tool_simple_02", category="simple", tools=[FX], user="How much is 250 US dollars in euros?",
+         gold_calls=[{"name": "convert_currency", "arguments": {"amount": [250], "from_currency": ["USD"], "to_currency": ["EUR"]}}]),
+    dict(id="tool_simple_03", category="simple", tools=[TIMER], user="Set a 25 minute timer labelled pomodoro.",
+         gold_calls=[{"name": "set_timer", "arguments": {"minutes": [25], "label": ["pomodoro", "Pomodoro"]}}]),
+    dict(id="tool_simple_04", category="simple", tools=[STOCK], user="What's NVIDIA trading at? (ticker NVDA)",
+         gold_calls=[{"name": "get_stock_price", "arguments": {"ticker": ["NVDA"]}}]),
+    dict(id="tool_simple_05", category="simple", tools=[BMI], user="I weigh 72.5 kg and I'm 1.80 m tall. What's my BMI?",
+         gold_calls=[{"name": "calculate_bmi", "arguments": {"weight_kg": [72.5], "height_m": [1.8]}}]),
+    dict(id="tool_simple_06", category="simple", tools=[FILES], user="List everything under /var/log recursively.",
+         gold_calls=[{"name": "list_files", "arguments": {"path": ["/var/log", "/var/log/"], "recursive": [True]}}]),
+    dict(id="tool_simple_07", category="simple", tools=[TZ], user="What time is it in Tokyo right now? The IANA zone is Asia/Tokyo.",
+         gold_calls=[{"name": "get_time_in_timezone", "arguments": {"timezone": ["Asia/Tokyo"]}}]),
+    dict(id="tool_simple_08", category="simple", tools=[LIGHT], user="Turn on the kitchen light at 40% brightness.",
+         gold_calls=[{"name": "set_light_state", "arguments": {"room": ["kitchen"], "on": [True], "brightness": [40]}}]),
+    # ---------------- multiple (several tools available, pick one)
+    dict(id="tool_multiple_01", category="multiple", tools=[WEATHER, FLIGHTS, FX, TIMER],
+         user="Find me flights from SFO to JFK on 2026-12-03 for 2 passengers.",
+         gold_calls=[{"name": "search_flights", "arguments": {"origin": ["SFO"], "destination": ["JFK"], "date": ["2026-12-03"], "passengers": [2]}}]),
+    dict(id="tool_multiple_02", category="multiple", tools=[EMAIL, CAL, TICKET],
+         user="Email bob@example.com with subject 'Lunch' and body 'Are we still on for noon tomorrow?'",
+         gold_calls=[{"name": "send_email", "arguments": {"to": ["bob@example.com"], "subject": ["Lunch"], "body": ["Are we still on for noon tomorrow?"]}}]),
+    dict(id="tool_multiple_03", category="multiple", tools=[TRANSLATE, STOCK, WEATHER],
+         user="Translate 'Where is the train station?' into German.",
+         gold_calls=[{"name": "translate_text", "arguments": {"text": ["Where is the train station?"], "target_language": ["de"]}}]),
+    dict(id="tool_multiple_04", category="multiple", tools=[CAL, TIMER, EMAIL],
+         user="Put 'Quarterly review' on my calendar for 2026-11-14 at 15:30, lasting 90 minutes.",
+         gold_calls=[{"name": "create_calendar_event", "arguments": {"title": ["Quarterly review"], "date": ["2026-11-14"], "start_time": ["15:30"], "duration_minutes": [90]}}]),
+    dict(id="tool_multiple_05", category="multiple", tools=[SQL, FILES, STOCK],
+         user="Run this query on the warehouse: SELECT COUNT(*) FROM orders",
+         gold_calls=[{"name": "run_sql_query", "arguments": {"query": ["SELECT COUNT(*) FROM orders", "SELECT COUNT(*) FROM orders;"]}}]),
+    dict(id="tool_multiple_06", category="multiple", tools=[TICKET, EMAIL, LIGHT],
+         system="You are a support agent for Acme Corp.",
+         user="Customer C-1042 says their router keeps rebooting. Open a high priority ticket with summary 'Router reboot loop'.",
+         gold_calls=[{"name": "create_support_ticket", "arguments": {"customer_id": ["C-1042"], "priority": ["high"], "summary": ["Router reboot loop"]}}]),
+    # ---------------- parallel (two calls expected)
+    dict(id="tool_parallel_01", category="parallel", tools=[WEATHER],
+         user="What's the current weather in Oslo and in Cairo? Use celsius for both.",
+         gold_calls=[{"name": "get_current_weather", "arguments": {"city": ["Oslo"], "unit": ["celsius"]}},
+                     {"name": "get_current_weather", "arguments": {"city": ["Cairo"], "unit": ["celsius"]}}]),
+    dict(id="tool_parallel_02", category="parallel", tools=[STOCK, FX],
+         user="Get me the stock prices of AAPL and MSFT.",
+         gold_calls=[{"name": "get_stock_price", "arguments": {"ticker": ["AAPL"]}},
+                     {"name": "get_stock_price", "arguments": {"ticker": ["MSFT"]}}]),
+    dict(id="tool_parallel_03", category="parallel", tools=[TIMER, LIGHT],
+         user="Turn off the bedroom light and set a 10 minute timer.",
+         gold_calls=[{"name": "set_light_state", "arguments": {"room": ["bedroom"], "on": [False], "brightness": [O, 0]}},
+                     {"name": "set_timer", "arguments": {"minutes": [10], "label": [O]}}]),
+    # ---------------- irrelevance (tools present but none applicable)
+    dict(id="tool_irrelevance_01", category="irrelevance", tools=[WEATHER, FX],
+         user="Who wrote the novel 'Pride and Prejudice'?", gold_calls=[]),
+    dict(id="tool_irrelevance_02", category="irrelevance", tools=[STOCK, TIMER],
+         user="What is 17 multiplied by 23?", gold_calls=[]),
+    dict(id="tool_irrelevance_03", category="irrelevance", tools=[FLIGHTS, CAL],
+         user="Write a two-line poem about autumn leaves.", gold_calls=[]),
+]

evaluation/hf_bf16_checks.json

@@ -0,0 +1,85 @@
+{
+  "model_dir": "upstream/SenseNova-Loop-9B-Preview",
+  "torch": "2.11.0+cu130",
+  "load_s": 12.2,
+  "gpu_mem_after_load_GiB": 17.54,
+  "loop_plan_len": 56,
+  "mhc_param_count": 96,
+  "mhc_all_fp32": true,
+  "layer_param_dtypes": [
+    "torch.bfloat16"
+  ],
+  "text_mult": {
+    "prompt_tokens": 27,
+    "gen_tokens": 3,
+    "seconds": 1.61,
+    "text": "91<|im_end|>"
+  },
+  "loop_calls_per_layer": {
+    "0": 3,
+    "1": 3,
+    "2": 3,
+    "3": 3,
+    "4": 6,
+    "5": 6,
+    "6": 6,
+    "7": 6,
+    "8": 6,
+    "9": 6,
+    "10": 6,
+    "11": 6,
+    "12": 6,
+    "13": 6,
+    "14": 6,
+    "15": 6,
+    "16": 6,
+    "17": 6,
+    "18": 6,
+    "19": 6,
+    "20": 6,
+    "21": 6,
+    "22": 6,
+    "23": 6,
+    "24": 6,
+    "25": 6,
+    "26": 6,
+    "27": 6,
+    "28": 3,
+    "29": 3,
+    "30": 3,
+    "31": 3
+  },
+  "looped_layers_called_twice": true,
+  "text_haiku": {
+    "prompt_tokens": 21,
+    "gen_tokens": 20,
+    "seconds": 5.08,
+    "text": "Green sphere, white seam,\nBounces high in summer sun,\nA game begins.<|im_end|>"
+  },
+  "thinking": {
+    "prompt_tokens": 32,
+    "gen_tokens": 262,
+    "seconds": 66.44,
+    "text": "The train goes 60 km in 45 minutes.\n\nTo find the speed in km/h, I need to convert 45 minutes to hours.\n\n45 minutes = 45/60 hours = 3/4 hours\n\nSpeed = Distance / Time = 60 km / (3/4 hours) = 60 × 4/3 = 80 km/h\n</think>\n\n# Finding the Train's Speed\n\n## Given Information\n- Distance = 60 km\n- Time = 45 minutes\n\n## Solution\n\n**Step 1:** Convert time to hours.\n\n$$45 \\text{ minutes} = \\frac{45}{60} \\text{ hours} = \\frac{3}{4} \\text{ hours}$$\n\n**Step 2:** Use the speed formula.\n\n$$\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}} = \\frac{60 \\text{ km}}{\\frac{3}{4} \\text{ hr}}$$\n\n**Step 3:** Calculate.\n\n$$\\text{Speed} = 60 \\times \\frac{4}{3} = \\boxed{80 \\text{ km/h}}$$<|im_end|>"
+  },
+  "tool_call": {
+    "prompt_tokens": 300,
+    "gen_tokens": 39,
+    "seconds": 10.13,
+    "text": "<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n<parameter=unit>\ncelsius\n</parameter>\n</function>\n</tool_call><|im_end|>",
+    "has_tool_call_xml": true
+  },
+  "image": {
+    "prompt_tokens": 1013,
+    "gen_tokens": 61,
+    "seconds": 17.08,
+    "text": "A yellow mobile tyre fitting service van, branded with \"ATS\" and \"Euromaster\" logos, is driving on a road with a driver visible inside wearing a blue shirt. Behind it, a white van with a red warning light on top and a dark car are also present on the road.<|im_end|>"
+  },
+  "cache_consistency": {
+    "n": 64,
+    "match": 64,
+    "first_mismatch": null,
+    "finite_logits": true
+  },
+  "peak_gpu_mem_GiB": 17.98
+}

evaluation/kernel_summary.txt

@@ -0,0 +1,28 @@
+distinct kernels: 97 total GPU us: 553403.1010000036
+{'other': 273.4, 'GEMM(other)': 81.8, 'FP4-related': 198.2} (ms)
+--- top 25 kernels by time ---
+   130.2 ms    672x  [FP4-related] cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu
+    71.5 ms    216x  [GEMM(other)] std::enable_if<!(false), void>::type internal::gemvx::kernel<int, int, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, float, false, true, true, false, 7
+    67.4 ms    672x  [FP4-related] cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu_stream_k
+    41.3 ms  46080x  [other] void at::native::elementwise_kernel<128, 2, at::native::gpu_kernel_impl_nocast<at::native::CUDAFunctor_add<float> >(at::TensorIteratorBase&, at::nativ
+    23.7 ms  23616x  [other] void at::native::vectorized_elementwise_kernel<4, at::native::exp_kernel_cuda(at::TensorIteratorBase&)::{lambda()#2}::operator()() const::{lambda()#2}
+    23.4 ms  23040x  [other] void at::native::vectorized_elementwise_kernel<4, at::native::log_kernel_cuda(at::TensorIteratorBase&)::{lambda()#2}::operator()() const::{lambda()#2}
+    22.8 ms  23040x  [other] void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous namespace)::masked_fill_kernel(at::TensorIterator&, c10::Scalar const&)::{lam
+    21.8 ms  23040x  [other] void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctor_add<float>, std::array<char*, 3ul>, false>(int, at::native::CUDAFunctor_add<
+    21.4 ms  23040x  [other] void at::native::vectorized_elementwise_kernel<4, at::native::AbsFunctor<float>, std::array<char*, 2ul>, false>(int, at::native::AbsFunctor<float>, st
+    21.2 ms  11520x  [other] void at::native::reduce_kernel<256, 2, at::native::ReduceOp<float, at::native::func_wrapper_t<float, at::native::MaxNanFunctor<float> >, unsigned int,
+    21.0 ms  23040x  [other] void at::native::vectorized_elementwise_kernel<4, at::native::AUnaryFunctor<float, float, bool, at::native::(anonymous namespace)::CompareEqFunctor<fl
+    20.6 ms  11520x  [other] void at::native::reduce_kernel<256, 2, at::native::ReduceOp<float, at::native::func_wrapper_t<float, at::native::sum_functor<float, float, float>::ope
+    18.1 ms  11520x  [other] void at::native::reduce_kernel<512, 1, at::native::ReduceOp<float, at::native::func_wrapper_t<float, at::native::MaxNanFunctor<float> >, unsigned int,
+    17.2 ms  11520x  [other] void at::native::reduce_kernel<512, 1, at::native::ReduceOp<float, at::native::func_wrapper_t<float, at::native::sum_functor<float, float, float>::ope
+     3.4 ms     96x  [GEMM(other)] void sgemm_largek_lds64<true, false, 5, 5, 4, 4, 4, 34>(float*, float const*, float const*, int, int, int, int, int, int, float const*, float const*, 
+     2.6 ms    210x  [other] fused_recurrent_gated_delta_rule_packed_decode_kernel
+     2.3 ms   1092x  [other] void tensorrt_llm::kernels::quantize_with_block_size<(tensorrt_llm::BlockScaleQuantizationType)0, __nv_bfloat16, 16, false, false, false, false, std::
+     1.7 ms   1728x  [other] void at::native::vectorized_elementwise_kernel<4, at::native::bfloat16_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda(float)#1}, std::array<char*,
+     1.4 ms    480x  [GEMM(other)] void dot_kernel<float, 128, 0, cublasDotParams<cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float> > >(cublasDotParams<
+     1.3 ms    576x  [other] void at::native::reduce_kernel<512, 1, at::native::ReduceOp<float, at::native::MeanOps<float, float, float, float>, unsigned int, float, 4, 4> >(at::n
+     1.3 ms    577x  [other] void at::native::unrolled_elementwise_kernel<at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()
+     1.3 ms     96x  [GEMM(other)] void cutlass::Kernel2<cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_32x1_nn_align2>(cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_32x1_n
+     1.2 ms    576x  [other] void at::native::elementwise_kernel<128, 4, at::native::gpu_kernel_impl_nocast<at::native::BinaryFunctor<c10::BFloat16, c10::BFloat16, c10::BFloat16, 
+     1.1 ms    588x  [GEMM(other)] kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64409640961_tensorptrbf16gmemalign16o40961_tenso
+     1.0 ms    480x  [GEMM(other)] void cutlass::Kernel2<cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_16x16_32x1_nn_align2>(cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_16x16_32x1_n

evaluation/nvfp4_v1_mtp_release_probe.json

@@ -0,0 +1,82 @@
+{
+ "mult": {
+  "content": "91",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 27,
+   "total_tokens": 30,
+   "completion_tokens": 3,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "haiku": {
+  "content": "Green sphere, white seam,\nBounces true on court's warm ground,\nA fleeting, bright joy.",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 21,
+   "total_tokens": 45,
+   "completion_tokens": 24,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "thinking": {
+  "content": "\n\n# Finding the Train's Speed\n\n**Given:**\n- Distance = 60 km\n- Time = 45 minutes\n\n**Step 1: Convert time to hours**\n\n45 minutes = 45/60 = **3/4 hour**\n\n**Step 2: Calculate speed**\n\n$$\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}} = \\frac{60 \\text{ km}}{3/4 \\text{ hour}} = 60 \\times \\frac{4}{3} = \\boxed{80 \\text{ km/h}}$$",
+  "reasoning": "The train goes 60 km in 45 minutes.\n\nTo find the speed in km/h:\n\nSpeed = Distance / Time\n\nTime = 45 minutes = 45/60 hours = 3/4 hours\n\nSpeed = 60 km / (3/4 hours) = 60 × 4/3 = 80 km/h\n",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 32,
+   "total_tokens": 244,
+   "completion_tokens": 212,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 80,
+   "completion_tokens_details": {
+    "reasoning_tokens": 80
+   }
+  }
+ },
+ "tool": {
+  "content": "",
+  "reasoning": "",
+  "tool_calls": [
+   {
+    "id": "call_3bdd055af15c4337a2b9fe21",
+    "index": 0,
+    "type": "function",
+    "function": {
+     "name": "get_weather",
+     "arguments": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
+    }
+   }
+  ],
+  "usage": {
+   "prompt_tokens": 306,
+   "total_tokens": 345,
+   "completion_tokens": 39,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "image": {
+  "content": "A yellow mobile tyre fitting service van, branded with \"ATS\" and \"Euromaster\" logos, is driving on a road with its driver visible inside. Behind it, a white van and a dark car are also traveling on the same road.",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 1013,
+   "total_tokens": 1064,
+   "completion_tokens": 51,
+   "prompt_tokens_details": {
+    "cached_tokens": 0,
+    "image_tokens": 992
+   },
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ }
+}

evaluation/nvfp4_v1_probe.json

@@ -0,0 +1,82 @@
+{
+ "mult": {
+  "content": "91",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 27,
+   "total_tokens": 30,
+   "completion_tokens": 3,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "haiku": {
+  "content": "Green and white, it bounces true,\nA fleeting arc through summer air,\nA brief, bright, spinning joy.",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 21,
+   "total_tokens": 48,
+   "completion_tokens": 27,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "thinking": {
+  "content": "\n\n# Finding the Train's Speed\n\n**Given:**\n- Distance = 60 km\n- Time = 45 minutes\n\n**Step 1:** Convert time to hours.\n\n$$45 \\text{ minutes} = \\frac{45}{60} \\text{ hours} = \\frac{3}{4} \\text{ hours}$$\n\n**Step 2:** Calculate speed.\n\n$$\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}} = \\frac{60}{\\frac{3}{4}} = 60 \\times \\frac{4}{3} = \\boxed{80 \\text{ km/h}}$$",
+  "reasoning": "The train goes 60 km in 45 minutes.\n\nTo find the speed in km/h:\n\nSpeed = Distance / Time\n\nTime = 45 minutes = 45/60 hours = 3/4 hours\n\nSpeed = 60 km / (3/4 hours) = 60 × 4/3 = 80 km/h\n",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 32,
+   "total_tokens": 258,
+   "completion_tokens": 226,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 80,
+   "completion_tokens_details": {
+    "reasoning_tokens": 80
+   }
+  }
+ },
+ "tool": {
+  "content": "",
+  "reasoning": "",
+  "tool_calls": [
+   {
+    "id": "call_f059ff08a40946de871cc501",
+    "index": 0,
+    "type": "function",
+    "function": {
+     "name": "get_weather",
+     "arguments": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
+    }
+   }
+  ],
+  "usage": {
+   "prompt_tokens": 306,
+   "total_tokens": 345,
+   "completion_tokens": 39,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "image": {
+  "content": "A yellow mobile tyre fitting service van, branded with \"ATS\" and \"Euromaster\" logos, is driving on a road with its amber warning lights on. Behind it, a white van with the license plate \"WG53 JNU\" and a dark car are also visible on the road.",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 1013,
+   "total_tokens": 1075,
+   "completion_tokens": 62,
+   "prompt_tokens_details": {
+    "cached_tokens": 0,
+    "image_tokens": 992
+   },
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ }
+}

evaluation/perf_sglang_bf16.json

@@ -0,0 +1,170 @@
+{
+  "label": "",
+  "model": "sensenova-loop-bf16",
+  "base_url": "http://127.0.0.1:30080/v1",
+  "host": "tux",
+  "time": "2026-10-30T15:47:00-0700",
+  "config": {
+    "base_url": "http://127.0.0.1:30080/v1",
+    "model": "sensenova-loop-bf16",
+    "concurrency": "1,2,4,8",
+    "num_requests": 16,
+    "warmup": 2,
+    "input_tokens": 1024,
+    "output_tokens": 256,
+    "tokenizer": "upstream/SenseNova-Loop-9B-Preview/tokenizer.json",
+    "allow_prefix_cache": false,
+    "no_ignore_eos": false,
+    "timeout": 600,
+    "label": "",
+    "out": "eval/runs/perf_sglang_bf16.json",
+    "prompt_text_tokens": 1024,
+    "prompt_build": "tokenizer.json"
+  },
+  "levels": [
+    {
+      "concurrency": 1,
+      "n_requests": 16,
+      "n_ok": 16,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 712.4458575863391,
+      "ttft_s": {
+        "p50": 0.7494839327409863,
+        "p90": 0.752388886641711,
+        "mean": 0.7490631011896767
+      },
+      "e2e_s": {
+        "p50": 44.54232191387564,
+        "p90": 44.74176003783941,
+        "mean": 44.52777633216465
+      },
+      "decode_tok_s_per_req": {
+        "p50": 5.82298062979468,
+        "p90": 5.857639877869307,
+        "mean": 5.8248643173510715
+      },
+      "itl_s": {
+        "p50": 0.17173366795531383,
+        "p90": 0.17253968634587877,
+        "mean": 0.17168066543298682
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 5.7492088084793425,
+      "requests_per_s": 0.02245784690812243,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 2,
+      "n_requests": 16,
+      "n_ok": 16,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 358.13540337421,
+      "ttft_s": {
+        "p50": 1.4381381664425135,
+        "p90": 1.493556933477521,
+        "mean": 1.451384060550481
+      },
+      "e2e_s": {
+        "p50": 44.76612635422498,
+        "p90": 45.01789597328752,
+        "mean": 44.76674999552779
+      },
+      "decode_tok_s_per_req": {
+        "p50": 5.888547972827849,
+        "p90": 5.919728871447871,
+        "mean": 5.8871749230165396
+      },
+      "itl_s": {
+        "p50": 0.16982132055997556,
+        "p90": 0.1708354555380841,
+        "mean": 0.16986371961846838
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 11.43701505466678,
+      "requests_per_s": 0.04467584005729211,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 4,
+      "n_requests": 16,
+      "n_ok": 16,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 187.37603424675763,
+      "ttft_s": {
+        "p50": 2.734584681224078,
+        "p90": 2.7569230226799846,
+        "mean": 2.7020455568563193
+      },
+      "e2e_s": {
+        "p50": 46.840107245370746,
+        "p90": 46.998931530397385,
+        "mean": 46.84355494793272
+      },
+      "decode_tok_s_per_req": {
+        "p50": 5.772806069829697,
+        "p90": 5.803642233569628,
+        "mean": 5.776986593431426
+      },
+      "itl_s": {
+        "p50": 0.17322622279947003,
+        "p90": 0.1738359775942038,
+        "mean": 0.1731030160166305
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 21.859785945762578,
+      "requests_per_s": 0.08538978885063507,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 8,
+      "n_requests": 16,
+      "n_ok": 16,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 102.59108853153884,
+      "ttft_s": {
+        "p50": 4.974256161600351,
+        "p90": 5.0004629092291,
+        "mean": 4.947074577212334
+      },
+      "e2e_s": {
+        "p50": 51.294219580013305,
+        "p90": 51.56152600981295,
+        "mean": 51.29352270014351
+      },
+      "decode_tok_s_per_req": {
+        "p50": 5.488342640760974,
+        "p90": 5.537196663306641,
+        "mean": 5.502280519349593
+      },
+      "itl_s": {
+        "p50": 0.18220517470811803,
+        "p90": 0.18259358133244163,
+        "mean": 0.18174918959711622
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 39.925495075927536,
+      "requests_per_s": 0.15595896514034194,
+      "usage_reported": true
+    }
+  ]
+}

evaluation/perf_sglang_bf16_mtp.json

@@ -0,0 +1,170 @@
+{
+  "label": "",
+  "model": "sensenova-loop-bf16",
+  "base_url": "http://127.0.0.1:30080/v1",
+  "host": "tux",
+  "time": "2026-10-30T16:33:57-0700",
+  "config": {
+    "base_url": "http://127.0.0.1:30080/v1",
+    "model": "sensenova-loop-bf16",
+    "concurrency": "1,2,4,8",
+    "num_requests": 8,
+    "warmup": 2,
+    "input_tokens": 1024,
+    "output_tokens": 256,
+    "tokenizer": "upstream/SenseNova-Loop-9B-Preview/tokenizer.json",
+    "allow_prefix_cache": false,
+    "no_ignore_eos": false,
+    "timeout": 600,
+    "label": "",
+    "out": "eval/runs/perf_sglang_bf16_mtp.json",
+    "prompt_text_tokens": 1024,
+    "prompt_build": "tokenizer.json"
+  },
+  "levels": [
+    {
+      "concurrency": 1,
+      "n_requests": 8,
+      "n_ok": 8,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 272.73132864851505,
+      "ttft_s": {
+        "p50": 0.7848195834085345,
+        "p90": 0.9759742130525411,
+        "mean": 0.8312031434616074
+      },
+      "e2e_s": {
+        "p50": 34.21052242349833,
+        "p90": 34.438923329673706,
+        "mean": 34.09125282650348
+      },
+      "decode_tok_s_per_req": {
+        "p50": 7.627452517131835,
+        "p90": 7.763279273008121,
+        "mean": 7.669396340870895
+      },
+      "itl_s": {
+        "p50": 0.1311053762084567,
+        "p90": 0.13172920723317885,
+        "mean": 0.13043050938103276
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 7.50922165835733,
+      "requests_per_s": 0.02933289710295832,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 2,
+      "n_requests": 8,
+      "n_ok": 8,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 138.61350492294878,
+      "ttft_s": {
+        "p50": 1.1976498886942863,
+        "p90": 1.6063555277884007,
+        "mean": 1.2973694901447743
+      },
+      "e2e_s": {
+        "p50": 34.72376540116966,
+        "p90": 36.45519359242171,
+        "mean": 34.5663175820373
+      },
+      "decode_tok_s_per_req": {
+        "p50": 7.653260406224514,
+        "p90": 8.09628260642012,
+        "mean": 7.677757556692128
+      },
+      "itl_s": {
+        "p50": 0.13067035353651235,
+        "p90": 0.13736926173243452,
+        "mean": 0.13046496708274763
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 14.774895138380808,
+      "requests_per_s": 0.05771443413430003,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 4,
+      "n_requests": 8,
+      "n_ok": 8,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 76.38755708001554,
+      "ttft_s": {
+        "p50": 2.5858272635377944,
+        "p90": 2.779972269944847,
+        "mean": 2.2637667186791077
+      },
+      "e2e_s": {
+        "p50": 36.801751386839896,
+        "p90": 39.171385447308424,
+        "mean": 37.270905990269966
+      },
+      "decode_tok_s_per_req": {
+        "p50": 7.2633816269398785,
+        "p90": 7.592177491148207,
+        "mean": 7.294118130503968
+      },
+      "itl_s": {
+        "p50": 0.1376772181057901,
+        "p90": 0.1427121850625411,
+        "mean": 0.13728188767883123
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 26.810649250829314,
+      "requests_per_s": 0.10472909863605201,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 8,
+      "n_requests": 8,
+      "n_ok": 8,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 44.81087386980653,
+      "ttft_s": {
+        "p50": 5.374245874583721,
+        "p90": 5.41043642014265,
+        "mean": 5.388463915442117
+      },
+      "e2e_s": {
+        "p50": 41.73927395185456,
+        "p90": 43.86301381308586,
+        "mean": 41.677275480818935
+      },
+      "decode_tok_s_per_req": {
+        "p50": 7.0119672437329355,
+        "p90": 7.471562651588895,
+        "mean": 7.0494192349641605
+      },
+      "itl_s": {
+        "p50": 0.1426148278529153,
+        "p90": 0.1509358527029262,
+        "mean": 0.14230802183349928
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 45.7031926212878,
+      "requests_per_s": 0.17852809617690546,
+      "usage_reported": true
+    }
+  ]
+}

evaluation/perf_sglang_nvfp4_v1.json

@@ -0,0 +1,170 @@
+{
+  "label": "",
+  "model": "sensenova-loop-nvfp4",
+  "base_url": "http://127.0.0.1:30080/v1",
+  "host": "tux",
+  "time": "2026-10-30T17:00:57-0700",
+  "config": {
+    "base_url": "http://127.0.0.1:30080/v1",
+    "model": "sensenova-loop-nvfp4",
+    "concurrency": "1,2,4,8",
+    "num_requests": 16,
+    "warmup": 2,
+    "input_tokens": 1024,
+    "output_tokens": 256,
+    "tokenizer": "upstream/SenseNova-Loop-9B-Preview/tokenizer.json",
+    "allow_prefix_cache": false,
+    "no_ignore_eos": false,
+    "timeout": 600,
+    "label": "",
+    "out": "eval/runs/perf_sglang_nvfp4_v1.json",
+    "prompt_text_tokens": 1024,
+    "prompt_build": "tokenizer.json"
+  },
+  "levels": [
+    {
+      "concurrency": 1,
+      "n_requests": 16,
+      "n_ok": 16,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 436.37869714573026,
+      "ttft_s": {
+        "p50": 0.5826272442936897,
+        "p90": 0.5880275997333229,
+        "mean": 0.5825185888679698
+      },
+      "e2e_s": {
+        "p50": 27.272629408165812,
+        "p90": 27.33341192267835,
+        "mean": 27.273564630304463
+      },
+      "decode_tok_s_per_req": {
+        "p50": 9.555136327634862,
+        "p90": 9.571729349173973,
+        "mean": 9.55383492852507
+      },
+      "itl_s": {
+        "p50": 0.10465575453697466,
+        "p90": 0.10489086669814937,
+        "mean": 0.10467025827383622
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 9.386342703690978,
+      "requests_per_s": 0.036665401186292884,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 2,
+      "n_requests": 16,
+      "n_ok": 16,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 232.5841392017901,
+      "ttft_s": {
+        "p50": 1.1243764483369887,
+        "p90": 1.1582226920872927,
+        "mean": 1.117659447598271
+      },
+      "e2e_s": {
+        "p50": 29.07659900886938,
+        "p90": 29.158572073560208,
+        "mean": 29.072850470547564
+      },
+      "decode_tok_s_per_req": {
+        "p50": 9.119639853361988,
+        "p90": 9.150102465833314,
+        "mean": 9.121841848607799
+      },
+      "itl_s": {
+        "p50": 0.1096534530486108,
+        "p90": 0.10988617092918823,
+        "mean": 0.10962762238854981
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 17.610831134303226,
+      "requests_per_s": 0.06879230911837197,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 4,
+      "n_requests": 16,
+      "n_ok": 16,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 122.35653864778578,
+      "ttft_s": {
+        "p50": 2.017746684141457,
+        "p90": 2.0371825736947358,
+        "mean": 2.0016646458534524
+      },
+      "e2e_s": {
+        "p50": 30.58990833349526,
+        "p90": 30.627535844687372,
+        "mean": 30.58872388175223
+      },
+      "decode_tok_s_per_req": {
+        "p50": 8.92044357838708,
+        "p90": 8.939535982689941,
+        "mean": 8.920202608225608
+      },
+      "itl_s": {
+        "p50": 0.11210204872540107,
+        "p90": 0.11229545274091994,
+        "mean": 0.1121052856137063
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 33.47593880365235,
+      "requests_per_s": 0.13076538595176698,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 8,
+      "n_requests": 16,
+      "n_ok": 16,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 63.220375599339604,
+      "ttft_s": {
+        "p50": 4.205650780815631,
+        "p90": 4.5575914164073765,
+        "mean": 4.21742238936713
+      },
+      "e2e_s": {
+        "p50": 31.60961770033464,
+        "p90": 32.07297795498744,
+        "mean": 31.608251873170957
+      },
+      "decode_tok_s_per_req": {
+        "p50": 9.306845748863292,
+        "p90": 9.356793657252013,
+        "mean": 9.310041888446369
+      },
+      "itl_s": {
+        "p50": 0.10744962619946283,
+        "p90": 0.10790295124090478,
+        "mean": 0.10741330675379027
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 64.78923861443788,
+      "requests_per_s": 0.25308296333764796,
+      "usage_reported": true
+    }
+  ]
+}

evaluation/perf_sglang_nvfp4_v1_marlin_w4a16.json

@@ -0,0 +1,98 @@
+{
+  "label": "",
+  "model": "sensenova-loop-nvfp4",
+  "base_url": "http://127.0.0.1:30080/v1",
+  "host": "tux",
+  "time": "2026-10-30T17:34:28-0700",
+  "config": {
+    "base_url": "http://127.0.0.1:30080/v1",
+    "model": "sensenova-loop-nvfp4",
+    "concurrency": "1,8",
+    "num_requests": 8,
+    "warmup": 2,
+    "input_tokens": 1024,
+    "output_tokens": 256,
+    "tokenizer": "upstream/SenseNova-Loop-9B-Preview/tokenizer.json",
+    "allow_prefix_cache": false,
+    "no_ignore_eos": false,
+    "timeout": 600,
+    "label": "",
+    "out": "eval/runs/perf_sglang_nvfp4_v1_marlin_w4a16.json",
+    "prompt_text_tokens": 1024,
+    "prompt_build": "tokenizer.json"
+  },
+  "levels": [
+    {
+      "concurrency": 1,
+      "n_requests": 8,
+      "n_ok": 8,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 186.50683353003114,
+      "ttft_s": {
+        "p50": 0.7810423010960221,
+        "p90": 0.7850782623514533,
+        "mean": 0.7793308835243806
+      },
+      "e2e_s": {
+        "p50": 23.208507381845266,
+        "p90": 23.798335312213748,
+        "mean": 23.31315671908669
+      },
+      "decode_tok_s_per_req": {
+        "p50": 11.369868579529511,
+        "p90": 11.574229702673124,
+        "mean": 11.320382232025935
+      },
+      "itl_s": {
+        "p50": 0.08795635091761747,
+        "p90": 0.09028583205250257,
+        "mean": 0.0883674864250911
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 10.980830896312618,
+      "requests_per_s": 0.042893870688721164,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 8,
+      "n_requests": 8,
+      "n_ok": 8,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 35.205060383304954,
+      "ttft_s": {
+        "p50": 6.181970591191202,
+        "p90": 6.18461575973779,
+        "mean": 6.14046993048396
+      },
+      "e2e_s": {
+        "p50": 35.20080000348389,
+        "p90": 35.20238197837025,
+        "mean": 35.2009118568385
+      },
+      "decode_tok_s_per_req": {
+        "p50": 8.787708328163664,
+        "p90": 8.78777517236657,
+        "mean": 8.774984883438263
+      },
+      "itl_s": {
+        "p50": 0.11379531075191848,
+        "p90": 0.11419632067502129,
+        "mean": 0.11396200469439374
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 58.17345511417468,
+      "requests_per_s": 0.22724005903974484,
+      "usage_reported": true
+    }
+  ]
+}

evaluation/perf_sglang_nvfp4_v1_mtp.json

@@ -0,0 +1,170 @@
+{
+  "label": "",
+  "model": "sensenova-loop-nvfp4",
+  "base_url": "http://127.0.0.1:30080/v1",
+  "host": "tux",
+  "time": "2026-10-30T17:21:08-0700",
+  "config": {
+    "base_url": "http://127.0.0.1:30080/v1",
+    "model": "sensenova-loop-nvfp4",
+    "concurrency": "1,2,4,8",
+    "num_requests": 8,
+    "warmup": 2,
+    "input_tokens": 1024,
+    "output_tokens": 256,
+    "tokenizer": "upstream/SenseNova-Loop-9B-Preview/tokenizer.json",
+    "allow_prefix_cache": false,
+    "no_ignore_eos": false,
+    "timeout": 600,
+    "label": "",
+    "out": "eval/runs/perf_sglang_nvfp4_v1_mtp.json",
+    "prompt_text_tokens": 1024,
+    "prompt_build": "tokenizer.json"
+  },
+  "levels": [
+    {
+      "concurrency": 1,
+      "n_requests": 8,
+      "n_ok": 8,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 167.7497703190893,
+      "ttft_s": {
+        "p50": 0.7368478225544095,
+        "p90": 0.7410881330259145,
+        "mean": 0.7100226980401203
+      },
+      "e2e_s": {
+        "p50": 20.956795112695545,
+        "p90": 22.442169491481035,
+        "mean": 20.96854761627037
+      },
+      "decode_tok_s_per_req": {
+        "p50": 12.574976715274474,
+        "p90": 13.522581057360785,
+        "mean": 12.629198610683233
+      },
+      "itl_s": {
+        "p50": 0.07952795094609552,
+        "p90": 0.08523689763364839,
+        "mean": 0.07944462552721447
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 12.208660531125302,
+      "requests_per_s": 0.04769008019970821,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 2,
+      "n_requests": 8,
+      "n_ok": 8,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 82.62508355267346,
+      "ttft_s": {
+        "p50": 0.8884775410406291,
+        "p90": 1.168848348222673,
+        "mean": 0.9600589442998171
+      },
+      "e2e_s": {
+        "p50": 20.495888442732394,
+        "p90": 23.237484936509283,
+        "mean": 20.539230286842212
+      },
+      "decode_tok_s_per_req": {
+        "p50": 13.113690417487145,
+        "p90": 15.40404216440473,
+        "mean": 13.185971019742919
+      },
+      "itl_s": {
+        "p50": 0.07629031224687602,
+        "p90": 0.08763312555082581,
+        "mean": 0.0767804972402861
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 24.786661773169655,
+      "requests_per_s": 0.09682289755144396,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 4,
+      "n_requests": 8,
+      "n_ok": 8,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 48.18132451828569,
+      "ttft_s": {
+        "p50": 1.4830374312587082,
+        "p90": 2.075450401753187,
+        "mean": 1.469463941641152
+      },
+      "e2e_s": {
+        "p50": 22.7819732921198,
+        "p90": 24.27312577702105,
+        "mean": 22.48564152873587
+      },
+      "decode_tok_s_per_req": {
+        "p50": 12.02420852876317,
+        "p90": 13.446927005044758,
+        "mean": 12.240232030320389
+      },
+      "itl_s": {
+        "p50": 0.08323863003512516,
+        "p90": 0.09169133495101157,
+        "mean": 0.08241570999380202
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 42.50609588830931,
+      "requests_per_s": 0.16603943706370825,
+      "usage_reported": true
+    },
+    {
+      "concurrency": 8,
+      "n_requests": 8,
+      "n_ok": 8,
+      "n_errors": 0,
+      "errors": [],
+      "wall_s": 31.786477480083704,
+      "ttft_s": {
+        "p50": 5.049089141655713,
+        "p90": 5.0829355555586515,
+        "mean": 5.062669638660736
+      },
+      "e2e_s": {
+        "p50": 25.993340300396085,
+        "p90": 29.076695342641322,
+        "mean": 26.93958732835017
+      },
+      "decode_tok_s_per_req": {
+        "p50": 12.175041260252009,
+        "p90": 12.51746500245328,
+        "mean": 11.74804181220008
+      },
+      "itl_s": {
+        "p50": 0.08213639315948182,
+        "p90": 0.09391973348115297,
+        "mean": 0.08579115118813135
+      },
+      "output_tokens_mean": 256.0,
+      "output_tokens_min": 256,
+      "output_tokens_max": 256,
+      "prompt_tokens_mean": 1051.0,
+      "n_short_outputs": 0,
+      "aggregate_output_tok_s": 64.42991367266805,
+      "requests_per_s": 0.25167935028385957,
+      "usage_reported": true
+    }
+  ]
+}

evaluation/sglang_bf16_mtp_probe.json

@@ -0,0 +1,82 @@
+{
+ "mult": {
+  "content": "91",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 27,
+   "total_tokens": 30,
+   "completion_tokens": 3,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "haiku": {
+  "content": "Green and white, it flies,\nBounces high in summer air,\nA game of joy.",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 21,
+   "total_tokens": 43,
+   "completion_tokens": 22,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "thinking": {
+  "content": "\n\n# Finding the Train's Speed\n\n## Given Information\n- Distance = 60 km\n- Time = 45 minutes\n\n## Solution\n\n**Step 1:** Convert time to hours.\n\n$$45 \\text{ minutes} = \\frac{45}{60} \\text{ hours} = \\frac{3}{4} \\text{ hours}$$\n\n**Step 2:** Use the speed formula.\n\n$$\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}} = \\frac{60 \\text{ km}}{\\frac{3}{4} \\text{ hr}}$$\n\n**Step 3:** Calculate.\n\n$$\\text{Speed} = 60 \\times \\frac{4}{3} = \\boxed{80 \\text{ km/h}}$$",
+  "reasoning": "The train goes 60 km in 45 minutes.\n\nTo find the speed in km/h, I need to convert 45 minutes to hours.\n\n45 minutes = 45/60 hours = 3/4 hours\n\nSpeed = Distance / Time = 60 km / (3/4 hours) = 60 × 4/3 = 80 km/h\n",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 32,
+   "total_tokens": 294,
+   "completion_tokens": 262,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 86,
+   "completion_tokens_details": {
+    "reasoning_tokens": 86
+   }
+  }
+ },
+ "tool": {
+  "content": "",
+  "reasoning": "",
+  "tool_calls": [
+   {
+    "id": "call_ecd429c744ef4cdc945856fd",
+    "index": 0,
+    "type": "function",
+    "function": {
+     "name": "get_weather",
+     "arguments": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
+    }
+   }
+  ],
+  "usage": {
+   "prompt_tokens": 306,
+   "total_tokens": 345,
+   "completion_tokens": 39,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "image": {
+  "content": "A yellow mobile tyre fitting service van, branded with \"ATS\" and \"Euromaster\" logos, is driving on a road with a white van and a dark car visible in the background. The yellow van's license plate reads \"AJ52 UYV,\" and it features orange warning lights on its roof along with text advertising services such as tyres, batteries, exhausts, shocks, and brakes.",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 1013,
+   "total_tokens": 1096,
+   "completion_tokens": 83,
+   "prompt_tokens_details": {
+    "cached_tokens": 0,
+    "image_tokens": 992
+   },
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ }
+}

evaluation/sglang_bf16_probe.json

@@ -0,0 +1,82 @@
+{
+ "mult": {
+  "content": "91",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 27,
+   "total_tokens": 30,
+   "completion_tokens": 3,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "haiku": {
+  "content": "Green sphere, white seam,\nBounces high in summer sun,\nA game begins.",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 21,
+   "total_tokens": 41,
+   "completion_tokens": 20,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "thinking": {
+  "content": "\n\n# Finding the Train's Speed\n\n## Given Information\n- Distance = 60 km\n- Time = 45 minutes\n\n## Solution\n\n**Step 1:** Convert time to hours.\n\n$$45 \\text{ minutes} = \\frac{45}{60} \\text{ hours} = \\frac{3}{4} \\text{ hours}$$\n\n**Step 2:** Calculate speed using the formula:\n\n$$\\text{Speed} = \\frac{\\text{Distance}}{\\text{Time}}$$\n\n$$\\text{Speed} = \\frac{60 \\text{ km}}{\\frac{3}{4} \\text{ hours}} = 60 \\times \\frac{4}{3} = \\boxed{80 \\text{ km/h}}$$",
+  "reasoning": "The train goes 60 km in 45 minutes.\n\nTo find the speed in km/h, I need to convert 45 minutes to hours.\n\n45 minutes = 45/60 hours = 3/4 hours\n\nSpeed = Distance / Time = 60 km / (3/4 hours) = 60 × 4/3 = 80 km/h\n",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 32,
+   "total_tokens": 287,
+   "completion_tokens": 255,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 86,
+   "completion_tokens_details": {
+    "reasoning_tokens": 86
+   }
+  }
+ },
+ "tool": {
+  "content": "",
+  "reasoning": "",
+  "tool_calls": [
+   {
+    "id": "call_5048fd622c914bf7acade944",
+    "index": 0,
+    "type": "function",
+    "function": {
+     "name": "get_weather",
+     "arguments": "{\"city\": \"Paris\", \"unit\": \"celsius\"}"
+    }
+   }
+  ],
+  "usage": {
+   "prompt_tokens": 306,
+   "total_tokens": 345,
+   "completion_tokens": 39,
+   "prompt_tokens_details": null,
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ },
+ "image": {
+  "content": "A yellow mobile tyre fitting service van, branded with \"ATS\" and \"Euromaster\" logos, is driving on a road with a white van and a dark car visible in the background. The yellow van's license plate reads \"AJ52 UYV,\" and it features orange warning lights on its roof along with text advertising services such as tyres, batteries, exhausts, shocks, and brakes.",
+  "reasoning": "",
+  "tool_calls": null,
+  "usage": {
+   "prompt_tokens": 1013,
+   "total_tokens": 1096,
+   "completion_tokens": 83,
+   "prompt_tokens_details": {
+    "cached_tokens": 0,
+    "image_tokens": 992
+   },
+   "reasoning_tokens": 0,
+   "completion_tokens_details": null
+  }
+ }
+}

evaluation/summary_sglang_bf16.json

@@ -0,0 +1,148 @@
+{
+  "eval_version": "sn-loop-eval-v1",
+  "model": "sensenova-loop-bf16",
+  "base_url": "http://127.0.0.1:30080/v1",
+  "started": "2026-10-30T15:15:24-0700",
+  "enable_thinking": false,
+  "sampling": {
+    "temperature": 0.0,
+    "top_p": 1.0,
+    "seed": 0,
+    "max_tokens": {
+      "ifeval": 1024,
+      "gsm8k": 1024,
+      "mmlu_pro": 1024,
+      "humaneval": 1024,
+      "tools": 512,
+      "vqa": 64
+    }
+  },
+  "concurrency": 16,
+  "limit": null,
+  "host": "tux",
+  "data_sha256": {
+    "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+    "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+    "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+    "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+    "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+    "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
+  },
+  "tasks": {
+    "ifeval": {
+      "n": 60,
+      "correct": 47,
+      "accuracy": 0.7833333333333333,
+      "wilson95": [
+        0.6637997232345094,
+        0.8687695600460952
+      ],
+      "n_errors": 0,
+      "n_truncated": 5,
+      "mean_completion_tokens": 318.95,
+      "sum_completion_tokens": 19137,
+      "instruction_level_acc": 0.8433734939759037,
+      "n_instructions": 83,
+      "wall_s": 392.1
+    },
+    "gsm8k": {
+      "n": 80,
+      "correct": 74,
+      "accuracy": 0.925,
+      "wilson95": [
+        0.8458798383979047,
+        0.9651747509106547
+      ],
+      "n_errors": 0,
+      "n_truncated": 1,
+      "mean_completion_tokens": 191.5375,
+      "sum_completion_tokens": 15323,
+      "wall_s": 370.3
+    },
+    "mmlu_pro": {
+      "n": 60,
+      "correct": 39,
+      "accuracy": 0.65,
+      "wilson95": [
+        0.5236261874086521,
+        0.7583222566810799
+      ],
+      "n_errors": 0,
+      "n_truncated": 13,
+      "mean_completion_tokens": 622.15,
+      "sum_completion_tokens": 37329,
+      "wall_s": 707.9
+    },
+    "humaneval": {
+      "n": 40,
+      "correct": 33,
+      "accuracy": 0.825,
+      "wilson95": [
+        0.6805000953869541,
+        0.9125458630150444
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 226.575,
+      "sum_completion_tokens": 9063,
+      "wall_s": 233.7
+    },
+    "tools": {
+      "n": 20,
+      "correct": 19,
+      "accuracy": 0.95,
+      "wilson95": [
+        0.7638688045357189,
+        0.9911185512976646
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 64.25,
+      "sum_completion_tokens": 1285,
+      "names_correct": 0.95,
+      "parse_source_counts": {
+        "server": 16,
+        "none": 4
+      },
+      "format_well_formed": 1.0,
+      "raw_tool_call_tags_in_content": 0,
+      "by_category": {
+        "irrelevance": "3/3",
+        "multiple": "6/6",
+        "parallel": "3/3",
+        "simple": "7/8"
+      },
+      "wall_s": 55.8
+    },
+    "vqa": {
+      "n": 40,
+      "correct": 36,
+      "accuracy": 0.9,
+      "wilson95": [
+        0.7694822464406407,
+        0.9604204715925885
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 4.125,
+      "sum_completion_tokens": 165,
+      "vqa_soft_acc": 0.8833333333333334,
+      "wall_s": 38.3
+    }
+  },
+  "server_models": {
+    "object": "list",
+    "data": [
+      {
+        "id": "sensenova-loop-bf16",
+        "object": "model",
+        "created": 1793398524,
+        "owned_by": "sglang",
+        "root": "sensenova-loop-bf16",
+        "parent": null,
+        "max_model_len": 32768
+      }
+    ]
+  },
+  "wall_s": 1798.0
+}

evaluation/summary_sglang_bf16_mtp.json

@@ -0,0 +1,103 @@
+{
+  "eval_version": "sn-loop-eval-v1",
+  "model": "sensenova-loop-bf16",
+  "base_url": "http://127.0.0.1:30080/v1",
+  "started": "2026-10-30T16:28:25-0700",
+  "enable_thinking": false,
+  "sampling": {
+    "temperature": 0.0,
+    "top_p": 1.0,
+    "seed": 0,
+    "max_tokens": {
+      "ifeval": 1024,
+      "gsm8k": 1024,
+      "mmlu_pro": 1024,
+      "humaneval": 1024,
+      "tools": 512,
+      "vqa": 64
+    }
+  },
+  "concurrency": 16,
+  "limit": null,
+  "host": "tux",
+  "data_sha256": {
+    "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+    "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+    "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+    "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+    "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+    "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
+  },
+  "tasks": {
+    "gsm8k": {
+      "n": 80,
+      "correct": 74,
+      "accuracy": 0.925,
+      "wilson95": [
+        0.8458798383979047,
+        0.9651747509106547
+      ],
+      "n_errors": 0,
+      "n_truncated": 1,
+      "mean_completion_tokens": 186.425,
+      "sum_completion_tokens": 14914,
+      "wall_s": 138.4
+    },
+    "humaneval": {
+      "n": 40,
+      "correct": 32,
+      "accuracy": 0.8,
+      "wilson95": [
+        0.6524269352437855,
+        0.8950001032811363
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 230.5,
+      "sum_completion_tokens": 9220,
+      "wall_s": 94.4
+    },
+    "tools": {
+      "n": 20,
+      "correct": 19,
+      "accuracy": 0.95,
+      "wilson95": [
+        0.7638688045357189,
+        0.9911185512976646
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 64.9,
+      "sum_completion_tokens": 1298,
+      "names_correct": 0.95,
+      "parse_source_counts": {
+        "server": 16,
+        "none": 4
+      },
+      "format_well_formed": 1.0,
+      "raw_tool_call_tags_in_content": 0,
+      "by_category": {
+        "irrelevance": "3/3",
+        "multiple": "6/6",
+        "parallel": "3/3",
+        "simple": "7/8"
+      },
+      "wall_s": 24.8
+    }
+  },
+  "server_models": {
+    "object": "list",
+    "data": [
+      {
+        "id": "sensenova-loop-bf16",
+        "object": "model",
+        "created": 1793402905,
+        "owned_by": "sglang",
+        "root": "sensenova-loop-bf16",
+        "parent": null,
+        "max_model_len": 32768
+      }
+    ]
+  },
+  "wall_s": 257.7
+}

evaluation/summary_sglang_nvfp4_v1.json

@@ -0,0 +1,148 @@
+{
+  "eval_version": "sn-loop-eval-v1",
+  "model": "sensenova-loop-nvfp4",
+  "base_url": "http://127.0.0.1:30080/v1",
+  "started": "2026-10-30T16:45:48-0700",
+  "enable_thinking": false,
+  "sampling": {
+    "temperature": 0.0,
+    "top_p": 1.0,
+    "seed": 0,
+    "max_tokens": {
+      "ifeval": 1024,
+      "gsm8k": 1024,
+      "mmlu_pro": 1024,
+      "humaneval": 1024,
+      "tools": 512,
+      "vqa": 64
+    }
+  },
+  "concurrency": 16,
+  "limit": null,
+  "host": "tux",
+  "data_sha256": {
+    "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+    "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+    "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+    "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+    "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+    "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
+  },
+  "tasks": {
+    "ifeval": {
+      "n": 60,
+      "correct": 45,
+      "accuracy": 0.75,
+      "wilson95": [
+        0.6276792981955739,
+        0.8422347752873125
+      ],
+      "n_errors": 0,
+      "n_truncated": 4,
+      "mean_completion_tokens": 272.6666666666667,
+      "sum_completion_tokens": 16360,
+      "instruction_level_acc": 0.7831325301204819,
+      "n_instructions": 83,
+      "wall_s": 196.0
+    },
+    "gsm8k": {
+      "n": 80,
+      "correct": 76,
+      "accuracy": 0.95,
+      "wilson95": [
+        0.8783772371965909,
+        0.9803864456007073
+      ],
+      "n_errors": 0,
+      "n_truncated": 1,
+      "mean_completion_tokens": 187.2,
+      "sum_completion_tokens": 14976,
+      "wall_s": 156.1
+    },
+    "mmlu_pro": {
+      "n": 60,
+      "correct": 41,
+      "accuracy": 0.6833333333333333,
+      "wilson95": [
+        0.5576623287169538,
+        0.7869413251704964
+      ],
+      "n_errors": 0,
+      "n_truncated": 9,
+      "mean_completion_tokens": 588.55,
+      "sum_completion_tokens": 35313,
+      "wall_s": 352.6
+    },
+    "humaneval": {
+      "n": 40,
+      "correct": 32,
+      "accuracy": 0.8,
+      "wilson95": [
+        0.6524269352437855,
+        0.8950001032811363
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 197.975,
+      "sum_completion_tokens": 7919,
+      "wall_s": 79.5
+    },
+    "tools": {
+      "n": 20,
+      "correct": 19,
+      "accuracy": 0.95,
+      "wilson95": [
+        0.7638688045357189,
+        0.9911185512976646
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 54.3,
+      "sum_completion_tokens": 1086,
+      "names_correct": 0.95,
+      "parse_source_counts": {
+        "server": 16,
+        "none": 4
+      },
+      "format_well_formed": 1.0,
+      "raw_tool_call_tags_in_content": 0,
+      "by_category": {
+        "irrelevance": "3/3",
+        "multiple": "5/6",
+        "parallel": "3/3",
+        "simple": "8/8"
+      },
+      "wall_s": 24.6
+    },
+    "vqa": {
+      "n": 40,
+      "correct": 36,
+      "accuracy": 0.9,
+      "wilson95": [
+        0.7694822464406407,
+        0.9604204715925885
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 4.075,
+      "sum_completion_tokens": 163,
+      "vqa_soft_acc": 0.8833333333333334,
+      "wall_s": 25.5
+    }
+  },
+  "server_models": {
+    "object": "list",
+    "data": [
+      {
+        "id": "sensenova-loop-nvfp4",
+        "object": "model",
+        "created": 1793403948,
+        "owned_by": "sglang",
+        "root": "sensenova-loop-nvfp4",
+        "parent": null,
+        "max_model_len": 32768
+      }
+    ]
+  },
+  "wall_s": 834.3
+}

evaluation/summary_sglang_nvfp4_v1_marlin_w4a16_gsm8k.json

@@ -0,0 +1,62 @@
+{
+  "eval_version": "sn-loop-eval-v1",
+  "model": "sensenova-loop-nvfp4",
+  "base_url": "http://127.0.0.1:30080/v1",
+  "started": "2026-10-30T17:38:10-0700",
+  "enable_thinking": false,
+  "sampling": {
+    "temperature": 0.0,
+    "top_p": 1.0,
+    "seed": 0,
+    "max_tokens": {
+      "ifeval": 1024,
+      "gsm8k": 1024,
+      "mmlu_pro": 1024,
+      "humaneval": 1024,
+      "tools": 512,
+      "vqa": 64
+    }
+  },
+  "concurrency": 16,
+  "limit": null,
+  "host": "tux",
+  "data_sha256": {
+    "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+    "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+    "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+    "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+    "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+    "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
+  },
+  "tasks": {
+    "gsm8k": {
+      "n": 80,
+      "correct": 77,
+      "accuracy": 0.9625,
+      "wilson95": [
+        0.8954527983617241,
+        0.9871654311799435
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 184.425,
+      "sum_completion_tokens": 14754,
+      "wall_s": 137.2
+    }
+  },
+  "server_models": {
+    "object": "list",
+    "data": [
+      {
+        "id": "sensenova-loop-nvfp4",
+        "object": "model",
+        "created": 1793407090,
+        "owned_by": "sglang",
+        "root": "sensenova-loop-nvfp4",
+        "parent": null,
+        "max_model_len": 32768
+      }
+    ]
+  },
+  "wall_s": 137.2
+}

evaluation/summary_sglang_nvfp4_v1_mtp.json

@@ -0,0 +1,103 @@
+{
+  "eval_version": "sn-loop-eval-v1",
+  "model": "sensenova-loop-nvfp4",
+  "base_url": "http://127.0.0.1:30080/v1",
+  "started": "2026-10-30T17:17:37-0700",
+  "enable_thinking": false,
+  "sampling": {
+    "temperature": 0.0,
+    "top_p": 1.0,
+    "seed": 0,
+    "max_tokens": {
+      "ifeval": 1024,
+      "gsm8k": 1024,
+      "mmlu_pro": 1024,
+      "humaneval": 1024,
+      "tools": 512,
+      "vqa": 64
+    }
+  },
+  "concurrency": 16,
+  "limit": null,
+  "host": "tux",
+  "data_sha256": {
+    "ifeval": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
+    "gsm8k": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
+    "mmlu_pro": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
+    "humaneval": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
+    "tools": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
+    "vqa": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d"
+  },
+  "tasks": {
+    "gsm8k": {
+      "n": 80,
+      "correct": 77,
+      "accuracy": 0.9625,
+      "wilson95": [
+        0.8954527983617241,
+        0.9871654311799435
+      ],
+      "n_errors": 0,
+      "n_truncated": 1,
+      "mean_completion_tokens": 190.3875,
+      "sum_completion_tokens": 15231,
+      "wall_s": 94.6
+    },
+    "humaneval": {
+      "n": 40,
+      "correct": 32,
+      "accuracy": 0.8,
+      "wilson95": [
+        0.6524269352437855,
+        0.8950001032811363
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 191.3,
+      "sum_completion_tokens": 7652,
+      "wall_s": 46.1
+    },
+    "tools": {
+      "n": 20,
+      "correct": 18,
+      "accuracy": 0.9,
+      "wilson95": [
+        0.6989663528135878,
+        0.9721335190383086
+      ],
+      "n_errors": 0,
+      "n_truncated": 0,
+      "mean_completion_tokens": 60.2,
+      "sum_completion_tokens": 1204,
+      "names_correct": 0.9,
+      "parse_source_counts": {
+        "server": 15,
+        "none": 5
+      },
+      "format_well_formed": 1.0,
+      "raw_tool_call_tags_in_content": 0,
+      "by_category": {
+        "irrelevance": "3/3",
+        "multiple": "5/6",
+        "parallel": "3/3",
+        "simple": "7/8"
+      },
+      "wall_s": 16.6
+    }
+  },
+  "server_models": {
+    "object": "list",
+    "data": [
+      {
+        "id": "sensenova-loop-nvfp4",
+        "object": "model",
+        "created": 1793405857,
+        "owned_by": "sglang",
+        "root": "sensenova-loop-nvfp4",
+        "parent": null,
+        "max_model_len": 32768
+      }
+    ]
+  },
+  "wall_s": 157.4
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": [
+    248044,
+    248046
+  ],
+  "transformers_version": "5.18.0",
+  "use_cache": true,
+  "temperature": 1.0,
+  "do_sample": true,
+  "top_p": 0.95
+}

hf_quant_config.json

@@ -0,0 +1,103 @@
+{
+    "producer": {
+        "name": "modelopt",
+        "version": "0.48.0rc1"
+    },
+    "quantization": {
+        "quant_algo": "NVFP4",
+        "kv_cache_quant_algo": null,
+        "group_size": 16,
+        "exclude_modules": [
+            "lm_head",
+            "model.language_model.embed_tokens",
+            "model.language_model.layers.0.linear_attn.conv1d",
+            "model.language_model.layers.0.linear_attn.in_proj_a",
+            "model.language_model.layers.0.linear_attn.in_proj_b",
+            "model.language_model.layers.1.linear_attn.conv1d",
+            "model.language_model.layers.1.linear_attn.in_proj_a",
+            "model.language_model.layers.1.linear_attn.in_proj_b",
+            "model.language_model.layers.10.linear_attn.conv1d",
+            "model.language_model.layers.10.linear_attn.in_proj_a",
+            "model.language_model.layers.10.linear_attn.in_proj_b",
+            "model.language_model.layers.12.linear_attn.conv1d",
+            "model.language_model.layers.12.linear_attn.in_proj_a",
+            "model.language_model.layers.12.linear_attn.in_proj_b",
+            "model.language_model.layers.13.linear_attn.conv1d",
+            "model.language_model.layers.13.linear_attn.in_proj_a",
+            "model.language_model.layers.13.linear_attn.in_proj_b",
+            "model.language_model.layers.14.linear_attn.conv1d",
+            "model.language_model.layers.14.linear_attn.in_proj_a",
+            "model.language_model.layers.14.linear_attn.in_proj_b",
+            "model.language_model.layers.16.linear_attn.conv1d",
+            "model.language_model.layers.16.linear_attn.in_proj_a",
+            "model.language_model.layers.16.linear_attn.in_proj_b",
+            "model.language_model.layers.17.linear_attn.conv1d",
+            "model.language_model.layers.17.linear_attn.in_proj_a",
+            "model.language_model.layers.17.linear_attn.in_proj_b",
+            "model.language_model.layers.18.linear_attn.conv1d",
+            "model.language_model.layers.18.linear_attn.in_proj_a",
+            "model.language_model.layers.18.linear_attn.in_proj_b",
+            "model.language_model.layers.2.linear_attn.conv1d",
+            "model.language_model.layers.2.linear_attn.in_proj_a",
+            "model.language_model.layers.2.linear_attn.in_proj_b",
+            "model.language_model.layers.20.linear_attn.conv1d",
+            "model.language_model.layers.20.linear_attn.in_proj_a",
+            "model.language_model.layers.20.linear_attn.in_proj_b",
+            "model.language_model.layers.21.linear_attn.conv1d",
+            "model.language_model.layers.21.linear_attn.in_proj_a",
+            "model.language_model.layers.21.linear_attn.in_proj_b",
+            "model.language_model.layers.22.linear_attn.conv1d",
+            "model.language_model.layers.22.linear_attn.in_proj_a",
+            "model.language_model.layers.22.linear_attn.in_proj_b",
+            "model.language_model.layers.24.linear_attn.conv1d",
+            "model.language_model.layers.24.linear_attn.in_proj_a",
+            "model.language_model.layers.24.linear_attn.in_proj_b",
+            "model.language_model.layers.25.linear_attn.conv1d",
+            "model.language_model.layers.25.linear_attn.in_proj_a",
+            "model.language_model.layers.25.linear_attn.in_proj_b",
+            "model.language_model.layers.26.linear_attn.conv1d",
+            "model.language_model.layers.26.linear_attn.in_proj_a",
+            "model.language_model.layers.26.linear_attn.in_proj_b",
+            "model.language_model.layers.28.linear_attn.conv1d",
+            "model.language_model.layers.28.linear_attn.in_proj_a",
+            "model.language_model.layers.28.linear_attn.in_proj_b",
+            "model.language_model.layers.29.linear_attn.conv1d",
+            "model.language_model.layers.29.linear_attn.in_proj_a",
+            "model.language_model.layers.29.linear_attn.in_proj_b",
+            "model.language_model.layers.30.linear_attn.conv1d",
+            "model.language_model.layers.30.linear_attn.in_proj_a",
+            "model.language_model.layers.30.linear_attn.in_proj_b",
+            "model.language_model.layers.4.linear_attn.conv1d",
+            "model.language_model.layers.4.linear_attn.in_proj_a",
+            "model.language_model.layers.4.linear_attn.in_proj_b",
+            "model.language_model.layers.5.linear_attn.conv1d",
+            "model.language_model.layers.5.linear_attn.in_proj_a",
+            "model.language_model.layers.5.linear_attn.in_proj_b",
+            "model.language_model.layers.6.linear_attn.conv1d",
+            "model.language_model.layers.6.linear_attn.in_proj_a",
+            "model.language_model.layers.6.linear_attn.in_proj_b",
+            "model.language_model.layers.8.linear_attn.conv1d",
+            "model.language_model.layers.8.linear_attn.in_proj_a",
+            "model.language_model.layers.8.linear_attn.in_proj_b",
+            "model.language_model.layers.9.linear_attn.conv1d",
+            "model.language_model.layers.9.linear_attn.in_proj_a",
+            "model.language_model.layers.9.linear_attn.in_proj_b",
+            "model.visual*",
+            "mtp.fc",
+            "mtp.layers.0.input_layernorm",
+            "mtp.layers.0.mlp.down_proj",
+            "mtp.layers.0.mlp.gate_proj",
+            "mtp.layers.0.mlp.up_proj",
+            "mtp.layers.0.post_attention_layernorm",
+            "mtp.layers.0.self_attn.k_norm",
+            "mtp.layers.0.self_attn.k_proj",
+            "mtp.layers.0.self_attn.o_proj",
+            "mtp.layers.0.self_attn.q_norm",
+            "mtp.layers.0.self_attn.q_proj",
+            "mtp.layers.0.self_attn.v_proj",
+            "mtp.norm",
+            "mtp.pre_fc_norm_embedding",
+            "mtp.pre_fc_norm_hidden"
+        ]
+    }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e420ea69111b7bb3c7c529cdbbb4ff4e6ebb531c0cd2be5c634cd88cd40eaca
+size 9382671592

modeling_sensenova_loop.py

@@ -0,0 +1,395 @@
+"""Hugging Face remote-code support for shared-weight SenseNova Loop and MHC.
+
+This single file contains configuration, plan validation, MHC routing and the
+model implementation. The original physical layer weights are reused on every
+Loop visit; autoregressive cache states are independent per visit.
+"""
+import copy
+import math
+from contextvars import ContextVar
+
+import torch
+from torch import nn
+from transformers import Qwen3_5Config
+from transformers.cache_utils import DynamicCache
+from transformers.models.qwen3_5 import modeling_qwen3_5 as hf
+
+
+_attention_capture = ContextVar("sensenova_attention_capture", default=None)
+
+
+class SenseNovaLoopConfig(Qwen3_5Config):
+    model_type = "sensenova_loop"
+
+
+def execution_plan(config):
+    """Return physical layer indices in the order they execute during inference."""
+    loop = config.get("loop_settings")
+    if not isinstance(loop, dict):
+        raise ValueError("This implementation requires explicit loop_settings")
+
+    text_config = config["text_config"]
+    num_layers = text_config["num_hidden_layers"]
+    start, end, count = loop.get("start"), loop.get("end"), loop.get("eval_count")
+    if any(type(value) is not int for value in (num_layers, start, end, count)) or not (
+        0 <= start < end <= num_layers and count >= 1
+    ):
+        raise ValueError("Invalid Loop interval or eval_count")
+    if loop.get("counts") != [count]:
+        raise ValueError("Only fixed-count Loop checkpoints are supported")
+
+    layer_types = text_config["layer_types"]
+    if len(layer_types) != num_layers or set(layer_types) - {"full_attention", "linear_attention"}:
+        raise ValueError("Unexpected physical layer types")
+    if "full_attention" not in layer_types:
+        raise ValueError("At least one full-attention layer is required for sequence-length tracking")
+
+    mhc = config.get("mhc")
+    if mhc is not None:
+        mhc_start, mhc_end, streams, iterations = (
+            mhc.get(key) for key in ("start", "end", "streams", "iterations")
+        )
+        if (
+            any(type(value) is not int for value in (mhc_start, mhc_end, streams, iterations))
+            or not 0 <= mhc_start < mhc_end <= num_layers
+            or streams not in (2, 4)
+            or iterations < 1
+        ):
+            raise ValueError("Invalid MHC range, stream count, or iterations")
+        if type(mhc.get("fixed_identity")) is not bool:
+            raise ValueError("MHC fixed_identity must be a boolean")
+        scale = mhc.get("pre_scale", 1.0)
+        if not isinstance(scale, (int, float)) or not math.isfinite(scale) or scale <= 0:
+            raise ValueError("Invalid MHC pre_scale")
+        allowed = {"start", "end", "streams", "iterations", "fixed_identity", "pre_scale"}
+        if set(mhc) - allowed:
+            raise ValueError("Unknown MHC configuration fields")
+        phase_boundaries = (0, start, end, num_layers)
+        if not any(
+            left <= mhc_start < mhc_end <= right
+            for left, right in zip(phase_boundaries, phase_boundaries[1:])
+        ):
+            raise ValueError("MHC cannot cross a Loop phase boundary")
+
+    return tuple(range(start)) + tuple(range(start, end)) * count + tuple(range(end, num_layers))
+
+
+def sinkhorn(logits, iterations):
+    """Normalize routing logits across rows and columns in log space."""
+    logits = logits.float()
+    for _ in range(iterations):
+        logits = logits - torch.logsumexp(logits, dim=-1, keepdim=True)
+        logits = logits - torch.logsumexp(logits, dim=-2, keepdim=True)
+    return logits.exp()
+
+
+class MHCRouter(nn.Module):
+    def __init__(self, dim, settings):
+        super().__init__()
+        self.streams = settings["streams"]
+        self.iterations = settings["iterations"]
+        self.fixed_identity = settings["fixed_identity"]
+        self.pre_scale = settings.get("pre_scale", 1.0)
+        streams = self.streams
+        output_size = 2 * streams + (0 if self.fixed_identity else streams * streams)
+        self.weight = nn.Parameter(torch.zeros(output_size, streams * dim, dtype=torch.float32))
+        self.bias = nn.Parameter(torch.zeros(output_size, dtype=torch.float32))
+        with torch.no_grad():
+            initial_probabilities = {2: (0.25, 0.75), 4: (0.0625, 0.1875, 0.25, 0.5)}[streams]
+            self.bias[:streams].copy_(torch.logit(torch.tensor(initial_probabilities, device=self.bias.device)))
+            if not self.fixed_identity:
+                diagonal = math.log((1 - (streams - 1) / 16) * 16)
+                self.bias[2 * streams:].copy_((torch.eye(streams, device=self.bias.device) * diagonal).flatten())
+
+    def _apply(self, fn, recurse=True):
+        # Keep the original FP32 values when the surrounding model is cast to BF16.
+        def keep_fp32(tensor):
+            result = fn(tensor)
+            if tensor.is_floating_point() and result.dtype != torch.float32:
+                if tensor.device.type == "meta" and result.device.type != "meta":
+                    return torch.empty_like(result, dtype=torch.float32)
+                return tensor.to(device=result.device, dtype=torch.float32)
+            return result
+
+        return super()._apply(keep_fp32, recurse=recurse)
+
+    def forward(self, residual):
+        if self.weight.dtype != torch.float32 or self.bias.dtype != torch.float32:
+            raise RuntimeError("mHC routers require FP32 parameters; configure FSDP per-module precision")
+        with torch.autocast(device_type=residual.device.type, enabled=False):
+            flat = residual.flatten(-2).float()
+            normalized = flat * torch.rsqrt(flat.square().mean(-1, keepdim=True) + 1e-6)
+            logits = torch.nn.functional.linear(normalized, self.weight, self.bias)
+            streams = self.streams
+            pre = logits[..., :streams].sigmoid()
+            post = 2 * logits[..., streams:2 * streams].sigmoid()
+            if self.fixed_identity:
+                mix = torch.eye(streams, device=flat.device, dtype=torch.float32).expand(
+                    *flat.shape[:-1], streams, streams
+                )
+            else:
+                mix = sinkhorn(logits[..., 2 * streams:].unflatten(-1, (streams, streams)), self.iterations)
+            branch = torch.matmul(pre.to(residual.dtype).unsqueeze(-2), residual).squeeze(-2)
+            if self.pre_scale != 1.0:
+                branch = branch * self.pre_scale
+        return branch, post, mix
+
+    @staticmethod
+    def merge(residual, branch, post, mix):
+        """Mix residual streams and add the branch output to each stream."""
+        with torch.autocast(device_type=residual.device.type, enabled=False):
+            mixed_residual = torch.matmul(mix.to(residual.dtype), residual)
+            branch_update = post.to(branch.dtype).unsqueeze(-1) * branch.unsqueeze(-2)
+            return mixed_residual + branch_update
+
+
+def enter(hidden, physical, settings):
+    """Expand to residual streams at the first mHC layer, or restore their shape."""
+    if physical == settings["start"]:
+        return hidden.unsqueeze(-2).expand(
+            *hidden.shape[:-1], settings["streams"], hidden.shape[-1]
+        ).contiguous()
+    return hidden.unflatten(-1, (settings["streams"], -1))
+
+
+def leave(residual, physical, settings):
+    """Reduce streams at the last mHC layer; otherwise carry them forward."""
+    return residual.mean(-2) if physical == settings["end"] - 1 else residual.flatten(-2)
+
+
+class _CacheLayersView:
+    """Expose the execution slot when a layer accesses its physical index."""
+    def __init__(self, cache, physical, virtual):
+        self.cache = cache
+        self.physical = physical
+        self.virtual = virtual
+
+    def __getitem__(self, index):
+        if index != self.physical:
+            raise IndexError("A physical layer may only access its own virtual cache slot")
+        return self.cache.layers[self.virtual]
+
+
+class _VisitCache:
+    """Map a physical attention module's layer_idx to one execution slot.
+
+    Mapping is local to this forward call: shared modules and their layer_idx are
+    never mutated. `layers` supports Transformers 5.12; state accessors support
+    newer official backbone cache APIs as well.
+    """
+    def __init__(self, cache, physical, virtual):
+        self.cache = cache
+        self.virtual = virtual
+        self.layers = _CacheLayersView(cache, physical, virtual)
+
+    def update(self, key_states, value_states, layer_idx, **kwargs):
+        return self.cache.update(key_states, value_states, self.virtual, **kwargs)
+
+    def has_previous_state(self, layer_idx=None, **kwargs):
+        return self.cache.has_previous_state(self.virtual, **kwargs)
+
+    def update_conv_state(self, state, layer_idx, **kwargs):
+        return self.cache.update_conv_state(state, self.virtual, **kwargs)
+
+    def update_recurrent_state(self, state, layer_idx, **kwargs):
+        return self.cache.update_recurrent_state(state, self.virtual, **kwargs)
+
+    def get_conv_state(self, layer_idx, **kwargs):
+        return self.cache.get_conv_state(self.virtual, **kwargs)
+
+    def get_recurrent_state(self, layer_idx, **kwargs):
+        return self.cache.get_recurrent_state(self.virtual, **kwargs)
+
+    def is_recording_past(self, layer_idx):
+        return self.cache.is_recording_past(self.virtual)
+
+
+class SenseNovaLoopTextModel(hf.Qwen3_5TextModel):
+    def forward(self, input_ids=None, attention_mask=None, position_ids=None,
+                past_key_values=None, inputs_embeds=None, use_cache=None,
+                output_hidden_states=None, output_attentions=None, **kwargs):
+        """Run Loop visits, optionally returning their states and attention weights.
+
+        Hidden states include the initial embedding and one state per visit.
+        MHC states between its boundaries retain their flattened residual streams.
+        Attention entries follow visit order, with None for linear-attention layers.
+        """
+        if (input_ids is None) == (inputs_embeds is None):
+            raise ValueError("Specify exactly one of input_ids or inputs_embeds")
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+        use_cache = self.config.use_cache if use_cache is None else use_cache
+        if output_hidden_states is None:
+            output_hidden_states = getattr(self.config, "output_hidden_states", False)
+        if output_attentions is None:
+            output_attentions = getattr(self.config, "output_attentions", False)
+        if not isinstance(output_hidden_states, bool):
+            raise TypeError("output_hidden_states must be a boolean.")
+        if output_attentions and self.config._attn_implementation != "eager":
+            raise ValueError(
+                "Attention weights require attn_implementation='eager' when loading the model."
+            )
+        if self.gradient_checkpointing and self.training and use_cache:
+            raise ValueError("Disable use_cache when training with gradient checkpointing")
+        if not use_cache and past_key_values is not None:
+            raise ValueError("past_key_values requires use_cache=True")
+        if past_key_values is not None and not isinstance(past_key_values, DynamicCache):
+            raise TypeError("Loop generation currently supports DynamicCache only")
+        if use_cache:
+            if past_key_values is None:
+                past_key_values = DynamicCache(config=self._loop_cache_config)
+                past_key_values._loop_execution_plan = self._loop_plan
+            elif getattr(past_key_values, "_loop_execution_plan", None) != self._loop_plan:
+                # GenerationMixin may allocate an empty physical-depth cache.
+                # A populated foreign cache cannot be reinterpreted safely.
+                if past_key_values.get_seq_length() != 0 or past_key_values.has_previous_state():
+                    raise ValueError("A populated cache from another execution plan cannot be reused")
+                past_key_values = DynamicCache(config=self._loop_cache_config)
+                past_key_values._loop_execution_plan = self._loop_plan
+
+        if position_ids is None:
+            seen = past_key_values.get_seq_length() if past_key_values is not None else 0
+            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + seen
+            position_ids = position_ids.view(1, 1, -1).expand(4, inputs_embeds.shape[0], -1)
+        elif position_ids.ndim == 2:
+            position_ids = position_ids.unsqueeze(0).expand(4, -1, -1)
+        if position_ids.ndim == 3 and position_ids.shape[0] == 4:
+            text_positions, position_ids = position_ids[0], position_ids[1:]
+        else:
+            text_positions = None
+
+        if isinstance(attention_mask, dict):
+            masks = attention_mask
+        else:
+            mask_args = dict(
+                config=self._loop_cache_config,
+                inputs_embeds=inputs_embeds,
+                attention_mask=attention_mask,
+                past_key_values=past_key_values,
+                position_ids=text_positions,
+            )
+            if hasattr(hf, "create_recurrent_attention_mask"):
+                linear_mask = hf.create_recurrent_attention_mask(**mask_args)
+            else:
+                linear_mask = self._update_linear_attn_mask(attention_mask, past_key_values)
+            masks = {
+                "full_attention": hf.create_causal_mask(**mask_args),
+                "linear_attention": linear_mask,
+            }
+
+        hidden = inputs_embeds
+        position_embeddings = self.rotary_emb(hidden, position_ids)
+        hidden_states = [hidden] if output_hidden_states else None
+        attentions = [] if output_attentions else None
+        hooks = []
+        capture_token = None
+
+        def record_attention(_module, _inputs, output):
+            if _attention_capture.get() is attentions:
+                if output[1] is None:
+                    raise RuntimeError("The attention backend did not return attention weights.")
+                attentions.append(output[1])
+
+        # Repeated visits share weights, but each visit has its own cache slot.
+        try:
+            if attentions is not None:
+                for layer in self.layers:
+                    if hasattr(layer, "self_attn"):
+                        hooks.append(layer.self_attn.register_forward_hook(record_attention))
+                capture_token = _attention_capture.set(attentions)
+            for execution_index, physical_index in enumerate(self._loop_plan):
+                layer_type = self.config.layer_types[physical_index]
+                if attentions is not None and layer_type == "linear_attention":
+                    attentions.append(None)
+                cache = (
+                    _VisitCache(past_key_values, physical_index, execution_index)
+                    if use_cache else None
+                )
+                hidden = self.layers[physical_index](
+                    hidden, position_embeddings=position_embeddings,
+                    attention_mask=masks[layer_type],
+                    position_ids=text_positions, past_key_values=cache,
+                    use_cache=use_cache, **kwargs,
+                )
+                if hidden_states is not None:
+                    hidden_states.append(hidden)
+        finally:
+            if capture_token is not None:
+                _attention_capture.reset(capture_token)
+            for hook in hooks:
+                hook.remove()
+
+        hidden = self.norm(hidden)
+        if hidden_states is not None:
+            hidden_states[-1] = hidden
+        return hf.Qwen3_5ModelOutputWithPast(
+            last_hidden_state=hidden, past_key_values=past_key_values,
+            hidden_states=tuple(hidden_states) if hidden_states is not None else None,
+            attentions=tuple(attentions) if attentions is not None else None,
+        )
+
+
+class MHCDecoderLayer(hf.Qwen3_5DecoderLayer):
+    def forward(self, hidden_states, position_embeddings, attention_mask=None,
+                position_ids=None, past_key_values=None, **kwargs):
+        residual = enter(hidden_states, self._mhc_physical, self._mhc_settings)
+        branch, post, mix = self.mhc_attn(residual)
+        branch = self.input_layernorm(branch)
+        kind = self.layer_type if hasattr(self, "layer_type") else self.block_type
+        if kind == "linear_attention":
+            branch = self.linear_attn(hidden_states=branch, cache_params=past_key_values,
+                                      attention_mask=attention_mask, **kwargs)
+        else:
+            branch, _ = self.self_attn(
+                hidden_states=branch, position_embeddings=position_embeddings,
+                attention_mask=attention_mask, position_ids=position_ids,
+                past_key_values=past_key_values, **kwargs)
+        residual = MHCRouter.merge(residual, branch, post, mix)
+        branch, post, mix = self.mhc_ffn(residual)
+        branch = self.mlp(self.post_attention_layernorm(branch))
+        return leave(MHCRouter.merge(residual, branch, post, mix),
+                     self._mhc_physical, self._mhc_settings)
+
+
+def _configure_text_model(text, config):
+    raw = config.to_dict()
+    plan = execution_plan(raw)
+    # Change only these new model instances, preserving canonical state_dict
+    # names and avoiding a second allocation of the full backbone.
+    text.__class__ = SenseNovaLoopTextModel
+    text._loop_plan = plan
+    text._loop_cache_config = copy.deepcopy(text.config)
+    text._loop_cache_config.num_hidden_layers = len(plan)
+    text._loop_cache_config.layer_types = [text.config.layer_types[i] for i in plan]
+    if raw.get("mhc") is not None:
+        settings = raw["mhc"]
+        for i in range(settings["start"], settings["end"]):
+            layer = text.layers[i]
+            layer.__class__ = MHCDecoderLayer
+            layer._mhc_physical, layer._mhc_settings = i, dict(settings)
+            layer.mhc_attn = MHCRouter(text.config.hidden_size, settings)
+            layer.mhc_ffn = MHCRouter(text.config.hidden_size, settings)
+
+
+class SenseNovaLoopModel(hf.Qwen3_5Model):
+    config_class = SenseNovaLoopConfig
+    _keep_in_fp32_modules = ["mhc_attn", "mhc_ffn"]
+    _no_split_modules = ["Qwen3_5DecoderLayer", "MHCDecoderLayer", "Qwen3_5VisionBlock"]
+    _keys_to_ignore_on_load_unexpected = [r"^mtp\."]
+
+    def __init__(self, config):
+        execution_plan(config.to_dict())
+        super().__init__(config)
+        _configure_text_model(self.language_model, config)
+
+
+class SenseNovaLoopForConditionalGeneration(hf.Qwen3_5ForConditionalGeneration):
+    config_class = SenseNovaLoopConfig
+    _keep_in_fp32_modules = ["mhc_attn", "mhc_ffn"]
+    _no_split_modules = ["Qwen3_5DecoderLayer", "MHCDecoderLayer", "Qwen3_5VisionBlock"]
+    _keys_to_ignore_on_load_unexpected = [r"^mtp\."]
+
+    def __init__(self, config):
+        execution_plan(config.to_dict())
+        super().__init__(config)
+        _configure_text_model(self.model.language_model, config)

preprocessor_config.json

@@ -0,0 +1,26 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "merge_size": 2,
+  "patch_size": 16,
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 16777216,
+    "shortest_edge": 65536
+  },
+  "temporal_patch_size": 2
+}

processor_config.json

@@ -0,0 +1,60 @@
+{
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Qwen2VLImageProcessor",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "merge_size": 2,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 16777216,
+      "shortest_edge": 65536
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Qwen3VLProcessor",
+  "video_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 25165824,
+      "shortest_edge": 4096
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen3VLVideoProcessor"
+  }
+}

quantization/10_reference_checks.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""HF-transformers functional checks for SenseNova-Loop (reference BF16 backbone + FP32 mHC).
+
+Checks: mHC router dtype, shared-weight looping, text greedy, thinking, tool-call format,
+image understanding, and cached-vs-uncached decoding consistency (teacher-forced argmax).
+Writes a JSON report; does not modify the model directory.
+
+usage: python -I scripts/10_reference_checks.py MODEL_DIR OUT_JSON [IMAGE]
+"""
+import json, sys, time
+import torch
+from PIL import Image
+from transformers import AutoModelForImageTextToText, AutoProcessor
+
+model_dir, out_path = sys.argv[1], sys.argv[2]
+image_path = sys.argv[3] if len(sys.argv) > 3 else None
+dev = "cuda:0"
+report = {"model_dir": model_dir, "torch": torch.__version__}
+
+t0 = time.time()
+proc = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
+model = AutoModelForImageTextToText.from_pretrained(
+    model_dir, trust_remote_code=True, dtype=torch.bfloat16, device_map={"": dev}).eval()
+report["load_s"] = round(time.time() - t0, 1)
+report["gpu_mem_after_load_GiB"] = round(torch.cuda.memory_allocated() / 2**30, 2)
+
+text = model.model.language_model
+report["loop_plan_len"] = len(text._loop_plan)
+routers = {n: p.dtype for n, p in model.named_parameters() if ".mhc_" in n}
+report["mhc_param_count"] = len(routers)
+report["mhc_all_fp32"] = all(d == torch.float32 for d in routers.values())
+report["layer_param_dtypes"] = sorted({str(p.dtype) for n, p in model.named_parameters() if ".mhc_" not in n})
+
+# Each physical looped layer must be called twice per forward (shared weights).
+calls = {}
+hooks = [l.register_forward_hook(lambda m, i, o, k=k: calls.__setitem__(k, calls.get(k, 0) + 1))
+         for k, l in enumerate(text.layers)]
+tok = proc.tokenizer
+
+
+def chat(messages, max_new=128, tools=None, thinking=False, images=None):
+    prompt = proc.apply_chat_template(messages, tools=tools, tokenize=False, add_generation_prompt=True,
+                                      enable_thinking=thinking)
+    inputs = proc(text=[prompt], images=images, return_tensors="pt").to(dev)
+    t = time.time()
+    with torch.inference_mode():
+        out = model.generate(**inputs, max_new_tokens=max_new, do_sample=False, use_cache=True)
+    gen = out[0, inputs["input_ids"].shape[1]:]
+    return {"prompt_tokens": int(inputs["input_ids"].shape[1]), "gen_tokens": int(gen.shape[0]),
+            "seconds": round(time.time() - t, 2), "text": tok.decode(gen, skip_special_tokens=False)}, out, inputs
+
+
+r, _, _ = chat([{"role": "user", "content": "What is 13 times 7? Reply with the number only."}], 16)
+report["text_mult"] = r
+report["loop_calls_per_layer"] = {str(k): v for k, v in sorted(calls.items())}
+report["looped_layers_called_twice"] = all(calls[k] == (2 if 4 <= k < 28 else 1) * calls[0] for k in calls)
+for h in hooks:
+    h.remove()
+
+report["text_haiku"] = chat([{"role": "user", "content": "Write a haiku about a tennis ball."}], 64)[0]
+report["thinking"] = chat([{"role": "user", "content": "If a train goes 60 km in 45 minutes, what is its speed in km/h?"}],
+                          384, thinking=True)[0]
+tools = [{"type": "function", "function": {
+    "name": "get_weather", "description": "Get current weather for a city.",
+    "parameters": {"type": "object", "properties": {
+        "city": {"type": "string"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}},
+        "required": ["city"]}}}]
+r = chat([{"role": "user", "content": "What's the weather in Paris in celsius?"}], 96, tools=tools)[0]
+r["has_tool_call_xml"] = ("<tool_call>" in r["text"] and "<function=get_weather>" in r["text"]
+                          and "<parameter=city>" in r["text"])
+report["tool_call"] = r
+
+if image_path:
+    img = Image.open(image_path).convert("RGB")
+    msgs = [{"role": "user", "content": [{"type": "image"},
+                                          {"type": "text", "text": "Describe this image in two sentences."}]}]
+    report["image"] = chat(msgs, 96, images=[img])[0]
+
+# Cached generation consistency: greedy tokens from cached decoding must equal the
+# argmax of a single uncached forward over prompt+generation (teacher forcing).
+r, out, inputs = chat([{"role": "user", "content": "Explain in one paragraph why the sky is blue."}], 64)
+plen = inputs["input_ids"].shape[1]
+with torch.inference_mode():
+    logits = model(input_ids=out, use_cache=False).logits[0, plen - 1:-1].float()
+pred = logits.argmax(-1)
+gen = out[0, plen:]
+report["cache_consistency"] = {"n": int(gen.numel()), "match": int((pred == gen).sum()),
+                               "first_mismatch": int((pred != gen).nonzero()[0]) if (pred != gen).any() else None,
+                               "finite_logits": bool(torch.isfinite(logits).all())}
+report["peak_gpu_mem_GiB"] = round(torch.cuda.max_memory_allocated() / 2**30, 2)
+json.dump(report, open(out_path, "w"), indent=2, ensure_ascii=False)
+print(json.dumps(report, indent=2, ensure_ascii=False))

quantization/20_quantize.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""ModelOpt NVFP4 PTQ of SenseNova-Loop-9B-Preview through the real looped (2-visit) forward.
+
+Recipes:
+  mlp      - NVFP4 on decoder MLP gate/up/down only (conservative)
+  decoder  - NVFP4 on all decoder linears (full-attn q/k/v/o, GDN in_proj_qkv/in_proj_z/out_proj, MLP)
+Always higher precision: vision tower, lm_head, embeddings, GDN in_proj_a/in_proj_b (tiny gate projs),
+GDN conv1d/A_log/dt_bias, norms, mHC routers (FP32, not nn.Linear), MTP head (not loaded by HF; copied BF16).
+KV cache: not quantized.
+
+usage: python -I scripts/20_quantize.py --recipe decoder --calib calib/calib_v1.jsonl --out work/nvfp4-decoder [--limit N]
+"""
+import argparse, copy, json, os, sys, time
+import torch
+from PIL import Image
+from transformers import AutoModelForImageTextToText, AutoProcessor
+import modelopt
+import modelopt.torch.quantization as mtq
+from modelopt.torch.export import export_hf_checkpoint
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--model", default="upstream/SenseNova-Loop-9B-Preview")
+ap.add_argument("--recipe", choices=["mlp", "decoder"], required=True)
+ap.add_argument("--calib", default="calib/calib_v1.jsonl")
+ap.add_argument("--root", default=".", help="base dir for calib image paths")
+ap.add_argument("--out", required=True)
+ap.add_argument("--limit", type=int, default=0)
+ap.add_argument("--max-len", type=int, default=2048)
+args = ap.parse_args()
+dev = "cuda:0"
+torch.manual_seed(0)
+
+proc = AutoProcessor.from_pretrained(args.model, trust_remote_code=True)
+model = AutoModelForImageTextToText.from_pretrained(
+    args.model, trust_remote_code=True, dtype=torch.bfloat16, device_map={"": dev}).eval()
+text = model.model.language_model
+
+# --- quantization config -------------------------------------------------------------
+cfg = copy.deepcopy(mtq.NVFP4_DEFAULT_CFG)
+qc = cfg["quant_cfg"]  # ModelOpt >=0.48: ordered list, later entries win
+for pat in ["*lm_head*", "*visual*", "*vision*", "*mtp*", "*in_proj_a*", "*in_proj_b*",
+            "*mhc_*", "*embed_tokens*", "*conv1d*"]:
+    qc.append({"quantizer_name": pat, "enable": False})
+if args.recipe == "mlp":
+    for pat in ["*self_attn*", "*linear_attn*"]:
+        qc.append({"quantizer_name": pat, "enable": False})
+
+# --- calibration data: rendered with the model's own template + processor ----------------
+rows = [json.loads(l) for l in open(args.calib)]
+if args.limit:
+    rows = rows[: args.limit]
+
+
+def render(row):
+    msgs, images = [], []
+    for m in row["messages"]:
+        m = dict(m)
+        if isinstance(m.get("content"), list):
+            parts = []
+            for p in m["content"]:
+                if p.get("type") == "image":
+                    images.append(Image.open(os.path.join(args.root, p["image"])).convert("RGB"))
+                    parts.append({"type": "image"})
+                else:
+                    parts.append(p)
+            m["content"] = parts
+        msgs.append(m)
+    prompt = proc.apply_chat_template(msgs, tools=row.get("tools"), tokenize=False,
+                                      add_generation_prompt=False,
+                                      enable_thinking=bool(row.get("enable_thinking")))
+    batch = proc(text=[prompt], images=images or None, return_tensors="pt")
+    if batch["input_ids"].shape[1] > args.max_len and not images:
+        batch = {k: v[:, : args.max_len] for k, v in batch.items()}
+    return batch
+
+
+batches = [render(r) for r in rows]
+ntok = sum(int(b["input_ids"].shape[1]) for b in batches)
+print(f"calib: {len(batches)} samples, {ntok} tokens, {sum(r.get('n_images', 0) for r in rows)} images", flush=True)
+
+visits = {}
+hooks = [l.register_forward_hook(lambda m, i, o, k=k: visits.__setitem__(k, visits.get(k, 0) + 1))
+         for k, l in enumerate(text.layers)]
+
+
+def forward_loop(m):
+    t = time.time()
+    for i, b in enumerate(batches):
+        b = {k: v.to(dev) for k, v in b.items()}
+        with torch.inference_mode():
+            m(**b, use_cache=False)
+        if i % 32 == 0:
+            print(f"  calib {i}/{len(batches)} {time.time() - t:.0f}s", flush=True)
+
+
+t0 = time.time()
+model = mtq.quantize(model, cfg, forward_loop)
+for h in hooks:
+    h.remove()
+calib_s = time.time() - t0
+mtq.print_quant_summary(model) if hasattr(mtq, "print_quant_summary") else None
+
+# --- bookkeeping: what got quantized ----------------------------------------------------
+quantized, excluded = [], []
+for name, mod in model.named_modules():
+    if isinstance(mod, torch.nn.Linear):
+        wq = getattr(mod, "weight_quantizer", None)
+        (quantized if (wq is not None and wq.is_enabled) else excluded).append((name, mod.weight.numel()))
+total_params = sum(p.numel() for p in model.parameters())
+q_params = sum(n for _, n in quantized)
+looped = [k for k, v in visits.items() if v == 2 * len(batches)]
+stats = {
+    "recipe": args.recipe, "modelopt": modelopt.__version__, "torch": torch.__version__,
+    "calib_file": args.calib, "calib_samples": len(batches), "calib_tokens": ntok, "calib_seconds": round(calib_s, 1),
+    "layers_visited_twice_per_sample": looped,
+    "quantized_linear_modules": len(quantized), "excluded_linear_modules": [n for n, _ in excluded],
+    "quantized_params": q_params, "total_params_hf_loaded": total_params,
+    "fraction_params_quantized": round(q_params / total_params, 4),
+    "quant_cfg": qc,
+}
+
+# Sanity: logits stay finite through the fake-quant model.
+with torch.inference_mode():
+    b = {k: v.to(dev) for k, v in batches[0].items()}
+    stats["fakequant_logits_finite"] = bool(torch.isfinite(model(**b, use_cache=False).logits).all())
+
+os.makedirs(args.out, exist_ok=True)
+t0 = time.time()
+with torch.inference_mode():
+    export_hf_checkpoint(model, export_dir=args.out)
+stats["export_seconds"] = round(time.time() - t0, 1)
+json.dump(stats, open(os.path.join(args.out, "quantization_stats.json"), "w"), indent=2)
+print(json.dumps({k: v for k, v in stats.items() if k != "excluded_linear_modules"}, indent=2))
+print("excluded linear modules:", len(excluded))

quantization/30_verify_export.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Independent structural verification of an exported NVFP4 checkpoint (reads files only).
+
+Checks against the upstream checkpoint: every upstream tensor is present (as BF16/FP32 or as an
+NVFP4 packed weight + scales), packed shapes are consistent (uint8 [out, in/2], fp8 e4m3 block scales
+[out, in/16], fp32 global scales), all scales finite/positive, mHC routers FP32, exactly one copy of
+each physical (shared, looped) layer, MTP tensors present, config/quant metadata consistent.
+
+usage: python -I scripts/30_verify_export.py EXPORT_DIR UPSTREAM_DIR OUT_JSON
+"""
+import json, os, sys, collections
+import torch
+from safetensors import safe_open
+
+exp, up, out = sys.argv[1:4]
+
+
+def index(d):
+    idx = os.path.join(d, "model.safetensors.index.json")
+    if os.path.exists(idx):
+        wm = json.load(open(idx))["weight_map"]
+    else:
+        wm = {}
+        for f in os.listdir(d):
+            if f.endswith(".safetensors"):
+                with safe_open(os.path.join(d, f), "pt") as fh:
+                    wm.update({k: f for k in fh.keys()})
+    return wm
+
+
+def header(d, wm):
+    meta = {}
+    for f in set(wm.values()):
+        with safe_open(os.path.join(d, f), "pt") as fh:
+            for k in fh.keys():
+                sl = fh.get_slice(k)
+                meta[k] = (sl.get_dtype(), tuple(sl.get_shape()), f)
+    return meta
+
+
+ewm, uwm = index(exp), index(up)
+E, U = header(exp, ewm), header(up, uwm)
+qcfg = json.load(open(os.path.join(exp, "hf_quant_config.json")))["quantization"]
+cfg = json.load(open(os.path.join(exp, "config.json")))
+rep = {"export_dir": exp, "quant_algo": qcfg.get("quant_algo"), "group_size": qcfg.get("group_size"),
+       "kv_cache_quant_algo": qcfg.get("kv_cache_quant_algo"),
+       "config_has_quantization_config": "quantization_config" in cfg,
+       "n_tensors_export": len(E), "n_tensors_upstream": len(U)}
+problems = []
+
+packed = [k[: -len(".weight")] for k, (dt, _, _) in E.items() if k.endswith(".weight") and dt == "U8"]
+rep["nvfp4_linear_modules"] = len(packed)
+q_params = 0
+for m in packed:
+    dt, shp, _ = E[m + ".weight"]
+    ref = U.get(m + ".weight")
+    if ref is None:
+        problems.append(f"packed {m} not in upstream"); continue
+    o, i = ref[1]
+    q_params += o * i
+    if shp != (o, i // 2):
+        problems.append(f"{m}: packed shape {shp} != {(o, i // 2)}")
+    s = E.get(m + ".weight_scale")
+    if s is None or s[0] != "F8_E4M3" or s[1] != (o, i // qcfg["group_size"]):
+        problems.append(f"{m}: weight_scale {s}")
+    for k in ("weight_scale_2", "input_scale"):
+        t = E.get(f"{m}.{k}")
+        if t is None or t[0] != "F32" or (len(t[1]) and t[1] != (1,)):
+            problems.append(f"{m}: {k} {t}")
+
+# Load all scales and check finite / positive.
+bad_scales, n_scales = 0, 0
+by_file = collections.defaultdict(list)
+for k in E:
+    if k.endswith(("weight_scale", "weight_scale_2", "input_scale")):
+        by_file[E[k][2]].append(k)
+for f, keys in by_file.items():
+    with safe_open(os.path.join(exp, f), "pt") as fh:
+        for k in keys:
+            t = fh.get_tensor(k).float()
+            n_scales += 1
+            if not torch.isfinite(t).all() or (t[t != 0] <= 0).any() or (k.endswith("input_scale") and (t <= 0).any()):
+                bad_scales += 1
+                problems.append(f"bad scale {k}")
+rep["scale_tensors_checked"] = n_scales
+rep["bad_scale_tensors"] = bad_scales
+
+# Upstream coverage: every upstream tensor present in export (either same name or packed).
+missing = [k for k in U if k not in E]
+rep["upstream_tensors_missing_in_export"] = missing[:20]
+rep["n_upstream_missing"] = len(missing)
+extra = [k for k in E if k not in U and not k.endswith(("weight_scale", "weight_scale_2", "input_scale"))]
+rep["unexpected_extra_tensors"] = extra[:20]
+
+mhc = {k: E[k][0] for k in E if ".mhc_" in k}
+rep["mhc_tensors"] = len(mhc)
+rep["mhc_all_fp32"] = all(v == "F32" for v in mhc.values()) and len(mhc) == 96
+mtp = [k for k in E if k.startswith("mtp.")]
+rep["mtp_tensors"] = len(mtp)
+rep["mtp_dtypes"] = sorted({E[k][0] for k in mtp})
+
+# Shared weights: layer ids present must be exactly 0..31 (no duplicated per-visit copies).
+lids = sorted({int(k.split(".layers.")[1].split(".")[0]) for k in E if "language_model.layers." in k})
+rep["physical_layer_ids"] = f"{lids[0]}..{lids[-1]} ({len(lids)})"
+if lids != list(range(32)):
+    problems.append(f"layer ids {lids}")
+
+dtypes = collections.Counter()
+nbytes = collections.Counter()
+size = {"U8": 1, "F8_E4M3": 1, "BF16": 2, "F16": 2, "F32": 4}
+import math
+for k, (dt, shp, _) in E.items():
+    n = math.prod(shp)
+    dtypes[dt] += n
+    nbytes[dt] += n * size.get(dt, 0)
+rep["elements_by_dtype"] = dict(dtypes)
+rep["GB_by_dtype"] = {k: round(v / 1e9, 3) for k, v in nbytes.items()}
+rep["total_file_GB"] = round(sum(os.path.getsize(os.path.join(exp, f)) for f in set(ewm.values())) / 1e9, 3)
+total_up = sum(math.prod(s) for (_, s, _) in U.values())
+rep["upstream_params_incl_mtp"] = total_up
+rep["nvfp4_params"] = q_params
+rep["fraction_params_nvfp4"] = round(q_params / total_up, 4)
+rep["problems"] = problems
+rep["ok"] = not problems and not missing and rep["mhc_all_fp32"] and len(mtp) > 0
+json.dump(rep, open(out, "w"), indent=2)
+print(json.dumps(rep, indent=2))

quantization/build_calib.py

@@ -0,0 +1,329 @@
+#!/usr/bin/env python
+"""Build calib/calib_v1.jsonl: 256 chat-format calibration examples for NVFP4 (ModelOpt) PTQ.
+
+  .venv-eval/bin/python calib/build_calib.py            # raw files are expected under calib/raw/
+  (downloads: see SOURCES; fetched with huggingface_hub.hf_hub_download at the pinned revisions)
+
+Each line: {"id", "category", "source", "messages": [...OpenAI-style...], "tools"?: [...],
+            "enable_thinking": bool, "approx_text_tokens": int, "n_images": int}
+Images: {"type": "image", "image": "calib/images/<file>.jpg"} (path relative to project root),
+which HF processors' apply_chat_template understand; the chat template also accepts image_url.
+Thinking examples carry the trace in the assistant's `reasoning_content` (the chat template renders
+it as <think>\n...\n</think>\n\n before the answer); all other final assistant turns render with
+the empty <think>\n\n</think>\n\n block, matching enable_thinking=False inference.
+Disjointness from eval/data is enforced by construction (different datasets/splits) AND checked
+by 13-gram overlap against every eval prompt (summary in calib/calib_v1.meta.json).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import random
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+import pyarrow.parquet as pq
+
+ROOT = Path(__file__).resolve().parent.parent
+CAL = ROOT / "calib"
+RAW = CAL / "raw"
+SEED = 20261030
+MAX_TEXT_TOKENS = 1800
+TARGET = {"chat": 77, "code": 51, "math": 51, "tools": 38, "image": 39}  # = 256 (30/20/20/15/15 %)
+N_THINK_MATH = 26
+
+SOURCES = {
+    "chat": dict(repo="OpenAssistant/oasst2", rev="179dd21fc55192153d94adb0e0ce8f69e222bf75",
+                 file="data/train-00000-of-00001-88ba0162028a73fc.parquet", split="train", license="Apache-2.0"),
+    "code": dict(repo="google-research-datasets/mbpp", rev="4bb6404fdc6cacfda99d4ac4205087b89d32030c",
+                 file="full/train-00000-of-00001.parquet", split="full/train", license="CC-BY-4.0"),
+    "math": dict(repo="openai/gsm8k", rev="740312add88f781978c0658806c59bc2815b9866",
+                 file="main/train-00000-of-00001.parquet", split="main/train", license="MIT"),
+    "tools": dict(repo="glaiveai/glaive-function-calling-v2", rev="e7f4b6456019f5d8bcb991ef0dd67d8ff23221ac",
+                  file="glaive-function-calling-v2.json", split="train", license="Apache-2.0"),
+    "image": dict(repo="lmms-lab/textvqa", rev="9c0699cd19768ac5ab97568f6b3cbac4c0062884",
+                  file="data/train-00000-of-00020.parquet", split="train (shard 0 of 20)",
+                  license="TextVQA annotations CC-BY-4.0; images OpenImages CC-BY-2.0 (Flickr attribution)"),
+}
+
+
+def raw(cat):
+    s = SOURCES[cat]
+    return RAW / s["repo"].replace("/", "__") / s["file"]
+
+
+_tok = None
+
+
+def ntok(text):
+    global _tok
+    if _tok is None:
+        from tokenizers import Tokenizer
+        _tok = Tokenizer.from_file(str(ROOT / "upstream/SenseNova-Loop-9B-Preview/tokenizer.json"))
+    return len(_tok.encode(text, add_special_tokens=False).ids)
+
+
+def ex_text(ex):
+    parts = [json.dumps(ex.get("tools") or "")]
+    for m in ex["messages"]:
+        c = m.get("content")
+        if isinstance(c, list):
+            c = " ".join(p.get("text", "") for p in c)
+        parts.append(c or "")
+        parts.append(m.get("reasoning_content") or "")
+        parts.append(json.dumps(m.get("tool_calls") or ""))
+    return "\n".join(parts)
+
+
+# ----------------------------------------------------------------------------- chat (oasst2)
+def build_chat(rng):
+    t = pq.read_table(raw("chat"), columns=["message_id", "parent_id", "text", "role", "lang", "deleted",
+                                             "review_result", "rank", "tree_state", "synthetic"]).to_pylist()
+    kids = {}
+    roots = []
+    for m in t:
+        if m["deleted"] or m["review_result"] is False or m["synthetic"]:
+            continue
+        if m["parent_id"] is None:
+            if m["role"] == "prompter" and m["tree_state"] == "ready_for_export":
+                roots.append(m)
+        else:
+            kids.setdefault(m["parent_id"], []).append(m)
+
+    def best(mid, role):
+        cs = [c for c in kids.get(mid, []) if c["role"] == role]
+        cs.sort(key=lambda c: (c["rank"] if c["rank"] is not None else 99, c["message_id"]))
+        return cs[0] if cs else None
+    roots.sort(key=lambda m: m["message_id"])
+    rng.shuffle(roots)
+    want = {"en": 58, "zh": 10, "other": 9}
+    got = {"en": [], "zh": [], "other": []}
+    for r in roots:
+        bucket = r["lang"] if r["lang"] in ("en", "zh") else "other"
+        if len(got[bucket]) >= want[bucket]:
+            continue
+        msgs, cur = [{"role": "user", "content": r["text"]}], r
+        turns = rng.choice([1, 1, 2])
+        for k in range(turns):
+            a = best(cur["message_id"], "assistant")
+            if a is None:
+                break
+            msgs.append({"role": "assistant", "content": a["text"]})
+            if k + 1 < turns:
+                u = best(a["message_id"], "prompter")
+                a2 = best(u["message_id"], "assistant") if u else None
+                if not (u and a2):
+                    break
+                msgs.append({"role": "user", "content": u["text"]})
+                cur = u
+        if msgs[-1]["role"] != "assistant":
+            continue
+        ex = {"category": "chat", "source": f"oasst2:{r['message_id']}", "lang": r["lang"], "messages": msgs}
+        if ntok(ex_text(ex)) > MAX_TEXT_TOKENS:
+            continue
+        got[bucket].append(ex)
+        if all(len(got[b]) >= want[b] for b in want):
+            break
+    return got["en"] + got["zh"] + got["other"]
+
+
+# ----------------------------------------------------------------------------- code (MBPP train)
+def build_code(rng):
+    rows = pq.read_table(raw("code")).to_pylist()
+    rows.sort(key=lambda r: r["task_id"])
+    idx = sorted(rng.sample(range(len(rows)), TARGET["code"]))
+    out = []
+    for i in idx:
+        r = rows[i]
+        tests = "\n".join(r["test_list"])
+        user = f"{r['text']} Your code should pass these tests:\n\n```python\n{tests}\n```"
+        asst = f"```python\n{r['code'].strip()}\n```"
+        out.append({"category": "code", "source": f"mbpp:task_id={r['task_id']}",
+                    "messages": [{"role": "user", "content": user}, {"role": "assistant", "content": asst}]})
+    return out
+
+
+# ----------------------------------------------------------------------------- math (GSM8K train)
+def build_math(rng):
+    rows = pq.read_table(raw("math")).to_pylist()
+    idx = sorted(rng.sample(range(len(rows)), TARGET["math"]))
+    out = []
+    for j, i in enumerate(idx):
+        q, sol = rows[i]["question"], rows[i]["answer"]
+        steps, final = sol.split("####")
+        steps = re.sub(r"<<[^>]*>>", "", steps).strip()
+        final = final.strip()
+        think = j < N_THINK_MATH
+        if think:
+            asst = {"role": "assistant", "reasoning_content": steps,
+                    "content": f"The final answer is {final}."}
+        else:
+            asst = {"role": "assistant", "content": f"{steps}\nThe final answer is {final}."}
+        out.append({"category": "math", "source": f"gsm8k:train:{i}", "enable_thinking": think,
+                    "messages": [{"role": "user", "content": q}, asst]})
+    return out
+
+
+# ----------------------------------------------------------------------------- tools (glaive v2)
+def parse_glaive(rec):
+    sysm = rec["system"]
+    if "following functions" not in sysm:
+        return None
+    body = sysm.split("Use them if required -", 1)[-1].strip()
+    dec, pos, fns = json.JSONDecoder(), 0, []
+    while pos < len(body):
+        while pos < len(body) and body[pos] in " \n\t":
+            pos += 1
+        if pos >= len(body):
+            break
+        try:
+            obj, pos = dec.raw_decode(body, pos)
+        except json.JSONDecodeError:
+            return None
+        fns.append({"type": "function", "function": obj})
+    if not fns:
+        return None
+    turns = re.split(r"\n*(USER|ASSISTANT|FUNCTION RESPONSE): ", "\n" + rec["chat"])
+    msgs, n_calls = [], 0
+    it = iter(turns[1:])
+    for role, text in zip(it, it):
+        text = text.replace("<|endoftext|>", "").strip()
+        if role == "USER":
+            msgs.append({"role": "user", "content": text})
+        elif role == "FUNCTION RESPONSE":
+            msgs.append({"role": "tool", "content": text})
+        else:
+            if "<functioncall>" in text:
+                pre, call = text.split("<functioncall>", 1)
+                m = re.match(r'\s*\{\s*"name"\s*:\s*"([^"]+)"\s*,\s*"arguments"\s*:\s*\'(.*)\'\s*\}\s*$', call, re.S)
+                if not m:
+                    return None
+                try:
+                    args = json.loads(m.group(2))
+                except json.JSONDecodeError:
+                    return None
+                if not isinstance(args, dict):
+                    return None
+                msgs.append({"role": "assistant", "content": pre.strip(),
+                             "tool_calls": [{"id": f"call_{n_calls}", "type": "function",
+                                             "function": {"name": m.group(1), "arguments": args}}]})
+                n_calls += 1
+            else:
+                msgs.append({"role": "assistant", "content": text})
+    if n_calls == 0 or not msgs or msgs[0]["role"] != "user" or msgs[-1]["role"] != "assistant":
+        return None
+    if not any(m["role"] == "tool" for m in msgs):
+        return None
+    return fns, msgs
+
+
+def build_tools(rng):
+    data = json.load(open(raw("tools")))
+    order = list(range(len(data)))
+    rng.shuffle(order)
+    out = []
+    for i in order:
+        p = parse_glaive(data[i])
+        if not p:
+            continue
+        ex = {"category": "tools", "source": f"glaive-fc-v2:{i}", "tools": p[0], "messages": p[1]}
+        if ntok(ex_text(ex)) > MAX_TEXT_TOKENS:
+            continue
+        out.append(ex)
+        if len(out) >= TARGET["tools"]:
+            break
+    return out
+
+
+# ----------------------------------------------------------------------------- image (TextVQA train)
+def build_image(rng, eval_image_ids):
+    from tasks import vqa_norm  # eval/ is on sys.path
+    t = pq.read_table(raw("image"), columns=["question_id", "question", "answers", "image_id"]).to_pylist()
+    cand = [i for i, r in enumerate(t) if r["image_id"] not in eval_image_ids
+            and Counter(vqa_norm(a) for a in r["answers"]).most_common(1)[0][1] >= 3]
+    rng.shuffle(cand)
+    seen_img, pickd = set(), []
+    for i in cand:
+        if t[i]["image_id"] in seen_img:
+            continue
+        seen_img.add(t[i]["image_id"])
+        pickd.append(i)
+        if len(pickd) >= TARGET["image"]:
+            break
+    pickd.sort()
+    imgs = pq.read_table(raw("image"), columns=["image"]).take(pickd).column("image").to_pylist()
+    (CAL / "images").mkdir(exist_ok=True)
+    out = []
+    for i, im in zip(pickd, imgs):
+        r = t[i]
+        b = im["bytes"]
+        ext = "png" if b[:4] == b"\x89PNG" else "jpg"
+        rel = f"calib/images/textvqa_train_{r['question_id']}.{ext}"
+        (ROOT / rel).write_bytes(b)
+        ans = Counter(a.strip() for a in r["answers"]).most_common(1)[0][0]
+        out.append({"category": "image", "source": f"textvqa:train:qid={r['question_id']}", "image_id": r["image_id"],
+                    "image_sha256": hashlib.sha256(b).hexdigest(), "messages": [
+                        {"role": "user", "content": [{"type": "image", "image": rel},
+                                                     {"type": "text", "text": r["question"].strip().capitalize()
+                                                      + "\nAnswer the question using a single word or phrase."}]},
+                        {"role": "assistant", "content": ans}]})
+    return out
+
+
+# ----------------------------------------------------------------------------- overlap check
+def shingles(text, n=13):
+    w = re.findall(r"\w+", text.lower())
+    return {" ".join(w[i:i + n]) for i in range(max(0, len(w) - n + 1))}
+
+
+def main():
+    sys.path.insert(0, str(ROOT / "eval"))
+    eval_dir = ROOT / "eval/data"
+    eval_sh, eval_imgs = set(), set()
+    for p in eval_dir.glob("*.jsonl"):
+        for l in open(p):
+            r = json.loads(l)
+            eval_sh |= shingles(json.dumps(r, ensure_ascii=False))
+            if "image_id" in r:
+                eval_imgs.add(r["image_id"])
+    exs = []
+    exs += build_chat(random.Random(f"{SEED}-chat"))
+    exs += build_code(random.Random(f"{SEED}-code"))
+    exs += build_math(random.Random(f"{SEED}-math"))
+    exs += build_tools(random.Random(f"{SEED}-tools"))
+    exs += build_image(random.Random(f"{SEED}-image"), eval_imgs)
+    cnt = Counter(e["category"] for e in exs)
+    assert dict(cnt) == TARGET, cnt
+    overlaps = []
+    for k, e in enumerate(exs):
+        e["id"] = f"calib_v1_{k:03d}"
+        e.setdefault("enable_thinking", False)
+        e["approx_text_tokens"] = ntok(ex_text(e))
+        e["n_images"] = sum(1 for m in e["messages"] if isinstance(m["content"], list)
+                            for p in m["content"] if p.get("type") == "image")
+        ov = shingles(ex_text(e)) & eval_sh
+        if ov:
+            overlaps.append({"id": e["id"], "source": e["source"], "n_shared_13grams": len(ov),
+                             "example": sorted(ov)[0]})
+    order = ["id", "category", "source", "enable_thinking", "approx_text_tokens", "n_images"]
+    with open(CAL / "calib_v1.jsonl", "w") as f:
+        for e in exs:
+            f.write(json.dumps({**{k: e[k] for k in order}, **{k: v for k, v in e.items() if k not in order}},
+                               ensure_ascii=False) + "\n")
+    toks = [e["approx_text_tokens"] for e in exs]
+    meta = {"version": "calib_v1", "seed": SEED, "n": len(exs), "counts": dict(cnt),
+            "n_thinking_examples": sum(e["enable_thinking"] for e in exs),
+            "langs_chat": dict(Counter(e.get("lang") for e in exs if e["category"] == "chat")),
+            "text_tokens": {"min": min(toks), "max": max(toks), "mean": sum(toks) / len(toks), "sum": sum(toks)},
+            "max_text_tokens_filter": MAX_TEXT_TOKENS, "sources": SOURCES,
+            "eval_overlap_13gram": overlaps,
+            "sha256": hashlib.sha256((CAL / "calib_v1.jsonl").read_bytes()).hexdigest()}
+    (CAL / "calib_v1.meta.json").write_text(json.dumps(meta, indent=2, ensure_ascii=False))
+    print(json.dumps({k: meta[k] for k in ("n", "counts", "n_thinking_examples", "langs_chat", "text_tokens")}))
+    print("13-gram overlaps with eval:", len(overlaps), overlaps[:3])
+
+
+if __name__ == "__main__":
+    main()

quantization/calib_v1.jsonl.sha256

@@ -0,0 +1 @@
+9a4d2112ef71820a5a6d8b015b4996092d4c08c3a2aa1abf7303aa5cd3b7ff83  calib/calib_v1.jsonl

quantization/calib_v1.meta.json

@@ -0,0 +1,72 @@
+{
+  "version": "calib_v1",
+  "seed": 20261030,
+  "n": 256,
+  "counts": {
+    "chat": 77,
+    "code": 51,
+    "math": 51,
+    "tools": 38,
+    "image": 39
+  },
+  "n_thinking_examples": 26,
+  "langs_chat": {
+    "en": 58,
+    "zh": 10,
+    "es": 7,
+    "ca": 2
+  },
+  "text_tokens": {
+    "min": 25,
+    "max": 1551,
+    "mean": 233.29296875,
+    "sum": 59723
+  },
+  "max_text_tokens_filter": 1800,
+  "sources": {
+    "chat": {
+      "repo": "OpenAssistant/oasst2",
+      "rev": "179dd21fc55192153d94adb0e0ce8f69e222bf75",
+      "file": "data/train-00000-of-00001-88ba0162028a73fc.parquet",
+      "split": "train",
+      "license": "Apache-2.0"
+    },
+    "code": {
+      "repo": "google-research-datasets/mbpp",
+      "rev": "4bb6404fdc6cacfda99d4ac4205087b89d32030c",
+      "file": "full/train-00000-of-00001.parquet",
+      "split": "full/train",
+      "license": "CC-BY-4.0"
+    },
+    "math": {
+      "repo": "openai/gsm8k",
+      "rev": "740312add88f781978c0658806c59bc2815b9866",
+      "file": "main/train-00000-of-00001.parquet",
+      "split": "main/train",
+      "license": "MIT"
+    },
+    "tools": {
+      "repo": "glaiveai/glaive-function-calling-v2",
+      "rev": "e7f4b6456019f5d8bcb991ef0dd67d8ff23221ac",
+      "file": "glaive-function-calling-v2.json",
+      "split": "train",
+      "license": "Apache-2.0"
+    },
+    "image": {
+      "repo": "lmms-lab/textvqa",
+      "rev": "9c0699cd19768ac5ab97568f6b3cbac4c0062884",
+      "file": "data/train-00000-of-00020.parquet",
+      "split": "train (shard 0 of 20)",
+      "license": "TextVQA annotations CC-BY-4.0; images OpenImages CC-BY-2.0 (Flickr attribution)"
+    }
+  },
+  "eval_overlap_13gram": [
+    {
+      "id": "calib_v1_203",
+      "source": "glaive-fc-v2:108408",
+      "n_shared_13grams": 1,
+      "example": "type function function name send_email description send an email parameters type object properties"
+    }
+  ],
+  "sha256": "9a4d2112ef71820a5a6d8b015b4996092d4c08c3a2aa1abf7303aa5cd3b7ff83"
+}

quantization/quantization_stats.json

@@ -0,0 +1,389 @@
+{
+  "recipe": "decoder",
+  "modelopt": "0.48.0rc1",
+  "torch": "2.11.0+cu130",
+  "calib_file": "calib/calib_v1.jsonl",
+  "calib_samples": 256,
+  "calib_tokens": 98458,
+  "calib_seconds": 1579.2,
+  "layers_visited_twice_per_sample": [
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    11,
+    12,
+    13,
+    14,
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+    23,
+    24,
+    25,
+    26,
+    27
+  ],
+  "quantized_linear_modules": 200,
+  "excluded_linear_modules": [
+    "model.visual.blocks.0.attn.qkv",
+    "model.visual.blocks.0.attn.proj",
+    "model.visual.blocks.0.mlp.linear_fc1",
+    "model.visual.blocks.0.mlp.linear_fc2",
+    "model.visual.blocks.1.attn.qkv",
+    "model.visual.blocks.1.attn.proj",
+    "model.visual.blocks.1.mlp.linear_fc1",
+    "model.visual.blocks.1.mlp.linear_fc2",
+    "model.visual.blocks.2.attn.qkv",
+    "model.visual.blocks.2.attn.proj",
+    "model.visual.blocks.2.mlp.linear_fc1",
+    "model.visual.blocks.2.mlp.linear_fc2",
+    "model.visual.blocks.3.attn.qkv",
+    "model.visual.blocks.3.attn.proj",
+    "model.visual.blocks.3.mlp.linear_fc1",
+    "model.visual.blocks.3.mlp.linear_fc2",
+    "model.visual.blocks.4.attn.qkv",
+    "model.visual.blocks.4.attn.proj",
+    "model.visual.blocks.4.mlp.linear_fc1",
+    "model.visual.blocks.4.mlp.linear_fc2",
+    "model.visual.blocks.5.attn.qkv",
+    "model.visual.blocks.5.attn.proj",
+    "model.visual.blocks.5.mlp.linear_fc1",
+    "model.visual.blocks.5.mlp.linear_fc2",
+    "model.visual.blocks.6.attn.qkv",
+    "model.visual.blocks.6.attn.proj",
+    "model.visual.blocks.6.mlp.linear_fc1",
+    "model.visual.blocks.6.mlp.linear_fc2",
+    "model.visual.blocks.7.attn.qkv",
+    "model.visual.blocks.7.attn.proj",
+    "model.visual.blocks.7.mlp.linear_fc1",
+    "model.visual.blocks.7.mlp.linear_fc2",
+    "model.visual.blocks.8.attn.qkv",
+    "model.visual.blocks.8.attn.proj",
+    "model.visual.blocks.8.mlp.linear_fc1",
+    "model.visual.blocks.8.mlp.linear_fc2",
+    "model.visual.blocks.9.attn.qkv",
+    "model.visual.blocks.9.attn.proj",
+    "model.visual.blocks.9.mlp.linear_fc1",
+    "model.visual.blocks.9.mlp.linear_fc2",
+    "model.visual.blocks.10.attn.qkv",
+    "model.visual.blocks.10.attn.proj",
+    "model.visual.blocks.10.mlp.linear_fc1",
+    "model.visual.blocks.10.mlp.linear_fc2",
+    "model.visual.blocks.11.attn.qkv",
+    "model.visual.blocks.11.attn.proj",
+    "model.visual.blocks.11.mlp.linear_fc1",
+    "model.visual.blocks.11.mlp.linear_fc2",
+    "model.visual.blocks.12.attn.qkv",
+    "model.visual.blocks.12.attn.proj",
+    "model.visual.blocks.12.mlp.linear_fc1",
+    "model.visual.blocks.12.mlp.linear_fc2",
+    "model.visual.blocks.13.attn.qkv",
+    "model.visual.blocks.13.attn.proj",
+    "model.visual.blocks.13.mlp.linear_fc1",
+    "model.visual.blocks.13.mlp.linear_fc2",
+    "model.visual.blocks.14.attn.qkv",
+    "model.visual.blocks.14.attn.proj",
+    "model.visual.blocks.14.mlp.linear_fc1",
+    "model.visual.blocks.14.mlp.linear_fc2",
+    "model.visual.blocks.15.attn.qkv",
+    "model.visual.blocks.15.attn.proj",
+    "model.visual.blocks.15.mlp.linear_fc1",
+    "model.visual.blocks.15.mlp.linear_fc2",
+    "model.visual.blocks.16.attn.qkv",
+    "model.visual.blocks.16.attn.proj",
+    "model.visual.blocks.16.mlp.linear_fc1",
+    "model.visual.blocks.16.mlp.linear_fc2",
+    "model.visual.blocks.17.attn.qkv",
+    "model.visual.blocks.17.attn.proj",
+    "model.visual.blocks.17.mlp.linear_fc1",
+    "model.visual.blocks.17.mlp.linear_fc2",
+    "model.visual.blocks.18.attn.qkv",
+    "model.visual.blocks.18.attn.proj",
+    "model.visual.blocks.18.mlp.linear_fc1",
+    "model.visual.blocks.18.mlp.linear_fc2",
+    "model.visual.blocks.19.attn.qkv",
+    "model.visual.blocks.19.attn.proj",
+    "model.visual.blocks.19.mlp.linear_fc1",
+    "model.visual.blocks.19.mlp.linear_fc2",
+    "model.visual.blocks.20.attn.qkv",
+    "model.visual.blocks.20.attn.proj",
+    "model.visual.blocks.20.mlp.linear_fc1",
+    "model.visual.blocks.20.mlp.linear_fc2",
+    "model.visual.blocks.21.attn.qkv",
+    "model.visual.blocks.21.attn.proj",
+    "model.visual.blocks.21.mlp.linear_fc1",
+    "model.visual.blocks.21.mlp.linear_fc2",
+    "model.visual.blocks.22.attn.qkv",
+    "model.visual.blocks.22.attn.proj",
+    "model.visual.blocks.22.mlp.linear_fc1",
+    "model.visual.blocks.22.mlp.linear_fc2",
+    "model.visual.blocks.23.attn.qkv",
+    "model.visual.blocks.23.attn.proj",
+    "model.visual.blocks.23.mlp.linear_fc1",
+    "model.visual.blocks.23.mlp.linear_fc2",
+    "model.visual.blocks.24.attn.qkv",
+    "model.visual.blocks.24.attn.proj",
+    "model.visual.blocks.24.mlp.linear_fc1",
+    "model.visual.blocks.24.mlp.linear_fc2",
+    "model.visual.blocks.25.attn.qkv",
+    "model.visual.blocks.25.attn.proj",
+    "model.visual.blocks.25.mlp.linear_fc1",
+    "model.visual.blocks.25.mlp.linear_fc2",
+    "model.visual.blocks.26.attn.qkv",
+    "model.visual.blocks.26.attn.proj",
+    "model.visual.blocks.26.mlp.linear_fc1",
+    "model.visual.blocks.26.mlp.linear_fc2",
+    "model.visual.merger.linear_fc1",
+    "model.visual.merger.linear_fc2",
+    "model.language_model.layers.0.linear_attn.in_proj_b",
+    "model.language_model.layers.0.linear_attn.in_proj_a",
+    "model.language_model.layers.1.linear_attn.in_proj_b",
+    "model.language_model.layers.1.linear_attn.in_proj_a",
+    "model.language_model.layers.2.linear_attn.in_proj_b",
+    "model.language_model.layers.2.linear_attn.in_proj_a",
+    "model.language_model.layers.4.linear_attn.in_proj_b",
+    "model.language_model.layers.4.linear_attn.in_proj_a",
+    "model.language_model.layers.5.linear_attn.in_proj_b",
+    "model.language_model.layers.5.linear_attn.in_proj_a",
+    "model.language_model.layers.6.linear_attn.in_proj_b",
+    "model.language_model.layers.6.linear_attn.in_proj_a",
+    "model.language_model.layers.8.linear_attn.in_proj_b",
+    "model.language_model.layers.8.linear_attn.in_proj_a",
+    "model.language_model.layers.9.linear_attn.in_proj_b",
+    "model.language_model.layers.9.linear_attn.in_proj_a",
+    "model.language_model.layers.10.linear_attn.in_proj_b",
+    "model.language_model.layers.10.linear_attn.in_proj_a",
+    "model.language_model.layers.12.linear_attn.in_proj_b",
+    "model.language_model.layers.12.linear_attn.in_proj_a",
+    "model.language_model.layers.13.linear_attn.in_proj_b",
+    "model.language_model.layers.13.linear_attn.in_proj_a",
+    "model.language_model.layers.14.linear_attn.in_proj_b",
+    "model.language_model.layers.14.linear_attn.in_proj_a",
+    "model.language_model.layers.16.linear_attn.in_proj_b",
+    "model.language_model.layers.16.linear_attn.in_proj_a",
+    "model.language_model.layers.17.linear_attn.in_proj_b",
+    "model.language_model.layers.17.linear_attn.in_proj_a",
+    "model.language_model.layers.18.linear_attn.in_proj_b",
+    "model.language_model.layers.18.linear_attn.in_proj_a",
+    "model.language_model.layers.20.linear_attn.in_proj_b",
+    "model.language_model.layers.20.linear_attn.in_proj_a",
+    "model.language_model.layers.21.linear_attn.in_proj_b",
+    "model.language_model.layers.21.linear_attn.in_proj_a",
+    "model.language_model.layers.22.linear_attn.in_proj_b",
+    "model.language_model.layers.22.linear_attn.in_proj_a",
+    "model.language_model.layers.24.linear_attn.in_proj_b",
+    "model.language_model.layers.24.linear_attn.in_proj_a",
+    "model.language_model.layers.25.linear_attn.in_proj_b",
+    "model.language_model.layers.25.linear_attn.in_proj_a",
+    "model.language_model.layers.26.linear_attn.in_proj_b",
+    "model.language_model.layers.26.linear_attn.in_proj_a",
+    "model.language_model.layers.28.linear_attn.in_proj_b",
+    "model.language_model.layers.28.linear_attn.in_proj_a",
+    "model.language_model.layers.29.linear_attn.in_proj_b",
+    "model.language_model.layers.29.linear_attn.in_proj_a",
+    "model.language_model.layers.30.linear_attn.in_proj_b",
+    "model.language_model.layers.30.linear_attn.in_proj_a",
+    "lm_head"
+  ],
+  "quantized_params": 6912212992,
+  "total_params_hf_loaded": 9412959856,
+  "fraction_params_quantized": 0.7343,
+  "quant_cfg": [
+    {
+      "quantizer_name": "*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*weight_quantizer",
+      "cfg": {
+        "num_bits": [
+          2,
+          1
+        ],
+        "effective_bits": 4.5,
+        "block_sizes": {
+          "-1": 16,
+          "type": "dynamic",
+          "scale_bits": [
+            4,
+            3
+          ]
+        }
+      }
+    },
+    {
+      "quantizer_name": "*input_quantizer",
+      "cfg": {
+        "num_bits": [
+          2,
+          1
+        ],
+        "effective_bits": 4.5,
+        "block_sizes": {
+          "-1": 16,
+          "type": "dynamic",
+          "scale_bits": [
+            4,
+            3
+          ]
+        }
+      }
+    },
+    {
+      "quantizer_name": "*gdn_state_quantizer",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*gdn_w_quantizer",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*block_sparse_moe.gate*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*linear_attn.conv1d*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*self_attention.conv1d*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*linear_attn.in_proj_a*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*linear_attn.in_proj_b*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*lm_head*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*mixer.conv1d*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*mlp.gate.*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*mlp.shared_expert_gate.*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*output_layer*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*proj_out.*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*router*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "mtp.*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "output.*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*embed_vision*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*vision_tower*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*visual*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*vision_model*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*multi_modal_projector*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*",
+      "parent_class": "nn.BatchNorm1d",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*",
+      "parent_class": "nn.BatchNorm2d",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*",
+      "parent_class": "nn.BatchNorm3d",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*",
+      "parent_class": "nn.LeakyReLU",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*",
+      "parent_class": "nn.Embedding",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*lm_head*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*visual*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*vision*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*mtp*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*in_proj_a*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*in_proj_b*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*mhc_*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*embed_tokens*",
+      "enable": false
+    },
+    {
+      "quantizer_name": "*conv1d*",
+      "enable": false
+    }
+  ],
+  "fakequant_logits_finite": true,
+  "export_seconds": 140.7
+}

quantization/requirements-quant.lock

@@ -0,0 +1,107 @@
+accelerate==1.15.0
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.5
+aiosignal==1.4.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anyio==4.15.1
+attrs==26.1.0
+certifi==2026.10.23
+charset-normalizer==3.5.2
+click==8.5.0
+cloudpickle==3.1.2
+cuda-bindings==13.4.4
+cuda-pathfinder==1.8.4
+cuda-toolkit==13.0.2
+datasets==5.1.0
+deepspeed==0.19.9
+defusedxml==0.7.1
+diffusers==0.42.1
+dill==0.4.1
+einops==0.8.2
+filelock==4.2.4
+frozenlist==1.8.0
+fsspec==2026.7.0
+h11==0.16.0
+hf-xet==1.8.0
+hjson==3.1.0
+httpcore==1.0.9
+httpcore2==2.16.0
+httpx==0.28.1
+httpx2==2.16.0
+huggingface-hub==2.4.0
+idna==3.20
+importlib-metadata==10.1.0
+jinja2==3.1.6
+joblib==1.6.0
+markdown-it-py==4.2.0
+markupsafe==3.0.3
+mdurl==0.1.2
+mpmath==1.3.0
+msgpack==1.2.4
+multidict==7.2.1
+multiprocess==0.70.19
+networkx==3.7
+ninja==1.13.2
+nltk==3.10.4
+numpy==2.5.4
+nvidia-cublas==13.1.0.3
+nvidia-cuda-cccl==13.3.4.3.1
+nvidia-cuda-cupti==13.0.85
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-runtime==13.0.96
+nvidia-cudnn-cu13==9.19.0.56
+nvidia-cufft==12.0.0.61
+nvidia-cufile==1.15.1.6
+nvidia-curand==10.4.0.35
+nvidia-cusolver==12.0.4.66
+nvidia-cusparse==12.6.3.3
+nvidia-cusparselt-cu13==0.8.0
+nvidia-ml-py==13.615.71
+nvidia-modelopt==0.48.0rc1
+nvidia-nccl-cu13==2.28.9
+nvidia-nvjitlink==13.0.88
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvtx==13.0.85
+omegaconf==2.4.0
+packaging==26.3
+pandas==3.1.0
+peft==0.21.2
+pillow==13.0.0
+propcache==0.5.4
+psutil==7.2.2
+pulp==3.3.2
+py-cpuinfo==9.0.0
+pyarrow==26.0.0
+pydantic==2.14.2
+pydantic-core==2.50.2
+pygments==2.21.0
+python-dateutil==3.0.0
+pyyaml==6.0.3
+regex==2026.9.29
+requests==2.34.2
+rich==15.0.0
+safetensors==0.8.0
+scipy==1.18.1
+sentencepiece==0.2.2
+setuptools==81.0.0
+shellingham==1.5.4
+six==1.17.0
+sympy==1.14.0
+tiktoken==0.14.0
+tokenizers==0.23.3
+torch==2.11.0+cu130
+torchvision==0.26.0+cu130
+tqdm==4.70.1
+transformers==5.18.0
+triton==3.6.0
+truststore==0.10.5
+typer==0.27.3
+typing-extensions==4.16.0
+typing-inspection==0.4.4
+tzdata==2026.6
+urllib3==2.8.0
+wonderwords==3.0.1
+xxhash==4.0.2
+yarl==1.25.1
+zipp==4.1.2