Sariel00 commited on 15 days ago

Commit

3706f1c

verified ·

1 Parent(s): 022e6e4

Keep MTP opt-in: default-off build, isolated experimental package and measurements

Add explicit experimental MTP compile gating and resident probe tools/results. Normal terminal/API inference does not initialize MTP. Keep final v6 model and portable binary unchanged; exclude MTP build artifacts.

Files changed (19) hide show

.gitignore +33 -0
CMakeLists.txt +2 -0
README.md +5 -0
docs/MTP_LATEST_20261002.md +168 -0
docs/README.md +1 -0
docs/measurements/mtp-latest-20261002/baseline-32k/metadata.json +64 -0
docs/measurements/mtp-latest-20261002/combined-32k/metadata.json +64 -0
docs/measurements/mtp-latest-20261002/ling3-tiny-v6-mtp-experimental.manifest.json +13 -0
docs/measurements/mtp-latest-20261002/summary.json +200 -0
include/ling3/decoder.h +8 -0
include/ling3/precision_policy.h +11 -1
models/local-experimental/mtp/README.md +15 -0
models/local-experimental/mtp/ling3-tiny-v6-mtp-experimental.manifest.json +13 -0
src/decoder.cpp +131 -1
src/main.cpp +134 -0
tests/data/mtp_probe_questions.txt +3 -0
tests/precision_policy_test.cpp +15 -0
tools/append_mtp_package.py +133 -0
tools/mtp_resident_benchmark.py +134 -0

.gitignore ADDED Viewed

	@@ -0,0 +1,33 @@

+# Build and local caches
+build/
+build-mtp*/
+engine/source/build/
+**/__pycache__/
+*.pyc
+# Runtime and benchmark output
+*.log
+*.jsonl
+board-test-result.json
+*-result.json
+artifacts/benchmarks/
+docs/measurements/**/*.f32
+# Large model/checkpoint/release files; publish through Release or model hosting
+models/**/*.safetensors
+models/**/*.l3r
+models/**/*.gguf
+models/local-final/**/*.l3r
+models/local-final/**/mindnano-infer
+models/local-experimental/**/ling3-rknn
+releases/
+# The final model is published separately through Hugging Face / Git LFS.
+!models/local-final/
+!models/local-final/v6-board-test/
+!models/local-final/v6-board-test/ling3-tiny-w4.l3r
+models/local-final/v6-board-test/mindnano-infer
+# Generated quantization payloads
+calibration/assets/quantized-assets/
+work/

CMakeLists.txt CHANGED Viewed

@@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 3.20)
 project(ling3_rknn VERSION 0.1.0 LANGUAGES CXX)
 option(LING3_WITH_RKNN "Build the direct RKNN backend" OFF)
 option(LING3_TUNE_A76 "Tune release builds for Cortex-A76" ON)
 option(LING3_WITH_ICU "Use ICU for exact tokenizer normalization and splitting" ON)
 set(RKNN_ROOT "" CACHE PATH "Local RKNPU2 root from https://github.com/airockchip/rknpu2 (contains include/ and lib/)")
@@ -28,6 +29,7 @@ target_include_directories(ling3_core PUBLIC include)
 target_compile_features(ling3_core PUBLIC cxx_std_20)
 target_compile_options(ling3_core PRIVATE -Wall -Wextra -Wpedantic)
 target_compile_definitions(ling3_core PUBLIC LING3_WITH_RKNN=$<BOOL:${LING3_WITH_RKNN}>)
 if(LING3_WITH_ICU)
     find_package(ICU COMPONENTS uc i18n QUIET)

 project(ling3_rknn VERSION 0.1.0 LANGUAGES CXX)
 option(LING3_WITH_RKNN "Build the direct RKNN backend" OFF)
+option(LING3_EXPERIMENTAL_MTP "Build experimental NEXTN/MTP probes (not production decoding)" OFF)
 option(LING3_TUNE_A76 "Tune release builds for Cortex-A76" ON)
 option(LING3_WITH_ICU "Use ICU for exact tokenizer normalization and splitting" ON)
 set(RKNN_ROOT "" CACHE PATH "Local RKNPU2 root from https://github.com/airockchip/rknpu2 (contains include/ and lib/)")
 target_compile_features(ling3_core PUBLIC cxx_std_20)
 target_compile_options(ling3_core PRIVATE -Wall -Wextra -Wpedantic)
 target_compile_definitions(ling3_core PUBLIC LING3_WITH_RKNN=$<BOOL:${LING3_WITH_RKNN}>)
+target_compile_definitions(ling3_core PUBLIC LING3_EXPERIMENTAL_MTP=$<BOOL:${LING3_EXPERIMENTAL_MTP}>)
 if(LING3_WITH_ICU)
     find_package(ICU COMPONENTS uc i18n QUIET)

README.md CHANGED Viewed

@@ -70,6 +70,8 @@ Ling-3.0-rknn 是在 RK3588/RK3588S 上的专用 C++ 推理工程。
 当前限制：一次只执行一个推理请求，忙时可能返回 429；接口没有用户鉴权和 TLS；KV 缓存是进程内资源，不是跨重启的持久聊天记录；128K/256K 只提供容量选项，尚未完成完整板端质量和性能认证；MTP 未启用。
 ## 板端性能
 下表来自 NanoPi M6（RK3588、16GB）2026-09-22 实测。CPU、NPU、DMC 测试期间固定最高频率；模型为 `ling3-tiny-w4.l3r`，BF16 KV，32K 初始化，关闭 thinking，temperature=0，每次冷启动会话输入后生成 64 tokens，未命中 KV 缓存。TTFT 不包括服务启动时间。
@@ -86,6 +88,8 @@ Ling-3.0-rknn 是在 RK3588/RK3588S 上的专用 C++ 推理工程。
 TTFT 随输入长度增加，解码速度也会因每个新 token 需要访问更长的历史 KV 而下降；因此 128/64 的结果不能代表长上下文表现。完整原始结果和测试环境见 [BOARD_BENCHMARK_AND_INT4_20260922.md](docs/BOARD_BENCHMARK_AND_INT4_20260922.md)。频率、散热和后台负载改变时应重新测量。
 ## 量化与数值损失
 当前发布模型不是全 W4：部分线性层和专家使用 W4A8，其他指定投影使用 W8A8。默认 KV 为 BF16，KDA 状态保留 FP32。量化存在损失，以下是固定 teacher-forcing 序列、4,322 个预测位置、完整 157,184 词表，以 BF16 为参考的 logits 分布指标：
@@ -137,6 +141,7 @@ python3 tools/pack_model.py \
 cmake -S . -B build \
   -DCMAKE_BUILD_TYPE=Release \
   -DLING3_WITH_RKNN=ON \
   -DRKNN_ROOT=./vendor/rknpu2
 cmake --build build -j4 --target mindnano-infer ling3-rknn
 ```

 当前限制：一次只执行一个推理请求，忙时可能返回 429；接口没有用户鉴权和 TLS；KV 缓存是进程内资源，不是跨重启的持久聊天记录；128K/256K 只提供容量选项，尚未完成完整板端质量和性能认证；MTP 未启用。
+MTP 仅作为实验保留，编译开关为 `LING3_EXPERIMENTAL_MTP`，默认 `OFF`。默认构建不包含 MTP 前向实现、实验 API 和 `mtp-benchmark` 命令，不初始化 MTP 权重或 KV。即使显式编译为 `ON`，终端对话和 OpenAI API 也不自动启用 MTP；只有实验探针调用 `EnableMtp()` 后才初始化。实验模型不加入正常部署流程，启用方法见 [MTP 实验说明](docs/MTP_LATEST_20261002.md)。
 ## 板端性能
 下表来自 NanoPi M6（RK3588、16GB）2026-09-22 实测。CPU、NPU、DMC 测试期间固定最高频率；模型为 `ling3-tiny-w4.l3r`，BF16 KV，32K 初始化，关闭 thinking，temperature=0，每次冷启动会话输入后生成 64 tokens，未命中 KV 缓存。TTFT 不包括服务启动时间。
 TTFT 随输入长度增加，解码速度也会因每个新 token 需要访问更长的历史 KV 而下降；因此 128/64 的结果不能代表长上下文表现。完整原始结果和测试环境见 [BOARD_BENCHMARK_AND_INT4_20260922.md](docs/BOARD_BENCHMARK_AND_INT4_20260922.md)。频率、散热和后台负载改变时应重新测量。
+2026-10-02 保留最新混合模型＋MTP 的[实验驻留前向测试](docs/MTP_LATEST_20261002.md)：统一 32K 初始化，主模型约 18.69 token/s；MTP 每步追加约 10.54 ms，驻留 RSS 增加约 497 MiB。四条不同序列的下一 token Top1 命中为 17/252，尚未证明投机加速。这些 MTP 指标只适用于显式启用的实验探针，不是默认引擎的开销。实验模型位于 `models/local-experimental/mtp/`，正式 v6 模型、便携发布二进制和默认服务未替换。
 ## 量化与数值损失
 当前发布模型不是全 W4：部分线性层和专家使用 W4A8，其他指定投影使用 W8A8。默认 KV 为 BF16，KDA 状态保留 FP32。量化存在损失，以下是固定 teacher-forcing 序列、4,322 个预测位置、完整 157,184 词表，以 BF16 为参考的 logits 分布指标：
 cmake -S . -B build \
   -DCMAKE_BUILD_TYPE=Release \
   -DLING3_WITH_RKNN=ON \
+  -DLING3_EXPERIMENTAL_MTP=OFF \
   -DRKNN_ROOT=./vendor/rknpu2
 cmake --build build -j4 --target mindnano-infer ling3-rknn
 ```

docs/MTP_LATEST_20261002.md ADDED Viewed

	@@ -0,0 +1,168 @@

+# 最新混合量化模型与 MTP 组合测试
+2026-10-02，在 NanoPi M6 / RK3588 / 16GB（`10.1.20.33`）测试。
+结论：最新主模型与 MTP 可以一起加载并连续前向，未改变本组主模型生成
+token；但预测命中偏低，尚未证明投机解码收益。
+## 实验开关（默认关闭）
+`LING3_EXPERIMENTAL_MTP=OFF` 是 CMake 默认值。关闭时，MTP 层、状态、
+前向函数、实验 API 和 `mtp-benchmark` 命令均不编译；正常推理只运行原主干，
+不初始化 MTP 权重和 KV。追加了 MTP 的模型文件不会自动激活它。
+开启编译也不会把 MTP 加入终端或 OpenAI API 对话流程。`Decoder` 构造函数
+不再自动加载 sidecar，只有实验探针显式调用 `EnableMtp()` 才会初始化。
+现有正常生成、KV 复用和服务流程没有调用此接口。
+以下性能和内存数据为之前显式运行实验探针的记录，不代表默认构建开销。
+实验模型保留在 `models/local-experimental/mtp/`，不进入正式 v6 发布包。
+本地及板端的独立 MTP 编译目录、旧实验二进制已清理；源码、模型和测试记录
+保留，可按下方命令重新构建。当前源码复现实验时必须单独开启编译开关；默认二进制的
+帮助信息不包含 `mtp-benchmark`，测试脚本会在锁频前提示编译开关未开启。
+## 版本与口径
+主干使用本工作区 `models/local-final/v6-board-test/ling3-tiny-w4.l3r`，
+SHA256 为 `80f0086247328fe77b2f939df460b4955287b6f75f6aaada85512c56e652f435`。
+它是混合 W4A8/W8A8 包，包含 91 个 BF16 源线性权重张量，在运行时走 W8A8；
+不是上一轮 `minimind-o/Ling3RKNN` 的旧 W4 实验主干。
+MTP 来自 `inclusionAI/Ling-3.0-tiny-base` 的 layer 24，使用此前已转换的
+W4A8 资产。仅追加 798 个 MTP 张量，原主干 18,029 个张量保留原数据并逐项
+验证 SHA256；不重新量化主模型。MTP 使用主模型的共享 W8A8 输出头。
+组合包：`models/local-experimental/mtp/ling3-tiny-v6-mtp-experimental.l3r`，
+5,188,395,008 bytes，约 4.832 GiB。
+SHA256 为 `edfc670a0d27d9e5db1a49e18584adb1f47af1120b2060cfa5042ec9332ce8a9`。
+正式 v6 模型和便携发布二进制未替换。
+使用本工作区根目录源码的实验 `HasMtp` / `EvalMtp` 与 `mtp-benchmark`，
+在板端重新编译。该源码与 README 中 2026-09-22 的 `engine/source/`
+历史性能测试版本不同，不把新旧结果视为同成本优化对比。
+两个模型均初始化 **32K 容量**，实际输入为 128、44、42、39 token，
+输出均为 64 token；这是短请求测试，不是 32K 实际输入测试。每题在同一个
+驻留进程中预热一次，再测三轮；切换题目时重置对话状态，保留权重和 RKNN
+context。固定中文性能题、故事题、办公题、解释题均关闭 thinking，贪心解码。
+CPU 大核 2.304/2.352 GHz、NPU 1 GHz、DMC 2.4 GHz，固定最高频率。
+启用专家预加载、三核平衡调度、共享输入、SIMD/vector math。
+MLA attention 使用 CPU，GDN 使用 FP32 CPU 路径；线性层和专家矩阵使用 NPU。
+采样未发现降频，最高温度 58.23°C；推理进程未使用 swap，测试后频率已恢复。
+## 连续速度与内存
+速度取预热后三轮合计解码时间，均计入首 token 后的 63 次计算。
+| 项目 | 最新主模型 | 加载 MTP、只运行主模型 | 主模型逐步追加 MTP |
+|---|---:|---:|---:|
+| 四题合计解码速度 | 18.69 token/s | 18.71 token/s | 15.54 token/s |
+| 固定 128/64 题速度 | 18.43 token/s | 18.56 token/s | 15.34 token/s |
+| 主模型每步平均 | 53.35 ms | 53.32 ms | 53.54 ms |
+| MTP 每步平均 | — | — | 10.54 ms |
+| MTP 层 / 输出头 | — | — | 2.03 / 8.51 ms |
+| 预热后进程 RSS | 6,516～6,524 MiB | 7,012～7,021 MiB | 7,012～7,021 MiB |
+| 进程峰值 RSS | 6,952 MiB | 7,021 MiB | 7,021 MiB |
+RSS 约从 **6.37 GiB 增至 6.86 GiB**，同题稳定态增量约 **497 MiB**。
+MTP 增加一个 MLA 层，32K BF16 KV 本体增加 320 MiB；其他增量包括专家与
+投影权重、RKNN context 和工作内存。共享输出头没有另复制一整套权重。
+峰值还包含构造期间临时映射，不能直接用两次冷启动峰值差计算 MTP 的内存增量。
+RSS 包含 NPU 共享映射，是进程统计，不等于整个系统内存差值。
+输出头约占 MTP 前向时间的 **81%**。上一轮旧 W4 主干的 MTP 为 6.85 ms、
+输出头 4.76 ms；最新模型使用共享 W8A8 输出头，不能沿用旧包的成本估算。
+本探针逐 token 填满 MTP 缓存：固定 128-token 题的主模型填充约 6.8 秒，
+追加 MTP 后约 8.1 秒。**这不是正式批量 prefill 的 TTFT**，不能与 README
+的 753 ms 直接比较。
+## 预测与主模型一致性
+| 问题 | 输入 token | MTP 下一 token Top1 命中 |
+|---|---:|---:|
+| 固定工作安排性能题 | 128 | 3/63，4.76% |
+| 中文故事 | 44 | 6/63，9.52% |
+| 项目会议议程 | 42 | 3/63，4.76% |
+| 长上下文原理解释 | 39 | 5/63，7.94% |
+| 四条不同序列合计 | — | **17/252，6.75%** |
+每题三轮相同，重复轮次用于测速度，不能当作更多独立质量样本。
+加入 MTP 后每轮主模型 Top1 均为 64/64 一致；四题的生成 token hash
+也与不含 MTP 的原模型一致。这只证明本组生成序列一致，没有完整 logits
+数值等价认证，也不是通用问答正确率。
+目前测的是使用真实主干 hidden 和正确历史的单步 NEXTN；**不是连续预测
+四个候选后的实际投机接受率**。主模型批量验证、接受前缀提交、拒绝回滚及
+MTP 会话状态恢复尚未实现。直接顺序追加 MTP 会降低速度，当前命中表现
+也不足以支持“预测四个、接受两个”的假设。
+下一步应先对照桌面参考定位预测偏差，包括 MTP 量化损失、主干与 sidecar
+兼容性、位置和缓存对齐。已核对上游 HF 实现确实传入主模型最终归一化后的
+hidden；不应在没有数值对照时擅自去掉这一归一化。先修预测质量，再做完整
+投机验证和输出头优化。
+## 使用与复现
+在 RK3588 上显式开启实验编译，使用独立构建目录：
+```bash
+cmake -S . -B build-mtp -DCMAKE_BUILD_TYPE=Release \
+  -DLING3_WITH_RKNN=ON -DRKNN_ROOT=./vendor/rknpu2 \
+  -DLING3_EXPERIMENTAL_MTP=ON
+cmake --build build-mtp -j4 --target ling3-rknn
+```
+普通发布使用 README 中的默认 `OFF` 配置。CLI 参数示例如下，依赖匹配的
+RKNN/ICU 用户态运行库；定量复现请使用后面的脚本设置优化环境。正式便携
+二进制尚未加入这个实验 CLI。模型头中的 4096 是原包默认容量，本测试显式
+传入 32768 初始化，不代表实际输入被限制在 4096。
+```bash
+./build-mtp/ling3-rknn mtp-benchmark \
+  models/local-experimental/mtp/ling3-tiny-v6-mtp-experimental.l3r \
+  128 64 3 32768 tests/data/mtp_probe_questions.txt
+```
+`tools/mtp_resident_benchmark.py` 会设置优化环境、临时锁频、采样 CPU/NPU、
+内存及温度，正常结束和 SIGINT/SIGTERM 时恢复原设置：
+```bash
+sudo python3 tools/mtp_resident_benchmark.py \
+  ./build-mtp/ling3-rknn \
+  models/local-experimental/mtp/ling3-tiny-v6-mtp-experimental.l3r \
+  --runtime /path/to/rknn/lib --output ./results/mtp-combined-new \
+  --context 32768 --seqlen 128 --tokens 64 --rounds 3 \
+  --cases tests/data/mtp_probe_questions.txt
+```
+对照时将模型路径换成 `models/local-final/v6-board-test/ling3-tiny-w4.l3r`，
+使用新的输出目录。脚本拒绝覆盖旧结果，一次只运行一个模型进程。
+`tools/append_mtp_package.py` 可从含量化 MTP 张量的 `.l3r` 追加 layer 24，
+不要求原始主模型或重新量化主干：
+```bash
+python3 tools/append_mtp_package.py \
+  --base models/local-final/v6-board-test/ling3-tiny-w4.l3r \
+  --mtp-package /path/to/package-containing-layer24.l3r \
+  --output ./work/ling3-tiny-v6-mtp-new.l3r
+```
+MTP 缓存目前要求连续填满。跳过前缀、重复提交同一个位置、只恢复主模型
+缓存后直接调用 MTP 都会被接口检查拒绝；不能把主模型状态复用直接视为
+MTP 状态复用。
+主机侧 8 项机制测试通过，包含新增的 MTP 精度族分类检查。板端两组测试
+均 exit 0，本地生成的组合模型 SHA256 与板端一致。
+整理实验开关后，主机侧默认 `OFF` 与显式 `ON` 两种构建分别通过全部
+8 项测试；检查默认静态库不含 `EnableMtp`、`HasMtp`、`EvalMtp` 导出符号，
+默认 CLI 不含 `mtp-benchmark`，禁用的实验命令不会打开模型文件。此轮没有
+重跑板端性能测试，上面的数值仍是此前保存的实测记录。
+[结果摘要](measurements/mtp-latest-20261002/summary.json)与
+[合包审计](measurements/mtp-latest-20261002/ling3-tiny-v6-mtp-experimental.manifest.json)
+保存在工作区。完整前向日志、频率元数据和占用采样保留在同目录的
+`baseline-32k/`、`combined-32k/`，以及板端
+`/home/serial/ling3-latest-mtp-20261002/results/`。

docs/README.md CHANGED Viewed

@@ -4,6 +4,7 @@
 ## 核心报告
 - [BOARD_BENCHMARK_AND_INT4_20260922.md](BOARD_BENCHMARK_AND_INT4_20260922.md)：当前板端 128/64 至 16K/64 性能与官方 INT4 数值对比。
 - [INT4_BOARD_ACCURACY_20260922.md](INT4_BOARD_ACCURACY_20260922.md)：固定 teacher-forcing 条件下的 logits、概率分布和 Top1 一致率。
 - [KV_CACHE_BENCHMARK_20260922.md](KV_CACHE_BENCHMARK_20260922.md)：BF16/INT8 KV 的容量、内存和长输入性能。

 ## 核心报告
+- [MTP_LATEST_20261002.md](MTP_LATEST_20261002.md)：默认不编译的 MTP 实验开关，以及统一 32K 容量的驻留速度、内存和预测命中测试。
 - [BOARD_BENCHMARK_AND_INT4_20260922.md](BOARD_BENCHMARK_AND_INT4_20260922.md)：当前板端 128/64 至 16K/64 性能与官方 INT4 数值对比。
 - [INT4_BOARD_ACCURACY_20260922.md](INT4_BOARD_ACCURACY_20260922.md)：固定 teacher-forcing 条件下的 logits、概率分布和 Top1 一致率。
 - [KV_CACHE_BENCHMARK_20260922.md](KV_CACHE_BENCHMARK_20260922.md)：BF16/INT8 KV 的容量、内存和长输入性能。

docs/measurements/mtp-latest-20261002/baseline-32k/metadata.json ADDED Viewed

	@@ -0,0 +1,64 @@

+{
+  "started_utc": "2026-10-02T14:32:07.142908+00:00",
+  "command": [
+    "/home/serial/ling3-latest-mtp-20261002/build/ling3-rknn",
+    "mtp-benchmark",
+    "/home/serial/ling3-v6-workspace-test/ling3-tiny-w4.l3r",
+    "128",
+    "64",
+    "3",
+    "32768",
+    "/home/serial/ling3-latest-mtp-20261002/source/tests/data/mtp_probe_questions.txt"
+  ],
+  "original_clocks": {
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor": "ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_min_freq": "1800000",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_governor": "ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_min_freq": "408000",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_governor": "ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_min_freq": "408000",
+    "/sys/class/devfreq/fdab0000.npu/governor": "rknpu_ondemand",
+    "/sys/class/devfreq/fdab0000.npu/min_freq": "300000000",
+    "/sys/class/devfreq/dmc/governor": "dmc_ondemand",
+    "/sys/class/devfreq/dmc/min_freq": "534000000"
+  },
+  "meminfo_before": "MemTotal:       16342628 kB\nMemFree:          260916 kB\nMemAvailable:   14314584 kB\nBuffers:           87808 kB\nCached:         13925596 kB\nSwapCached:            0 kB\nActive:          7125980 kB\nInactive:        7658296 kB\nActive(anon):      23304 kB\nInactive(anon):   830208 kB\nActive(file):    7102676 kB\nInactive(file):  6828088 kB\nUnevictable:       14400 kB\nMlocked:               0 kB\nSwapTotal:       8171312 kB\nSwapFree:        8119344 kB\nZswap:                 0 kB\nZswapped:              0 kB\nDirty:               316 kB\nWriteback:             0 kB\nAnonPages:        785292 kB\nMapped:           276836 kB\nShmem:             82640 kB\nKReclaimable:     383648 kB\nSlab:             541800 kB\nSReclaimable:     383648 kB\nSUnreclaim:       158152 kB\nKernelStack:        9184 kB\nPageTables:        18632 kB\nSecPageTables:         0 kB\nNFS_Unstable:          0 kB\nBounce:                0 kB\nWritebackTmp:          0 kB\nCommitLimit:    16342624 kB\nCommitted_AS:    4934684 kB\nVmallocTotal:   133143592960 kB\nVmallocUsed:      104164 kB\nVmallocChunk:          0 kB\nPercpu:             7776 kB\nCmaTotal:         786432 kB\nCmaAllocated:       8780 kB\nCmaReleased:      777652 kB\nCmaFree:             588 kB\nHugePages_Total:       0\nHugePages_Free:        0\nHugePages_Rsvd:        0\nHugePages_Surp:        0\nHugepagesize:       2048 kB\nHugetlb:               0 kB",
+  "applied_clocks": {
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor": "performance",
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_min_freq": "1800000",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_governor": "performance",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_min_freq": "2304000",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_governor": "performance",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_min_freq": "2352000",
+    "/sys/class/devfreq/fdab0000.npu/governor": "performance",
+    "/sys/class/devfreq/fdab0000.npu/min_freq": "1000000000",
+    "/sys/class/devfreq/dmc/governor": "performance",
+    "/sys/class/devfreq/dmc/min_freq": "2400000000"
+  },
+  "profile": {
+    "LING3_PREWARM_EXPERTS": "1",
+    "LING3_EXPERT_BALANCED": "1",
+    "LING3_EXPERT_ZERO_COPY": "1",
+    "LING3_GDN_CPU_PREFILL": "1",
+    "LING3_GDN_CPU_FP32_STATE": "1",
+    "LING3_GDN_CPU_DECODE": "1",
+    "LING3_GDN_FULL_FP32": "1",
+    "LING3_MLA_SIMD": "1",
+    "LING3_VECTOR_MATH": "1",
+    "LING3_MLA_BACKEND": "cpu"
+  },
+  "exit_code": 0,
+  "elapsed_s": 138.48842622793745,
+  "restored_clocks": {
+    "/sys/class/devfreq/dmc/min_freq": "534000000",
+    "/sys/class/devfreq/dmc/governor": "dmc_ondemand",
+    "/sys/class/devfreq/fdab0000.npu/min_freq": "300000000",
+    "/sys/class/devfreq/fdab0000.npu/governor": "rknpu_ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_min_freq": "408000",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_governor": "ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_min_freq": "408000",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_governor": "ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_min_freq": "1800000",
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor": "ondemand"
+  }
+}

docs/measurements/mtp-latest-20261002/combined-32k/metadata.json ADDED Viewed

	@@ -0,0 +1,64 @@

+{
+  "started_utc": "2026-10-02T14:35:58.845242+00:00",
+  "command": [
+    "/home/serial/ling3-latest-mtp-20261002/build/ling3-rknn",
+    "mtp-benchmark",
+    "/home/serial/ling3-latest-mtp-20261002/artifacts/ling3-tiny-v6-mtp-experimental.l3r",
+    "128",
+    "64",
+    "3",
+    "32768",
+    "/home/serial/ling3-latest-mtp-20261002/source/tests/data/mtp_probe_questions.txt"
+  ],
+  "original_clocks": {
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor": "ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_min_freq": "1800000",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_governor": "ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_min_freq": "408000",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_governor": "ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_min_freq": "408000",
+    "/sys/class/devfreq/fdab0000.npu/governor": "rknpu_ondemand",
+    "/sys/class/devfreq/fdab0000.npu/min_freq": "300000000",
+    "/sys/class/devfreq/dmc/governor": "dmc_ondemand",
+    "/sys/class/devfreq/dmc/min_freq": "534000000"
+  },
+  "meminfo_before": "MemTotal:       16342628 kB\nMemFree:         7260096 kB\nMemAvailable:   14319844 kB\nBuffers:           52036 kB\nCached:          7005492 kB\nSwapCached:            0 kB\nActive:          3938108 kB\nInactive:        3889344 kB\nActive(anon):      28320 kB\nInactive(anon):   830228 kB\nActive(file):    3909788 kB\nInactive(file):  3059116 kB\nUnevictable:       14400 kB\nMlocked:               0 kB\nSwapTotal:       8171312 kB\nSwapFree:        8113968 kB\nZswap:                 0 kB\nZswapped:              0 kB\nDirty:                 0 kB\nWriteback:             0 kB\nAnonPages:        784344 kB\nMapped:           232044 kB\nShmem:             88876 kB\nKReclaimable:     351588 kB\nSlab:             506672 kB\nSReclaimable:     351588 kB\nSUnreclaim:       155084 kB\nKernelStack:        9104 kB\nPageTables:        18624 kB\nSecPageTables:         0 kB\nNFS_Unstable:          0 kB\nBounce:                0 kB\nWritebackTmp:          0 kB\nCommitLimit:    16342624 kB\nCommitted_AS:    4947628 kB\nVmallocTotal:   133143592960 kB\nVmallocUsed:      104052 kB\nVmallocChunk:          0 kB\nPercpu:             7712 kB\nCmaTotal:         786432 kB\nCmaAllocated:       8780 kB\nCmaReleased:      777652 kB\nCmaFree:          402076 kB\nHugePages_Total:       0\nHugePages_Free:        0\nHugePages_Rsvd:        0\nHugePages_Surp:        0\nHugepagesize:       2048 kB\nHugetlb:               0 kB",
+  "applied_clocks": {
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor": "performance",
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_min_freq": "1800000",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_governor": "performance",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_min_freq": "2304000",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_governor": "performance",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_min_freq": "2352000",
+    "/sys/class/devfreq/fdab0000.npu/governor": "performance",
+    "/sys/class/devfreq/fdab0000.npu/min_freq": "1000000000",
+    "/sys/class/devfreq/dmc/governor": "performance",
+    "/sys/class/devfreq/dmc/min_freq": "2400000000"
+  },
+  "profile": {
+    "LING3_PREWARM_EXPERTS": "1",
+    "LING3_EXPERT_BALANCED": "1",
+    "LING3_EXPERT_ZERO_COPY": "1",
+    "LING3_GDN_CPU_PREFILL": "1",
+    "LING3_GDN_CPU_FP32_STATE": "1",
+    "LING3_GDN_CPU_DECODE": "1",
+    "LING3_GDN_FULL_FP32": "1",
+    "LING3_MLA_SIMD": "1",
+    "LING3_VECTOR_MATH": "1",
+    "LING3_MLA_BACKEND": "cpu"
+  },
+  "exit_code": 0,
+  "elapsed_s": 262.595832146937,
+  "restored_clocks": {
+    "/sys/class/devfreq/dmc/min_freq": "534000000",
+    "/sys/class/devfreq/dmc/governor": "dmc_ondemand",
+    "/sys/class/devfreq/fdab0000.npu/min_freq": "300000000",
+    "/sys/class/devfreq/fdab0000.npu/governor": "rknpu_ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_min_freq": "408000",
+    "/sys/devices/system/cpu/cpufreq/policy6/scaling_governor": "ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_min_freq": "408000",
+    "/sys/devices/system/cpu/cpufreq/policy4/scaling_governor": "ondemand",
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_min_freq": "1800000",
+    "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor": "ondemand"
+  }
+}

docs/measurements/mtp-latest-20261002/ling3-tiny-v6-mtp-experimental.manifest.json ADDED Viewed

	@@ -0,0 +1,13 @@

+{
+  "base": "ling3-tiny-w4.l3r",
+  "base_sha256": "80f0086247328fe77b2f939df460b4955287b6f75f6aaada85512c56e652f435",
+  "mtp_package": "ling3-tiny-w4-mtp-experimental.l3r",
+  "mtp_source_sha256": "27911ca182d5a16d3c8e80e069e60a893ceb77f6ddfa3ac3da4182e5ed2e0485",
+  "output": "ling3-tiny-v6-mtp-experimental.l3r",
+  "output_sha256": "edfc670a0d27d9e5db1a49e18584adb1f47af1120b2060cfa5042ec9332ce8a9",
+  "bytes": 5188395008,
+  "flags": 513,
+  "trunk_tensors_unchanged": 18029,
+  "appended_mtp_tensors": 798,
+  "recomputed_quantization": false
+}

docs/measurements/mtp-latest-20261002/summary.json ADDED Viewed

	@@ -0,0 +1,200 @@

+{
+  "test": "resident NEXTN forward, not speculative decoding",
+  "context_capacity": 32768,
+  "new_tokens": 64,
+  "measured_rounds": 3,
+  "cases": [
+    {
+      "case": 0,
+      "label": "fixed_128_token_work_prompt",
+      "input_tokens": 128,
+      "baseline": {
+        "decode_wall_tokens_per_second": 18.429956132608858,
+        "target_mean_ms": 54.117866666666664,
+        "rss_mib_min": 6515.7969,
+        "rss_mib_max": 6515.7969,
+        "process_hwm_mib": 6952.4688
+      },
+      "combined_target_only": {
+        "decode_wall_tokens_per_second": 18.564440680217608,
+        "target_mean_ms": 53.7246,
+        "rss_mib_min": 7011.7422,
+        "rss_mib_max": 7011.7422,
+        "process_hwm_mib": 7011.7422
+      },
+      "target_plus_mtp": {
+        "decode_wall_tokens_per_second": 15.343605430662127,
+        "target_mean_ms": 54.265033333333335,
+        "rss_mib_min": 7011.7422,
+        "rss_mib_max": 7011.7422,
+        "process_hwm_mib": 7011.7422,
+        "mtp_mean_ms": 10.624833333333333,
+        "mtp_layers_mean_ms": 2.1126666666666667,
+        "mtp_head_mean_ms": 8.5121,
+        "mtp_p95_ms": 10.869200000000001
+      },
+      "teacher_hash": "658009ec40b8a851",
+      "predictions_unchanged": true,
+      "draft_top1_hits": 3,
+      "draft_positions": 63
+    },
+    {
+      "case": 1,
+      "label": "story",
+      "input_tokens": 44,
+      "baseline": {
+        "decode_wall_tokens_per_second": 18.7953734859991,
+        "target_mean_ms": 53.06296666666667,
+        "rss_mib_min": 6519.8555,
+        "rss_mib_max": 6519.8555,
+        "process_hwm_mib": 6952.4688
+      },
+      "combined_target_only": {
+        "decode_wall_tokens_per_second": 18.82802610460991,
+        "target_mean_ms": 52.9705,
+        "rss_mib_min": 7016.4453,
+        "rss_mib_max": 7016.4453,
+        "process_hwm_mib": 7016.4453
+      },
+      "target_plus_mtp": {
+        "decode_wall_tokens_per_second": 15.614778103593295,
+        "target_mean_ms": 53.24966666666667,
+        "rss_mib_min": 7016.4453,
+        "rss_mib_max": 7016.4453,
+        "process_hwm_mib": 7016.4453,
+        "mtp_mean_ms": 10.5083,
+        "mtp_layers_mean_ms": 2.0058,
+        "mtp_head_mean_ms": 8.502466666666667,
+        "mtp_p95_ms": 10.743
+      },
+      "teacher_hash": "871f43b0902ff7a0",
+      "predictions_unchanged": true,
+      "draft_top1_hits": 6,
+      "draft_positions": 63
+    },
+    {
+      "case": 2,
+      "label": "office_meeting",
+      "input_tokens": 42,
+      "baseline": {
+        "decode_wall_tokens_per_second": 18.730715539100714,
+        "target_mean_ms": 53.2465,
+        "rss_mib_min": 6521.8945,
+        "rss_mib_max": 6521.8945,
+        "process_hwm_mib": 6952.4688
+      },
+      "combined_target_only": {
+        "decode_wall_tokens_per_second": 18.6898446583178,
+        "target_mean_ms": 53.36313333333333,
+        "rss_mib_min": 7018.6602,
+        "rss_mib_max": 7018.6602,
+        "process_hwm_mib": 7018.6602
+      },
+      "target_plus_mtp": {
+        "decode_wall_tokens_per_second": 15.613767923149531,
+        "target_mean_ms": 53.2464,
+        "rss_mib_min": 7018.6602,
+        "rss_mib_max": 7018.6602,
+        "process_hwm_mib": 7018.6602,
+        "mtp_mean_ms": 10.515466666666667,
+        "mtp_layers_mean_ms": 2.0067666666666666,
+        "mtp_head_mean_ms": 8.5087,
+        "mtp_p95_ms": 10.748233333333333
+      },
+      "teacher_hash": "34da8f0ff021a5e",
+      "predictions_unchanged": true,
+      "draft_top1_hits": 3,
+      "draft_positions": 63
+    },
+    {
+      "case": 3,
+      "label": "long_context_explanation",
+      "input_tokens": 39,
+      "baseline": {
+        "decode_wall_tokens_per_second": 18.826060646809655,
+        "target_mean_ms": 52.97593333333333,
+        "rss_mib_min": 6524.0742,
+        "rss_mib_max": 6524.0742,
+        "process_hwm_mib": 6952.4688
+      },
+      "combined_target_only": {
+        "decode_wall_tokens_per_second": 18.744271096191156,
+        "target_mean_ms": 53.207766666666664,
+        "rss_mib_min": 7020.9922,
+        "rss_mib_max": 7020.9922,
+        "process_hwm_mib": 7020.9922
+      },
+      "target_plus_mtp": {
+        "decode_wall_tokens_per_second": 15.580993868400766,
+        "target_mean_ms": 53.38976666666667,
+        "rss_mib_min": 7020.9922,
+        "rss_mib_max": 7020.9922,
+        "process_hwm_mib": 7020.9922,
+        "mtp_mean_ms": 10.506966666666667,
+        "mtp_layers_mean_ms": 2.0087,
+        "mtp_head_mean_ms": 8.498233333333333,
+        "mtp_p95_ms": 10.7371
+      },
+      "teacher_hash": "d5dafb8e2a4e1f3f",
+      "predictions_unchanged": true,
+      "draft_top1_hits": 5,
+      "draft_positions": 63
+    }
+  ],
+  "baseline-32k": {
+    "initialization_ms": 26056.0407,
+    "clocks_restored": true,
+    "maximum_temperature_C": 57.307,
+    "minimum_system_mem_available_mib": 7443.515625,
+    "maximum_process_swap_kib": 0
+  },
+  "combined-32k": {
+    "initialization_ms": 21645.1046,
+    "clocks_restored": true,
+    "maximum_temperature_C": 58.23,
+    "minimum_system_mem_available_mib": 6909.07421875,
+    "maximum_process_swap_kib": 0
+  },
+  "distinct_draft_positions": 252,
+  "distinct_draft_hits": 17,
+  "draft_top1_agreement": 0.06746031746031746,
+  "package": {
+    "base": "ling3-tiny-w4.l3r",
+    "base_sha256": "80f0086247328fe77b2f939df460b4955287b6f75f6aaada85512c56e652f435",
+    "mtp_package": "ling3-tiny-w4-mtp-experimental.l3r",
+    "mtp_source_sha256": "27911ca182d5a16d3c8e80e069e60a893ceb77f6ddfa3ac3da4182e5ed2e0485",
+    "output": "ling3-tiny-v6-mtp-experimental.l3r",
+    "output_sha256": "edfc670a0d27d9e5db1a49e18584adb1f47af1120b2060cfa5042ec9332ce8a9",
+    "bytes": 5188395008,
+    "flags": 513,
+    "trunk_tensors_unchanged": 18029,
+    "appended_mtp_tensors": 798,
+    "recomputed_quantization": false
+  },
+  "aggregate": {
+    "baseline-32k/target_only": {
+      "decode_wall_tokens_per_second": 18.69419444496819,
+      "target_mean_ms": 53.35081666666667,
+      "rss_mib_min": 6515.7969,
+      "rss_mib_max": 6524.0742,
+      "process_hwm_mib": 6952.4688
+    },
+    "combined-32k/target_only": {
+      "decode_wall_tokens_per_second": 18.70615503663274,
+      "target_mean_ms": 53.3165,
+      "rss_mib_min": 7011.7422,
+      "rss_mib_max": 7020.9922,
+      "process_hwm_mib": 7020.9922
+    },
+    "combined-32k/target_plus_mtp": {
+      "decode_wall_tokens_per_second": 15.537454676617868,
+      "target_mean_ms": 53.53771666666667,
+      "rss_mib_min": 7011.7422,
+      "rss_mib_max": 7020.9922,
+      "process_hwm_mib": 7020.9922,
+      "mtp_mean_ms": 10.538891666666666,
+      "mtp_layers_mean_ms": 2.0334833333333333,
+      "mtp_head_mean_ms": 8.505375
+    }
+  }
+}

include/ling3/decoder.h CHANGED Viewed

@@ -39,6 +39,14 @@ public:
     std::size_t RestoreState(const DecoderState & state);
     const std::string & StateSignature() const;
     DecodeTimings Eval(std::uint32_t token, std::span<float> logits);
     DecodeTimings EvalBatch(
         std::span<const std::uint32_t> tokens,
         std::span<float> logits);

     std::size_t RestoreState(const DecoderState & state);
     const std::string & StateSignature() const;
     DecodeTimings Eval(std::uint32_t token, std::span<float> logits);
+#if LING3_EXPERIMENTAL_MTP
+    // Only the explicit experiment initializes MTP; normal chat does not.
+    bool EnableMtp();
+    // Experimental NEXTN probe; requires a contiguous MTP prefix. Restored
+    // trunk states and batch prefill alone do not populate the MTP cache.
+    bool HasMtp() const noexcept;
+    DecodeTimings EvalMtp(std::uint32_t next_token, std::span<float> logits);
+#endif
     DecodeTimings EvalBatch(
         std::span<const std::uint32_t> tokens,
         std::span<float> logits);

include/ling3/precision_policy.h CHANGED Viewed

@@ -25,9 +25,19 @@ inline std::string W8MatrixFamily(const std::string& base) {
         constexpr std::string_view prefix="model.layers.";
         if(!base.starts_with(prefix))throw std::invalid_argument("invalid attention matrix name");
         int layer=-1;auto [end,ec]=std::from_chars(base.data()+prefix.size(),base.data()+base.size(),layer);
-        if(ec!=std::errc{}||end==base.data()+base.size()||*end!='.'||layer<0||layer>=24)
             throw std::invalid_argument("invalid attention layer");
         return std::string((layer+1)%4==0?"mla_":"kda_")+(base.ends_with(".o_proj")?"out":"in");
     }
     if(base.find(".mlp.shared_experts.")!=std::string::npos)return "shared";
     if(base=="model.layers.0.mlp.gate_up"||base=="model.layers.0.mlp.down_proj")return "dense";

         constexpr std::string_view prefix="model.layers.";
         if(!base.starts_with(prefix))throw std::invalid_argument("invalid attention matrix name");
         int layer=-1;auto [end,ec]=std::from_chars(base.data()+prefix.size(),base.data()+base.size(),layer);
+#if LING3_EXPERIMENTAL_MTP
+        constexpr int maximum_layer=24;
+#else
+        constexpr int maximum_layer=23;
+#endif
+        if(ec!=std::errc{}||end==base.data()+base.size()||*end!='.'||layer<0||layer>maximum_layer)
             throw std::invalid_argument("invalid attention layer");
+#if LING3_EXPERIMENTAL_MTP
+        // Optional layer 24 is MTP MLA, outside the main 24-layer trunk.
+        return std::string((layer==24||(layer+1)%4==0)?"mla_":"kda_")+(base.ends_with(".o_proj")?"out":"in");
+#else
         return std::string((layer+1)%4==0?"mla_":"kda_")+(base.ends_with(".o_proj")?"out":"in");
+#endif
     }
     if(base.find(".mlp.shared_experts.")!=std::string::npos)return "shared";
     if(base=="model.layers.0.mlp.gate_up"||base=="model.layers.0.mlp.down_proj")return "dense";

models/local-experimental/mtp/README.md ADDED Viewed

	@@ -0,0 +1,15 @@

+# MTP 实验模型
+本目录仅供 NEXTN/MTP 驻留前向测试，不属于默认部署流程。尚未实现完整的投机验证、接受前缀提交和拒绝回滚；当前测试未证明加速收益。
+组合模型为 `ling3-tiny-v6-mtp-experimental.l3r`（5,188,395,008 bytes）：
+- 主干来自正式 v6 混合 W4A8/W8A8 模型，原 18,029 个张量数据保持不变。
+- MTP 来自 [inclusionAI/Ling-3.0-tiny-base](https://huggingface.co/inclusionAI/Ling-3.0-tiny-base) 的 layer 24，追加 798 个转换张量。
+- SHA256：`edfc670a0d27d9e5db1a49e18584adb1f47af1120b2060cfa5042ec9332ce8a9`。
+本项目转换模型资产使用 [CC BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/)，许可边界见 [LICENSES.md](../../../LICENSES.md)。上游来源及其原有许可须保留。
+需要单独使用 `-DLING3_EXPERIMENTAL_MTP=ON` 编译；编译产物不随本目录发布。默认 `OFF` 不编译 MTP，终端和 OpenAI API 对话不会自动启用它。详细命令、内存、速度和预测命中见 [实验报告](../../../docs/MTP_LATEST_20261002.md)。
+此包包含完整主干，约 4.832 GiB。普通推理只需下载正式 v6 模型，无需下载本目录。

models/local-experimental/mtp/ling3-tiny-v6-mtp-experimental.manifest.json ADDED Viewed

	@@ -0,0 +1,13 @@

+{
+  "base": "ling3-tiny-w4.l3r",
+  "base_sha256": "80f0086247328fe77b2f939df460b4955287b6f75f6aaada85512c56e652f435",
+  "mtp_package": "ling3-tiny-w4-mtp-experimental.l3r",
+  "mtp_source_sha256": "27911ca182d5a16d3c8e80e069e60a893ceb77f6ddfa3ac3da4182e5ed2e0485",
+  "output": "ling3-tiny-v6-mtp-experimental.l3r",
+  "output_sha256": "edfc670a0d27d9e5db1a49e18584adb1f47af1120b2060cfa5042ec9332ce8a9",
+  "bytes": 5188395008,
+  "flags": 513,
+  "trunk_tensors_unchanged": 18029,
+  "appended_mtp_tensors": 798,
+  "recomputed_quantization": false
+}

src/decoder.cpp CHANGED Viewed

@@ -133,10 +133,19 @@ std::unique_ptr<Linear> MakeLinear(
         const auto begin = base.data() + layer_prefix.size();
         const auto end = base.data() + base.size();
         const auto [parsed_end, error] = std::from_chars(begin, end, layer);
-        if (error != std::errc {} || parsed_end == begin || layer < 0 || layer >= 24) {
             throw std::runtime_error("cannot derive W4 IOMMU domain from " + base);
         }
         iommu_domain_id = 2 + layer / 2;
     } else if (base == "lm_head") {
         iommu_domain_id = 14;
     }
@@ -1414,6 +1423,62 @@ private:
     bool reuse_batch_input_ = false;
 };
 class DecoderLayer {
 public:
     DecoderLayer(
@@ -1543,6 +1608,12 @@ struct Decoder::Impl {
     std::vector<float> final_norm;
     DecoderScratch scratch; // Declared before layers: outlives all borrowers.
     std::vector<std::unique_ptr<DecoderLayer>> layers;
     std::unique_ptr<Linear> lm_head;
     std::vector<float> hidden;
     std::vector<float> normalized;
@@ -1595,11 +1666,31 @@ struct Decoder::Impl {
         lm_head = MakeLinear(package, "lm_head");
     }
     void Reset() {
         for (auto & weak : checkpoints) if (auto checkpoint = weak.lock()) checkpoint->valid = false;
         checkpoints.clear();
         current_position = 0;
         for (auto & layer : layers) layer->Reset();
     }
     void InvalidateOverwrittenCheckpoints() {
@@ -1649,6 +1740,9 @@ struct Decoder::Impl {
             throw std::invalid_argument("stale or foreign decoder checkpoint");
         for (std::size_t i = 0; i < layers.size(); ++i) layers[i]->RestoreCheckpoint(checkpoint.layers[i]);
         current_position = checkpoint.position;
         return current_position;
     }
@@ -1678,6 +1772,9 @@ struct Decoder::Impl {
         lm_head->Run(normalized, logits);
         const auto end = Clock::now();
         ++current_position;
         return {
             Milliseconds(begin, layers_end),
             Milliseconds(layers_end, end),
@@ -1727,6 +1824,9 @@ struct Decoder::Impl {
         }
         const auto end = Clock::now();
         current_position += rows;
         return {
             Milliseconds(begin, layers_end),
             Milliseconds(layers_end, end),
@@ -1734,6 +1834,28 @@ struct Decoder::Impl {
         };
     }
     void PrepareBatch(std::size_t rows) {
         if (rows < 1 || rows > kMaxBatch) {
             throw std::invalid_argument("decoder batch rows must be in [1, 128]");
@@ -1756,6 +1878,14 @@ MlaBackendStats Decoder::AttentionStats() const { return impl_->scratch.mla.npu.
 DecodeTimings Decoder::Eval(std::uint32_t token, std::span<float> logits) {
     return impl_->Eval(token, logits);
 }
 DecodeTimings Decoder::EvalBatch(
     std::span<const std::uint32_t> tokens,
     std::span<float> logits) {

         const auto begin = base.data() + layer_prefix.size();
         const auto end = base.data() + base.size();
         const auto [parsed_end, error] = std::from_chars(begin, end, layer);
+#if LING3_EXPERIMENTAL_MTP
+        constexpr int maximum_layer = 24;
+#else
+        constexpr int maximum_layer = 23;
+#endif
+        if (error != std::errc {} || parsed_end == begin || layer < 0 || layer > maximum_layer) {
             throw std::runtime_error("cannot derive W4 IOMMU domain from " + base);
         }
+#if LING3_EXPERIMENTAL_MTP
+        iommu_domain_id = layer == 24 ? 15 : 2 + layer / 2;
+#else
         iommu_domain_id = 2 + layer / 2;
+#endif
     } else if (base == "lm_head") {
         iommu_domain_id = 14;
     }
     bool reuse_batch_input_ = false;
 };
+#if LING3_EXPERIMENTAL_MTP
+// Layer 24 from the optional Ling NEXTN/MTP sidecar.  It is deliberately
+// separate from DecoderLayer: the main trunk has KDA/MLA grouping and its
+// state/checkpoints, while MTP consumes the trunk's normalized hidden state
+// together with the embedding of the candidate token.
+class MtpLayer {
+public:
+    MtpLayer(const ModelPackage & package, std::size_t max_context, DecoderScratch & scratch)
+        : input_norm_(DecodeBf16(package.tensor("model.layers.24.input_layernorm.weight"), kHidden)),
+          post_norm_(DecodeBf16(package.tensor("model.layers.24.post_attention_layernorm.weight"), kHidden)),
+          embedding_norm_(DecodeBf16(package.tensor("model.layers.24.enorm.weight"), kHidden)),
+          hidden_norm_(DecodeBf16(package.tensor("model.layers.24.hnorm.weight"), kHidden)),
+          final_norm_(DecodeBf16(package.tensor("model.layers.24.final_layernorm.weight"), kHidden)),
+          embedding_hidden_(kHidden * 2), projected_(kHidden), residual_(kHidden),
+          normalized_(kHidden), attention_output_(kHidden), ffn_output_(kHidden),
+          eh_projection_(MakeLinear(package, "model.layers.24.eh_proj")),
+          attention_(std::make_unique<MlaAttention>(package, 24, max_context, scratch.mla)),
+          feed_forward_(std::make_unique<SparseFeedForward>(package, 24, scratch.sparse)) {}
+    void Reset() { attention_->Reset(); }
+    void Run(std::span<const float> token_embedding, std::span<const float> trunk_hidden,
+             std::size_t position, std::span<float> output) {
+        if (token_embedding.size() != kHidden || trunk_hidden.size() != kHidden ||
+            output.size() != kHidden) {
+            throw std::invalid_argument("MTP hidden shape mismatch");
+        }
+        RmsNorm(token_embedding.data(), embedding_norm_.data(), embedding_hidden_.data(), kHidden, kEpsilon);
+        RmsNorm(trunk_hidden.data(), hidden_norm_.data(), embedding_hidden_.data() + kHidden, kHidden, kEpsilon);
+        eh_projection_->Run(embedding_hidden_, projected_);
+        std::copy(projected_.begin(), projected_.end(), residual_.begin());
+        RmsNorm(residual_.data(), input_norm_.data(), normalized_.data(), kHidden, kEpsilon);
+        attention_->Run(normalized_, position, attention_output_);
+        for (int i = 0; i < kHidden; ++i) residual_[i] += attention_output_[i];
+        RmsNorm(residual_.data(), post_norm_.data(), normalized_.data(), kHidden, kEpsilon);
+        feed_forward_->Run(normalized_, ffn_output_);
+        for (int i = 0; i < kHidden; ++i) residual_[i] += ffn_output_[i];
+        RmsNorm(residual_.data(), final_norm_.data(), output.data(), kHidden, kEpsilon);
+    }
+    void PrepareBatch(std::size_t rows) {
+        eh_projection_->PrepareBatch(rows);
+        attention_->PrepareBatch(rows);
+        feed_forward_->PrepareBatch(rows);
+    }
+private:
+    std::vector<float> input_norm_, post_norm_, embedding_norm_, hidden_norm_, final_norm_;
+    std::vector<float> embedding_hidden_, projected_, residual_, normalized_, attention_output_, ffn_output_;
+    std::unique_ptr<Linear> eh_projection_;
+    std::unique_ptr<MlaAttention> attention_;
+    std::unique_ptr<SparseFeedForward> feed_forward_;
+};
+#endif
 class DecoderLayer {
 public:
     DecoderLayer(
     std::vector<float> final_norm;
     DecoderScratch scratch; // Declared before layers: outlives all borrowers.
     std::vector<std::unique_ptr<DecoderLayer>> layers;
+#if LING3_EXPERIMENTAL_MTP
+    std::unique_ptr<MtpLayer> mtp;
+    std::vector<float> mtp_hidden, mtp_embedding;
+    std::size_t mtp_position = 0;
+    bool mtp_trunk_ready = false;
+#endif
     std::unique_ptr<Linear> lm_head;
     std::vector<float> hidden;
     std::vector<float> normalized;
         lm_head = MakeLinear(package, "lm_head");
     }
+#if LING3_EXPERIMENTAL_MTP
+    bool EnableMtp() {
+        if (current_position != 0)
+            throw std::invalid_argument("MTP must be enabled before processing the prefix");
+        if (mtp) return true;
+        if (std::any_of(package.tensors().begin(), package.tensors().end(),
+                       [](const TensorView & t) { return t.name == "model.layers.24.eh_proj.weight"; })) {
+            mtp = std::make_unique<MtpLayer>(package, context_capacity, scratch);
+            mtp_hidden.resize(kHidden);
+            mtp_embedding.resize(kHidden);
+        }
+        return mtp != nullptr;
+    }
+#endif
     void Reset() {
         for (auto & weak : checkpoints) if (auto checkpoint = weak.lock()) checkpoint->valid = false;
         checkpoints.clear();
         current_position = 0;
         for (auto & layer : layers) layer->Reset();
+#if LING3_EXPERIMENTAL_MTP
+        if (mtp) mtp->Reset();
+        mtp_position = 0;
+        mtp_trunk_ready = false;
+#endif
     }
     void InvalidateOverwrittenCheckpoints() {
             throw std::invalid_argument("stale or foreign decoder checkpoint");
         for (std::size_t i = 0; i < layers.size(); ++i) layers[i]->RestoreCheckpoint(checkpoint.layers[i]);
         current_position = checkpoint.position;
+#if LING3_EXPERIMENTAL_MTP
+        mtp_trunk_ready = false;
+#endif
         return current_position;
     }
         lm_head->Run(normalized, logits);
         const auto end = Clock::now();
         ++current_position;
+#if LING3_EXPERIMENTAL_MTP
+        mtp_trunk_ready = true;
+#endif
         return {
             Milliseconds(begin, layers_end),
             Milliseconds(layers_end, end),
         }
         const auto end = Clock::now();
         current_position += rows;
+#if LING3_EXPERIMENTAL_MTP
+        mtp_trunk_ready = compute_logits;
+#endif
         return {
             Milliseconds(begin, layers_end),
             Milliseconds(layers_end, end),
         };
     }
+#if LING3_EXPERIMENTAL_MTP
+    DecodeTimings EvalMtp(std::uint32_t token, std::span<float> logits) {
+        if (!mtp) throw std::runtime_error("model package has no MTP layer");
+        if (token >= package.header().vocab_size || logits.size() != package.header().vocab_size ||
+            current_position == 0 || !mtp_trunk_ready || mtp_position != current_position - 1)
+            throw std::invalid_argument("MTP requires a contiguous, aligned prefix and valid token/logits");
+        const auto begin = Clock::now();
+        const auto embedding = embeddings.subspan(static_cast<std::size_t>(token) * kHidden, kHidden);
+        for (int i = 0; i < kHidden; ++i) mtp_embedding[i] = BFloat16ToFloat(embedding[i]);
+        // Upstream passes the main model's final normalized hidden state.
+        // Keep MTP output separate so probing cannot overwrite trunk scratch.
+        mtp->Run(mtp_embedding, normalized, mtp_position, mtp_hidden);
+        const auto layers_end = Clock::now();
+        lm_head->Run(mtp_hidden, logits);
+        const auto end = Clock::now();
+        ++mtp_position;
+        mtp_trunk_ready = false;
+        return {Milliseconds(begin, layers_end), Milliseconds(layers_end, end), Milliseconds(begin, end)};
+    }
+#endif
     void PrepareBatch(std::size_t rows) {
         if (rows < 1 || rows > kMaxBatch) {
             throw std::invalid_argument("decoder batch rows must be in [1, 128]");
 DecodeTimings Decoder::Eval(std::uint32_t token, std::span<float> logits) {
     return impl_->Eval(token, logits);
 }
+#if LING3_EXPERIMENTAL_MTP
+bool Decoder::EnableMtp() { return impl_->EnableMtp(); }
+bool Decoder::HasMtp() const noexcept { return impl_->mtp != nullptr; }
+DecodeTimings Decoder::EvalMtp(std::uint32_t token, std::span<float> logits) {
+    return impl_->EvalMtp(token, logits);
+}
+#endif
 DecodeTimings Decoder::EvalBatch(
     std::span<const std::uint32_t> tokens,
     std::span<float> logits) {

src/main.cpp CHANGED Viewed

@@ -64,6 +64,9 @@ void PrintUsage(const char * program) {
               << "  " << program << " logits-smoke MODEL.l3r TOKEN [REF.f32]\n"
               << "  " << program << " generate MODEL.l3r USER_TEXT [MAX_NEW_TOKENS]\n"
               << "  " << program << " benchmark MODEL.l3r SEQLEN NEW_TOKENS [ITERATIONS]\n"
               << "  " << program << " sequence-check MODEL.l3r SEQLEN NEW_TOKENS OUT_PREFIX [REFERENCE_PREFIX [CORPUS.txt]]\n"
               << "  " << program << " serve MODEL.l3r [PORT]\n";
 }
@@ -1209,6 +1212,130 @@ struct BenchmarkSample {
     ProcessMemory memory;
 };
 int RunBenchmark(
     const std::string & package_path,
     std::size_t sequence_length,
@@ -1449,6 +1576,13 @@ int main(int argc, char ** argv) {
             const std::size_t iterations = argc == 6 ? ParseSize(argv[5], "iterations") : 3;
             return RunBenchmark(argv[2], sequence_length, new_tokens, iterations);
         }
         if (command == "sequence-check" && argc >= 6 && argc <= 8) {
             return RunSequenceCheck(argv[2], ParseSize(argv[3], "sequence length"),
                                     ParseSize(argv[4], "new tokens"), argv[5],

               << "  " << program << " logits-smoke MODEL.l3r TOKEN [REF.f32]\n"
               << "  " << program << " generate MODEL.l3r USER_TEXT [MAX_NEW_TOKENS]\n"
               << "  " << program << " benchmark MODEL.l3r SEQLEN NEW_TOKENS [ITERATIONS]\n"
+#if LING3_EXPERIMENTAL_MTP
+              << "  " << program << " mtp-benchmark MODEL.l3r SEQLEN NEW_TOKENS ITERATIONS CONTEXT [CASES.txt]\n"
+#endif
               << "  " << program << " sequence-check MODEL.l3r SEQLEN NEW_TOKENS OUT_PREFIX [REFERENCE_PREFIX [CORPUS.txt]]\n"
               << "  " << program << " serve MODEL.l3r [PORT]\n";
 }
     ProcessMemory memory;
 };
+#if LING3_EXPERIMENTAL_MTP
+// Same resident decoder for baseline and MTP. Populate every MTP prefix slot
+// sequentially, and teacher-force baseline tokens so the work and histories
+// stay comparable. This measures forward overhead, not speculative speedup.
+int RunMtpBenchmark(const std::string & path, std::size_t length,
+                    std::size_t steps, std::size_t repeats, std::size_t capacity,
+                    const std::string & cases_path) {
+    ConfigureInferenceProcess();
+    const ling3::ModelPackage package(path);
+    if (!length || steps < 2 || !repeats || length + steps > capacity || capacity > 262144)
+        throw std::invalid_argument("invalid MTP benchmark lengths");
+    const auto tokenizer = PackageTokenizer(package);
+    std::vector<std::vector<std::uint32_t>> prompts{BuildBenchmarkPrompt(tokenizer, length)};
+    if (!cases_path.empty()) {
+        std::ifstream file(cases_path);
+        if (!file) throw std::runtime_error("cannot open MTP cases file");
+        std::string question;
+        while (std::getline(file, question)) {
+            if (question.empty()) continue;
+            auto ids = tokenizer.Encode("<role>SYSTEM</role>detailed thinking off<|role_end|>"
+                "<role>HUMAN</role>" + question + "<|role_end|><role>ASSISTANT</role>\n<think></think>");
+            if (ids.empty() || ids.size() + steps > capacity)
+                throw std::invalid_argument("case does not fit the requested context");
+            prompts.push_back(std::move(ids));
+        }
+    }
+    const auto started = std::chrono::steady_clock::now();
+    ling3::Decoder decoder(package, capacity);
+    decoder.EnableMtp();
+    const std::vector<bool> modes = decoder.HasMtp() ? std::vector<bool>{false, true} : std::vector<bool>{false};
+    std::vector<float> logits(package.header().vocab_size), draft(logits.size());
+    const auto milliseconds = [](auto a, auto b) {
+        return std::chrono::duration<double, std::milli>(b - a).count();
+    };
+    std::cout << std::fixed << std::setprecision(4)
+              << "benchmark=resident_mtp_forward_not_speculative\n"
+              << "context_capacity=" << capacity << " cases=" << prompts.size()
+              << " mtp_enabled=" << decoder.HasMtp() << " new_tokens=" << steps << " iterations=" << repeats << '\n'
+              << "initialization_ms=" << milliseconds(started, std::chrono::steady_clock::now()) << std::endl;
+    const auto initial_memory = ReadProcessMemory();
+    std::cout << "initial_rss_mib=" << initial_memory.rss_kb / 1024.0
+              << " initial_hwm_mib=" << initial_memory.hwm_kb / 1024.0 << std::endl;
+    for (std::size_t case_index = 0; case_index < prompts.size(); ++case_index) {
+    const auto & prompt = prompts[case_index];
+    length = prompt.size();
+    for (std::size_t round = 0; round <= repeats; ++round) {
+        std::vector<std::uint32_t> teacher;
+        teacher.reserve(steps);
+        for (bool with_mtp : modes) {
+            decoder.Reset();
+            const auto attention_before = decoder.AttentionStats();
+            const auto begin = std::chrono::steady_clock::now();
+            double trunk = 0, mtp = 0, mtp_layers = 0, mtp_head = 0;
+            for (std::size_t i = 0; i < length; ++i) {
+                decoder.Eval(prompt[i], logits);
+                if (with_mtp && i + 1 < length) decoder.EvalMtp(prompt[i + 1], draft);
+            }
+            std::uint32_t token = GreedyToken(logits);
+            std::size_t target_agreement = 0, draft_hits = 0;
+            if (!with_mtp) teacher.push_back(token);
+            else {
+                target_agreement += token == teacher[0];
+                token = teacher[0];
+            }
+            const auto first = std::chrono::steady_clock::now();
+            std::vector<double> trunk_samples, mtp_samples;
+            for (std::size_t i = 1; i < steps; ++i) {
+                std::uint32_t proposed = 0;
+                if (with_mtp) {
+                    const auto t = decoder.EvalMtp(token, draft);
+                    mtp += t.total_ms; mtp_layers += t.layers_ms; mtp_head += t.output_head_ms;
+                    mtp_samples.push_back(t.total_ms);
+                    proposed = GreedyToken(draft);
+                }
+                const auto t = decoder.Eval(token, logits);
+                trunk += t.total_ms; trunk_samples.push_back(t.total_ms);
+                token = GreedyToken(logits);
+                if (!with_mtp) teacher.push_back(token);
+                else {
+                    target_agreement += token == teacher[i];
+                    draft_hits += proposed == teacher[i];
+                    token = teacher[i];
+                }
+            }
+            const auto end = std::chrono::steady_clock::now();
+            const auto percentile = [](std::vector<double> values, double q) {
+                std::sort(values.begin(), values.end());
+                return values[static_cast<std::size_t>(q * (values.size() - 1))];
+            };
+            const auto memory = ReadProcessMemory();
+            const auto attention = decoder.AttentionStats();
+            std::cout << "phase=" << (round == 0 ? "warmup" : "measured")
+                      << " case=" << case_index << " seqlen=" << length
+                      << " round=" << round << " mode=" << (with_mtp ? "target_plus_mtp" : "target_only")
+                      << " sequential_prefill_ms=" << milliseconds(begin, first)
+                      << " decode_wall_ms=" << milliseconds(first, end)
+                      << " tokens_per_second=" << 1000.0 * (steps - 1) / milliseconds(first, end)
+                      << " target_mean_ms=" << trunk / (steps - 1)
+                      << " target_p50_ms=" << percentile(trunk_samples, 0.5)
+                      << " target_p95_ms=" << percentile(trunk_samples, 0.95);
+            if (with_mtp)
+                std::cout << " mtp_mean_ms=" << mtp / (steps - 1)
+                          << " mtp_p50_ms=" << percentile(mtp_samples, 0.5)
+                          << " mtp_p95_ms=" << percentile(mtp_samples, 0.95)
+                          << " mtp_layers_mean_ms=" << mtp_layers / (steps - 1)
+                          << " mtp_head_mean_ms=" << mtp_head / (steps - 1)
+                          << " target_agreement=" << target_agreement << '/' << steps
+                          << " draft_top1_hits=" << draft_hits << '/' << (steps - 1);
+            std::cout << " mla_npu_calls=" << attention.npu_calls - attention_before.npu_calls
+                      << " mla_cpu_calls=" << attention.cpu_calls - attention_before.cpu_calls
+                      << " mla_fallbacks=" << attention.fallbacks - attention_before.fallbacks
+                      << " rss_mib=" << memory.rss_kb / 1024.0
+                      << " hwm_mib=" << memory.hwm_kb / 1024.0
+                      << " teacher_hash=" << std::hex << TokenHash(teacher) << std::dec << std::endl;
+            if (with_mtp && target_agreement != steps)
+                throw std::runtime_error("MTP probe changed target predictions for the same teacher-forced tokens");
+        }
+    }
+    }
+    return 0;
+}
+#endif
 int RunBenchmark(
     const std::string & package_path,
     std::size_t sequence_length,
             const std::size_t iterations = argc == 6 ? ParseSize(argv[5], "iterations") : 3;
             return RunBenchmark(argv[2], sequence_length, new_tokens, iterations);
         }
+#if LING3_EXPERIMENTAL_MTP
+        if (command == "mtp-benchmark" && (argc == 7 || argc == 8)) {
+            return RunMtpBenchmark(argv[2], ParseSize(argv[3], "sequence length"),
+                ParseSize(argv[4], "new tokens"), ParseSize(argv[5], "iterations"),
+                ParseSize(argv[6], "context capacity"), argc == 8 ? argv[7] : "");
+        }
+#endif
         if (command == "sequence-check" && argc >= 6 && argc <= 8) {
             return RunSequenceCheck(argv[2], ParseSize(argv[3], "sequence length"),
                                     ParseSize(argv[4], "new tokens"), argv[5],

tests/data/mtp_probe_questions.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+请用中文讲一个完整的小故事，主角是一个第一次出海的年轻人，故事要有起因、转折和结尾。
+我明天要召开一个项目进度会，请帮我设计会议议程，并说明如何跟踪会议后的任务。
+请解释为什么长上下文会影响大语言模型的解码速度，用通俗语言举一个例子。

tests/precision_policy_test.cpp CHANGED Viewed

@@ -16,6 +16,21 @@ int main()try{
     if(!SelectW8(small,"lm_head")||!SelectW8(small,"model.layers.0.mlp.down_proj")||
        !SelectW8(small,"model.layers.1.mlp.shared_experts.gate_up")||SelectW8({},"lm_head"))
         throw std::runtime_error("nonattention precision selection mismatch");
     for(auto bad:{"kda","head,","head,,shared"}){
         bool rejected=false;try{ParseW8Families(bad);}catch(const std::invalid_argument&){rejected=true;}
         if(!rejected)throw std::runtime_error("invalid selector accepted");

     if(!SelectW8(small,"lm_head")||!SelectW8(small,"model.layers.0.mlp.down_proj")||
        !SelectW8(small,"model.layers.1.mlp.shared_experts.gate_up")||SelectW8({},"lm_head"))
         throw std::runtime_error("nonattention precision selection mismatch");
+#if LING3_EXPERIMENTAL_MTP
+    if(W8MatrixFamily("model.layers.24.attention.qkv_gate_a")!="mla_in" ||
+       W8MatrixFamily("model.layers.24.attention.o_proj")!="mla_out" ||
+       SelectW8(all,"model.layers.24.mlp.experts.0.gate_up"))
+        throw std::runtime_error("MTP precision family mismatch");
+#else
+    bool disabled_mtp_rejected=false;
+    try{W8MatrixFamily("model.layers.24.attention.o_proj");}
+    catch(const std::invalid_argument&){disabled_mtp_rejected=true;}
+    if(!disabled_mtp_rejected)throw std::runtime_error("disabled MTP layer accepted");
+#endif
+    bool invalid_mtp_layer=false;
+    try{W8MatrixFamily("model.layers.25.attention.o_proj");}
+    catch(const std::invalid_argument&){invalid_mtp_layer=true;}
+    if(!invalid_mtp_layer)throw std::runtime_error("invalid MTP layer accepted");
     for(auto bad:{"kda","head,","head,,shared"}){
         bool rejected=false;try{ParseW8Families(bad);}catch(const std::invalid_argument&){rejected=true;}
         if(!rejected)throw std::runtime_error("invalid selector accepted");

tools/append_mtp_package.py ADDED Viewed

	@@ -0,0 +1,133 @@

+#!/usr/bin/env python3
+"""Append quantized layer-24 tensors without requantizing any trunk weight."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import zlib
+from pack_model import HEADER, ENTRY, align, pad_to
+def index(path):
+    with path.open("rb") as stream:
+        header = list(HEADER.unpack(stream.read(HEADER.size)))
+        if (header[0] != b"L3RKNN1\0" or header[1:4] != [1, HEADER.size, 0x01020304]
+                or header[5] != ENTRY.size or header[12] != path.stat().st_size):
+            raise ValueError("invalid package header: " + str(path))
+        crc = header[14]
+        header[14] = 0
+        if zlib.crc32(HEADER.pack(*header)) & 0xFFFFFFFF != crc:
+            raise ValueError("header CRC mismatch")
+        header[14] = crc
+        stream.seek(header[8])
+        names = stream.read(header[9])
+        stream.seek(header[7])
+        result = []
+        for _ in range(header[4]):
+            entry = list(ENTRY.unpack(stream.read(ENTRY.size)))
+            name = names[entry[0]:entry[0] + entry[1]].decode("utf-8")
+            if not name or entry[3] > 4:
+                raise ValueError("invalid tensor entry")
+            for offset, size in ((entry[12], entry[13]), (entry[14], entry[15])):
+                if size and (offset < header[10] or offset + size > header[12]):
+                    raise ValueError("out-of-range tensor: " + name)
+            result.append((path, name, entry))
+    if len({name for _, name, _ in result}) != len(result):
+        raise ValueError("duplicate tensor names")
+    return header, result
+def copy_range(source, output, offset, size):
+    source.seek(offset)
+    digest = hashlib.sha256()
+    remaining = size
+    while remaining:
+        chunk = source.read(min(remaining, 8 * 1024 * 1024))
+        if not chunk:
+            raise IOError("short tensor payload")
+        output.write(chunk)
+        digest.update(chunk)
+        remaining -= len(chunk)
+    return digest.digest()
+def file_hash(path):
+    with path.open("rb") as stream:
+        return hashlib.file_digest(stream, "sha256").hexdigest()
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base", type=Path, required=True)
+    parser.add_argument("--mtp-package", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    if args.output.exists():
+        parser.error("refusing to overwrite an existing package")
+    header, trunk = index(args.base)
+    other_header, other = index(args.mtp_package)
+    if header[15:35] != other_header[15:35]:
+        parser.error("MTP architecture differs from trunk")
+    mtp = [row for row in other if row[1].startswith("model.layers.24.")]
+    if len(mtp) != 798 or any(row[1].startswith("model.layers.24.") for row in trunk):
+        parser.error("expected 798 MTP tensors and a trunk without MTP")
+    tensors = trunk + mtp
+    names = bytearray()
+    for _, name, entry in tensors:
+        entry[0] = len(names)
+        encoded = name.encode("utf-8")
+        entry[1] = len(encoded)
+        names.extend(encoded)
+    header[4] = len(tensors)
+    header[7] = HEADER.size
+    header[8] = HEADER.size + len(tensors) * ENTRY.size
+    header[9] = len(names)
+    header[10] = align(header[8] + len(names))
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    sources = {}
+    entries = []
+    try:
+        with args.output.open("xb") as output:
+            output.write(bytes(HEADER.size + len(tensors) * ENTRY.size))
+            output.write(names)
+            pad_to(output, header[10])
+            for path, name, entry in tensors:
+                if path not in sources:
+                    sources[path] = path.open("rb")
+                source = sources[path]
+                data_offset = align(output.tell())
+                pad_to(output, data_offset)
+                if copy_range(source, output, entry[12], entry[13]) != entry[22]:
+                    raise ValueError("tensor SHA256 mismatch: " + name)
+                aux_offset = align(output.tell()) if entry[15] else 0
+                if entry[15]:
+                    pad_to(output, aux_offset)
+                    copy_range(source, output, entry[14], entry[15])
+                entry[12], entry[14] = data_offset, aux_offset
+                entries.append(ENTRY.pack(*entry))
+            header[12] = align(output.tell())
+            pad_to(output, header[12])
+            header[11] = header[12] - header[10]
+            header[14] = 0
+            header[14] = zlib.crc32(HEADER.pack(*header)) & 0xFFFFFFFF
+            output.seek(0)
+            output.write(HEADER.pack(*header))
+            output.writelines(entries)
+            output.flush()
+            os.fsync(output.fileno())
+    finally:
+        for source in sources.values():
+            source.close()
+    record = {"base": args.base.name, "base_sha256": file_hash(args.base),
+              "mtp_package": args.mtp_package.name, "mtp_source_sha256": file_hash(args.mtp_package),
+              "output": args.output.name, "output_sha256": file_hash(args.output),
+              "bytes": header[12], "flags": header[6], "trunk_tensors_unchanged": len(trunk),
+              "appended_mtp_tensors": len(mtp), "recomputed_quantization": False}
+    args.output.with_suffix(".manifest.json").write_text(json.dumps(record, indent=2) + "\n")
+    print(json.dumps(record, indent=2), flush=True)
+if __name__ == "__main__":
+    main()

tools/mtp_resident_benchmark.py ADDED Viewed

	@@ -0,0 +1,134 @@

+#!/usr/bin/env python3
+"""Resident MTP forward probe; temporarily lock clocks and restore on exit."""
+import argparse
+import datetime
+import glob
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import time
+def read(path):
+    try:
+        return Path(path).read_text().strip()
+    except OSError:
+        return None
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("binary")
+    parser.add_argument("model")
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--runtime", required=True)
+    parser.add_argument("--seqlen", type=int, default=128)
+    parser.add_argument("--tokens", type=int, default=64)
+    parser.add_argument("--rounds", type=int, default=3)
+    parser.add_argument("--context", type=int, default=32768)
+    parser.add_argument("--cases")
+    args = parser.parse_args()
+    if os.geteuid() != 0:
+        parser.error("run with sudo for temporary clock locking and NPU debugfs sampling")
+    help_env = os.environ.copy()
+    help_env["LD_LIBRARY_PATH"] = args.runtime + ":" + help_env.get("LD_LIBRARY_PATH", "")
+    probe = subprocess.run([args.binary], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                           text=True, env=help_env, timeout=15)
+    if "Usage:" not in probe.stdout:
+        parser.error("cannot query engine CLI: " + probe.stdout.strip())
+    if "mtp-benchmark" not in probe.stdout:
+        parser.error("MTP is not compiled; rebuild with -DLING3_EXPERIMENTAL_MTP=ON")
+    output = Path(args.output)
+    output.mkdir(parents=True, exist_ok=False)
+    original = {}
+    applied = {}
+    child = None
+    def write_clock(path, value):
+        original[path] = read(path)
+        if original[path] is None:
+            raise RuntimeError("cannot snapshot " + path)
+        Path(path).write_text(str(value))
+        applied[path] = read(path)
+    def stop(signum, frame):
+        raise KeyboardInterrupt
+    signal.signal(signal.SIGTERM, stop)
+    signal.signal(signal.SIGINT, stop)
+    try:
+        for policy in sorted(glob.glob("/sys/devices/system/cpu/cpufreq/policy*")):
+            write_clock(policy + "/scaling_governor", "performance")
+            write_clock(policy + "/scaling_min_freq", read(policy + "/scaling_max_freq"))
+        for device in ("fdab0000.npu", "dmc"):
+            base = "/sys/class/devfreq/" + device
+            frequencies = read(base + "/available_frequencies")
+            maximum = max(map(int, frequencies.split())) if frequencies else int(read(base + "/max_freq"))
+            write_clock(base + "/governor", "performance")
+            write_clock(base + "/min_freq", maximum)
+        env = os.environ.copy()
+        for key in ("LING3_GDN_PREFILL_DIR", "LING3_PREFILL_W4A4", "LING3_EXPERT_ALL_CORES",
+                    "LING3_MLA_DECODE_NPU", "LING3_DISABLE_BATCH_INPUT_REUSE",
+                    "LING3_DISABLE_PARALLEL_GATHER"):
+            env.pop(key, None)
+        for key in ("LING3_PREWARM_EXPERTS", "LING3_EXPERT_BALANCED", "LING3_EXPERT_ZERO_COPY",
+                    "LING3_GDN_CPU_PREFILL", "LING3_GDN_CPU_FP32_STATE", "LING3_GDN_CPU_DECODE",
+                    "LING3_GDN_FULL_FP32", "LING3_MLA_SIMD", "LING3_VECTOR_MATH"):
+            env[key] = "1"
+        env["LING3_MLA_BACKEND"] = "cpu"
+        env["LD_LIBRARY_PATH"] = args.runtime + ":" + env.get("LD_LIBRARY_PATH", "")
+        command = [args.binary, "mtp-benchmark", args.model, str(args.seqlen), str(args.tokens), str(args.rounds), str(args.context)]
+        if args.cases:
+            command.append(args.cases)
+        metadata = {"started_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+                    "command": command, "original_clocks": original, "meminfo_before": read("/proc/meminfo"), "applied_clocks": applied,
+                    "profile": {k: v for k, v in env.items() if k.startswith("LING3_")}}
+        (output / "metadata.json").write_text(json.dumps(metadata, indent=2))
+        with (output / "benchmark.log").open("w") as log, (output / "telemetry.jsonl").open("w") as telemetry:
+            child = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, env=env)
+            print("pid=" + str(child.pid) + " output=" + str(output), flush=True)
+            start = time.monotonic()
+            while child.poll() is None:
+                row = {"elapsed_s": time.monotonic() - start,
+                       "process_stat": read(f"/proc/{child.pid}/stat"),
+                       "process_status": read(f"/proc/{child.pid}/status"),
+                       "npu_load": read("/sys/kernel/debug/rknpu/load"),
+                       "meminfo": read("/proc/meminfo")}
+                row["frequencies"] = {p: read(p) for p in glob.glob("/sys/devices/system/cpu/cpufreq/policy*/scaling_cur_freq")
+                                      + ["/sys/class/devfreq/fdab0000.npu/cur_freq", "/sys/class/devfreq/dmc/cur_freq"]}
+                row["temperatures"] = {p: read(p) for p in glob.glob("/sys/class/thermal/thermal_zone*/temp")}
+                telemetry.write(json.dumps(row) + "\n")
+                telemetry.flush()
+                time.sleep(0.5)
+            metadata["exit_code"] = child.returncode
+            metadata["elapsed_s"] = time.monotonic() - start
+            print("exit_code=" + str(child.returncode), flush=True)
+    finally:
+        if child is not None and child.poll() is None:
+            child.terminate()
+            try:
+                child.wait(timeout=15)
+            except subprocess.TimeoutExpired:
+                child.kill()
+                child.wait()
+        restored = {}
+        for path, value in reversed(list(original.items())):
+            try:
+                Path(path).write_text(value)
+                restored[path] = read(path)
+            except OSError as error:
+                restored[path] = str(error)
+        final = locals().get("metadata", {"original_clocks": original, "applied_clocks": applied})
+        final["restored_clocks"] = restored
+        (output / "metadata.json").write_text(json.dumps(final, indent=2))
+        if "SUDO_UID" in os.environ:
+            for path in [output] + list(output.iterdir()):
+                os.chown(path, int(os.environ["SUDO_UID"]), int(os.environ["SUDO_GID"]))
+        print("clocks_restored", flush=True)
+    return child.returncode if child else 1
+if __name__ == "__main__":
+    raise SystemExit(main())