Upload folder using huggingface_hub

.meta.json

@@ -1,17 +1,22 @@
 {
-  "Pillow": "12.2.0",
-  "accelerate": "1.15.0",
-  "av": "19.0.2",
-  "flashinfer-python": "0.7.2",
-  "huggingface_hub": "1.33.0",
-  "kernels": "0.17.0",
-  "numpy": "2.3.5",
-  "safetensors": "0.8.0",
-  "sglang": "0.5.23.dev384+g9ab74b851",
-  "torch": "2.14.1",
-  "torchcodec": "0.17.0",
-  "transformers": "5.19.0",
-  "Python": "3.12.13",
-  "CUDA": "13.0",
-  "libstdc++": "13.4.0"
-}
+  "source_model": {
+    "name": "Qwen/Qwen3.8-Flash-Next",
+    "commit_id": "de4b8e4d43b917e7706784d8bb445c9af86a3540"
+  },
+  "env": {
+    "python": "3.12.13",
+    "packages": {
+      "Pillow": "13.0.0",
+      "flashinfer-python": "0.7.0.post1",
+      "huggingface_hub": "2.3.0",
+      "numpy": "2.3.5",
+      "safetensors": "0.8.0",
+      "tokenizers": "0.23.3",
+      "torch": "2.13.0",
+      "torchcodec": "0.17.0",
+      "transformers": "5.19.0",
+      "vllm": "0.32.0"
+    },
+    "cuda": "13.0"
+  }
+}

README.md

@@ -12,7 +12,7 @@ Structural comparison with `Qwen/Qwen3.8-Flash-Next`:
 |---|---|---|
 | Architecture / modalities | Qwen4Exp; text, image, video | Same |
 | Vocabulary / tied embeddings | 248,320 / false | Same |
-| Backbone layers / hidden size | 48 / 2,560 | 8 / 4 |
+| Backbone layers / hidden size | 48 / 2,560 | 8 / 8 |
 | Attention Q/KV heads; head dim | 24/2; 256 | 12/1; 32 |
 | Attention cycle | 3 linear + 1 QSA | Same; two cycles |
 | Linear key/value heads; head dim | 16/48; 128 | 4/12; 32 |
@@ -25,42 +25,58 @@ Structural comparison with `Qwen/Qwen3.8-Flash-Next`:
 | PLE hash bucket base | 20,000,000 | 512; official prime layout |
 | PLE convolution / serialized table shards | 4 / 128 | Same |
 | MTP | 1 QSA layer; HC fusion; shared token embeddings | Same |
-| Vision depth / hidden size / output size | 27 / 1,152 / 2,560 | 2 / 64 / 4 |
+| Vision depth / hidden size / output size | 27 / 1,152 / 2,560 | 2 / 64 / 8 |
 | Vision patch / temporal patch / merge | 16 / 2 / 2 | Same |
 
 PLE retains the source n-gram orders, head count, hash/prime-layout algorithm,
 projections and convolution with reduced auxiliary hash capacity. The main token
 vocabulary and IDs are unchanged.
-The complete BF16 checkpoint occupies 5,874,184 bytes.
+The Qwen4Exp Hyper-Connections/MTP TP=4 path requires an eight-wide hidden state:
+the four-wide configuration creates non-contiguous per-rank MTP block outputs in
+vLLM. The two untied BF16 token embedding/head matrices at width 8 occupy
+7,946,240 bytes; the complete verified checkpoint is 10,886,632 bytes.
 Tokenizer and processor assets are copied unchanged from the pinned source.
+vLLM 0.32.0 was verified on one four-NVIDIA-H20 node with TP=4, CUDA Graph,
+three-token MTP, and real text, image, and four-frame video requests.
 
 | File path | Size |
 |------|------|
-| model.safetensors | 5.9MB |
+| model.safetensors | 10.9MB |
 
 
 ### Example usage:
 
-- SGLang (✅ verified)
+- vLLM (✅ verified)
 
-  - Run on both two-GPU nodes with NODE_RANK set to 0 or 1, the same
-    DIST_INIT_ADDR, and an available local NCCL_PORT. Enables TP=4, NEXTN/MTP
-    and CUDA Graph.
-  - Verified on four NVIDIA H20 GPUs across two nodes for text, QSA long
-    prompts, image-plus-text and video-plus-text; see Test environment.
+```bash
+export VLLM_ALLREDUCE_USE_FLASHINFER=0
+export VLLM_ALLREDUCE_USE_SYMM_MEM=0
+vllm serve tiny-random/qwen3.8-flash-next \\
+  --tensor-parallel-size 4 \\
+  --disable-custom-all-reduce \\
+  --kernel-config '{"moe_backend":"triton"}' \\
+  --gdn-prefill-backend triton \\
+  --speculative-config '{"method":"mtp","num_speculative_tokens":3}' \\
+  --max-model-len 4096 \\
+  --max-num-batched-tokens 512 \\
+  --max-num-seqs 4 \\
+  --gpu-memory-utilization 0.3 \\
+  --compilation-config '{"cudagraph_capture_sizes":[1,2,4,8,16]}'
+```
+
+- SGLang (unverified)
+
+  - Run on one four-GPU node. Enables TP=4, NEXTN/MTP and CUDA Graph.
+  - Unverified on the tested NVIDIA H20 worker: its CUDA 12.2 driver cannot
+    load the CUDA 13 SGLang kernel and TileLang artifacts required by this
+    Qwen4Exp release. Use the Test environment versions with a CUDA 13-capable
+    driver for TP=4 validation.
 
 ```bash
 model_id=tiny-random/qwen3.8-flash-next
-node_rank=${NODE_RANK:-0}
-dist_init_addr=${DIST_INIT_ADDR}
-nccl_port=${NCCL_PORT}
 sglang serve \
   --model-path $model_id \
   --tp-size 4 \
-  --nnodes 2 \
-  --node-rank $node_rank \
-  --dist-init-addr $dist_init_addr \
-  --nccl-port $nccl_port \
   --fp8-gemm-backend triton \
   --moe-runner-backend triton \
   --speculative-algorithm NEXTN \
@@ -131,7 +147,7 @@ folder = Path(save_folder)
 folder.mkdir(parents=True, exist_ok=True)
 raw = json.loads(Path(hf_hub_download(source_model_id, 'config.json', revision=source_revision)).read_text())
 raw['text_config'].update(
-    hidden_size=4, ple_embed_dim=16, ngram_vocab_size_base=512, hc_lowrank=4, num_hidden_layers=8,
+    hidden_size=8, ple_embed_dim=16, ngram_vocab_size_base=512, hc_lowrank=4, num_hidden_layers=8,
     layer_types=(["linear_attention"] * 3 + ["full_attention"]) * 2,
     num_attention_heads=12, num_key_value_heads=1, head_dim=32,
     linear_num_key_heads=4, linear_num_value_heads=12,
@@ -139,7 +155,7 @@ raw['text_config'].update(
     num_experts=32, moe_intermediate_size=128, shared_expert_intermediate_size=128,
 )
 raw['text_config']['rope_parameters']['mrope_section'] = [2, 1, 1]
-raw['vision_config'].update(depth=2, hidden_size=64, intermediate_size=128, num_heads=4, out_hidden_size=4)
+raw['vision_config'].update(depth=2, hidden_size=64, intermediate_size=128, num_heads=4, out_hidden_size=8)
 (folder / 'config.json').write_text(json.dumps(raw, indent=2))
 config = AutoConfig.from_pretrained(folder)
 model = AutoModelForImageTextToText.from_config(config, dtype=torch.bfloat16)
@@ -223,21 +239,21 @@ Qwen4ExpForConditionalGeneration(
         (norm): LayerNorm((64,), eps=1e-06, elementwise_affine=True, bias=True)
         (linear_fc1): Linear(in_features=256, out_features=256, bias=True)
         (act_fn): GELU(approximate='none')
-        (linear_fc2): Linear(in_features=256, out_features=4, bias=True)
+        (linear_fc2): Linear(in_features=256, out_features=8, bias=True)
       )
     )
     (language_model): Qwen4ExpTextModel(
-      (embed_tokens): Embedding(248320, 4)
+      (embed_tokens): Embedding(248320, 8)
       (layers): ModuleList(
         (0): Qwen4ExpTextDecoderLayer(
           (linear_attn): Qwen4ExpTextGatedDeltaNet(
             (conv1d): Conv1d(640, 640, kernel_size=(4,), stride=(1,), padding=(3,), groups=640, bias=False)
             (norm): Qwen4ExpTextRMSNormGated()
-            (out_proj): Linear(in_features=384, out_features=4, bias=False)
-            (in_proj_qkv): Linear(in_features=4, out_features=640, bias=False)
-            (in_proj_z): Linear(in_features=4, out_features=384, bias=False)
-            (in_proj_b): Linear(in_features=4, out_features=12, bias=False)
-            (in_proj_a): Linear(in_features=4, out_features=12, bias=False)
+            (out_proj): Linear(in_features=384, out_features=8, bias=False)
+            (in_proj_qkv): Linear(in_features=8, out_features=640, bias=False)
+            (in_proj_z): Linear(in_features=8, out_features=384, bias=False)
+            (in_proj_b): Linear(in_features=8, out_features=12, bias=False)
+            (in_proj_a): Linear(in_features=8, out_features=12, bias=False)
           )
           (mlp): Qwen4ExpTextSparseMoeBlock(
             (gate): Qwen4ExpTextTopKRouter()
@@ -245,35 +261,35 @@ Qwen4ExpForConditionalGeneration(
               (act_fn): SiLUActivation()
             )
             (shared_expert): Qwen4ExpTextMLP(
-              (gate_proj): Linear(in_features=4, out_features=128, bias=False)
-              (up_proj): Linear(in_features=4, out_features=128, bias=False)
-              (down_proj): Linear(in_features=128, out_features=4, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=128, bias=False)
+              (up_proj): Linear(in_features=8, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=8, bias=False)
               (act_fn): SiLUActivation()
             )
-            (shared_expert_gate): Linear(in_features=4, out_features=1, bias=False)
+            (shared_expert_gate): Linear(in_features=8, out_features=1, bias=False)
           )
           (attn_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
           (mlp_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
         )
         (1): Qwen4ExpTextDecoderLayer(
           (linear_attn): Qwen4ExpTextGatedDeltaNet(
             (conv1d): Conv1d(640, 640, kernel_size=(4,), stride=(1,), padding=(3,), groups=640, bias=False)
             (norm): Qwen4ExpTextRMSNormGated()
-            (out_proj): Linear(in_features=384, out_features=4, bias=False)
-            (in_proj_qkv): Linear(in_features=4, out_features=640, bias=False)
-            (in_proj_z): Linear(in_features=4, out_features=384, bias=False)
-            (in_proj_b): Linear(in_features=4, out_features=12, bias=False)
-            (in_proj_a): Linear(in_features=4, out_features=12, bias=False)
+            (out_proj): Linear(in_features=384, out_features=8, bias=False)
+            (in_proj_qkv): Linear(in_features=8, out_features=640, bias=False)
+            (in_proj_z): Linear(in_features=8, out_features=384, bias=False)
+            (in_proj_b): Linear(in_features=8, out_features=12, bias=False)
+            (in_proj_a): Linear(in_features=8, out_features=12, bias=False)
           )
           (mlp): Qwen4ExpTextSparseMoeBlock(
             (gate): Qwen4ExpTextTopKRouter()
@@ -281,46 +297,46 @@ Qwen4ExpForConditionalGeneration(
               (act_fn): SiLUActivation()
             )
             (shared_expert): Qwen4ExpTextMLP(
-              (gate_proj): Linear(in_features=4, out_features=128, bias=False)
-              (up_proj): Linear(in_features=4, out_features=128, bias=False)
-              (down_proj): Linear(in_features=128, out_features=4, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=128, bias=False)
+              (up_proj): Linear(in_features=8, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=8, bias=False)
               (act_fn): SiLUActivation()
             )
-            (shared_expert_gate): Linear(in_features=4, out_features=1, bias=False)
+            (shared_expert_gate): Linear(in_features=8, out_features=1, bias=False)
           )
           (ple): Qwen4ExpTextPLELayer(
             (ple_embedding): Qwen4ExpTextNGramEmbedding(
               (ngram_embedding): Embedding(9216, 1)
             )
-            (key_proj): Linear(in_features=16, out_features=16, bias=False)
-            (value_proj): Linear(in_features=16, out_features=4, bias=False)
-            (norm_key): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (norm_query): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (norm_conv): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (conv1d): Conv1d(16, 16, kernel_size=(4,), stride=(1,), dilation=(3,), groups=16, bias=False)
+            (key_proj): Linear(in_features=16, out_features=32, bias=False)
+            (value_proj): Linear(in_features=16, out_features=8, bias=False)
+            (norm_key): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (norm_query): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (norm_conv): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), dilation=(3,), groups=32, bias=False)
           )
           (attn_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
           (mlp_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
         )
         (2): Qwen4ExpTextDecoderLayer(
           (linear_attn): Qwen4ExpTextGatedDeltaNet(
             (conv1d): Conv1d(640, 640, kernel_size=(4,), stride=(1,), padding=(3,), groups=640, bias=False)
             (norm): Qwen4ExpTextRMSNormGated()
-            (out_proj): Linear(in_features=384, out_features=4, bias=False)
-            (in_proj_qkv): Linear(in_features=4, out_features=640, bias=False)
-            (in_proj_z): Linear(in_features=4, out_features=384, bias=False)
-            (in_proj_b): Linear(in_features=4, out_features=12, bias=False)
-            (in_proj_a): Linear(in_features=4, out_features=12, bias=False)
+            (out_proj): Linear(in_features=384, out_features=8, bias=False)
+            (in_proj_qkv): Linear(in_features=8, out_features=640, bias=False)
+            (in_proj_z): Linear(in_features=8, out_features=384, bias=False)
+            (in_proj_b): Linear(in_features=8, out_features=12, bias=False)
+            (in_proj_a): Linear(in_features=8, out_features=12, bias=False)
           )
           (mlp): Qwen4ExpTextSparseMoeBlock(
             (gate): Qwen4ExpTextTopKRouter()
@@ -328,36 +344,36 @@ Qwen4ExpForConditionalGeneration(
               (act_fn): SiLUActivation()
             )
             (shared_expert): Qwen4ExpTextMLP(
-              (gate_proj): Linear(in_features=4, out_features=128, bias=False)
-              (up_proj): Linear(in_features=4, out_features=128, bias=False)
-              (down_proj): Linear(in_features=128, out_features=4, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=128, bias=False)
+              (up_proj): Linear(in_features=8, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=8, bias=False)
               (act_fn): SiLUActivation()
             )
-            (shared_expert_gate): Linear(in_features=4, out_features=1, bias=False)
+            (shared_expert_gate): Linear(in_features=8, out_features=1, bias=False)
           )
           (attn_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
           (mlp_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
         )
         (3): Qwen4ExpTextDecoderLayer(
           (self_attn): Qwen4ExpTextAttention(
-            (q_proj): Linear(in_features=4, out_features=768, bias=False)
-            (k_proj): Linear(in_features=4, out_features=32, bias=False)
-            (v_proj): Linear(in_features=4, out_features=32, bias=False)
-            (o_proj): Linear(in_features=384, out_features=4, bias=False)
+            (q_proj): Linear(in_features=8, out_features=768, bias=False)
+            (k_proj): Linear(in_features=8, out_features=32, bias=False)
+            (v_proj): Linear(in_features=8, out_features=32, bias=False)
+            (o_proj): Linear(in_features=384, out_features=8, bias=False)
             (q_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
             (k_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
             (indexer): Qwen4ExpTextQSAIndexer(
-              (index_qk_proj): Linear(in_features=4, out_features=640, bias=False)
+              (index_qk_proj): Linear(in_features=8, out_features=640, bias=False)
               (q_layernorm): Qwen4ExpTextRMSNorm((128,), eps=1e-06)
               (k_layernorm): Qwen4ExpTextRMSNorm((128,), eps=1e-06)
             )
@@ -368,35 +384,35 @@ Qwen4ExpForConditionalGeneration(
               (act_fn): SiLUActivation()
             )
             (shared_expert): Qwen4ExpTextMLP(
-              (gate_proj): Linear(in_features=4, out_features=128, bias=False)
-              (up_proj): Linear(in_features=4, out_features=128, bias=False)
-              (down_proj): Linear(in_features=128, out_features=4, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=128, bias=False)
+              (up_proj): Linear(in_features=8, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=8, bias=False)
               (act_fn): SiLUActivation()
             )
-            (shared_expert_gate): Linear(in_features=4, out_features=1, bias=False)
+            (shared_expert_gate): Linear(in_features=8, out_features=1, bias=False)
           )
           (attn_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
           (mlp_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
         )
         (4-6): 3 x Qwen4ExpTextDecoderLayer(
           (linear_attn): Qwen4ExpTextGatedDeltaNet(
             (conv1d): Conv1d(640, 640, kernel_size=(4,), stride=(1,), padding=(3,), groups=640, bias=False)
             (norm): Qwen4ExpTextRMSNormGated()
-            (out_proj): Linear(in_features=384, out_features=4, bias=False)
-            (in_proj_qkv): Linear(in_features=4, out_features=640, bias=False)
-            (in_proj_z): Linear(in_features=4, out_features=384, bias=False)
-            (in_proj_b): Linear(in_features=4, out_features=12, bias=False)
-            (in_proj_a): Linear(in_features=4, out_features=12, bias=False)
+            (out_proj): Linear(in_features=384, out_features=8, bias=False)
+            (in_proj_qkv): Linear(in_features=8, out_features=640, bias=False)
+            (in_proj_z): Linear(in_features=8, out_features=384, bias=False)
+            (in_proj_b): Linear(in_features=8, out_features=12, bias=False)
+            (in_proj_a): Linear(in_features=8, out_features=12, bias=False)
           )
           (mlp): Qwen4ExpTextSparseMoeBlock(
             (gate): Qwen4ExpTextTopKRouter()
@@ -404,36 +420,36 @@ Qwen4ExpForConditionalGeneration(
               (act_fn): SiLUActivation()
             )
             (shared_expert): Qwen4ExpTextMLP(
-              (gate_proj): Linear(in_features=4, out_features=128, bias=False)
-              (up_proj): Linear(in_features=4, out_features=128, bias=False)
-              (down_proj): Linear(in_features=128, out_features=4, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=128, bias=False)
+              (up_proj): Linear(in_features=8, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=8, bias=False)
               (act_fn): SiLUActivation()
             )
-            (shared_expert_gate): Linear(in_features=4, out_features=1, bias=False)
+            (shared_expert_gate): Linear(in_features=8, out_features=1, bias=False)
           )
           (attn_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
           (mlp_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
         )
         (7): Qwen4ExpTextDecoderLayer(
           (self_attn): Qwen4ExpTextAttention(
-            (q_proj): Linear(in_features=4, out_features=768, bias=False)
-            (k_proj): Linear(in_features=4, out_features=32, bias=False)
-            (v_proj): Linear(in_features=4, out_features=32, bias=False)
-            (o_proj): Linear(in_features=384, out_features=4, bias=False)
+            (q_proj): Linear(in_features=8, out_features=768, bias=False)
+            (k_proj): Linear(in_features=8, out_features=32, bias=False)
+            (v_proj): Linear(in_features=8, out_features=32, bias=False)
+            (o_proj): Linear(in_features=384, out_features=8, bias=False)
             (q_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
             (k_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
             (indexer): Qwen4ExpTextQSAIndexer(
-              (index_qk_proj): Linear(in_features=4, out_features=640, bias=False)
+              (index_qk_proj): Linear(in_features=8, out_features=640, bias=False)
               (q_layernorm): Qwen4ExpTextRMSNorm((128,), eps=1e-06)
               (k_layernorm): Qwen4ExpTextRMSNorm((128,), eps=1e-06)
             )
@@ -444,52 +460,52 @@ Qwen4ExpForConditionalGeneration(
               (act_fn): SiLUActivation()
             )
             (shared_expert): Qwen4ExpTextMLP(
-              (gate_proj): Linear(in_features=4, out_features=128, bias=False)
-              (up_proj): Linear(in_features=4, out_features=128, bias=False)
-              (down_proj): Linear(in_features=128, out_features=4, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=128, bias=False)
+              (up_proj): Linear(in_features=8, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=8, bias=False)
               (act_fn): SiLUActivation()
             )
-            (shared_expert_gate): Linear(in_features=4, out_features=1, bias=False)
+            (shared_expert_gate): Linear(in_features=8, out_features=1, bias=False)
           )
           (attn_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
           (mlp_hyper_connection): Qwen4ExpTextGatedResidual(
-            (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-            (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-            (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-            (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+            (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+            (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+            (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+            (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
           )
         )
       )
       (rotary_emb): Qwen4ExpTextRotaryEmbedding()
       (hyper_connection_mixer): Qwen4ExpTextGatedResidual(
-        (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-        (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-        (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
+        (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+        (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+        (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
       )
     )
   )
-  (lm_head): Linear(in_features=4, out_features=248320, bias=False)
+  (lm_head): Linear(in_features=8, out_features=248320, bias=False)
   (mtp): ModuleDict(
-    (fc_embedding): Linear(in_features=4, out_features=4, bias=False)
-    (fc_hidden): Linear(in_features=4, out_features=4, bias=False)
-    (pre_fc_norm_embedding): Qwen4ExpTextRMSNorm((4,), eps=1e-06)
-    (pre_fc_norm_hidden): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
+    (fc_embedding): Linear(in_features=8, out_features=8, bias=False)
+    (fc_hidden): Linear(in_features=8, out_features=8, bias=False)
+    (pre_fc_norm_embedding): Qwen4ExpTextRMSNorm((8,), eps=1e-06)
+    (pre_fc_norm_hidden): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
     (layers): ModuleList(
       (0): Qwen4ExpTextDecoderLayer(
         (self_attn): Qwen4ExpTextAttention(
-          (q_proj): Linear(in_features=4, out_features=768, bias=False)
-          (k_proj): Linear(in_features=4, out_features=32, bias=False)
-          (v_proj): Linear(in_features=4, out_features=32, bias=False)
-          (o_proj): Linear(in_features=384, out_features=4, bias=False)
+          (q_proj): Linear(in_features=8, out_features=768, bias=False)
+          (k_proj): Linear(in_features=8, out_features=32, bias=False)
+          (v_proj): Linear(in_features=8, out_features=32, bias=False)
+          (o_proj): Linear(in_features=384, out_features=8, bias=False)
           (q_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
           (k_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
           (indexer): Qwen4ExpTextQSAIndexer(
-            (index_qk_proj): Linear(in_features=4, out_features=640, bias=False)
+            (index_qk_proj): Linear(in_features=8, out_features=640, bias=False)
             (q_layernorm): Qwen4ExpTextRMSNorm((128,), eps=1e-06)
             (k_layernorm): Qwen4ExpTextRMSNorm((128,), eps=1e-06)
           )
@@ -500,31 +516,31 @@ Qwen4ExpForConditionalGeneration(
             (act_fn): SiLUActivation()
           )
           (shared_expert): Qwen4ExpTextMLP(
-            (gate_proj): Linear(in_features=4, out_features=128, bias=False)
-            (up_proj): Linear(in_features=4, out_features=128, bias=False)
-            (down_proj): Linear(in_features=128, out_features=4, bias=False)
+            (gate_proj): Linear(in_features=8, out_features=128, bias=False)
+            (up_proj): Linear(in_features=8, out_features=128, bias=False)
+            (down_proj): Linear(in_features=128, out_features=8, bias=False)
             (act_fn): SiLUActivation()
           )
-          (shared_expert_gate): Linear(in_features=4, out_features=1, bias=False)
+          (shared_expert_gate): Linear(in_features=8, out_features=1, bias=False)
         )
         (attn_hyper_connection): Qwen4ExpTextGatedResidual(
-          (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-          (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-          (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-          (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+          (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+          (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+          (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+          (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
         )
         (mlp_hyper_connection): Qwen4ExpTextGatedResidual(
-          (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-          (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-          (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
-          (block_inject_weight): Linear(in_features=16, out_features=4, bias=False)
+          (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+          (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+          (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
+          (block_inject_weight): Linear(in_features=32, out_features=4, bias=False)
         )
       )
     )
     (hyper_connection_mixer): Qwen4ExpTextGatedResidual(
-      (hc_norm): Qwen4ExpTextRMSNorm((16,), eps=1e-06)
-      (input_mix_weight_down): Linear(in_features=16, out_features=4, bias=False)
-      (input_mix_weight_up): Linear(in_features=4, out_features=16, bias=False)
+      (hc_norm): Qwen4ExpTextRMSNorm((32,), eps=1e-06)
+      (input_mix_weight_down): Linear(in_features=32, out_features=4, bias=False)
+      (input_mix_weight_up): Linear(in_features=4, out_features=32, bias=False)
     )
   )
 )
@@ -534,25 +550,26 @@ Qwen4ExpForConditionalGeneration(
 
 ### Test environment:
 
-- Pillow: 12.2.0
-- accelerate: 1.15.0
-- av: 19.0.2
-- flashinfer-python: 0.7.2
-- huggingface_hub: 1.33.0
-- kernels: 0.17.0
+- Pillow: 13.0.0
+- flashinfer-python: 0.7.0.post1
+- huggingface_hub: 2.3.0
 - numpy: 2.3.5
 - safetensors: 0.8.0
-- sglang: 0.5.23.dev384+g9ab74b851
-- torch: 2.14.1
+- tokenizers: 0.23.3
+- torch: 2.13.0
 - torchcodec: 0.17.0
 - transformers: 5.19.0
-- Python: 3.12.13
-- CUDA: 13.0
-- libstdc++: 13.4.0
+- python (vLLM): 3.12.13
+- CUDA (vLLM): 13.0
+- torch (vLLM): 2.13.0+cu130
+- transformers (vLLM): 5.19.0
+- vllm: 0.32.0
+- flashinfer-python (vLLM): 0.7.0.post1
 
 ### Change log:
 
 - 2026-10-23: Initial version.
 - 2026-10-23: Creation examples use only public dependencies; source assets are copied unchanged.
+- 2026-10-24: Verified vLLM TP=4 with CUDA Graph, MTP, text, image, and video requests.
 
 This repo is generated by GPT-6.1-Sol

config.json

@@ -17,7 +17,7 @@
     "head_dim": 32,
     "heads_per_ngram": 8,
     "hidden_act": "silu",
-    "hidden_size": 4,
+    "hidden_size": 8,
     "indexer_budget": 2048,
     "indexer_compress_ratio": 4,
     "indexer_head_dim": 128,
@@ -104,7 +104,7 @@
     "model_type": "qwen4_exp",
     "num_heads": 4,
     "num_position_embeddings": 2304,
-    "out_hidden_size": 4,
+    "out_hidden_size": 8,
     "patch_size": 16,
     "spatial_merge_size": 2,
     "temporal_patch_size": 2

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:351faaca22aef4a4d139e12feecacbdfdb564a985cbcfb01030a9d2eda61fea9
-size 5874184
+oid sha256:0fd07e2bb01c975f7059d68ccac1654338bae0324df77ad1c9d322a222b3d600
+size 10886632