Upload folder using huggingface_hub

.meta/create_model.py

@@ -195,6 +195,9 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
     )
     # Kernel-sensitive geometry mirrors tiny-random/kimi-k3: KDA head_dim and MLA ranks stay at
     # source values, only head count and q LoRA / layer count / widths shrink.
+    # Experts are shrunk aggressively (16 experts x 128 intermediate, top-k 8, still group-32
+    # aligned for MXFP4) because MoE tensors otherwise dominate the checkpoint; the source uses
+    # 896 experts at 3,072/3,584 with top-k 16.
     text_config.update(
         {
             "attn_res_block_size": 8,
@@ -202,13 +205,14 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
             "hidden_size": 8,
             "intermediate_size": 32,
             "kv_lora_rank": 512,
-            "moe_intermediate_size": 256,
+            "moe_intermediate_size": 128,
             "num_attention_heads": 8,
-            "num_experts": 64,
+            "num_experts": 16,
+            "num_experts_per_token": 8,
             "num_hidden_layers": 17,
             "num_key_value_heads": 8,
-            "q_lora_rank": 256,
-            "routed_expert_hidden_size": 256,
+            "q_lora_rank": 128,
+            "routed_expert_hidden_size": 128,
             "_attn_implementation": "eager",
         }
     )
@@ -337,5 +341,5 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
 create_model(
     source_model_id="moonshotai/Kimi-K3.1-Pro",
     source_revision="6d0a0202956796543364d666ab19c6ad1a494ee1",
-    save_folder="/tmp/kimi-k31",
+    save_folder="/tmp/kimi-k31-small",
 )

.meta/model.log

@@ -79,22 +79,22 @@ KimiK3ForConditionalGeneration(
           )
           (block_sparse_moe): KimiSparseMoeBlock(
             (experts): ModuleList(
-              (0-63): 64 x KimiBlockSparseMLP(
-                (w1): Linear(in_features=256, out_features=256, bias=False)
-                (w2): Linear(in_features=256, out_features=256, bias=False)
-                (w3): Linear(in_features=256, out_features=256, bias=False)
+              (0-15): 16 x KimiBlockSparseMLP(
+                (w1): Linear(in_features=128, out_features=128, bias=False)
+                (w2): Linear(in_features=128, out_features=128, bias=False)
+                (w3): Linear(in_features=128, out_features=128, bias=False)
                 (act_fn): SituAndMul()
               )
             )
             (gate): KimiMoEGate()
             (shared_experts): KimiMLP(
-              (gate_proj): Linear(in_features=8, out_features=512, bias=False)
-              (up_proj): Linear(in_features=8, out_features=512, bias=False)
-              (down_proj): Linear(in_features=512, out_features=8, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
+              (up_proj): Linear(in_features=8, out_features=256, bias=False)
+              (down_proj): Linear(in_features=256, out_features=8, bias=False)
               (act_fn): SituAndMul()
             )
-            (routed_expert_down_proj): Linear(in_features=8, out_features=256, bias=False)
-            (routed_expert_up_proj): Linear(in_features=256, out_features=8, bias=False)
+            (routed_expert_down_proj): Linear(in_features=8, out_features=128, bias=False)
+            (routed_expert_up_proj): Linear(in_features=128, out_features=8, bias=False)
             (routed_expert_norm): KimiRMSNorm()
           )
           (input_layernorm): KimiRMSNorm()
@@ -106,9 +106,9 @@ KimiK3ForConditionalGeneration(
         )
         (3): KimiDecoderLayer(
           (self_attn): KimiMLAAttention(
-            (q_a_proj): Linear(in_features=8, out_features=256, bias=False)
+            (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
             (q_a_layernorm): KimiRMSNorm()
-            (q_b_proj): Linear(in_features=256, out_features=1536, bias=False)
+            (q_b_proj): Linear(in_features=128, out_features=1536, bias=False)
             (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
             (kv_a_layernorm): KimiRMSNorm()
             (kv_b_proj): Linear(in_features=512, out_features=2048, bias=False)
@@ -117,22 +117,22 @@ KimiK3ForConditionalGeneration(
           )
           (block_sparse_moe): KimiSparseMoeBlock(
             (experts): ModuleList(
-              (0-63): 64 x KimiBlockSparseMLP(
-                (w1): Linear(in_features=256, out_features=256, bias=False)
-                (w2): Linear(in_features=256, out_features=256, bias=False)
-                (w3): Linear(in_features=256, out_features=256, bias=False)
+              (0-15): 16 x KimiBlockSparseMLP(
+                (w1): Linear(in_features=128, out_features=128, bias=False)
+                (w2): Linear(in_features=128, out_features=128, bias=False)
+                (w3): Linear(in_features=128, out_features=128, bias=False)
                 (act_fn): SituAndMul()
               )
             )
             (gate): KimiMoEGate()
             (shared_experts): KimiMLP(
-              (gate_proj): Linear(in_features=8, out_features=512, bias=False)
-              (up_proj): Linear(in_features=8, out_features=512, bias=False)
-              (down_proj): Linear(in_features=512, out_features=8, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
+              (up_proj): Linear(in_features=8, out_features=256, bias=False)
+              (down_proj): Linear(in_features=256, out_features=8, bias=False)
               (act_fn): SituAndMul()
             )
-            (routed_expert_down_proj): Linear(in_features=8, out_features=256, bias=False)
-            (routed_expert_up_proj): Linear(in_features=256, out_features=8, bias=False)
+            (routed_expert_down_proj): Linear(in_features=8, out_features=128, bias=False)
+            (routed_expert_up_proj): Linear(in_features=128, out_features=8, bias=False)
             (routed_expert_norm): KimiRMSNorm()
           )
           (input_layernorm): KimiRMSNorm()
@@ -159,22 +159,22 @@ KimiK3ForConditionalGeneration(
           )
           (block_sparse_moe): KimiSparseMoeBlock(
             (experts): ModuleList(
-              (0-63): 64 x KimiBlockSparseMLP(
-                (w1): Linear(in_features=256, out_features=256, bias=False)
-                (w2): Linear(in_features=256, out_features=256, bias=False)
-                (w3): Linear(in_features=256, out_features=256, bias=False)
+              (0-15): 16 x KimiBlockSparseMLP(
+                (w1): Linear(in_features=128, out_features=128, bias=False)
+                (w2): Linear(in_features=128, out_features=128, bias=False)
+                (w3): Linear(in_features=128, out_features=128, bias=False)
                 (act_fn): SituAndMul()
               )
             )
             (gate): KimiMoEGate()
             (shared_experts): KimiMLP(
-              (gate_proj): Linear(in_features=8, out_features=512, bias=False)
-              (up_proj): Linear(in_features=8, out_features=512, bias=False)
-              (down_proj): Linear(in_features=512, out_features=8, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
+              (up_proj): Linear(in_features=8, out_features=256, bias=False)
+              (down_proj): Linear(in_features=256, out_features=8, bias=False)
               (act_fn): SituAndMul()
             )
-            (routed_expert_down_proj): Linear(in_features=8, out_features=256, bias=False)
-            (routed_expert_up_proj): Linear(in_features=256, out_features=8, bias=False)
+            (routed_expert_down_proj): Linear(in_features=8, out_features=128, bias=False)
+            (routed_expert_up_proj): Linear(in_features=128, out_features=8, bias=False)
             (routed_expert_norm): KimiRMSNorm()
           )
           (input_layernorm): KimiRMSNorm()
@@ -186,9 +186,9 @@ KimiK3ForConditionalGeneration(
         )
         (7): KimiDecoderLayer(
           (self_attn): KimiMLAAttention(
-            (q_a_proj): Linear(in_features=8, out_features=256, bias=False)
+            (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
             (q_a_layernorm): KimiRMSNorm()
-            (q_b_proj): Linear(in_features=256, out_features=1536, bias=False)
+            (q_b_proj): Linear(in_features=128, out_features=1536, bias=False)
             (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
             (kv_a_layernorm): KimiRMSNorm()
             (kv_b_proj): Linear(in_features=512, out_features=2048, bias=False)
@@ -197,22 +197,22 @@ KimiK3ForConditionalGeneration(
           )
           (block_sparse_moe): KimiSparseMoeBlock(
             (experts): ModuleList(
-              (0-63): 64 x KimiBlockSparseMLP(
-                (w1): Linear(in_features=256, out_features=256, bias=False)
-                (w2): Linear(in_features=256, out_features=256, bias=False)
-                (w3): Linear(in_features=256, out_features=256, bias=False)
+              (0-15): 16 x KimiBlockSparseMLP(
+                (w1): Linear(in_features=128, out_features=128, bias=False)
+                (w2): Linear(in_features=128, out_features=128, bias=False)
+                (w3): Linear(in_features=128, out_features=128, bias=False)
                 (act_fn): SituAndMul()
               )
             )
             (gate): KimiMoEGate()
             (shared_experts): KimiMLP(
-              (gate_proj): Linear(in_features=8, out_features=512, bias=False)
-              (up_proj): Linear(in_features=8, out_features=512, bias=False)
-              (down_proj): Linear(in_features=512, out_features=8, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
+              (up_proj): Linear(in_features=8, out_features=256, bias=False)
+              (down_proj): Linear(in_features=256, out_features=8, bias=False)
               (act_fn): SituAndMul()
             )
-            (routed_expert_down_proj): Linear(in_features=8, out_features=256, bias=False)
-            (routed_expert_up_proj): Linear(in_features=256, out_features=8, bias=False)
+            (routed_expert_down_proj): Linear(in_features=8, out_features=128, bias=False)
+            (routed_expert_up_proj): Linear(in_features=128, out_features=8, bias=False)
             (routed_expert_norm): KimiRMSNorm()
           )
           (input_layernorm): KimiRMSNorm()
@@ -239,22 +239,22 @@ KimiK3ForConditionalGeneration(
           )
           (block_sparse_moe): KimiSparseMoeBlock(
             (experts): ModuleList(
-              (0-63): 64 x KimiBlockSparseMLP(
-                (w1): Linear(in_features=256, out_features=256, bias=False)
-                (w2): Linear(in_features=256, out_features=256, bias=False)
-                (w3): Linear(in_features=256, out_features=256, bias=False)
+              (0-15): 16 x KimiBlockSparseMLP(
+                (w1): Linear(in_features=128, out_features=128, bias=False)
+                (w2): Linear(in_features=128, out_features=128, bias=False)
+                (w3): Linear(in_features=128, out_features=128, bias=False)
                 (act_fn): SituAndMul()
               )
             )
             (gate): KimiMoEGate()
             (shared_experts): KimiMLP(
-              (gate_proj): Linear(in_features=8, out_features=512, bias=False)
-              (up_proj): Linear(in_features=8, out_features=512, bias=False)
-              (down_proj): Linear(in_features=512, out_features=8, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
+              (up_proj): Linear(in_features=8, out_features=256, bias=False)
+              (down_proj): Linear(in_features=256, out_features=8, bias=False)
               (act_fn): SituAndMul()
             )
-            (routed_expert_down_proj): Linear(in_features=8, out_features=256, bias=False)
-            (routed_expert_up_proj): Linear(in_features=256, out_features=8, bias=False)
+            (routed_expert_down_proj): Linear(in_features=8, out_features=128, bias=False)
+            (routed_expert_up_proj): Linear(in_features=128, out_features=8, bias=False)
             (routed_expert_norm): KimiRMSNorm()
           )
           (input_layernorm): KimiRMSNorm()
@@ -266,9 +266,9 @@ KimiK3ForConditionalGeneration(
         )
         (11): KimiDecoderLayer(
           (self_attn): KimiMLAAttention(
-            (q_a_proj): Linear(in_features=8, out_features=256, bias=False)
+            (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
             (q_a_layernorm): KimiRMSNorm()
-            (q_b_proj): Linear(in_features=256, out_features=1536, bias=False)
+            (q_b_proj): Linear(in_features=128, out_features=1536, bias=False)
             (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
             (kv_a_layernorm): KimiRMSNorm()
             (kv_b_proj): Linear(in_features=512, out_features=2048, bias=False)
@@ -277,22 +277,22 @@ KimiK3ForConditionalGeneration(
           )
           (block_sparse_moe): KimiSparseMoeBlock(
             (experts): ModuleList(
-              (0-63): 64 x KimiBlockSparseMLP(
-                (w1): Linear(in_features=256, out_features=256, bias=False)
-                (w2): Linear(in_features=256, out_features=256, bias=False)
-                (w3): Linear(in_features=256, out_features=256, bias=False)
+              (0-15): 16 x KimiBlockSparseMLP(
+                (w1): Linear(in_features=128, out_features=128, bias=False)
+                (w2): Linear(in_features=128, out_features=128, bias=False)
+                (w3): Linear(in_features=128, out_features=128, bias=False)
                 (act_fn): SituAndMul()
               )
             )
             (gate): KimiMoEGate()
             (shared_experts): KimiMLP(
-              (gate_proj): Linear(in_features=8, out_features=512, bias=False)
-              (up_proj): Linear(in_features=8, out_features=512, bias=False)
-              (down_proj): Linear(in_features=512, out_features=8, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
+              (up_proj): Linear(in_features=8, out_features=256, bias=False)
+              (down_proj): Linear(in_features=256, out_features=8, bias=False)
               (act_fn): SituAndMul()
             )
-            (routed_expert_down_proj): Linear(in_features=8, out_features=256, bias=False)
-            (routed_expert_up_proj): Linear(in_features=256, out_features=8, bias=False)
+            (routed_expert_down_proj): Linear(in_features=8, out_features=128, bias=False)
+            (routed_expert_up_proj): Linear(in_features=128, out_features=8, bias=False)
             (routed_expert_norm): KimiRMSNorm()
           )
           (input_layernorm): KimiRMSNorm()
@@ -319,22 +319,22 @@ KimiK3ForConditionalGeneration(
           )
           (block_sparse_moe): KimiSparseMoeBlock(
             (experts): ModuleList(
-              (0-63): 64 x KimiBlockSparseMLP(
-                (w1): Linear(in_features=256, out_features=256, bias=False)
-                (w2): Linear(in_features=256, out_features=256, bias=False)
-                (w3): Linear(in_features=256, out_features=256, bias=False)
+              (0-15): 16 x KimiBlockSparseMLP(
+                (w1): Linear(in_features=128, out_features=128, bias=False)
+                (w2): Linear(in_features=128, out_features=128, bias=False)
+                (w3): Linear(in_features=128, out_features=128, bias=False)
                 (act_fn): SituAndMul()
               )
             )
             (gate): KimiMoEGate()
             (shared_experts): KimiMLP(
-              (gate_proj): Linear(in_features=8, out_features=512, bias=False)
-              (up_proj): Linear(in_features=8, out_features=512, bias=False)
-              (down_proj): Linear(in_features=512, out_features=8, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
+              (up_proj): Linear(in_features=8, out_features=256, bias=False)
+              (down_proj): Linear(in_features=256, out_features=8, bias=False)
               (act_fn): SituAndMul()
             )
-            (routed_expert_down_proj): Linear(in_features=8, out_features=256, bias=False)
-            (routed_expert_up_proj): Linear(in_features=256, out_features=8, bias=False)
+            (routed_expert_down_proj): Linear(in_features=8, out_features=128, bias=False)
+            (routed_expert_up_proj): Linear(in_features=128, out_features=8, bias=False)
             (routed_expert_norm): KimiRMSNorm()
           )
           (input_layernorm): KimiRMSNorm()
@@ -346,9 +346,9 @@ KimiK3ForConditionalGeneration(
         )
         (15-16): 2 x KimiDecoderLayer(
           (self_attn): KimiMLAAttention(
-            (q_a_proj): Linear(in_features=8, out_features=256, bias=False)
+            (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
             (q_a_layernorm): KimiRMSNorm()
-            (q_b_proj): Linear(in_features=256, out_features=1536, bias=False)
+            (q_b_proj): Linear(in_features=128, out_features=1536, bias=False)
             (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
             (kv_a_layernorm): KimiRMSNorm()
             (kv_b_proj): Linear(in_features=512, out_features=2048, bias=False)
@@ -357,22 +357,22 @@ KimiK3ForConditionalGeneration(
           )
           (block_sparse_moe): KimiSparseMoeBlock(
             (experts): ModuleList(
-              (0-63): 64 x KimiBlockSparseMLP(
-                (w1): Linear(in_features=256, out_features=256, bias=False)
-                (w2): Linear(in_features=256, out_features=256, bias=False)
-                (w3): Linear(in_features=256, out_features=256, bias=False)
+              (0-15): 16 x KimiBlockSparseMLP(
+                (w1): Linear(in_features=128, out_features=128, bias=False)
+                (w2): Linear(in_features=128, out_features=128, bias=False)
+                (w3): Linear(in_features=128, out_features=128, bias=False)
                 (act_fn): SituAndMul()
               )
             )
             (gate): KimiMoEGate()
             (shared_experts): KimiMLP(
-              (gate_proj): Linear(in_features=8, out_features=512, bias=False)
-              (up_proj): Linear(in_features=8, out_features=512, bias=False)
-              (down_proj): Linear(in_features=512, out_features=8, bias=False)
+              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
+              (up_proj): Linear(in_features=8, out_features=256, bias=False)
+              (down_proj): Linear(in_features=256, out_features=8, bias=False)
               (act_fn): SituAndMul()
             )
-            (routed_expert_down_proj): Linear(in_features=8, out_features=256, bias=False)
-            (routed_expert_up_proj): Linear(in_features=256, out_features=8, bias=False)
+            (routed_expert_down_proj): Linear(in_features=8, out_features=128, bias=False)
+            (routed_expert_up_proj): Linear(in_features=128, out_features=8, bias=False)
             (routed_expert_norm): KimiRMSNorm()
           )
           (input_layernorm): KimiRMSNorm()

README.md

@@ -9,14 +9,14 @@ This is a randomly initialized tiny model based on [moonshotai/Kimi-K3.1-Pro](ht
 
 | Feature | Source | This repo |
 |---|---|---|
-| Checkpoint tensor elements | 57.36B BF16, 11.12M F32, 1.45T U8 | 17.73M BF16, 163.84K F32, 106.95M U8 |
-| Safetensors weight files | 1.56 TB (96 files) | 144.02 MB (model.safetensors) |
+| Checkpoint tensor elements | 57.36B BF16, 11.12M F32, 1.45T U8 | 13.98M BF16, 163.07K F32, 6.68M U8 |
+| Safetensors weight files | 1.56 TB (96 files) | 35.58 MB (model.safetensors) |
 | Total / DSA-gated layers | 93 / 24 (plus 69 KDA, 1 dense) | 17 / 5 (plus 11 KDA, 1 dense) |
 | Attention composition | 69 KDA + 24 gated DSA | 11 KDA + 5 DSA cadence |
 | Hidden width / MLP width | 7,168 / 33,792 | 8 / 32 |
 | Full-attn Q heads / head dim | 96 / 128 (nope 128, rope 64, v 128) | 8 / 128 (same per-head dims) |
-| MLA q / kv LoRA rank | 1,536 / 512 | 256 / 512 |
-| Routed / shared experts, top-k | 896 / 2, 16 | 64 / 2, 16 |
+| MLA q / kv LoRA rank | 1,536 / 512 | 128 / 512 |
+| Routed / shared experts, top-k | 896 / 2, 16 | 16 / 2, 8 |
 | LSA index heads / head dim / topk | 32 / 128 / 2,048 | 32 / 128 / 2,048 (unchanged) |
 | Physical DSA indexer layers | 24 (dsa_layers - 1) | [3, 7, 11, 15, 16] |
 | Routed expert precision | MXFP4 packed (group 32, E8M0 scales) | Same |
@@ -29,7 +29,7 @@ Kimi K3.1 Pro ships configuration and processor code but no modeling implementat
 
 | File path | Size |
 |------|------|
-| model.safetensors | 144.0MB |
+| model.safetensors | 35.6MB |
 
 ### Example usage
 
@@ -71,27 +71,33 @@ def transformers_example(model_id: str):
 
 
 if __name__ == "__main__":
-    transformers_example(model_id="scratch/kimi-k31")
+    transformers_example(model_id="scratch/kimi-k31-small")
 ```
 
 #### vLLM serving (unverified)
 
+Requires a vLLM build with KimiK31 gated-DSA support (newer than the released wheel at tiny-model creation time). See https://recipes.vllm.ai/moonshotai/Kimi-K3.1-Pro.
+
+```bash
 # K3.1 gated-DSA serving requires a vLLM build with KimiK31 DSA support (newer than the
 # released wheel at tiny-model creation time). See https://recipes.vllm.ai/moonshotai/Kimi-K3.1-Pro
-vllm serve scratch/kimi-k31 \
+vllm serve scratch/kimi-k31-small \
     --trust-remote-code \
     --tensor-parallel-size 8 \
     --load-format safetensors \
     --max-model-len 131072 \
     --gpu-memory-utilization 0.90
-
+```
 
 #### SGLang serving (unverified)
 
+Requires an SGLang build newer than 0.5.23 with the KimiK31 DSA indexer (the gated-DSA kernels had not landed in the released wheel at tiny-model creation time).
+
+```bash
 # K3.1 gated-DSA serving requires an SGLang build newer than 0.5.23 (the KimiK31 DSA indexer
 # kernels had not landed in the released wheel at tiny-model creation time).
 python3 -m sglang.launch_server \
-    --model-path scratch/kimi-k31 \
+    --model-path scratch/kimi-k31-small \
     --trust-remote-code \
     --tp-size 8 \
     --chunked-prefill-size 16384 \
@@ -101,6 +107,7 @@ python3 -m sglang.launch_server \
     --reasoning-parser kimi_k3 \
     --tool-call-parser kimi_k3 \
     --mem-fraction-static 0.85
+```
 
 ### Creating this repo
 
@@ -110,6 +117,6 @@ python3 -m sglang.launch_server \
 
 ### Change log
 
-- 2026-11-05: Initial version. Kimi K3.1 Pro is K3 (69 KDA + 24 attention layers) with the 24 full-attention layers replaced by gated DSA sparse attention carrying an LSA indexer. The tiny model follows the proven tiny-random/kimi-k3 geometry (residual 8, 8 attention heads, q LoRA 256, kv LoRA 512, 17 layers in the 12 KDA + 5 DSA cadence) and keeps the source DSA indexer contract (32 index heads, head dim 128, topk 2048) on five physical layers. The MXFP4 routed-expert packing and K3 KDA/MLA compatibility patches are preserved; the released K3 remote modeling constructs the DSA layers as dense MLA because no released Transformers/SGLang runtime yet ships the K3.1 gated-DSA operator (the indexer tensors are included forward-compatibly).
+- 2026-11-05: Initial version. Kimi K3.1 Pro is K3 (69 KDA + 24 attention layers) with the 24 full-attention layers replaced by gated DSA sparse attention carrying an LSA indexer. The tiny model follows the proven tiny-random/kimi-k3 geometry (residual 8, 8 attention heads, q LoRA 128, kv LoRA 512, 17 layers in the 12 KDA + 5 DSA cadence) and keeps the source DSA indexer contract (32 index heads, head dim 128, topk 2048) on five physical layers. The MXFP4 routed-expert packing and K3 KDA/MLA compatibility patches are preserved; the released K3 remote modeling constructs the DSA layers as dense MLA because no released Transformers/SGLang runtime yet ships the K3.1 gated-DSA operator (the indexer tensors are included forward-compatibly).
 
 *This repo is generated by GPT-6.1-Sol.*

config.json

@@ -97,14 +97,14 @@
     "mla_use_nope": true,
     "mla_use_output_gate": true,
     "model_type": "kimi_linear",
-    "moe_intermediate_size": 256,
+    "moe_intermediate_size": 128,
     "moe_layer_freq": 1,
     "moe_renormalize": true,
     "moe_router_activation_func": "sigmoid",
     "num_attention_heads": 8,
     "num_expert_group": 1,
-    "num_experts": 64,
-    "num_experts_per_token": 16,
+    "num_experts": 16,
+    "num_experts_per_token": 8,
     "num_hidden_layers": 17,
     "num_key_value_heads": 8,
     "num_nextn_predict_layers": 0,
@@ -115,7 +115,7 @@
     "prefix": null,
     "problem_type": null,
     "pruned_heads": {},
-    "q_lora_rank": 256,
+    "q_lora_rank": 128,
     "qk_nope_head_dim": 128,
     "qk_rope_head_dim": 64,
     "return_dict": true,
@@ -125,7 +125,7 @@
       "rope_type": "default"
     },
     "rope_theta": 10000.0,
-    "routed_expert_hidden_size": 256,
+    "routed_expert_hidden_size": 128,
     "routed_scaling_factor": 1.0,
     "sep_token_id": null,
     "task_specific_params": null,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:181c021dfd2926da51ec6b2fb8c96a57a2f39f86afbed4306b77a4c6bd9456de
-size 144021592
+oid sha256:35842b8a041956803e5ea97e515fe0463a22a4e8b5ec1a5151868af0bebfd4be
+size 35578920