Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.meta/create_model.py

@@ -0,0 +1,76 @@
+def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
+    import json
+    from pathlib import Path
+
+    import torch
+    from huggingface_hub import hf_hub_download
+    from transformers import AutoConfig, AutoModel, set_seed
+
+    folder = Path(save_folder)
+    folder.mkdir(parents=True, exist_ok=True)
+    for name in (
+        "action_head.py",
+        "configuration_minicpm_vla.py",
+        "modeling_minicpm_vla.py",
+        "chat_template.jinja",
+        "generation_config.json",
+        "preprocessor_config.json",
+        "tokenizer.json",
+        "tokenizer_config.json",
+    ):
+        (folder / name).write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
+
+    config = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
+    text = config["vlm_config"]["text_config"]
+    text.update(
+        hidden_size=32,
+        intermediate_size=64,
+        head_dim=32,
+        num_attention_heads=1,
+        num_key_value_heads=1,
+        num_hidden_layers=8,
+        layer_types=[
+            "linear_attention",
+            "linear_attention",
+            "linear_attention",
+            "full_attention",
+        ]
+        * 2,
+        linear_key_head_dim=32,
+        linear_value_head_dim=32,
+        linear_num_key_heads=1,
+        linear_num_value_heads=1,
+    )
+    vision = config["vlm_config"]["vision_config"]
+    vision.update(hidden_size=32, intermediate_size=64, num_attention_heads=1, num_hidden_layers=2)
+    config["vlm_config"]["image_size"] = 112
+    (folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
+
+    action_head = (folder / "action_head.py").read_text()
+    for old, new in (
+        ("self.input_embedding_dim = 768", "self.input_embedding_dim = 32"),
+        ("input_embedding_dim=768,", "input_embedding_dim=32,"),
+        ("attention_head_dim=64,", "attention_head_dim=32,"),
+        ("num_attention_heads=12,", "num_attention_heads=1,"),
+        ("cross_attention_dim=1024,", "cross_attention_dim=32,"),
+        ("num_layers=16,", "num_layers=2,"),
+        ("output_dim=1024,", "output_dim=32,"),
+        ("hidden_dim=hidden_size,", "hidden_dim=32,"),
+        ("input_dim=self.model.config.output_dim,", "input_dim=32,"),
+    ):
+        if old not in action_head:
+            raise ValueError(f"Expected RobotManip action-head fragment is missing: {old}")
+        action_head = action_head.replace(old, new)
+    (folder / "action_head.py").write_text(action_head)
+
+    set_seed(42)
+    model = AutoModel.from_config(AutoConfig.from_pretrained(folder, trust_remote_code=True), trust_remote_code=True)
+    model.save_pretrained(folder, safe_serialization=True)
+    return str(model)
+
+
+create_model(
+    source_model_id="openbmb/MiniCPM-RobotManip",
+    source_revision="29cb257541e9f96f493abe21174f8892e5dffe3f",
+    save_folder="/tmp/minicpm-robot-manip",
+)

.meta/env.json

@@ -0,0 +1,39 @@
+{
+  "source_model": {
+    "name": "openbmb/MiniCPM-RobotManip",
+    "commit_id": "29cb257541e9f96f493abe21174f8892e5dffe3f"
+  },
+  "environments": {
+    "creation": {
+      "python": "3.12.13",
+      "packages": {
+        "diffusers": "0.42.0.dev0",
+        "huggingface_hub": "2.3.0",
+        "safetensors": "0.8.0",
+        "tokenizers": "0.23.3",
+        "torch": "2.13.0+cu126",
+        "transformers": "5.20.2"
+      },
+      "cuda": "12.6",
+      "hardware": {
+        "gpu_model": "mlx lab master节点不包含GPU。请使用mlx worker launch -- nvidia-smi 来检查GPU状态, 或通过mlx worker launch -- bash登入gpu worker使用gpu相关命令。也可以通过mlx worker launch -h 查看更多launch命令使用方法。 如果mlx worker命令使用遇到问题，可运行 mlx upgrade将命令行工具升级至最新版"
+      }
+    },
+    "transformers": {
+      "python": "3.12.13",
+      "packages": {
+        "Pillow": "12.3.0",
+        "diffusers": "0.42.0.dev0",
+        "huggingface_hub": "2.3.0",
+        "safetensors": "0.8.0",
+        "tokenizers": "0.23.3",
+        "torch": "2.13.0+cu126",
+        "transformers": "5.20.2"
+      },
+      "cuda": "12.6",
+      "hardware": {
+        "gpu_model": "NVIDIA H20"
+      }
+    }
+  }
+}

.meta/model.log

@@ -0,0 +1,194 @@
+MiniCPMV_VLA(
+  (vlm): MiniCPMV4_6ForConditionalGeneration(
+    (model): MiniCPMV4_6Model(
+      (vision_tower): MiniCPMV4_6VisionModel(
+        (embeddings): MiniCPMV4_6VisionEmbeddings(
+          (patch_embedding): Conv2d(3, 32, kernel_size=(14, 14), stride=(14, 14), padding=valid)
+          (position_embedding): Embedding(4900, 32)
+        )
+        (encoder): MiniCPMV4_6VisionEncoder(
+          (layers): ModuleList(
+            (0-1): 2 x MiniCPMV4_6VisionEncoderLayer(
+              (layer_norm1): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
+              (self_attn): MiniCPMV4_6VisionAttention(
+                (k_proj): Linear(in_features=32, out_features=32, bias=True)
+                (v_proj): Linear(in_features=32, out_features=32, bias=True)
+                (q_proj): Linear(in_features=32, out_features=32, bias=True)
+                (out_proj): Linear(in_features=32, out_features=32, bias=True)
+              )
+              (layer_norm2): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
+              (mlp): MiniCPMV4_6VisionMLP(
+                (activation_fn): GELUTanh()
+                (fc1): Linear(in_features=32, out_features=64, bias=True)
+                (fc2): Linear(in_features=64, out_features=32, bias=True)
+              )
+            )
+          )
+        )
+        (post_layernorm): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
+        (vit_merger): MiniCPMV4_6ViTWindowAttentionMerger(
+          (self_attn): MiniCPMV4_6VisionAttention(
+            (k_proj): Linear(in_features=32, out_features=32, bias=True)
+            (v_proj): Linear(in_features=32, out_features=32, bias=True)
+            (q_proj): Linear(in_features=32, out_features=32, bias=True)
+            (out_proj): Linear(in_features=32, out_features=32, bias=True)
+          )
+          (layer_norm1): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
+          (pre_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+          (linear_1): Linear(in_features=128, out_features=256, bias=True)
+          (act): GELUTanh()
+          (linear_2): Linear(in_features=256, out_features=32, bias=True)
+        )
+      )
+      (language_model): Qwen3_5TextModel(
+        (embed_tokens): Embedding(248094, 32)
+        (layers): ModuleList(
+          (0-2): 3 x Qwen3_5DecoderLayer(
+            (linear_attn): Qwen3_5GatedDeltaNet(
+              (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
+              (norm): Qwen3_5RMSNormGated()
+              (out_proj): Linear(in_features=32, out_features=32, bias=False)
+              (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
+              (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
+              (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
+              (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
+            )
+            (mlp): Qwen3_5MLP(
+              (gate_proj): Linear(in_features=32, out_features=64, bias=False)
+              (up_proj): Linear(in_features=32, out_features=64, bias=False)
+              (down_proj): Linear(in_features=64, out_features=32, bias=False)
+              (act_fn): SiLUActivation()
+            )
+            (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
+            (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
+          )
+          (3): Qwen3_5DecoderLayer(
+            (self_attn): Qwen3_5Attention(
+              (q_proj): Linear(in_features=32, out_features=64, bias=False)
+              (k_proj): Linear(in_features=32, out_features=32, bias=False)
+              (v_proj): Linear(in_features=32, out_features=32, bias=False)
+              (o_proj): Linear(in_features=32, out_features=32, bias=False)
+              (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+              (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+            )
+            (mlp): Qwen3_5MLP(
+              (gate_proj): Linear(in_features=32, out_features=64, bias=False)
+              (up_proj): Linear(in_features=32, out_features=64, bias=False)
+              (down_proj): Linear(in_features=64, out_features=32, bias=False)
+              (act_fn): SiLUActivation()
+            )
+            (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
+            (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
+          )
+          (4-6): 3 x Qwen3_5DecoderLayer(
+            (linear_attn): Qwen3_5GatedDeltaNet(
+              (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
+              (norm): Qwen3_5RMSNormGated()
+              (out_proj): Linear(in_features=32, out_features=32, bias=False)
+              (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
+              (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
+              (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
+              (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
+            )
+            (mlp): Qwen3_5MLP(
+              (gate_proj): Linear(in_features=32, out_features=64, bias=False)
+              (up_proj): Linear(in_features=32, out_features=64, bias=False)
+              (down_proj): Linear(in_features=64, out_features=32, bias=False)
+              (act_fn): SiLUActivation()
+            )
+            (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
+            (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
+          )
+          (7): Qwen3_5DecoderLayer(
+            (self_attn): Qwen3_5Attention(
+              (q_proj): Linear(in_features=32, out_features=64, bias=False)
+              (k_proj): Linear(in_features=32, out_features=32, bias=False)
+              (v_proj): Linear(in_features=32, out_features=32, bias=False)
+              (o_proj): Linear(in_features=32, out_features=32, bias=False)
+              (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+              (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+            )
+            (mlp): Qwen3_5MLP(
+              (gate_proj): Linear(in_features=32, out_features=64, bias=False)
+              (up_proj): Linear(in_features=32, out_features=64, bias=False)
+              (down_proj): Linear(in_features=64, out_features=32, bias=False)
+              (act_fn): SiLUActivation()
+            )
+            (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
+            (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
+          )
+        )
+        (norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+        (rotary_emb): Qwen3_5TextRotaryEmbedding()
+      )
+      (merger): MiniCPMV4_6Merger(
+        (mlp): ModuleList(
+          (0): MiniCPMV4_6DownsampleMLP(
+            (pre_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+            (linear_1): Linear(in_features=128, out_features=128, bias=True)
+            (act): GELU(approximate='none')
+            (linear_2): Linear(in_features=128, out_features=32, bias=True)
+          )
+        )
+      )
+    )
+    (lm_head): Linear(in_features=32, out_features=248094, bias=False)
+  )
+  (action_head): MiniCPMV_VLA_ActionHead(
+    (model): DiT(
+      (timestep_encoder): TimestepEncoder(
+        (time_proj): Timesteps()
+        (timestep_embedder): TimestepEmbedding(
+          (linear_1): Linear(in_features=256, out_features=32, bias=True)
+          (act): SiLU()
+          (linear_2): Linear(in_features=32, out_features=32, bias=True)
+        )
+      )
+      (transformer_blocks): ModuleList(
+        (0-1): 2 x BasicTransformerBlock(
+          (norm1): AdaLayerNorm(
+            (silu): SiLU()
+            (linear): Linear(in_features=32, out_features=64, bias=True)
+            (norm): LayerNorm((32,), eps=1e-05, elementwise_affine=False, bias=False)
+          )
+          (attn1): Attention(
+            (to_q): Linear(in_features=32, out_features=32, bias=True)
+            (to_k): Linear(in_features=32, out_features=32, bias=True)
+            (to_v): Linear(in_features=32, out_features=32, bias=True)
+            (to_out): ModuleList(
+              (0): Linear(in_features=32, out_features=32, bias=True)
+              (1): Dropout(p=0.2, inplace=False)
+            )
+          )
+          (norm3): LayerNorm((32,), eps=1e-05, elementwise_affine=False, bias=False)
+          (ff): FeedForward(
+            (net): ModuleList(
+              (0): GELU(
+                (proj): Linear(in_features=32, out_features=128, bias=True)
+              )
+              (1): Dropout(p=0.2, inplace=False)
+              (2): Linear(in_features=128, out_features=32, bias=True)
+              (3): Dropout(p=0.2, inplace=False)
+            )
+          )
+          (final_dropout): Dropout(p=0.2, inplace=False)
+        )
+      )
+      (norm_out): LayerNorm((32,), eps=1e-06, elementwise_affine=False, bias=False)
+      (proj_out_1): Linear(in_features=32, out_features=64, bias=True)
+      (proj_out_2): Linear(in_features=32, out_features=32, bias=True)
+    )
+    (action_encoder): MultiEmbodimentActionEncoder(
+      (W1): CategorySpecificLinear()
+      (W2): CategorySpecificLinear()
+      (W3): CategorySpecificLinear()
+      (pos_encoding): SinusoidalPositionalEncoding()
+    )
+    (action_decoder): CategorySpecificMLP(
+      (layer1): CategorySpecificLinear()
+      (layer2): CategorySpecificLinear()
+    )
+    (future_tokens): Embedding(32, 32)
+    (position_embedding): Embedding(1024, 32)
+  )
+)

README.md

@@ -0,0 +1,78 @@
+---
+library_name: transformers
+pipeline_tag: robotics
+base_model:
+- openbmb/MiniCPM-RobotManip
+---
+
+This is a randomly initialized tiny model based on [openbmb/MiniCPM-RobotManip](https://huggingface.co/openbmb/MiniCPM-RobotManip) for testing and debugging.
+
+| Feature | Source | This repo |
+|---|---|---|
+| Checkpoint tensor elements | 1.30B BF16, 235.81M F32 | 8.29M BF16, 459.10K F32 |
+| Safetensors weight files | 3.54 GB (model.safetensors) | 18.45 MB (model.safetensors) |
+| VLM text layers / width / head dimension | 24 / 1024 / 256 | 8 / 32 / 32 |
+| VLM attention cadence | 3 linear + 1 full, repeated | Two 3-linear + 1-full cycles |
+| Vision layers / width | 27 / 1152 | 2 / 32 |
+| Action diffusion blocks / width | 16 / 768 | 2 / 32 |
+| Action horizon / action-state dimensions | 30 / 80-80 | Same |
+| Embodiment categories | 32 | Same |
+| Text vocabulary rows | 248094 | Same |
+| Stored VLM / action-head precision | BF16 / FP32 | Same |
+| Input / output | Image, instruction, state → action chunk | Same |
+
+| File path | Size |
+|------|------|
+| model.safetensors | 18.4MB |
+
+### Example usage
+
+#### Transformers (✅ verified)
+
+```python
+def run_transformers(model_id: str):
+    import torch
+    from PIL import Image
+    from transformers import AutoModel, AutoProcessor
+
+    model = AutoModel.from_pretrained(model_id, trust_remote_code=True, dtype=torch.bfloat16).eval().cuda()
+    processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
+    image = Image.new("RGB", (112, 112), "blue")
+    inputs = processor.apply_chat_template(
+        [
+            {
+                "role": "user",
+                "content": [{"type": "image", "image": image}, {"type": "text", "text": "Pick up the blue object."}],
+            }
+        ],
+        tokenize=True,
+        add_generation_prompt=True,
+        return_dict=True,
+        return_tensors="pt",
+        processor_kwargs={"padding": False},
+    ).to(model.device)
+    action = model.predict_action(
+        state=torch.zeros(1, 1, 80, device=model.device),
+        embodiment_id=torch.zeros(1, dtype=torch.long, device=model.device),
+        **inputs,
+    )
+    assert action.shape == (1, 30, 80)
+    assert torch.isfinite(action).all()
+    print(action.shape)
+
+
+if __name__ == "__main__":
+    run_transformers(model_id="tiny-random/minicpm-robot-manip")
+```
+
+### Creating this repo
+
+- Code: [.meta/create_model.py](.meta/create_model.py)
+- Model structure: [.meta/model.log](.meta/model.log)
+- Environment: [.meta/env.json](.meta/env.json)
+
+### Change log
+
+- 2026-10-27: Created a tiny MiniCPM RobotManip vision-language-action policy.
+
+*This repo is generated by GPT-6.1-Sol.*

action_head.py

@@ -0,0 +1,387 @@
+from __future__ import annotations
+
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+from diffusers import ConfigMixin, ModelMixin
+from diffusers.configuration_utils import register_to_config
+from diffusers.models.attention import Attention, FeedForward
+from diffusers.models.embeddings import SinusoidalPositionalEmbedding, TimestepEmbedding, Timesteps
+from torch import nn
+
+
+class TimestepEncoder(nn.Module):
+    def __init__(self, embedding_dim: int):
+        super().__init__()
+        self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=1)
+        self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim)
+
+    def forward(self, timesteps: torch.Tensor) -> torch.Tensor:
+        return self.timestep_embedder(self.time_proj(timesteps).to(next(self.parameters()).dtype))
+
+
+class AdaLayerNorm(nn.Module):
+    def __init__(
+        self,
+        embedding_dim: int,
+        norm_elementwise_affine: bool = False,
+        norm_eps: float = 1e-5,
+    ):
+        super().__init__()
+        self.silu = nn.SiLU()
+        self.linear = nn.Linear(embedding_dim, embedding_dim * 2)
+        self.norm = nn.LayerNorm(embedding_dim, norm_eps, norm_elementwise_affine)
+
+    def forward(self, x: torch.Tensor, temb: torch.Tensor) -> torch.Tensor:
+        scale, shift = self.linear(self.silu(temb)).chunk(2, dim=1)
+        return self.norm(x) * (1 + scale[:, None]) + shift[:, None]
+
+
+class BasicTransformerBlock(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_attention_heads: int,
+        attention_head_dim: int,
+        dropout: float = 0.0,
+        cross_attention_dim: Optional[int] = None,
+        activation_fn: str = "geglu",
+        attention_bias: bool = False,
+        upcast_attention: bool = False,
+        norm_elementwise_affine: bool = True,
+        norm_type: str = "layer_norm",
+        norm_eps: float = 1e-5,
+        final_dropout: bool = False,
+        positional_embeddings: Optional[str] = None,
+        num_positional_embeddings: Optional[int] = None,
+        ff_inner_dim: Optional[int] = None,
+        ff_bias: bool = True,
+        attention_out_bias: bool = True,
+    ):
+        super().__init__()
+        if positional_embeddings and num_positional_embeddings is None:
+            raise ValueError("num_positional_embeddings is required for positional embeddings")
+        self.norm_type = norm_type
+        self.pos_embed = (
+            SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings)
+            if positional_embeddings == "sinusoidal"
+            else None
+        )
+        self.norm1 = (
+            AdaLayerNorm(dim)
+            if norm_type == "ada_norm"
+            else nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)
+        )
+        self.attn1 = Attention(
+            query_dim=dim,
+            heads=num_attention_heads,
+            dim_head=attention_head_dim,
+            dropout=dropout,
+            bias=attention_bias,
+            cross_attention_dim=cross_attention_dim,
+            upcast_attention=upcast_attention,
+            out_bias=attention_out_bias,
+        )
+        self.norm3 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine)
+        self.ff = FeedForward(
+            dim,
+            dropout=dropout,
+            activation_fn=activation_fn,
+            final_dropout=final_dropout,
+            inner_dim=ff_inner_dim,
+            bias=ff_bias,
+        )
+        self.final_dropout = nn.Dropout(dropout) if final_dropout else None
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.Tensor] = None,
+        temb: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        norm_hidden_states = (
+            self.norm1(hidden_states, temb) if self.norm_type == "ada_norm" else self.norm1(hidden_states)
+        )
+        if self.pos_embed is not None:
+            norm_hidden_states = self.pos_embed(norm_hidden_states)
+        attention_output = self.attn1(
+            norm_hidden_states,
+            encoder_hidden_states=encoder_hidden_states,
+            attention_mask=encoder_attention_mask,
+        )
+        if self.final_dropout is not None:
+            attention_output = self.final_dropout(attention_output)
+        hidden_states = attention_output + hidden_states
+        if hidden_states.ndim == 4:
+            hidden_states = hidden_states.squeeze(1)
+        hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states
+        if hidden_states.ndim == 4:
+            hidden_states = hidden_states.squeeze(1)
+        return hidden_states
+
+
+class DiT(ModelMixin, ConfigMixin):
+    _supports_gradient_checkpointing = True
+
+    @register_to_config
+    def __init__(
+        self,
+        num_attention_heads: int = 8,
+        attention_head_dim: int = 64,
+        output_dim: int = 26,
+        num_layers: int = 12,
+        dropout: float = 0.1,
+        attention_bias: bool = True,
+        activation_fn: str = "gelu-approximate",
+        num_embeds_ada_norm: Optional[int] = 1000,
+        upcast_attention: bool = False,
+        norm_type: str = "ada_norm",
+        norm_elementwise_affine: bool = False,
+        norm_eps: float = 1e-5,
+        max_num_positional_embeddings: int = 512,
+        compute_dtype: torch.dtype = torch.float32,
+        final_dropout: bool = True,
+        positional_embeddings: Optional[str] = "sinusoidal",
+        interleave_self_attention: bool = False,
+        cross_attention_dim: Optional[int] = None,
+        **kwargs,
+    ):
+        super().__init__()
+        self.attention_head_dim = attention_head_dim
+        self.inner_dim = self.config.num_attention_heads * self.config.attention_head_dim
+        self.gradient_checkpointing = False
+        self.timestep_encoder = TimestepEncoder(self.inner_dim)
+        self.transformer_blocks = nn.ModuleList(
+            [
+                BasicTransformerBlock(
+                    self.inner_dim,
+                    self.config.num_attention_heads,
+                    self.config.attention_head_dim,
+                    dropout=self.config.dropout,
+                    activation_fn=self.config.activation_fn,
+                    attention_bias=self.config.attention_bias,
+                    upcast_attention=self.config.upcast_attention,
+                    norm_type=norm_type,
+                    norm_elementwise_affine=self.config.norm_elementwise_affine,
+                    norm_eps=self.config.norm_eps,
+                    positional_embeddings=positional_embeddings,
+                    num_positional_embeddings=self.config.max_num_positional_embeddings,
+                    final_dropout=final_dropout,
+                    cross_attention_dim=(
+                        None if index % 2 == 1 and interleave_self_attention else cross_attention_dim
+                    ),
+                )
+                for index in range(self.config.num_layers)
+            ]
+        )
+        self.norm_out = nn.LayerNorm(self.inner_dim, elementwise_affine=False, eps=1e-6)
+        self.proj_out_1 = nn.Linear(self.inner_dim, 2 * self.inner_dim)
+        self.proj_out_2 = nn.Linear(self.inner_dim, self.config.output_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.Tensor,
+        timestep: Optional[torch.LongTensor] = None,
+        return_all_hidden_states: bool = False,
+        encoder_attention_mask: Optional[torch.Tensor] = None,
+    ):
+        time_embedding = self.timestep_encoder(timestep)
+        hidden_states = hidden_states.contiguous()
+        encoder_hidden_states = encoder_hidden_states.contiguous()
+        all_hidden_states = [hidden_states]
+        for index, block in enumerate(self.transformer_blocks):
+            self_attention = index % 2 == 1 and self.config.interleave_self_attention
+            hidden_states = block(
+                hidden_states,
+                encoder_hidden_states=None if self_attention else encoder_hidden_states,
+                encoder_attention_mask=None if self_attention else encoder_attention_mask,
+                temb=time_embedding,
+            )
+            all_hidden_states.append(hidden_states)
+        shift, scale = self.proj_out_1(F.silu(time_embedding)).chunk(2, dim=1)
+        hidden_states = self.norm_out(hidden_states) * (1 + scale[:, None]) + shift[:, None]
+        output = self.proj_out_2(hidden_states)
+        return (output, all_hidden_states) if return_all_hidden_states else output
+
+
+class SinusoidalPositionalEncoding(nn.Module):
+    def __init__(self, embedding_dim: int):
+        super().__init__()
+        self.embedding_dim = embedding_dim
+
+    def forward(self, timesteps: torch.Tensor) -> torch.Tensor:
+        timesteps = timesteps.float()
+        half_dim = self.embedding_dim // 2
+        exponent = -torch.arange(half_dim, dtype=torch.float, device=timesteps.device) * (
+            torch.log(torch.tensor(10000.0, device=timesteps.device)) / half_dim
+        )
+        frequencies = timesteps.unsqueeze(-1) * exponent.exp()
+        return torch.cat([torch.sin(frequencies), torch.cos(frequencies)], dim=-1)
+
+
+class CategorySpecificLinear(nn.Module):
+    def __init__(self, num_categories: int, input_dim: int, output_dim: int):
+        super().__init__()
+        self.num_categories = num_categories
+        self.W = nn.Parameter(0.02 * torch.randn(num_categories, input_dim, output_dim))
+        self.b = nn.Parameter(torch.zeros(num_categories, output_dim))
+
+    def forward(self, x: torch.Tensor, category_ids: torch.Tensor) -> torch.Tensor:
+        if category_ids is None:
+            raise ValueError("embodiment_id (B,) is required")
+        return torch.bmm(x, self.W[category_ids]) + self.b[category_ids].unsqueeze(1)
+
+
+class CategorySpecificMLP(nn.Module):
+    def __init__(self, num_categories: int, input_dim: int, hidden_dim: int, output_dim: int):
+        super().__init__()
+        self.layer1 = CategorySpecificLinear(num_categories, input_dim, hidden_dim)
+        self.layer2 = CategorySpecificLinear(num_categories, hidden_dim, output_dim)
+
+    def forward(self, x: torch.Tensor, category_ids: torch.Tensor) -> torch.Tensor:
+        return self.layer2(F.relu(self.layer1(x, category_ids)), category_ids)
+
+
+class MultiEmbodimentActionEncoder(nn.Module):
+    def __init__(self, action_dim: int, hidden_size: int, num_embodiments: int):
+        super().__init__()
+        self.W1 = CategorySpecificLinear(num_embodiments, action_dim, hidden_size)
+        self.W2 = CategorySpecificLinear(num_embodiments, 2 * hidden_size, hidden_size)
+        self.W3 = CategorySpecificLinear(num_embodiments, hidden_size, hidden_size)
+        self.pos_encoding = SinusoidalPositionalEncoding(hidden_size)
+
+    def forward(
+        self,
+        actions: torch.Tensor,
+        timesteps: torch.Tensor,
+        embodiment_id: torch.Tensor,
+    ) -> torch.Tensor:
+        batch_size, horizon, _ = actions.shape
+        if timesteps.dim() != 1 or timesteps.shape[0] != batch_size:
+            raise ValueError("timesteps must have shape (B,)")
+        timesteps = timesteps.unsqueeze(1).expand(-1, horizon)
+        action_embedding = self.W1(actions, embodiment_id)
+        time_embedding = self.pos_encoding(timesteps).to(dtype=action_embedding.dtype)
+        hidden = self.W2(torch.cat([action_embedding, time_embedding], dim=-1), embodiment_id)
+        return self.W3(hidden * torch.sigmoid(hidden), embodiment_id)
+
+
+class MiniCPMV_VLA_ActionHead(nn.Module):
+    """80-D, 32-embodiment action head for MiniCPM-VLA."""
+
+    def __init__(
+        self,
+        hidden_size: int = 1024,
+        action_dim: int = 80,
+        state_dim: int = 80,
+        action_horizon: int = 30,
+        num_inference_timesteps: int = 4,
+        num_target_vision_tokens: int = 32,
+        max_seq_len: int = 1024,
+        num_timestep_buckets: int = 1000,
+        max_num_embodiments: int = 32,
+    ):
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.input_embedding_dim = 32
+        self.model = DiT(
+            input_embedding_dim=32,
+            attention_head_dim=32,
+            num_attention_heads=1,
+            cross_attention_dim=32,
+            dropout=0.2,
+            final_dropout=True,
+            interleave_self_attention=True,
+            norm_type="ada_norm",
+            num_layers=2,
+            output_dim=32,
+            positional_embeddings=None,
+        )
+        self.action_dim = action_dim
+        self.state_dim = state_dim
+        self.action_horizon = action_horizon
+        self.num_inference_timesteps = num_inference_timesteps
+        self.max_num_embodiments = max_num_embodiments
+        self.multi_embodiment = True
+        self.proprio_inject = "concat"
+        self.state_encoder = None
+        self.action_encoder = MultiEmbodimentActionEncoder(
+            action_dim=action_dim + state_dim,
+            hidden_size=self.input_embedding_dim,
+            num_embodiments=max_num_embodiments,
+        )
+        self.action_decoder = CategorySpecificMLP(
+            num_categories=max_num_embodiments,
+            input_dim=32,
+            hidden_dim=32,
+            output_dim=action_dim,
+        )
+        self.future_tokens = nn.Embedding(num_target_vision_tokens, self.input_embedding_dim)
+        nn.init.normal_(self.future_tokens.weight, mean=0.0, std=0.02)
+        self.position_embedding = nn.Embedding(max_seq_len, self.input_embedding_dim)
+        nn.init.normal_(self.position_embedding.weight, mean=0.0, std=0.02)
+        self.num_timestep_buckets = num_timestep_buckets
+
+    def _encode_action_tokens(
+        self,
+        noisy_actions: torch.Tensor,
+        state: torch.Tensor,
+        timesteps: torch.Tensor,
+        embodiment_id: torch.Tensor,
+    ) -> torch.Tensor:
+        if state is None:
+            raise ValueError("state is required because PROPRIO_INJECT=concat")
+        state = state.expand(-1, noisy_actions.shape[1], -1)
+        inputs = torch.cat([noisy_actions, state], dim=-1)
+        return self.action_encoder(inputs, timesteps, embodiment_id)
+
+    def _build_sequence(self, action_features: torch.Tensor) -> torch.Tensor:
+        future_tokens = self.future_tokens.weight.unsqueeze(0).expand(action_features.shape[0], -1, -1)
+        return torch.cat((future_tokens, action_features), dim=1)
+
+    def _predict(
+        self,
+        noisy_actions: torch.Tensor,
+        vl_embs: torch.Tensor,
+        state: torch.Tensor,
+        timesteps: torch.Tensor,
+        embodiment_id: torch.Tensor,
+    ) -> torch.Tensor:
+        features = self._encode_action_tokens(noisy_actions, state, timesteps, embodiment_id)
+        position_ids = torch.arange(features.shape[1], dtype=torch.long, device=features.device)
+        features = features + self.position_embedding(position_ids).unsqueeze(0)
+        output = self.model(
+            hidden_states=self._build_sequence(features),
+            encoder_hidden_states=vl_embs,
+            timestep=timesteps,
+        )
+        return self.action_decoder(output, embodiment_id)[:, -self.action_horizon :]
+
+    @torch.no_grad()
+    def predict_action(
+        self,
+        vl_embs: torch.Tensor,
+        state: torch.Tensor,
+        embodiment_id: torch.Tensor,
+    ) -> torch.Tensor:
+        actions = torch.zeros(
+            (vl_embs.shape[0], self.action_horizon, self.action_dim),
+            dtype=vl_embs.dtype,
+            device=vl_embs.device,
+        )
+        noise = torch.randn_like(actions)
+        for step in range(self.num_inference_timesteps, 0, -1):
+            time = step / float(self.num_inference_timesteps)
+            timestep = min(int(time * self.num_timestep_buckets), self.num_timestep_buckets - 1)
+            timesteps = torch.full(
+                (vl_embs.shape[0],),
+                timestep,
+                device=vl_embs.device,
+                dtype=torch.long,
+            )
+            noisy_actions = time * noise + (1 - time) * actions
+            actions = self._predict(noisy_actions, vl_embs, state, timesteps, embodiment_id)
+        return actions

chat_template.jinja

@@ -0,0 +1,145 @@
+{%- if enable_thinking is not defined -%}
+    {%- set enable_thinking = false -%}
+{%- endif -%}
+{%- macro render_content(content, is_system_content=false) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- set ns = namespace(parts=[]) -%}
+        {%- for item in content -%}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}
+                {%- if is_system_content -%}
+                    {{- raise_exception('System message cannot contain images.') -}}
+                {%- endif -%}
+                {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%}
+            {%- elif 'video' in item or item.type == 'video' -%}
+                {%- if is_system_content -%}
+                    {{- raise_exception('System message cannot contain videos.') -}}
+                {%- endif -%}
+                {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%}
+            {%- elif 'text' in item -%}
+                {%- set ns.parts = ns.parts + [item.text] -%}
+            {%- else -%}
+                {{- raise_exception('Unexpected item type in content.') -}}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- ns.parts | join('\n') -}}
+    {%- elif content is none or content is undefined -%}
+        {{- '' -}}
+    {%- else -%}
+        {{- raise_exception('Unexpected content type.') -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,154 @@
+{
+  "action_dim": 80,
+  "action_head_dtype": "float32",
+  "action_horizon": 30,
+  "architectures": [
+    "MiniCPMV_VLA"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_minicpm_vla.MiniCPMVLAConfig",
+    "AutoModel": "modeling_minicpm_vla.MiniCPMV_VLA"
+  },
+  "dtype": "bfloat16",
+  "max_num_embodiments": 32,
+  "model_type": "minicpm_vla",
+  "num_inference_timesteps": 4,
+  "state_dim": 80,
+  "transformers_version": "5.20.2",
+  "vlm_config": {
+    "architectures": [
+      "MiniCPMV4_6ForConditionalGeneration"
+    ],
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "downsample_mode": "16x",
+    "drop_vision_last_layer": false,
+    "dtype": null,
+    "eos_token_id": 248044,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 112,
+    "image_token_id": 248056,
+    "insert_layer_id": 6,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "merge_kernel_size": [
+      2,
+      2
+    ],
+    "merger_times": 1,
+    "model_type": "minicpmv4_6",
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "problem_type": null,
+    "return_dict": true,
+    "text_config": {
+      "architectures": null,
+      "attention_bias": false,
+      "attention_dropout": 0.0,
+      "attn_output_gate": true,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "dtype": null,
+      "eos_token_id": null,
+      "full_attention_interval": 4,
+      "head_dim": 32,
+      "hidden_act": "silu",
+      "hidden_size": 32,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "initializer_range": 0.02,
+      "intermediate_size": 64,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_types": [
+        "linear_attention",
+        "linear_attention",
+        "linear_attention",
+        "full_attention",
+        "linear_attention",
+        "linear_attention",
+        "linear_attention",
+        "full_attention"
+      ],
+      "linear_conv_kernel_dim": 4,
+      "linear_key_head_dim": 32,
+      "linear_num_key_heads": 1,
+      "linear_num_value_heads": 1,
+      "linear_value_head_dim": 32,
+      "mamba_ssm_dtype": "float32",
+      "max_position_embeddings": 262144,
+      "mlp_only_layers": [],
+      "model_type": "qwen3_5_text",
+      "mtp_num_hidden_layers": 1,
+      "mtp_use_dedicated_embeddings": false,
+      "num_attention_heads": 1,
+      "num_hidden_layers": 8,
+      "num_key_value_heads": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "pad_token_id": null,
+      "partial_rotary_factor": 0.25,
+      "problem_type": null,
+      "return_dict": true,
+      "rms_norm_eps": 1e-06,
+      "rope_parameters": {
+        "partial_rotary_factor": 0.25,
+        "rope_theta": 10000000,
+        "rope_type": "default"
+      },
+      "tie_word_embeddings": true,
+      "use_cache": true,
+      "vocab_size": 248094
+    },
+    "tie_word_embeddings": true,
+    "transformers_version": "5.7.0",
+    "video_token_id": 248057,
+    "vision_config": {
+      "architectures": null,
+      "attention_dropout": 0.0,
+      "chunk_size_feed_forward": 0,
+      "dtype": null,
+      "hidden_act": "gelu_pytorch_tanh",
+      "hidden_size": 32,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "image_size": 980,
+      "insert_layer_id": 6,
+      "intermediate_size": 64,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-06,
+      "num_attention_heads": 1,
+      "num_channels": 3,
+      "num_hidden_layers": 2,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "patch_size": 14,
+      "problem_type": null,
+      "return_dict": true,
+      "window_kernel_size": [
+        2,
+        2
+      ]
+    }
+  },
+  "vlm_dtype": "bfloat16"
+}

configuration_minicpm_vla.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from transformers import PretrainedConfig
+
+
+class MiniCPMVLAConfig(PretrainedConfig):
+    """Configuration for the complete VLM and action-head VLA model."""
+
+    model_type = "minicpm_vla"
+    is_composition = True
+
+    def __init__(
+        self,
+        vlm_config: dict | PretrainedConfig | None = None,
+        action_dim: int = 80,
+        state_dim: int = 80,
+        action_horizon: int = 30,
+        max_num_embodiments: int = 32,
+        num_inference_timesteps: int = 4,
+        vlm_dtype: str = "bfloat16",
+        action_head_dtype: str = "float32",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        if isinstance(vlm_config, PretrainedConfig):
+            vlm_config = vlm_config.to_dict()
+        self.vlm_config = vlm_config
+        self.action_dim = action_dim
+        self.state_dim = state_dim
+        self.action_horizon = action_horizon
+        self.max_num_embodiments = max_num_embodiments
+        self.num_inference_timesteps = num_inference_timesteps
+        self.vlm_dtype = vlm_dtype
+        self.action_head_dtype = action_head_dtype
+        self.architectures = ["MiniCPMV_VLA"]

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 248045,
+  "do_sample": true,
+  "eos_token_id": [
+    248044,
+    248046
+  ],
+  "temperature": 0.7,
+  "top_k": 0,
+  "top_p": 1.0,
+  "repetition_penalty": 1.0,
+  "transformers_version": "5.7.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c8076646005e1ef1748225910dacfe2760cb880d053e8c400fc589b29842325
+size 18447424

modeling_minicpm_vla.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+import torch
+from transformers import AutoConfig, AutoModel, AutoModelForImageTextToText, PretrainedConfig, PreTrainedModel
+
+try:
+    from .action_head import MiniCPMV_VLA_ActionHead
+    from .configuration_minicpm_vla import MiniCPMVLAConfig
+except ImportError:  # pragma: no cover - fallback for non-package loaders
+    from action_head import MiniCPMV_VLA_ActionHead
+    from configuration_minicpm_vla import MiniCPMVLAConfig
+
+
+def _torch_dtype(name: str) -> torch.dtype:
+    try:
+        return getattr(torch, name)
+    except AttributeError as exc:
+        raise ValueError(f"Unsupported torch dtype: {name}") from exc
+
+
+def _build_vlm_config(config_dict: dict) -> PretrainedConfig:
+    values = dict(config_dict)
+    model_type = values.pop("model_type")
+    return AutoConfig.for_model(model_type, **values)
+
+
+class MiniCPMV_VLA(PreTrainedModel):
+    """Complete VLA model loadable with ``AutoModel.from_pretrained``."""
+
+    config_class = MiniCPMVLAConfig
+    base_model_prefix = ""
+    _tied_weights_keys = {
+        "vlm.lm_head.weight": "vlm.model.language_model.embed_tokens.weight",
+    }
+    _no_split_modules = ["BasicTransformerBlock"]
+
+    def __init__(self, config: MiniCPMVLAConfig):
+        super().__init__(config)
+        if config.vlm_config is None:
+            raise ValueError("config.vlm_config is required")
+        vlm_config = _build_vlm_config(config.vlm_config)
+        self.vlm_dtype = _torch_dtype(config.vlm_dtype)
+        self.action_head_dtype = _torch_dtype(config.action_head_dtype)
+        self.vlm = AutoModelForImageTextToText.from_config(vlm_config).to(self.vlm_dtype)
+        self.vlm.config.hidden_size = self.vlm.config.text_config.hidden_size
+        self.action_head = MiniCPMV_VLA_ActionHead(
+            action_dim=config.action_dim,
+            state_dim=config.state_dim,
+            action_horizon=config.action_horizon,
+            num_inference_timesteps=config.num_inference_timesteps,
+            max_num_embodiments=config.max_num_embodiments,
+        ).to(self.action_head_dtype)
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.vlm.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        return self.vlm.set_input_embeddings(value)
+
+    def _vlm_forward(self, data: Mapping[str, object]):
+        inputs = {key: value for key, value in data.items() if value is not None}
+        if "language_attention_mask" in inputs and "attention_mask" not in inputs:
+            inputs["attention_mask"] = inputs.pop("language_attention_mask")
+        inputs.setdefault("output_hidden_states", True)
+        inputs.setdefault("return_dict", True)
+        inputs.setdefault("use_cache", False)
+        inputs.setdefault("logits_to_keep", 1)
+        with torch.autocast("cuda", dtype=self.vlm_dtype, enabled=torch.cuda.is_available()):
+            try:
+                return self.vlm(**inputs)
+            except TypeError as exc:
+                if "logits_to_keep" not in str(exc):
+                    raise
+                inputs.pop("logits_to_keep")
+                return self.vlm(**inputs)
+
+    @torch.no_grad()
+    def predict_action(
+        self,
+        state: torch.Tensor,
+        embodiment_id: torch.Tensor,
+        **vlm_inputs,
+    ) -> torch.Tensor:
+        vl_embs = self._vlm_forward(vlm_inputs).hidden_states[-1].to(self.action_head_dtype)
+        return self.action_head.predict_action(vl_embs, state, embodiment_id)
+
+
+AutoConfig.register("minicpm_vla", MiniCPMVLAConfig)
+AutoModel.register(MiniCPMVLAConfig, MiniCPMV_VLA)

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "image_processor_type": "MiniCPMV4_6ImageProcessor",
+  "processor_class": "MiniCPMV4_6Processor",
+  "max_slice_nums": 9,
+  "scale_resolution": 448,
+  "patch_size": 14,
+  "use_image_id": true,
+  "slice_mode": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ]
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33861e37bb955af1e3f3061182b820f347eba2b9c2c1011c82794bf0d6e77b54
+size 19992481

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "image_start_token": "<image>",
+    "image_end_token": "</image>",
+    "slice_start_token": "<slice>",
+    "slice_end_token": "</slice>",
+    "image_id_start_token": "<image_id>",
+    "image_id_end_token": "</image_id>"
+  },
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "split_special_tokens": false,
+  "unk_token": "<unk>",
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>",
+  "chat_template": "{%- if enable_thinking is not defined -%}\n    {%- set enable_thinking = false -%}\n{%- endif -%}\n{%- macro render_content(content, is_system_content=false) -%}\n    {%- if content is string -%}\n        {{- content -}}\n    {%- elif content is iterable and content is not mapping -%}\n        {%- set ns = namespace(parts=[]) -%}\n        {%- for item in content -%}\n            {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}\n                {%- if is_system_content -%}\n                    {{- raise_exception('System message cannot contain images.') -}}\n                {%- endif -%}\n                {%- set ns.parts = ns.parts + ['<|image_pad|>'] -%}\n            {%- elif 'video' in item or item.type == 'video' -%}\n                {%- if is_system_content -%}\n                    {{- raise_exception('System message cannot contain videos.') -}}\n                {%- endif -%}\n                {%- set ns.parts = ns.parts + ['<|video_pad|>'] -%}\n            {%- elif 'text' in item -%}\n                {%- set ns.parts = ns.parts + [item.text] -%}\n            {%- else -%}\n                {{- raise_exception('Unexpected item type in content.') -}}\n            {%- endif -%}\n        {%- endfor -%}\n        {{- ns.parts | join('\\n') -}}\n    {%- elif content is none or content is undefined -%}\n        {{- '' -}}\n    {%- else -%}\n        {{- raise_exception('Unexpected content type.') -}}\n    {%- endif -%}\n{%- endmacro -%}\n{%- if not messages %}\n    {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n    {{- '<|im_start|>system\\n' }}\n    {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\" }}\n    {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n    {%- if messages[0].role == 'system' %}\n        {%- set content = render_content(messages[0].content, true)|trim %}\n        {%- if content %}\n            {{- '\\n\\n' + content }}\n        {%- endif %}\n    {%- endif %}\n    {{- '<|im_end|>\\n' }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {%- set content = render_content(messages[0].content, true)|trim %}\n        {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" %}\n        {%- set content = render_content(message.content)|trim %}\n        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n            {%- set ns.multi_step_tool = false %}\n            {%- set ns.last_query_index = index %}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n    {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n    {%- set content = render_content(message.content)|trim %}\n    {%- if message.role == \"system\" %}\n        {%- if not loop.first %}\n            {{- raise_exception('System message must be at the beginning.') }}\n        {%- endif %}\n    {%- elif message.role == \"user\" %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if '</think>' in content %}\n                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n        {%- set reasoning_content = reasoning_content|trim %}\n        {%- if loop.index0 > ns.last_query_index %}\n            {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n        {%- else %}\n            {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- endif %}\n        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if tool_call.function is defined %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {%- if loop.first %}\n                    {%- if content|trim %}\n                        {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                    {%- else %}\n                        {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                    {%- endif %}\n                {%- else %}\n                    {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                {%- endif %}\n                {%- if tool_call.arguments is defined %}\n                    {%- for args_name, args_value in tool_call.arguments|items %}\n                        {{- '<parameter=' + args_name + '>\\n' }}\n                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n                        {{- args_value }}\n                        {{- '\\n</parameter>\\n' }}\n                    {%- endfor %}\n                {%- endif %}\n                {{- '</function>\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n            {{- '<|im_end|>\\n' }}\n        {%- elif loop.last %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- else %}\n        {{- raise_exception('Unexpected message role.') }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- else %}\n        {{- '<think>\\n' }}\n    {%- endif %}\n{%- endif %}\n",
+  "tokenizer_class": "Qwen2Tokenizer"
+}