Upload tied tiny Mistral Large 4 checkpoint

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tekken.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.meta/create_model.py

@@ -0,0 +1,307 @@
+def create_model(source_model_id: str, source_revision: str, save_folder: str):
+    import json
+    from pathlib import Path
+
+    import torch
+    from huggingface_hub import hf_hub_download
+    from safetensors import safe_open
+    from safetensors.torch import save_file
+    from transformers import AutoConfig, AutoModelForImageTextToText
+
+    folder = Path(save_folder)
+    folder.mkdir(parents=True, exist_ok=True)
+    torch.manual_seed(42)
+
+    def read_json(name: str):
+        return json.loads(
+            Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_text(encoding="utf-8")
+        )
+
+    source_config, params = read_json("config.json"), read_json("params.json")
+    text = source_config["text_config"]
+    text.update(
+        architectures=["MistralLarge4ForCausalLM"],
+        tie_word_embeddings=True,
+        hidden_size=128,
+        intermediate_size=128,
+        moe_intermediate_size=128,
+        num_hidden_layers=12,
+        num_attention_heads=4,
+        num_key_value_heads=4,
+        head_dim=32,
+        qk_nope_head_dim=64,
+        qk_rope_head_dim=64,
+        v_head_dim=128,
+        kv_lora_rank=512,
+        index_head_dim=128,
+        index_n_heads=4,
+        index_topk=128,
+        n_routed_experts=32,
+        n_shared_experts=1,
+        num_experts_per_tok=4,
+        first_k_dense_replace=1,
+        layer_types=["indexed_attention"] * 12,
+        mlp_layer_types=["dense"] + ["sparse"] * 11,
+        no_rope_layers=[1, 1, 1, 0] * 3,
+        indexer_types=["full"] * 4 + ["full", "full", "shared", "shared"] * 2,
+        kv_source_layers=[0, 1, 2, 3, 4, 5, 6, 3, 8, 9, 10, 3],
+    )
+    source_config["tie_word_embeddings"] = True
+    source_config["vision_config"].update(
+        hidden_size=64,
+        intermediate_size=128,
+        num_hidden_layers=4,
+        num_attention_heads=2,
+        head_dim=32,
+        image_size=1540,
+        patch_size=14,
+    )
+    source_config["quantization_config"] = {
+        "quant_method": "fp8",
+        "activation_scheme": "dynamic",
+        "weight_block_size": [64, 64],
+        "scale_fmt": "ue8m0",
+        "modules_to_not_convert": [
+            "model.vision_tower",
+            "model.multi_modal_projector",
+            "lm_head",
+            "kv_b_proj",
+            "indexer.wk",
+            "indexer.weights_proj",
+            "kv_a_proj_with_mqa",
+        ],
+    }
+    source_config["transformers_weights"] = "consolidated.safetensors"
+    params.update(
+        dim=128,
+        tied_embeddings=True,
+        n_layers=12,
+        head_dim=32,
+        hidden_dim=128,
+        n_heads=4,
+        n_kv_heads=4,
+        kv_lora_rank=512,
+        qk_rope_head_dim=64,
+        qk_nope_head_dim=64,
+        v_head_dim=128,
+        nope="rope,rope,rope,nope",
+        kv_sharing={"source_layers_str": "0,1,2,3,4,5,6,3,8,9,10,3"},
+        dsa={
+            "topk": 128,
+            "n_heads": 4,
+            "head_dim": 128,
+            "rope_head_dim": 64,
+            "match_nope": True,
+            "ssmax_scaling": True,
+            "norm_eps": 1e-6,
+            "sharing": "full,full,full,full,full,full,shared,shared,full,full,shared,shared",
+            "fp8_indexer": True,
+            "fp8_kv_cache": True,
+        },
+        quantization={
+            "qscheme_act": "BLOCK",
+            "qformat_weight": "fp8_e4m3",
+            "qdynamic_act": True,
+            "qscheme_weight": "BLOCK",
+            "use_ue8m0_for_qscales": True,
+            "block_size_out": 64,
+            "block_size_in": 64,
+            "quantize_kv_a": False,
+        },
+        moe={
+            "expert_parallel": 1,
+            "expert_model_parallel": 1,
+            "route_every_n": 1,
+            "first_k_dense_replace": 1,
+            "num_experts": 32,
+            "num_experts_per_tok": 4,
+            "num_expert_groups": 1,
+            "num_expert_groups_per_tok": 1,
+            "routed_scale": 1.0,
+            "expert_hidden_dim": 128,
+            "num_shared_experts": 1,
+        },
+        vision_encoder={
+            **params["vision_encoder"],
+            "intermediate_size": 128,
+            "num_hidden_layers": 4,
+            "num_attention_heads": 2,
+            "hidden_size": 64,
+            "image_size": 1540,
+            "max_image_size": 1540,
+            "patch_size": 14,
+        },
+        mtp={
+            "n_layers": 3,
+            "depth": 3,
+            "ragged_attention": "32,32,32",
+            "target_activation_layers": "1,4,7,10,11",
+            "norm_strategy": "eagle",
+            "dspark": {"markov_rank": 4, "head_type": "rnn", "draft_input": "position_specific"},
+        },
+    )
+    # The public HF config is used by Transformers while vLLM also needs this
+    # native Mistral block to build and enable the in-checkpoint DSpark draft.
+    text["mtp_config"] = params["mtp"]
+    config_for_build = dict(source_config)
+    config_for_build.pop("quantization_config")
+    (folder / "config.json").write_text(json.dumps(config_for_build, indent=2) + "\n", encoding="utf-8")
+    model = AutoModelForImageTextToText.from_config(AutoConfig.from_pretrained(folder), dtype=torch.bfloat16)
+    with torch.no_grad():
+        for name, value in model.named_parameters():
+            if name.endswith("weight") and "norm" in name:
+                value.fill_(1)
+            elif name.endswith("bias"):
+                value.zero_()
+            else:
+                value.normal_(mean=0.0, std=0.02)
+    model.tie_weights()
+    model.save_pretrained(folder, safe_serialization=True)
+    state = {}
+    with safe_open(folder / "model.safetensors", framework="pt", device="cpu") as reader:
+        for name in reader.keys():
+            state[name] = reader.get_tensor(name).contiguous()
+    (folder / "model.safetensors").unlink()
+
+    def should_quantize(name: str, value: torch.Tensor) -> bool:
+        return (
+            name.endswith(".weight")
+            and value.ndim == 2
+            and value.shape[0] % 64 == 0
+            and value.shape[1] % 64 == 0
+            and not name.startswith(
+                (
+                    "vision_encoder.",
+                    "vision_language_adapter.",
+                    "patch_merger.",
+                    "pre_mm_projector_norm.",
+                    "tok_embeddings.",
+                    "output.",
+                )
+            )
+            and not any(
+                piece in name
+                for piece in (
+                    ".wkv_a_with_mqa.",
+                    ".wkv_b.",
+                    ".indexer.wk.",
+                    ".indexer.weights_proj.",
+                    ".attention.wo.",
+                    ".feed_forward.",
+                    ".shared_experts.",
+                    ".experts.",
+                )
+            )
+            and not name.endswith(".gate.weight")
+        )
+
+    def quantize(name: str, value: torch.Tensor) -> None:
+        rows, columns = value.shape
+        blocks = value.float().reshape(rows // 64, 64, columns // 64, 64)
+        maximum = blocks.abs().amax(dim=(1, 3)).clamp_min(torch.finfo(torch.float32).tiny)
+        inverse = torch.pow(2.0, torch.ceil(torch.log2(1.0 / maximum))).to(torch.float8_e8m0fnu)
+        state[name] = (
+            (blocks * inverse.float()[:, None, :, None])
+            .clamp(-448, 448)
+            .reshape(rows, columns)
+            .to(torch.float8_e4m3fn)
+            .contiguous()
+        )
+        state[name.removesuffix(".weight") + ".qscale_weight"] = (1.0 / inverse.float()).to(torch.float8_e8m0fnu)
+
+    for name, value in list(state.items()):
+        if should_quantize(name, value):
+            quantize(name, value)
+
+    for layer in range(3):
+        prefix = f"mtp_dspark.backbone.layers.{layer}"
+        shapes = {
+            "attention_norm.weight": (128,),
+            "attention.wq.weight": (512, 128),
+            "attention.wkv_a_with_mqa.weight": (576, 128),
+            "attention.kv_a_norm.weight": (512,),
+            "attention.wkv_b.weight": (768, 512),
+            "attention.wo.weight": (128, 512),
+            "ffn_norm.weight": (128,),
+            "feed_forward.w1.weight": (128, 128),
+            "feed_forward.w2.weight": (128, 128),
+            "feed_forward.w3.weight": (128, 128),
+        }
+        for name, shape in shapes.items():
+            state[f"{prefix}.{name}"] = (
+                torch.ones(shape, dtype=torch.bfloat16)
+                if name.endswith("norm.weight")
+                else torch.randn(shape, dtype=torch.bfloat16) * 0.02
+            )
+    state.update(
+        {
+            "mtp_dspark.backbone.target_projection.weight": torch.randn(128, 640, dtype=torch.bfloat16) * 0.02,
+            "mtp_dspark.backbone.target_norm.weight": torch.ones(128, dtype=torch.bfloat16),
+            "mtp_dspark.backbone.final_norm.weight": torch.ones(128, dtype=torch.bfloat16),
+            "mtp_dspark.backbone.mask_embedding": torch.randn(2, 128, dtype=torch.bfloat16) * 0.02,
+            "mtp_dspark.autoregressive_head.embedding.weight": torch.randn(204800, 4, dtype=torch.bfloat16) * 0.02,
+            "mtp_dspark.autoregressive_head.output.weight": torch.randn(204800, 4, dtype=torch.bfloat16) * 0.02,
+            "mtp_dspark.autoregressive_head.joint_proj.weight": torch.randn(12, 136, dtype=torch.bfloat16) * 0.02,
+            "mtp_dspark.autoregressive_head.joint_proj.bias": torch.zeros(12, dtype=torch.bfloat16),
+            "lerope.alpha": torch.zeros(32, dtype=torch.bfloat16),
+        }
+    )
+    save_file(state, folder / "consolidated.safetensors", metadata={"format": "pt"})
+    (folder / "config.json").write_text(json.dumps(source_config, indent=2) + "\n", encoding="utf-8")
+    (folder / "params.json").write_text(json.dumps(params, indent=2) + "\n", encoding="utf-8")
+    for name in (
+        "tekken.json",
+        "tokenizer.json",
+        "tokenizer_config.json",
+        "processor_config.json",
+        "chat_template.jinja",
+        "generation_config.json",
+        "LICENSE",
+    ):
+        (folder / name).write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
+    (folder / "vllm_mistral_large4_tied.py").write_text(
+        "from vllm.distributed import get_pp_group\n"
+        "from vllm.models.mistral_large4 import MistralLarge4ForCausalLM\n\n\n"
+        "class MistralLarge4TiedForCausalLM(MistralLarge4ForCausalLM):\n"
+        '    def __init__(self, *, vllm_config, prefix=""):\n'
+        "        super().__init__(vllm_config=vllm_config, prefix=prefix)\n"
+        "        if self.config.tie_word_embeddings and get_pp_group().is_last_rank:\n"
+        "            self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)\n",
+        encoding="utf-8",
+    )
+    (folder / "vllm_plugin.py").write_text(
+        "def register():\n"
+        "    from vllm import ModelRegistry\n\n"
+        "    ModelRegistry.register_model(\n"
+        '        "MistralLarge4ForCausalLM",\n'
+        '        "vllm_mistral_large4_tied:MistralLarge4TiedForCausalLM",\n'
+        "    )\n",
+        encoding="utf-8",
+    )
+    (folder / "pyproject.toml").write_text(
+        "[build-system]\n"
+        "requires = ['setuptools>=61']\n"
+        "build-backend = 'setuptools.build_meta'\n\n"
+        "[project]\n"
+        "name = 'mistral-large-4-tied-vllm'\n"
+        "version = '0.1.0'\n\n"
+        "[project.entry-points.'vllm.general_plugins']\n"
+        "mistral_large4_tied = 'vllm_plugin:register'\n\n"
+        "[tool.setuptools]\n"
+        "py-modules = ['vllm_mistral_large4_tied', 'vllm_plugin']\n",
+        encoding="utf-8",
+    )
+    model.add_module(
+        "mtp_dspark",
+        torch.nn.ModuleDict(
+            {"backbone": torch.nn.ModuleDict({"layers": torch.nn.ModuleList([torch.nn.Identity() for _ in range(3)])})}
+        ),
+    )
+    return str(model)
+
+
+create_model(
+    source_model_id="mistralai/Mistral-Large-4-1T-A52B",
+    source_revision="f12dabdc0c1e99e7f7924461a6f308859b05af4d",
+    save_folder="/tmp/mistral-large-4",
+)

.meta/env.json

@@ -0,0 +1,53 @@
+{
+  "source_model": {
+    "name": "mistralai/Mistral-Large-4-1T-A52B",
+    "commit_id": "f12dabdc0c1e99e7f7924461a6f308859b05af4d"
+  },
+  "environments": {
+    "creation": {
+      "python": "3.12.13",
+      "packages": {
+        "huggingface_hub": "2.3.0",
+        "safetensors": "0.8.0",
+        "tokenizers": "0.23.3",
+        "torch": "2.13.0+cu126",
+        "transformers": "5.22.0.dev0"
+      },
+      "cuda": "12.6"
+    },
+    "transformers": {
+      "python": "3.12.13",
+      "packages": {
+        "Pillow": "12.3.0",
+        "huggingface_hub": "2.3.0",
+        "mistral-large-4-tied-vllm": "0.1.0",
+        "safetensors": "0.8.0",
+        "tokenizers": "0.23.3",
+        "torch": "2.13.0+cu126",
+        "transformers": "5.22.0.dev0",
+        "vllm": "0.1.dev1+g5a3eba034.d20260730.cu126"
+      },
+      "cuda": "12.6",
+      "hardware": {
+        "gpu_model": "NVIDIA H20"
+      }
+    },
+    "vllm-offline": {
+      "python": "3.12.13",
+      "packages": {
+        "Pillow": "12.3.0",
+        "huggingface_hub": "2.3.0",
+        "mistral-large-4-tied-vllm": "0.1.0",
+        "safetensors": "0.8.0",
+        "tokenizers": "0.23.3",
+        "torch": "2.13.0+cu126",
+        "transformers": "5.22.0.dev0",
+        "vllm": "0.1.dev1+g5a3eba034.d20260730.cu126"
+      },
+      "cuda": "12.6",
+      "hardware": {
+        "gpu_model": "NVIDIA H20"
+      }
+    }
+  }
+}

.meta/model.log

@@ -0,0 +1,299 @@
+MistralLarge4ForConditionalGeneration(
+  (model): MistralLarge4Model(
+    (vision_tower): PixtralVisionModel(
+      (patch_conv): Conv2d(3, 64, kernel_size=(14, 14), stride=(14, 14), bias=False)
+      (ln_pre): PixtralRMSNorm((64,), eps=1e-05)
+      (transformer): PixtralTransformer(
+        (layers): ModuleList(
+          (0-3): 4 x PixtralAttentionLayer(
+            (attention_norm): PixtralRMSNorm((64,), eps=1e-05)
+            (feed_forward): PixtralMLP(
+              (gate_proj): Linear(in_features=64, out_features=128, bias=False)
+              (up_proj): Linear(in_features=64, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=64, bias=False)
+              (act_fn): SiLUActivation()
+            )
+            (attention): PixtralAttention(
+              (k_proj): Linear(in_features=64, out_features=64, bias=False)
+              (v_proj): Linear(in_features=64, out_features=64, bias=False)
+              (q_proj): Linear(in_features=64, out_features=64, bias=False)
+              (o_proj): Linear(in_features=64, out_features=64, bias=False)
+            )
+            (ffn_norm): PixtralRMSNorm((64,), eps=1e-05)
+          )
+        )
+      )
+      (patch_positional_embedding): PixtralVisionRotaryEmbedding()
+    )
+    (multi_modal_projector): MistralLarge4MultiModalProjector(
+      (norm): MistralLarge4RMSNorm((64,), eps=1e-05)
+      (patch_merger): MistralLarge4PatchMerger(
+        (merging_layer): Linear(in_features=256, out_features=64, bias=False)
+      )
+      (linear_1): Linear(in_features=64, out_features=128, bias=False)
+      (act): GELUActivation()
+      (linear_2): Linear(in_features=128, out_features=128, bias=False)
+    )
+    (language_model): MistralLarge4TextModel(
+      (embed_tokens): Embedding(204800, 128)
+      (layers): ModuleList(
+        (0): MistralLarge4TextDecoderLayer(
+          (self_attn): MistralLarge4TextAttention(
+            (kv_a_proj_with_mqa): Linear(in_features=128, out_features=576, bias=False)
+            (kv_a_layernorm): MistralLarge4RMSNorm((512,), eps=1e-05)
+            (kv_b_proj): Linear(in_features=512, out_features=768, bias=False)
+            (o_proj): Linear(in_features=512, out_features=128, bias=False)
+            (indexer): MistralLarge4TextIndexer(
+              (wq_b): Linear(in_features=128, out_features=512, bias=False)
+              (wk): Linear(in_features=128, out_features=128, bias=False)
+              (k_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+              (weights_proj): Linear(in_features=128, out_features=4, bias=False)
+              (q_input_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+              (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+            )
+            (q_proj): Linear(in_features=128, out_features=512, bias=False)
+            (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+          )
+          (mlp): MistralLarge4TextMLP(
+            (gate_proj): Linear(in_features=128, out_features=128, bias=False)
+            (up_proj): Linear(in_features=128, out_features=128, bias=False)
+            (down_proj): Linear(in_features=128, out_features=128, bias=False)
+            (act_fn): SiLUActivation()
+          )
+          (input_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+          (post_attention_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+        )
+        (1-2): 2 x MistralLarge4TextDecoderLayer(
+          (self_attn): MistralLarge4TextAttention(
+            (kv_a_proj_with_mqa): Linear(in_features=128, out_features=576, bias=False)
+            (kv_a_layernorm): MistralLarge4RMSNorm((512,), eps=1e-05)
+            (kv_b_proj): Linear(in_features=512, out_features=768, bias=False)
+            (o_proj): Linear(in_features=512, out_features=128, bias=False)
+            (indexer): MistralLarge4TextIndexer(
+              (wq_b): Linear(in_features=128, out_features=512, bias=False)
+              (wk): Linear(in_features=128, out_features=128, bias=False)
+              (k_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+              (weights_proj): Linear(in_features=128, out_features=4, bias=False)
+              (q_input_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+              (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+            )
+            (q_proj): Linear(in_features=128, out_features=512, bias=False)
+            (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+          )
+          (mlp): MistralLarge4TextMoE(
+            (experts): MistralLarge4TextExperts(
+              (act_fn): SiLUActivation()
+            )
+            (gate): MistralLarge4TextTopkRouter()
+            (shared_experts): MistralLarge4TextMLP(
+              (gate_proj): Linear(in_features=128, out_features=128, bias=False)
+              (up_proj): Linear(in_features=128, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=128, bias=False)
+              (act_fn): SiLUActivation()
+            )
+          )
+          (input_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+          (post_attention_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+        )
+        (3): MistralLarge4TextDecoderLayer(
+          (self_attn): MistralLarge4TextAttention(
+            (kv_a_proj_with_mqa): Linear(in_features=128, out_features=512, bias=False)
+            (kv_a_layernorm): MistralLarge4RMSNorm((512,), eps=1e-05)
+            (kv_b_proj): Linear(in_features=512, out_features=1024, bias=False)
+            (o_proj): Linear(in_features=512, out_features=128, bias=False)
+            (indexer): MistralLarge4TextIndexer(
+              (wq_b): Linear(in_features=128, out_features=512, bias=False)
+              (wk): Linear(in_features=128, out_features=128, bias=False)
+              (k_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+              (weights_proj): Linear(in_features=128, out_features=4, bias=False)
+              (q_input_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+              (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+            )
+            (q_proj): Linear(in_features=128, out_features=512, bias=False)
+            (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+          )
+          (mlp): MistralLarge4TextMoE(
+            (experts): MistralLarge4TextExperts(
+              (act_fn): SiLUActivation()
+            )
+            (gate): MistralLarge4TextTopkRouter()
+            (shared_experts): MistralLarge4TextMLP(
+              (gate_proj): Linear(in_features=128, out_features=128, bias=False)
+              (up_proj): Linear(in_features=128, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=128, bias=False)
+              (act_fn): SiLUActivation()
+            )
+          )
+          (input_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+          (post_attention_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+        )
+        (4-5): 2 x MistralLarge4TextDecoderLayer(
+          (self_attn): MistralLarge4TextAttention(
+            (kv_a_proj_with_mqa): Linear(in_features=128, out_features=576, bias=False)
+            (kv_a_layernorm): MistralLarge4RMSNorm((512,), eps=1e-05)
+            (kv_b_proj): Linear(in_features=512, out_features=768, bias=False)
+            (o_proj): Linear(in_features=512, out_features=128, bias=False)
+            (indexer): MistralLarge4TextIndexer(
+              (wq_b): Linear(in_features=128, out_features=512, bias=False)
+              (wk): Linear(in_features=128, out_features=128, bias=False)
+              (k_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+              (weights_proj): Linear(in_features=128, out_features=4, bias=False)
+              (q_input_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+              (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+            )
+            (q_proj): Linear(in_features=128, out_features=512, bias=False)
+            (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+          )
+          (mlp): MistralLarge4TextMoE(
+            (experts): MistralLarge4TextExperts(
+              (act_fn): SiLUActivation()
+            )
+            (gate): MistralLarge4TextTopkRouter()
+            (shared_experts): MistralLarge4TextMLP(
+              (gate_proj): Linear(in_features=128, out_features=128, bias=False)
+              (up_proj): Linear(in_features=128, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=128, bias=False)
+              (act_fn): SiLUActivation()
+            )
+          )
+          (input_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+          (post_attention_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+        )
+        (6): MistralLarge4TextDecoderLayer(
+          (self_attn): MistralLarge4TextAttention(
+            (kv_a_proj_with_mqa): Linear(in_features=128, out_features=576, bias=False)
+            (kv_a_layernorm): MistralLarge4RMSNorm((512,), eps=1e-05)
+            (kv_b_proj): Linear(in_features=512, out_features=768, bias=False)
+            (o_proj): Linear(in_features=512, out_features=128, bias=False)
+            (q_proj): Linear(in_features=128, out_features=512, bias=False)
+            (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+          )
+          (mlp): MistralLarge4TextMoE(
+            (experts): MistralLarge4TextExperts(
+              (act_fn): SiLUActivation()
+            )
+            (gate): MistralLarge4TextTopkRouter()
+            (shared_experts): MistralLarge4TextMLP(
+              (gate_proj): Linear(in_features=128, out_features=128, bias=False)
+              (up_proj): Linear(in_features=128, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=128, bias=False)
+              (act_fn): SiLUActivation()
+            )
+          )
+          (input_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+          (post_attention_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+        )
+        (7): MistralLarge4TextDecoderLayer(
+          (self_attn): MistralLarge4TextAttention(
+            (kv_b_proj): Linear(in_features=512, out_features=1024, bias=False)
+            (o_proj): Linear(in_features=512, out_features=128, bias=False)
+            (q_proj): Linear(in_features=128, out_features=512, bias=False)
+            (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+          )
+          (mlp): MistralLarge4TextMoE(
+            (experts): MistralLarge4TextExperts(
+              (act_fn): SiLUActivation()
+            )
+            (gate): MistralLarge4TextTopkRouter()
+            (shared_experts): MistralLarge4TextMLP(
+              (gate_proj): Linear(in_features=128, out_features=128, bias=False)
+              (up_proj): Linear(in_features=128, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=128, bias=False)
+              (act_fn): SiLUActivation()
+            )
+          )
+          (input_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+          (post_attention_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+        )
+        (8-9): 2 x MistralLarge4TextDecoderLayer(
+          (self_attn): MistralLarge4TextAttention(
+            (kv_a_proj_with_mqa): Linear(in_features=128, out_features=576, bias=False)
+            (kv_a_layernorm): MistralLarge4RMSNorm((512,), eps=1e-05)
+            (kv_b_proj): Linear(in_features=512, out_features=768, bias=False)
+            (o_proj): Linear(in_features=512, out_features=128, bias=False)
+            (indexer): MistralLarge4TextIndexer(
+              (wq_b): Linear(in_features=128, out_features=512, bias=False)
+              (wk): Linear(in_features=128, out_features=128, bias=False)
+              (k_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+              (weights_proj): Linear(in_features=128, out_features=4, bias=False)
+              (q_input_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=True)
+              (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+            )
+            (q_proj): Linear(in_features=128, out_features=512, bias=False)
+            (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+          )
+          (mlp): MistralLarge4TextMoE(
+            (experts): MistralLarge4TextExperts(
+              (act_fn): SiLUActivation()
+            )
+            (gate): MistralLarge4TextTopkRouter()
+            (shared_experts): MistralLarge4TextMLP(
+              (gate_proj): Linear(in_features=128, out_features=128, bias=False)
+              (up_proj): Linear(in_features=128, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=128, bias=False)
+              (act_fn): SiLUActivation()
+            )
+          )
+          (input_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+          (post_attention_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+        )
+        (10): MistralLarge4TextDecoderLayer(
+          (self_attn): MistralLarge4TextAttention(
+            (kv_a_proj_with_mqa): Linear(in_features=128, out_features=576, bias=False)
+            (kv_a_layernorm): MistralLarge4RMSNorm((512,), eps=1e-05)
+            (kv_b_proj): Linear(in_features=512, out_features=768, bias=False)
+            (o_proj): Linear(in_features=512, out_features=128, bias=False)
+            (q_proj): Linear(in_features=128, out_features=512, bias=False)
+            (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+          )
+          (mlp): MistralLarge4TextMoE(
+            (experts): MistralLarge4TextExperts(
+              (act_fn): SiLUActivation()
+            )
+            (gate): MistralLarge4TextTopkRouter()
+            (shared_experts): MistralLarge4TextMLP(
+              (gate_proj): Linear(in_features=128, out_features=128, bias=False)
+              (up_proj): Linear(in_features=128, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=128, bias=False)
+              (act_fn): SiLUActivation()
+            )
+          )
+          (input_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+          (post_attention_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+        )
+        (11): MistralLarge4TextDecoderLayer(
+          (self_attn): MistralLarge4TextAttention(
+            (kv_b_proj): Linear(in_features=512, out_features=1024, bias=False)
+            (o_proj): Linear(in_features=512, out_features=128, bias=False)
+            (q_proj): Linear(in_features=128, out_features=512, bias=False)
+            (softmax_scaling): MistralLarge4TextSoftmaxScaling()
+          )
+          (mlp): MistralLarge4TextMoE(
+            (experts): MistralLarge4TextExperts(
+              (act_fn): SiLUActivation()
+            )
+            (gate): MistralLarge4TextTopkRouter()
+            (shared_experts): MistralLarge4TextMLP(
+              (gate_proj): Linear(in_features=128, out_features=128, bias=False)
+              (up_proj): Linear(in_features=128, out_features=128, bias=False)
+              (down_proj): Linear(in_features=128, out_features=128, bias=False)
+              (act_fn): SiLUActivation()
+            )
+          )
+          (input_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+          (post_attention_layernorm): MistralLarge4RMSNorm((128,), eps=1e-05)
+        )
+      )
+      (norm): MistralLarge4RMSNorm((128,), eps=1e-05)
+      (rotary_emb): MistralLarge4TextRotaryEmbedding()
+    )
+  )
+  (lm_head): Linear(in_features=128, out_features=204800, bias=False)
+  (mtp_dspark): ModuleDict(
+    (backbone): ModuleDict(
+      (layers): ModuleList(
+        (0-2): 3 x Identity()
+      )
+    )
+  )
+)

LICENSE

@@ -0,0 +1,18 @@
+Mistral Ecosystem Open-Weight License 
+
+Mistral Ecosystem Open-Weight License. This license is a modified version of the MIT license. 
+Attribution notice: 2026 - Mistral 
+
+Permission is hereby granted, free of charge, to any person (the “Licensee”) obtaining a copy of this model and associated documentation files - including the model weights, parameters, configuration files, and associated documentation (the “Model”), to deal in the Model without restriction, including without limitation the rights to (a) use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Model; (b) run, deploy, further train or otherwise modify the Model and create derivative works from it; and (c) permit persons to whom the Model is furnished to do so, subject to the following conditions: 
+
+1. The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Model. 
+
+2. If the Licensee or any of its affiliates operates a Model as a Service business, the Licensee must enter into a separate commercial license with Mistral before using the Model or its derivative works for any commercial purpose. The Licensee may contact Mistral, using this form: https://mistral.ai/contact/, to request a commercial license. 
+
+"Model as a Service" means giving a third party access to the Model or its derivative works for inference or post-training by any means (for instance via API or hosted endpoints), in a manner that allows such third party to exercise meaningful control over the inputs, parameters, or training data. This does not include (a) end-user products with model capabilities solely embedded within specific features or harnesses, or (b) mere relaying of requests to models hosted by others. 
+
+3. The requirements set forth in Section 2 do not apply to: (a) internal use of the Model or its derivative works, defined as any use that does not make the Model or its derivative works available to third parties; or (b) any use of the Model accessed through Mistral’s official products. 
+
+THE MODEL AND ANY OUTPUT THEREFROM ARE PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL MISTRAL BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE MODEL OR THE USE OR OTHER DEALINGS IN THE MODEL.
+
+For any questions regarding this license, please contact us using the form here: https://mistral.ai/contact/

README.md

@@ -0,0 +1,162 @@
+---
+library_name: transformers
+pipeline_tag: image-text-to-text
+base_model:
+- mistralai/Mistral-Large-4-1T-A52B
+---
+
+This is a randomly initialized tiny model based on [mistralai/Mistral-Large-4-1T-A52B](https://huggingface.co/mistralai/Mistral-Large-4-1T-A52B) for testing and debugging.
+
+| Feature | Source | This repo |
+|---|---|---|
+| Checkpoint tensor elements | 7.13B BF16, 63.84M F32, 1.05T F8_E4M3 | 54.84M BF16, 1.31M F8_E4M3, 320 F8_E8M0 |
+| Safetensors weight files | 1.06 TB (54 files) | 111.12 MB (consolidated.safetensors) |
+| Architecture / input | Multimodal MoE; text and image | Same |
+| Vocabulary rows / tied embeddings | 204,800 / false | 204,800 / true |
+| Backbone layers | 72 | 12: three complete RoPE/RoPE/RoPE/NoPE cycles |
+| Text hidden size / attention head dimension | 6,144 / 64 | 128 / 32 |
+| Routed / selected / shared experts | 128 / 4 / 1 | 32 / 4 / 1 |
+| DSA indexer | 32 heads; 128-wide; top-2,048 | 4 heads; 128-wide; top-128 |
+| Cross-layer KV sharing | Source pattern | Two complete three-own/one-shared NoPE cycles |
+| Vision tower | 30 layers; 2,048 width | 4 layers; 64 width |
+| DSpark | 3 dense MLA stages; rank 512; depth 3 | 3 dense MLA stages; rank 8; depth 3 |
+| Checkpoint precision | E4M3 block-FP8 + UE8M0 scales; BF16 exclusions | Same format; 64×64 blocks |
+
+TP=2 vLLM uses a 64-wide FP8 activation group per rank, which fixes the residual width at 128. The checkpoint retains 64×64 FP8 blocks, a 512-wide MLA latent KV required by H20 sparse MLA, and four 32-wide attention heads. The input/output BF16 token table is tied, saving one 52.43 MB matrix; the optimized vLLM kernel constraints still place the checkpoint above the 30 MB soft target.
+
+| File path | Size |
+|------|------|
+| consolidated.safetensors | 111.1MB |
+
+### Example usage
+
+#### Transformers (✅ verified)
+
+```python
+def run_transformers(model_id: str):
+    import torch
+    from PIL import Image
+    from transformers import AutoModelForImageTextToText, AutoProcessor
+    from transformers.utils.quantization_config import FineGrainedFP8Config
+
+    model = AutoModelForImageTextToText.from_pretrained(
+        str(model_id),
+        dtype=torch.bfloat16,
+        device_map="cuda",
+        attn_implementation="sdpa",
+        quantization_config=FineGrainedFP8Config(dequantize=True),
+    )
+    processor = AutoProcessor.from_pretrained(str(model_id))
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "image", "image": Image.new("RGB", (56, 56), (30, 120, 220))},
+                {"type": "text", "text": "Describe this image in one word."},
+            ],
+        }
+    ]
+    inputs = processor.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        tokenize=True,
+        return_dict=True,
+        return_tensors="pt",
+        reasoning_effort="low",
+    ).to(model.device)
+    output = model.generate(**inputs, do_sample=False, max_new_tokens=10)
+    print(processor.decode(output[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True))
+
+
+if __name__ == "__main__":
+    run_transformers(model_id="tiny-random/mistral-large-4")
+```
+
+#### vLLM offline (✅ verified)
+
+Install the snapshot package with `pip install --no-deps <model directory>` before starting vLLM.
+
+```python
+def run_vllm(model_id: str):
+    import os
+
+    os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0")
+    os.environ.setdefault("VLLM_ALLREDUCE_USE_FLASHINFER", "0")
+    os.environ.setdefault("VLLM_ALLREDUCE_USE_SYMM_MEM", "0")
+    from PIL import Image
+    from transformers import AutoProcessor
+    from vllm import LLM, SamplingParams
+
+    model_id = str(model_id)
+    processor = AutoProcessor.from_pretrained(model_id)
+    llm = LLM(
+        model=model_id,
+        trust_remote_code=True,
+        tensor_parallel_size=2,
+        speculative_config={"method": "dspark", "num_speculative_tokens": 3},
+        kv_cache_dtype="bfloat16",
+        max_model_len=512,
+        max_num_batched_tokens=1024,
+        max_num_seqs=2,
+        gpu_memory_utilization=0.1,
+        disable_custom_all_reduce=True,
+        kernel_config={"linear_backend": "triton"},
+        attention_config={"backend": "FLASHINFER_MLA_SPARSE_SM90"},
+    )
+    sampling = SamplingParams(temperature=0, max_tokens=10, ignore_eos=True)
+    try:
+        print(llm.generate(["Hello"], sampling)[0].outputs[0].text)
+        image = Image.new("RGB", (56, 56), "blue")
+        inputs = processor.apply_chat_template(
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "image", "image": image},
+                        {"type": "text", "text": "Describe this image in one word."},
+                    ],
+                }
+            ],
+            add_generation_prompt=True,
+            tokenize=True,
+            return_dict=True,
+            return_tensors="pt",
+            reasoning_effort="low",
+        )
+        result = llm.generate(
+            {"prompt_token_ids": inputs["input_ids"][0].tolist(), "multi_modal_data": {"image": image}},
+            sampling,
+        )
+        print(result[0].outputs[0].text)
+    finally:
+        llm.llm_engine.engine_core.shutdown(timeout=30)
+
+
+if __name__ == "__main__":
+    run_vllm(model_id="tiny-random/mistral-large-4")
+```
+
+#### vLLM serving (unverified)
+
+Use the official Mistral Large 4 vLLM image.
+
+```bash
+docker run --rm --runtime nvidia --gpus all --ipc=host -p 8000:8000 --entrypoint bash \
+  -v "$(pwd)/mistral-large-4:/model:ro" vllm/vllm-openai:mistral-large-4 \
+  -lc 'pip install --no-deps /model && vllm serve /model \
+  --tensor-parallel-size 2 \
+  --speculative-config '{"method":"dspark","num_speculative_tokens":3}' \
+  --kv-cache-dtype bfloat16 --disable-custom-all-reduce \
+  --kernel-config '{"linear_backend":"triton"}' \
+  --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_SM90"}' \
+  --limit-mm-per-prompt '{"image":1}' --max-model-len 2048 \
+  --max-num-batched-tokens 256 --max-num-seqs 4 --gpu-memory-utilization 0.2'
+```
+
+### Creating this repo
+
+- Code: [.meta/create_model.py](.meta/create_model.py)
+- Model structure: [.meta/model.log](.meta/model.log)
+- Environment: [.meta/env.json](.meta/env.json)
+
+*This repo is generated by GPT-6.1-Sol.*

chat_template.jinja

@@ -0,0 +1,258 @@
+{#- Default system message if no system prompt is passed. #}
+{%- set default_system_message = '' %}
+
+{#- Begin of sequence token. #}
+{{- bos_token }}
+
+
+{#- Handle system prompt if it exists. #}
+{%- set loop_messages = messages %}
+{%- if messages[0]['role'] != 'system' and default_system_message != '' %}
+    {{- '[SYSTEM_PROMPT]' ~ default_system_message ~ '[/SYSTEM_PROMPT]' }}
+{%- endif %}
+
+
+{#- Tools and model settings definition #}
+{%- set available_tools = '' %}
+{%- set has_tools = false %}
+{%- if tools is defined and tools is not none and tools|length > 0 %}
+    {%- set has_tools = true %}
+    {%- set available_tools = '[AVAILABLE_TOOLS]' ~ (tools| tojson) ~ '[/AVAILABLE_TOOLS]' %}
+{%- endif %}
+{%- set json_schema = none %}
+{%- if response_format is defined and response_format is not none %}
+    {%- set response_format_type = response_format.get('type', 'text') %}
+    {%- set response_json_schema = response_format.get('json_schema') %}
+    {%- if response_format_type not in ['text', 'json_object', 'json_schema'] %}
+        {{- raise_exception('Response format type must be one of text, json_object or json_schema, got ' ~ response_format_type ~ '.') }}
+    {%- elif response_format_type == 'json_schema' %}
+        {%- if response_json_schema is none or response_json_schema.get('schema') is none %}
+            {{- raise_exception('Response format `json_schema` must define the schema') }}
+        {%- else %}
+            {%- set json_schema = response_json_schema['schema'] %}
+        {%- endif %}
+    {%- elif response_format_type == 'json_object' %}
+        {%- set json_schema = {'anyOf': [{'type': 'object'}, {'type': 'array'}]} %}
+    {%- endif %}
+{%- endif %}
+{%- if reasoning_effort is not defined or reasoning_effort is none %}
+    {%- set reasoning_effort = 'high' %}
+{%- endif %}
+{%- if reasoning_effort not in ['low', 'medium', 'high'] %}
+    {{- raise_exception('reasoning_effort must be one of "low", "medium", "high"') }}
+{%- endif %}
+{%- set model_settings_parts = [] %}
+{%- if json_schema is not none %}
+    {%- set model_settings_parts = model_settings_parts + ['"json_schema": ' ~ (json_schema|tojson)] %}
+{%- endif %}
+{%- set model_settings_parts = model_settings_parts + ['"reasoning_effort": "' ~ reasoning_effort ~ '"'] %}
+{%- set model_settings = '' %}
+{%- if model_settings_parts %}
+    {%- set model_settings = '[MODEL_SETTINGS]{' ~ model_settings_parts|join(', ') ~ '}[/MODEL_SETTINGS]' %}
+{%- endif %}
+
+{#- Macros #}
+{%- macro render_content(content, context_name, supported_types_desc, support_thinking, support_images) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content -%}
+        {%- for block in content -%}
+            {%- if block['type'] == 'text' -%}
+                {{- block['text'] -}}
+            {%- elif support_thinking and block['type'] == 'thinking' -%}
+                {{- '[THINK]' ~ block['thinking'] -}}
+                {%- if block.get('closed', true) -%}{{- '[/THINK]' -}}{%- endif -%}
+            {%- elif support_images and block['type'] in ['image', 'image_url'] -%}
+                {{- '[IMG]' -}}
+            {%- else -%}
+                {{- raise_exception('Only ' ~ supported_types_desc ~ ' chunks are supported in ' ~ context_name ~ '.') -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{- raise_exception(context_name ~ ' must have non-empty content.') -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{#- Aggregate consecutive messages with the same role except system and tool. #}
+{#- A sentinel message is appended so the last group gets flushed inside the loop. #}
+{%- set ns_agg = namespace(messages=[], current_group=[], current_role=none) %}
+{%- for message in loop_messages + [{'role': '__sentinel__'}] %}
+    {%- if message['role'] != ns_agg.current_role or message['role'] == 'system' or message['role'] == 'tool' %}
+        {%- if ns_agg.current_role == 'tool' %}
+            {%- set ns_agg.messages = ns_agg.messages + ns_agg.current_group %}
+        {%- elif ns_agg.current_role is not none %}
+            {%- set ns_c = namespace(text_parts=[], chunks=[], has_non_text=false, tool_calls=[]) %}
+            {%- for msg in ns_agg.current_group %}
+                {#- Convert reasoning / reasoning_content to a leading thinking chunk. #}
+                {%- set reasoning = msg.get('reasoning_content', msg.get('reasoning', none)) %}
+                {%- if reasoning is not none and reasoning != '' %}
+                    {%- if msg['content'] is not none and msg['content'] is not string %}
+                        {%- for block in msg['content'] %}
+                            {%- if block['type'] == 'thinking' %}
+                                {{- raise_exception('Message cannot have both thinking chunks in content and a top-level `reasoning` or `reasoning_content` field.') }}
+                            {%- endif %}
+                        {%- endfor %}
+                    {%- endif %}
+                    {%- set think_chunk = {'type': 'thinking', 'thinking': reasoning} %}
+                    {%- if msg['content'] is string and msg['content'] != '' %}
+                        {%- set new_content = [think_chunk, {'type': 'text', 'text': msg['content']}] %}
+                    {%- elif msg['content'] is not none and msg['content'] is not string and msg['content'] | length > 0 %}
+                        {%- set new_content = [think_chunk] + msg['content'] | list %}
+                    {%- else %}
+                        {%- set new_content = [think_chunk] %}
+                    {%- endif %}
+                    {%- if msg['tool_calls'] is defined and msg['tool_calls'] is not none %}
+                        {%- set msg = {'role': msg['role'], 'content': new_content, 'tool_calls': msg['tool_calls']} %}
+                    {%- else %}
+                        {%- set msg = {'role': msg['role'], 'content': new_content} %}
+                    {%- endif %}
+                {%- endif %}
+                {%- if msg['content'] is string %}
+                    {%- set ns_c.text_parts = ns_c.text_parts + [msg['content']] %}
+                {%- elif msg['content'] is not none %}
+                    {%- set ns_msg = namespace(msg_text_parts=[]) %}
+                    {%- for block in msg['content'] %}
+                        {%- if block['type'] == 'text' %}
+                            {%- set ns_msg.msg_text_parts = ns_msg.msg_text_parts + [block['text']] %}
+                        {%- else %}
+                            {%- if ns_msg.msg_text_parts | length > 0 %}
+                                {%- set ns_c.text_parts = ns_c.text_parts + [ns_msg.msg_text_parts | join('')] %}
+                                {%- set ns_msg.msg_text_parts = [] %}
+                            {%- endif %}
+                            {%- if ns_c.text_parts | length > 0 %}
+                                {%- set ns_c.chunks = ns_c.chunks + [{'type': 'text', 'text': ns_c.text_parts | join('\n\n')}] %}
+                                {%- set ns_c.text_parts = [] %}
+                            {%- endif %}
+                            {%- set ns_c.chunks = ns_c.chunks + [block] %}
+                            {%- set ns_c.has_non_text = true %}
+                        {%- endif %}
+                    {%- endfor %}
+                    {%- if ns_msg.msg_text_parts | length > 0 %}
+                        {%- set ns_c.text_parts = ns_c.text_parts + [ns_msg.msg_text_parts | join('')] %}
+                    {%- endif %}
+                {%- endif %}
+                {%- if msg['tool_calls'] is defined and msg['tool_calls'] is not none %}
+                    {%- set ns_c.tool_calls = ns_c.tool_calls + msg['tool_calls'] | list %}
+                {%- endif %}
+            {%- endfor %}
+            {%- if ns_c.has_non_text %}
+                {%- if ns_c.text_parts | length > 0 %}
+                    {%- set ns_c.chunks = ns_c.chunks + [{'type': 'text', 'text': ns_c.text_parts | join('\n\n')}] %}
+                {%- endif %}
+                {%- set merged_content = ns_c.chunks %}
+            {%- else %}
+                {%- set merged_content = ns_c.text_parts | join('\n\n') %}
+            {%- endif %}
+            {%- if ns_c.tool_calls | length > 0 %}
+                {%- set ns_agg.messages = ns_agg.messages + [{'role': ns_agg.current_role, 'content': merged_content, 'tool_calls': ns_c.tool_calls}] %}
+            {%- else %}
+                {%- set ns_agg.messages = ns_agg.messages + [{'role': ns_agg.current_role, 'content': merged_content}] %}
+            {%- endif %}
+        {%- endif %}
+        {%- if message['role'] != '__sentinel__' %}
+            {%- set ns_agg.current_group = [message] %}
+            {%- set ns_agg.current_role = message['role'] %}
+        {%- endif %}
+    {%- else %}
+        {%- set ns_agg.current_group = ns_agg.current_group + [message] %}
+    {%- endif %}
+{%- endfor %}
+{%- set loop_messages = ns_agg.messages %}
+
+{#- Validates message ordering. #}
+{%- set ns = namespace(available_tools_and_settings_emitted=false) %}
+{%- if loop_messages | length > 0 and loop_messages[0]['role'] not in ['user', 'system'] %}
+    {{- raise_exception('Conversation must start with a user or system message, got ' ~ loop_messages[0]['role'] ~ '.') }}
+{%- endif %}
+{%- set ns_order = namespace(previous_role=none) %}
+{%- for message in loop_messages %}
+    {%- set current_role = message['role'] %}
+    {%- if ns_order.previous_role is not none %}
+        {%- if ns_order.previous_role == 'system' %}
+            {%- if current_role not in ['user', 'assistant', 'system'] %}
+                {{- raise_exception('Unexpected role \'' ~ current_role ~ '\' after role \'' ~ ns_order.previous_role ~ '\'') }}
+            {%- endif %}
+        {%- elif ns_order.previous_role == 'user' %}
+            {%- if current_role not in ['assistant', 'system', 'user'] %}
+                {{- raise_exception('Unexpected role \'' ~ current_role ~ '\' after role \'' ~ ns_order.previous_role ~ '\'') }}
+            {%- endif %}
+        {%- elif ns_order.previous_role == 'assistant' %}
+            {%- if current_role not in ['assistant', 'user', 'tool'] %}
+                {{- raise_exception('Unexpected role \'' ~ current_role ~ '\' after role \'' ~ ns_order.previous_role ~ '\'') }}
+            {%- endif %}
+        {%- elif ns_order.previous_role == 'tool' %}
+            {%- if current_role not in ['assistant', 'tool', 'user'] %}
+                {{- raise_exception('Unexpected role \'' ~ current_role ~ '\' after role \'' ~ ns_order.previous_role ~ '\'') }}
+            {%- endif %}
+        {%- endif %}
+    {%- endif %}
+    {%- set ns_order.previous_role = current_role %}
+{%- endfor %}
+
+{#- Handle conversation messages. #}
+{%- for message in loop_messages %}
+    {#- User messages supports text, image and image_url content. #}
+    {%- if message['role'] == 'user' %}
+        {%- if not ns.available_tools_and_settings_emitted %}
+            {{- available_tools }}
+            {{- model_settings }}
+            {%- set ns.available_tools_and_settings_emitted = true %}
+        {%- endif %}
+        {%- if message['content'] is not string and message['content'] %}
+            {#- When content has exactly one image and one text block, put image first. #}
+            {%- if message['content'] | length == 2 and message['content'][0]['type'] == 'text' and message['content'][1]['type'] in ['image', 'image_url'] %}
+                {%- set blocks = [message['content'][1], message['content'][0]] %}
+            {%- else %}
+                {%- set blocks = message['content'] %}
+            {%- endif %}
+            {%- set user_content = blocks %}
+        {%- else %}
+            {%- set user_content = message['content'] %}
+        {%- endif %}
+        {{- '[INST]' -}}
+        {{- render_content(content=user_content, context_name='user message content', supported_types_desc='text, image and image_url', support_thinking=false, support_images=true) -}}
+        {{- '[/INST]' }}
+
+    {#- Assistant messages supports text and thinking content. #}
+    {%- elif message['role'] == 'assistant' %}
+        {%- if (message['content'] is none or message['content'] == '' or message['content']|length == 0) and (message['tool_calls'] is not defined or message['tool_calls'] is none or message['tool_calls']|length == 0) %}
+            {{- raise_exception('Assistant message must have a string or a list of chunks in content or a list of tool calls.') }}
+        {%- endif %}
+
+        {%- if message['content'] %}
+            {{- render_content(content=message['content'], context_name='assistant message contents', supported_types_desc='text and thinking', support_thinking=true, support_images=false) -}}
+        {%- endif %}
+
+        {%- if message['tool_calls'] is defined and message['tool_calls'] is not none and message['tool_calls']|length > 0 %}
+            {%- for tool in message['tool_calls'] %}
+                {{- '[TOOL_CALLS]' }}
+                {%- set name = tool['function']['name'] %}
+                {%- set arguments = tool['function']['arguments'] %}
+                {%- if arguments is not string %}
+                    {%- set arguments = arguments|tojson|safe %}
+                {%- elif arguments == '' %}
+                    {%- set arguments = '{}' %}
+                {%- endif %}
+                {{- name ~ '[ARGS]' ~ arguments }}
+            {%- endfor %}
+        {%- endif %}
+
+        {{- eos_token }}
+
+    {#- Tool messages (multimodal). #}
+    {%- elif message['role'] == 'tool' %}
+        {{- '[TOOL_RESULTS]' -}}
+        {{- render_content(content=message['content'], context_name='tool message contents', supported_types_desc='text and image', support_thinking=false, support_images=true) -}}
+        {{- '[/TOOL_RESULTS]' }}
+
+    {#- System messages. #}
+    {%- elif message['role'] == 'system' %}
+        {{- '[SYSTEM_PROMPT]' -}}
+        {{- render_content(content=message['content'], context_name='system message contents', supported_types_desc='text', support_thinking=false, support_images=false) -}}
+        {{- '[/SYSTEM_PROMPT]' -}}
+
+    {#- Raise exception for unsupported roles. #}
+    {%- else %}
+        {{- raise_exception('Only user, assistant, system and tool roles are supported, got ' ~ message['role'] ~ '.') }}
+    {%- endif %}
+{%- endfor %}

config.json

@@ -0,0 +1,185 @@
+{
+  "architectures": [
+    "MistralLarge4ForConditionalGeneration"
+  ],
+  "dtype": "bfloat16",
+  "image_token_index": 10,
+  "model_type": "mistral_large4",
+  "multimodal_projector_bias": false,
+  "projector_hidden_act": "gelu",
+  "quantization_config": {
+    "quant_method": "fp8",
+    "activation_scheme": "dynamic",
+    "weight_block_size": [
+      64,
+      64
+    ],
+    "scale_fmt": "ue8m0",
+    "modules_to_not_convert": [
+      "model.vision_tower",
+      "model.multi_modal_projector",
+      "lm_head",
+      "kv_b_proj",
+      "indexer.wk",
+      "indexer.weights_proj",
+      "kv_a_proj_with_mqa"
+    ]
+  },
+  "spatial_merge_size": 2,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "final_logit_softcapping": 30.0,
+    "first_k_dense_replace": 1,
+    "head_dim": 32,
+    "hidden_act": "silu",
+    "hidden_size": 128,
+    "index_head_dim": 128,
+    "index_n_heads": 4,
+    "index_topk": 128,
+    "indexer_types": [
+      "full",
+      "full",
+      "full",
+      "full",
+      "full",
+      "full",
+      "shared",
+      "shared",
+      "full",
+      "full",
+      "shared",
+      "shared"
+    ],
+    "initializer_range": 0.02,
+    "intermediate_size": 128,
+    "kv_lora_rank": 512,
+    "kv_source_layers": [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      3,
+      8,
+      9,
+      10,
+      3
+    ],
+    "layer_types": [
+      "indexed_attention",
+      "indexed_attention",
+      "indexed_attention",
+      "indexed_attention",
+      "indexed_attention",
+      "indexed_attention",
+      "indexed_attention",
+      "indexed_attention",
+      "indexed_attention",
+      "indexed_attention",
+      "indexed_attention",
+      "indexed_attention"
+    ],
+    "max_position_embeddings": 1048576,
+    "mlp_layer_types": [
+      "dense",
+      "sparse",
+      "sparse",
+      "sparse",
+      "sparse",
+      "sparse",
+      "sparse",
+      "sparse",
+      "sparse",
+      "sparse",
+      "sparse",
+      "sparse"
+    ],
+    "model_type": "mistral_large4_text",
+    "moe_intermediate_size": 128,
+    "n_group": 1,
+    "n_routed_experts": 32,
+    "n_shared_experts": 1,
+    "no_rope_layer_interval": 4,
+    "no_rope_layers": [
+      1,
+      1,
+      1,
+      0,
+      1,
+      1,
+      1,
+      0,
+      1,
+      1,
+      1,
+      0
+    ],
+    "num_attention_heads": 4,
+    "num_experts_per_tok": 4,
+    "num_hidden_layers": 12,
+    "num_key_value_heads": 4,
+    "output_router_logits": false,
+    "pad_token_id": null,
+    "qk_head_dim": 128,
+    "qk_nope_head_dim": 64,
+    "qk_rope_head_dim": 64,
+    "rms_norm_eps": 1e-05,
+    "rope_parameters": {
+      "attention_factor": 1.0,
+      "beta_fast": 32,
+      "beta_slow": 1,
+      "factor": 2048,
+      "original_max_position_embeddings": 512,
+      "rope_theta": 10000.0,
+      "rope_type": "yarn"
+    },
+    "routed_scaling_factor": 1.0,
+    "tie_word_embeddings": true,
+    "topk_group": 1,
+    "use_cache": true,
+    "v_head_dim": 128,
+    "vocab_size": 204800,
+    "architectures": [
+      "MistralLarge4ForCausalLM"
+    ],
+    "mtp_config": {
+      "n_layers": 3,
+      "depth": 3,
+      "ragged_attention": "32,32,32",
+      "target_activation_layers": "1,4,7,10,11",
+      "norm_strategy": "eagle",
+      "dspark": {
+        "markov_rank": 4,
+        "head_type": "rnn",
+        "draft_input": "position_specific"
+      }
+    }
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.21.0.dev0",
+  "transformers_weights": "consolidated.safetensors",
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "head_dim": 32,
+    "hidden_act": "silu",
+    "hidden_size": 64,
+    "image_size": 1540,
+    "initializer_range": 0.02,
+    "intermediate_size": 128,
+    "model_type": "pixtral",
+    "num_attention_heads": 2,
+    "num_channels": 3,
+    "num_hidden_layers": 4,
+    "patch_size": 14,
+    "rope_parameters": {
+      "rope_theta": 10000.0,
+      "rope_type": "axial"
+    }
+  },
+  "vision_feature_layer": -1
+}

consolidated.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:233e1a76d998d19bf117cf6acc381c31eeb46a3313811543d60f7d42cfaada56
+size 111124456

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "max_length": 1048576,
+  "pad_token_id": 11,
+  "temperature": 1.0,
+  "top_p": 0.95,
+  "transformers_version": "5.21.0.dev0"
+}

params.json

@@ -0,0 +1,99 @@
+{
+  "dim": 128,
+  "n_layers": 12,
+  "head_dim": 32,
+  "hidden_dim": 128,
+  "n_heads": 4,
+  "n_kv_heads": 4,
+  "rope_theta": 10000.0,
+  "norm_eps": 1e-05,
+  "vocab_size": 204800,
+  "tied_embeddings": true,
+  "max_position_embeddings": 1048576,
+  "ragged_attention": null,
+  "nope": "rope,rope,rope,nope",
+  "softmax_tanh_gating": 30.0,
+  "llama_4_scaling": null,
+  "q_lora_rank": null,
+  "qk_rope_head_dim": 64,
+  "qk_nope_head_dim": 64,
+  "kv_lora_rank": 512,
+  "v_head_dim": 128,
+  "ssmax_scaling": true,
+  "lerope": true,
+  "dsa": {
+    "topk": 128,
+    "n_heads": 4,
+    "head_dim": 128,
+    "rope_head_dim": 64,
+    "match_nope": true,
+    "ssmax_scaling": true,
+    "norm_eps": 1e-06,
+    "sharing": "full,full,full,full,full,full,shared,shared,full,full,shared,shared",
+    "fp8_indexer": true,
+    "fp8_kv_cache": true
+  },
+  "quantization": {
+    "qscheme_act": "BLOCK",
+    "qformat_weight": "fp8_e4m3",
+    "qdynamic_act": true,
+    "qscheme_weight": "BLOCK",
+    "use_ue8m0_for_qscales": true,
+    "block_size_out": 64,
+    "block_size_in": 64,
+    "quantize_kv_a": false
+  },
+  "yarn": {
+    "original_max_position_embeddings": 512,
+    "factor": 2048,
+    "apply_scale": false,
+    "beta": 32,
+    "alpha": 1
+  },
+  "kv_sharing": {
+    "source_layers_str": "0,1,2,3,4,5,6,3,8,9,10,3"
+  },
+  "moe": {
+    "expert_parallel": 1,
+    "expert_model_parallel": 1,
+    "route_every_n": 1,
+    "first_k_dense_replace": 1,
+    "num_experts": 32,
+    "num_experts_per_tok": 4,
+    "num_expert_groups": 1,
+    "num_expert_groups_per_tok": 1,
+    "routed_scale": 1.0,
+    "expert_hidden_dim": 128,
+    "num_shared_experts": 1
+  },
+  "vision_encoder": {
+    "image_token_id": 10,
+    "image_break_token_id": 12,
+    "image_end_token_id": 13,
+    "intermediate_size": 128,
+    "num_hidden_layers": 4,
+    "num_attention_heads": 2,
+    "mm_projector_id": "patch_merge",
+    "spatial_merge_size": 2,
+    "hidden_size": 64,
+    "num_channels": 3,
+    "image_size": 1540,
+    "max_image_size": 1540,
+    "patch_size": 14,
+    "rope_theta": 10000.0,
+    "add_pre_mm_projector_layer_norm": true,
+    "adapter_bias": false
+  },
+  "mtp": {
+    "n_layers": 3,
+    "depth": 3,
+    "ragged_attention": "32,32,32",
+    "target_activation_layers": "1,4,7,10,11",
+    "norm_strategy": "eagle",
+    "dspark": {
+      "markov_rank": 4,
+      "head_type": "rnn",
+      "draft_input": "position_specific"
+    }
+  }
+}

processor_config.json

@@ -0,0 +1,32 @@
+{
+  "image_break_token": "[IMG_BREAK]",
+  "image_end_token": "[IMG_END]",
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_pad": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "PixtralImageProcessor",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "patch_size": 14,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 1540
+    }
+  },
+  "image_token": "[IMG]",
+  "patch_size": 14,
+  "processor_class": "PixtralProcessor",
+  "spatial_merge_size": 2
+}

pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ['setuptools>=61']
+build-backend = 'setuptools.build_meta'
+
+[project]
+name = 'mistral-large-4-tied-vllm'
+version = '0.1.0'
+
+[project.entry-points.'vllm.general_plugins']
+mistral_large4_tied = 'vllm_plugin:register'
+
+[tool.setuptools]
+py-modules = ['vllm_mistral_large4_tied', 'vllm_plugin']

tekken.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc05cc0f7d8ef11e61151df069ffa99499b040e29309006f7bac5a470b8ddb67
+size 25813790

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23a444357919e98375ac640bdd08723430ba5965da96c5f920ee2bad2390dc3e
+size 27851075

tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "model_max_length": 1048576,
+  "pad_token": "<pad>",
+  "processor_class": "PixtralProcessor",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}

vllm_mistral_large4_tied.py

@@ -0,0 +1,9 @@
+from vllm.distributed import get_pp_group
+from vllm.models.mistral_large4 import MistralLarge4ForCausalLM
+
+
+class MistralLarge4TiedForCausalLM(MistralLarge4ForCausalLM):
+    def __init__(self, *, vllm_config, prefix=""):
+        super().__init__(vllm_config=vllm_config, prefix=prefix)
+        if self.config.tie_word_embeddings and get_pp_group().is_last_rank:
+            self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)

vllm_plugin.py

@@ -0,0 +1,7 @@
+def register():
+    from vllm import ModelRegistry
+
+    ModelRegistry.register_model(
+        "MistralLarge4ForCausalLM",
+        "vllm_mistral_large4_tied:MistralLarge4TiedForCausalLM",
+    )