remove vLLM plugin

plugins/vllm_glm5v/glm5v.py

@@ -1,241 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""
-glm5v: out-of-tree vLLM model = MoonViT (Kimi-K2.6) vision tower + trained MLP
-projector + GLM-5.2 (GlmMoeDsa) text decoder.
-
-This is a thin frankenstein over vLLM's in-tree `kimi_k25` VLM:
-  * vision tower + projector classes are imported VERBATIM from kimi_k25_vit
-    (the MoonViT3d encoder + KimiK25MultiModalProjector are bit-identical to the
-    encoder/projector we trained against).
-  * the text backbone is swapped from DeepseekV2ForCausalLM -> GlmMoeDsaForCausalLM
-    (GLM-5.2; same `deepseek_v2.py` MLA+MoE+DSA stack in vLLM).
-  * the image placeholder is GLM's `<|image|>` (154854), span
-    `<|begin_of_image|><|image|><|end_of_image|>`, instead of Kimi's `<|media_pad|>`.
-  * projector `linear_2` outputs GLM hidden (6144) via vision_config.mm_hidden_size.
-
-Register out-of-tree via `ModelRegistry.register_model` (see register.py) with the
-config.json `architectures = ["Glm5vForConditionalGeneration"]`.
-"""
-
-from collections.abc import Iterable, Mapping
-from typing import Any
-
-import torch
-from torch import nn
-
-from vllm.config import VllmConfig
-from vllm.logger import init_logger
-from vllm.model_executor.layers.quantization import QuantizationConfig
-from vllm.model_executor.layers.quantization.compressed_tensors import (
-    compressed_tensors,
-)
-from vllm.model_executor.models.interfaces import (
-    SupportsMultiModal,
-    SupportsPP,
-    SupportsQuant,
-)
-from vllm.model_executor.models.kimi_k25 import (
-    KimiK25DummyInputsBuilder,
-    KimiK25MultiModalProcessor,
-    KimiK25ProcessingInfo,
-)
-from vllm.model_executor.models.kimi_k25_vit import (
-    KimiK25MultiModalProjector,
-    MoonViT3dPretrainedModel,
-    vision_tower_forward,
-)
-from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.inputs import NestedTensors
-from vllm.platforms import current_platform
-from vllm.sequence import IntermediateTensors
-
-from .glm5v_config import Glm5vConfig
-from vllm.model_executor.models.utils import (
-    AutoWeightsLoader,
-    WeightsMapper,
-    init_vllm_registered_model,
-    maybe_prefix,
-)
-
-logger = init_logger(__name__)
-
-
-class Glm5vProcessingInfo(KimiK25ProcessingInfo):
-    """Processing info: GLM tokenizer + `<|image|>` placeholder + MoonViT image proc.
-
-    Reuses Kimi-K2.6's image processor (identical MoonViT preprocessing) and its
-    `KimiK25Processor` wiring; only the config type and placeholder token differ.
-    """
-
-    def get_hf_config(self):
-        return self.ctx.get_hf_config(Glm5vConfig)
-
-
-class Glm5vDummyInputsBuilder(KimiK25DummyInputsBuilder):
-    pass
-
-
-class Glm5vMultiModalProcessor(KimiK25MultiModalProcessor):
-    pass
-
-
-@MULTIMODAL_REGISTRY.register_processor(
-    Glm5vMultiModalProcessor,
-    info=Glm5vProcessingInfo,
-    dummy_inputs=Glm5vDummyInputsBuilder,
-)
-class Glm5vForConditionalGeneration(
-    nn.Module,
-    SupportsMultiModal,
-    SupportsPP,
-    SupportsQuant,
-):
-    """glm5v: MoonViT vision + trained projector + GLM-5.2 text."""
-
-    supports_encoder_tp_data = True
-
-    hf_to_vllm_mapper = WeightsMapper(
-        orig_to_new_prefix={
-            # GLM-5.2 text shards are stored with raw `model.*` / `lm_head.*` keys;
-            # nest them under the `language_model` submodule. (vision_tower.* and
-            # mm_projector.* keys are already correctly namespaced in our assembly.)
-            "lm_head.": "language_model.lm_head.",
-            "model.": "language_model.model.",
-        }
-    )
-
-    @classmethod
-    def get_placeholder_str(cls, modality: str, i: int) -> str | None:
-        if modality == "image":
-            # GLM image span; the inner <|image|> token is the one expanded.
-            return "<|begin_of_image|><|image|><|end_of_image|>"
-        raise ValueError(f"Unsupported modality: {modality}")
-
-    def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
-        super().__init__()
-        model_config = vllm_config.model_config
-        config: Glm5vConfig = model_config.hf_config
-        self.config = config
-        quant_config = vllm_config.quant_config
-
-        self.use_data_parallel = (
-            model_config.multimodal_config.mm_encoder_tp_mode == "data"
-        )
-        self.hidden_size = config.text_config.hidden_size
-        self.device = current_platform.current_device()
-
-        # Vision tower + projector (MoonViT, kept out of compressed-tensors quant).
-        with self._mark_tower_model(vllm_config, "vision_chunk"):
-            self.vision_tower = MoonViT3dPretrainedModel(
-                config.vision_config,
-                quant_config=self._maybe_ignore_quant_config(quant_config),
-                prefix=maybe_prefix(prefix, "vision_tower"),
-            )
-            self.vision_tower = self.vision_tower.to(
-                device=self.device, dtype=model_config.dtype
-            )
-            self.mm_projector = KimiK25MultiModalProjector(
-                config=config.vision_config,
-                use_data_parallel=self.use_data_parallel,
-                quant_config=self._maybe_ignore_quant_config(quant_config),
-                prefix=maybe_prefix(prefix, "mm_projector"),
-            )
-            self.mm_projector = self.mm_projector.to(
-                device=self.device, dtype=model_config.dtype
-            )
-
-        self.quant_config = quant_config
-        with self._mark_language_model(vllm_config):
-            self.language_model = init_vllm_registered_model(
-                vllm_config=vllm_config,
-                hf_config=config.text_config,
-                prefix=maybe_prefix(prefix, "language_model"),
-                architectures=["GlmMoeDsaForCausalLM"],
-            )
-        self.make_empty_intermediate_tensors = (
-            self.language_model.make_empty_intermediate_tensors
-        )
-        self.media_placeholder: int = self.config.media_placeholder_token_id
-
-    def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
-        # Keep the MoonViT vision tower + trained projector in bf16; only the GLM
-        # text backbone is quantized. The GLM-5.2-NVFP4 checkpoint uses ModelOpt
-        # NVFP4 (modelopt_fp4) for the text Linears, but our vision/projector
-        # weights are bf16 -- so skip quant for them regardless of method.
-        from vllm.model_executor.layers.quantization.modelopt import (
-            ModelOptNvFp4Config,
-            ModelOptFp8Config,
-        )
-
-        if isinstance(
-            quant_config,
-            (
-                compressed_tensors.CompressedTensorsConfig,
-                ModelOptNvFp4Config,
-                ModelOptFp8Config,
-            ),
-        ):
-            return None
-        return quant_config
-
-    def _parse_and_validate_media_input(self, **kwargs: object):
-        pixel_values = kwargs.pop("pixel_values", None)
-        grid_thws = kwargs.pop("grid_thws", None)
-        if pixel_values is None:
-            return None
-        if isinstance(pixel_values, list):
-            pixel_values = torch.cat(pixel_values, dim=0)
-        if len(pixel_values.shape) == 5 or len(pixel_values.shape) == 3:
-            pixel_values = pixel_values.reshape(
-                pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
-            )
-        target_dtype = next(self.vision_tower.parameters()).dtype
-        pixel_values = pixel_values.to(target_dtype)
-        assert isinstance(grid_thws, torch.Tensor)
-        grid_thws = grid_thws.reshape(-1, grid_thws.shape[-1])
-        assert grid_thws.ndim == 2 and grid_thws.size(1) == 3
-        return {"pixel_values": pixel_values, "grid_thws": grid_thws}
-
-    def _process_media_input(self, media_input) -> list[torch.Tensor]:
-        return vision_tower_forward(
-            self.vision_tower,
-            media_input["pixel_values"],
-            media_input["grid_thws"],
-            mm_projector=self.mm_projector,
-            use_data_parallel=self.use_data_parallel,
-        )
-
-    def embed_multimodal(self, **kwargs: object) -> NestedTensors | None:
-        media_input = self._parse_and_validate_media_input(**kwargs)
-        if media_input is None:
-            return None
-        return self._process_media_input(media_input)
-
-    def forward(
-        self,
-        input_ids: torch.Tensor,
-        positions: torch.Tensor,
-        intermediate_tensors: IntermediateTensors | None = None,
-        inputs_embeds: torch.Tensor | None = None,
-        **kwargs: object,
-    ) -> IntermediateTensors:
-        if intermediate_tensors is not None:
-            inputs_embeds = None
-        return self.language_model(
-            input_ids=input_ids,
-            positions=positions,
-            intermediate_tensors=intermediate_tensors,
-            inputs_embeds=inputs_embeds,
-        )
-
-    def compute_logits(self, hidden_states: torch.Tensor, **kwargs) -> torch.Tensor:
-        return self.language_model.compute_logits(hidden_states)
-
-    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
-        loader = AutoWeightsLoader(self)
-        loaded = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
-        # Sanity log: confirm the trained projector tensors were actually loaded
-        # (not left at init). Helps catch key-name mismatches.
-        proj = sorted(k for k in loaded if "mm_projector" in k)
-        logger.info("[glm5v] mm_projector keys loaded (%d): %s", len(proj), proj)
-        return loaded