configuration_groundinganything.py

from dataclasses import dataclass

from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
try:
    from transformers.configuration_utils import PreTrainedConfig
except ImportError:
    from transformers.configuration_utils import PretrainedConfig as PreTrainedConfig


@dataclass(init=False)
class GroundAnythingVLMVisionConfig(PreTrainedConfig):
    model_type = "groundinganything_vision"
    base_config_key = "vision_config"

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    hidden_size: int = 1024
    intermediate_size: int = 4096
    num_hidden_layers: int = 24
    num_attention_heads: int = 16
    num_channels: int = 3
    image_size: int = 448
    patch_size: int = 14
    hidden_act: str = "gelu"
    layer_norm_eps: float = 1e-6
    layer_norm_type: str = "layer_norm"
    attention_dropout: float = 0.0
    initializer_range: float = 0.02
    rope_theta: float = 10000.0
    use_head: bool = False
    out_hidden_size: int = 1024
    spatial_merge_size: int = 2
    tokens_per_second: int = 1
    frame_windows_size: int = 4
    use_patch_position_encoding: bool = False
    patch_position_encoding_type: str = "absolute"
    max_position_embeddings: int = 8192
    init_pos_emb_height: int = 64
    init_pos_emb_width: int = 64
    init_pos_emb_time: int = 4
    pos_emb_type: str = "divided_fixed"
    merge_kernel_size: list | None = None
    merge_type: str = "sd2_tpool"
    qkv_hidden_size: int = 1536
    norm_type: str = "rmsnorm"
    attn_bias: bool = False
    patch_embed_proj_bias: bool = False
    mlp_type: str = "mlp2"
    linear_bias: bool = False
    activation_func: str = "gelu_pytorch_tanh"
    pos_emb_interpolation_mode: str = "bilinear"
    projector_hidden_size: int = 4096
    projector_hidden_act: str = "gelu"
    projector_ln_eps: float = 1e-5


@dataclass(init=False)
class GroundAnythingVLMConfig(PreTrainedConfig):
    r"""
    This is the configuration class to store the configuration of a [`GroundAnythingVLMBaseModel`]. It is used to instantiate a
    GroundAnythingVLMBaseModel model according to the specified arguments, defining the model architecture. Instantiating a configuration
    with the defaults will yield a GroundAnything-VLM configuration.

    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PreTrainedConfig`] for more information.

    Args:
        text_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `Qwen3Config`):
            The config object or dictionary of the text backbone.
        vision_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `GroundAnythingVLMVisionConfig`):
            The config object or dictionary of the vision backbone.
        image_token_id (`int`, *optional*, defaults to 151655):
            The image token index to encode the image prompt.
        video_token_id (`int`, *optional*, defaults to 151656):
            The video token index to encode the image prompt.
        vision_start_token_id (`int`, *optional*, defaults to 151652):
            The token index to denote start of vision input.
        vision_end_token_id (`int`, *optional*, defaults to 151653):
            The token index to denote end of vision input.
    """

    model_type = "groundinganything_vlm"
    # `text_config` is resolved dynamically based on its `model_type` (defaults to `qwen3`),
    # so we use `AutoConfig` here as a placeholder; `__post_init__` swaps it for the
    # concrete config class via `CONFIG_MAPPING`.
    sub_configs = {"vision_config": GroundAnythingVLMVisionConfig, "text_config": Qwen3Config}
    keys_to_ignore_at_inference = ["past_key_values"]


    def __init__(self, **kwargs):
        self.text_config = kwargs.pop("text_config", None)
        self.vision_config = kwargs.pop("vision_config", None)
        self.image_token_id = kwargs.pop("image_token_id", 151655)
        self.video_token_id = kwargs.pop("video_token_id", 151656)
        self.vision_start_token_id = kwargs.pop("vision_start_token_id", 151652)
        self.vision_end_token_id = kwargs.pop("vision_end_token_id", 151653)
        self.tie_word_embeddings = kwargs.pop("tie_word_embeddings", False)
        self.bos_token_id = kwargs.pop("bos_token_id", None)
        self.eos_token_id = kwargs.pop("eos_token_id", None)
        self.pad_token_id = kwargs.pop("pad_token_id", None)
        if isinstance(self.vision_config, dict):
            self.vision_config = GroundAnythingVLMVisionConfig(**self.vision_config)
        if isinstance(self.text_config, dict):
            text_model_type = self.text_config.get("model_type", "qwen3")
            if text_model_type != "qwen3":
                raise ValueError(f"unsupported text model type: {text_model_type}")
            text_config_cls = Qwen3Config
            self.sub_configs["text_config"] = text_config_cls
            self.text_config = text_config_cls(**self.text_config)
        # Transformers 5.x uses a generated dataclass __init__ that dispatches
        # to this subclass' __post_init__. Call the base hook explicitly so its
        # private attention/config state is initialized. Keep 4.x compatible.
        if hasattr(PreTrainedConfig, "__post_init__"):
            PreTrainedConfig.__post_init__(self, **kwargs)
        else:
            super().__init__(**kwargs)
        self.__post_init__()

    text_config: dict | PreTrainedConfig | None = None
    vision_config: dict | PreTrainedConfig | None = None
    image_token_id: int = 151655
    video_token_id: int = 151656
    vision_start_token_id: int = 151652
    vision_end_token_id: int = 151653
    tie_word_embeddings: bool = False
    # Generation-related token ids are mirrored from `text_config` in `__post_init__`
    # so downstream tools (e.g. `generate`, vLLM) that read them at the top level keep working.
    bos_token_id: int | None = None
    eos_token_id: int | list[int] | None = None
    pad_token_id: int | None = None

    def __post_init__(self, **kwargs):
        # Resolve vision_config
        if isinstance(self.vision_config, dict):
            self.vision_config = self.sub_configs["vision_config"](**self.vision_config)
        elif self.vision_config is None:
            self.vision_config = self.sub_configs["vision_config"]()

        # Resolve text_config dynamically via CONFIG_MAPPING (defaults to qwen3)
        if isinstance(self.text_config, dict):
            text_model_type = self.text_config.get("model_type", "qwen3")
            self.text_config["model_type"] = text_model_type
            if text_model_type != "qwen3":
                raise ValueError(f"unsupported text model type: {text_model_type}")
            text_config_cls = Qwen3Config
            self.sub_configs["text_config"] = text_config_cls
            self.text_config = text_config_cls(**self.text_config)
        elif self.text_config is None:
            text_config_cls = Qwen3Config
            self.sub_configs["text_config"] = text_config_cls
            self.text_config = text_config_cls()

        # Mirror generation-related token ids from text_config to the top level so
        # downstream tools (e.g. `generate`, chat templates, vLLM) that read them
        # from the top-level config keep working.
        for tok_key in ("bos_token_id", "eos_token_id", "pad_token_id"):
            text_val = getattr(self.text_config, tok_key, None)
            if text_val is not None and getattr(self, tok_key, None) is None:
                setattr(self, tok_key, text_val)

        del kwargs


__all__ = ["GroundAnythingVLMConfig", "GroundAnythingVLMVisionConfig", "GroundAnythingConfig"]


class GroundAnythingConfig(GroundAnythingVLMConfig):
    """DLM release identity for the shared Qwen3 VLM configuration."""

    model_type = "groundinganything"