hrm_penguin/vision/encoder.py

from __future__ import annotations

from types import MethodType

import torch
import torch.nn.functional as F
from torch import nn


def _rotate_half(value: torch.Tensor) -> torch.Tensor:
    first, second = value.chunk(2, dim=-1)
    return torch.cat((-second, first), dim=-1)


def _apply_penguin_rotary(query, key, cos, sin):
    section = [cos.shape[-1] // 2, cos.shape[-1] // 2]
    cos = torch.cat([axis[index % 2] for index, axis in enumerate(cos.split(section, dim=-1))], dim=-1)
    sin = torch.cat([axis[index % 2] for index, axis in enumerate(sin.split(section, dim=-1))], dim=-1)
    cos = cos.unsqueeze(1)
    sin = sin.unsqueeze(1)
    return query * cos + _rotate_half(query) * sin, key * cos + _rotate_half(key) * sin


def _penguin_sdpa_attention_forward(
    module,
    hidden_states: torch.Tensor,
    position_embeddings,
    attention_mask=None,
    past_key_value=None,
    cache_position=None,
    cu_seqlens=None,
    **kwargs,
):
    """SDPA equivalent of Penguin's packed, non-causal FlashAttention path."""
    input_shape = hidden_states.shape[:-1]
    hidden_shape = (*input_shape, -1, module.head_dim)
    query = module.q_norm(module.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
    key = module.k_norm(module.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
    value = module.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
    cos, sin = position_embeddings
    query, key = _apply_penguin_rotary(query, key, cos, sin)
    if past_key_value is not None:
        cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
        key, value = past_key_value.update(key, value, module.layer_idx, cache_kwargs)

    if cu_seqlens is None:
        attended = F.scaled_dot_product_attention(
            query,
            key,
            value,
            attn_mask=attention_mask,
            dropout_p=0.0,
            is_causal=False,
            enable_gqa=query.shape[1] != key.shape[1],
        )
    else:
        # Penguin packs image sequences along S inside a singleton batch. Splitting
        # here prevents patches from different images attending to one another.
        boundaries = cu_seqlens.detach().cpu().tolist()
        chunks = []
        for start, end in zip(boundaries[:-1], boundaries[1:]):
            chunks.append(
                F.scaled_dot_product_attention(
                    query[:, :, start:end],
                    key[:, :, start:end],
                    value[:, :, start:end],
                    dropout_p=0.0,
                    is_causal=False,
                    enable_gqa=query.shape[1] != key.shape[1],
                )
            )
        attended = torch.cat(chunks, dim=2)
    attended = attended.transpose(1, 2).reshape(*input_shape, -1).contiguous()
    return module.o_proj(attended), None


def install_penguin_sdpa(model: nn.Module) -> int:
    patched = 0
    for module in model.modules():
        if module.__class__.__name__ == "PenguinVLAttention":
            module.forward = MethodType(_penguin_sdpa_attention_forward, module)
            patched += 1
    if patched == 0:
        raise RuntimeError("No PenguinVLAttention modules found for SDPA patch")
    return patched


def install_penguin_transformers_compatibility() -> None:
    """Bridge Penguin remote code written for Transformers 4.51 to 5.x RoPE APIs."""
    from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS

    if "default" not in ROPE_INIT_FUNCTIONS:
        ROPE_INIT_FUNCTIONS["default"] = compute_default_rope_parameters


def compute_default_rope_parameters(config, device=None, **kwargs):
    parameters = getattr(config, "rope_parameters", None) or getattr(config, "rope_scaling", None) or {}
    theta = float(parameters.get("rope_theta", getattr(config, "rope_theta", 10000.0)))
    dimension = int(getattr(config, "head_dim", config.hidden_size // config.num_attention_heads))
    inverse_frequency = 1.0 / (
        theta ** (torch.arange(0, dimension, 2, dtype=torch.float32, device=device) / dimension)
    )
    return inverse_frequency, 1.0


class PenguinVisionEncoder(nn.Module):
    """Thin checked wrapper around the official Penguin encoder."""

    def __init__(self, model: nn.Module, hidden_size: int = 1024):
        super().__init__()
        self.model = model
        self.hidden_size = hidden_size

    def forward(self, pixel_values, grid_sizes, merge_sizes) -> torch.Tensor:
        features = self.model(
            pixel_values=pixel_values,
            grid_sizes=grid_sizes,
            merge_sizes=merge_sizes,
        )
        if hasattr(features, "last_hidden_state"):
            features = features.last_hidden_state
        if features.ndim != 2 or features.shape[-1] != self.hidden_size:
            raise RuntimeError(
                f"Expected packed Penguin output [sum(N_i), {self.hidden_size}], "
                f"got {tuple(features.shape)}"
            )
        if grid_sizes.ndim != 2 or grid_sizes.shape[1] != 3:
            raise RuntimeError(f"Invalid grid_sizes shape: {tuple(grid_sizes.shape)}")
        if merge_sizes.ndim != 1 or merge_sizes.shape[0] != grid_sizes.shape[0]:
            raise RuntimeError(f"Invalid merge_sizes shape: {tuple(merge_sizes.shape)}")
        spatial = grid_sizes[:, 1:] // merge_sizes[:, None]
        if bool((spatial * merge_sizes[:, None] != grid_sizes[:, 1:]).any()):
            raise RuntimeError("Grid dimensions must be divisible by merge_size")
        counts = grid_sizes[:, 0] * spatial[:, 0] * spatial[:, 1]
        if features.shape[0] != int(counts.sum()):
            raise RuntimeError(
                "Packed Penguin features do not match dynamic grids: "
                f"features={features.shape[0]}, counts={counts.tolist()}"
            )
        return features