invoker-render / h3vae /vae_vit.py
multimodalart's picture
multimodalart HF Staff
Invoker-Render demo
10cdaf1 verified
Raw History Blame Contribute Delete
13.4 kB
# SPDX-License-Identifier: Apache-2.0
# ViT3D decoder for the MiniMax H3 visual VAE (inference-only bundle).
import torch
import torch.distributed as dist
import torch.nn as nn
from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.models.modeling_utils import ModelMixin
from diffusers.utils import logging
from .base_module import RotaryEmbeddingND, TransformerBlock
from .flash import make_block_causal_mask_mod
from .vit_utils import create_token_ids, prepare_rotary_pos_emb
logger = logging.get_logger(__name__)
def _linear_with_module_dtype(linear, tensor, out_dtype=None):
weight = getattr(linear, "weight", None)
target_dtype = getattr(weight, "dtype", tensor.dtype)
output = linear(tensor.to(target_dtype))
if out_dtype is not None and output.dtype != out_dtype:
output = output.to(out_dtype)
return output
def _pack_tensors_3d(tensors, patch_size, patch_size_t):
batch_size, num_channels_tensors, temporal, height, width = tensors.shape
tensors = tensors.view(
batch_size,
num_channels_tensors,
temporal // patch_size_t,
patch_size_t,
height // patch_size,
patch_size,
width // patch_size,
patch_size,
)
tensors = tensors.permute(0, 2, 4, 6, 1, 3, 5, 7)
tensors = tensors.reshape(
batch_size,
(temporal // patch_size_t) * (height // patch_size) * (width // patch_size),
num_channels_tensors * patch_size_t * patch_size * patch_size,
)
return tensors
def _unpack_tensors_3d(tensors, patch_size, patch_size_t, temporal, height, width):
batch_size, num_patches, channels = tensors.shape
num_channels_tensors = channels // (patch_size_t * patch_size * patch_size)
tensors = tensors.view(
batch_size,
temporal // patch_size_t,
height // patch_size,
width // patch_size,
num_channels_tensors,
patch_size_t,
patch_size,
patch_size,
)
tensors = tensors.permute(0, 4, 1, 5, 2, 6, 3, 7).contiguous()
tensors = tensors.reshape(batch_size, num_channels_tensors, temporal, height, width)
return tensors
class ViTBase(ModelMixin, ConfigMixin):
"""Base class for ViT Encoder and Decoder with common functionality."""
_no_split_modules = ["TransformerBlock"]
def _init_weights(self):
def basic_init(m):
if isinstance(m, nn.Linear):
nn.init.xavier_uniform_(m.weight)
if m.bias is not None:
nn.init.constant_(m.bias, 0)
self.apply(basic_init)
def init_mask_config(self, dim, is_3d=False):
self._mask_dim = dim
self._mask_is_3d = is_3d
self.register_buffer("mask_token", torch.zeros(1, 1, dim))
def set_mask_config(self, mask_config):
self.mask_prob = mask_config.get("mask_prob", 0.0)
self.mask_enabled = self.mask_prob > 0
self.mask_style = mask_config.get("mask_style", "replace")
if self.mask_enabled and self.mask_style == "drop" and self.mask_prob < 1.0:
logger.warning("mask_style='drop' with mask_prob < 1.0")
if self._mask_is_3d:
self.temporal_scale_range = mask_config.get(
"temporal_scale_range", (0.3, 0.5)
)
self.spatial_scale_range = mask_config.get(
"spatial_scale_range", (0.1, 0.25)
)
self.min_mask_ratio = mask_config.get("min_mask_ratio", 0.75)
self.max_mask_ratio = mask_config.get("max_mask_ratio", 0.95)
else:
self.spatial_scale_range = mask_config.get(
"spatial_scale_range", (0.15, 0.15)
)
self.min_mask_ratio = mask_config.get("min_mask_ratio", 0.5)
self.max_mask_ratio = mask_config.get("max_mask_ratio", 0.75)
self.aspect_ratio_range = mask_config.get("aspect_ratio_range", (0.75, 1.5))
self.max_retries = mask_config.get("max_retries", 100)
if (
self.mask_enabled
and self.mask_style == "drop"
and getattr(self, "t_causal", False)
):
logger.warning("mask_style='drop' with t_causal may cause issues")
if self.mask_enabled and "mask_token" in self._buffers:
del self._buffers["mask_token"]
self.mask_token = nn.Parameter(torch.randn(1, 1, self._mask_dim) * 0.02)
def init_suffix_tokens(self, dim, num_register_tokens, has_cls_token=True):
self.num_register_tokens = num_register_tokens
if num_register_tokens > 0:
self.register_tokens = nn.Parameter(
torch.randn(1, num_register_tokens, dim) * 0.02
)
else:
self.register_tokens = None
if has_cls_token:
self.cls_token = nn.Parameter(torch.randn(1, 1, dim) * 0.02)
def apply_mask_preprocess(self, hidden_states, img_ids, patch_dims, num_suffix):
if self.training and self.mask_enabled:
raise NotImplementedError(
"mask modeling is not supported in this inference-only bundle"
)
return hidden_states, img_ids
def forward_transformer_blocks(self, hidden_states, rotary_pos_emb, pack_info=None):
if pack_info is None:
pack_info = {}
for block in self.transformer_blocks:
hidden_states = block(hidden_states, rotary_pos_emb, pack_info)
return hidden_states
def apply_mask_postprocess(self, hidden_states, num_patches):
if self.training and self.mask_enabled and self.mask_style == "drop":
raise NotImplementedError(
"mask modeling is not supported in this inference-only bundle"
)
return hidden_states
class ViT3DDecoder(ViTBase):
"""Vision Transformer Video Decoder using TransformerBlock."""
@register_to_config
def __init__(
self,
patch_size: int = 16,
patch_size_t: int = 4,
t_causal: bool = False,
in_channels: int = 16,
out_channels: int = 3,
num_layers: int = 24,
heads: int = 16,
dim_head: int = 64,
norm_type: str = "layer_norm",
norm_affine: bool = True,
qk_norm_type: str = None,
qk_norm_affine: bool = False,
ffn_activation_fn: str = "gelu",
ffn_use_gated: bool = False,
rope_theta: float = 100.0,
rope_dim_ratio: float = 1.0,
bias: bool = True,
eps: float = 1e-5,
num_register_tokens: int = 4,
mask_config: dict = {},
**kwargs,
):
super().__init__()
dim = heads * dim_head
rope_apply_dim = int(dim_head * rope_dim_ratio)
self.pos_embed = RotaryEmbeddingND(
rope_apply_dim, rope_theta, n_dim=3, use_angle=True
)
self.x_embedder = nn.Linear(in_channels, dim)
self.init_suffix_tokens(dim, num_register_tokens, has_cls_token=False)
self.t_causal = t_causal
self.transformer_blocks = nn.ModuleList(
[
TransformerBlock(
heads=heads,
dim_head=dim_head,
norm_type=norm_type,
norm_affine=norm_affine,
qk_norm_type=qk_norm_type,
qk_norm_affine=qk_norm_affine,
ffn_activation_fn=ffn_activation_fn,
ffn_use_gated=ffn_use_gated,
bias=bias,
eps=eps,
**kwargs,
)
for _ in range(num_layers)
]
)
self.norm_out = nn.LayerNorm(dim, elementwise_affine=norm_affine, eps=eps)
patch_dim = out_channels * patch_size_t * patch_size * patch_size
self.proj_out = nn.Linear(dim, patch_dim)
self.init_mask_config(dim, is_3d=True)
self.set_mask_config(mask_config)
self._rotary_pos_emb_cache = None
self._autocast_linear_dtype = None
if len(kwargs) > 0 and (not dist.is_initialized() or dist.get_rank() == 0):
logger.warning(f"Unused kwargs: {kwargs}")
def _apply(self, fn, recurse=True):
result = super()._apply(fn, recurse=recurse)
self._rotary_pos_emb_cache = None
self._autocast_linear_dtype = None
return result
def prepare_autocast_linear_weights(self, dtype: torch.dtype) -> int:
"""Keep decoder-block linear weights in their autocast compute dtype.
PyTorch autocast does not cache casts for these frozen parameters, so
tiled decode otherwise converts every FP32 weight and bias once per
block invocation. Persisting the rounded values is numerically
equivalent to the per-call autocast conversion. The embedding and
output projections stay FP32 because their calls explicitly disable
autocast.
"""
if dtype not in (torch.float16, torch.bfloat16):
raise ValueError(
"MiniMax H3 decoder autocast weights require fp16 or bf16, "
f"got {dtype}"
)
if self._autocast_linear_dtype == dtype:
return 0
converted = 0
for block in self.transformer_blocks:
for linear in (
block.attn.to_qkv,
block.attn.to_out,
block.ff.w1,
block.ff.w2,
):
if linear.weight.dtype != dtype:
linear.to(dtype=dtype)
converted += 1
self._autocast_linear_dtype = dtype
return converted
def forward(self, x: torch.Tensor) -> torch.Tensor:
B, C, latent_T, latent_H, latent_W = x.shape
patch_size = self.config.patch_size
patch_size_t = self.config.patch_size_t
num_suffix = 1 + self.num_register_tokens
hidden_states = _pack_tensors_3d(x, 1, 1)
latent_size = (latent_T, latent_H, latent_W)
with torch.autocast("cuda", enabled=False):
hidden_states = _linear_with_module_dtype(
self.x_embedder, hidden_states, hidden_states.dtype
)
num_patches = hidden_states.shape[1]
tokens = [hidden_states]
if self.register_tokens is not None:
register_tokens = self.register_tokens.expand(B, -1, -1)
tokens.append(register_tokens)
cls_token = torch.zeros_like(hidden_states[:, 0:1, :])
tokens.append(cls_token)
hidden_states = torch.cat(tokens, dim=1)
patch_dims = [latent_T, latent_H, latent_W]
rotary_dtype = (
torch.get_autocast_dtype("cuda")
if x.is_cuda and torch.is_autocast_enabled("cuda")
else hidden_states.dtype
)
cache_enabled = (
not self.training
and not self.mask_enabled
and not torch.compiler.is_compiling()
)
cache_key = (
B,
latent_T,
latent_H,
latent_W,
num_suffix,
x.device,
x.dtype,
rotary_dtype,
)
cache_record = self._rotary_pos_emb_cache if cache_enabled else None
cache_hit = cache_record is not None and cache_record[0] == cache_key
if cache_hit:
img_ids = cache_record[1]
else:
img_ids = create_token_ids(latent_size, x.device, x.dtype).expand(B, -1, -1)
suffix_ids = torch.zeros(
(B, num_suffix, 3), device=x.device, dtype=img_ids.dtype
)
img_ids = torch.cat([img_ids, suffix_ids], dim=1)
hidden_states, img_ids = self.apply_mask_preprocess(
hidden_states, img_ids, patch_dims, num_suffix
)
cache_img_ids = img_ids
pack_info = {}
if self.t_causal:
spatial_size = latent_H * latent_W
mask_mod = make_block_causal_mask_mod(
num_tokens=num_patches,
block_size=spatial_size,
suffix=True,
)
pack_info["mask_mod"] = mask_mod
if cache_hit:
rotary_pos_emb = cache_record[2]
else:
rotary_pos_emb = prepare_rotary_pos_emb(
self.pos_embed(img_ids),
dtype=rotary_dtype,
)
if cache_enabled:
self._rotary_pos_emb_cache = (
cache_key,
cache_img_ids,
rotary_pos_emb,
)
for block in self.transformer_blocks:
hidden_states = block(hidden_states, rotary_pos_emb, pack_info)
hidden_states = self.norm_out(hidden_states)
hidden_states = self.apply_mask_postprocess(hidden_states, num_patches)
with torch.autocast("cuda", enabled=False):
output = _linear_with_module_dtype(
self.proj_out, hidden_states, hidden_states.dtype
)
output = output[:, :num_patches, :]
video_t = latent_size[0] * patch_size_t
video_h = latent_size[1] * patch_size
video_w = latent_size[2] * patch_size
output = _unpack_tensors_3d(
output, patch_size, patch_size_t, video_t, video_h, video_w
)
return output