Spaces:
Running on Zero
Running on Zero
Download h3vae/vae_vit.py from hugging-apps/invoker-render: direct link, hf CLI and curl.
- Browser
- Download file 13.4 kB
-
https://huggingface.co/spaces/hugging-apps/invoker-render/resolve/main/h3vae/vae_vit.py
- Command line
-
hf download hf://spaces/hugging-apps/invoker-render/h3vae/vae_vit.py
-
curl -L -o vae_vit.py https://huggingface.co/spaces/hugging-apps/invoker-render/resolve/main/h3vae/vae_vit.py
13.4 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| # ViT3D decoder for the MiniMax H3 visual VAE (inference-only bundle). | |
| import torch | |
| import torch.distributed as dist | |
| import torch.nn as nn | |
| from diffusers.configuration_utils import ConfigMixin, register_to_config | |
| from diffusers.models.modeling_utils import ModelMixin | |
| from diffusers.utils import logging | |
| from .base_module import RotaryEmbeddingND, TransformerBlock | |
| from .flash import make_block_causal_mask_mod | |
| from .vit_utils import create_token_ids, prepare_rotary_pos_emb | |
| logger = logging.get_logger(__name__) | |
| def _linear_with_module_dtype(linear, tensor, out_dtype=None): | |
| weight = getattr(linear, "weight", None) | |
| target_dtype = getattr(weight, "dtype", tensor.dtype) | |
| output = linear(tensor.to(target_dtype)) | |
| if out_dtype is not None and output.dtype != out_dtype: | |
| output = output.to(out_dtype) | |
| return output | |
| def _pack_tensors_3d(tensors, patch_size, patch_size_t): | |
| batch_size, num_channels_tensors, temporal, height, width = tensors.shape | |
| tensors = tensors.view( | |
| batch_size, | |
| num_channels_tensors, | |
| temporal // patch_size_t, | |
| patch_size_t, | |
| height // patch_size, | |
| patch_size, | |
| width // patch_size, | |
| patch_size, | |
| ) | |
| tensors = tensors.permute(0, 2, 4, 6, 1, 3, 5, 7) | |
| tensors = tensors.reshape( | |
| batch_size, | |
| (temporal // patch_size_t) * (height // patch_size) * (width // patch_size), | |
| num_channels_tensors * patch_size_t * patch_size * patch_size, | |
| ) | |
| return tensors | |
| def _unpack_tensors_3d(tensors, patch_size, patch_size_t, temporal, height, width): | |
| batch_size, num_patches, channels = tensors.shape | |
| num_channels_tensors = channels // (patch_size_t * patch_size * patch_size) | |
| tensors = tensors.view( | |
| batch_size, | |
| temporal // patch_size_t, | |
| height // patch_size, | |
| width // patch_size, | |
| num_channels_tensors, | |
| patch_size_t, | |
| patch_size, | |
| patch_size, | |
| ) | |
| tensors = tensors.permute(0, 4, 1, 5, 2, 6, 3, 7).contiguous() | |
| tensors = tensors.reshape(batch_size, num_channels_tensors, temporal, height, width) | |
| return tensors | |
| class ViTBase(ModelMixin, ConfigMixin): | |
| """Base class for ViT Encoder and Decoder with common functionality.""" | |
| _no_split_modules = ["TransformerBlock"] | |
| def _init_weights(self): | |
| def basic_init(m): | |
| if isinstance(m, nn.Linear): | |
| nn.init.xavier_uniform_(m.weight) | |
| if m.bias is not None: | |
| nn.init.constant_(m.bias, 0) | |
| self.apply(basic_init) | |
| def init_mask_config(self, dim, is_3d=False): | |
| self._mask_dim = dim | |
| self._mask_is_3d = is_3d | |
| self.register_buffer("mask_token", torch.zeros(1, 1, dim)) | |
| def set_mask_config(self, mask_config): | |
| self.mask_prob = mask_config.get("mask_prob", 0.0) | |
| self.mask_enabled = self.mask_prob > 0 | |
| self.mask_style = mask_config.get("mask_style", "replace") | |
| if self.mask_enabled and self.mask_style == "drop" and self.mask_prob < 1.0: | |
| logger.warning("mask_style='drop' with mask_prob < 1.0") | |
| if self._mask_is_3d: | |
| self.temporal_scale_range = mask_config.get( | |
| "temporal_scale_range", (0.3, 0.5) | |
| ) | |
| self.spatial_scale_range = mask_config.get( | |
| "spatial_scale_range", (0.1, 0.25) | |
| ) | |
| self.min_mask_ratio = mask_config.get("min_mask_ratio", 0.75) | |
| self.max_mask_ratio = mask_config.get("max_mask_ratio", 0.95) | |
| else: | |
| self.spatial_scale_range = mask_config.get( | |
| "spatial_scale_range", (0.15, 0.15) | |
| ) | |
| self.min_mask_ratio = mask_config.get("min_mask_ratio", 0.5) | |
| self.max_mask_ratio = mask_config.get("max_mask_ratio", 0.75) | |
| self.aspect_ratio_range = mask_config.get("aspect_ratio_range", (0.75, 1.5)) | |
| self.max_retries = mask_config.get("max_retries", 100) | |
| if ( | |
| self.mask_enabled | |
| and self.mask_style == "drop" | |
| and getattr(self, "t_causal", False) | |
| ): | |
| logger.warning("mask_style='drop' with t_causal may cause issues") | |
| if self.mask_enabled and "mask_token" in self._buffers: | |
| del self._buffers["mask_token"] | |
| self.mask_token = nn.Parameter(torch.randn(1, 1, self._mask_dim) * 0.02) | |
| def init_suffix_tokens(self, dim, num_register_tokens, has_cls_token=True): | |
| self.num_register_tokens = num_register_tokens | |
| if num_register_tokens > 0: | |
| self.register_tokens = nn.Parameter( | |
| torch.randn(1, num_register_tokens, dim) * 0.02 | |
| ) | |
| else: | |
| self.register_tokens = None | |
| if has_cls_token: | |
| self.cls_token = nn.Parameter(torch.randn(1, 1, dim) * 0.02) | |
| def apply_mask_preprocess(self, hidden_states, img_ids, patch_dims, num_suffix): | |
| if self.training and self.mask_enabled: | |
| raise NotImplementedError( | |
| "mask modeling is not supported in this inference-only bundle" | |
| ) | |
| return hidden_states, img_ids | |
| def forward_transformer_blocks(self, hidden_states, rotary_pos_emb, pack_info=None): | |
| if pack_info is None: | |
| pack_info = {} | |
| for block in self.transformer_blocks: | |
| hidden_states = block(hidden_states, rotary_pos_emb, pack_info) | |
| return hidden_states | |
| def apply_mask_postprocess(self, hidden_states, num_patches): | |
| if self.training and self.mask_enabled and self.mask_style == "drop": | |
| raise NotImplementedError( | |
| "mask modeling is not supported in this inference-only bundle" | |
| ) | |
| return hidden_states | |
| class ViT3DDecoder(ViTBase): | |
| """Vision Transformer Video Decoder using TransformerBlock.""" | |
| def __init__( | |
| self, | |
| patch_size: int = 16, | |
| patch_size_t: int = 4, | |
| t_causal: bool = False, | |
| in_channels: int = 16, | |
| out_channels: int = 3, | |
| num_layers: int = 24, | |
| heads: int = 16, | |
| dim_head: int = 64, | |
| norm_type: str = "layer_norm", | |
| norm_affine: bool = True, | |
| qk_norm_type: str = None, | |
| qk_norm_affine: bool = False, | |
| ffn_activation_fn: str = "gelu", | |
| ffn_use_gated: bool = False, | |
| rope_theta: float = 100.0, | |
| rope_dim_ratio: float = 1.0, | |
| bias: bool = True, | |
| eps: float = 1e-5, | |
| num_register_tokens: int = 4, | |
| mask_config: dict = {}, | |
| **kwargs, | |
| ): | |
| super().__init__() | |
| dim = heads * dim_head | |
| rope_apply_dim = int(dim_head * rope_dim_ratio) | |
| self.pos_embed = RotaryEmbeddingND( | |
| rope_apply_dim, rope_theta, n_dim=3, use_angle=True | |
| ) | |
| self.x_embedder = nn.Linear(in_channels, dim) | |
| self.init_suffix_tokens(dim, num_register_tokens, has_cls_token=False) | |
| self.t_causal = t_causal | |
| self.transformer_blocks = nn.ModuleList( | |
| [ | |
| TransformerBlock( | |
| heads=heads, | |
| dim_head=dim_head, | |
| norm_type=norm_type, | |
| norm_affine=norm_affine, | |
| qk_norm_type=qk_norm_type, | |
| qk_norm_affine=qk_norm_affine, | |
| ffn_activation_fn=ffn_activation_fn, | |
| ffn_use_gated=ffn_use_gated, | |
| bias=bias, | |
| eps=eps, | |
| **kwargs, | |
| ) | |
| for _ in range(num_layers) | |
| ] | |
| ) | |
| self.norm_out = nn.LayerNorm(dim, elementwise_affine=norm_affine, eps=eps) | |
| patch_dim = out_channels * patch_size_t * patch_size * patch_size | |
| self.proj_out = nn.Linear(dim, patch_dim) | |
| self.init_mask_config(dim, is_3d=True) | |
| self.set_mask_config(mask_config) | |
| self._rotary_pos_emb_cache = None | |
| self._autocast_linear_dtype = None | |
| if len(kwargs) > 0 and (not dist.is_initialized() or dist.get_rank() == 0): | |
| logger.warning(f"Unused kwargs: {kwargs}") | |
| def _apply(self, fn, recurse=True): | |
| result = super()._apply(fn, recurse=recurse) | |
| self._rotary_pos_emb_cache = None | |
| self._autocast_linear_dtype = None | |
| return result | |
| def prepare_autocast_linear_weights(self, dtype: torch.dtype) -> int: | |
| """Keep decoder-block linear weights in their autocast compute dtype. | |
| PyTorch autocast does not cache casts for these frozen parameters, so | |
| tiled decode otherwise converts every FP32 weight and bias once per | |
| block invocation. Persisting the rounded values is numerically | |
| equivalent to the per-call autocast conversion. The embedding and | |
| output projections stay FP32 because their calls explicitly disable | |
| autocast. | |
| """ | |
| if dtype not in (torch.float16, torch.bfloat16): | |
| raise ValueError( | |
| "MiniMax H3 decoder autocast weights require fp16 or bf16, " | |
| f"got {dtype}" | |
| ) | |
| if self._autocast_linear_dtype == dtype: | |
| return 0 | |
| converted = 0 | |
| for block in self.transformer_blocks: | |
| for linear in ( | |
| block.attn.to_qkv, | |
| block.attn.to_out, | |
| block.ff.w1, | |
| block.ff.w2, | |
| ): | |
| if linear.weight.dtype != dtype: | |
| linear.to(dtype=dtype) | |
| converted += 1 | |
| self._autocast_linear_dtype = dtype | |
| return converted | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| B, C, latent_T, latent_H, latent_W = x.shape | |
| patch_size = self.config.patch_size | |
| patch_size_t = self.config.patch_size_t | |
| num_suffix = 1 + self.num_register_tokens | |
| hidden_states = _pack_tensors_3d(x, 1, 1) | |
| latent_size = (latent_T, latent_H, latent_W) | |
| with torch.autocast("cuda", enabled=False): | |
| hidden_states = _linear_with_module_dtype( | |
| self.x_embedder, hidden_states, hidden_states.dtype | |
| ) | |
| num_patches = hidden_states.shape[1] | |
| tokens = [hidden_states] | |
| if self.register_tokens is not None: | |
| register_tokens = self.register_tokens.expand(B, -1, -1) | |
| tokens.append(register_tokens) | |
| cls_token = torch.zeros_like(hidden_states[:, 0:1, :]) | |
| tokens.append(cls_token) | |
| hidden_states = torch.cat(tokens, dim=1) | |
| patch_dims = [latent_T, latent_H, latent_W] | |
| rotary_dtype = ( | |
| torch.get_autocast_dtype("cuda") | |
| if x.is_cuda and torch.is_autocast_enabled("cuda") | |
| else hidden_states.dtype | |
| ) | |
| cache_enabled = ( | |
| not self.training | |
| and not self.mask_enabled | |
| and not torch.compiler.is_compiling() | |
| ) | |
| cache_key = ( | |
| B, | |
| latent_T, | |
| latent_H, | |
| latent_W, | |
| num_suffix, | |
| x.device, | |
| x.dtype, | |
| rotary_dtype, | |
| ) | |
| cache_record = self._rotary_pos_emb_cache if cache_enabled else None | |
| cache_hit = cache_record is not None and cache_record[0] == cache_key | |
| if cache_hit: | |
| img_ids = cache_record[1] | |
| else: | |
| img_ids = create_token_ids(latent_size, x.device, x.dtype).expand(B, -1, -1) | |
| suffix_ids = torch.zeros( | |
| (B, num_suffix, 3), device=x.device, dtype=img_ids.dtype | |
| ) | |
| img_ids = torch.cat([img_ids, suffix_ids], dim=1) | |
| hidden_states, img_ids = self.apply_mask_preprocess( | |
| hidden_states, img_ids, patch_dims, num_suffix | |
| ) | |
| cache_img_ids = img_ids | |
| pack_info = {} | |
| if self.t_causal: | |
| spatial_size = latent_H * latent_W | |
| mask_mod = make_block_causal_mask_mod( | |
| num_tokens=num_patches, | |
| block_size=spatial_size, | |
| suffix=True, | |
| ) | |
| pack_info["mask_mod"] = mask_mod | |
| if cache_hit: | |
| rotary_pos_emb = cache_record[2] | |
| else: | |
| rotary_pos_emb = prepare_rotary_pos_emb( | |
| self.pos_embed(img_ids), | |
| dtype=rotary_dtype, | |
| ) | |
| if cache_enabled: | |
| self._rotary_pos_emb_cache = ( | |
| cache_key, | |
| cache_img_ids, | |
| rotary_pos_emb, | |
| ) | |
| for block in self.transformer_blocks: | |
| hidden_states = block(hidden_states, rotary_pos_emb, pack_info) | |
| hidden_states = self.norm_out(hidden_states) | |
| hidden_states = self.apply_mask_postprocess(hidden_states, num_patches) | |
| with torch.autocast("cuda", enabled=False): | |
| output = _linear_with_module_dtype( | |
| self.proj_out, hidden_states, hidden_states.dtype | |
| ) | |
| output = output[:, :num_patches, :] | |
| video_t = latent_size[0] * patch_size_t | |
| video_h = latent_size[1] * patch_size | |
| video_w = latent_size[2] * patch_size | |
| output = _unpack_tensors_3d( | |
| output, patch_size, patch_size_t, video_t, video_h, video_w | |
| ) | |
| return output | |