Spaces:
Running on Zero
Running on Zero
Download direct3d/models/decoder.py from hugging-apps/direct3d-v3: direct link, hf CLI and curl.
- Browser
- Download file 4.73 kB
-
https://huggingface.co/spaces/hugging-apps/direct3d-v3/resolve/main/direct3d/models/decoder.py
- Command line
-
hf download hf://spaces/hugging-apps/direct3d-v3/direct3d/models/decoder.py
-
curl -L -o decoder.py https://huggingface.co/spaces/hugging-apps/direct3d-v3/resolve/main/direct3d/models/decoder.py
4.73 kB
| from torch import nn | |
| from .sparse_atten import MLP, Attention | |
| class DecoderFinalLayer(nn.Module): | |
| """ | |
| The final layer adopted from DiT. | |
| """ | |
| def __init__(self, model_channels, out_channels, norm_layer=nn.LayerNorm, *, elementwise_affine=False): | |
| super().__init__() | |
| self.norm_final = norm_layer(model_channels, elementwise_affine=elementwise_affine, eps=1e-6) | |
| self.linear = nn.Linear(model_channels, out_channels, bias=True) | |
| def forward(self, x): | |
| x = self.norm_final(x) | |
| x = self.linear(x) | |
| return x | |
| class PixelAttentionBlock(nn.Module): | |
| """Mix patch features with compact window attention and a per-pixel MLP.""" | |
| def __init__( | |
| self, | |
| channels, | |
| z_channels, | |
| patch_size, | |
| num_heads, | |
| norm_layer=nn.LayerNorm, | |
| qkv_bias=True, | |
| qk_norm=False, | |
| qk_norm_layer=nn.RMSNorm, | |
| ): | |
| super().__init__() | |
| self.channels = channels | |
| self.patch_size = patch_size | |
| self.norm1 = norm_layer(channels, elementwise_affine=True, eps=1e-6) | |
| self.compact_proj = nn.Linear(patch_size**3 * channels, z_channels) | |
| self.compact_unproj = nn.Linear(z_channels, patch_size**3 * channels) | |
| self.attn1 = Attention( | |
| z_channels, num_heads=num_heads, qkv_bias=qkv_bias, qk_norm=qk_norm, norm_layer=qk_norm_layer | |
| ) | |
| self.norm2 = norm_layer(channels, elementwise_affine=True, eps=1e-6) | |
| self.mlp = MLP(width=channels) | |
| self.adaLN_modulation = nn.Sequential( | |
| nn.SiLU(), nn.Linear(z_channels, patch_size**3 * 6 * channels, bias=True) | |
| ) | |
| def forward(self, x, y, index, *, window_size, window_cache): | |
| # Modulation and the pixel MLP are patch-local. Only the compact tokens | |
| # need to be resident together for attention across sparse windows. | |
| chunk_size = 1024 | |
| patches = self.patch_size**3 | |
| compact = x.new_empty(len(x), self.compact_proj.out_features) | |
| for start in range(0, len(x), chunk_size): | |
| part = slice(start, start + chunk_size) | |
| scale_shift = self.adaLN_modulation(y[part]).reshape(-1, patches, 6, self.channels) | |
| shift, scale = scale_shift[:, :, 0], scale_shift[:, :, 1] | |
| normalized = (1 + scale) * self.norm1(x[part]) + shift | |
| compact[part] = self.compact_proj(normalized.flatten(1)) | |
| attended = self.attn1(compact, index, window_size=window_size, window_cache=window_cache) | |
| for start in range(0, len(x), chunk_size): | |
| part = slice(start, start + chunk_size) | |
| modulation = self.adaLN_modulation(y[part]).reshape(-1, patches, 6, self.channels) | |
| gate, shift, scale, mlp_gate = modulation[:, :, 2:].unbind(dim=2) | |
| residual = x[part] + gate * self.compact_unproj(attended[part]).reshape( | |
| -1, patches, self.channels | |
| ) | |
| x[part] = residual + mlp_gate * self.mlp((1 + scale) * self.norm2(residual) + shift) | |
| return x | |
| class PixelAttentionDecoder(nn.Module): | |
| def __init__( | |
| self, | |
| in_channels, | |
| model_channels, | |
| out_channels, | |
| z_channels, | |
| num_res_blocks, | |
| patch_size, | |
| num_heads, | |
| norm_layer=nn.LayerNorm, | |
| qkv_bias=True, | |
| qk_norm=False, | |
| qk_norm_layer=nn.RMSNorm, | |
| ): | |
| super().__init__() | |
| self.in_channels = in_channels | |
| self.model_channels = model_channels | |
| self.out_channels = out_channels | |
| self.num_res_blocks = num_res_blocks | |
| self.patch_size = patch_size | |
| self.input_proj = nn.Linear(in_channels, model_channels) | |
| self.res_blocks = nn.ModuleList( | |
| [ | |
| PixelAttentionBlock( | |
| model_channels, | |
| z_channels, | |
| patch_size, | |
| num_heads=num_heads, | |
| norm_layer=norm_layer, | |
| qkv_bias=qkv_bias, | |
| qk_norm=qk_norm, | |
| qk_norm_layer=qk_norm_layer, | |
| ) | |
| for _ in range(num_res_blocks) | |
| ] | |
| ) | |
| self.final_layer = DecoderFinalLayer(model_channels, out_channels, norm_layer=norm_layer) | |
| def forward(self, x, c, index, *, window_cache, window_size): | |
| """ | |
| :param x: [L, patch_size**3, in_channels] | |
| :param c: [L, z_channels] | |
| :param index: [L, 4] | |
| """ | |
| x = self.input_proj(x) # [L, patch_size**3, model_channels] | |
| for block in self.res_blocks: | |
| x = block(x, c, index, window_cache=window_cache, window_size=window_size) | |
| x = self.final_layer(x) | |
| return x | |