direct3d-v3 / direct3d /models /decoder.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
35462eb verified
Raw History Blame Contribute Delete
4.73 kB
from torch import nn
from .sparse_atten import MLP, Attention
class DecoderFinalLayer(nn.Module):
"""
The final layer adopted from DiT.
"""
def __init__(self, model_channels, out_channels, norm_layer=nn.LayerNorm, *, elementwise_affine=False):
super().__init__()
self.norm_final = norm_layer(model_channels, elementwise_affine=elementwise_affine, eps=1e-6)
self.linear = nn.Linear(model_channels, out_channels, bias=True)
def forward(self, x):
x = self.norm_final(x)
x = self.linear(x)
return x
class PixelAttentionBlock(nn.Module):
"""Mix patch features with compact window attention and a per-pixel MLP."""
def __init__(
self,
channels,
z_channels,
patch_size,
num_heads,
norm_layer=nn.LayerNorm,
qkv_bias=True,
qk_norm=False,
qk_norm_layer=nn.RMSNorm,
):
super().__init__()
self.channels = channels
self.patch_size = patch_size
self.norm1 = norm_layer(channels, elementwise_affine=True, eps=1e-6)
self.compact_proj = nn.Linear(patch_size**3 * channels, z_channels)
self.compact_unproj = nn.Linear(z_channels, patch_size**3 * channels)
self.attn1 = Attention(
z_channels, num_heads=num_heads, qkv_bias=qkv_bias, qk_norm=qk_norm, norm_layer=qk_norm_layer
)
self.norm2 = norm_layer(channels, elementwise_affine=True, eps=1e-6)
self.mlp = MLP(width=channels)
self.adaLN_modulation = nn.Sequential(
nn.SiLU(), nn.Linear(z_channels, patch_size**3 * 6 * channels, bias=True)
)
def forward(self, x, y, index, *, window_size, window_cache):
# Modulation and the pixel MLP are patch-local. Only the compact tokens
# need to be resident together for attention across sparse windows.
chunk_size = 1024
patches = self.patch_size**3
compact = x.new_empty(len(x), self.compact_proj.out_features)
for start in range(0, len(x), chunk_size):
part = slice(start, start + chunk_size)
scale_shift = self.adaLN_modulation(y[part]).reshape(-1, patches, 6, self.channels)
shift, scale = scale_shift[:, :, 0], scale_shift[:, :, 1]
normalized = (1 + scale) * self.norm1(x[part]) + shift
compact[part] = self.compact_proj(normalized.flatten(1))
attended = self.attn1(compact, index, window_size=window_size, window_cache=window_cache)
for start in range(0, len(x), chunk_size):
part = slice(start, start + chunk_size)
modulation = self.adaLN_modulation(y[part]).reshape(-1, patches, 6, self.channels)
gate, shift, scale, mlp_gate = modulation[:, :, 2:].unbind(dim=2)
residual = x[part] + gate * self.compact_unproj(attended[part]).reshape(
-1, patches, self.channels
)
x[part] = residual + mlp_gate * self.mlp((1 + scale) * self.norm2(residual) + shift)
return x
class PixelAttentionDecoder(nn.Module):
def __init__(
self,
in_channels,
model_channels,
out_channels,
z_channels,
num_res_blocks,
patch_size,
num_heads,
norm_layer=nn.LayerNorm,
qkv_bias=True,
qk_norm=False,
qk_norm_layer=nn.RMSNorm,
):
super().__init__()
self.in_channels = in_channels
self.model_channels = model_channels
self.out_channels = out_channels
self.num_res_blocks = num_res_blocks
self.patch_size = patch_size
self.input_proj = nn.Linear(in_channels, model_channels)
self.res_blocks = nn.ModuleList(
[
PixelAttentionBlock(
model_channels,
z_channels,
patch_size,
num_heads=num_heads,
norm_layer=norm_layer,
qkv_bias=qkv_bias,
qk_norm=qk_norm,
qk_norm_layer=qk_norm_layer,
)
for _ in range(num_res_blocks)
]
)
self.final_layer = DecoderFinalLayer(model_channels, out_channels, norm_layer=norm_layer)
def forward(self, x, c, index, *, window_cache, window_size):
"""
:param x: [L, patch_size**3, in_channels]
:param c: [L, z_channels]
:param index: [L, 4]
"""
x = self.input_proj(x) # [L, patch_size**3, model_channels]
for block in self.res_blocks:
x = block(x, c, index, window_cache=window_cache, window_size=window_size)
x = self.final_layer(x)
return x