models/light_decoder.py

# coding=utf-8
"""
Lightweight SD2 VAE Decoder with halved channel dimensions.

Original SD2 VAE Decoder:
    block_out_channels = [128, 256, 512, 512]
    layers_per_block = 2  (results in 3 resnets per UpBlock)
    mid_block: 2 ResnetBlock2D + 1 Attention (512-dim)
    conv_in: 4 -> 512
    conv_out: 128 -> 3

Light Decoder (this file):
    block_out_channels = [64, 128, 256, 256]
    layers_per_block = 2  (results in 3 resnets per UpBlock, same structure)
    mid_block: 2 ResnetBlock2D + 1 Attention (256-dim)
    conv_in: 4 -> 256
    conv_out: 64 -> 3

Self-attention in mid_block is preserved. Channel dimensions are halved.
"""

import torch
import torch.nn as nn
from diffusers.models.unets.unet_2d_blocks import UpDecoderBlock2D, UNetMidBlock2D


class LightDecoder(nn.Module):
    """
    Lightweight VAE Decoder for SD2 with halved channels.
    
    Accepts latent z of shape [B, 4, H/8, W/8] and outputs RGB image [B, 3, H, W].
    """

    def __init__(
        self,
        in_channels: int = 4,
        out_channels: int = 3,
        block_out_channels=(64, 128, 256, 256),
        layers_per_block: int = 2,
        norm_num_groups: int = 32,
        act_fn: str = "silu",
        mid_block_add_attention: bool = True,
    ):
        super().__init__()

        self.in_channels = in_channels
        self.out_channels = out_channels
        self.block_out_channels = block_out_channels
        self.layers_per_block = layers_per_block

        # Reversed block_out_channels for decoder (goes from deepest to shallowest)
        reversed_block_out_channels = list(reversed(block_out_channels))

        # conv_in: latent_channels -> deepest channel dim
        self.conv_in = nn.Conv2d(
            in_channels,
            reversed_block_out_channels[0],
            kernel_size=3,
            stride=1,
            padding=1,
        )

        # Mid block with attention
        # attention_head_dim should match in_channels (single-head attention),
        # consistent with diffusers Decoder which uses attention_head_dim=block_out_channels[-1]
        self.mid_block = UNetMidBlock2D(
            in_channels=reversed_block_out_channels[0],
            temb_channels=None,
            dropout=0.0,
            num_layers=1,
            resnet_eps=1e-6,
            resnet_act_fn=act_fn,
            resnet_groups=norm_num_groups,
            add_attention=mid_block_add_attention,
            attention_head_dim=reversed_block_out_channels[0],
        )

        # Up blocks
        self.up_blocks = nn.ModuleList([])
        output_channel = reversed_block_out_channels[0]
        for i, up_block_out_channel in enumerate(reversed_block_out_channels):
            prev_output_channel = output_channel
            output_channel = up_block_out_channel
            is_final_block = i == len(reversed_block_out_channels) - 1

            up_block = UpDecoderBlock2D(
                in_channels=prev_output_channel,
                out_channels=output_channel,
                num_layers=layers_per_block + 1,  # diffusers convention: layers_per_block + 1 resnets
                resnet_eps=1e-6,
                resnet_act_fn=act_fn,
                resnet_groups=norm_num_groups,
                add_upsample=not is_final_block,
            )
            self.up_blocks.append(up_block)

        # Output layers
        self.conv_norm_out = nn.GroupNorm(
            num_channels=block_out_channels[0],
            num_groups=norm_num_groups,
            eps=1e-6,
        )
        self.conv_act = nn.SiLU()
        self.conv_out = nn.Conv2d(
            block_out_channels[0],
            out_channels,
            kernel_size=3,
            stride=1,
            padding=1,
        )

    def forward(self, z: torch.Tensor) -> torch.Tensor:
        """
        Args:
            z: Latent tensor of shape [B, in_channels, H', W']
        Returns:
            Decoded image of shape [B, out_channels, H, W]
        """
        # conv_in
        h = self.conv_in(z)

        # mid block
        h = self.mid_block(h)

        # up blocks
        for up_block in self.up_blocks:
            h = up_block(h)

        # output
        h = self.conv_norm_out(h)
        h = self.conv_act(h)
        h = self.conv_out(h)

        return h