HiUni / diffusion_head.py
hanghang214's picture
Add files using upload-large-folder tool
bc25b25 verified
Raw History Blame Contribute Delete
25.8 kB
# Adapted from model/transfusion/diffusion_head.py; inference uses PyTorch SDPA.
"""Multi-layer Diffusion Head for Transfusion Qwen3-VL.
Ported from tuna-2/models/backbones/modules_new_mm.py and adapted for the
HiUni / Megatron-based training pipeline.
Key adaptations vs. tuna-2:
- ``adaln_input`` is **per-token** ``[N_gen, H]`` (already expanded by
``TimestepEmbedder`` in the model), NOT per-image ``[B*num_imgs, H]``.
This matches ``self._gen_time_emb`` which is produced as
``time_embeddings(timesteps_packed)`` where ``timesteps_packed`` is
already repeated per token.
- Modulation scatter uses a boolean ``gen_token_mask`` of shape ``[B, S]``
(True at generation image token positions) instead of ``modality_positions``
``[B, num_imgs, 2]``, because the latter is never built in training_step.
- ``position_ids`` supports Qwen3-VL M-RoPE format ``[3, B, S]``
(temporal, height, width), directly reusing the backbone's 3-D position IDs.
Text tokens have all 3 channels equal (degenerates to 1-D); image tokens
carry spatial position information via the H/W channels.
Public API
----------
``DiffHeadConfig`` – standalone config (no Megatron dependency).
``ModulatedAttentionBlock`` – adaLN-Zero conditioned GQA + SwiGLU block.
``DiffusionHeadFinalLayer`` – adaLN-Zero modulated RMSNorm + linear head.
Usage inside TransfusionQwen3VLModel (post_process stage)::
# init
self.diffusion_head_a = nn.ModuleList([
ModulatedAttentionBlock(diff_head_cfg, i)
for i in range(num_diffusion_layers)
])
self.diffusion_head_b = DiffusionHeadFinalLayer(lm_dim, diff_out_dim)
# forward (hidden_states: [B, S, H], already gathered from SP)
# gen_token_mask : [B, S] bool – True at generation image token positions
# gen_time_emb : [total_gen_tokens, H] – per-token timestep embedding
# position_ids : [3, B, S] – backbone M-RoPE position IDs
for layer in self.diffusion_head_a:
hidden_states = layer(
hidden_states=hidden_states,
gen_time_emb=gen_time_emb,
gen_token_mask=gen_token_mask,
attention_mask=attention_mask,
position_ids=position_ids,
)[0]
x0_full = self.diffusion_head_b(hidden_states, gen_time_emb, gen_token_mask)
image_pred = x0_full[gen_token_mask] # [total_gen_tokens, out_channels]
"""
import math
from typing import List, Optional, Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers.activations import ACT2FN
from transformers.configuration_utils import PretrainedConfig
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
"""Rotate paired rotary coordinates by ninety degrees."""
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
def _apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim: int = 1):
"""Apply the shared rotary positions to query and key heads."""
cos = cos.unsqueeze(unsqueeze_dim)
sin = sin.unsqueeze(unsqueeze_dim)
return q * cos + _rotate_half(q) * sin, k * cos + _rotate_half(k) * sin
def _repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
"""Expand grouped key/value heads to the query head count."""
batch, n_kv, slen, head_dim = hidden_states.shape
if n_rep == 1:
return hidden_states
return (
hidden_states[:, :, None, :, :]
.expand(batch, n_kv, n_rep, slen, head_dim)
.reshape(batch, n_kv * n_rep, slen, head_dim)
)
def _modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
"""(1 + scale) * x + shift, computed in fp32 for numerical stability."""
input_dtype = x.dtype
return (
x.float() * (1 + scale.float()) + shift.float()
).to(input_dtype)
def _apply_interleaved_mrope(freqs: torch.Tensor, mrope_section: List[int]) -> torch.Tensor:
"""Merge 3-channel M-RoPE freqs into single tensor via interleaved pattern.
Same algorithm as Qwen3VLTextRotaryEmbedding.apply_interleaved_mrope.
Args:
freqs: [3, B, S, head_dim//2] – per-channel rotation frequencies.
mrope_section: [3] – e.g. [24, 20, 20], dimensions per channel.
Returns:
[B, S, head_dim//2] – interleaved merged frequencies.
"""
freqs_t = freqs[0].clone() # start with temporal as base
for dim, offset in enumerate((1, 2), start=1): # H, W channels
length = mrope_section[dim] * 3
idx = slice(offset, length, 3)
freqs_t[..., idx] = freqs[dim, ..., idx]
return freqs_t
# ---------------------------------------------------------------------------
# Building blocks
# ---------------------------------------------------------------------------
class DiffHeadRMSNorm(nn.Module):
"""RMSNorm with learnable weight (no Megatron dependency)."""
def __init__(self, hidden_size: int, eps: float = 1e-6):
"""Create the learned RMS scale with the training epsilon."""
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""对输入做 RMS 归一化后乘以可学习权重。
Args:
hidden_states: 任意形状的输入张量,最后一维为 hidden_size。
Returns:
与输入同形状的归一化结果,dtype 与输入保持一致。
"""
input_dtype = hidden_states.dtype
hidden_states = hidden_states.float()
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
return self.weight * hidden_states.to(input_dtype)
class DiffHeadConfig(PretrainedConfig):
"""Config for the multi-layer Diffusion Head.
Mirrors tuna-2's DiffusionHeadConfig.
Defaults are sized for Qwen3-7B (hidden_size=3584).
"""
def __init__(
self,
hidden_size: int = 3584,
num_attention_heads: int = 28,
num_key_value_heads: int = 4,
intermediate_size: int = 18944,
max_position_embeddings: int = 131072,
):
"""Set architecture and rotary defaults used by the training head."""
self.attn_implementation = "sdpa"
self.max_position_embeddings = max_position_embeddings
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.hidden_act = "silu"
self.rms_norm_eps = 1e-05
self.mlp_bias = False
self.head_dim = hidden_size // num_attention_heads
self.attention_bias = False
self.attention_dropout = 0.0
self.rope_theta = 5000000.0 # 与 Qwen3-VL backbone 一致 (5M)
self.rope_scaling = {
"rope_type": "default", # 与 backbone 一致,无 wavelength scaling
"mrope_section": [24, 20, 20],
}
self.qk_norm = True
super().__init__()
class _DiffHeadRotaryEmbedding(nn.Module):
"""RoPE embedding supporting both 1-D [B, S] and 3-D M-RoPE [3, B, S] position_ids."""
# Qwen3-VL default mrope_section
MROPE_SECTION = [24, 20, 20]
def __init__(self, config: DiffHeadConfig):
"""Initialize nonpersistent rotary frequencies from the head config."""
super().__init__()
rope_type = config.rope_scaling.get("rope_type", "default")
self.config = config
if rope_type == "default":
# Transformers 5 no longer exposes the original implementation in
# ROPE_INIT_FUNCTIONS. Keep the standard Qwen RoPE computation
# local so old and new Transformers releases behave identically.
dim = config.head_dim
inv_freq = 1.0 / (
config.rope_theta
** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)
)
self.attention_scaling = 1.0
else:
init_fn = ROPE_INIT_FUNCTIONS[rope_type]
inv_freq, self.attention_scaling = init_fn(config, None)
self.register_buffer("inv_freq", inv_freq, persistent=False)
@torch.no_grad()
def forward(self, x: torch.Tensor, position_ids: torch.Tensor):
"""计算 RoPE cos/sin 旋转系数,支持 1-D 和 3-D (M-RoPE) position_ids。
Args:
x: 参考张量,仅用于获取目标 dtype。
position_ids: 位置索引:
- ``[B, S]`` 或 ``[1, S]``: 标准 1-D RoPE
- ``[3, B, S]``: Qwen3-VL M-RoPE (temporal, height, width)
Returns:
cos: shape ``[B, S, head_dim]``
sin: shape ``[B, S, head_dim]``
"""
device_type = x.device.type if x.device.type != "mps" else "cpu"
if position_ids.ndim == 3 and position_ids.shape[0] == 3:
# M-RoPE mode: position_ids [3, B, S]
# 按照 Qwen3VLTextRotaryEmbedding 的方式计算 interleaved mrope
inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(
3, position_ids.shape[1], -1, 1
) # [3, B, dim, 1]
pos_expanded = position_ids[:, :, None, :].float() # [3, B, 1, S]
with torch.autocast(device_type=device_type, enabled=False):
freqs = (inv_freq_expanded.float() @ pos_expanded.float()).transpose(2, 3)
# freqs: [3, B, S, dim]
freqs = _apply_interleaved_mrope(freqs, self.MROPE_SECTION)
# freqs: [B, S, dim]
emb = torch.cat((freqs, freqs), dim=-1) # [B, S, head_dim]
cos = emb.cos() * self.attention_scaling
sin = emb.sin() * self.attention_scaling
else:
# Standard 1-D mode: position_ids [B, S] or [1, S]
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(
position_ids.shape[0], -1, 1
)
pos_expanded = position_ids[:, None, :].float()
with torch.autocast(device_type=device_type, enabled=False):
freqs = (inv_freq_expanded.float() @ pos_expanded.float()).transpose(1, 2)
emb = torch.cat((freqs, freqs), dim=-1)
cos = emb.cos() * self.attention_scaling
sin = emb.sin() * self.attention_scaling
return cos.to(x.dtype), sin.to(x.dtype)
class _DiffHeadMLP(nn.Module):
"""Training-compatible SwiGLU projection."""
def __init__(self, config: DiffHeadConfig):
"""Create the gate, up and down linear projections."""
super().__init__()
self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=config.mlp_bias)
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=config.mlp_bias)
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=config.mlp_bias)
self.act_fn = ACT2FN[config.hidden_act]
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""SwiGLU 前馈网络:gate_proj × up_proj 经 SiLU 激活后由 down_proj 投影回原维度。
Args:
x: 输入张量,shape ``[..., hidden_size]``。
Returns:
与输入同形状的输出张量。
"""
return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
class _DiffHeadSdpaAttention(nn.Module):
"""GQA attention with M-RoPE and an explicit PyTorch SDPA mask."""
def __init__(self, config: DiffHeadConfig, layer_idx: int):
"""Create the checkpoint-compatible projections and query/key normalization."""
super().__init__()
self.num_heads = config.num_attention_heads
self.head_dim = config.head_dim
self.num_kv_heads = config.num_key_value_heads
self.num_kv_groups = self.num_heads // self.num_kv_heads
self.attention_dropout = config.attention_dropout
self.q_proj = nn.Linear(config.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias)
self.k_proj = nn.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=config.attention_bias)
self.v_proj = nn.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=config.attention_bias)
self.o_proj = nn.Linear(self.num_heads * self.head_dim, config.hidden_size, bias=config.attention_bias)
if getattr(config, "qk_norm", False):
self.q_norm = DiffHeadRMSNorm(self.head_dim)
self.k_norm = DiffHeadRMSNorm(self.head_dim)
else:
self.q_norm = None
self.k_norm = None
self.rotary_emb = _DiffHeadRotaryEmbedding(config)
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: Optional[torch.Tensor] = None,
position_ids: Optional[torch.Tensor] = None,
startend_row_indices: Optional[torch.Tensor] = None,
**kwargs,
) -> Tuple[torch.Tensor, None, None]:
"""GQA 自注意力前向,支持 M-RoPE + FlashMask。
Args:
hidden_states: 输入序列,shape ``[B, S, H]``。
attention_mask: 加性 attention mask ``[B, 1, S, S]`` 或 None (fallback)。
position_ids: ``[B, S]``/``[1, S]`` (1-D) 或 ``[3, B, S]`` (M-RoPE)。
startend_row_indices: ``[B, 1, S, 2]`` FlashMask 格式 (优先使用,无需 dense mask)。
Returns:
Tuple: (attention output [B, S, H], None, None)
"""
bsz, q_len, _ = hidden_states.size()
q = self.q_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2)
k = self.k_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2)
v = self.v_proj(hidden_states).view(bsz, q_len, -1, self.head_dim).transpose(1, 2)
if self.q_norm is not None:
q = self.q_norm(q)
k = self.k_norm(k)
cos, sin = self.rotary_emb(v, position_ids)
q, k = _apply_rotary_pos_emb(q, k, cos, sin)
k = _repeat_kv(k, self.num_kv_groups)
v = _repeat_kv(v, self.num_kv_groups)
if startend_row_indices is not None:
raise ValueError("HF inference requires an explicit dense attention mask")
out = F.scaled_dot_product_attention(
q.contiguous(), k.contiguous(), v.contiguous(),
attn_mask=attention_mask, dropout_p=0.0,
is_causal=attention_mask is None and q_len > 1,
)
out = out.transpose(1, 2).contiguous().view(bsz, q_len, -1)
return self.o_proj(out), None, None
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
class ModulatedAttentionBlock(nn.Module):
"""adaLN-Zero conditioned transformer block for the diffusion head.
Conditioning design (adapted for HiUni data flow):
- ``gen_time_emb`` : ``[total_gen_tokens, H]`` – per-token timestep
embedding produced by ``TimestepEmbedder(timesteps_packed)``.
- ``gen_token_mask``: ``[B, S]`` bool – True at generation image token
positions; used to scatter per-token modulation into the full sequence.
Text and understanding-image positions keep identity modulation
(zero shift/scale, unit gate), so gradients do not flow into them
from the adaLN path.
"""
def __init__(self, config: DiffHeadConfig, layer_idx: int, adaln_post_modulation: bool = False,
block_adaln: bool = True):
"""Build a pre-norm head block with optional time modulation."""
super().__init__()
self.hidden_size = config.hidden_size
self.self_attn = _DiffHeadSdpaAttention(config=config, layer_idx=layer_idx)
self.mlp = _DiffHeadMLP(config)
self.input_layernorm = DiffHeadRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = DiffHeadRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
# block_adaln=False: transformer block 内不做任何 adaLN 调制 (纯 pre-norm
# transformer, 无 time/gate 条件), 时间条件仅由 head 最后一层的 adaLN 注入。
# 用于 "no adaln in transformer block" 变体; 默认 True 保持原状。
self.block_adaln = bool(block_adaln)
# post-modulation (NVlabs PixelDiT #6): 4-way scale/shift 施加在 attn/mlp
# 输出上 (无 gate), 缓解训练 spike; 默认 False 保持原 6-way pre-mod。
self.adaln_post_modulation = bool(adaln_post_modulation)
if self.block_adaln:
n_mod = 4 if self.adaln_post_modulation else 6
self.adaLN_modulation = nn.Sequential(
nn.SiLU(),
nn.Linear(config.hidden_size, n_mod * config.hidden_size, bias=True),
)
nn.init.zeros_(self.adaLN_modulation[1].weight)
nn.init.zeros_(self.adaLN_modulation[1].bias)
def forward(
self,
hidden_states: torch.Tensor, # [B, S, H]
gen_time_emb: torch.Tensor, # [total_gen_tokens, H] per-token timestep emb
gen_token_mask: torch.Tensor, # [B, S] bool, True = generation image token
attention_mask: Optional[torch.Tensor] = None,
position_ids: Optional[torch.Tensor] = None,
startend_row_indices: Optional[torch.Tensor] = None,
**kwargs,
) -> Tuple[torch.Tensor, ...]:
"""adaLN-Zero 条件化的 transformer block 前向。
Args:
hidden_states: 完整序列的隐状态,shape ``[B, S, H]``。
gen_time_emb: per-token 时间步 embedding,shape ``[total_gen_tokens, H]``。
gen_token_mask: 布尔掩码,shape ``[B, S]``,True = 生成图像 token。
attention_mask: 加性 attention mask ``[B, 1, S, S]`` 或 None (fallback)。
position_ids: ``[B, S]``/``[1, S]`` (1-D) 或 ``[3, B, S]`` (M-RoPE)。
startend_row_indices: ``[B, 1, S, 2]`` FlashMask 格式 (优先使用)。
**kwargs: 兼容性占位。
Returns:
Tuple,第 0 元素为更新后的 hidden_states,shape ``[B, S, H]``。
"""
# block_adaln=False: 纯 pre-norm transformer, 无 adaLN/gate 调制
# (时间条件仅在 head 末层注入)。忽略 gen_time_emb / gen_token_mask。
if not self.block_adaln:
residual = hidden_states
hidden_states, _, _ = self.self_attn(
hidden_states=self.input_layernorm(hidden_states),
attention_mask=attention_mask,
position_ids=position_ids,
startend_row_indices=startend_row_indices,
)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.mlp(self.post_attention_layernorm(hidden_states))
hidden_states = residual + hidden_states
return (hidden_states,)
# Compute per-token adaLN modulation from per-token time embedding.
modulation = self.adaLN_modulation(gen_time_emb) # [N_gen, n_mod*H]
if self.adaln_post_modulation:
# post-mod (NVlabs PixelDiT #6): 4-way scale/shift 施加在 attn/mlp 输出上 (无 gate)。
scale_msa, shift_msa, scale_mlp, shift_mlp = modulation.chunk(4, dim=-1)
# 非 gen 位置默认 scale=0/shift=0 → _modulate 恒等 (等价 residual+out)。
shift_msa_n = torch.zeros_like(hidden_states)
scale_msa_n = torch.zeros_like(hidden_states)
shift_mlp_n = torch.zeros_like(hidden_states)
scale_mlp_n = torch.zeros_like(hidden_states)
shift_msa_n[gen_token_mask] = shift_msa.to(hidden_states.dtype)
scale_msa_n[gen_token_mask] = scale_msa.to(hidden_states.dtype)
shift_mlp_n[gen_token_mask] = shift_mlp.to(hidden_states.dtype)
scale_mlp_n[gen_token_mask] = scale_mlp.to(hidden_states.dtype)
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states, _, _ = self.self_attn(
hidden_states=hidden_states,
attention_mask=attention_mask,
position_ids=position_ids,
startend_row_indices=startend_row_indices,
)
hidden_states = residual + _modulate(hidden_states, shift_msa_n, scale_msa_n)
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + _modulate(hidden_states, shift_mlp_n, scale_mlp_n)
return (hidden_states,)
# pre-mod (default): Compute per-token 6-chunk adaLN modulation.
# [total_gen_tokens, 6*H] → 6 × [total_gen_tokens, H]
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = modulation.chunk(6, dim=-1)
# Scatter into full-sequence tensors; non-gen positions keep identity defaults.
shift_msa_n = torch.zeros_like(hidden_states)
scale_msa_n = torch.zeros_like(hidden_states)
gate_msa_n = torch.ones_like(hidden_states)
shift_mlp_n = torch.zeros_like(hidden_states)
scale_mlp_n = torch.zeros_like(hidden_states)
gate_mlp_n = torch.ones_like(hidden_states)
shift_msa_n[gen_token_mask] = shift_msa.to(hidden_states.dtype)
scale_msa_n[gen_token_mask] = scale_msa.to(hidden_states.dtype)
gate_msa_n[gen_token_mask] = gate_msa.to(hidden_states.dtype)
shift_mlp_n[gen_token_mask] = shift_mlp.to(hidden_states.dtype)
scale_mlp_n[gen_token_mask] = scale_mlp.to(hidden_states.dtype)
gate_mlp_n[gen_token_mask] = gate_mlp.to(hidden_states.dtype)
residual = hidden_states
hidden_states = _modulate(self.input_layernorm(hidden_states), shift_msa_n, scale_msa_n)
hidden_states, _, _ = self.self_attn(
hidden_states=hidden_states,
attention_mask=attention_mask,
position_ids=position_ids,
startend_row_indices=startend_row_indices,
)
hidden_states = residual + gate_msa_n * hidden_states
residual = hidden_states
hidden_states = _modulate(self.post_attention_layernorm(hidden_states), shift_mlp_n, scale_mlp_n)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + gate_mlp_n * hidden_states
return (hidden_states,)
class DiffusionHeadFinalLayer(nn.Module):
"""adaLN-modulated RMSNorm + linear output layer.
Same conditioning design as ``ModulatedAttentionBlock``:
uses per-token ``gen_time_emb`` and a boolean ``gen_token_mask``
to apply modulation only at generation image token positions.
Args:
hidden_size: LLM hidden dimension.
out_channels: Output channel count (e.g. VAE latent dim = 32).
"""
def __init__(self, hidden_size: int, out_channels: int):
"""Create the normalized, time-modulated output projection."""
super().__init__()
self.norm_final = DiffHeadRMSNorm(hidden_size)
self.linear = nn.Linear(hidden_size, out_channels, bias=True)
self.adaLN_modulation = nn.Sequential(
nn.SiLU(),
nn.Linear(hidden_size, 2 * hidden_size, bias=True),
)
# Zero-init (standard DiT initialisation – identity at the start)
nn.init.constant_(self.adaLN_modulation[-1].weight, 0)
nn.init.constant_(self.adaLN_modulation[-1].bias, 0)
nn.init.constant_(self.linear.weight, 0)
nn.init.constant_(self.linear.bias, 0)
def forward(
self,
x: torch.Tensor, # [B, S, H]
gen_time_emb: torch.Tensor, # [total_gen_tokens, H]
gen_token_mask: torch.Tensor, # [B, S] bool
) -> torch.Tensor:
"""adaLN 调制的最终归一化与线性投影。
流程:
1. 从 gen_time_emb 计算 per-token 的 shift 和 scale。
2. 将 shift/scale 按 gen_token_mask scatter 到完整序列,
非生成位置保持 shift=0 / scale=0(恒等)。
3. adaLN 调制后的 RMSNorm → 线性投影到输出通道数。
输出张量保持完整序列形状 ``[B, S, out_channels]``,调用方
用 ``x0_full[gen_token_mask]`` 提取生成 token 的预测值。
Args:
x: 完整序列的隐状态,shape ``[B, S, H]``。
gen_time_emb: 生成 token 的 per-token 时间步 embedding,
shape ``[total_gen_tokens, H]``。
gen_token_mask: 布尔掩码,shape ``[B, S]``,True 表示生成图像 token。
Returns:
shape ``[B, S, out_channels]`` 的输出张量;
通过 ``result[gen_token_mask]`` 可得 ``[total_gen_tokens, out_channels]`` 的预测。
"""
# [N_gen, 2*H] → shift [N_gen, H], scale [N_gen, H]
shift, scale = self.adaLN_modulation(gen_time_emb).chunk(2, dim=-1)
shift_new = torch.zeros_like(x)
scale_new = torch.zeros_like(x)
shift_new[gen_token_mask] = shift.to(x.dtype)
scale_new[gen_token_mask] = scale.to(x.dtype)
x = _modulate(self.norm_final(x), shift_new, scale_new)
return self.linear(x)